From e3e12176bfacf0bfe4dd6fb7096c98e7954f12e4 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Mon, 18 Nov 2024 18:16:21 +1100 Subject: [PATCH 01/36] push manifest with subject Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Content/Digest.cs | 14 +- .../Exceptions/NoReferrerUpdateException.cs | 20 +++ src/OrasProject.Oras/Oci/Artifact.cs | 26 ++++ src/OrasProject.Oras/Oci/Descriptor.cs | 13 ++ src/OrasProject.Oras/Oci/Index.cs | 22 ++++ .../Registry/Remote/Auth/Client.cs | 13 ++ .../Remote/HttpResponseMessageExtensions.cs | 12 +- .../Registry/Remote/ManifestStore.cs | 124 +++++++++++++++++- .../Registry/Remote/Referrers.cs | 124 ++++++++++++++++++ .../Registry/Remote/Repository.cs | 1 + .../Registry/Remote/RepositoryOptions.cs | 7 + 11 files changed, 369 insertions(+), 7 deletions(-) create mode 100644 src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs create mode 100644 src/OrasProject.Oras/Oci/Artifact.cs create mode 100644 src/OrasProject.Oras/Registry/Remote/Auth/Client.cs create mode 100644 src/OrasProject.Oras/Registry/Remote/Referrers.cs diff --git a/src/OrasProject.Oras/Content/Digest.cs b/src/OrasProject.Oras/Content/Digest.cs index 2e8c0350..e87ed5c0 100644 --- a/src/OrasProject.Oras/Content/Digest.cs +++ b/src/OrasProject.Oras/Content/Digest.cs @@ -47,6 +47,18 @@ internal static string Validate(string? digest) return digest; } + internal static string GetAlgorithm(string digest) + { + var validatedDigest = Validate(digest); + return validatedDigest.Split(':')[0]; + } + + internal static string GetRef(string digest) + { + var validatedDigest = Validate(digest); + return validatedDigest.Split(':')[1]; + } + /// /// Generates a SHA-256 digest from a byte array. /// @@ -59,4 +71,4 @@ internal static string ComputeSHA256(byte[] content) var output = $"sha256:{BitConverter.ToString(hash).Replace("-", "")}"; return output.ToLower(); } -} \ No newline at end of file +} diff --git a/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs b/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs new file mode 100644 index 00000000..a7eaf799 --- /dev/null +++ b/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs @@ -0,0 +1,20 @@ +using System; + +namespace OrasProject.Oras.Exceptions; + +public class NoReferrerUpdateException : Exception +{ + public NoReferrerUpdateException() + { + } + + public NoReferrerUpdateException(string message) + : base(message) + { + } + + public NoReferrerUpdateException(string message, Exception innerException) + : base(message, innerException) + { + } +} diff --git a/src/OrasProject.Oras/Oci/Artifact.cs b/src/OrasProject.Oras/Oci/Artifact.cs new file mode 100644 index 00000000..4edaa82f --- /dev/null +++ b/src/OrasProject.Oras/Oci/Artifact.cs @@ -0,0 +1,26 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace OrasProject.Oras.Oci; + +public class Artifact +{ + [JsonPropertyName("mediaType")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string? MediaType { get; set; } + + [JsonPropertyName("artifactType")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public string? ArtifactType { get; set; } + + [JsonPropertyName("blobs")] + public required IList Blobs { get; set; } + + [JsonPropertyName("subject")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public Descriptor? Subject { get; set; } + + [JsonPropertyName("annotations")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public IDictionary? Annotations { get; set; } +} diff --git a/src/OrasProject.Oras/Oci/Descriptor.cs b/src/OrasProject.Oras/Oci/Descriptor.cs index 9720e151..f06dd30e 100644 --- a/src/OrasProject.Oras/Oci/Descriptor.cs +++ b/src/OrasProject.Oras/Oci/Descriptor.cs @@ -13,6 +13,7 @@ using System.Collections.Generic; using System.Text.Json.Serialization; +using OrasProject.Oras.Content; namespace OrasProject.Oras.Oci; @@ -48,4 +49,16 @@ public class Descriptor public string? ArtifactType { get; set; } internal BasicDescriptor BasicDescriptor => new BasicDescriptor(MediaType, Digest, Size); + + internal static bool IsEmptyOrNull(Descriptor? descriptor) + { + return descriptor == null || descriptor.Size == 0 || string.IsNullOrEmpty(descriptor.Digest) || string.IsNullOrEmpty(descriptor.MediaType); + } + + internal static Descriptor EmptyDescriptor() => new Descriptor + { + MediaType = "", + Digest = "", + Size = 0 + }; } diff --git a/src/OrasProject.Oras/Oci/Index.cs b/src/OrasProject.Oras/Oci/Index.cs index e11eff26..e6329232 100644 --- a/src/OrasProject.Oras/Oci/Index.cs +++ b/src/OrasProject.Oras/Oci/Index.cs @@ -12,7 +12,10 @@ // limitations under the License. using System.Collections.Generic; +using System.Text; +using System.Text.Json; using System.Text.Json.Serialization; +using OrasProject.Oras.Content; namespace OrasProject.Oras.Oci; @@ -39,4 +42,23 @@ public class Index : Versioned [JsonPropertyName("annotations")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public IDictionary? Annotations { get; set; } + + internal static (Descriptor, byte[]) GenerateIndex(IList manifests) + { + var index = new Index() + { + Manifests = manifests, + MediaType = Oci.MediaType.ImageIndex, + SchemaVersion = 2 + }; + var indexContent = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(index)); + var indexDesc = new Descriptor() + { + Digest = Digest.ComputeSHA256(indexContent), + MediaType = Oci.MediaType.ImageIndex, + Size = indexContent.Length + }; + + return (indexDesc, indexContent); + } } diff --git a/src/OrasProject.Oras/Registry/Remote/Auth/Client.cs b/src/OrasProject.Oras/Registry/Remote/Auth/Client.cs new file mode 100644 index 00000000..0decf9aa --- /dev/null +++ b/src/OrasProject.Oras/Registry/Remote/Auth/Client.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace OrasProject.Oras.Registry.Remote.Auth +{ + internal class Client + { + + } +} diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index eb08dca3..0076c065 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -26,7 +26,7 @@ namespace OrasProject.Oras.Registry.Remote; internal static class HttpResponseMessageExtensions { private const string _dockerContentDigestHeader = "Docker-Content-Digest"; - + /// /// Parses the error returned by the remote registry. /// @@ -101,6 +101,14 @@ public static void VerifyContentDigest(this HttpResponseMessage response, string throw new HttpIOException(HttpRequestError.InvalidResponse, $"{response.RequestMessage!.Method} {response.RequestMessage.RequestUri}: invalid response; digest mismatch in Docker-Content-Digest: received {contentDigest} when expecting {digestStr}"); } } + + public static void CheckOciSubjectHeader(this HttpResponseMessage response, Repository repository) + { + if (response.Headers.TryGetValues("OCI-Subject", out var values)) + { + repository.ReferrerState = Referrers.ReferrerState.ReferrerSupported; + } + } /// /// Returns a descriptor generated from the response. @@ -160,7 +168,7 @@ public static async Task GenerateDescriptorAsync(this HttpResponseMe { serverDigest = serverHeaderDigest.FirstOrDefault(); if (!string.IsNullOrEmpty(serverDigest)) - { + { response.VerifyContentDigest(serverDigest); } } diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 549a02bb..881a55ed 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -14,11 +14,15 @@ using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; using System; +using System.Collections.Generic; using System.IO; using System.Net; using System.Net.Http; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using OrasProject.Oras.Content; +using Index = OrasProject.Oras.Oci.Index; namespace OrasProject.Oras.Registry.Remote; @@ -140,7 +144,9 @@ public async Task ExistsAsync(Descriptor target, CancellationToken cancell /// /// public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) - => await InternalPushAsync(expected, content, expected.Digest, cancellationToken).ConfigureAwait(false); + { + await PushWithIndexingAsync(expected, content, expected.Digest, cancellationToken).ConfigureAwait(false); + } /// /// PushReferenceASync pushes the manifest with a reference tag. @@ -153,9 +159,116 @@ public async Task PushAsync(Descriptor expected, Stream content, CancellationTok public async Task PushAsync(Descriptor expected, Stream content, string reference, CancellationToken cancellationToken = default) { var contentReference = Repository.ParseReference(reference).ContentReference!; - await InternalPushAsync(expected, content, contentReference, cancellationToken).ConfigureAwait(false); + await PushWithIndexingAsync(expected, content, contentReference, cancellationToken).ConfigureAwait(false); + } + + private async Task PushWithIndexingAsync(Descriptor expected, Stream content, string reference, + CancellationToken cancellationToken = default) + { + switch (expected.MediaType) + { + case MediaType.ImageManifest: + case MediaType.ImageIndex: + if (Repository.ReferrerState == Referrers.ReferrerState.ReferrerSupported) + { + await InternalPushAsync(expected, content, reference, cancellationToken).ConfigureAwait(false); + return; + } + + var contentBytes = await content.ReadAllAsync(expected, cancellationToken); + // var initPosition = content.Position; + await InternalPushAsync(expected, new MemoryStream(contentBytes), reference, cancellationToken).ConfigureAwait(false); + if (Repository.ReferrerState == Referrers.ReferrerState.ReferrerSupported) + { + return; + } + // content.Seek(initPosition, SeekOrigin.Begin); + await IndexReferrersToPush(expected, new MemoryStream(contentBytes)); + break; + default: + await InternalPushAsync(expected, content, reference, cancellationToken); + break; + } + } + + private async Task IndexReferrersToPush(Descriptor desc, Stream content, CancellationToken cancellationToken = default) + { + Descriptor? subject = null; + switch (desc.MediaType) + { + case MediaType.ImageIndex: + var indexManifest = JsonSerializer.Deserialize(content); + if (indexManifest?.Subject == null) return; + subject = indexManifest.Subject; + desc.ArtifactType = indexManifest.ArtifactType; + desc.Annotations = indexManifest.Annotations; + break; + case MediaType.ImageManifest: + var imageManifest = JsonSerializer.Deserialize(content); + if (imageManifest?.Subject == null) return; + desc.ArtifactType = string.IsNullOrEmpty(imageManifest.ArtifactType) ? imageManifest.Config.MediaType : imageManifest.ArtifactType; + desc.Annotations = imageManifest.Annotations; + break; + default: + return; + } + + Repository.ReferrerState = Referrers.ReferrerState.ReferrerNotSupported; + if (subject == null) + { + throw new InvalidOperationException("Subject was not initialized"); + } + await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd)); } + private async Task UpdateReferrersIndex(Descriptor subject, + Referrers.ReferrerChange referrerChange, CancellationToken cancellationToken = default) + { + try + { + var referrersTag = Referrers.BuildReferrersTag(subject); + var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag); + var updatedReferrers = + Referrers.ApplyReferrerChanges(oldReferrers, new List { referrerChange }); + + if (updatedReferrers.Count > 0 || repository.Options.SkipReferrersGc) + { + var (indexDesc, indexContent) = Index.GenerateIndex(updatedReferrers); + await InternalPushAsync(indexDesc, new MemoryStream(indexContent), referrersTag, cancellationToken).ConfigureAwait(false); + } + + if (repository.Options.SkipReferrersGc || Descriptor.IsEmptyOrNull(oldDesc)) + { + return; + } + // delete oldIndexDesc + await DeleteAsync(oldDesc, cancellationToken).ConfigureAwait(false); + } + catch (NoReferrerUpdateException) + { + return; + } + } + + internal async Task<(Descriptor, IList)> PullReferrersIndexList(string referrersTag, CancellationToken cancellationToken = default) + { + try + { + var (desc, content) = await FetchAsync(referrersTag); + var index = JsonSerializer.Deserialize(content); + if (index == null) + { + throw new JsonException("null index manifests list"); + } + return (desc, index.Manifests); + } + catch (NotFoundException) + { + return (Descriptor.EmptyDescriptor(), new List()); + } + } + + /// /// Pushes the manifest content, matching the expected descriptor. /// @@ -163,20 +276,23 @@ public async Task PushAsync(Descriptor expected, Stream content, string referenc /// /// /// - private async Task InternalPushAsync(Descriptor expected, Stream stream, string contentReference, CancellationToken cancellationToken) + private async Task InternalPushAsync(Descriptor expected, Stream stream, string contentReference, + CancellationToken cancellationToken) { - var remoteReference = Repository.ParseReference(contentReference); + var remoteReference = Repository.ParseReference(contentReference); // duplicate? var url = new UriFactory(remoteReference, Repository.Options.PlainHttp).BuildRepositoryManifest(); var request = new HttpRequestMessage(HttpMethod.Put, url); request.Content = new StreamContent(stream); request.Content.Headers.ContentLength = expected.Size; request.Content.Headers.Add("Content-Type", expected.MediaType); + var client = Repository.Options.HttpClient; using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); if (response.StatusCode != HttpStatusCode.Created) { throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); } + response.CheckOciSubjectHeader(Repository); response.VerifyContentDigest(expected.Digest); } diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs new file mode 100644 index 00000000..ac25d8e9 --- /dev/null +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -0,0 +1,124 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Collections.Generic; +using OrasProject.Oras.Content; +using OrasProject.Oras.Exceptions; +using OrasProject.Oras.Oci; + +namespace OrasProject.Oras.Registry.Remote; + +public class Referrers +{ + internal enum ReferrerState + { + ReferrerUnknown, + ReferrerSupported, + ReferrerNotSupported + } + + internal record ReferrerChange(Descriptor Referrer, ReferrerOperation ReferrerOperation); + + internal enum ReferrerOperation + { + ReferrerAdd, + ReferrerDelete, + } + + public static string BuildReferrersTag(Descriptor descriptor) + { + return Digest.GetAlgorithm(descriptor.Digest) + "-" + Digest.GetRef(descriptor.Digest); + } + + internal static IList ApplyReferrerChanges(IList oldReferrers, IList referrerChanges) + { + if (oldReferrers == null || referrerChanges == null) + { + throw new NoReferrerUpdateException("referrerChanges or oldReferrers is null in this request"); + } + var updatedReferrers = new List(); + var referrerToIndex = new Dictionary(); + + var updateRequired = false; + foreach (var oldReferrer in oldReferrers) + { + if (Descriptor.IsEmptyOrNull(oldReferrer)) + { + updateRequired = true; + continue; + } + var basicDesc = oldReferrer.BasicDescriptor; + if (referrerToIndex.ContainsKey(basicDesc)) + { + updateRequired = true; + continue; + } + updatedReferrers.Add(oldReferrer); + referrerToIndex[basicDesc] = updatedReferrers.Count - 1; + } + + foreach (var change in referrerChanges) + { + if (Descriptor.IsEmptyOrNull(change.Referrer)) continue; + var basicDesc = change.Referrer.BasicDescriptor; + switch (change.ReferrerOperation) + { + case ReferrerOperation.ReferrerAdd: + if (!referrerToIndex.ContainsKey(basicDesc)) + { + updatedReferrers.Add(change.Referrer); + referrerToIndex[basicDesc] = updatedReferrers.Count - 1; + } + break; + + case ReferrerOperation.ReferrerDelete: + if (referrerToIndex.TryGetValue(basicDesc, out var index)) + { + updatedReferrers[index] = Descriptor.EmptyDescriptor(); + referrerToIndex.Remove(basicDesc); + } + break; + default: + break; + } + } + + if (!updateRequired && referrerToIndex.Count == oldReferrers.Count) + { + foreach (var oldReferrer in oldReferrers) + { + var basicDesc = oldReferrer.BasicDescriptor; + if (!referrerToIndex.ContainsKey(basicDesc)) updateRequired = true; + } + + if (!updateRequired) throw new NoReferrerUpdateException("no referrer update in this request"); + } + + RemoveEmptyDescriptors(updatedReferrers, referrerToIndex.Count); + return updatedReferrers; + } + + internal static void RemoveEmptyDescriptors(List updatedReferrers, int numNonEmptyReferrers) + { + var lastEmptyIndex = 0; + for (var i = 0; i < updatedReferrers.Count; ++i) + { + if (Descriptor.IsEmptyOrNull(updatedReferrers[i])) continue; + + if (i > lastEmptyIndex) updatedReferrers[lastEmptyIndex] = updatedReferrers[i]; + ++lastEmptyIndex; + if (lastEmptyIndex == numNonEmptyReferrers) break; + } + updatedReferrers.RemoveRange(lastEmptyIndex, updatedReferrers.Count - lastEmptyIndex); + } +} diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 62d73bc2..33c77207 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -46,6 +46,7 @@ public class Repository : IRepository public IManifestStore Manifests => new ManifestStore(this); public RepositoryOptions Options => _opts; + internal Referrers.ReferrerState ReferrerState { get; set; } = Referrers.ReferrerState.ReferrerUnknown; internal static readonly string[] DefaultManifestMediaTypes = [ diff --git a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs index 0302ea68..1d7b45a4 100644 --- a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs +++ b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs @@ -50,4 +50,11 @@ public struct RepositoryOptions /// Reference: https://docs.docker.com/registry/spec/api/#tags /// public int TagListPageSize { get; set; } + + // SkipReferrersGc specifies whether to delete the dangling referrers + // index when referrers tag schema is utilized. + // - If false, the old referrers index will be deleted after the new one is successfully uploaded. + // - If true, the old referrers index is kept. + // By default, it is disabled (set to false). See also: + public bool SkipReferrersGc { get; set; } } From 846e1739728a30d24abb5a37c4b089e5f4f59770 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 19 Nov 2024 17:45:31 +1100 Subject: [PATCH 02/36] add unit tests Signed-off-by: Patrick Pan --- .../Exceptions/NoReferrerUpdateException.cs | 19 +- .../Registry/Remote/ManifestStore.cs | 69 +- .../Registry/Remote/Referrers.cs | 39 +- .../Content/MemoryStoreTest.cs | 3 +- .../Exceptions/ExceptionTest.cs | 8 + tests/OrasProject.Oras.Tests/Oci/IndexTest.cs | 66 + .../Remote/ManifestStoreTest.cs | 359 +++ .../Remote/ReferrersTest.cs | 227 ++ .../Remote/RepositoryTest.cs | 2285 +---------------- .../Remote/Util/RandomDataGenerator.cs | 76 + .../Remote/Util/Util.cs | 41 + 11 files changed, 882 insertions(+), 2310 deletions(-) create mode 100644 tests/OrasProject.Oras.Tests/Oci/IndexTest.cs create mode 100644 tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs create mode 100644 tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs create mode 100644 tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs create mode 100644 tests/OrasProject.Oras.Tests/Remote/Util/Util.cs diff --git a/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs b/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs index a7eaf799..87ab2b5a 100644 --- a/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs +++ b/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs @@ -1,7 +1,24 @@ -using System; +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; namespace OrasProject.Oras.Exceptions; + +/// +/// NoReferrerUpdateException is thrown when no referrer update is needed. +/// public class NoReferrerUpdateException : Exception { public NoReferrerUpdateException() diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 881a55ed..5a6a0907 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -162,6 +162,16 @@ public async Task PushAsync(Descriptor expected, Stream content, string referenc await PushWithIndexingAsync(expected, content, contentReference, cancellationToken).ConfigureAwait(false); } + /// + /// PushWithIndexingAsync pushes the given manifest to the repository with indexing support. + /// If referrer support is not enabled, the function will first push the content, then process and update + /// the referrers index before pushing the content again. It handles both image manifests and index manifests. + /// + /// + /// + /// + /// + /// private async Task PushWithIndexingAsync(Descriptor expected, Stream content, string reference, CancellationToken cancellationToken = default) { @@ -176,14 +186,19 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, st } var contentBytes = await content.ReadAllAsync(expected, cancellationToken); - // var initPosition = content.Position; - await InternalPushAsync(expected, new MemoryStream(contentBytes), reference, cancellationToken).ConfigureAwait(false); + using (var contentDuplicate = new MemoryStream(contentBytes)) + { + await InternalPushAsync(expected, contentDuplicate, reference, cancellationToken).ConfigureAwait(false); + } if (Repository.ReferrerState == Referrers.ReferrerState.ReferrerSupported) { return; } - // content.Seek(initPosition, SeekOrigin.Begin); - await IndexReferrersToPush(expected, new MemoryStream(contentBytes)); + + using (var contentDuplicate = new MemoryStream(contentBytes)) + { + await ProcessReferrersAndPushIndex(expected, contentDuplicate); + } break; default: await InternalPushAsync(expected, content, reference, cancellationToken); @@ -191,7 +206,17 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, st } } - private async Task IndexReferrersToPush(Descriptor desc, Stream content, CancellationToken cancellationToken = default) + /// + /// ProcessReferrersAndPushIndex processes the referrers for the given descriptor by deserializing its content + /// (either as an image manifest or image index), extracting relevant metadata + /// such as the subject, artifact type, and annotations, and then updates the + /// referrers index if applicable. + /// + /// + /// + /// + /// + private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, CancellationToken cancellationToken = default) { Descriptor? subject = null; switch (desc.MediaType) @@ -206,6 +231,7 @@ private async Task IndexReferrersToPush(Descriptor desc, Stream content, Cancell case MediaType.ImageManifest: var imageManifest = JsonSerializer.Deserialize(content); if (imageManifest?.Subject == null) return; + subject = imageManifest.Subject; desc.ArtifactType = string.IsNullOrEmpty(imageManifest.ArtifactType) ? imageManifest.Config.MediaType : imageManifest.ArtifactType; desc.Annotations = imageManifest.Annotations; break; @@ -214,13 +240,18 @@ private async Task IndexReferrersToPush(Descriptor desc, Stream content, Cancell } Repository.ReferrerState = Referrers.ReferrerState.ReferrerNotSupported; - if (subject == null) - { - throw new InvalidOperationException("Subject was not initialized"); - } await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd)); } + /// + /// UpdateReferrersIndex updates the referrers index for a given subject by applying the specified referrer changes. + /// If the referrers index is updated, the new index is pushed to the repository. If referrers + /// garbage collection is not skipped, the old index is deleted. + /// + /// + /// + /// + /// private async Task UpdateReferrersIndex(Descriptor subject, Referrers.ReferrerChange referrerChange, CancellationToken cancellationToken = default) { @@ -229,19 +260,22 @@ private async Task UpdateReferrersIndex(Descriptor subject, var referrersTag = Referrers.BuildReferrersTag(subject); var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag); var updatedReferrers = - Referrers.ApplyReferrerChanges(oldReferrers, new List { referrerChange }); + Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); if (updatedReferrers.Count > 0 || repository.Options.SkipReferrersGc) { var (indexDesc, indexContent) = Index.GenerateIndex(updatedReferrers); - await InternalPushAsync(indexDesc, new MemoryStream(indexContent), referrersTag, cancellationToken).ConfigureAwait(false); + using (var content = new MemoryStream(indexContent)) + { + await InternalPushAsync(indexDesc, content, referrersTag, cancellationToken).ConfigureAwait(false); + } } if (repository.Options.SkipReferrersGc || Descriptor.IsEmptyOrNull(oldDesc)) { return; } - // delete oldIndexDesc + await DeleteAsync(oldDesc, cancellationToken).ConfigureAwait(false); } catch (NoReferrerUpdateException) @@ -250,6 +284,15 @@ private async Task UpdateReferrersIndex(Descriptor subject, } } + /// + /// PullReferrersIndexList retrieves the referrers index list associated with the given referrers tag. + /// It fetches the index manifest from the repository, deserializes it into an `Index` object, + /// and returns the descriptor along with the list of manifests (referrers). If the referrers index is not found, + /// an empty descriptor and an empty list are returned. + /// + /// + /// + /// internal async Task<(Descriptor, IList)> PullReferrersIndexList(string referrersTag, CancellationToken cancellationToken = default) { try @@ -279,7 +322,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, private async Task InternalPushAsync(Descriptor expected, Stream stream, string contentReference, CancellationToken cancellationToken) { - var remoteReference = Repository.ParseReference(contentReference); // duplicate? + var remoteReference = Repository.ParseReference(contentReference); var url = new UriFactory(remoteReference, Repository.Options.PlainHttp).BuildRepositoryManifest(); var request = new HttpRequestMessage(HttpMethod.Put, url); request.Content = new StreamContent(stream); diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index ac25d8e9..bbc1a7c0 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -35,16 +35,24 @@ internal enum ReferrerOperation ReferrerDelete, } - public static string BuildReferrersTag(Descriptor descriptor) + internal static string BuildReferrersTag(Descriptor descriptor) { return Digest.GetAlgorithm(descriptor.Digest) + "-" + Digest.GetRef(descriptor.Digest); } - internal static IList ApplyReferrerChanges(IList oldReferrers, IList referrerChanges) + /// + /// ApplyReferrerChanges applies the specified referrer change (either add or delete) to the existing list of referrers. + /// It updates the list based on the operation defined in the provided `referrerChange`. + /// If the referrer to be added or deleted already exists in the list, it is handled accordingly. + /// + /// + /// + /// The updated referrers list + internal static IList ApplyReferrerChanges(IList oldReferrers, ReferrerChange referrerChange) { - if (oldReferrers == null || referrerChanges == null) + if (oldReferrers == null || referrerChange == null) { - throw new NoReferrerUpdateException("referrerChanges or oldReferrers is null in this request"); + throw new NoReferrerUpdateException("referrerChange or oldReferrers is null in this request"); } var updatedReferrers = new List(); var referrerToIndex = new Dictionary(); @@ -67,32 +75,34 @@ internal static IList ApplyReferrerChanges(IList oldRefe referrerToIndex[basicDesc] = updatedReferrers.Count - 1; } - foreach (var change in referrerChanges) + + if (!Descriptor.IsEmptyOrNull(referrerChange.Referrer)) { - if (Descriptor.IsEmptyOrNull(change.Referrer)) continue; - var basicDesc = change.Referrer.BasicDescriptor; - switch (change.ReferrerOperation) + var basicDesc = referrerChange.Referrer.BasicDescriptor; + switch (referrerChange.ReferrerOperation) { case ReferrerOperation.ReferrerAdd: if (!referrerToIndex.ContainsKey(basicDesc)) { - updatedReferrers.Add(change.Referrer); + updatedReferrers.Add(referrerChange.Referrer); referrerToIndex[basicDesc] = updatedReferrers.Count - 1; } + break; - + case ReferrerOperation.ReferrerDelete: if (referrerToIndex.TryGetValue(basicDesc, out var index)) { updatedReferrers[index] = Descriptor.EmptyDescriptor(); referrerToIndex.Remove(basicDesc); } + break; default: break; } } - + if (!updateRequired && referrerToIndex.Count == oldReferrers.Count) { foreach (var oldReferrer in oldReferrers) @@ -108,6 +118,13 @@ internal static IList ApplyReferrerChanges(IList oldRefe return updatedReferrers; } + /// + /// RemoveEmptyDescriptors removes any empty or null descriptors from the provided list of referrers, ensuring that only non-empty + /// descriptors remain in the list. It optimizes the list by shifting valid descriptors forward and trimming + /// the remaining elements at the end. The list is truncated to only contain non-empty descriptors up to the specified count. + /// + /// + /// internal static void RemoveEmptyDescriptors(List updatedReferrers, int numNonEmptyReferrers) { var lastEmptyIndex = 0; diff --git a/tests/OrasProject.Oras.Tests/Content/MemoryStoreTest.cs b/tests/OrasProject.Oras.Tests/Content/MemoryStoreTest.cs index 42c76763..d7a28ea1 100644 --- a/tests/OrasProject.Oras.Tests/Content/MemoryStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Content/MemoryStoreTest.cs @@ -14,6 +14,7 @@ using OrasProject.Oras.Content; using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; +using Index = OrasProject.Oras.Oci.Index; using System.Text; using System.Text.Json; using Xunit; @@ -195,7 +196,7 @@ public async Task ShouldReturnPredecessorsOfNodes() var generateIndex = (List manifests) => { - var index = new Oci.Index + var index = new Index { Manifests = manifests }; diff --git a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs index 0a426e78..84ee0e62 100644 --- a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs +++ b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs @@ -49,4 +49,12 @@ public async Task NotFoundException() await Assert.ThrowsAsync(() => throw new NotFoundException("Not found")); await Assert.ThrowsAsync(() => throw new NotFoundException("Not found", null)); } + + [Fact] + public async Task NoReferrerUpdateException() + { + await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException()); + await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException("No referrer update")); + await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException("No referrer update", null)); + } } diff --git a/tests/OrasProject.Oras.Tests/Oci/IndexTest.cs b/tests/OrasProject.Oras.Tests/Oci/IndexTest.cs new file mode 100644 index 00000000..dacf8b25 --- /dev/null +++ b/tests/OrasProject.Oras.Tests/Oci/IndexTest.cs @@ -0,0 +1,66 @@ +using System.Text.Json; +using OrasProject.Oras.Content; +using OrasProject.Oras.Oci; +using static OrasProject.Oras.Tests.Remote.Util.Util; +using Xunit; +using Index = OrasProject.Oras.Oci.Index; + +namespace OrasProject.Oras.Tests.Oci; + +public class IndexTest +{ + [Fact] + public void GenerateIndex_CorrectlyGeneratesIndexDescriptor() + { + var expectedManifests = new List + { + new Descriptor + { + Digest = "digest1", + MediaType = MediaType.ImageManifest, + Size = 100 + }, + new Descriptor + { + Digest = "digest2", + MediaType = MediaType.ImageManifest, + Size = 200 + } + }; + + var (generatedIndexDesc, generatedIndexContent) = Index.GenerateIndex(expectedManifests); + Assert.NotNull(generatedIndexDesc); + Assert.Equal(MediaType.ImageIndex, generatedIndexDesc.MediaType); + Assert.Equal(generatedIndexContent.Length, generatedIndexDesc.Size); + Assert.Equal(Digest.ComputeSHA256(generatedIndexContent), generatedIndexDesc.Digest); + + var generatedIndex = JsonSerializer.Deserialize(generatedIndexContent); + Assert.NotNull(generatedIndex); + Assert.Equal(2, generatedIndex.Manifests.Count); + for (var i = 0; i < generatedIndex.Manifests.Count; ++i) + { + Assert.True(AreDescriptorsEqual(generatedIndex.Manifests[i], expectedManifests[i])); + } + Assert.Equal(MediaType.ImageIndex, generatedIndex.MediaType); + Assert.Equal(2, generatedIndex.SchemaVersion); + } + + [Fact] + public void GenerateIndex_CorrectlyGeneratesIndexDescriptorWithEmptyManifests() + { + var expectedManifests = new List(); + var (generatedIndexDesc, generatedIndexContent) = Index.GenerateIndex(expectedManifests); + + Assert.NotNull(generatedIndexDesc); + Assert.Equal(MediaType.ImageIndex, generatedIndexDesc.MediaType); + Assert.Equal(generatedIndexContent.Length, generatedIndexDesc.Size); + Assert.Equal(Digest.ComputeSHA256(generatedIndexContent), generatedIndexDesc.Digest); + + var generatedIndex = JsonSerializer.Deserialize(generatedIndexContent); + Assert.NotNull(generatedIndex); + Assert.Empty(generatedIndex.Manifests); + Assert.Equal(expectedManifests, generatedIndex.Manifests); + Assert.Equal(MediaType.ImageIndex, generatedIndex.MediaType); + Assert.Equal(2, generatedIndex.SchemaVersion); + } +} diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs new file mode 100644 index 00000000..83fd80fa --- /dev/null +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -0,0 +1,359 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using OrasProject.Oras.Oci; +using OrasProject.Oras.Registry; +using OrasProject.Oras.Registry.Remote; +using static OrasProject.Oras.Tests.Remote.Util.RandomDataGenerator; +using static OrasProject.Oras.Tests.Remote.Util.Util; +using static OrasProject.Oras.Content.Digest; +using Index = OrasProject.Oras.Oci.Index; + + +using Xunit; +using Xunit.Abstractions; + +namespace OrasProject.Oras.Tests.Remote; + +public class ManifestStoreTest +{ + private const string _dockerContentDigestHeader = "Docker-Content-Digest"; + + private ITestOutputHelper _output; + + public ManifestStoreTest(ITestOutputHelper output) + { + _output = output; + } + + /// + /// ManifestStore_PushAsyncWithSubjectAndReferrerSupported tests PushAsync method for pushing manifest with subject when registry supports referrers API + /// + [Fact] + public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() + { + var (_, manifestBytes) = RandomManifestWithSubject(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifestBytes), + Size = manifestBytes.Length + }; + byte[]? receivedManifest = null; + + var mockHttpRequestHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + if (req.Content?.Headers?.ContentLength != null) + { + var buf = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buf, 0); + receivedManifest = buf; + } + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.StatusCode = HttpStatusCode.Created; + res.Headers.Add("OCI-Subject", "test"); + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + await store.PushAsync(manifestDesc, new MemoryStream(manifestBytes), cancellationToken); + Assert.Equal(manifestBytes, receivedManifest); + Assert.Equal(Referrers.ReferrerState.ReferrerSupported, repo.ReferrerState); + } + + [Fact] + public async Task ManifestStore_PullReferrersIndexListSuccessfully() + { + var expectedIndex = RandomIndex(); + var expectedIndexBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(expectedIndex)); + var expectedIndexDesc = new Descriptor() + { + Digest = ComputeSHA256(expectedIndexBytes), + MediaType = MediaType.ImageIndex, + Size = expectedIndexBytes.Length + }; + + var mockedHttpHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedIndexDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(expectedIndexBytes); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedIndexDesc.Digest }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockedHttpHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + var (receivedDesc, receivedManifests) = await store.PullReferrersIndexList(expectedIndexDesc.Digest, cancellationToken); + Assert.True(AreDescriptorsEqual(expectedIndexDesc, receivedDesc)); + for (var i = 0; i < receivedManifests.Count; ++i) + { + Assert.True(AreDescriptorsEqual(expectedIndex.Manifests[i], receivedManifests[i])); + } + } + + [Fact] + public async Task ManifestStore_PullReferrersIndexListNotFound() + { + var mockedHttpHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockedHttpHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + var (receivedDesc, receivedManifests) = await store.PullReferrersIndexList("test", cancellationToken); + Assert.True(Descriptor.IsEmptyOrNull(receivedDesc)); + Assert.Empty(receivedManifests); + } + + + [Fact] + public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupported() + { + var oldIndex = RandomIndex(); + var oldIndexBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(oldIndex)); + var oldIndexDesc = new Descriptor() + { + Digest = ComputeSHA256(oldIndexBytes), + MediaType = MediaType.ImageIndex, + Size = oldIndexBytes.Length + }; + + // first push + var (firstExpectedManifest, firstExpectedManifestBytes) = RandomManifestWithSubject(); + var firstExpectedManifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(firstExpectedManifestBytes), + Size = firstExpectedManifestBytes.Length, + ArtifactType = MediaType.ImageConfig, + }; + var firstExpectedReferrersList = new List(oldIndex.Manifests); + firstExpectedReferrersList.Add(firstExpectedManifestDesc); + var (firstExpectedIndexReferrersDesc, firstExpectedIndexReferrersBytes) = Index.GenerateIndex(firstExpectedReferrersList); + + // second push + var (_, secondExpectedManifestBytes) = RandomManifestWithSubject(firstExpectedManifest.Subject); + var secondExpectedManifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(secondExpectedManifestBytes), + Size = secondExpectedManifestBytes.Length, + ArtifactType = MediaType.ImageConfig, + }; + var secondExpectedReferrersList = new List(oldIndex.Manifests); + secondExpectedReferrersList.Add(secondExpectedManifestDesc); + var (secondExpectedIndexReferrersDesc, secondExpectedIndexReferrersBytes) = Index.GenerateIndex(secondExpectedReferrersList); + + byte[]? receivedManifestContent = null; + byte[]? receivedIndexContent = null; + var referrersTag = Referrers.BuildReferrersTag(firstExpectedManifest.Subject); + var oldIndexDeleted = false; + var firstIndexDeleted = false; + var mockHttpRequestHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var response = new HttpResponseMessage(); + response.RequestMessage = req; + + if (req.Method == HttpMethod.Put && ( + req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{firstExpectedManifestDesc.Digest}" || + req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{secondExpectedManifestDesc.Digest}" || + req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{referrersTag}")) + { + if (req.Content?.Headers?.ContentLength != null) + { + var buffer = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buffer, 0); + if (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{firstExpectedManifestDesc.Digest}" || + req.RequestUri.AbsolutePath == $"/v2/test/manifests/{secondExpectedManifestDesc.Digest}") receivedManifestContent = buffer; + else receivedIndexContent = buffer; + } + + if (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{firstExpectedManifestDesc.Digest}") + response.Headers.Add(_dockerContentDigestHeader, new[] { firstExpectedManifestDesc.Digest }); + else if (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{secondExpectedManifestDesc.Digest}") + response.Headers.Add(_dockerContentDigestHeader, new[] { secondExpectedManifestDesc.Digest }); + else if (!oldIndexDeleted) response.Headers.Add(_dockerContentDigestHeader, new[] { firstExpectedIndexReferrersDesc.Digest }); + else response.Headers.Add(_dockerContentDigestHeader, new[] { secondExpectedIndexReferrersDesc.Digest }); + + response.StatusCode = HttpStatusCode.Created; + return response; + } else if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{referrersTag}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(oldIndexBytes); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + if (oldIndexDeleted) response.Headers.Add(_dockerContentDigestHeader, new string[] { firstExpectedIndexReferrersDesc.Digest }); + else response.Headers.Add(_dockerContentDigestHeader, new string[] { oldIndexDesc.Digest }); + response.StatusCode = HttpStatusCode.OK; + return response; + } else if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{oldIndexDesc.Digest}") + { + response.Headers.Add(_dockerContentDigestHeader, new[] { oldIndexDesc.Digest }); + response.StatusCode = HttpStatusCode.Accepted; + oldIndexDeleted = true; + return response; + } else if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{firstExpectedIndexReferrersDesc.Digest}") + { + response.Headers.Add(_dockerContentDigestHeader, new[] { firstExpectedIndexReferrersDesc.Digest }); + response.StatusCode = HttpStatusCode.Accepted; + firstIndexDeleted = true; + return response; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + // First push with referrer tag schema + Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + await store.PushAsync(firstExpectedManifestDesc, new MemoryStream(firstExpectedManifestBytes), cancellationToken); + Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + Assert.Equal(firstExpectedManifestBytes, receivedManifestContent); + Assert.True(oldIndexDeleted); + Assert.Equal(firstExpectedIndexReferrersBytes, receivedIndexContent); + + + // Second push with referrer tag schema + Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + await store.PushAsync(secondExpectedManifestDesc, new MemoryStream(secondExpectedManifestBytes), cancellationToken); + Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + Assert.Equal(secondExpectedManifestBytes, receivedManifestContent); + Assert.True(firstIndexDeleted); + Assert.Equal(secondExpectedIndexReferrersBytes, receivedIndexContent); + } + + [Fact] + public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupportedWithoutOldIndex() + { + var expectedIndexManifest = new Index() + { + Subject = RandomDescriptor(), + Manifests = new List{ RandomDescriptor(), RandomDescriptor() }, + MediaType = MediaType.ImageIndex, + ArtifactType = MediaType.ImageIndex, + }; + + var expectedIndexManifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(expectedIndexManifest)); + var expectedIndexManifestDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(expectedIndexManifestBytes), + Size = expectedIndexManifestBytes.Length, + ArtifactType = MediaType.ImageIndex, + }; + var expectedReferrers = new List + { + expectedIndexManifestDesc, + }; + + var (expectedIndexReferrersDesc, expectedIndexReferrersBytes) = Index.GenerateIndex(expectedReferrers); + + byte[]? receivedIndexManifestContent = null; + byte[]? receivedIndexReferrersContent = null; + var referrersTag = Referrers.BuildReferrersTag(expectedIndexManifest.Subject); + + var mockHttpRequestHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var response = new HttpResponseMessage(); + response.RequestMessage = req; + + if (req.Method == HttpMethod.Put && ( + req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedIndexManifestDesc.Digest}" || + req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{referrersTag}")) + { + if (req.Content?.Headers?.ContentLength != null) + { + var buffer = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buffer, 0); + if (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{expectedIndexManifestDesc.Digest}") receivedIndexManifestContent = buffer; + else receivedIndexReferrersContent = buffer; + } + if (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{expectedIndexManifestDesc.Digest}") + { + response.Headers.Add(_dockerContentDigestHeader, new[] { expectedIndexManifestDesc.Digest }); + } else response.Headers.Add(_dockerContentDigestHeader, new[] { expectedIndexReferrersDesc.Digest }); + response.StatusCode = HttpStatusCode.Created; + return response; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + await store.PushAsync(expectedIndexManifestDesc, new MemoryStream(expectedIndexManifestBytes), cancellationToken); + Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + Assert.Equal(expectedIndexManifestBytes, receivedIndexManifestContent); + Assert.Equal(expectedIndexReferrersBytes, receivedIndexReferrersContent); + } +} diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs new file mode 100644 index 00000000..a3615ff8 --- /dev/null +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -0,0 +1,227 @@ +using OrasProject.Oras.Exceptions; +using OrasProject.Oras.Oci; +using OrasProject.Oras.Registry.Remote; +using static OrasProject.Oras.Tests.Remote.Util.Util; +using static OrasProject.Oras.Tests.Remote.Util.RandomDataGenerator; +using Xunit; + +namespace OrasProject.Oras.Tests.Remote; + +public class ReferrersTest +{ + [Fact] + public void ApplyReferrerChanges_ShouldAddNewReferrers() + { + var oldDescriptor1 = RandomDescriptor(); + var oldDescriptor2 = RandomDescriptor(); + var newDescriptor = RandomDescriptor(); + var oldReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + }; + + var expectedReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + newDescriptor, + }; + var referrerChange = new Referrers.ReferrerChange( + newDescriptor, + Referrers.ReferrerOperation.ReferrerAdd + ); + + var updatedReferrers = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Equal(3, updatedReferrers.Count); + for (var i = 0; i < updatedReferrers.Count; ++i) + { + Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); + } + } + + [Fact] + public void ApplyReferrerChanges_ShouldDiscardDuplicateReferrers() + { + var oldDescriptor1 = RandomDescriptor(); + var oldDescriptor2 = RandomDescriptor(); + var newDescriptor1 = RandomDescriptor(); + + var oldReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + oldDescriptor2, + oldDescriptor1, + }; + + var expectedReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + newDescriptor1, + }; + var referrerChange = new Referrers.ReferrerChange( + newDescriptor1, + Referrers.ReferrerOperation.ReferrerAdd + ); + + var updatedReferrers = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Equal(3, updatedReferrers.Count); + for (var i = 0; i < updatedReferrers.Count; ++i) + { + Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); + } + } + + [Fact] + public void ApplyReferrerChanges_ShouldNotAddNewDuplicateReferrers() + { + var oldDescriptor1 = RandomDescriptor(); + var oldDescriptor2 = RandomDescriptor(); + var oldReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + }; + var referrerChange = new Referrers.ReferrerChange( + oldDescriptor1, + Referrers.ReferrerOperation.ReferrerAdd + ); + + + var exception = Assert.Throws(() => Referrers.ApplyReferrerChanges(oldReferrers, referrerChange)); + Assert.Equal("no referrer update in this request", exception.Message); + } + + [Fact] + public void ApplyReferrerChanges_ShouldNotKeepOldEmptyReferrers() + { + var emptyDesc1 = Descriptor.EmptyDescriptor(); + Descriptor? emptyDesc2 = null; + var newDescriptor = RandomDescriptor(); + + var oldReferrers = new List + { + emptyDesc1, + emptyDesc2, + }; + var expectedReferrers = new List + { + newDescriptor, + }; + var referrerChange = new Referrers.ReferrerChange( + newDescriptor, + Referrers.ReferrerOperation.ReferrerAdd + ); + + var updatedReferrers = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + + Assert.Single(updatedReferrers); + for (var i = 0; i < updatedReferrers.Count; ++i) + { + Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); + } + } + + [Fact] + public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreNull() + { + IList oldReferrers = null; + Referrers.ReferrerChange referrerChange = null; + + var exception = Assert.Throws(() => Referrers.ApplyReferrerChanges(oldReferrers, referrerChange)); + Assert.Equal("referrerChange or oldReferrers is null in this request", exception.Message); + } + + [Fact] + public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreEmpty() + { + var oldReferrers = new List(); + var referrerChange = new Referrers.ReferrerChange(Descriptor.EmptyDescriptor(), Referrers.ReferrerOperation.ReferrerAdd); + + var exception = Assert.Throws(() => Referrers.ApplyReferrerChanges(oldReferrers, referrerChange)); + Assert.Equal("no referrer update in this request", exception.Message); + } + + [Fact] + public void RemoveEmptyDescriptors_ShouldRemoveEmptyDescriptors() + { + var randomDescriptor1 = RandomDescriptor(); + var randomDescriptor2 = RandomDescriptor(); + var randomDescriptor3 = RandomDescriptor(); + var randomDescriptor4 = RandomDescriptor(); + var descriptors = new List + { + Descriptor.EmptyDescriptor(), + randomDescriptor1, + Descriptor.EmptyDescriptor(), + randomDescriptor2, + Descriptor.EmptyDescriptor(), + Descriptor.EmptyDescriptor(), + randomDescriptor3, + randomDescriptor4, + }; + + var expectedDescriptors = new List + { + randomDescriptor1, + randomDescriptor2, + randomDescriptor3, + randomDescriptor4 + }; + Referrers.RemoveEmptyDescriptors(descriptors, 4); + + Assert.Equal(4, descriptors.Count); + Assert.DoesNotContain(Descriptor.EmptyDescriptor(), descriptors); + for (var i = 0; i < descriptors.Count; ++i) + { + Assert.True(AreDescriptorsEqual(descriptors[i], expectedDescriptors[i])); + } + } + + [Fact] + public void RemoveEmptyDescriptors_ShouldReturnAllNonEmptyDescriptors() + { + var randomDescriptor1 = RandomDescriptor(); + var randomDescriptor2 = RandomDescriptor(); + var randomDescriptor3 = RandomDescriptor(); + var randomDescriptor4 = RandomDescriptor(); + var descriptors = new List + { + randomDescriptor1, + randomDescriptor2, + randomDescriptor3, + randomDescriptor4, + }; + + var expectedDescriptors = new List + { + randomDescriptor1, + randomDescriptor2, + randomDescriptor3, + randomDescriptor4 + }; + Referrers.RemoveEmptyDescriptors(descriptors, 4); + Assert.Equal(4, descriptors.Count); + for (var i = 0; i < descriptors.Count; ++i) + { + Assert.True(AreDescriptorsEqual(descriptors[i], expectedDescriptors[i])); + } + } + + [Fact] + public void RemoveEmptyDescriptors_ShouldRemoveAllEmptyDescriptors() + { + var descriptors = new List + { + Descriptor.EmptyDescriptor(), + Descriptor.EmptyDescriptor(), + Descriptor.EmptyDescriptor(), + Descriptor.EmptyDescriptor(), + }; + + Referrers.RemoveEmptyDescriptors(descriptors, 0); + Assert.Empty(descriptors); + } +} diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index 84df80a9..acecad7c 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -1,2284 +1 @@ -// Copyright The ORAS Authors. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using Moq; -using Moq.Protected; -using OrasProject.Oras.Content; -using OrasProject.Oras.Exceptions; -using OrasProject.Oras.Oci; -using OrasProject.Oras.Registry; -using OrasProject.Oras.Registry.Remote; -using System.Collections.Immutable; -using System.Diagnostics; -using System.Net; -using System.Net.Http.Headers; -using System.Text; -using System.Text.Json; -using System.Text.RegularExpressions; -using System.Web; -using Xunit; -using static OrasProject.Oras.Content.Digest; - -namespace OrasProject.Oras.Tests.Remote; - -public class RepositoryTest -{ - public struct TestIOStruct - { - public bool IsTag; - public bool ErrExpectedOnHEAD; - public string ServerCalculatedDigest; - public string ClientSuppliedReference; - public bool ErrExpectedOnGET; - } - - private byte[] _theAmazingBanClan = "Ban Gu, Ban Chao, Ban Zhao"u8.ToArray(); - private const string _theAmazingBanDigest = "b526a4f2be963a2f9b0990c001255669eab8a254ab1a6e3f84f1820212ac7078"; - - private const string _dockerContentDigestHeader = "Docker-Content-Digest"; - - // The following truth table aims to cover the expected GET/HEAD request outcome - // for all possible permutations of the client/server "containing a digest", for - // both Manifests and Blobs. Where the results between the two differ, the index - // of the first column has an exclamation mark. - // - // The client is said to "contain a digest" if the user-supplied reference string - // is of the form that contains a digest rather than a tag. The server, on the - // other hand, is said to "contain a digest" if the server responded with the - // special header `Docker-Content-Digest`. - // - // In this table, anything denoted with an asterisk indicates that the true - // response should actually be the opposite of what's expected; for example, - // `*PASS` means we will get a `PASS`, even though the true answer would be its - // diametric opposite--a `FAIL`. This may seem odd, and deserves an explanation. - // This function has blind-spots, and while it can expend power to gain sight, - // i.e., perform the expensive validation, we chose not to. The reason is two- - // fold: a) we "know" that even if we say "!PASS", it will eventually fail later - // when checks are performed, and with that assumption, we have the luxury for - // the second point, which is b) performance. - // - // _______________________________________________________________________________________________________________ - // | ID | CLIENT | SERVER | Manifest.GET | Blob.GET | Manifest.HEAD | Blob.HEAD | - // |----+-----------------+------------------+-----------------------+-----------+---------------------+-----------+ - // | 1 | tag | missing | CALCULATE,PASS | n/a | FAIL | n/a | - // | 2 | tag | presentCorrect | TRUST,PASS | n/a | TRUST,PASS | n/a | - // | 3 | tag | presentIncorrect | TRUST,*PASS | n/a | TRUST,*PASS | n/a | - // | 4 | correctDigest | missing | TRUST,PASS | PASS | TRUST,PASS | PASS | - // | 5 | correctDigest | presentCorrect | TRUST,COMPARE,PASS | PASS | TRUST,COMPARE,PASS | PASS | - // | 6 | correctDigest | presentIncorrect | TRUST,COMPARE,FAIL | FAIL | TRUST,COMPARE,FAIL | FAIL | - // --------------------------------------------------------------------------------------------------------------- - - /// - /// GetTestIOStructMapForGetDescriptorClass returns a map of test cases for different - /// GET/HEAD request outcome for all possible permutations of the client/server "containing a digest", for - /// both Manifests and Blobs. - /// - /// - public static Dictionary GetTestIOStructMapForGetDescriptorClass() - { - string correctDigest = $"sha256:{_theAmazingBanDigest}"; - string incorrectDigest = $"sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"; - - return new Dictionary - { - ["1. Client:Tag & Server:DigestMissing"] = new TestIOStruct - { - IsTag = true, - ErrExpectedOnHEAD = true - }, - ["2. Client:Tag & Server:DigestValid"] = new TestIOStruct - { - IsTag = true, - ServerCalculatedDigest = correctDigest - }, - ["3. Client:Tag & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct - { - IsTag = true, - ServerCalculatedDigest = incorrectDigest - }, - ["4. Client:DigestValid & Server:DigestMissing"] = new TestIOStruct - { - ClientSuppliedReference = correctDigest - }, - ["5. Client:DigestValid & Server:DigestValid"] = new TestIOStruct - { - ClientSuppliedReference = correctDigest, - ServerCalculatedDigest = correctDigest - }, - ["6. Client:DigestValid & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct - { - ClientSuppliedReference = correctDigest, - ServerCalculatedDigest = incorrectDigest, - ErrExpectedOnHEAD = true, - ErrExpectedOnGET = true - } - }; - } - - /// - /// AreDescriptorsEqual compares two descriptors and returns true if they are equal. - /// - /// - /// - /// - public bool AreDescriptorsEqual(Descriptor a, Descriptor b) - { - return a.MediaType == b.MediaType && a.Digest == b.Digest && a.Size == b.Size; - } - - public static HttpClient CustomClient(Func func) - { - var moqHandler = new Mock(); - moqHandler.Protected().Setup>( - "SendAsync", - ItExpr.IsAny(), - ItExpr.IsAny() - ).ReturnsAsync(func); - return new HttpClient(moqHandler.Object); - } - - private HttpClient CustomClient(Func> func) - { - var moqHandler = new Mock(); - moqHandler.Protected().Setup>( - "SendAsync", - ItExpr.IsAny(), - ItExpr.IsAny() - ).Returns(func); - return new HttpClient(moqHandler.Object); - } - - /// - /// Repository_FetchAsync tests the FetchAsync method of the Repository. - /// - /// - [Fact] - public async Task Repository_FetchAsync() - { - var blob = Encoding.UTF8.GetBytes("hello world"); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var index = """{"manifests":[]}"""u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var resp = new HttpResponseMessage(); - resp.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - Debug.WriteLine("Expected GET request"); - resp.StatusCode = HttpStatusCode.BadRequest; - return resp; - } - - var path = req.RequestUri!.AbsolutePath; - if (path == "/v2/test/blobs/" + blobDesc.Digest) - { - resp.Content = new ByteArrayContent(blob); - resp.Content.Headers.Add("Content-Type", "application/octet-stream"); - resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return resp; - } - - if (path == "/v2/test/manifests/" + indexDesc.Digest) - { - if (!req.Headers.Accept.Contains(new MediaTypeWithQualityHeaderValue(MediaType.ImageIndex))) - { - resp.StatusCode = HttpStatusCode.BadRequest; - Debug.WriteLine("manifest not convertable: " + req.Headers.Accept); - return resp; - } - - resp.Content = new ByteArrayContent(index); - resp.Content.Headers.Add("Content-Type", indexDesc.MediaType); - resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - return resp; - } - - resp.StatusCode = HttpStatusCode.NotFound; - return resp; - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var stream = await repo.FetchAsync(blobDesc, cancellationToken); - var buf = new byte[stream.Length]; - await stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob, buf); - stream = await repo.FetchAsync(indexDesc, cancellationToken); - buf = new byte[stream.Length]; - await stream.ReadAsync(buf, cancellationToken); - Assert.Equal(index, buf); - - } - - /// - /// Repository_PushAsync tests the PushAsync method of the Repository - /// - /// - [Fact] - public async Task Repository_PushAsync() - { - var blob = @"hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var index = @"{""manifests"":[]}"u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - var uuid = Guid.NewGuid().ToString(); - var gotBlob = new byte[blobDesc.Size]; - var gotIndex = new byte[indexDesc.Size]; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var resp = new HttpResponseMessage(); - resp.RequestMessage = req; - if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/") - { - resp.Headers.Location = new Uri("http://localhost:5000/v2/test/blobs/uploads/" + uuid); - resp.StatusCode = HttpStatusCode.Accepted; - return resp; - } - - if (req.Method == HttpMethod.Put && - req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) - { - if (req.Headers.TryGetValues("Content-Type", out var values) && - !values.Contains("application/octet-stream")) - { - resp.StatusCode = HttpStatusCode.BadRequest; - return resp; - - } - - var queries = HttpUtility.ParseQueryString(req.RequestUri.Query); - if (queries["digest"] != blobDesc.Digest) - { - resp.StatusCode = HttpStatusCode.BadRequest; - return resp; - } - - var stream = req.Content!.ReadAsStream(cancellationToken); - stream.Read(gotBlob); - resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - resp.StatusCode = HttpStatusCode.Created; - return resp; - - } - - if (req.Method == HttpMethod.Put && - req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) - { - if (req.Headers.TryGetValues("Content-Type", out var values) && - !values.Contains(MediaType.ImageIndex)) - { - resp.StatusCode = HttpStatusCode.BadRequest; - return resp; - } - - var stream = req.Content!.ReadAsStream(cancellationToken); - stream.Read(gotIndex); - resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - resp.StatusCode = HttpStatusCode.Created; - return resp; - } - - resp.StatusCode = HttpStatusCode.Forbidden; - return resp; - - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - await repo.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); - Assert.Equal(blob, gotBlob); - await repo.PushAsync(indexDesc, new MemoryStream(index), cancellationToken); - Assert.Equal(index, gotIndex); - } - - /// - /// Repository_ExistsAsync tests the ExistsAsync method of the Repository - /// - /// - [Fact] - public async Task Repository_ExistsAsync() - { - var blob = @"hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var index = @"{""manifests"":[]}"u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Head) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) - { - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) - { - if (req.Headers.TryGetValues("Accept", out var values) && - !values.Contains(MediaType.ImageIndex)) - { - return new HttpResponseMessage(HttpStatusCode.NotAcceptable); - } - - res.Content.Headers.Add("Content-Type", indexDesc.MediaType); - res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); - res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var exists = await repo.ExistsAsync(blobDesc, cancellationToken); - Assert.True(exists); - exists = await repo.ExistsAsync(indexDesc, cancellationToken); - Assert.True(exists); - } - - /// - /// Repository_DeleteAsync tests the DeleteAsync method of the Repository - /// - /// - [Fact] - public async Task Repository_DeleteAsync() - { - var blob = @"hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var blobDeleted = false; - var index = @"{""manifests"":[]}"u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - var indexDeleted = false; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Delete) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) - { - blobDeleted = true; - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - res.StatusCode = HttpStatusCode.Accepted; - return res; - } - - if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) - { - indexDeleted = true; - // no dockerContentDigestHeader header for manifest deletion - res.StatusCode = HttpStatusCode.Accepted; - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - await repo.DeleteAsync(blobDesc, cancellationToken); - Assert.True(blobDeleted); - await repo.DeleteAsync(indexDesc, cancellationToken); - Assert.True(indexDeleted); - } - - /// - /// Repository_ResolveAsync tests the ResolveAsync method of the Repository - /// - /// - [Fact] - public async Task Repository_ResolveAsync() - { - var blob = @"hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var index = @"{""manifests"":[]}"u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - var reference = "foobar"; - - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Head) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) - { - return new HttpResponseMessage(HttpStatusCode.NotFound); - } - - if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest - || req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + reference) - - { - if (req.Headers.TryGetValues("Accept", out var values) && - !values.Contains(MediaType.ImageIndex)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - res.Content.Headers.Add("Content-Type", indexDesc.MediaType); - res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); - res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - await Assert.ThrowsAsync(async () => - await repo.ResolveAsync(blobDesc.Digest, cancellationToken)); - // await repo.ResolveAsync(blobDesc.Digest, cancellationToken); - var got = await repo.ResolveAsync(indexDesc.Digest, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, got)); - - got = await repo.ResolveAsync(reference, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, got)); - var tagDigestRef = "whatever" + "@" + indexDesc.Digest; - got = await repo.ResolveAsync(tagDigestRef, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, got)); - var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; - got = await repo.ResolveAsync(fqdnRef, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, got)); - } - - /// - /// Repository_ResolveAsync tests the ResolveAsync method of the Repository - /// - /// - [Fact] - public async Task Repository_TagAsync() - { - var blob = "hello"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var index = @"{""manifests"":[]}"u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - byte[]? gotIndex = null; - var reference = "foobar"; - - var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method == HttpMethod.Get && - req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) - { - return new HttpResponseMessage(HttpStatusCode.Found); - } - - if (req.Method == HttpMethod.Get && - req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) - { - if (req.Headers.TryGetValues("Accept", out var values) && - !values.Contains(MediaType.ImageIndex)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - res.Content = new ByteArrayContent(index); - res.Content.Headers.Add("Content-Type", indexDesc.MediaType); - res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - return res; - } - - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference - || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) - - { - if (req.Headers.TryGetValues("Content-Type", out var values) && - !values.Contains(MediaType.ImageIndex)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - if (req.Content != null) - { - gotIndex = await req.Content.ReadAsByteArrayAsync(cancellationToken); - } - res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - res.StatusCode = HttpStatusCode.Created; - return res; - } - - return new HttpResponseMessage(HttpStatusCode.Forbidden); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - await Assert.ThrowsAnyAsync( - async () => await repo.TagAsync(blobDesc, reference, cancellationToken)); - await repo.TagAsync(indexDesc, reference, cancellationToken); - Assert.Equal(index, gotIndex); - await repo.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); - Assert.Equal(index, gotIndex); - } - - /// - /// Repository_PushReferenceAsync tests the PushReferenceAsync method of the Repository - /// - /// - [Fact] - public async Task Repository_PushReferenceAsync() - { - var index = @"{""manifests"":[]}"u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - byte[]? gotIndex = null; - var reference = "foobar"; - - var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) - { - if (req.Headers.TryGetValues("Content-Type", out var values) && - !values.Contains(MediaType.ImageIndex)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - if (req.Content != null) - { - gotIndex = await req.Content.ReadAsByteArrayAsync(); - } - res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - res.StatusCode = HttpStatusCode.Created; - return res; - } - - return new HttpResponseMessage(HttpStatusCode.Forbidden); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var streamContent = new MemoryStream(index); - await repo.PushAsync(indexDesc, streamContent, reference, cancellationToken); - Assert.Equal(index, gotIndex); - } - - /// - /// Repository_FetchReferenceAsync tests the FetchReferenceAsync method of the Repository - /// - /// - [Fact] - public async Task Repository_FetchReferenceAsyc() - { - var blob = "hello"u8.ToArray(); - var blobDesc = new Descriptor() - { - Digest = ComputeSHA256(blob), - MediaType = "test", - Size = (uint)blob.Length - }; - var index = @"{""manifests"":[]}"u8.ToArray(); - var indexDesc = new Descriptor() - { - Digest = ComputeSHA256(index), - MediaType = MediaType.ImageIndex, - Size = index.Length - }; - var reference = "foobar"; - - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) - { - return new HttpResponseMessage(HttpStatusCode.NotFound); - } - - if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest - || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) - { - if (req.Headers.TryGetValues("Accept", out var values) && - !values.Contains(MediaType.ImageIndex)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - res.Content = new ByteArrayContent(index); - res.Content.Headers.Add("Content-Type", indexDesc.MediaType); - res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.Found); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - - // test with blob digest - await Assert.ThrowsAsync( - async () => await repo.FetchAsync(blobDesc.Digest, cancellationToken)); - - // test with manifest digest - var data = await repo.FetchAsync(indexDesc.Digest, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); - var buf = new byte[data.Stream.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(index, buf); - - // test with manifest tag - data = await repo.FetchAsync(reference, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); - buf = new byte[data.Stream.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(index, buf); - - // test with manifest tag@digest - var tagDigestRef = "whatever" + "@" + indexDesc.Digest; - data = await repo.FetchAsync(tagDigestRef, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); - buf = new byte[data.Stream.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(index, buf); - - // test with manifest FQDN - var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; - data = await repo.FetchAsync(fqdnRef, cancellationToken); - Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); - - buf = new byte[data.Stream.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(index, buf); - } - - /// - /// Repository_TagsAsync tests the TagsAsync method of the Repository - /// to check if the tags are returned correctly - /// - /// - /// - [Fact] - public async Task Repository_TagsAsync() - { - var tagSet = new List>() - { - new() {"the", "quick", "brown", "fox"}, - new() {"jumps", "over", "the", "lazy"}, - new() {"dog"} - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get || - req.RequestUri?.AbsolutePath != "/v2/test/tags/list" - ) - { - return new HttpResponseMessage(HttpStatusCode.NotFound); - } - - var q = req.RequestUri.Query; - try - { - var n = int.Parse(Regex.Match(q, @"(?<=n=)\d+").Value); - if (n != 4) throw new Exception(); - } - catch - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - var tags = new List(); - var serverUrl = "http://localhost:5000"; - var matched = Regex.Match(q, @"(?<=test=)\w+").Value; - switch (matched) - { - case "foo": - tags = tagSet[1]; - res.Headers.Add("Link", $"<{serverUrl}/v2/test/tags/list?n=4&test=bar>; rel=\"next\""); - break; - case "bar": - tags = tagSet[2]; - break; - default: - tags = tagSet[0]; - res.Headers.Add("Link", $"; rel=\"next\""); - break; - } - - var listOfTags = new Repository.TagList - { - Tags = tags.ToArray() - }; - res.Content = new StringContent(JsonSerializer.Serialize(listOfTags)); - return res; - - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - TagListPageSize = 4, - }); - - var cancellationToken = new CancellationToken(); - - var wantTags = new List(); - foreach (var set in tagSet) - { - wantTags.AddRange(set); - } - var gotTags = new List(); - await foreach (var tag in repo.ListTagsAsync().WithCancellation(cancellationToken)) - { - gotTags.Add(tag); - } - Assert.Equal(wantTags, gotTags); - } - - /// - /// BlobStore_FetchAsync tests the FetchAsync method of the BlobStore - /// - /// - [Fact] - public async Task BlobStore_FetchAsync() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - res.Content = new ByteArrayContent(blob); - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - var stream = await store.FetchAsync(blobDesc, cancellationToken); - var buf = new byte[stream.Length]; - await stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob, buf); - - } - - /// - /// BlobStore_FetchAsync_CanSeek tests the FetchAsync method of the BlobStore for a stream that can seek - /// - /// - [Fact] - public async Task BlobStore_FetchAsync_CanSeek() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var seekable = false; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - if (seekable) - { - res.Headers.AcceptRanges.Add("bytes"); - } - - if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) - { - } - - - if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") - { - res.StatusCode = HttpStatusCode.OK; - res.Content = new ByteArrayContent(blob); - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - - long start = -1, end = -1; - var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); - if (hv != null && hv.From.HasValue && hv.To.HasValue) - { - start = hv.From.Value; - end = hv.To.Value; - } - - if (start < 0 || start > end || start >= blobDesc.Size) - { - return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); - } - - end++; - if (end > blobDesc.Size) - { - end = blobDesc.Size; - } - - res.StatusCode = HttpStatusCode.PartialContent; - res.Content = new ByteArrayContent(blob[(int)start..(int)end]); - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - res.StatusCode = HttpStatusCode.NotFound; - return res; - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - var stream = await store.FetchAsync(blobDesc, cancellationToken); - var buf = new byte[stream.Length]; - await stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob, buf); - - seekable = true; - stream = await store.FetchAsync(blobDesc, cancellationToken); - buf = new byte[stream.Length]; - await stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob, buf); - - buf = new byte[stream.Length - 3]; - stream.Seek(3, SeekOrigin.Begin); - await stream.ReadAsync(buf, cancellationToken); - var seg = blob[3..]; - Assert.Equal(seg, buf); - } - - /// - /// BlobStore_FetchAsync_ZeroSizedBlob tests the FetchAsync method of the BlobStore for a zero sized blob - /// - /// - [Fact] - public async Task BlobStore_FetchAsync_ZeroSizedBlob() - { - var blob = ""u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - if (req.Headers.TryGetValues("Range", out var rangeHeader)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - var stream = await store.FetchAsync(blobDesc, cancellationToken); - var buf = new byte[stream.Length]; - await stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob, buf); - } - - /// - /// BlobStore_PushAsync tests the PushAsync method of the BlobStore. - /// - /// - [Fact] - public async Task BlobStore_PushAsync() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var gotBlob = new byte[blob.Length]; - var uuid = Guid.NewGuid().ToString(); - var existingQueryParameter = "existingParam=value"; - - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method == HttpMethod.Post && req.RequestUri?.AbsolutePath == $"/v2/test/blobs/uploads/") - { - res.StatusCode = HttpStatusCode.Accepted; - res.Headers.Add("Location", $"/v2/test/blobs/uploads/{uuid}?{existingQueryParameter}"); - return res; - } - - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) - { - // Assert that the existing query parameter is present - var queryParameters = HttpUtility.ParseQueryString(req.RequestUri.Query); - Assert.Equal("value", queryParameters["existingParam"]); - - if (req.Headers.TryGetValues("Content-Type", out var contentType) && - contentType.FirstOrDefault() != "application/octet-stream") - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - if (HttpUtility.ParseQueryString(req.RequestUri.Query)["digest"] != blobDesc.Digest) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - // read content into buffer - var stream = req.Content!.ReadAsStream(cancellationToken); - stream.Read(gotBlob); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - res.StatusCode = HttpStatusCode.Created; - return res; - } - - return new HttpResponseMessage(HttpStatusCode.Forbidden); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - await store.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); - Assert.Equal(blob, gotBlob); - } - - /// - /// BlobStore_ExistsAsync tests the ExistsAsync method of the BlobStore. - /// - /// - [Fact] - public async Task BlobStore_ExistsAsync() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var content = "foobar"u8.ToArray(); - var contentDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(content), - Size = content.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Head) - { - res.StatusCode = HttpStatusCode.MethodNotAllowed; - return res; - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - var exists = await store.ExistsAsync(blobDesc, cancellationToken); - Assert.True(exists); - exists = await store.ExistsAsync(contentDesc, cancellationToken); - Assert.False(exists); - } - - /// - /// BlobStore_DeleteAsync tests the DeleteAsync method of the BlobStore. - /// - /// - [Fact] - public async Task BlobStore_DeleteAsync() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var blobDeleted = false; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Delete) - { - res.StatusCode = HttpStatusCode.MethodNotAllowed; - return res; - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - blobDeleted = true; - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - res.StatusCode = HttpStatusCode.Accepted; - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - await store.DeleteAsync(blobDesc, cancellationToken); - Assert.True(blobDeleted); - - var content = "foobar"u8.ToArray(); - var contentDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(content), - Size = content.Length - }; - await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); - } - - /// - /// BlobStore_ResolveAsync tests the ResolveAsync method of the BlobStore. - /// - /// - [Fact] - public async Task BlobStore_ResolveAsync() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Head) - { - res.StatusCode = HttpStatusCode.MethodNotAllowed; - return res; - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - var got = await store.ResolveAsync(blobDesc.Digest, cancellationToken); - Assert.Equal(blobDesc.Digest, got.Digest); - Assert.Equal(blobDesc.Size, got.Size); - - var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; - got = await store.ResolveAsync(fqdnRef, cancellationToken); - Assert.Equal(blobDesc.Digest, got.Digest); - - var content = "foobar"u8.ToArray(); - var contentDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(content), - Size = content.Length - }; - await Assert.ThrowsAsync(async () => - await store.ResolveAsync(contentDesc.Digest, cancellationToken)); - } - - /// - /// BlobStore_FetchReferenceAsync tests the FetchReferenceAsync method of BlobStore - /// - /// - [Fact] - public async Task BlobStore_FetchReferenceAsync() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - res.StatusCode = HttpStatusCode.MethodNotAllowed; - return res; - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - res.Content = new ByteArrayContent(blob); - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new BlobStore(repo); - - // test with digest - var gotDesc = await store.FetchAsync(blobDesc.Digest, cancellationToken); - Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); - Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); - - var buf = new byte[gotDesc.Descriptor.Size]; - await gotDesc.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob, buf); - - // test with FQDN reference - var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; - gotDesc = await store.FetchAsync(fqdnRef, cancellationToken); - Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); - Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); - - var content = "foobar"u8.ToArray(); - var contentDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(content), - Size = content.Length - }; - // test with other digest - await Assert.ThrowsAsync(async () => - await store.FetchAsync(contentDesc.Digest, cancellationToken)); - } - - /// - /// BlobStore_FetchAsyncReferenceAsync_Seek tests the FetchAsync method of BlobStore with seek. - /// - /// - [Fact] - public async Task BlobStore_FetchReferenceAsync_Seek() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor() - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var seekable = false; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - - if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") - { - if (seekable) - { - res.Headers.AcceptRanges.Add("bytes"); - } - - if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) - { - } - - - if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") - { - res.StatusCode = HttpStatusCode.OK; - res.Content = new ByteArrayContent(blob); - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - - var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); - var start = hv != null && hv.To.HasValue ? hv.To.Value : -1; - if (start < 0 || start >= blobDesc.Size) - { - return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); - } - - res.StatusCode = HttpStatusCode.PartialContent; - res.Content = new ByteArrayContent(blob[(int)start..]); - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - return res; - } - - res.Content.Headers.Add("Content-Type", "application/octet-stream"); - res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); - res.StatusCode = HttpStatusCode.NotFound; - return res; - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - - var store = new BlobStore(repo); - - // test non-seekable content - - var data = await store.FetchAsync(blobDesc.Digest, cancellationToken); - - Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); - Assert.Equal(data.Descriptor.Size, blobDesc.Size); - - var buf = new byte[data.Descriptor.Size]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob, buf); - - // test seekable content - seekable = true; - data = await store.FetchAsync(blobDesc.Digest, cancellationToken); - Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); - Assert.Equal(data.Descriptor.Size, blobDesc.Size); - - data.Stream.Seek(3, SeekOrigin.Begin); - buf = new byte[data.Descriptor.Size - 3]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(blob[3..], buf); - } - - - /// - /// GenerateBlobDescriptor_WithVariusDockerContentDigestHeaders tests the GenerateBlobDescriptor method of BlobStore with various Docker-Content-Digest headers. - /// - /// - /// - [Fact] - public void GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() - { - var reference = new Reference("eastern.haan.com", "from25to220ce"); - var tests = GetTestIOStructMapForGetDescriptorClass(); - foreach ((string testName, TestIOStruct dcdIOStruct) in tests) - { - if (dcdIOStruct.IsTag) - { - continue; - } - HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; - foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) - { - reference.ContentReference = dcdIOStruct.ClientSuppliedReference; - var resp = new HttpResponseMessage(); - if (method == HttpMethod.Get) - { - resp.Content = new ByteArrayContent(_theAmazingBanClan); - resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); - resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); - } - if (!resp.Headers.TryGetValues(_dockerContentDigestHeader, out IEnumerable? values)) - { - resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); - resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); - resp.RequestMessage = new HttpRequestMessage() - { - Method = method - }; - - } - else - { - resp.RequestMessage = new HttpRequestMessage() - { - Method = method - }; - } - - var d = string.Empty; - try - { - d = reference.Digest; - } - catch - { - throw new Exception( - $"[Blob.{method}] {testName}; got digest from a tag reference unexpectedly"); - } - - var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; - if (d.Length == 0) - { - // To avoid an otherwise impossible scenario in the tested code - // path, we set d so that verifyContentDigest does not break. - d = dcdIOStruct.ServerCalculatedDigest; - } - - var err = false; - try - { - resp.GenerateBlobDescriptor(d); - } - catch (Exception e) - { - err = true; - if (!errExpected) - { - throw new Exception( - $"[Blob.{method}] {testName}; expected no error for request, but got err; {e.Message}"); - } - - } - - if (errExpected && !err) - { - throw new Exception($"[Blob.{method}] {testName}; expected error for request, but got none"); - } - } - } - } - - - /// - /// ManifestStore_FetchAsync tests the FetchAsync method of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_FetchAsync() - { - var manifest = """{"layers":[]}"""u8.ToArray(); - var manifestDesc = new Descriptor - { - MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifest), - Size = manifest.Length - }; - - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") - { - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - res.Content = new ByteArrayContent(manifest); - res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); - return res; - } - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - var data = await store.FetchAsync(manifestDesc, cancellationToken); - var buf = new byte[data.Length]; - await data.ReadAsync(buf, cancellationToken); - Assert.Equal(manifest, buf); - - var content = """{"manifests":[]}"""u8.ToArray(); - var contentDesc = new Descriptor - { - MediaType = MediaType.ImageIndex, - Digest = ComputeSHA256(content), - Size = content.Length - }; - await Assert.ThrowsAsync(async () => await store.FetchAsync(contentDesc, cancellationToken)); - } - - [Fact] - public async Task ManifestStore_FetchAsync_ManifestUnknown() - { - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(HttpStatusCode.Unauthorized); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - res.Content = new StringContent("""{"errors":[{"code":"UNAUTHORIZED","message":"authentication required","detail":[{"Type":"repository","Class":"","Name":"repo","Action":"pull"}]}]}"""); - return res; - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - try - { - var data = await store.FetchAsync("hello", cancellationToken); - Assert.Fail(); - } - catch (ResponseException e) - { - Assert.Equal("UNAUTHORIZED", e.Errors?[0].Code); - } - } - - /// - /// ManifestStore_PushAsync tests the PushAsync method of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_PushAsync() - { - var manifest = """{"layers":[]}"""u8.ToArray(); - var manifestDesc = new Descriptor - { - MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifest), - Size = manifest.Length - }; - byte[]? gotManifest = null; - - var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") - { - if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - if (req.Content?.Headers?.ContentLength != null) - { - var buf = new byte[req.Content.Headers.ContentLength.Value]; - (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); - gotManifest = buf; - } - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); - res.StatusCode = HttpStatusCode.Created; - return res; - } - else - { - return new HttpResponseMessage(HttpStatusCode.Forbidden); - } - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - await store.PushAsync(manifestDesc, new MemoryStream(manifest), cancellationToken); - Assert.Equal(manifest, gotManifest); - } - - /// - /// ManifestStore_ExistAsync tests the ExistAsync method of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_ExistAsync() - { - var manifest = """{"layers":[]}"""u8.ToArray(); - var manifestDesc = new Descriptor - { - MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifest), - Size = manifest.Length - }; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Head) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") - { - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); - res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); - res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); - return res; - } - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - var exist = await store.ExistsAsync(manifestDesc, cancellationToken); - Assert.True(exist); - - var content = """{"manifests":[]}"""u8.ToArray(); - var contentDesc = new Descriptor - { - MediaType = MediaType.ImageIndex, - Digest = ComputeSHA256(content), - Size = content.Length - }; - exist = await store.ExistsAsync(contentDesc, cancellationToken); - Assert.False(exist); - } - - /// - /// ManifestStore_DeleteAsync tests the DeleteAsync method of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_DeleteAsync() - { - var manifest = """{"layers":[]}"""u8.ToArray(); - var manifestDesc = new Descriptor - { - MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifest), - Size = manifest.Length - }; - var manifestDeleted = false; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") - { - manifestDeleted = true; - res.StatusCode = HttpStatusCode.Accepted; - return res; - } - if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") - { - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - res.Content = new ByteArrayContent(manifest); - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); - res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); - return res; - } - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - await store.DeleteAsync(manifestDesc, cancellationToken); - Assert.True(manifestDeleted); - - var content = """{"manifests":[]}"""u8.ToArray(); - var contentDesc = new Descriptor - { - MediaType = MediaType.ImageIndex, - Digest = ComputeSHA256(content), - Size = content.Length - }; - await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); - } - - /// - /// ManifestStore_ResolveAsync tests the ResolveAsync method of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_ResolveAsync() - { - var manifest = """{"layers":[]}"""u8.ToArray(); - var manifestDesc = new Descriptor - { - MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifest), - Size = manifest.Length - }; - var reference = "foobar"; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Head) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") - { - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); - res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); - res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); - return res; - } - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - var got = await store.ResolveAsync(manifestDesc.Digest, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, got)); - got = await store.ResolveAsync(reference, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, got)); - - var tagDigestRef = "whatever" + "@" + manifestDesc.Digest; - got = await store.ResolveAsync(tagDigestRef, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, got)); - - var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; - got = await store.ResolveAsync(fqdnRef, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, got)); - - var content = """{"manifests":[]}"""u8.ToArray(); - var contentDesc = new Descriptor - { - MediaType = MediaType.ImageIndex, - Digest = ComputeSHA256(content), - Size = content.Length - }; - - await Assert.ThrowsAsync(async () => await store.ResolveAsync(contentDesc.Digest, cancellationToken)); - - } - - /// - /// ManifestStore_FetchReferenceAsync tests the FetchReferenceAsync method of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_FetchReferenceAsync() - { - var manifest = """{"layers":[]}"""u8.ToArray(); - var manifestDesc = new Descriptor - { - MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifest), - Size = manifest.Length - }; - var reference = "foobar"; - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method != HttpMethod.Get) - { - return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); - } - if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") - { - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - res.Content = new ByteArrayContent(manifest); - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); - res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); - return res; - } - return new HttpResponseMessage(HttpStatusCode.NotFound); - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - - // test with tag - var data = await store.FetchAsync(reference, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); - var buf = new byte[manifest.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(manifest, buf); - - // test with other tag - var randomRef = "whatever"; - await Assert.ThrowsAsync(async () => await store.FetchAsync(randomRef, cancellationToken)); - - // test with digest - data = await store.FetchAsync(manifestDesc.Digest, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); - - buf = new byte[manifest.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(manifest, buf); - - // test with tag@digest - var tagDigestRef = randomRef + "@" + manifestDesc.Digest; - data = await store.FetchAsync(tagDigestRef, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); - buf = new byte[manifest.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(manifest, buf); - - // test with FQDN - var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; - data = await store.FetchAsync(fqdnRef, cancellationToken); - Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); - buf = new byte[manifest.Length]; - await data.Stream.ReadAsync(buf, cancellationToken); - Assert.Equal(manifest, buf); - } - - /// - /// ManifestStore_TagAsync tests the TagAsync method of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_TagAsync() - { - var blob = "hello world"u8.ToArray(); - var blobDesc = new Descriptor - { - MediaType = "test", - Digest = ComputeSHA256(blob), - Size = blob.Length - }; - var index = """{"manifests":[]}"""u8.ToArray(); - var indexDesc = new Descriptor - { - MediaType = MediaType.ImageIndex, - Digest = ComputeSHA256(index), - Size = index.Length - }; - var gotIndex = new byte[index.Length]; - var reference = "foobar"; - - var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{blobDesc.Digest}") - { - res.StatusCode = HttpStatusCode.NotFound; - return res; - } - if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") - { - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - res.Content = new ByteArrayContent(index); - res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); - res.Content.Headers.Add("Content-Type", new string[] { indexDesc.MediaType }); - return res; - } - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") - { - if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) - { - res.StatusCode = HttpStatusCode.BadRequest; - return res; - } - if (req.Content?.Headers?.ContentLength != null) - { - var buf = new byte[req.Content.Headers.ContentLength.Value]; - (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); - gotIndex = buf; - } - - res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); - res.StatusCode = HttpStatusCode.Created; - return res; - } - - res.StatusCode = HttpStatusCode.Forbidden; - return res; - }; - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - - await Assert.ThrowsAnyAsync(async () => await store.TagAsync(blobDesc, reference, cancellationToken)); - - await store.TagAsync(indexDesc, reference, cancellationToken); - Assert.Equal(index, gotIndex); - - gotIndex = null; - await store.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); - Assert.Equal(index, gotIndex); - } - - /// - /// ManifestStore_PushReferenceAsync tests the PushReferenceAsync of ManifestStore. - /// - /// - [Fact] - public async Task ManifestStore_PushReferenceAsync() - { - var index = """{"manifests":[]}"""u8.ToArray(); - var indexDesc = new Descriptor - { - MediaType = MediaType.ImageIndex, - Digest = ComputeSHA256(index), - Size = index.Length - }; - var gotIndex = new byte[index.Length]; - var reference = "foobar"; - - var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") - { - if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) - { - res.StatusCode = HttpStatusCode.BadRequest; - return res; - } - - if (req.Content?.Headers?.ContentLength != null) - { - var buf = new byte[req.Content.Headers.ContentLength.Value]; - (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); - gotIndex = buf; - } - - res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); - res.StatusCode = HttpStatusCode.Created; - return res; - } - res.StatusCode = HttpStatusCode.Forbidden; - return res; - }; - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(func), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - await store.PushAsync(indexDesc, new MemoryStream(index), reference, cancellationToken); - Assert.Equal(index, gotIndex); - } - - /// - /// This test tries copying artifacts from the remote target to the memory target - /// - /// - [Fact] - public async Task CopyFromRepositoryToMemory() - { - var exampleManifest = @"hello world"u8.ToArray(); - - var exampleManifestDescriptor = new Descriptor - { - MediaType = MediaType.Descriptor, - Digest = ComputeSHA256(exampleManifest), - Size = exampleManifest.Length - }; - var exampleUploadUUid = new Guid().ToString(); - var func = (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - var path = req.RequestUri != null ? req.RequestUri.AbsolutePath : string.Empty; - var method = req.Method; - if (path.Contains("/blobs/uploads/") && method == HttpMethod.Post) - { - res.StatusCode = HttpStatusCode.Accepted; - res.Headers.Location = new Uri($"{path}/{exampleUploadUUid}"); - res.Headers.Add("Content-Type", MediaType.ImageManifest); - return res; - } - if (path.Contains("/blobs/uploads/" + exampleUploadUUid) && method == HttpMethod.Get) - { - res.StatusCode = HttpStatusCode.Created; - return res; - } - - if (path.Contains("/manifests/latest") && method == HttpMethod.Put) - { - res.StatusCode = HttpStatusCode.Created; - return res; - } - - if (path.Contains("/manifests/" + exampleManifestDescriptor.Digest) || path.Contains("/manifests/latest") && method == HttpMethod.Head) - { - if (method == HttpMethod.Get) - { - res.Content = new ByteArrayContent(exampleManifest); - res.Content.Headers.Add("Content-Type", MediaType.Descriptor); - res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); - res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); - return res; - } - res.Content.Headers.Add("Content-Type", MediaType.Descriptor); - res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); - res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); - return res; - } - - - if (path.Contains("/blobs/") && (method == HttpMethod.Get || method == HttpMethod.Head)) - { - var arr = path.Split("/"); - var digest = arr[arr.Length - 1]; - - - if (digest == exampleManifestDescriptor.Digest) - { - byte[] content = exampleManifest; - res.Content = new ByteArrayContent(content); - res.Content.Headers.Add("Content-Type", exampleManifestDescriptor.MediaType); - res.Content.Headers.Add("Content-Length", content.Length.ToString()); - } - - res.Headers.Add(_dockerContentDigestHeader, digest); - - return res; - } - - if (path.Contains("/manifests/") && method == HttpMethod.Put) - { - res.StatusCode = HttpStatusCode.Created; - return res; - } - - return res; - }; - - var reg = new Registry.Remote.Registry(new RepositoryOptions() - { - Reference = new Reference("localhost:5000"), - HttpClient = CustomClient(func), - }); - var src = await reg.GetRepositoryAsync("source", CancellationToken.None); - - var dst = new MemoryStore(); - var tagName = "latest"; - var desc = await src.CopyAsync(tagName, dst, tagName, CancellationToken.None); - } - - [Fact] - public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigestHeaders() - { - var reference = new Reference("eastern.haan.com", "from25to220ce"); - var tests = GetTestIOStructMapForGetDescriptorClass(); - foreach ((string testName, TestIOStruct dcdIOStruct) in tests) - { - var repo = new Repository(reference.Repository + "/" + reference.Repository); - HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; - var s = new ManifestStore(repo); - foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) - { - reference.ContentReference = dcdIOStruct.ClientSuppliedReference; - var resp = new HttpResponseMessage(); - if (method == HttpMethod.Get) - { - resp.Content = new ByteArrayContent(_theAmazingBanClan); - resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); - resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); - } - else - { - resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); - resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); - } - resp.RequestMessage = new HttpRequestMessage() - { - Method = method - }; - - var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; - - var err = false; - try - { - await resp.GenerateDescriptorAsync(reference, CancellationToken.None); - } - catch (Exception e) - { - err = true; - if (!errExpected) - { - throw new Exception( - $"[Manifest.{method}] {testName}; expected no error for request, but got err; {e.Message}"); - } - - } - if (errExpected && !err) - { - throw new Exception($"[Manifest.{method}] {testName}; expected error for request, but got none"); - } - } - } - - } -} +// Copyright The ORAS Authors. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. using OrasProject.Oras.Content; using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; using OrasProject.Oras.Registry; using OrasProject.Oras.Registry.Remote; using static OrasProject.Oras.Tests.Remote.Util.Util; using System.Diagnostics; using System.Net; using System.Net.Http.Headers; using System.Text; using System.Text.Json; using System.Text.RegularExpressions; using System.Web; using Xunit; using static OrasProject.Oras.Content.Digest; namespace OrasProject.Oras.Tests.Remote; public class RepositoryTest { public struct TestIOStruct { public bool IsTag; public bool ErrExpectedOnHEAD; public string ServerCalculatedDigest; public string ClientSuppliedReference; public bool ErrExpectedOnGET; } private byte[] _theAmazingBanClan = "Ban Gu, Ban Chao, Ban Zhao"u8.ToArray(); private const string _theAmazingBanDigest = "b526a4f2be963a2f9b0990c001255669eab8a254ab1a6e3f84f1820212ac7078"; private const string _dockerContentDigestHeader = "Docker-Content-Digest"; // The following truth table aims to cover the expected GET/HEAD request outcome // for all possible permutations of the client/server "containing a digest", for // both Manifests and Blobs. Where the results between the two differ, the index // of the first column has an exclamation mark. // // The client is said to "contain a digest" if the user-supplied reference string // is of the form that contains a digest rather than a tag. The server, on the // other hand, is said to "contain a digest" if the server responded with the // special header `Docker-Content-Digest`. // // In this table, anything denoted with an asterisk indicates that the true // response should actually be the opposite of what's expected; for example, // `*PASS` means we will get a `PASS`, even though the true answer would be its // diametric opposite--a `FAIL`. This may seem odd, and deserves an explanation. // This function has blind-spots, and while it can expend power to gain sight, // i.e., perform the expensive validation, we chose not to. The reason is two- // fold: a) we "know" that even if we say "!PASS", it will eventually fail later // when checks are performed, and with that assumption, we have the luxury for // the second point, which is b) performance. // // _______________________________________________________________________________________________________________ // | ID | CLIENT | SERVER | Manifest.GET | Blob.GET | Manifest.HEAD | Blob.HEAD | // |----+-----------------+------------------+-----------------------+-----------+---------------------+-----------+ // | 1 | tag | missing | CALCULATE,PASS | n/a | FAIL | n/a | // | 2 | tag | presentCorrect | TRUST,PASS | n/a | TRUST,PASS | n/a | // | 3 | tag | presentIncorrect | TRUST,*PASS | n/a | TRUST,*PASS | n/a | // | 4 | correctDigest | missing | TRUST,PASS | PASS | TRUST,PASS | PASS | // | 5 | correctDigest | presentCorrect | TRUST,COMPARE,PASS | PASS | TRUST,COMPARE,PASS | PASS | // | 6 | correctDigest | presentIncorrect | TRUST,COMPARE,FAIL | FAIL | TRUST,COMPARE,FAIL | FAIL | // --------------------------------------------------------------------------------------------------------------- /// /// GetTestIOStructMapForGetDescriptorClass returns a map of test cases for different /// GET/HEAD request outcome for all possible permutations of the client/server "containing a digest", for /// both Manifests and Blobs. /// /// public static Dictionary GetTestIOStructMapForGetDescriptorClass() { string correctDigest = $"sha256:{_theAmazingBanDigest}"; string incorrectDigest = $"sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"; return new Dictionary { ["1. Client:Tag & Server:DigestMissing"] = new TestIOStruct { IsTag = true, ErrExpectedOnHEAD = true }, ["2. Client:Tag & Server:DigestValid"] = new TestIOStruct { IsTag = true, ServerCalculatedDigest = correctDigest }, ["3. Client:Tag & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct { IsTag = true, ServerCalculatedDigest = incorrectDigest }, ["4. Client:DigestValid & Server:DigestMissing"] = new TestIOStruct { ClientSuppliedReference = correctDigest }, ["5. Client:DigestValid & Server:DigestValid"] = new TestIOStruct { ClientSuppliedReference = correctDigest, ServerCalculatedDigest = correctDigest }, ["6. Client:DigestValid & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct { ClientSuppliedReference = correctDigest, ServerCalculatedDigest = incorrectDigest, ErrExpectedOnHEAD = true, ErrExpectedOnGET = true } }; } /// /// Repository_FetchAsync tests the FetchAsync method of the Repository. /// /// [Fact] public async Task Repository_FetchAsync() { var blob = Encoding.UTF8.GetBytes("hello world"); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = """{"manifests":[]}"""u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var resp = new HttpResponseMessage(); resp.RequestMessage = req; if (req.Method != HttpMethod.Get) { Debug.WriteLine("Expected GET request"); resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var path = req.RequestUri!.AbsolutePath; if (path == "/v2/test/blobs/" + blobDesc.Digest) { resp.Content = new ByteArrayContent(blob); resp.Content.Headers.Add("Content-Type", "application/octet-stream"); resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return resp; } if (path == "/v2/test/manifests/" + indexDesc.Digest) { if (!req.Headers.Accept.Contains(new MediaTypeWithQualityHeaderValue(MediaType.ImageIndex))) { resp.StatusCode = HttpStatusCode.BadRequest; Debug.WriteLine("manifest not convertable: " + req.Headers.Accept); return resp; } resp.Content = new ByteArrayContent(index); resp.Content.Headers.Add("Content-Type", indexDesc.MediaType); resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return resp; } resp.StatusCode = HttpStatusCode.NotFound; return resp; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var stream = await repo.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); stream = await repo.FetchAsync(indexDesc, cancellationToken); buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); } /// /// Repository_PushAsync tests the PushAsync method of the Repository /// /// [Fact] public async Task Repository_PushAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var uuid = Guid.NewGuid().ToString(); var gotBlob = new byte[blobDesc.Size]; var gotIndex = new byte[indexDesc.Size]; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var resp = new HttpResponseMessage(); resp.RequestMessage = req; if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/") { resp.Headers.Location = new Uri("http://localhost:5000/v2/test/blobs/uploads/" + uuid); resp.StatusCode = HttpStatusCode.Accepted; return resp; } if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains("application/octet-stream")) { resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var queries = HttpUtility.ParseQueryString(req.RequestUri.Query); if (queries["digest"] != blobDesc.Digest) { resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var stream = req.Content!.ReadAsStream(cancellationToken); stream.Read(gotBlob); resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); resp.StatusCode = HttpStatusCode.Created; return resp; } if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(MediaType.ImageIndex)) { resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var stream = req.Content!.ReadAsStream(cancellationToken); stream.Read(gotIndex); resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); resp.StatusCode = HttpStatusCode.Created; return resp; } resp.StatusCode = HttpStatusCode.Forbidden; return resp; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await repo.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); Assert.Equal(blob, gotBlob); await repo.PushAsync(indexDesc, new MemoryStream(index), cancellationToken); Assert.Equal(index, gotIndex); } /// /// Repository_ExistsAsync tests the ExistsAsync method of the Repository /// /// [Fact] public async Task Repository_ExistsAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) { res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.NotAcceptable); } res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var exists = await repo.ExistsAsync(blobDesc, cancellationToken); Assert.True(exists); exists = await repo.ExistsAsync(indexDesc, cancellationToken); Assert.True(exists); } /// /// Repository_DeleteAsync tests the DeleteAsync method of the Repository /// /// [Fact] public async Task Repository_DeleteAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var blobDeleted = false; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var indexDeleted = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Delete) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) { blobDeleted = true; res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.Accepted; return res; } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { indexDeleted = true; // no dockerContentDigestHeader header for manifest deletion res.StatusCode = HttpStatusCode.Accepted; return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await repo.DeleteAsync(blobDesc, cancellationToken); Assert.True(blobDeleted); await repo.DeleteAsync(indexDesc, cancellationToken); Assert.True(indexDeleted); } /// /// Repository_ResolveAsync tests the ResolveAsync method of the Repository /// /// [Fact] public async Task Repository_ResolveAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.NotFound); } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest || req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + reference) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await Assert.ThrowsAsync(async () => await repo.ResolveAsync(blobDesc.Digest, cancellationToken)); // await repo.ResolveAsync(blobDesc.Digest, cancellationToken); var got = await repo.ResolveAsync(indexDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); got = await repo.ResolveAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); var tagDigestRef = "whatever" + "@" + indexDesc.Digest; got = await repo.ResolveAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; got = await repo.ResolveAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); } /// /// Repository_ResolveAsync tests the ResolveAsync method of the Repository /// /// [Fact] public async Task Repository_TagAsync() { var blob = "hello"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; byte[]? gotIndex = null; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.Found); } if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(index); res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (req.Content != null) { gotIndex = await req.Content.ReadAsByteArrayAsync(cancellationToken); } res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); res.StatusCode = HttpStatusCode.Created; return res; } return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await Assert.ThrowsAnyAsync( async () => await repo.TagAsync(blobDesc, reference, cancellationToken)); await repo.TagAsync(indexDesc, reference, cancellationToken); Assert.Equal(index, gotIndex); await repo.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); Assert.Equal(index, gotIndex); } /// /// Repository_PushReferenceAsync tests the PushReferenceAsync method of the Repository /// /// [Fact] public async Task Repository_PushReferenceAsync() { var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; byte[]? gotIndex = null; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (req.Content != null) { gotIndex = await req.Content.ReadAsByteArrayAsync(); } res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); res.StatusCode = HttpStatusCode.Created; return res; } return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var streamContent = new MemoryStream(index); await repo.PushAsync(indexDesc, streamContent, reference, cancellationToken); Assert.Equal(index, gotIndex); } /// /// Repository_FetchReferenceAsync tests the FetchReferenceAsync method of the Repository /// /// [Fact] public async Task Repository_FetchReferenceAsyc() { var blob = "hello"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.NotFound); } if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(index); res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.Found); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); // test with blob digest await Assert.ThrowsAsync( async () => await repo.FetchAsync(blobDesc.Digest, cancellationToken)); // test with manifest digest var data = await repo.FetchAsync(indexDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); var buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); // test with manifest tag data = await repo.FetchAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); // test with manifest tag@digest var tagDigestRef = "whatever" + "@" + indexDesc.Digest; data = await repo.FetchAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); // test with manifest FQDN var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; data = await repo.FetchAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); } /// /// Repository_TagsAsync tests the TagsAsync method of the Repository /// to check if the tags are returned correctly /// /// /// [Fact] public async Task Repository_TagsAsync() { var tagSet = new List>() { new() {"the", "quick", "brown", "fox"}, new() {"jumps", "over", "the", "lazy"}, new() {"dog"} }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get || req.RequestUri?.AbsolutePath != "/v2/test/tags/list" ) { return new HttpResponseMessage(HttpStatusCode.NotFound); } var q = req.RequestUri.Query; try { var n = int.Parse(Regex.Match(q, @"(?<=n=)\d+").Value); if (n != 4) throw new Exception(); } catch { return new HttpResponseMessage(HttpStatusCode.BadRequest); } var tags = new List(); var serverUrl = "http://localhost:5000"; var matched = Regex.Match(q, @"(?<=test=)\w+").Value; switch (matched) { case "foo": tags = tagSet[1]; res.Headers.Add("Link", $"<{serverUrl}/v2/test/tags/list?n=4&test=bar>; rel=\"next\""); break; case "bar": tags = tagSet[2]; break; default: tags = tagSet[0]; res.Headers.Add("Link", $"; rel=\"next\""); break; } var listOfTags = new Repository.TagList { Tags = tags.ToArray() }; res.Content = new StringContent(JsonSerializer.Serialize(listOfTags)); return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, TagListPageSize = 4, }); var cancellationToken = new CancellationToken(); var wantTags = new List(); foreach (var set in tagSet) { wantTags.AddRange(set); } var gotTags = new List(); await foreach (var tag in repo.ListTagsAsync().WithCancellation(cancellationToken)) { gotTags.Add(tag); } Assert.Equal(wantTags, gotTags); } /// /// BlobStore_FetchAsync tests the FetchAsync method of the BlobStore /// /// [Fact] public async Task BlobStore_FetchAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var stream = await store.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); } /// /// BlobStore_FetchAsync_CanSeek tests the FetchAsync method of the BlobStore for a stream that can seek /// /// [Fact] public async Task BlobStore_FetchAsync_CanSeek() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var seekable = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { if (seekable) { res.Headers.AcceptRanges.Add("bytes"); } if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) { } if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") { res.StatusCode = HttpStatusCode.OK; res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } long start = -1, end = -1; var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); if (hv != null && hv.From.HasValue && hv.To.HasValue) { start = hv.From.Value; end = hv.To.Value; } if (start < 0 || start > end || start >= blobDesc.Size) { return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); } end++; if (end > blobDesc.Size) { end = blobDesc.Size; } res.StatusCode = HttpStatusCode.PartialContent; res.Content = new ByteArrayContent(blob[(int)start..(int)end]); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.NotFound; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var stream = await store.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); seekable = true; stream = await store.FetchAsync(blobDesc, cancellationToken); buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); buf = new byte[stream.Length - 3]; stream.Seek(3, SeekOrigin.Begin); await stream.ReadAsync(buf, cancellationToken); var seg = blob[3..]; Assert.Equal(seg, buf); } /// /// BlobStore_FetchAsync_ZeroSizedBlob tests the FetchAsync method of the BlobStore for a zero sized blob /// /// [Fact] public async Task BlobStore_FetchAsync_ZeroSizedBlob() { var blob = ""u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { if (req.Headers.TryGetValues("Range", out var rangeHeader)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var stream = await store.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); } /// /// BlobStore_PushAsync tests the PushAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_PushAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var gotBlob = new byte[blob.Length]; var uuid = Guid.NewGuid().ToString(); var existingQueryParameter = "existingParam=value"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Post && req.RequestUri?.AbsolutePath == $"/v2/test/blobs/uploads/") { res.StatusCode = HttpStatusCode.Accepted; res.Headers.Add("Location", $"/v2/test/blobs/uploads/{uuid}?{existingQueryParameter}"); return res; } if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) { // Assert that the existing query parameter is present var queryParameters = HttpUtility.ParseQueryString(req.RequestUri.Query); Assert.Equal("value", queryParameters["existingParam"]); if (req.Headers.TryGetValues("Content-Type", out var contentType) && contentType.FirstOrDefault() != "application/octet-stream") { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (HttpUtility.ParseQueryString(req.RequestUri.Query)["digest"] != blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } // read content into buffer var stream = req.Content!.ReadAsStream(cancellationToken); stream.Read(gotBlob); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.Created; return res; } return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); await store.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); Assert.Equal(blob, gotBlob); } /// /// BlobStore_ExistsAsync tests the ExistsAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_ExistsAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var exists = await store.ExistsAsync(blobDesc, cancellationToken); Assert.True(exists); exists = await store.ExistsAsync(contentDesc, cancellationToken); Assert.False(exists); } /// /// BlobStore_DeleteAsync tests the DeleteAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_DeleteAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var blobDeleted = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Delete) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { blobDeleted = true; res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.Accepted; return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); await store.DeleteAsync(blobDesc, cancellationToken); Assert.True(blobDeleted); var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); } /// /// BlobStore_ResolveAsync tests the ResolveAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_ResolveAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var got = await store.ResolveAsync(blobDesc.Digest, cancellationToken); Assert.Equal(blobDesc.Digest, got.Digest); Assert.Equal(blobDesc.Size, got.Size); var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; got = await store.ResolveAsync(fqdnRef, cancellationToken); Assert.Equal(blobDesc.Digest, got.Digest); var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.ResolveAsync(contentDesc.Digest, cancellationToken)); } /// /// BlobStore_FetchReferenceAsync tests the FetchReferenceAsync method of BlobStore /// /// [Fact] public async Task BlobStore_FetchReferenceAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); // test with digest var gotDesc = await store.FetchAsync(blobDesc.Digest, cancellationToken); Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); var buf = new byte[gotDesc.Descriptor.Size]; await gotDesc.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); // test with FQDN reference var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; gotDesc = await store.FetchAsync(fqdnRef, cancellationToken); Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; // test with other digest await Assert.ThrowsAsync(async () => await store.FetchAsync(contentDesc.Digest, cancellationToken)); } /// /// BlobStore_FetchAsyncReferenceAsync_Seek tests the FetchAsync method of BlobStore with seek. /// /// [Fact] public async Task BlobStore_FetchReferenceAsync_Seek() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var seekable = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { if (seekable) { res.Headers.AcceptRanges.Add("bytes"); } if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) { } if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") { res.StatusCode = HttpStatusCode.OK; res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); var start = hv != null && hv.To.HasValue ? hv.To.Value : -1; if (start < 0 || start >= blobDesc.Size) { return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); } res.StatusCode = HttpStatusCode.PartialContent; res.Content = new ByteArrayContent(blob[(int)start..]); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.NotFound; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); // test non-seekable content var data = await store.FetchAsync(blobDesc.Digest, cancellationToken); Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); Assert.Equal(data.Descriptor.Size, blobDesc.Size); var buf = new byte[data.Descriptor.Size]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); // test seekable content seekable = true; data = await store.FetchAsync(blobDesc.Digest, cancellationToken); Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); Assert.Equal(data.Descriptor.Size, blobDesc.Size); data.Stream.Seek(3, SeekOrigin.Begin); buf = new byte[data.Descriptor.Size - 3]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob[3..], buf); } /// /// GenerateBlobDescriptor_WithVariusDockerContentDigestHeaders tests the GenerateBlobDescriptor method of BlobStore with various Docker-Content-Digest headers. /// /// /// [Fact] public void GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() { var reference = new Reference("eastern.haan.com", "from25to220ce"); var tests = GetTestIOStructMapForGetDescriptorClass(); foreach ((string testName, TestIOStruct dcdIOStruct) in tests) { if (dcdIOStruct.IsTag) { continue; } HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) { reference.ContentReference = dcdIOStruct.ClientSuppliedReference; var resp = new HttpResponseMessage(); if (method == HttpMethod.Get) { resp.Content = new ByteArrayContent(_theAmazingBanClan); resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } if (!resp.Headers.TryGetValues(_dockerContentDigestHeader, out IEnumerable? values)) { resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); resp.RequestMessage = new HttpRequestMessage() { Method = method }; } else { resp.RequestMessage = new HttpRequestMessage() { Method = method }; } var d = string.Empty; try { d = reference.Digest; } catch { throw new Exception( $"[Blob.{method}] {testName}; got digest from a tag reference unexpectedly"); } var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; if (d.Length == 0) { // To avoid an otherwise impossible scenario in the tested code // path, we set d so that verifyContentDigest does not break. d = dcdIOStruct.ServerCalculatedDigest; } var err = false; try { resp.GenerateBlobDescriptor(d); } catch (Exception e) { err = true; if (!errExpected) { throw new Exception( $"[Blob.{method}] {testName}; expected no error for request, but got err; {e.Message}"); } } if (errExpected && !err) { throw new Exception($"[Blob.{method}] {testName}; expected error for request, but got none"); } } } } /// /// ManifestStore_FetchAsync tests the FetchAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_FetchAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(manifest); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var data = await store.FetchAsync(manifestDesc, cancellationToken); var buf = new byte[data.Length]; await data.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.FetchAsync(contentDesc, cancellationToken)); } [Fact] public async Task ManifestStore_FetchAsync_ManifestUnknown() { var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(HttpStatusCode.Unauthorized); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new StringContent( """{"errors":[{"code":"UNAUTHORIZED","message":"authentication required","detail":[{"Type":"repository","Class":"","Name":"repo","Action":"pull"}]}]}"""); return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); try { var data = await store.FetchAsync("hello", cancellationToken); Assert.Fail(); } catch (ResponseException e) { Assert.Equal("UNAUTHORIZED", e.Errors?[0].Code); } } /// /// ManifestStore_PushAsync tests the PushAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_PushAsync() { var configBlob = """config"""u8.ToArray(); var manifestStr = $@"{{""layers"": [], ""size"": 0, ""config"": {{""mediaType"": ""{MediaType.ImageConfig}"", ""digest"": ""{ComputeSHA256(configBlob)}"", ""size"": {configBlob.Length}}}}}"; var manifest = Encoding.UTF8.GetBytes(manifestStr); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; byte[]? gotManifest = null; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (req.Content?.Headers?.ContentLength != null) { var buf = new byte[req.Content.Headers.ContentLength.Value]; (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); gotManifest = buf; } res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.StatusCode = HttpStatusCode.Created; return res; } else { return new HttpResponseMessage(HttpStatusCode.Forbidden); } }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await store.PushAsync(manifestDesc, new MemoryStream(manifest), cancellationToken); Assert.Equal(manifest, gotManifest); } /// /// ManifestStore_ExistAsync tests the ExistAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_ExistAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var exist = await store.ExistsAsync(manifestDesc, cancellationToken); Assert.True(exist); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; exist = await store.ExistsAsync(contentDesc, cancellationToken); Assert.False(exist); } /// /// ManifestStore_DeleteAsync tests the DeleteAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_DeleteAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var manifestDeleted = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { manifestDeleted = true; res.StatusCode = HttpStatusCode.Accepted; return res; } if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(manifest); res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await store.DeleteAsync(manifestDesc, cancellationToken); Assert.True(manifestDeleted); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); } /// /// ManifestStore_ResolveAsync tests the ResolveAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_ResolveAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var got = await store.ResolveAsync(manifestDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); got = await store.ResolveAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); var tagDigestRef = "whatever" + "@" + manifestDesc.Digest; got = await store.ResolveAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; got = await store.ResolveAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.ResolveAsync(contentDesc.Digest, cancellationToken)); } /// /// ManifestStore_FetchReferenceAsync tests the FetchReferenceAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_FetchReferenceAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(manifest); res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); // test with tag var data = await store.FetchAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); var buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); // test with other tag var randomRef = "whatever"; await Assert.ThrowsAsync(async () => await store.FetchAsync(randomRef, cancellationToken)); // test with digest data = await store.FetchAsync(manifestDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); // test with tag@digest var tagDigestRef = randomRef + "@" + manifestDesc.Digest; data = await store.FetchAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); // test with FQDN var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; data = await store.FetchAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); } /// /// ManifestStore_TagAsync tests the TagAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_TagAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var index = """{"manifests":[]}"""u8.ToArray(); var indexDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(index), Size = index.Length }; var gotIndex = new byte[index.Length]; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{blobDesc.Digest}") { res.StatusCode = HttpStatusCode.NotFound; return res; } if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(index); res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { indexDesc.MediaType }); return res; } if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") { if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) { res.StatusCode = HttpStatusCode.BadRequest; return res; } if (req.Content?.Headers?.ContentLength != null) { var buf = new byte[req.Content.Headers.ContentLength.Value]; (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); gotIndex = buf; } res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); res.StatusCode = HttpStatusCode.Created; return res; } res.StatusCode = HttpStatusCode.Forbidden; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await Assert.ThrowsAnyAsync(async () => await store.TagAsync(blobDesc, reference, cancellationToken)); await store.TagAsync(indexDesc, reference, cancellationToken); Assert.Equal(index, gotIndex); gotIndex = null; await store.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); Assert.Equal(index, gotIndex); } /// /// ManifestStore_PushReferenceAsync tests the PushReferenceAsync of ManifestStore. /// /// [Fact] public async Task ManifestStore_PushReferenceAsync() { var manifest = Encoding.UTF8.GetBytes($@"{{""layers"": []}}"); var indexStr = $@"{{""manifests"":[{{""mediaType"": ""{MediaType.ImageManifest}"", ""digest"": ""{ComputeSHA256(manifest)}"", ""size"": {manifest.Length}}}]}}"; var index = Encoding.UTF8.GetBytes(indexStr); var indexDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(index), Size = index.Length }; var gotIndex = new byte[index.Length]; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") { if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) { res.StatusCode = HttpStatusCode.BadRequest; return res; } if (req.Content?.Headers?.ContentLength != null) { var buf = new byte[req.Content.Headers.ContentLength.Value]; (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); gotIndex = buf; } res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); res.StatusCode = HttpStatusCode.Created; return res; } res.StatusCode = HttpStatusCode.Forbidden; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await store.PushAsync(indexDesc, new MemoryStream(index), reference, cancellationToken); Assert.Equal(index, gotIndex); } /// /// This test tries copying artifacts from the remote target to the memory target /// /// [Fact] public async Task CopyFromRepositoryToMemory() { var exampleManifest = @"hello world"u8.ToArray(); var exampleManifestDescriptor = new Descriptor { MediaType = MediaType.Descriptor, Digest = ComputeSHA256(exampleManifest), Size = exampleManifest.Length }; var exampleUploadUUid = new Guid().ToString(); var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; var path = req.RequestUri != null ? req.RequestUri.AbsolutePath : string.Empty; var method = req.Method; if (path.Contains("/blobs/uploads/") && method == HttpMethod.Post) { res.StatusCode = HttpStatusCode.Accepted; res.Headers.Location = new Uri($"{path}/{exampleUploadUUid}"); res.Headers.Add("Content-Type", MediaType.ImageManifest); return res; } if (path.Contains("/blobs/uploads/" + exampleUploadUUid) && method == HttpMethod.Get) { res.StatusCode = HttpStatusCode.Created; return res; } if (path.Contains("/manifests/latest") && method == HttpMethod.Put) { res.StatusCode = HttpStatusCode.Created; return res; } if (path.Contains("/manifests/" + exampleManifestDescriptor.Digest) || path.Contains("/manifests/latest") && method == HttpMethod.Head) { if (method == HttpMethod.Get) { res.Content = new ByteArrayContent(exampleManifest); res.Content.Headers.Add("Content-Type", MediaType.Descriptor); res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); return res; } res.Content.Headers.Add("Content-Type", MediaType.Descriptor); res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); return res; } if (path.Contains("/blobs/") && (method == HttpMethod.Get || method == HttpMethod.Head)) { var arr = path.Split("/"); var digest = arr[arr.Length - 1]; if (digest == exampleManifestDescriptor.Digest) { byte[] content = exampleManifest; res.Content = new ByteArrayContent(content); res.Content.Headers.Add("Content-Type", exampleManifestDescriptor.MediaType); res.Content.Headers.Add("Content-Length", content.Length.ToString()); } res.Headers.Add(_dockerContentDigestHeader, digest); return res; } if (path.Contains("/manifests/") && method == HttpMethod.Put) { res.StatusCode = HttpStatusCode.Created; return res; } return res; }; var reg = new Registry.Remote.Registry(new RepositoryOptions() { Reference = new Reference("localhost:5000"), HttpClient = CustomClient(func), }); var src = await reg.GetRepositoryAsync("source", CancellationToken.None); var dst = new MemoryStore(); var tagName = "latest"; var desc = await src.CopyAsync(tagName, dst, tagName, CancellationToken.None); } [Fact] public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigestHeaders() { var reference = new Reference("eastern.haan.com", "from25to220ce"); var tests = GetTestIOStructMapForGetDescriptorClass(); foreach ((string testName, TestIOStruct dcdIOStruct) in tests) { var repo = new Repository(reference.Repository + "/" + reference.Repository); HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; var s = new ManifestStore(repo); foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) { reference.ContentReference = dcdIOStruct.ClientSuppliedReference; var resp = new HttpResponseMessage(); if (method == HttpMethod.Get) { resp.Content = new ByteArrayContent(_theAmazingBanClan); resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } else { resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } resp.RequestMessage = new HttpRequestMessage() { Method = method }; var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; var err = false; try { await resp.GenerateDescriptorAsync(reference, CancellationToken.None); } catch (Exception e) { err = true; if (!errExpected) { throw new Exception( $"[Manifest.{method}] {testName}; expected no error for request, but got err; {e.Message}"); } } if (errExpected && !err) { throw new Exception($"[Manifest.{method}] {testName}; expected error for request, but got none"); } } } } } \ No newline at end of file diff --git a/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs b/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs new file mode 100644 index 00000000..70803a84 --- /dev/null +++ b/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs @@ -0,0 +1,76 @@ +using System.Text; +using System.Text.Json; +using OrasProject.Oras.Content; +using OrasProject.Oras.Oci; +using Index = OrasProject.Oras.Oci.Index; + +namespace OrasProject.Oras.Tests.Remote.Util; + +public class RandomDataGenerator +{ + public static int RandomInt(int min, int max) + { + return new Random().Next(min, max); + } + + public static string RandomString() + { + const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + var length = RandomInt(1, chars.Length); + char[] stringChars = new char[length]; + for (int i = 0; i < length; ++i) + { + stringChars[i] = chars[RandomInt(0, chars.Length)]; + } + return new string(stringChars); + } + + public static Descriptor RandomDescriptor(string mediaType = MediaType.ImageManifest) + { + var randomBytes = RandomBytes(); + return new Descriptor + { MediaType = mediaType, Digest = Digest.ComputeSHA256(randomBytes), Size = randomBytes.Length }; + } + + public static (Manifest, byte[]) RandomManifest() + { + var manifest = new Manifest + { + Layers = new List(), + Config = new Descriptor{MediaType = MediaType.ImageConfig, Digest = Guid.NewGuid().ToString("N")}, + }; + return (manifest, Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest))); + } + + public static (Manifest, byte[]) RandomManifestWithSubject(Descriptor? subject = null) + { + var manifest = new Manifest + { + Layers = new List(), + Config = new Descriptor{MediaType = MediaType.ImageConfig, Digest = Guid.NewGuid().ToString("N")}, + }; + if (subject == null) manifest.Subject = RandomDescriptor(); + else manifest.Subject = subject; + return (manifest, Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest))); + } + + public static byte[] RandomBytes() + { + return Encoding.UTF8.GetBytes(RandomString()); + } + + public static Index RandomIndex() + { + return new Index() + { + Manifests = new List + { + RandomDescriptor(), + RandomDescriptor(), + }, + MediaType = MediaType.ImageIndex, + }; + } + + +} diff --git a/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs b/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs new file mode 100644 index 00000000..52392475 --- /dev/null +++ b/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs @@ -0,0 +1,41 @@ +using Moq; +using Moq.Protected; +using OrasProject.Oras.Oci; + +namespace OrasProject.Oras.Tests.Remote.Util; + +public class Util +{ + /// + /// AreDescriptorsEqual compares two descriptors and returns true if they are equal. + /// + /// + /// + /// + public static bool AreDescriptorsEqual(Descriptor a, Descriptor b) + { + return a.MediaType == b.MediaType && a.Digest == b.Digest && a.Size == b.Size; + } + + public static HttpClient CustomClient(Func func) + { + var moqHandler = new Mock(); + moqHandler.Protected().Setup>( + "SendAsync", + ItExpr.IsAny(), + ItExpr.IsAny() + ).ReturnsAsync(func); + return new HttpClient(moqHandler.Object); + } + + public static HttpClient CustomClient(Func> func) + { + var moqHandler = new Mock(); + moqHandler.Protected().Setup>( + "SendAsync", + ItExpr.IsAny(), + ItExpr.IsAny() + ).Returns(func); + return new HttpClient(moqHandler.Object); + } +} From 7fca0cbef92b2e8b89852ad52772e2dd51317f5c Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 19 Nov 2024 17:56:34 +1100 Subject: [PATCH 03/36] add unit tests Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Oci/Artifact.cs | 26 - .../Registry/Remote/Auth/Client.cs | 13 - .../Remote/HttpResponseMessageExtensions.cs | 1 - .../Registry/Remote/ManifestStore.cs | 6 +- tests/OrasProject.Oras.Tests/Oci/IndexTest.cs | 15 +- .../Remote/ManifestStoreTest.cs | 133 +- .../Remote/ReferrersTest.cs | 15 +- .../Remote/RepositoryTest.cs | 2253 ++++++++++++++++- .../Remote/Util/RandomDataGenerator.cs | 15 +- .../Remote/Util/Util.cs | 15 +- 10 files changed, 2378 insertions(+), 114 deletions(-) delete mode 100644 src/OrasProject.Oras/Oci/Artifact.cs delete mode 100644 src/OrasProject.Oras/Registry/Remote/Auth/Client.cs diff --git a/src/OrasProject.Oras/Oci/Artifact.cs b/src/OrasProject.Oras/Oci/Artifact.cs deleted file mode 100644 index 4edaa82f..00000000 --- a/src/OrasProject.Oras/Oci/Artifact.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace OrasProject.Oras.Oci; - -public class Artifact -{ - [JsonPropertyName("mediaType")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public string? MediaType { get; set; } - - [JsonPropertyName("artifactType")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public string? ArtifactType { get; set; } - - [JsonPropertyName("blobs")] - public required IList Blobs { get; set; } - - [JsonPropertyName("subject")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public Descriptor? Subject { get; set; } - - [JsonPropertyName("annotations")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public IDictionary? Annotations { get; set; } -} diff --git a/src/OrasProject.Oras/Registry/Remote/Auth/Client.cs b/src/OrasProject.Oras/Registry/Remote/Auth/Client.cs deleted file mode 100644 index 0decf9aa..00000000 --- a/src/OrasProject.Oras/Registry/Remote/Auth/Client.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace OrasProject.Oras.Registry.Remote.Auth -{ - internal class Client - { - - } -} diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 0076c065..5fff8f35 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -26,7 +26,6 @@ namespace OrasProject.Oras.Registry.Remote; internal static class HttpResponseMessageExtensions { private const string _dockerContentDigestHeader = "Docker-Content-Digest"; - /// /// Parses the error returned by the remote registry. /// diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 5a6a0907..da024672 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -144,9 +144,8 @@ public async Task ExistsAsync(Descriptor target, CancellationToken cancell /// /// public async Task PushAsync(Descriptor expected, Stream content, CancellationToken cancellationToken = default) - { - await PushWithIndexingAsync(expected, content, expected.Digest, cancellationToken).ConfigureAwait(false); - } + => await PushWithIndexingAsync(expected, content, expected.Digest, cancellationToken).ConfigureAwait(false); + /// /// PushReferenceASync pushes the manifest with a reference tag. @@ -328,7 +327,6 @@ private async Task InternalPushAsync(Descriptor expected, Stream stream, string request.Content = new StreamContent(stream); request.Content.Headers.ContentLength = expected.Size; request.Content.Headers.Add("Content-Type", expected.MediaType); - var client = Repository.Options.HttpClient; using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); if (response.StatusCode != HttpStatusCode.Created) diff --git a/tests/OrasProject.Oras.Tests/Oci/IndexTest.cs b/tests/OrasProject.Oras.Tests/Oci/IndexTest.cs index dacf8b25..ca1e066a 100644 --- a/tests/OrasProject.Oras.Tests/Oci/IndexTest.cs +++ b/tests/OrasProject.Oras.Tests/Oci/IndexTest.cs @@ -1,4 +1,17 @@ -using System.Text.Json; +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Text.Json; using OrasProject.Oras.Content; using OrasProject.Oras.Oci; using static OrasProject.Oras.Tests.Remote.Util.Util; diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index 83fd80fa..3540f3b6 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -1,4 +1,17 @@ -using System.Net; +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Net; using System.Text; using System.Text.Json; using OrasProject.Oras.Oci; @@ -8,77 +21,13 @@ using static OrasProject.Oras.Tests.Remote.Util.Util; using static OrasProject.Oras.Content.Digest; using Index = OrasProject.Oras.Oci.Index; - - using Xunit; -using Xunit.Abstractions; namespace OrasProject.Oras.Tests.Remote; public class ManifestStoreTest { private const string _dockerContentDigestHeader = "Docker-Content-Digest"; - - private ITestOutputHelper _output; - - public ManifestStoreTest(ITestOutputHelper output) - { - _output = output; - } - - /// - /// ManifestStore_PushAsyncWithSubjectAndReferrerSupported tests PushAsync method for pushing manifest with subject when registry supports referrers API - /// - [Fact] - public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() - { - var (_, manifestBytes) = RandomManifestWithSubject(); - var manifestDesc = new Descriptor - { - MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifestBytes), - Size = manifestBytes.Length - }; - byte[]? receivedManifest = null; - - var mockHttpRequestHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => - { - var res = new HttpResponseMessage(); - res.RequestMessage = req; - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") - { - if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - if (req.Content?.Headers?.ContentLength != null) - { - var buf = new byte[req.Content.Headers.ContentLength.Value]; - (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buf, 0); - receivedManifest = buf; - } - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); - res.StatusCode = HttpStatusCode.Created; - res.Headers.Add("OCI-Subject", "test"); - return res; - } - return new HttpResponseMessage(HttpStatusCode.Forbidden); - }; - - - var repo = new Repository(new RepositoryOptions() - { - Reference = Reference.Parse("localhost:5000/test"), - HttpClient = CustomClient(mockHttpRequestHandler), - PlainHttp = true, - }); - var cancellationToken = new CancellationToken(); - var store = new ManifestStore(repo); - Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); - await store.PushAsync(manifestDesc, new MemoryStream(manifestBytes), cancellationToken); - Assert.Equal(manifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrerState.ReferrerSupported, repo.ReferrerState); - } [Fact] public async Task ManifestStore_PullReferrersIndexListSuccessfully() @@ -155,6 +104,60 @@ public async Task ManifestStore_PullReferrersIndexListNotFound() Assert.Empty(receivedManifests); } + /// + /// ManifestStore_PushAsyncWithSubjectAndReferrerSupported tests PushAsync method for pushing manifest with subject when registry supports referrers API + /// + [Fact] + public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() + { + var (_, manifestBytes) = RandomManifestWithSubject(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifestBytes), + Size = manifestBytes.Length + }; + byte[]? receivedManifest = null; + + var mockHttpRequestHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + if (req.Content?.Headers?.ContentLength != null) + { + var buf = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buf, 0); + receivedManifest = buf; + } + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.StatusCode = HttpStatusCode.Created; + res.Headers.Add("OCI-Subject", "test"); + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + await store.PushAsync(manifestDesc, new MemoryStream(manifestBytes), cancellationToken); + Assert.Equal(manifestBytes, receivedManifest); + Assert.Equal(Referrers.ReferrerState.ReferrerSupported, repo.ReferrerState); + } + [Fact] public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupported() diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs index a3615ff8..3f12ba12 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -1,4 +1,17 @@ -using OrasProject.Oras.Exceptions; +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; using OrasProject.Oras.Registry.Remote; using static OrasProject.Oras.Tests.Remote.Util.Util; diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index acecad7c..23ed423d 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -1 +1,2252 @@ -// Copyright The ORAS Authors. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. using OrasProject.Oras.Content; using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; using OrasProject.Oras.Registry; using OrasProject.Oras.Registry.Remote; using static OrasProject.Oras.Tests.Remote.Util.Util; using System.Diagnostics; using System.Net; using System.Net.Http.Headers; using System.Text; using System.Text.Json; using System.Text.RegularExpressions; using System.Web; using Xunit; using static OrasProject.Oras.Content.Digest; namespace OrasProject.Oras.Tests.Remote; public class RepositoryTest { public struct TestIOStruct { public bool IsTag; public bool ErrExpectedOnHEAD; public string ServerCalculatedDigest; public string ClientSuppliedReference; public bool ErrExpectedOnGET; } private byte[] _theAmazingBanClan = "Ban Gu, Ban Chao, Ban Zhao"u8.ToArray(); private const string _theAmazingBanDigest = "b526a4f2be963a2f9b0990c001255669eab8a254ab1a6e3f84f1820212ac7078"; private const string _dockerContentDigestHeader = "Docker-Content-Digest"; // The following truth table aims to cover the expected GET/HEAD request outcome // for all possible permutations of the client/server "containing a digest", for // both Manifests and Blobs. Where the results between the two differ, the index // of the first column has an exclamation mark. // // The client is said to "contain a digest" if the user-supplied reference string // is of the form that contains a digest rather than a tag. The server, on the // other hand, is said to "contain a digest" if the server responded with the // special header `Docker-Content-Digest`. // // In this table, anything denoted with an asterisk indicates that the true // response should actually be the opposite of what's expected; for example, // `*PASS` means we will get a `PASS`, even though the true answer would be its // diametric opposite--a `FAIL`. This may seem odd, and deserves an explanation. // This function has blind-spots, and while it can expend power to gain sight, // i.e., perform the expensive validation, we chose not to. The reason is two- // fold: a) we "know" that even if we say "!PASS", it will eventually fail later // when checks are performed, and with that assumption, we have the luxury for // the second point, which is b) performance. // // _______________________________________________________________________________________________________________ // | ID | CLIENT | SERVER | Manifest.GET | Blob.GET | Manifest.HEAD | Blob.HEAD | // |----+-----------------+------------------+-----------------------+-----------+---------------------+-----------+ // | 1 | tag | missing | CALCULATE,PASS | n/a | FAIL | n/a | // | 2 | tag | presentCorrect | TRUST,PASS | n/a | TRUST,PASS | n/a | // | 3 | tag | presentIncorrect | TRUST,*PASS | n/a | TRUST,*PASS | n/a | // | 4 | correctDigest | missing | TRUST,PASS | PASS | TRUST,PASS | PASS | // | 5 | correctDigest | presentCorrect | TRUST,COMPARE,PASS | PASS | TRUST,COMPARE,PASS | PASS | // | 6 | correctDigest | presentIncorrect | TRUST,COMPARE,FAIL | FAIL | TRUST,COMPARE,FAIL | FAIL | // --------------------------------------------------------------------------------------------------------------- /// /// GetTestIOStructMapForGetDescriptorClass returns a map of test cases for different /// GET/HEAD request outcome for all possible permutations of the client/server "containing a digest", for /// both Manifests and Blobs. /// /// public static Dictionary GetTestIOStructMapForGetDescriptorClass() { string correctDigest = $"sha256:{_theAmazingBanDigest}"; string incorrectDigest = $"sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"; return new Dictionary { ["1. Client:Tag & Server:DigestMissing"] = new TestIOStruct { IsTag = true, ErrExpectedOnHEAD = true }, ["2. Client:Tag & Server:DigestValid"] = new TestIOStruct { IsTag = true, ServerCalculatedDigest = correctDigest }, ["3. Client:Tag & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct { IsTag = true, ServerCalculatedDigest = incorrectDigest }, ["4. Client:DigestValid & Server:DigestMissing"] = new TestIOStruct { ClientSuppliedReference = correctDigest }, ["5. Client:DigestValid & Server:DigestValid"] = new TestIOStruct { ClientSuppliedReference = correctDigest, ServerCalculatedDigest = correctDigest }, ["6. Client:DigestValid & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct { ClientSuppliedReference = correctDigest, ServerCalculatedDigest = incorrectDigest, ErrExpectedOnHEAD = true, ErrExpectedOnGET = true } }; } /// /// Repository_FetchAsync tests the FetchAsync method of the Repository. /// /// [Fact] public async Task Repository_FetchAsync() { var blob = Encoding.UTF8.GetBytes("hello world"); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = """{"manifests":[]}"""u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var resp = new HttpResponseMessage(); resp.RequestMessage = req; if (req.Method != HttpMethod.Get) { Debug.WriteLine("Expected GET request"); resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var path = req.RequestUri!.AbsolutePath; if (path == "/v2/test/blobs/" + blobDesc.Digest) { resp.Content = new ByteArrayContent(blob); resp.Content.Headers.Add("Content-Type", "application/octet-stream"); resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return resp; } if (path == "/v2/test/manifests/" + indexDesc.Digest) { if (!req.Headers.Accept.Contains(new MediaTypeWithQualityHeaderValue(MediaType.ImageIndex))) { resp.StatusCode = HttpStatusCode.BadRequest; Debug.WriteLine("manifest not convertable: " + req.Headers.Accept); return resp; } resp.Content = new ByteArrayContent(index); resp.Content.Headers.Add("Content-Type", indexDesc.MediaType); resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return resp; } resp.StatusCode = HttpStatusCode.NotFound; return resp; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var stream = await repo.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); stream = await repo.FetchAsync(indexDesc, cancellationToken); buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); } /// /// Repository_PushAsync tests the PushAsync method of the Repository /// /// [Fact] public async Task Repository_PushAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var uuid = Guid.NewGuid().ToString(); var gotBlob = new byte[blobDesc.Size]; var gotIndex = new byte[indexDesc.Size]; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var resp = new HttpResponseMessage(); resp.RequestMessage = req; if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/") { resp.Headers.Location = new Uri("http://localhost:5000/v2/test/blobs/uploads/" + uuid); resp.StatusCode = HttpStatusCode.Accepted; return resp; } if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains("application/octet-stream")) { resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var queries = HttpUtility.ParseQueryString(req.RequestUri.Query); if (queries["digest"] != blobDesc.Digest) { resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var stream = req.Content!.ReadAsStream(cancellationToken); stream.Read(gotBlob); resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); resp.StatusCode = HttpStatusCode.Created; return resp; } if (req.Method == HttpMethod.Put && req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(MediaType.ImageIndex)) { resp.StatusCode = HttpStatusCode.BadRequest; return resp; } var stream = req.Content!.ReadAsStream(cancellationToken); stream.Read(gotIndex); resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); resp.StatusCode = HttpStatusCode.Created; return resp; } resp.StatusCode = HttpStatusCode.Forbidden; return resp; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await repo.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); Assert.Equal(blob, gotBlob); await repo.PushAsync(indexDesc, new MemoryStream(index), cancellationToken); Assert.Equal(index, gotIndex); } /// /// Repository_ExistsAsync tests the ExistsAsync method of the Repository /// /// [Fact] public async Task Repository_ExistsAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) { res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.NotAcceptable); } res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var exists = await repo.ExistsAsync(blobDesc, cancellationToken); Assert.True(exists); exists = await repo.ExistsAsync(indexDesc, cancellationToken); Assert.True(exists); } /// /// Repository_DeleteAsync tests the DeleteAsync method of the Repository /// /// [Fact] public async Task Repository_DeleteAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var blobDeleted = false; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var indexDeleted = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Delete) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) { blobDeleted = true; res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.Accepted; return res; } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { indexDeleted = true; // no dockerContentDigestHeader header for manifest deletion res.StatusCode = HttpStatusCode.Accepted; return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await repo.DeleteAsync(blobDesc, cancellationToken); Assert.True(blobDeleted); await repo.DeleteAsync(indexDesc, cancellationToken); Assert.True(indexDeleted); } /// /// Repository_ResolveAsync tests the ResolveAsync method of the Repository /// /// [Fact] public async Task Repository_ResolveAsync() { var blob = @"hello world"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.NotFound); } if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest || req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + reference) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await Assert.ThrowsAsync(async () => await repo.ResolveAsync(blobDesc.Digest, cancellationToken)); // await repo.ResolveAsync(blobDesc.Digest, cancellationToken); var got = await repo.ResolveAsync(indexDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); got = await repo.ResolveAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); var tagDigestRef = "whatever" + "@" + indexDesc.Digest; got = await repo.ResolveAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; got = await repo.ResolveAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, got)); } /// /// Repository_ResolveAsync tests the ResolveAsync method of the Repository /// /// [Fact] public async Task Repository_TagAsync() { var blob = "hello"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; byte[]? gotIndex = null; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.Found); } if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(index); res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (req.Content != null) { gotIndex = await req.Content.ReadAsByteArrayAsync(cancellationToken); } res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); res.StatusCode = HttpStatusCode.Created; return res; } return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); await Assert.ThrowsAnyAsync( async () => await repo.TagAsync(blobDesc, reference, cancellationToken)); await repo.TagAsync(indexDesc, reference, cancellationToken); Assert.Equal(index, gotIndex); await repo.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); Assert.Equal(index, gotIndex); } /// /// Repository_PushReferenceAsync tests the PushReferenceAsync method of the Repository /// /// [Fact] public async Task Repository_PushReferenceAsync() { var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; byte[]? gotIndex = null; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) { if (req.Headers.TryGetValues("Content-Type", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (req.Content != null) { gotIndex = await req.Content.ReadAsByteArrayAsync(); } res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); res.StatusCode = HttpStatusCode.Created; return res; } return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var streamContent = new MemoryStream(index); await repo.PushAsync(indexDesc, streamContent, reference, cancellationToken); Assert.Equal(index, gotIndex); } /// /// Repository_FetchReferenceAsync tests the FetchReferenceAsync method of the Repository /// /// [Fact] public async Task Repository_FetchReferenceAsyc() { var blob = "hello"u8.ToArray(); var blobDesc = new Descriptor() { Digest = ComputeSHA256(blob), MediaType = "test", Size = (uint)blob.Length }; var index = @"{""manifests"":[]}"u8.ToArray(); var indexDesc = new Descriptor() { Digest = ComputeSHA256(index), MediaType = MediaType.ImageIndex, Size = index.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.NotFound); } if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) { if (req.Headers.TryGetValues("Accept", out var values) && !values.Contains(MediaType.ImageIndex)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(index); res.Content.Headers.Add("Content-Type", indexDesc.MediaType); res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.Found); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); // test with blob digest await Assert.ThrowsAsync( async () => await repo.FetchAsync(blobDesc.Digest, cancellationToken)); // test with manifest digest var data = await repo.FetchAsync(indexDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); var buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); // test with manifest tag data = await repo.FetchAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); // test with manifest tag@digest var tagDigestRef = "whatever" + "@" + indexDesc.Digest; data = await repo.FetchAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); // test with manifest FQDN var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; data = await repo.FetchAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); buf = new byte[data.Stream.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(index, buf); } /// /// Repository_TagsAsync tests the TagsAsync method of the Repository /// to check if the tags are returned correctly /// /// /// [Fact] public async Task Repository_TagsAsync() { var tagSet = new List>() { new() {"the", "quick", "brown", "fox"}, new() {"jumps", "over", "the", "lazy"}, new() {"dog"} }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get || req.RequestUri?.AbsolutePath != "/v2/test/tags/list" ) { return new HttpResponseMessage(HttpStatusCode.NotFound); } var q = req.RequestUri.Query; try { var n = int.Parse(Regex.Match(q, @"(?<=n=)\d+").Value); if (n != 4) throw new Exception(); } catch { return new HttpResponseMessage(HttpStatusCode.BadRequest); } var tags = new List(); var serverUrl = "http://localhost:5000"; var matched = Regex.Match(q, @"(?<=test=)\w+").Value; switch (matched) { case "foo": tags = tagSet[1]; res.Headers.Add("Link", $"<{serverUrl}/v2/test/tags/list?n=4&test=bar>; rel=\"next\""); break; case "bar": tags = tagSet[2]; break; default: tags = tagSet[0]; res.Headers.Add("Link", $"; rel=\"next\""); break; } var listOfTags = new Repository.TagList { Tags = tags.ToArray() }; res.Content = new StringContent(JsonSerializer.Serialize(listOfTags)); return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, TagListPageSize = 4, }); var cancellationToken = new CancellationToken(); var wantTags = new List(); foreach (var set in tagSet) { wantTags.AddRange(set); } var gotTags = new List(); await foreach (var tag in repo.ListTagsAsync().WithCancellation(cancellationToken)) { gotTags.Add(tag); } Assert.Equal(wantTags, gotTags); } /// /// BlobStore_FetchAsync tests the FetchAsync method of the BlobStore /// /// [Fact] public async Task BlobStore_FetchAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var stream = await store.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); } /// /// BlobStore_FetchAsync_CanSeek tests the FetchAsync method of the BlobStore for a stream that can seek /// /// [Fact] public async Task BlobStore_FetchAsync_CanSeek() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var seekable = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { if (seekable) { res.Headers.AcceptRanges.Add("bytes"); } if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) { } if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") { res.StatusCode = HttpStatusCode.OK; res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } long start = -1, end = -1; var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); if (hv != null && hv.From.HasValue && hv.To.HasValue) { start = hv.From.Value; end = hv.To.Value; } if (start < 0 || start > end || start >= blobDesc.Size) { return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); } end++; if (end > blobDesc.Size) { end = blobDesc.Size; } res.StatusCode = HttpStatusCode.PartialContent; res.Content = new ByteArrayContent(blob[(int)start..(int)end]); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.NotFound; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var stream = await store.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); seekable = true; stream = await store.FetchAsync(blobDesc, cancellationToken); buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); buf = new byte[stream.Length - 3]; stream.Seek(3, SeekOrigin.Begin); await stream.ReadAsync(buf, cancellationToken); var seg = blob[3..]; Assert.Equal(seg, buf); } /// /// BlobStore_FetchAsync_ZeroSizedBlob tests the FetchAsync method of the BlobStore for a zero sized blob /// /// [Fact] public async Task BlobStore_FetchAsync_ZeroSizedBlob() { var blob = ""u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { if (req.Headers.TryGetValues("Range", out var rangeHeader)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var stream = await store.FetchAsync(blobDesc, cancellationToken); var buf = new byte[stream.Length]; await stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); } /// /// BlobStore_PushAsync tests the PushAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_PushAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var gotBlob = new byte[blob.Length]; var uuid = Guid.NewGuid().ToString(); var existingQueryParameter = "existingParam=value"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Post && req.RequestUri?.AbsolutePath == $"/v2/test/blobs/uploads/") { res.StatusCode = HttpStatusCode.Accepted; res.Headers.Add("Location", $"/v2/test/blobs/uploads/{uuid}?{existingQueryParameter}"); return res; } if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) { // Assert that the existing query parameter is present var queryParameters = HttpUtility.ParseQueryString(req.RequestUri.Query); Assert.Equal("value", queryParameters["existingParam"]); if (req.Headers.TryGetValues("Content-Type", out var contentType) && contentType.FirstOrDefault() != "application/octet-stream") { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (HttpUtility.ParseQueryString(req.RequestUri.Query)["digest"] != blobDesc.Digest) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } // read content into buffer var stream = req.Content!.ReadAsStream(cancellationToken); stream.Read(gotBlob); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.Created; return res; } return new HttpResponseMessage(HttpStatusCode.Forbidden); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); await store.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); Assert.Equal(blob, gotBlob); } /// /// BlobStore_ExistsAsync tests the ExistsAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_ExistsAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var exists = await store.ExistsAsync(blobDesc, cancellationToken); Assert.True(exists); exists = await store.ExistsAsync(contentDesc, cancellationToken); Assert.False(exists); } /// /// BlobStore_DeleteAsync tests the DeleteAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_DeleteAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var blobDeleted = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Delete) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { blobDeleted = true; res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.Accepted; return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); await store.DeleteAsync(blobDesc, cancellationToken); Assert.True(blobDeleted); var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); } /// /// BlobStore_ResolveAsync tests the ResolveAsync method of the BlobStore. /// /// [Fact] public async Task BlobStore_ResolveAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); var got = await store.ResolveAsync(blobDesc.Digest, cancellationToken); Assert.Equal(blobDesc.Digest, got.Digest); Assert.Equal(blobDesc.Size, got.Size); var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; got = await store.ResolveAsync(fqdnRef, cancellationToken); Assert.Equal(blobDesc.Digest, got.Digest); var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.ResolveAsync(contentDesc.Digest, cancellationToken)); } /// /// BlobStore_FetchReferenceAsync tests the FetchReferenceAsync method of BlobStore /// /// [Fact] public async Task BlobStore_FetchReferenceAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { res.StatusCode = HttpStatusCode.MethodNotAllowed; return res; } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); // test with digest var gotDesc = await store.FetchAsync(blobDesc.Digest, cancellationToken); Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); var buf = new byte[gotDesc.Descriptor.Size]; await gotDesc.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); // test with FQDN reference var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; gotDesc = await store.FetchAsync(fqdnRef, cancellationToken); Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); var content = "foobar"u8.ToArray(); var contentDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(content), Size = content.Length }; // test with other digest await Assert.ThrowsAsync(async () => await store.FetchAsync(contentDesc.Digest, cancellationToken)); } /// /// BlobStore_FetchAsyncReferenceAsync_Seek tests the FetchAsync method of BlobStore with seek. /// /// [Fact] public async Task BlobStore_FetchReferenceAsync_Seek() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor() { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var seekable = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") { if (seekable) { res.Headers.AcceptRanges.Add("bytes"); } if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) { } if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") { res.StatusCode = HttpStatusCode.OK; res.Content = new ByteArrayContent(blob); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); var start = hv != null && hv.To.HasValue ? hv.To.Value : -1; if (start < 0 || start >= blobDesc.Size) { return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); } res.StatusCode = HttpStatusCode.PartialContent; res.Content = new ByteArrayContent(blob[(int)start..]); res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); return res; } res.Content.Headers.Add("Content-Type", "application/octet-stream"); res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); res.StatusCode = HttpStatusCode.NotFound; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new BlobStore(repo); // test non-seekable content var data = await store.FetchAsync(blobDesc.Digest, cancellationToken); Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); Assert.Equal(data.Descriptor.Size, blobDesc.Size); var buf = new byte[data.Descriptor.Size]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob, buf); // test seekable content seekable = true; data = await store.FetchAsync(blobDesc.Digest, cancellationToken); Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); Assert.Equal(data.Descriptor.Size, blobDesc.Size); data.Stream.Seek(3, SeekOrigin.Begin); buf = new byte[data.Descriptor.Size - 3]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(blob[3..], buf); } /// /// GenerateBlobDescriptor_WithVariusDockerContentDigestHeaders tests the GenerateBlobDescriptor method of BlobStore with various Docker-Content-Digest headers. /// /// /// [Fact] public void GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() { var reference = new Reference("eastern.haan.com", "from25to220ce"); var tests = GetTestIOStructMapForGetDescriptorClass(); foreach ((string testName, TestIOStruct dcdIOStruct) in tests) { if (dcdIOStruct.IsTag) { continue; } HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) { reference.ContentReference = dcdIOStruct.ClientSuppliedReference; var resp = new HttpResponseMessage(); if (method == HttpMethod.Get) { resp.Content = new ByteArrayContent(_theAmazingBanClan); resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } if (!resp.Headers.TryGetValues(_dockerContentDigestHeader, out IEnumerable? values)) { resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); resp.RequestMessage = new HttpRequestMessage() { Method = method }; } else { resp.RequestMessage = new HttpRequestMessage() { Method = method }; } var d = string.Empty; try { d = reference.Digest; } catch { throw new Exception( $"[Blob.{method}] {testName}; got digest from a tag reference unexpectedly"); } var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; if (d.Length == 0) { // To avoid an otherwise impossible scenario in the tested code // path, we set d so that verifyContentDigest does not break. d = dcdIOStruct.ServerCalculatedDigest; } var err = false; try { resp.GenerateBlobDescriptor(d); } catch (Exception e) { err = true; if (!errExpected) { throw new Exception( $"[Blob.{method}] {testName}; expected no error for request, but got err; {e.Message}"); } } if (errExpected && !err) { throw new Exception($"[Blob.{method}] {testName}; expected error for request, but got none"); } } } } /// /// ManifestStore_FetchAsync tests the FetchAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_FetchAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(manifest); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var data = await store.FetchAsync(manifestDesc, cancellationToken); var buf = new byte[data.Length]; await data.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.FetchAsync(contentDesc, cancellationToken)); } [Fact] public async Task ManifestStore_FetchAsync_ManifestUnknown() { var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(HttpStatusCode.Unauthorized); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new StringContent( """{"errors":[{"code":"UNAUTHORIZED","message":"authentication required","detail":[{"Type":"repository","Class":"","Name":"repo","Action":"pull"}]}]}"""); return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); try { var data = await store.FetchAsync("hello", cancellationToken); Assert.Fail(); } catch (ResponseException e) { Assert.Equal("UNAUTHORIZED", e.Errors?[0].Code); } } /// /// ManifestStore_PushAsync tests the PushAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_PushAsync() { var configBlob = """config"""u8.ToArray(); var manifestStr = $@"{{""layers"": [], ""size"": 0, ""config"": {{""mediaType"": ""{MediaType.ImageConfig}"", ""digest"": ""{ComputeSHA256(configBlob)}"", ""size"": {configBlob.Length}}}}}"; var manifest = Encoding.UTF8.GetBytes(manifestStr); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; byte[]? gotManifest = null; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } if (req.Content?.Headers?.ContentLength != null) { var buf = new byte[req.Content.Headers.ContentLength.Value]; (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); gotManifest = buf; } res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.StatusCode = HttpStatusCode.Created; return res; } else { return new HttpResponseMessage(HttpStatusCode.Forbidden); } }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await store.PushAsync(manifestDesc, new MemoryStream(manifest), cancellationToken); Assert.Equal(manifest, gotManifest); } /// /// ManifestStore_ExistAsync tests the ExistAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_ExistAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var exist = await store.ExistsAsync(manifestDesc, cancellationToken); Assert.True(exist); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; exist = await store.ExistsAsync(contentDesc, cancellationToken); Assert.False(exist); } /// /// ManifestStore_DeleteAsync tests the DeleteAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_DeleteAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var manifestDeleted = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { manifestDeleted = true; res.StatusCode = HttpStatusCode.Accepted; return res; } if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(manifest); res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await store.DeleteAsync(manifestDesc, cancellationToken); Assert.True(manifestDeleted); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); } /// /// ManifestStore_ResolveAsync tests the ResolveAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_ResolveAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Head) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var got = await store.ResolveAsync(manifestDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); got = await store.ResolveAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); var tagDigestRef = "whatever" + "@" + manifestDesc.Digest; got = await store.ResolveAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; got = await store.ResolveAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, got)); var content = """{"manifests":[]}"""u8.ToArray(); var contentDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(content), Size = content.Length }; await Assert.ThrowsAsync(async () => await store.ResolveAsync(contentDesc.Digest, cancellationToken)); } /// /// ManifestStore_FetchReferenceAsync tests the FetchReferenceAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_FetchReferenceAsync() { var manifest = """{"layers":[]}"""u8.ToArray(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, Digest = ComputeSHA256(manifest), Size = manifest.Length }; var reference = "foobar"; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(manifest); res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); return res; } return new HttpResponseMessage(HttpStatusCode.NotFound); }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); // test with tag var data = await store.FetchAsync(reference, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); var buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); // test with other tag var randomRef = "whatever"; await Assert.ThrowsAsync(async () => await store.FetchAsync(randomRef, cancellationToken)); // test with digest data = await store.FetchAsync(manifestDesc.Digest, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); // test with tag@digest var tagDigestRef = randomRef + "@" + manifestDesc.Digest; data = await store.FetchAsync(tagDigestRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); // test with FQDN var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; data = await store.FetchAsync(fqdnRef, cancellationToken); Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); buf = new byte[manifest.Length]; await data.Stream.ReadAsync(buf, cancellationToken); Assert.Equal(manifest, buf); } /// /// ManifestStore_TagAsync tests the TagAsync method of ManifestStore. /// /// [Fact] public async Task ManifestStore_TagAsync() { var blob = "hello world"u8.ToArray(); var blobDesc = new Descriptor { MediaType = "test", Digest = ComputeSHA256(blob), Size = blob.Length }; var index = """{"manifests":[]}"""u8.ToArray(); var indexDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(index), Size = index.Length }; var gotIndex = new byte[index.Length]; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{blobDesc.Digest}") { res.StatusCode = HttpStatusCode.NotFound; return res; } if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") { if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } res.Content = new ByteArrayContent(index); res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { indexDesc.MediaType }); return res; } if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") { if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) { res.StatusCode = HttpStatusCode.BadRequest; return res; } if (req.Content?.Headers?.ContentLength != null) { var buf = new byte[req.Content.Headers.ContentLength.Value]; (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); gotIndex = buf; } res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); res.StatusCode = HttpStatusCode.Created; return res; } res.StatusCode = HttpStatusCode.Forbidden; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await Assert.ThrowsAnyAsync(async () => await store.TagAsync(blobDesc, reference, cancellationToken)); await store.TagAsync(indexDesc, reference, cancellationToken); Assert.Equal(index, gotIndex); gotIndex = null; await store.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); Assert.Equal(index, gotIndex); } /// /// ManifestStore_PushReferenceAsync tests the PushReferenceAsync of ManifestStore. /// /// [Fact] public async Task ManifestStore_PushReferenceAsync() { var manifest = Encoding.UTF8.GetBytes($@"{{""layers"": []}}"); var indexStr = $@"{{""manifests"":[{{""mediaType"": ""{MediaType.ImageManifest}"", ""digest"": ""{ComputeSHA256(manifest)}"", ""size"": {manifest.Length}}}]}}"; var index = Encoding.UTF8.GetBytes(indexStr); var indexDesc = new Descriptor { MediaType = MediaType.ImageIndex, Digest = ComputeSHA256(index), Size = index.Length }; var gotIndex = new byte[index.Length]; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") { if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) { res.StatusCode = HttpStatusCode.BadRequest; return res; } if (req.Content?.Headers?.ContentLength != null) { var buf = new byte[req.Content.Headers.ContentLength.Value]; (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); gotIndex = buf; } res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); res.StatusCode = HttpStatusCode.Created; return res; } res.StatusCode = HttpStatusCode.Forbidden; return res; }; var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), HttpClient = CustomClient(func), PlainHttp = true, }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); await store.PushAsync(indexDesc, new MemoryStream(index), reference, cancellationToken); Assert.Equal(index, gotIndex); } /// /// This test tries copying artifacts from the remote target to the memory target /// /// [Fact] public async Task CopyFromRepositoryToMemory() { var exampleManifest = @"hello world"u8.ToArray(); var exampleManifestDescriptor = new Descriptor { MediaType = MediaType.Descriptor, Digest = ComputeSHA256(exampleManifest), Size = exampleManifest.Length }; var exampleUploadUUid = new Guid().ToString(); var func = (HttpRequestMessage req, CancellationToken cancellationToken) => { var res = new HttpResponseMessage(); res.RequestMessage = req; var path = req.RequestUri != null ? req.RequestUri.AbsolutePath : string.Empty; var method = req.Method; if (path.Contains("/blobs/uploads/") && method == HttpMethod.Post) { res.StatusCode = HttpStatusCode.Accepted; res.Headers.Location = new Uri($"{path}/{exampleUploadUUid}"); res.Headers.Add("Content-Type", MediaType.ImageManifest); return res; } if (path.Contains("/blobs/uploads/" + exampleUploadUUid) && method == HttpMethod.Get) { res.StatusCode = HttpStatusCode.Created; return res; } if (path.Contains("/manifests/latest") && method == HttpMethod.Put) { res.StatusCode = HttpStatusCode.Created; return res; } if (path.Contains("/manifests/" + exampleManifestDescriptor.Digest) || path.Contains("/manifests/latest") && method == HttpMethod.Head) { if (method == HttpMethod.Get) { res.Content = new ByteArrayContent(exampleManifest); res.Content.Headers.Add("Content-Type", MediaType.Descriptor); res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); return res; } res.Content.Headers.Add("Content-Type", MediaType.Descriptor); res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); return res; } if (path.Contains("/blobs/") && (method == HttpMethod.Get || method == HttpMethod.Head)) { var arr = path.Split("/"); var digest = arr[arr.Length - 1]; if (digest == exampleManifestDescriptor.Digest) { byte[] content = exampleManifest; res.Content = new ByteArrayContent(content); res.Content.Headers.Add("Content-Type", exampleManifestDescriptor.MediaType); res.Content.Headers.Add("Content-Length", content.Length.ToString()); } res.Headers.Add(_dockerContentDigestHeader, digest); return res; } if (path.Contains("/manifests/") && method == HttpMethod.Put) { res.StatusCode = HttpStatusCode.Created; return res; } return res; }; var reg = new Registry.Remote.Registry(new RepositoryOptions() { Reference = new Reference("localhost:5000"), HttpClient = CustomClient(func), }); var src = await reg.GetRepositoryAsync("source", CancellationToken.None); var dst = new MemoryStore(); var tagName = "latest"; var desc = await src.CopyAsync(tagName, dst, tagName, CancellationToken.None); } [Fact] public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigestHeaders() { var reference = new Reference("eastern.haan.com", "from25to220ce"); var tests = GetTestIOStructMapForGetDescriptorClass(); foreach ((string testName, TestIOStruct dcdIOStruct) in tests) { var repo = new Repository(reference.Repository + "/" + reference.Repository); HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; var s = new ManifestStore(repo); foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) { reference.ContentReference = dcdIOStruct.ClientSuppliedReference; var resp = new HttpResponseMessage(); if (method == HttpMethod.Get) { resp.Content = new ByteArrayContent(_theAmazingBanClan); resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } else { resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); } resp.RequestMessage = new HttpRequestMessage() { Method = method }; var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; var err = false; try { await resp.GenerateDescriptorAsync(reference, CancellationToken.None); } catch (Exception e) { err = true; if (!errExpected) { throw new Exception( $"[Manifest.{method}] {testName}; expected no error for request, but got err; {e.Message}"); } } if (errExpected && !err) { throw new Exception($"[Manifest.{method}] {testName}; expected error for request, but got none"); } } } } } \ No newline at end of file +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using OrasProject.Oras.Content; +using OrasProject.Oras.Exceptions; +using OrasProject.Oras.Oci; +using OrasProject.Oras.Registry; +using OrasProject.Oras.Registry.Remote; +using System.Diagnostics; +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Web; +using Xunit; +using static OrasProject.Oras.Content.Digest; +using static OrasProject.Oras.Tests.Remote.Util.Util; +using static OrasProject.Oras.Tests.Remote.Util.RandomDataGenerator; + +namespace OrasProject.Oras.Tests.Remote; + +public class RepositoryTest +{ + public struct TestIOStruct + { + public bool IsTag; + public bool ErrExpectedOnHEAD; + public string ServerCalculatedDigest; + public string ClientSuppliedReference; + public bool ErrExpectedOnGET; + } + + private byte[] _theAmazingBanClan = "Ban Gu, Ban Chao, Ban Zhao"u8.ToArray(); + private const string _theAmazingBanDigest = "b526a4f2be963a2f9b0990c001255669eab8a254ab1a6e3f84f1820212ac7078"; + + private const string _dockerContentDigestHeader = "Docker-Content-Digest"; + + // The following truth table aims to cover the expected GET/HEAD request outcome + // for all possible permutations of the client/server "containing a digest", for + // both Manifests and Blobs. Where the results between the two differ, the index + // of the first column has an exclamation mark. + // + // The client is said to "contain a digest" if the user-supplied reference string + // is of the form that contains a digest rather than a tag. The server, on the + // other hand, is said to "contain a digest" if the server responded with the + // special header `Docker-Content-Digest`. + // + // In this table, anything denoted with an asterisk indicates that the true + // response should actually be the opposite of what's expected; for example, + // `*PASS` means we will get a `PASS`, even though the true answer would be its + // diametric opposite--a `FAIL`. This may seem odd, and deserves an explanation. + // This function has blind-spots, and while it can expend power to gain sight, + // i.e., perform the expensive validation, we chose not to. The reason is two- + // fold: a) we "know" that even if we say "!PASS", it will eventually fail later + // when checks are performed, and with that assumption, we have the luxury for + // the second point, which is b) performance. + // + // _______________________________________________________________________________________________________________ + // | ID | CLIENT | SERVER | Manifest.GET | Blob.GET | Manifest.HEAD | Blob.HEAD | + // |----+-----------------+------------------+-----------------------+-----------+---------------------+-----------+ + // | 1 | tag | missing | CALCULATE,PASS | n/a | FAIL | n/a | + // | 2 | tag | presentCorrect | TRUST,PASS | n/a | TRUST,PASS | n/a | + // | 3 | tag | presentIncorrect | TRUST,*PASS | n/a | TRUST,*PASS | n/a | + // | 4 | correctDigest | missing | TRUST,PASS | PASS | TRUST,PASS | PASS | + // | 5 | correctDigest | presentCorrect | TRUST,COMPARE,PASS | PASS | TRUST,COMPARE,PASS | PASS | + // | 6 | correctDigest | presentIncorrect | TRUST,COMPARE,FAIL | FAIL | TRUST,COMPARE,FAIL | FAIL | + // --------------------------------------------------------------------------------------------------------------- + + /// + /// GetTestIOStructMapForGetDescriptorClass returns a map of test cases for different + /// GET/HEAD request outcome for all possible permutations of the client/server "containing a digest", for + /// both Manifests and Blobs. + /// + /// + public static Dictionary GetTestIOStructMapForGetDescriptorClass() + { + string correctDigest = $"sha256:{_theAmazingBanDigest}"; + string incorrectDigest = $"sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"; + + return new Dictionary + { + ["1. Client:Tag & Server:DigestMissing"] = new TestIOStruct + { + IsTag = true, + ErrExpectedOnHEAD = true + }, + ["2. Client:Tag & Server:DigestValid"] = new TestIOStruct + { + IsTag = true, + ServerCalculatedDigest = correctDigest + }, + ["3. Client:Tag & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct + { + IsTag = true, + ServerCalculatedDigest = incorrectDigest + }, + ["4. Client:DigestValid & Server:DigestMissing"] = new TestIOStruct + { + ClientSuppliedReference = correctDigest + }, + ["5. Client:DigestValid & Server:DigestValid"] = new TestIOStruct + { + ClientSuppliedReference = correctDigest, + ServerCalculatedDigest = correctDigest + }, + ["6. Client:DigestValid & Server:DigestWrongButSyntacticallyValid"] = new TestIOStruct + { + ClientSuppliedReference = correctDigest, + ServerCalculatedDigest = incorrectDigest, + ErrExpectedOnHEAD = true, + ErrExpectedOnGET = true + } + }; + } + + /// + /// Repository_FetchAsync tests the FetchAsync method of the Repository. + /// + /// + [Fact] + public async Task Repository_FetchAsync() + { + var blob = Encoding.UTF8.GetBytes("hello world"); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = """{"manifests":[]}"""u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + resp.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + Debug.WriteLine("Expected GET request"); + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + + var path = req.RequestUri!.AbsolutePath; + if (path == "/v2/test/blobs/" + blobDesc.Digest) + { + resp.Content = new ByteArrayContent(blob); + resp.Content.Headers.Add("Content-Type", "application/octet-stream"); + resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return resp; + } + + if (path == "/v2/test/manifests/" + indexDesc.Digest) + { + if (!req.Headers.Accept.Contains(new MediaTypeWithQualityHeaderValue(MediaType.ImageIndex))) + { + resp.StatusCode = HttpStatusCode.BadRequest; + Debug.WriteLine("manifest not convertable: " + req.Headers.Accept); + return resp; + } + + resp.Content = new ByteArrayContent(index); + resp.Content.Headers.Add("Content-Type", indexDesc.MediaType); + resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + return resp; + } + + resp.StatusCode = HttpStatusCode.NotFound; + return resp; + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var stream = await repo.FetchAsync(blobDesc, cancellationToken); + var buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + stream = await repo.FetchAsync(indexDesc, cancellationToken); + buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(index, buf); + + } + + /// + /// Repository_PushAsync tests the PushAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_PushAsync() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = @"{""manifests"":[]}"u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + var uuid = Guid.NewGuid().ToString(); + var gotBlob = new byte[blobDesc.Size]; + var gotIndex = new byte[indexDesc.Size]; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var resp = new HttpResponseMessage(); + resp.RequestMessage = req; + if (req.Method == HttpMethod.Post && req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/") + { + resp.Headers.Location = new Uri("http://localhost:5000/v2/test/blobs/uploads/" + uuid); + resp.StatusCode = HttpStatusCode.Accepted; + return resp; + } + + if (req.Method == HttpMethod.Put && + req.RequestUri!.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) + { + if (req.Headers.TryGetValues("Content-Type", out var values) && + !values.Contains("application/octet-stream")) + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + + } + + var queries = HttpUtility.ParseQueryString(req.RequestUri.Query); + if (queries["digest"] != blobDesc.Digest) + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + + var stream = req.Content!.ReadAsStream(cancellationToken); + stream.Read(gotBlob); + resp.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + resp.StatusCode = HttpStatusCode.Created; + return resp; + + } + + if (req.Method == HttpMethod.Put && + req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + { + if (req.Headers.TryGetValues("Content-Type", out var values) && + !values.Contains(MediaType.ImageIndex)) + { + resp.StatusCode = HttpStatusCode.BadRequest; + return resp; + } + + var stream = req.Content!.ReadAsStream(cancellationToken); + stream.Read(gotIndex); + resp.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + resp.StatusCode = HttpStatusCode.Created; + return resp; + } + + resp.StatusCode = HttpStatusCode.Forbidden; + return resp; + + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + await repo.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); + Assert.Equal(blob, gotBlob); + await repo.PushAsync(indexDesc, new MemoryStream(index), cancellationToken); + Assert.Equal(index, gotIndex); + } + + /// + /// Repository_ExistsAsync tests the ExistsAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_ExistsAsync() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = @"{""manifests"":[]}"u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) + { + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + { + if (req.Headers.TryGetValues("Accept", out var values) && + !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.NotAcceptable); + } + + res.Content.Headers.Add("Content-Type", indexDesc.MediaType); + res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); + res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var exists = await repo.ExistsAsync(blobDesc, cancellationToken); + Assert.True(exists); + exists = await repo.ExistsAsync(indexDesc, cancellationToken); + Assert.True(exists); + } + + /// + /// Repository_DeleteAsync tests the DeleteAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_DeleteAsync() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var blobDeleted = false; + var index = @"{""manifests"":[]}"u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + var indexDeleted = false; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Delete) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) + { + blobDeleted = true; + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + { + indexDeleted = true; + // no dockerContentDigestHeader header for manifest deletion + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + await repo.DeleteAsync(blobDesc, cancellationToken); + Assert.True(blobDeleted); + await repo.DeleteAsync(indexDesc, cancellationToken); + Assert.True(indexDeleted); + } + + /// + /// Repository_ResolveAsync tests the ResolveAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_ResolveAsync() + { + var blob = @"hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = @"{""manifests"":[]}"u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + var reference = "foobar"; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) + { + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest + || req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + reference) + + { + if (req.Headers.TryGetValues("Accept", out var values) && + !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + res.Content.Headers.Add("Content-Type", indexDesc.MediaType); + res.Content.Headers.Add("Content-Length", indexDesc.Size.ToString()); + res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + await Assert.ThrowsAsync(async () => + await repo.ResolveAsync(blobDesc.Digest, cancellationToken)); + // await repo.ResolveAsync(blobDesc.Digest, cancellationToken); + var got = await repo.ResolveAsync(indexDesc.Digest, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, got)); + + got = await repo.ResolveAsync(reference, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, got)); + var tagDigestRef = "whatever" + "@" + indexDesc.Digest; + got = await repo.ResolveAsync(tagDigestRef, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, got)); + var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; + got = await repo.ResolveAsync(fqdnRef, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, got)); + } + + /// + /// Repository_ResolveAsync tests the ResolveAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_TagAsync() + { + var blob = "hello"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = @"{""manifests"":[]}"u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + byte[]? gotIndex = null; + var reference = "foobar"; + + var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Get && + req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) + { + return new HttpResponseMessage(HttpStatusCode.Found); + } + + if (req.Method == HttpMethod.Get && + req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + { + if (req.Headers.TryGetValues("Accept", out var values) && + !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + res.Content = new ByteArrayContent(index); + res.Content.Headers.Add("Content-Type", indexDesc.MediaType); + res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + return res; + } + + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference + || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + + { + if (req.Headers.TryGetValues("Content-Type", out var values) && + !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + if (req.Content != null) + { + gotIndex = await req.Content.ReadAsByteArrayAsync(cancellationToken); + } + res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + res.StatusCode = HttpStatusCode.Created; + return res; + } + + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + await Assert.ThrowsAnyAsync( + async () => await repo.TagAsync(blobDesc, reference, cancellationToken)); + await repo.TagAsync(indexDesc, reference, cancellationToken); + Assert.Equal(index, gotIndex); + await repo.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); + Assert.Equal(index, gotIndex); + } + + /// + /// Repository_PushReferenceAsync tests the PushReferenceAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_PushReferenceAsync() + { + var index = @"{""manifests"":[]}"u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + byte[]? gotIndex = null; + var reference = "foobar"; + + var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) + { + if (req.Headers.TryGetValues("Content-Type", out var values) && + !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + if (req.Content != null) + { + gotIndex = await req.Content.ReadAsByteArrayAsync(); + } + res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + res.StatusCode = HttpStatusCode.Created; + return res; + } + + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var streamContent = new MemoryStream(index); + await repo.PushAsync(indexDesc, streamContent, reference, cancellationToken); + Assert.Equal(index, gotIndex); + } + + /// + /// Repository_FetchReferenceAsync tests the FetchReferenceAsync method of the Repository + /// + /// + [Fact] + public async Task Repository_FetchReferenceAsyc() + { + var blob = "hello"u8.ToArray(); + var blobDesc = new Descriptor() + { + Digest = ComputeSHA256(blob), + MediaType = "test", + Size = (uint)blob.Length + }; + var index = @"{""manifests"":[]}"u8.ToArray(); + var indexDesc = new Descriptor() + { + Digest = ComputeSHA256(index), + MediaType = MediaType.ImageIndex, + Size = index.Length + }; + var reference = "foobar"; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + blobDesc.Digest) + { + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + if (req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest + || req.RequestUri?.AbsolutePath == "/v2/test/manifests/" + reference) + { + if (req.Headers.TryGetValues("Accept", out var values) && + !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + res.Content = new ByteArrayContent(index); + res.Content.Headers.Add("Content-Type", indexDesc.MediaType); + res.Headers.Add(_dockerContentDigestHeader, indexDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.Found); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + + // test with blob digest + await Assert.ThrowsAsync( + async () => await repo.FetchAsync(blobDesc.Digest, cancellationToken)); + + // test with manifest digest + var data = await repo.FetchAsync(indexDesc.Digest, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); + var buf = new byte[data.Stream.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(index, buf); + + // test with manifest tag + data = await repo.FetchAsync(reference, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); + buf = new byte[data.Stream.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(index, buf); + + // test with manifest tag@digest + var tagDigestRef = "whatever" + "@" + indexDesc.Digest; + data = await repo.FetchAsync(tagDigestRef, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); + buf = new byte[data.Stream.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(index, buf); + + // test with manifest FQDN + var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; + data = await repo.FetchAsync(fqdnRef, cancellationToken); + Assert.True(AreDescriptorsEqual(indexDesc, data.Descriptor)); + + buf = new byte[data.Stream.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(index, buf); + } + + /// + /// Repository_TagsAsync tests the TagsAsync method of the Repository + /// to check if the tags are returned correctly + /// + /// + /// + [Fact] + public async Task Repository_TagsAsync() + { + var tagSet = new List>() + { + new() {"the", "quick", "brown", "fox"}, + new() {"jumps", "over", "the", "lazy"}, + new() {"dog"} + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get || + req.RequestUri?.AbsolutePath != "/v2/test/tags/list" + ) + { + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + var q = req.RequestUri.Query; + try + { + var n = int.Parse(Regex.Match(q, @"(?<=n=)\d+").Value); + if (n != 4) throw new Exception(); + } + catch + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + var tags = new List(); + var serverUrl = "http://localhost:5000"; + var matched = Regex.Match(q, @"(?<=test=)\w+").Value; + switch (matched) + { + case "foo": + tags = tagSet[1]; + res.Headers.Add("Link", $"<{serverUrl}/v2/test/tags/list?n=4&test=bar>; rel=\"next\""); + break; + case "bar": + tags = tagSet[2]; + break; + default: + tags = tagSet[0]; + res.Headers.Add("Link", $"; rel=\"next\""); + break; + } + + var listOfTags = new Repository.TagList + { + Tags = tags.ToArray() + }; + res.Content = new StringContent(JsonSerializer.Serialize(listOfTags)); + return res; + + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + TagListPageSize = 4, + }); + + var cancellationToken = new CancellationToken(); + + var wantTags = new List(); + foreach (var set in tagSet) + { + wantTags.AddRange(set); + } + var gotTags = new List(); + await foreach (var tag in repo.ListTagsAsync().WithCancellation(cancellationToken)) + { + gotTags.Add(tag); + } + Assert.Equal(wantTags, gotTags); + } + + /// + /// BlobStore_FetchAsync tests the FetchAsync method of the BlobStore + /// + /// + [Fact] + public async Task BlobStore_FetchAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + res.Content = new ByteArrayContent(blob); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + var stream = await store.FetchAsync(blobDesc, cancellationToken); + var buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + + } + + /// + /// BlobStore_FetchAsync_CanSeek tests the FetchAsync method of the BlobStore for a stream that can seek + /// + /// + [Fact] + public async Task BlobStore_FetchAsync_CanSeek() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var seekable = false; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + if (seekable) + { + res.Headers.AcceptRanges.Add("bytes"); + } + + if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) + { + } + + + if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") + { + res.StatusCode = HttpStatusCode.OK; + res.Content = new ByteArrayContent(blob); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + + long start = -1, end = -1; + var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); + if (hv != null && hv.From.HasValue && hv.To.HasValue) + { + start = hv.From.Value; + end = hv.To.Value; + } + + if (start < 0 || start > end || start >= blobDesc.Size) + { + return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); + } + + end++; + if (end > blobDesc.Size) + { + end = blobDesc.Size; + } + + res.StatusCode = HttpStatusCode.PartialContent; + res.Content = new ByteArrayContent(blob[(int)start..(int)end]); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + res.StatusCode = HttpStatusCode.NotFound; + return res; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + var stream = await store.FetchAsync(blobDesc, cancellationToken); + var buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + + seekable = true; + stream = await store.FetchAsync(blobDesc, cancellationToken); + buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + + buf = new byte[stream.Length - 3]; + stream.Seek(3, SeekOrigin.Begin); + await stream.ReadAsync(buf, cancellationToken); + var seg = blob[3..]; + Assert.Equal(seg, buf); + } + + /// + /// BlobStore_FetchAsync_ZeroSizedBlob tests the FetchAsync method of the BlobStore for a zero sized blob + /// + /// + [Fact] + public async Task BlobStore_FetchAsync_ZeroSizedBlob() + { + var blob = ""u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + if (req.Headers.TryGetValues("Range", out var rangeHeader)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + var stream = await store.FetchAsync(blobDesc, cancellationToken); + var buf = new byte[stream.Length]; + await stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + } + + /// + /// BlobStore_PushAsync tests the PushAsync method of the BlobStore. + /// + /// + [Fact] + public async Task BlobStore_PushAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var gotBlob = new byte[blob.Length]; + var uuid = Guid.NewGuid().ToString(); + var existingQueryParameter = "existingParam=value"; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Post && req.RequestUri?.AbsolutePath == $"/v2/test/blobs/uploads/") + { + res.StatusCode = HttpStatusCode.Accepted; + res.Headers.Add("Location", $"/v2/test/blobs/uploads/{uuid}?{existingQueryParameter}"); + return res; + } + + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == "/v2/test/blobs/uploads/" + uuid) + { + // Assert that the existing query parameter is present + var queryParameters = HttpUtility.ParseQueryString(req.RequestUri.Query); + Assert.Equal("value", queryParameters["existingParam"]); + + if (req.Headers.TryGetValues("Content-Type", out var contentType) && + contentType.FirstOrDefault() != "application/octet-stream") + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + if (HttpUtility.ParseQueryString(req.RequestUri.Query)["digest"] != blobDesc.Digest) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + // read content into buffer + var stream = req.Content!.ReadAsStream(cancellationToken); + stream.Read(gotBlob); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + res.StatusCode = HttpStatusCode.Created; + return res; + } + + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + await store.PushAsync(blobDesc, new MemoryStream(blob), cancellationToken); + Assert.Equal(blob, gotBlob); + } + + /// + /// BlobStore_ExistsAsync tests the ExistsAsync method of the BlobStore. + /// + /// + [Fact] + public async Task BlobStore_ExistsAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var content = "foobar"u8.ToArray(); + var contentDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(content), + Size = content.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + res.StatusCode = HttpStatusCode.MethodNotAllowed; + return res; + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + var exists = await store.ExistsAsync(blobDesc, cancellationToken); + Assert.True(exists); + exists = await store.ExistsAsync(contentDesc, cancellationToken); + Assert.False(exists); + } + + /// + /// BlobStore_DeleteAsync tests the DeleteAsync method of the BlobStore. + /// + /// + [Fact] + public async Task BlobStore_DeleteAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var blobDeleted = false; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Delete) + { + res.StatusCode = HttpStatusCode.MethodNotAllowed; + return res; + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + blobDeleted = true; + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + await store.DeleteAsync(blobDesc, cancellationToken); + Assert.True(blobDeleted); + + var content = "foobar"u8.ToArray(); + var contentDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(content), + Size = content.Length + }; + await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); + } + + /// + /// BlobStore_ResolveAsync tests the ResolveAsync method of the BlobStore. + /// + /// + [Fact] + public async Task BlobStore_ResolveAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + res.StatusCode = HttpStatusCode.MethodNotAllowed; + return res; + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + res.Content.Headers.Add("Content-Length", blobDesc.Size.ToString()); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + var got = await store.ResolveAsync(blobDesc.Digest, cancellationToken); + Assert.Equal(blobDesc.Digest, got.Digest); + Assert.Equal(blobDesc.Size, got.Size); + + var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; + got = await store.ResolveAsync(fqdnRef, cancellationToken); + Assert.Equal(blobDesc.Digest, got.Digest); + + var content = "foobar"u8.ToArray(); + var contentDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(content), + Size = content.Length + }; + await Assert.ThrowsAsync(async () => + await store.ResolveAsync(contentDesc.Digest, cancellationToken)); + } + + /// + /// BlobStore_FetchReferenceAsync tests the FetchReferenceAsync method of BlobStore + /// + /// + [Fact] + public async Task BlobStore_FetchReferenceAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + res.StatusCode = HttpStatusCode.MethodNotAllowed; + return res; + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + res.Content = new ByteArrayContent(blob); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new BlobStore(repo); + + // test with digest + var gotDesc = await store.FetchAsync(blobDesc.Digest, cancellationToken); + Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); + Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); + + var buf = new byte[gotDesc.Descriptor.Size]; + await gotDesc.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + + // test with FQDN reference + var fqdnRef = $"localhost:5000/test@{blobDesc.Digest}"; + gotDesc = await store.FetchAsync(fqdnRef, cancellationToken); + Assert.Equal(blobDesc.Digest, gotDesc.Descriptor.Digest); + Assert.Equal(blobDesc.Size, gotDesc.Descriptor.Size); + + var content = "foobar"u8.ToArray(); + var contentDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(content), + Size = content.Length + }; + // test with other digest + await Assert.ThrowsAsync(async () => + await store.FetchAsync(contentDesc.Digest, cancellationToken)); + } + + /// + /// BlobStore_FetchAsyncReferenceAsync_Seek tests the FetchAsync method of BlobStore with seek. + /// + /// + [Fact] + public async Task BlobStore_FetchReferenceAsync_Seek() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor() + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var seekable = false; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.RequestUri?.AbsolutePath == $"/v2/test/blobs/{blobDesc.Digest}") + { + if (seekable) + { + res.Headers.AcceptRanges.Add("bytes"); + } + + if (req.Headers.TryGetValues("Range", out IEnumerable? rangeHeader)) + { + } + + + if (!seekable || rangeHeader == null || rangeHeader.FirstOrDefault() == "") + { + res.StatusCode = HttpStatusCode.OK; + res.Content = new ByteArrayContent(blob); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + + var hv = req.Headers?.Range?.Ranges?.FirstOrDefault(); + var start = hv != null && hv.To.HasValue ? hv.To.Value : -1; + if (start < 0 || start >= blobDesc.Size) + { + return new HttpResponseMessage(HttpStatusCode.RequestedRangeNotSatisfiable); + } + + res.StatusCode = HttpStatusCode.PartialContent; + res.Content = new ByteArrayContent(blob[(int)start..]); + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + return res; + } + + res.Content.Headers.Add("Content-Type", "application/octet-stream"); + res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); + res.StatusCode = HttpStatusCode.NotFound; + return res; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + + var store = new BlobStore(repo); + + // test non-seekable content + + var data = await store.FetchAsync(blobDesc.Digest, cancellationToken); + + Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); + Assert.Equal(data.Descriptor.Size, blobDesc.Size); + + var buf = new byte[data.Descriptor.Size]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob, buf); + + // test seekable content + seekable = true; + data = await store.FetchAsync(blobDesc.Digest, cancellationToken); + Assert.Equal(data.Descriptor.Digest, blobDesc.Digest); + Assert.Equal(data.Descriptor.Size, blobDesc.Size); + + data.Stream.Seek(3, SeekOrigin.Begin); + buf = new byte[data.Descriptor.Size - 3]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(blob[3..], buf); + } + + + /// + /// GenerateBlobDescriptor_WithVariusDockerContentDigestHeaders tests the GenerateBlobDescriptor method of BlobStore with various Docker-Content-Digest headers. + /// + /// + /// + [Fact] + public void GenerateBlobDescriptor_WithVariousDockerContentDigestHeaders() + { + var reference = new Reference("eastern.haan.com", "from25to220ce"); + var tests = GetTestIOStructMapForGetDescriptorClass(); + foreach ((string testName, TestIOStruct dcdIOStruct) in tests) + { + if (dcdIOStruct.IsTag) + { + continue; + } + HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; + foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) + { + reference.ContentReference = dcdIOStruct.ClientSuppliedReference; + var resp = new HttpResponseMessage(); + if (method == HttpMethod.Get) + { + resp.Content = new ByteArrayContent(_theAmazingBanClan); + resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); + } + if (!resp.Headers.TryGetValues(_dockerContentDigestHeader, out IEnumerable? values)) + { + resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); + resp.RequestMessage = new HttpRequestMessage() + { + Method = method + }; + + } + else + { + resp.RequestMessage = new HttpRequestMessage() + { + Method = method + }; + } + + var d = string.Empty; + try + { + d = reference.Digest; + } + catch + { + throw new Exception( + $"[Blob.{method}] {testName}; got digest from a tag reference unexpectedly"); + } + + var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; + if (d.Length == 0) + { + // To avoid an otherwise impossible scenario in the tested code + // path, we set d so that verifyContentDigest does not break. + d = dcdIOStruct.ServerCalculatedDigest; + } + + var err = false; + try + { + resp.GenerateBlobDescriptor(d); + } + catch (Exception e) + { + err = true; + if (!errExpected) + { + throw new Exception( + $"[Blob.{method}] {testName}; expected no error for request, but got err; {e.Message}"); + } + + } + + if (errExpected && !err) + { + throw new Exception($"[Blob.{method}] {testName}; expected error for request, but got none"); + } + } + } + } + + + /// + /// ManifestStore_FetchAsync tests the FetchAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_FetchAsync() + { + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifest), + Size = manifest.Length + }; + + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(manifest); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + var data = await store.FetchAsync(manifestDesc, cancellationToken); + var buf = new byte[data.Length]; + await data.ReadAsync(buf, cancellationToken); + Assert.Equal(manifest, buf); + + var content = """{"manifests":[]}"""u8.ToArray(); + var contentDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(content), + Size = content.Length + }; + await Assert.ThrowsAsync(async () => await store.FetchAsync(contentDesc, cancellationToken)); + } + + [Fact] + public async Task ManifestStore_FetchAsync_ManifestUnknown() + { + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(HttpStatusCode.Unauthorized); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && + !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new StringContent("""{"errors":[{"code":"UNAUTHORIZED","message":"authentication required","detail":[{"Type":"repository","Class":"","Name":"repo","Action":"pull"}]}]}"""); + return res; + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + try + { + var data = await store.FetchAsync("hello", cancellationToken); + Assert.Fail(); + } + catch (ResponseException e) + { + Assert.Equal("UNAUTHORIZED", e.Errors?[0].Code); + } + } + + /// + /// ManifestStore_PushAsync tests the PushAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_PushAsync() + { + var (_, manifestBytes) = RandomManifest(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifestBytes), + Size = manifestBytes.Length + }; + byte[]? gotManifest = null; + + var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + if (req.Content?.Headers?.ContentLength != null) + { + var buf = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); + gotManifest = buf; + } + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.StatusCode = HttpStatusCode.Created; + return res; + } + else + { + return new HttpResponseMessage(HttpStatusCode.Forbidden); + } + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + await store.PushAsync(manifestDesc, new MemoryStream(manifestBytes), cancellationToken); + Assert.Equal(manifestBytes, gotManifest); + } + + /// + /// ManifestStore_ExistAsync tests the ExistAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_ExistAsync() + { + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifest), + Size = manifest.Length + }; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); + res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + var exist = await store.ExistsAsync(manifestDesc, cancellationToken); + Assert.True(exist); + + var content = """{"manifests":[]}"""u8.ToArray(); + var contentDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(content), + Size = content.Length + }; + exist = await store.ExistsAsync(contentDesc, cancellationToken); + Assert.False(exist); + } + + /// + /// ManifestStore_DeleteAsync tests the DeleteAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_DeleteAsync() + { + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifest), + Size = manifest.Length + }; + var manifestDeleted = false; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + manifestDeleted = true; + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(manifest); + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + await store.DeleteAsync(manifestDesc, cancellationToken); + Assert.True(manifestDeleted); + + var content = """{"manifests":[]}"""u8.ToArray(); + var contentDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(content), + Size = content.Length + }; + await Assert.ThrowsAsync(async () => await store.DeleteAsync(contentDesc, cancellationToken)); + } + + /// + /// ManifestStore_ResolveAsync tests the ResolveAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_ResolveAsync() + { + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifest), + Size = manifest.Length + }; + var reference = "foobar"; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Head) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); + res.Content.Headers.Add("Content-Length", new string[] { manifest.Length.ToString() }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + var got = await store.ResolveAsync(manifestDesc.Digest, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, got)); + got = await store.ResolveAsync(reference, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, got)); + + var tagDigestRef = "whatever" + "@" + manifestDesc.Digest; + got = await store.ResolveAsync(tagDigestRef, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, got)); + + var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; + got = await store.ResolveAsync(fqdnRef, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, got)); + + var content = """{"manifests":[]}"""u8.ToArray(); + var contentDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(content), + Size = content.Length + }; + + await Assert.ThrowsAsync(async () => await store.ResolveAsync(contentDesc.Digest, cancellationToken)); + + } + + /// + /// ManifestStore_FetchReferenceAsync tests the FetchReferenceAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_FetchReferenceAsync() + { + var manifest = """{"layers":[]}"""u8.ToArray(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifest), + Size = manifest.Length + }; + var reference = "foobar"; + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(manifest); + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + // test with tag + var data = await store.FetchAsync(reference, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); + var buf = new byte[manifest.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(manifest, buf); + + // test with other tag + var randomRef = "whatever"; + await Assert.ThrowsAsync(async () => await store.FetchAsync(randomRef, cancellationToken)); + + // test with digest + data = await store.FetchAsync(manifestDesc.Digest, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); + + buf = new byte[manifest.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(manifest, buf); + + // test with tag@digest + var tagDigestRef = randomRef + "@" + manifestDesc.Digest; + data = await store.FetchAsync(tagDigestRef, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); + buf = new byte[manifest.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(manifest, buf); + + // test with FQDN + var fqdnRef = "localhost:5000/test" + ":" + tagDigestRef; + data = await store.FetchAsync(fqdnRef, cancellationToken); + Assert.True(AreDescriptorsEqual(manifestDesc, data.Descriptor)); + buf = new byte[manifest.Length]; + await data.Stream.ReadAsync(buf, cancellationToken); + Assert.Equal(manifest, buf); + } + + /// + /// ManifestStore_TagAsync tests the TagAsync method of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_TagAsync() + { + var blob = "hello world"u8.ToArray(); + var blobDesc = new Descriptor + { + MediaType = "test", + Digest = ComputeSHA256(blob), + Size = blob.Length + }; + var index = """{"manifests":[]}"""u8.ToArray(); + var indexDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(index), + Size = index.Length + }; + var gotIndex = new byte[index.Length]; + var reference = "foobar"; + + var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{blobDesc.Digest}") + { + res.StatusCode = HttpStatusCode.NotFound; + return res; + } + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(index); + res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { indexDesc.MediaType }); + return res; + } + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}" || req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") + { + if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) + { + res.StatusCode = HttpStatusCode.BadRequest; + return res; + } + if (req.Content?.Headers?.ContentLength != null) + { + var buf = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); + gotIndex = buf; + } + + res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); + res.StatusCode = HttpStatusCode.Created; + return res; + } + + res.StatusCode = HttpStatusCode.Forbidden; + return res; + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + await Assert.ThrowsAnyAsync(async () => await store.TagAsync(blobDesc, reference, cancellationToken)); + + await store.TagAsync(indexDesc, reference, cancellationToken); + Assert.Equal(index, gotIndex); + + gotIndex = null; + await store.TagAsync(indexDesc, indexDesc.Digest, cancellationToken); + Assert.Equal(index, gotIndex); + } + + /// + /// ManifestStore_PushReferenceAsync tests the PushReferenceAsync of ManifestStore. + /// + /// + [Fact] + public async Task ManifestStore_PushReferenceAsync() + { + var manifest = Encoding.UTF8.GetBytes($@"{{""layers"": []}}"); + var indexStr = $@"{{""manifests"":[{{""mediaType"": ""{MediaType.ImageManifest}"", ""digest"": ""{ComputeSHA256(manifest)}"", ""size"": {manifest.Length}}}]}}"; + var index = Encoding.UTF8.GetBytes(indexStr); + var indexDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(index), + Size = index.Length + }; + var gotIndex = new byte[index.Length]; + var reference = "foobar"; + + var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{reference}") + { + if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(indexDesc.MediaType)) + { + res.StatusCode = HttpStatusCode.BadRequest; + return res; + } + + if (req.Content?.Headers?.ContentLength != null) + { + var buf = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync()).CopyTo(buf, 0); + gotIndex = buf; + } + + res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); + res.StatusCode = HttpStatusCode.Created; + return res; + } + res.StatusCode = HttpStatusCode.Forbidden; + return res; + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(func), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + await store.PushAsync(indexDesc, new MemoryStream(index), reference, cancellationToken); + Assert.Equal(index, gotIndex); + } + + /// + /// This test tries copying artifacts from the remote target to the memory target + /// + /// + [Fact] + public async Task CopyFromRepositoryToMemory() + { + var exampleManifest = @"hello world"u8.ToArray(); + + var exampleManifestDescriptor = new Descriptor + { + MediaType = MediaType.Descriptor, + Digest = ComputeSHA256(exampleManifest), + Size = exampleManifest.Length + }; + var exampleUploadUUid = new Guid().ToString(); + var func = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + var path = req.RequestUri != null ? req.RequestUri.AbsolutePath : string.Empty; + var method = req.Method; + if (path.Contains("/blobs/uploads/") && method == HttpMethod.Post) + { + res.StatusCode = HttpStatusCode.Accepted; + res.Headers.Location = new Uri($"{path}/{exampleUploadUUid}"); + res.Headers.Add("Content-Type", MediaType.ImageManifest); + return res; + } + if (path.Contains("/blobs/uploads/" + exampleUploadUUid) && method == HttpMethod.Get) + { + res.StatusCode = HttpStatusCode.Created; + return res; + } + + if (path.Contains("/manifests/latest") && method == HttpMethod.Put) + { + res.StatusCode = HttpStatusCode.Created; + return res; + } + + if (path.Contains("/manifests/" + exampleManifestDescriptor.Digest) || path.Contains("/manifests/latest") && method == HttpMethod.Head) + { + if (method == HttpMethod.Get) + { + res.Content = new ByteArrayContent(exampleManifest); + res.Content.Headers.Add("Content-Type", MediaType.Descriptor); + res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); + res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); + return res; + } + res.Content.Headers.Add("Content-Type", MediaType.Descriptor); + res.Headers.Add(_dockerContentDigestHeader, exampleManifestDescriptor.Digest); + res.Content.Headers.Add("Content-Length", exampleManifest.Length.ToString()); + return res; + } + + + if (path.Contains("/blobs/") && (method == HttpMethod.Get || method == HttpMethod.Head)) + { + var arr = path.Split("/"); + var digest = arr[arr.Length - 1]; + + + if (digest == exampleManifestDescriptor.Digest) + { + byte[] content = exampleManifest; + res.Content = new ByteArrayContent(content); + res.Content.Headers.Add("Content-Type", exampleManifestDescriptor.MediaType); + res.Content.Headers.Add("Content-Length", content.Length.ToString()); + } + + res.Headers.Add(_dockerContentDigestHeader, digest); + + return res; + } + + if (path.Contains("/manifests/") && method == HttpMethod.Put) + { + res.StatusCode = HttpStatusCode.Created; + return res; + } + + return res; + }; + + var reg = new Registry.Remote.Registry(new RepositoryOptions() + { + Reference = new Reference("localhost:5000"), + HttpClient = CustomClient(func), + }); + var src = await reg.GetRepositoryAsync("source", CancellationToken.None); + + var dst = new MemoryStore(); + var tagName = "latest"; + var desc = await src.CopyAsync(tagName, dst, tagName, CancellationToken.None); + } + + [Fact] + public async Task ManifestStore_generateDescriptorWithVariousDockerContentDigestHeaders() + { + var reference = new Reference("eastern.haan.com", "from25to220ce"); + var tests = GetTestIOStructMapForGetDescriptorClass(); + foreach ((string testName, TestIOStruct dcdIOStruct) in tests) + { + var repo = new Repository(reference.Repository + "/" + reference.Repository); + HttpMethod[] methods = new HttpMethod[] { HttpMethod.Get, HttpMethod.Head }; + var s = new ManifestStore(repo); + foreach ((int i, HttpMethod method) in methods.Select((value, i) => (i, value))) + { + reference.ContentReference = dcdIOStruct.ClientSuppliedReference; + var resp = new HttpResponseMessage(); + if (method == HttpMethod.Get) + { + resp.Content = new ByteArrayContent(_theAmazingBanClan); + resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); + } + else + { + resp.Content.Headers.Add("Content-Type", new string[] { "application/vnd.docker.distribution.manifest.v2+json" }); + resp.Headers.Add(_dockerContentDigestHeader, new string[] { dcdIOStruct.ServerCalculatedDigest }); + } + resp.RequestMessage = new HttpRequestMessage() + { + Method = method + }; + + var errExpected = new bool[] { dcdIOStruct.ErrExpectedOnGET, dcdIOStruct.ErrExpectedOnHEAD }[i]; + + var err = false; + try + { + await resp.GenerateDescriptorAsync(reference, CancellationToken.None); + } + catch (Exception e) + { + err = true; + if (!errExpected) + { + throw new Exception( + $"[Manifest.{method}] {testName}; expected no error for request, but got err; {e.Message}"); + } + + } + if (errExpected && !err) + { + throw new Exception($"[Manifest.{method}] {testName}; expected error for request, but got none"); + } + } + } + + } +} diff --git a/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs b/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs index 70803a84..f34df44f 100644 --- a/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs +++ b/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs @@ -1,4 +1,17 @@ -using System.Text; +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Text; using System.Text.Json; using OrasProject.Oras.Content; using OrasProject.Oras.Oci; diff --git a/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs b/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs index 52392475..6da68cab 100644 --- a/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs +++ b/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs @@ -1,4 +1,17 @@ -using Moq; +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Moq; using Moq.Protected; using OrasProject.Oras.Oci; From 7809549243e9dbd7f2820c1b4a69d065cd7cc18b Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 19 Nov 2024 18:24:51 +1100 Subject: [PATCH 04/36] resolve merge conflicts Signed-off-by: Patrick Pan --- .../Exceptions/NoReferrerUpdateException.cs | 2 +- tests/OrasProject.Oras.Tests/PackerTest.cs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs b/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs index 87ab2b5a..54ff5d94 100644 --- a/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs +++ b/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs @@ -30,7 +30,7 @@ public NoReferrerUpdateException(string message) { } - public NoReferrerUpdateException(string message, Exception innerException) + public NoReferrerUpdateException(string message, Exception? innerException) : base(message, innerException) { } diff --git a/tests/OrasProject.Oras.Tests/PackerTest.cs b/tests/OrasProject.Oras.Tests/PackerTest.cs index c656d872..129d2143 100644 --- a/tests/OrasProject.Oras.Tests/PackerTest.cs +++ b/tests/OrasProject.Oras.Tests/PackerTest.cs @@ -146,12 +146,12 @@ public async Task TestPackManifestImageV1_0_WithOptions() Layers = layers }; var manifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(manifest)); - appendBlob(Oci.MediaType.ImageManifest, manifestBytes); + appendBlob(MediaType.ImageManifest, manifestBytes); }; var getBytes = (string data) => Encoding.UTF8.GetBytes(data); - appendBlob(Oci.MediaType.ImageConfig, getBytes("config")); // blob 0 - appendBlob(Oci.MediaType.ImageLayer, getBytes("hello world")); // blob 1 - appendBlob(Oci.MediaType.ImageLayer, getBytes("goodbye world")); // blob 2 + appendBlob(MediaType.ImageConfig, getBytes("config")); // blob 0 + appendBlob(MediaType.ImageLayer, getBytes("hello world")); // blob 1 + appendBlob(MediaType.ImageLayer, getBytes("goodbye world")); // blob 2 var layers = descs.GetRange(1, 2); var configBytes = Encoding.UTF8.GetBytes("{}"); var configDesc = new Descriptor From c7e441830a1dc9ebbee2e1ed63432805f3023d73 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 19 Nov 2024 18:31:59 +1100 Subject: [PATCH 05/36] add tests Signed-off-by: Patrick Pan --- .../Remote/RepositoryTest.cs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index ded9fca8..11dc7304 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -1602,8 +1602,7 @@ public async Task ManifestStore_FetchAsync_ManifestUnknown() { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } - if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && - !values.Contains(MediaType.ImageManifest)) + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) { return new HttpResponseMessage(HttpStatusCode.BadRequest); } @@ -2045,16 +2044,15 @@ public async Task ManifestStore_TagAsync() [Fact] public async Task ManifestStore_PushReferenceAsync() { - var manifest = Encoding.UTF8.GetBytes($@"{{""layers"": []}}"); - var indexStr = $@"{{""manifests"":[{{""mediaType"": ""{MediaType.ImageManifest}"", ""digest"": ""{ComputeSHA256(manifest)}"", ""size"": {manifest.Length}}}]}}"; - var index = Encoding.UTF8.GetBytes(indexStr); + var index = RandomIndex(); + var indexBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(index)); var indexDesc = new Descriptor { MediaType = MediaType.ImageIndex, - Digest = ComputeSHA256(index), - Size = index.Length + Digest = ComputeSHA256(indexBytes), + Size = indexBytes.Length }; - var gotIndex = new byte[index.Length]; + var gotIndex = new byte[indexBytes.Length]; var reference = "foobar"; var func = async (HttpRequestMessage req, CancellationToken cancellationToken) => @@ -2092,8 +2090,8 @@ public async Task ManifestStore_PushReferenceAsync() }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - await store.PushAsync(indexDesc, new MemoryStream(index), reference, cancellationToken); - Assert.Equal(index, gotIndex); + await store.PushAsync(indexDesc, new MemoryStream(indexBytes), reference, cancellationToken); + Assert.Equal(indexBytes, gotIndex); } /// From a6d552f73edef36b2ed494362b997a68ff43d2b9 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Wed, 20 Nov 2024 09:21:24 +1100 Subject: [PATCH 06/36] add comments Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Oci/Descriptor.cs | 2 +- .../Registry/Remote/HttpResponseMessageExtensions.cs | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/OrasProject.Oras/Oci/Descriptor.cs b/src/OrasProject.Oras/Oci/Descriptor.cs index 8b006266..8e552c62 100644 --- a/src/OrasProject.Oras/Oci/Descriptor.cs +++ b/src/OrasProject.Oras/Oci/Descriptor.cs @@ -77,7 +77,7 @@ internal static bool IsEmptyOrNull(Descriptor? descriptor) return descriptor == null || descriptor.Size == 0 || string.IsNullOrEmpty(descriptor.Digest) || string.IsNullOrEmpty(descriptor.MediaType); } - internal static Descriptor EmptyDescriptor() => new Descriptor + internal static Descriptor EmptyDescriptor() => new () { MediaType = "", Digest = "", diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 5fff8f35..908ab97a 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -101,6 +101,12 @@ public static void VerifyContentDigest(this HttpResponseMessage response, string } } + /// + /// CheckOciSubjectHeader checks if the response header contains "OCI-Subject", + /// repository ReferrerState is set to supported if it is present + /// + /// + /// public static void CheckOciSubjectHeader(this HttpResponseMessage response, Repository repository) { if (response.Headers.TryGetValues("OCI-Subject", out var values)) From 685ab702f5663b8a5a3b141a172f1f86095e171c Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Wed, 20 Nov 2024 11:16:31 +1100 Subject: [PATCH 07/36] add unit test Signed-off-by: Patrick Pan --- .../Remote/ManifestStoreTest.cs | 54 +++++++++++++++---- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index 3540f3b6..d13a7654 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -110,12 +110,30 @@ public async Task ManifestStore_PullReferrersIndexListNotFound() [Fact] public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() { - var (_, manifestBytes) = RandomManifestWithSubject(); - var manifestDesc = new Descriptor + // first push with image manifest + var (_, expectedManifestBytes) = RandomManifestWithSubject(); + var expectedManifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifestBytes), - Size = manifestBytes.Length + Digest = ComputeSHA256(expectedManifestBytes), + Size = expectedManifestBytes.Length + }; + + // second push with index manifest + var expectedIndexManifest = new Index() + { + Subject = RandomDescriptor(), + Manifests = new List{ RandomDescriptor(), RandomDescriptor() }, + MediaType = MediaType.ImageIndex, + ArtifactType = MediaType.ImageIndex, + }; + var expectedIndexManifestBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(expectedIndexManifest)); + var expectedIndexManifestDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(expectedIndexManifestBytes), + Size = expectedIndexManifestBytes.Length, + ArtifactType = MediaType.ImageIndex, }; byte[]? receivedManifest = null; @@ -123,11 +141,18 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() { var res = new HttpResponseMessage(); res.RequestMessage = req; - if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + if (req.Method == HttpMethod.Put && (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}" || + req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedIndexManifestDesc.Digest}" )) { - if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values)) { - return new HttpResponseMessage(HttpStatusCode.BadRequest); + if ((req.RequestUri.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}" && + !values.Contains(MediaType.ImageManifest)) || + (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{expectedIndexManifestDesc.Digest}" && + !values.Contains(MediaType.ImageIndex))) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } } if (req.Content?.Headers?.ContentLength != null) { @@ -135,7 +160,8 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buf, 0); receivedManifest = buf; } - res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}") res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedManifestDesc.Digest }); + else res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedIndexManifestDesc.Digest }); res.StatusCode = HttpStatusCode.Created; res.Headers.Add("OCI-Subject", "test"); return res; @@ -143,7 +169,6 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() return new HttpResponseMessage(HttpStatusCode.Forbidden); }; - var repo = new Repository(new RepositoryOptions() { Reference = Reference.Parse("localhost:5000/test"), @@ -152,9 +177,16 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() }); var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); + + // first push with image manifest Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); - await store.PushAsync(manifestDesc, new MemoryStream(manifestBytes), cancellationToken); - Assert.Equal(manifestBytes, receivedManifest); + await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); + Assert.Equal(expectedManifestBytes, receivedManifest); + Assert.Equal(Referrers.ReferrerState.ReferrerSupported, repo.ReferrerState); + + // second push with index manifest + await store.PushAsync(expectedIndexManifestDesc, new MemoryStream(expectedIndexManifestBytes), cancellationToken); + Assert.Equal(expectedIndexManifestBytes, receivedManifest); Assert.Equal(Referrers.ReferrerState.ReferrerSupported, repo.ReferrerState); } From 1b9ade38b4fb447aacfb044c67b96ffdbc2b3c40 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Wed, 20 Nov 2024 15:25:07 +1100 Subject: [PATCH 08/36] add unit tests Signed-off-by: Patrick Pan --- .../Remote/ManifestStoreTest.cs | 77 ++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index d13a7654..2e21eef4 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -104,6 +104,80 @@ public async Task ManifestStore_PullReferrersIndexListNotFound() Assert.Empty(receivedManifests); } + [Fact] + public async Task ManifestStore_PushAsyncWithoutSubject() + { + // first push with image manifest + var (_, expectedManifestBytes) = RandomManifest(); + var expectedManifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(expectedManifestBytes), + Size = expectedManifestBytes.Length + }; + + // second push with image config + var expectedConfigBytes = """config"""u8.ToArray(); + var expectedConfigDesc = new Descriptor + { + MediaType = MediaType.ImageConfig, + Digest = ComputeSHA256(expectedConfigBytes), + Size = expectedConfigBytes.Length + }; + + byte[]? receivedManifest = null; + var mockHttpRequestHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Put && (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedConfigDesc.Digest}" || + req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}")) + { + if (req.Headers.TryGetValues("Content-Type", out IEnumerable? values)) + { + if ((req.RequestUri.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}" && + !values.Contains(MediaType.ImageManifest)) || + (req.RequestUri.AbsolutePath == $"/v2/test/manifests/{expectedConfigDesc.Digest}" && + !values.Contains(MediaType.ImageConfig))) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + } + if (req.Content?.Headers?.ContentLength != null) + { + var buf = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buf, 0); + receivedManifest = buf; + } + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}") + res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedManifestDesc.Digest }); + else res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedConfigDesc.Digest }); + res.StatusCode = HttpStatusCode.Created; + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); + Assert.Equal(expectedManifestBytes, receivedManifest); + + Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + await store.PushAsync(expectedConfigDesc, new MemoryStream(expectedConfigBytes), cancellationToken); + Assert.Equal(expectedConfigBytes, receivedManifest); + Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + } + + /// /// ManifestStore_PushAsyncWithSubjectAndReferrerSupported tests PushAsync method for pushing manifest with subject when registry supports referrers API /// @@ -160,7 +234,8 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buf, 0); receivedManifest = buf; } - if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}") res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedManifestDesc.Digest }); + if (req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}") + res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedManifestDesc.Digest }); else res.Headers.Add(_dockerContentDigestHeader, new string[] { expectedIndexManifestDesc.Digest }); res.StatusCode = HttpStatusCode.Created; res.Headers.Add("OCI-Subject", "test"); From 504159214ab551a24a61627790564d99ce64490b Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Thu, 21 Nov 2024 15:44:35 +1100 Subject: [PATCH 09/36] resolve comments Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Content/Digest.cs | 13 ------ src/OrasProject.Oras/Oci/Index.cs | 11 +---- .../Remote/HttpResponseMessageExtensions.cs | 9 ++++- .../Registry/Remote/ManifestStore.cs | 40 ++++++++++++++----- .../Registry/Remote/Referrers.cs | 3 +- .../Remote/ReferrersTest.cs | 17 ++++++++ 6 files changed, 59 insertions(+), 34 deletions(-) diff --git a/src/OrasProject.Oras/Content/Digest.cs b/src/OrasProject.Oras/Content/Digest.cs index e87ed5c0..15febfb4 100644 --- a/src/OrasProject.Oras/Content/Digest.cs +++ b/src/OrasProject.Oras/Content/Digest.cs @@ -46,19 +46,6 @@ internal static string Validate(string? digest) return digest; } - - internal static string GetAlgorithm(string digest) - { - var validatedDigest = Validate(digest); - return validatedDigest.Split(':')[0]; - } - - internal static string GetRef(string digest) - { - var validatedDigest = Validate(digest); - return validatedDigest.Split(':')[1]; - } - /// /// Generates a SHA-256 digest from a byte array. /// diff --git a/src/OrasProject.Oras/Oci/Index.cs b/src/OrasProject.Oras/Oci/Index.cs index e6329232..dc81f8d4 100644 --- a/src/OrasProject.Oras/Oci/Index.cs +++ b/src/OrasProject.Oras/Oci/Index.cs @@ -51,14 +51,7 @@ internal static (Descriptor, byte[]) GenerateIndex(IList manifests) MediaType = Oci.MediaType.ImageIndex, SchemaVersion = 2 }; - var indexContent = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(index)); - var indexDesc = new Descriptor() - { - Digest = Digest.ComputeSHA256(indexContent), - MediaType = Oci.MediaType.ImageIndex, - Size = indexContent.Length - }; - - return (indexDesc, indexContent); + var indexContent = JsonSerializer.SerializeToUtf8Bytes(index); + return (Descriptor.Create(indexContent, Oci.MediaType.ImageIndex), indexContent); } } diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 908ab97a..00473d91 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -107,12 +107,19 @@ public static void VerifyContentDigest(this HttpResponseMessage response, string /// /// /// - public static void CheckOciSubjectHeader(this HttpResponseMessage response, Repository repository) + public static void CheckOCISubjectHeader(this HttpResponseMessage response, Repository repository) { if (response.Headers.TryGetValues("OCI-Subject", out var values)) { + // Set it to ReferrerSupported when the response header contains OCI-Subject repository.ReferrerState = Referrers.ReferrerState.ReferrerSupported; } + + // If the "OCI-Subject" header is NOT set, it means that either the manifest + // has no subject OR the referrers API is NOT supported by the registry. + // + // Since we don't know whether the pushed manifest has a subject or not, + // we do not set the ReferrerState to ReferrerNotSupported here. } /// diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index da024672..7dbb810b 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -180,23 +180,30 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, st case MediaType.ImageIndex: if (Repository.ReferrerState == Referrers.ReferrerState.ReferrerSupported) { + // Push the manifest straightaway when the registry supports referrers API await InternalPushAsync(expected, content, reference, cancellationToken).ConfigureAwait(false); return; } - + var contentBytes = await content.ReadAllAsync(expected, cancellationToken); using (var contentDuplicate = new MemoryStream(contentBytes)) { + // Push the manifest when ReferrerState is Unknown or NotSupported await InternalPushAsync(expected, contentDuplicate, reference, cancellationToken).ConfigureAwait(false); } if (Repository.ReferrerState == Referrers.ReferrerState.ReferrerSupported) { + // Early exit when the registry supports Referrers API + // No need to index referrers list return; } using (var contentDuplicate = new MemoryStream(contentBytes)) { - await ProcessReferrersAndPushIndex(expected, contentDuplicate); + // 1. Index the referrers list using referrers tag schema when manifest contains a subject field + // And the ReferrerState is not supported + // 2. Or do nothing when the manifest does not contain a subject field when ReferrerState is not supported/unknown + await ProcessReferrersAndPushIndex(expected, contentDuplicate, cancellationToken); } break; default: @@ -239,13 +246,16 @@ private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, } Repository.ReferrerState = Referrers.ReferrerState.ReferrerNotSupported; - await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd)); + await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd), cancellationToken); } /// /// UpdateReferrersIndex updates the referrers index for a given subject by applying the specified referrer changes. /// If the referrers index is updated, the new index is pushed to the repository. If referrers /// garbage collection is not skipped, the old index is deleted. + /// References: + /// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#pushing-manifests-with-subject + /// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#deleting-manifests /// /// /// @@ -256,25 +266,36 @@ private async Task UpdateReferrersIndex(Descriptor subject, { try { + // 1. pull the original referrers index list using referrers tag schema var referrersTag = Referrers.BuildReferrersTag(subject); - var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag); + var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag, cancellationToken); + + // 2. apply the referrer change to referrers list var updatedReferrers = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + // 3. push the updated referrers list using referrers tag schema if (updatedReferrers.Count > 0 || repository.Options.SkipReferrersGc) { + // push a new index in either case: + // 1. the referrers list has been updated with a non-zero size + // 2. OR the updated referrers list is empty but referrers GC + // is skipped, in this case an empty index should still be pushed + // as the old index won't get deleted var (indexDesc, indexContent) = Index.GenerateIndex(updatedReferrers); using (var content = new MemoryStream(indexContent)) { await InternalPushAsync(indexDesc, content, referrersTag, cancellationToken).ConfigureAwait(false); } } - + if (repository.Options.SkipReferrersGc || Descriptor.IsEmptyOrNull(oldDesc)) { + // Skip the delete process if SkipReferrersGc is set to true or the old Descriptor is empty or null return; } - + + // 4. delete the dangling original referrers index, if applicable await DeleteAsync(oldDesc, cancellationToken).ConfigureAwait(false); } catch (NoReferrerUpdateException) @@ -296,7 +317,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, { try { - var (desc, content) = await FetchAsync(referrersTag); + var (desc, content) = await FetchAsync(referrersTag, cancellationToken); var index = JsonSerializer.Deserialize(content); if (index == null) { @@ -318,8 +339,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, /// /// /// - private async Task InternalPushAsync(Descriptor expected, Stream stream, string contentReference, - CancellationToken cancellationToken) + private async Task InternalPushAsync(Descriptor expected, Stream stream, string contentReference, CancellationToken cancellationToken) { var remoteReference = Repository.ParseReference(contentReference); var url = new UriFactory(remoteReference, Repository.Options.PlainHttp).BuildRepositoryManifest(); @@ -333,7 +353,7 @@ private async Task InternalPushAsync(Descriptor expected, Stream stream, string { throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); } - response.CheckOciSubjectHeader(Repository); + response.CheckOCISubjectHeader(Repository); response.VerifyContentDigest(expected.Digest); } diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index bbc1a7c0..1e851eab 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -37,7 +37,8 @@ internal enum ReferrerOperation internal static string BuildReferrersTag(Descriptor descriptor) { - return Digest.GetAlgorithm(descriptor.Digest) + "-" + Digest.GetRef(descriptor.Digest); + var validatedDigest = Digest.Validate(descriptor.Digest); + return validatedDigest.Substring(0, validatedDigest.IndexOf(':')) + "-" + validatedDigest.Substring(validatedDigest.IndexOf(':') + 1); } /// diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs index 3f12ba12..f992b3a8 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -22,6 +22,23 @@ namespace OrasProject.Oras.Tests.Remote; public class ReferrersTest { + [Fact] + public void BuildReferrersTag_ShouldReturnReferrersTagSuccessfully() + { + var desc = RandomDescriptor(); + var index = desc.Digest.IndexOf(':'); + var expected = desc.Digest.Substring(0, index) + "-" + desc.Digest.Substring(index + 1); + Assert.Equal(expected, Referrers.BuildReferrersTag(desc)); + } + + [Fact] + public void BuildReferrersTag_ShouldThrowInvalidDigestException() + { + var desc = RandomDescriptor(); + desc.Digest = "sha123321"; + Assert.Throws(() => Referrers.BuildReferrersTag(desc)); + } + [Fact] public void ApplyReferrerChanges_ShouldAddNewReferrers() { From c26ccb6dbdd0b6546aa25b795abe41518856ffd2 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Fri, 22 Nov 2024 00:40:40 +1100 Subject: [PATCH 10/36] add SetReferrersSupportLevel func and unit tests Signed-off-by: Patrick Pan --- ...eferrersSupportLevelAlreadySetException.cs | 20 ++++++++++ .../Remote/HttpResponseMessageExtensions.cs | 2 +- .../Registry/Remote/ManifestStore.cs | 8 ++-- .../Registry/Remote/Referrers.cs | 8 ++-- .../Registry/Remote/Repository.cs | 31 ++++++++++++++- .../Exceptions/ExceptionTest.cs | 8 ++++ .../Remote/ManifestStoreTest.cs | 24 ++++++------ .../Remote/RepositoryTest.cs | 38 +++++++++++++++++++ 8 files changed, 116 insertions(+), 23 deletions(-) create mode 100644 src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs diff --git a/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs b/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs new file mode 100644 index 00000000..5a1662d2 --- /dev/null +++ b/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs @@ -0,0 +1,20 @@ +using System; + +namespace OrasProject.Oras.Exceptions; + +public class ReferrersSupportLevelAlreadySetException : Exception +{ + public ReferrersSupportLevelAlreadySetException() + { + } + + public ReferrersSupportLevelAlreadySetException(string? message) + : base(message) + { + } + + public ReferrersSupportLevelAlreadySetException(string? message, Exception? inner) + : base(message, inner) + { + } +} diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 00473d91..354d3ac9 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -112,7 +112,7 @@ public static void CheckOCISubjectHeader(this HttpResponseMessage response, Repo if (response.Headers.TryGetValues("OCI-Subject", out var values)) { // Set it to ReferrerSupported when the response header contains OCI-Subject - repository.ReferrerState = Referrers.ReferrerState.ReferrerSupported; + repository.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); } // If the "OCI-Subject" header is NOT set, it means that either the manifest diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 7dbb810b..460244ab 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -178,7 +178,7 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, st { case MediaType.ImageManifest: case MediaType.ImageIndex: - if (Repository.ReferrerState == Referrers.ReferrerState.ReferrerSupported) + if (Repository.ReferrersSupportLevel == Referrers.ReferrersSupportLevel.ReferrersSupported) { // Push the manifest straightaway when the registry supports referrers API await InternalPushAsync(expected, content, reference, cancellationToken).ConfigureAwait(false); @@ -191,7 +191,7 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, st // Push the manifest when ReferrerState is Unknown or NotSupported await InternalPushAsync(expected, contentDuplicate, reference, cancellationToken).ConfigureAwait(false); } - if (Repository.ReferrerState == Referrers.ReferrerState.ReferrerSupported) + if (Repository.ReferrersSupportLevel == Referrers.ReferrersSupportLevel.ReferrersSupported) { // Early exit when the registry supports Referrers API // No need to index referrers list @@ -244,8 +244,8 @@ private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, default: return; } - - Repository.ReferrerState = Referrers.ReferrerState.ReferrerNotSupported; + + Repository.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersNotSupported); await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd), cancellationToken); } diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 1e851eab..9e0aff2f 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -20,11 +20,11 @@ namespace OrasProject.Oras.Registry.Remote; public class Referrers { - internal enum ReferrerState + internal enum ReferrersSupportLevel { - ReferrerUnknown, - ReferrerSupported, - ReferrerNotSupported + ReferrersUnknown, + ReferrersSupported, + ReferrersNotSupported } internal record ReferrerChange(Descriptor Referrer, ReferrerOperation ReferrerOperation); diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 5d7104e7..44f6f202 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -46,8 +46,9 @@ public class Repository : IRepository public IManifestStore Manifests => new ManifestStore(this); public RepositoryOptions Options => _opts; - internal Referrers.ReferrerState ReferrerState { get; set; } = Referrers.ReferrerState.ReferrerUnknown; - + + internal Referrers.ReferrersSupportLevel ReferrersSupportLevel { get; set; } = Referrers.ReferrersSupportLevel.ReferrersUnknown; + internal static readonly string[] DefaultManifestMediaTypes = [ Docker.MediaType.Manifest, @@ -86,6 +87,32 @@ public Repository(RepositoryOptions options) _opts = options; } + /// + /// SetReferrerSupportLevel indicates the Referrers API support level of the remote repository. + /// + /// SetReferrerSupportLevel is valid only when it is called for the first time. + /// SetReferrerSupportLevel returns ReferrersSupportLevelAlreadySetException if the + /// Referrers API support level has been already set. + /// - When the level is set to ReferrersSupported, the Referrers() function will always + /// request the Referrers API. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#listing-referrers + /// - When the level is set to ReferrersNotSupported, the Referrers() function will always + /// request the Referrers Tag. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#referrers-tag-schema + /// - When the capability is not set, the Referrers() function will automatically + /// determine which API to use. + /// + /// + /// + internal void SetReferrerSupportLevel(Referrers.ReferrersSupportLevel level) + { + if (ReferrersSupportLevel == Referrers.ReferrersSupportLevel.ReferrersUnknown) + { + ReferrersSupportLevel = level; + } else if (ReferrersSupportLevel != level) + { + throw new ReferrersSupportLevelAlreadySetException($"current support level: {ReferrersSupportLevel}, latest support level: {level}"); + } + } + /// /// FetchAsync fetches the content identified by the descriptor. /// diff --git a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs index 84ee0e62..ef92d28a 100644 --- a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs +++ b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs @@ -57,4 +57,12 @@ public async Task NoReferrerUpdateException() await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException("No referrer update")); await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException("No referrer update", null)); } + + [Fact] + public async Task ReferrersSupportLevelAlreadySetException() + { + await Assert.ThrowsAsync(() => throw new ReferrersSupportLevelAlreadySetException()); + await Assert.ThrowsAsync(() => throw new ReferrersSupportLevelAlreadySetException("Referrers support level has already been set")); + await Assert.ThrowsAsync(() => throw new ReferrersSupportLevelAlreadySetException("Referrers support level has already been set", null)); + } } diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index 2e21eef4..df2bf4b2 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -167,14 +167,14 @@ public async Task ManifestStore_PushAsyncWithoutSubject() var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); Assert.Equal(expectedManifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); await store.PushAsync(expectedConfigDesc, new MemoryStream(expectedConfigBytes), cancellationToken); Assert.Equal(expectedConfigBytes, receivedManifest); - Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); } @@ -254,15 +254,15 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() var store = new ManifestStore(repo); // first push with image manifest - Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); Assert.Equal(expectedManifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrerState.ReferrerSupported, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); // second push with index manifest await store.PushAsync(expectedIndexManifestDesc, new MemoryStream(expectedIndexManifestBytes), cancellationToken); Assert.Equal(expectedIndexManifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrerState.ReferrerSupported, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); } @@ -377,18 +377,18 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupported() var store = new ManifestStore(repo); // First push with referrer tag schema - Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); await store.PushAsync(firstExpectedManifestDesc, new MemoryStream(firstExpectedManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); Assert.Equal(firstExpectedManifestBytes, receivedManifestContent); Assert.True(oldIndexDeleted); Assert.Equal(firstExpectedIndexReferrersBytes, receivedIndexContent); // Second push with referrer tag schema - Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); await store.PushAsync(secondExpectedManifestDesc, new MemoryStream(secondExpectedManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); Assert.Equal(secondExpectedManifestBytes, receivedManifestContent); Assert.True(firstIndexDeleted); Assert.Equal(secondExpectedIndexReferrersBytes, receivedIndexContent); @@ -460,9 +460,9 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupportedWitho var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - Assert.Equal(Referrers.ReferrerState.ReferrerUnknown, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); await store.PushAsync(expectedIndexManifestDesc, new MemoryStream(expectedIndexManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrerState.ReferrerNotSupported, repo.ReferrerState); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); Assert.Equal(expectedIndexManifestBytes, receivedIndexManifestContent); Assert.Equal(expectedIndexReferrersBytes, receivedIndexReferrersContent); } diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index 11dc7304..5efe8899 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -2544,4 +2544,42 @@ public async Task Repository_MountAsync_Fallback_GetContentError() Assert.Equal(testErr, ex); Assert.Equal("post ", sequence); } + + [Fact] + public void SetReferrersSupportLevel_ShouldSet_WhenInitiallyUnknown() + { + var repo = new Repository("localhost:5000/test2"); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + } + + [Fact] + public void SetReferrersSupportLevel_ShouldThrowException_WhenChangingAfterSet() + { + var repo = new Repository("localhost:5000/test2"); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + + var exception = Assert.Throws(() => + repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersNotSupported) + ); + + Assert.Equal("current support level: ReferrersSupported, latest support level: ReferrersNotSupported", exception.Message); + } + + [Fact] + public void SetReferrersSupportLevel_ShouldNotThrowException_WhenSettingSameValue() + { + var repo = new Repository("localhost:5000/test2"); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + + var exception = Record.Exception(() => repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported)); + Assert.Null(exception); + } + + } From ddbf048ac8b72712755840e70a053d798bb6181d Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Fri, 22 Nov 2024 16:06:20 +1100 Subject: [PATCH 11/36] remove NoReferrerUpdateException and update tests accordingly Signed-off-by: Patrick Pan --- .../Exceptions/NoReferrerUpdateException.cs | 37 -------- .../Registry/Remote/ManifestStore.cs | 56 ++++++------ .../Registry/Remote/Referrers.cs | 87 ++++++++++++------- .../Registry/Remote/RepositoryOptions.cs | 7 +- .../Exceptions/ExceptionTest.cs | 8 -- .../Remote/ManifestStoreTest.cs | 86 ++++++++++++++++++ .../Remote/ReferrersTest.cs | 27 +++--- 7 files changed, 186 insertions(+), 122 deletions(-) delete mode 100644 src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs diff --git a/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs b/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs deleted file mode 100644 index 54ff5d94..00000000 --- a/src/OrasProject.Oras/Exceptions/NoReferrerUpdateException.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright The ORAS Authors. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; - -namespace OrasProject.Oras.Exceptions; - - -/// -/// NoReferrerUpdateException is thrown when no referrer update is needed. -/// -public class NoReferrerUpdateException : Exception -{ - public NoReferrerUpdateException() - { - } - - public NoReferrerUpdateException(string message) - : base(message) - { - } - - public NoReferrerUpdateException(string message, Exception? innerException) - : base(message, innerException) - { - } -} diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 460244ab..c90799cf 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -264,44 +264,38 @@ private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, private async Task UpdateReferrersIndex(Descriptor subject, Referrers.ReferrerChange referrerChange, CancellationToken cancellationToken = default) { - try - { - // 1. pull the original referrers index list using referrers tag schema - var referrersTag = Referrers.BuildReferrersTag(subject); - var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag, cancellationToken); - - // 2. apply the referrer change to referrers list - var updatedReferrers = - Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + // 1. pull the original referrers index list using referrers tag schema + var referrersTag = Referrers.BuildReferrersTag(subject); + var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag, cancellationToken); + + // 2. apply the referrer change to referrers list + var (updatedReferrers, updateRequired) = + Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + if (!updateRequired) return; - // 3. push the updated referrers list using referrers tag schema - if (updatedReferrers.Count > 0 || repository.Options.SkipReferrersGc) - { - // push a new index in either case: - // 1. the referrers list has been updated with a non-zero size - // 2. OR the updated referrers list is empty but referrers GC - // is skipped, in this case an empty index should still be pushed - // as the old index won't get deleted - var (indexDesc, indexContent) = Index.GenerateIndex(updatedReferrers); - using (var content = new MemoryStream(indexContent)) - { - await InternalPushAsync(indexDesc, content, referrersTag, cancellationToken).ConfigureAwait(false); - } - } - - if (repository.Options.SkipReferrersGc || Descriptor.IsEmptyOrNull(oldDesc)) + // 3. push the updated referrers list using referrers tag schema + if (updatedReferrers.Count > 0 || repository.Options.SkipReferrersGC) + { + // push a new index in either case: + // 1. the referrers list has been updated with a non-zero size + // 2. OR the updated referrers list is empty but referrers GC + // is skipped, in this case an empty index should still be pushed + // as the old index won't get deleted + var (indexDesc, indexContent) = Index.GenerateIndex(updatedReferrers); + using (var content = new MemoryStream(indexContent)) { - // Skip the delete process if SkipReferrersGc is set to true or the old Descriptor is empty or null - return; + await InternalPushAsync(indexDesc, content, referrersTag, cancellationToken).ConfigureAwait(false); } - - // 4. delete the dangling original referrers index, if applicable - await DeleteAsync(oldDesc, cancellationToken).ConfigureAwait(false); } - catch (NoReferrerUpdateException) + + if (repository.Options.SkipReferrersGC || Descriptor.IsEmptyOrNull(oldDesc)) { + // Skip the delete process if SkipReferrersGC is set to true or the old Descriptor is empty or null return; } + + // 4. delete the dangling original referrers index, if applicable + await DeleteAsync(oldDesc, cancellationToken).ConfigureAwait(false); } /// diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 9e0aff2f..061127ce 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -48,95 +48,118 @@ internal static string BuildReferrersTag(Descriptor descriptor) /// /// /// - /// The updated referrers list - internal static IList ApplyReferrerChanges(IList oldReferrers, ReferrerChange referrerChange) + /// The updated referrers list, updateRequired + internal static (IList, bool) ApplyReferrerChanges(IList oldReferrers, ReferrerChange referrerChange) { if (oldReferrers == null || referrerChange == null) { - throw new NoReferrerUpdateException("referrerChange or oldReferrers is null in this request"); + return (new List(), false); } + // updatedReferrers is a list to store the updated referrers var updatedReferrers = new List(); - var referrerToIndex = new Dictionary(); + // referrerIndexMap is a Dictionary to store referrer as the key + // and index(int) in the updatedReferrers list as the value + var referrerIndexMap = new Dictionary(); var updateRequired = false; foreach (var oldReferrer in oldReferrers) { if (Descriptor.IsEmptyOrNull(oldReferrer)) { + // Skip any empty or null referrers updateRequired = true; continue; } var basicDesc = oldReferrer.BasicDescriptor; - if (referrerToIndex.ContainsKey(basicDesc)) + if (referrerIndexMap.ContainsKey(basicDesc)) { + // Skip any duplicate referrers updateRequired = true; continue; } + // Update the updatedReferrers list + // Add referrer index in the referrerIndexMap updatedReferrers.Add(oldReferrer); - referrerToIndex[basicDesc] = updatedReferrers.Count - 1; + referrerIndexMap[basicDesc] = updatedReferrers.Count - 1; } - - + if (!Descriptor.IsEmptyOrNull(referrerChange.Referrer)) { var basicDesc = referrerChange.Referrer.BasicDescriptor; switch (referrerChange.ReferrerOperation) { case ReferrerOperation.ReferrerAdd: - if (!referrerToIndex.ContainsKey(basicDesc)) + if (!referrerIndexMap.ContainsKey(basicDesc)) { + // Add the new referrer only when it has not already existed in the referrerIndexMap updatedReferrers.Add(referrerChange.Referrer); - referrerToIndex[basicDesc] = updatedReferrers.Count - 1; + referrerIndexMap[basicDesc] = updatedReferrers.Count - 1; } - + break; - case ReferrerOperation.ReferrerDelete: - if (referrerToIndex.TryGetValue(basicDesc, out var index)) + if (referrerIndexMap.TryGetValue(basicDesc, out var index)) { + // Delete the referrer only when it existed in the referrerIndexMap updatedReferrers[index] = Descriptor.EmptyDescriptor(); - referrerToIndex.Remove(basicDesc); + referrerIndexMap.Remove(basicDesc); } - break; default: break; } } - if (!updateRequired && referrerToIndex.Count == oldReferrers.Count) + // Skip unnecessary update + if (!updateRequired && referrerIndexMap.Count == oldReferrers.Count) { + // Check for any new referrers in the referrerIndexMap that are not present in the oldReferrers list foreach (var oldReferrer in oldReferrers) { var basicDesc = oldReferrer.BasicDescriptor; - if (!referrerToIndex.ContainsKey(basicDesc)) updateRequired = true; + if (!referrerIndexMap.ContainsKey(basicDesc)) + { + updateRequired = true; + break; + } } - if (!updateRequired) throw new NoReferrerUpdateException("no referrer update in this request"); + if (!updateRequired) + { + return (updatedReferrers, false); + } } - RemoveEmptyDescriptors(updatedReferrers, referrerToIndex.Count); - return updatedReferrers; + RemoveEmptyDescriptors(updatedReferrers, referrerIndexMap.Count); + return (updatedReferrers, true); } /// - /// RemoveEmptyDescriptors removes any empty or null descriptors from the provided list of referrers, ensuring that only non-empty - /// descriptors remain in the list. It optimizes the list by shifting valid descriptors forward and trimming - /// the remaining elements at the end. The list is truncated to only contain non-empty descriptors up to the specified count. + /// RemoveEmptyDescriptors removes any empty or null descriptors from the provided list of descriptors, + /// ensuring that only non-empty descriptors remain in the list. + /// It optimizes the list by shifting valid descriptors forward and trimming the remaining elements at the end. + /// The list is truncated to only contain non-empty descriptors up to the specified count. /// - /// - /// - internal static void RemoveEmptyDescriptors(List updatedReferrers, int numNonEmptyReferrers) + /// + /// + internal static void RemoveEmptyDescriptors(List descriptors, int numNonEmptyDescriptors) { var lastEmptyIndex = 0; - for (var i = 0; i < updatedReferrers.Count; ++i) + for (var i = 0; i < descriptors.Count; ++i) { - if (Descriptor.IsEmptyOrNull(updatedReferrers[i])) continue; - - if (i > lastEmptyIndex) updatedReferrers[lastEmptyIndex] = updatedReferrers[i]; + if (Descriptor.IsEmptyOrNull(descriptors[i])) continue; + if (i > lastEmptyIndex) + { + // Move the descriptor at index i to lastEmptyIndex + descriptors[lastEmptyIndex] = descriptors[i]; + } ++lastEmptyIndex; - if (lastEmptyIndex == numNonEmptyReferrers) break; + if (lastEmptyIndex == numNonEmptyDescriptors) + { + // Break the loop when lastEmptyIndex reaches the number of Non-Empty descriptors + break; + } } - updatedReferrers.RemoveRange(lastEmptyIndex, updatedReferrers.Count - lastEmptyIndex); + descriptors.RemoveRange(lastEmptyIndex, descriptors.Count - lastEmptyIndex); } } diff --git a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs index 1d7b45a4..4742f9d8 100644 --- a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs +++ b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs @@ -51,10 +51,13 @@ public struct RepositoryOptions /// public int TagListPageSize { get; set; } - // SkipReferrersGc specifies whether to delete the dangling referrers + // SkipReferrersGC specifies whether to delete the dangling referrers // index when referrers tag schema is utilized. // - If false, the old referrers index will be deleted after the new one is successfully uploaded. // - If true, the old referrers index is kept. // By default, it is disabled (set to false). See also: - public bool SkipReferrersGc { get; set; } + // - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#referrers-tag-schema + // - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#pushing-manifests-with-subject + // - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#deleting-manifests + public bool SkipReferrersGC { get; set; } } diff --git a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs index ef92d28a..13888f31 100644 --- a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs +++ b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs @@ -50,14 +50,6 @@ public async Task NotFoundException() await Assert.ThrowsAsync(() => throw new NotFoundException("Not found", null)); } - [Fact] - public async Task NoReferrerUpdateException() - { - await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException()); - await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException("No referrer update")); - await Assert.ThrowsAsync(() => throw new NoReferrerUpdateException("No referrer update", null)); - } - [Fact] public async Task ReferrersSupportLevelAlreadySetException() { diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index df2bf4b2..350ae65f 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -466,4 +466,90 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupportedWitho Assert.Equal(expectedIndexManifestBytes, receivedIndexManifestContent); Assert.Equal(expectedIndexReferrersBytes, receivedIndexReferrersContent); } + + [Fact] + public async Task ManifestStore_PushAsyncWithSubjectAndNoUpdateRequired() + { + var (oldManifest, oldManifestBytes) = RandomManifestWithSubject(); + var oldIndex = new Index() + { + Manifests = new List + { + new () + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(oldManifestBytes), + Size = oldManifestBytes.Length, + ArtifactType = MediaType.ImageManifest, + } + }, + MediaType = MediaType.ImageIndex, + }; + var oldIndexBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(oldIndex)); + var oldIndexDesc = new Descriptor() + { + Digest = ComputeSHA256(oldIndexBytes), + MediaType = MediaType.ImageIndex, + Size = oldIndexBytes.Length + }; + + var expectedManifest = oldManifest; + var expectedManifestBytes = oldManifestBytes; + var expectedManifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(oldManifestBytes), + Size = oldManifestBytes.Length, + ArtifactType = MediaType.ImageManifest, + }; + + byte[]? receivedManifestContent = null; + var referrersTag = Referrers.BuildReferrersTag(expectedManifest.Subject); + + var mockHttpRequestHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var response = new HttpResponseMessage(); + response.RequestMessage = req; + + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{expectedManifestDesc.Digest}") + { + if (req.Content?.Headers?.ContentLength != null) + { + var buffer = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buffer, 0); + receivedManifestContent = buffer; + } + response.Headers.Add(_dockerContentDigestHeader, new[] { expectedManifestDesc.Digest }); + response.StatusCode = HttpStatusCode.Created; + return response; + } else if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{referrersTag}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(oldIndexBytes); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + response.Headers.Add(_dockerContentDigestHeader, new string[] { oldIndexDesc.Digest }); + response.StatusCode = HttpStatusCode.OK; + return response; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); + Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); + Assert.Equal(expectedManifestBytes, receivedManifestContent); + } } diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs index f992b3a8..5f39fd51 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -62,12 +62,13 @@ public void ApplyReferrerChanges_ShouldAddNewReferrers() Referrers.ReferrerOperation.ReferrerAdd ); - var updatedReferrers = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); Assert.Equal(3, updatedReferrers.Count); for (var i = 0; i < updatedReferrers.Count; ++i) { Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); } + Assert.True(updateRequired); } [Fact] @@ -96,12 +97,13 @@ public void ApplyReferrerChanges_ShouldDiscardDuplicateReferrers() Referrers.ReferrerOperation.ReferrerAdd ); - var updatedReferrers = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); Assert.Equal(3, updatedReferrers.Count); for (var i = 0; i < updatedReferrers.Count; ++i) { Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); } + Assert.True(updateRequired); } [Fact] @@ -118,10 +120,9 @@ public void ApplyReferrerChanges_ShouldNotAddNewDuplicateReferrers() oldDescriptor1, Referrers.ReferrerOperation.ReferrerAdd ); - - - var exception = Assert.Throws(() => Referrers.ApplyReferrerChanges(oldReferrers, referrerChange)); - Assert.Equal("no referrer update in this request", exception.Message); + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Equal(2, updatedReferrers.Count); + Assert.False(updateRequired); } [Fact] @@ -145,13 +146,13 @@ public void ApplyReferrerChanges_ShouldNotKeepOldEmptyReferrers() Referrers.ReferrerOperation.ReferrerAdd ); - var updatedReferrers = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); - + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); Assert.Single(updatedReferrers); for (var i = 0; i < updatedReferrers.Count; ++i) { Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); } + Assert.True(updateRequired); } [Fact] @@ -160,8 +161,9 @@ public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreNull() IList oldReferrers = null; Referrers.ReferrerChange referrerChange = null; - var exception = Assert.Throws(() => Referrers.ApplyReferrerChanges(oldReferrers, referrerChange)); - Assert.Equal("referrerChange or oldReferrers is null in this request", exception.Message); + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Empty(updatedReferrers); + Assert.False(updateRequired); } [Fact] @@ -170,8 +172,9 @@ public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreEmpty() var oldReferrers = new List(); var referrerChange = new Referrers.ReferrerChange(Descriptor.EmptyDescriptor(), Referrers.ReferrerOperation.ReferrerAdd); - var exception = Assert.Throws(() => Referrers.ApplyReferrerChanges(oldReferrers, referrerChange)); - Assert.Equal("no referrer update in this request", exception.Message); + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Empty(updatedReferrers); + Assert.False(updateRequired); } [Fact] From d6e849945d6e48c9364f6edfe4a3f1842e2a8d7f Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Mon, 25 Nov 2024 10:32:51 +1100 Subject: [PATCH 12/36] add Index constructor Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Content/Digest.cs | 3 ++- src/OrasProject.Oras/Oci/Index.cs | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/OrasProject.Oras/Content/Digest.cs b/src/OrasProject.Oras/Content/Digest.cs index 15febfb4..2e8c0350 100644 --- a/src/OrasProject.Oras/Content/Digest.cs +++ b/src/OrasProject.Oras/Content/Digest.cs @@ -46,6 +46,7 @@ internal static string Validate(string? digest) return digest; } + /// /// Generates a SHA-256 digest from a byte array. /// @@ -58,4 +59,4 @@ internal static string ComputeSHA256(byte[] content) var output = $"sha256:{BitConverter.ToString(hash).Replace("-", "")}"; return output.ToLower(); } -} +} \ No newline at end of file diff --git a/src/OrasProject.Oras/Oci/Index.cs b/src/OrasProject.Oras/Oci/Index.cs index dc81f8d4..918588d4 100644 --- a/src/OrasProject.Oras/Oci/Index.cs +++ b/src/OrasProject.Oras/Oci/Index.cs @@ -12,6 +12,7 @@ // limitations under the License. using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; @@ -43,14 +44,19 @@ public class Index : Versioned [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public IDictionary? Annotations { get; set; } + public Index() {} + + [SetsRequiredMembers] + public Index(IList manifests) + { + Manifests = manifests; + MediaType = Oci.MediaType.ImageIndex; + SchemaVersion = 2; + } + internal static (Descriptor, byte[]) GenerateIndex(IList manifests) { - var index = new Index() - { - Manifests = manifests, - MediaType = Oci.MediaType.ImageIndex, - SchemaVersion = 2 - }; + var index = new Index(manifests); var indexContent = JsonSerializer.SerializeToUtf8Bytes(index); return (Descriptor.Create(indexContent, Oci.MediaType.ImageIndex), indexContent); } From cf431f077f91b2eb1744be0f42d072f48de31c8b Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 26 Nov 2024 13:55:50 +1100 Subject: [PATCH 13/36] add license header Signed-off-by: Patrick Pan --- .../ReferrersSupportLevelAlreadySetException.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs b/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs index 5a1662d2..87ffd005 100644 --- a/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs +++ b/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs @@ -1,4 +1,17 @@ -using System; +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; namespace OrasProject.Oras.Exceptions; From 856c0efef880356750cda114a6e8bad71e511500 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Fri, 29 Nov 2024 15:34:14 +1100 Subject: [PATCH 14/36] add lock on SetReferrerState Signed-off-by: Patrick Pan --- ...s => ReferrersStateAlreadySetException.cs} | 8 ++-- .../Remote/HttpResponseMessageExtensions.cs | 2 +- .../Registry/Remote/ManifestStore.cs | 6 +-- .../Registry/Remote/Referrers.cs | 28 ++++++++++---- .../Registry/Remote/Repository.cs | 38 ++++++++++--------- .../Exceptions/ExceptionTest.cs | 6 +-- .../Remote/ManifestStoreTest.cs | 28 +++++++------- .../Remote/ReferrersTest.cs | 11 ------ .../Remote/RepositoryTest.cs | 34 ++++++++--------- 9 files changed, 82 insertions(+), 79 deletions(-) rename src/OrasProject.Oras/Exceptions/{ReferrersSupportLevelAlreadySetException.cs => ReferrersStateAlreadySetException.cs} (72%) diff --git a/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs b/src/OrasProject.Oras/Exceptions/ReferrersStateAlreadySetException.cs similarity index 72% rename from src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs rename to src/OrasProject.Oras/Exceptions/ReferrersStateAlreadySetException.cs index 87ffd005..4c0b7add 100644 --- a/src/OrasProject.Oras/Exceptions/ReferrersSupportLevelAlreadySetException.cs +++ b/src/OrasProject.Oras/Exceptions/ReferrersStateAlreadySetException.cs @@ -15,18 +15,18 @@ namespace OrasProject.Oras.Exceptions; -public class ReferrersSupportLevelAlreadySetException : Exception +public class ReferrersStateAlreadySetException : Exception { - public ReferrersSupportLevelAlreadySetException() + public ReferrersStateAlreadySetException() { } - public ReferrersSupportLevelAlreadySetException(string? message) + public ReferrersStateAlreadySetException(string? message) : base(message) { } - public ReferrersSupportLevelAlreadySetException(string? message, Exception? inner) + public ReferrersStateAlreadySetException(string? message, Exception? inner) : base(message, inner) { } diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 354d3ac9..0e4585ce 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -112,7 +112,7 @@ public static void CheckOCISubjectHeader(this HttpResponseMessage response, Repo if (response.Headers.TryGetValues("OCI-Subject", out var values)) { // Set it to ReferrerSupported when the response header contains OCI-Subject - repository.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); + repository.SetReferrersState(Referrers.ReferrersState.ReferrersSupported); } // If the "OCI-Subject" header is NOT set, it means that either the manifest diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 5f55f995..c4675c13 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -177,7 +177,7 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, Re { case MediaType.ImageManifest: case MediaType.ImageIndex: - if (Repository.ReferrersSupportLevel == Referrers.ReferrersSupportLevel.ReferrersSupported) + if (Repository.ReferrersState == Referrers.ReferrersState.ReferrersSupported) { // Push the manifest straightaway when the registry supports referrers API await DoPushAsync(expected, content, reference, cancellationToken).ConfigureAwait(false); @@ -190,7 +190,7 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, Re // Push the manifest when ReferrerState is Unknown or NotSupported await DoPushAsync(expected, contentDuplicate, reference, cancellationToken).ConfigureAwait(false); } - if (Repository.ReferrersSupportLevel == Referrers.ReferrersSupportLevel.ReferrersSupported) + if (Repository.ReferrersState == Referrers.ReferrersState.ReferrersSupported) { // Early exit when the registry supports Referrers API // No need to index referrers list @@ -244,7 +244,7 @@ private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, return; } - Repository.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersNotSupported); + Repository.SetReferrersState(Referrers.ReferrersState.ReferrersNotSupported); await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd), cancellationToken); } diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 061127ce..369b537c 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -20,11 +20,11 @@ namespace OrasProject.Oras.Registry.Remote; public class Referrers { - internal enum ReferrersSupportLevel + internal enum ReferrersState { - ReferrersUnknown, - ReferrersSupported, - ReferrersNotSupported + ReferrersUnknown = 0, + ReferrersSupported = 1, + ReferrersNotSupported = 2 } internal record ReferrerChange(Descriptor Referrer, ReferrerOperation ReferrerOperation); @@ -51,10 +51,6 @@ internal static string BuildReferrersTag(Descriptor descriptor) /// The updated referrers list, updateRequired internal static (IList, bool) ApplyReferrerChanges(IList oldReferrers, ReferrerChange referrerChange) { - if (oldReferrers == null || referrerChange == null) - { - return (new List(), false); - } // updatedReferrers is a list to store the updated referrers var updatedReferrers = new List(); // referrerIndexMap is a Dictionary to store referrer as the key @@ -79,10 +75,24 @@ internal static (IList, bool) ApplyReferrerChanges(IList } // Update the updatedReferrers list // Add referrer index in the referrerIndexMap + + // delete + // ...... + if (referrerChange.ReferrerOperation == ReferrerOperation.ReferrerDelete) + { + var toBeDeletedBasicDesc = referrerChange.Referrer.BasicDescriptor; + if (basicDesc == toBeDeletedBasicDesc) + { + updateRequired = true; + continue; + } + } updatedReferrers.Add(oldReferrer); referrerIndexMap[basicDesc] = updatedReferrers.Count - 1; } + // old => 1, 1 + // new => nil if (!Descriptor.IsEmptyOrNull(referrerChange.Referrer)) { var basicDesc = referrerChange.Referrer.BasicDescriptor; @@ -101,6 +111,8 @@ internal static (IList, bool) ApplyReferrerChanges(IList if (referrerIndexMap.TryGetValue(basicDesc, out var index)) { // Delete the referrer only when it existed in the referrerIndexMap + // updatedReferrers.Remove(basicDesc); + updatedReferrers[index] = Descriptor.EmptyDescriptor(); referrerIndexMap.Remove(basicDesc); } diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 44f6f202..613400f5 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -47,8 +47,14 @@ public class Repository : IRepository public RepositoryOptions Options => _opts; - internal Referrers.ReferrersSupportLevel ReferrersSupportLevel { get; set; } = Referrers.ReferrersSupportLevel.ReferrersUnknown; - + private int _referrersState = (int) Referrers.ReferrersState.ReferrersUnknown; + + internal Referrers.ReferrersState ReferrersState + { + get => (Referrers.ReferrersState) _referrersState; + private set => _referrersState = (int) value; + } + internal static readonly string[] DefaultManifestMediaTypes = [ Docker.MediaType.Manifest, @@ -88,28 +94,26 @@ public Repository(RepositoryOptions options) } /// - /// SetReferrerSupportLevel indicates the Referrers API support level of the remote repository. + /// SetReferrersState indicates the Referrers API state of the remote repository. /// - /// SetReferrerSupportLevel is valid only when it is called for the first time. - /// SetReferrerSupportLevel returns ReferrersSupportLevelAlreadySetException if the - /// Referrers API support level has been already set. - /// - When the level is set to ReferrersSupported, the Referrers() function will always + /// SetReferrersState is valid only when it is called for the first time. + /// SetReferrersState returns ReferrersStateAlreadySetException if the + /// Referrers API state has been already set. + /// - When the state is set to ReferrersSupported, the Referrers() function will always /// request the Referrers API. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#listing-referrers - /// - When the level is set to ReferrersNotSupported, the Referrers() function will always + /// - When the state is set to ReferrersNotSupported, the Referrers() function will always /// request the Referrers Tag. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#referrers-tag-schema - /// - When the capability is not set, the Referrers() function will automatically + /// - When the state is not set, the Referrers() function will automatically /// determine which API to use. /// - /// - /// - internal void SetReferrerSupportLevel(Referrers.ReferrersSupportLevel level) + /// + /// + internal void SetReferrersState(Referrers.ReferrersState state) { - if (ReferrersSupportLevel == Referrers.ReferrersSupportLevel.ReferrersUnknown) - { - ReferrersSupportLevel = level; - } else if (ReferrersSupportLevel != level) + var originalReferrersState = (Referrers.ReferrersState) Interlocked.CompareExchange(ref _referrersState, (int)state, (int)Referrers.ReferrersState.ReferrersUnknown); + if (originalReferrersState != Referrers.ReferrersState.ReferrersUnknown && _referrersState != (int) state) { - throw new ReferrersSupportLevelAlreadySetException($"current support level: {ReferrersSupportLevel}, latest support level: {level}"); + throw new ReferrersStateAlreadySetException($"current referrers state: {ReferrersState}, latest referrers state: {state}"); } } diff --git a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs index 13888f31..d9fb8a5c 100644 --- a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs +++ b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs @@ -53,8 +53,8 @@ public async Task NotFoundException() [Fact] public async Task ReferrersSupportLevelAlreadySetException() { - await Assert.ThrowsAsync(() => throw new ReferrersSupportLevelAlreadySetException()); - await Assert.ThrowsAsync(() => throw new ReferrersSupportLevelAlreadySetException("Referrers support level has already been set")); - await Assert.ThrowsAsync(() => throw new ReferrersSupportLevelAlreadySetException("Referrers support level has already been set", null)); + await Assert.ThrowsAsync(() => throw new ReferrersStateAlreadySetException()); + await Assert.ThrowsAsync(() => throw new ReferrersStateAlreadySetException("Referrers state has already been set")); + await Assert.ThrowsAsync(() => throw new ReferrersStateAlreadySetException("Referrers state has already been set", null)); } } diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index 350ae65f..e0721e99 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -167,14 +167,14 @@ public async Task ManifestStore_PushAsyncWithoutSubject() var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); Assert.Equal(expectedManifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); await store.PushAsync(expectedConfigDesc, new MemoryStream(expectedConfigBytes), cancellationToken); Assert.Equal(expectedConfigBytes, receivedManifest); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); } @@ -254,15 +254,15 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerSupported() var store = new ManifestStore(repo); // first push with image manifest - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); Assert.Equal(expectedManifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); // second push with index manifest await store.PushAsync(expectedIndexManifestDesc, new MemoryStream(expectedIndexManifestBytes), cancellationToken); Assert.Equal(expectedIndexManifestBytes, receivedManifest); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); } @@ -377,18 +377,18 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupported() var store = new ManifestStore(repo); // First push with referrer tag schema - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); await store.PushAsync(firstExpectedManifestDesc, new MemoryStream(firstExpectedManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); Assert.Equal(firstExpectedManifestBytes, receivedManifestContent); Assert.True(oldIndexDeleted); Assert.Equal(firstExpectedIndexReferrersBytes, receivedIndexContent); // Second push with referrer tag schema - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); await store.PushAsync(secondExpectedManifestDesc, new MemoryStream(secondExpectedManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); Assert.Equal(secondExpectedManifestBytes, receivedManifestContent); Assert.True(firstIndexDeleted); Assert.Equal(secondExpectedIndexReferrersBytes, receivedIndexContent); @@ -460,9 +460,9 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupportedWitho var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); await store.PushAsync(expectedIndexManifestDesc, new MemoryStream(expectedIndexManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); Assert.Equal(expectedIndexManifestBytes, receivedIndexManifestContent); Assert.Equal(expectedIndexReferrersBytes, receivedIndexReferrersContent); } @@ -547,9 +547,9 @@ public async Task ManifestStore_PushAsyncWithSubjectAndNoUpdateRequired() var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); await store.PushAsync(expectedManifestDesc, new MemoryStream(expectedManifestBytes), cancellationToken); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersNotSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); Assert.Equal(expectedManifestBytes, receivedManifestContent); } } diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs index 5f39fd51..215d5a8c 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -155,17 +155,6 @@ public void ApplyReferrerChanges_ShouldNotKeepOldEmptyReferrers() Assert.True(updateRequired); } - [Fact] - public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreNull() - { - IList oldReferrers = null; - Referrers.ReferrerChange referrerChange = null; - - var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); - Assert.Empty(updatedReferrers); - Assert.False(updateRequired); - } - [Fact] public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreEmpty() { diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index 5efe8899..6aae7e15 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -2546,40 +2546,38 @@ public async Task Repository_MountAsync_Fallback_GetContentError() } [Fact] - public void SetReferrersSupportLevel_ShouldSet_WhenInitiallyUnknown() + public void SetReferrersState_ShouldSet_WhenInitiallyUnknown() { var repo = new Repository("localhost:5000/test2"); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); - repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + repo.SetReferrersState(Referrers.ReferrersState.ReferrersSupported); + Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); } [Fact] - public void SetReferrersSupportLevel_ShouldThrowException_WhenChangingAfterSet() + public void SetReferrersState_ShouldThrowException_WhenChangingAfterSet() { var repo = new Repository("localhost:5000/test2"); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); - repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + repo.SetReferrersState(Referrers.ReferrersState.ReferrersSupported); + Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); - var exception = Assert.Throws(() => - repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersNotSupported) + var exception = Assert.Throws(() => + repo.SetReferrersState(Referrers.ReferrersState.ReferrersNotSupported) ); - Assert.Equal("current support level: ReferrersSupported, latest support level: ReferrersNotSupported", exception.Message); + Assert.Equal("current referrers state: ReferrersSupported, latest referrers state: ReferrersNotSupported", exception.Message); } [Fact] - public void SetReferrersSupportLevel_ShouldNotThrowException_WhenSettingSameValue() + public void SetReferrersState_ShouldNotThrowException_WhenSettingSameValue() { var repo = new Repository("localhost:5000/test2"); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersUnknown, repo.ReferrersSupportLevel); - repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported); - Assert.Equal(Referrers.ReferrersSupportLevel.ReferrersSupported, repo.ReferrersSupportLevel); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + repo.SetReferrersState(Referrers.ReferrersState.ReferrersSupported); + Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); - var exception = Record.Exception(() => repo.SetReferrerSupportLevel(Referrers.ReferrersSupportLevel.ReferrersSupported)); + var exception = Record.Exception(() => repo.SetReferrersState(Referrers.ReferrersState.ReferrersSupported)); Assert.Null(exception); } - - } From fcb121e207e7fc7802e79c7d495b3d9bba99fbad Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Fri, 29 Nov 2024 16:50:05 +1100 Subject: [PATCH 15/36] simplify ApplyReferrerChanges Signed-off-by: Patrick Pan --- .../Registry/Remote/Referrers.cs | 93 ++------- .../Remote/ReferrersTest.cs | 185 ++++++++++-------- 2 files changed, 122 insertions(+), 156 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 369b537c..6d15a268 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -12,6 +12,7 @@ // limitations under the License. using System.Collections.Generic; +using System.Linq; using OrasProject.Oras.Content; using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; @@ -53,9 +54,8 @@ internal static (IList, bool) ApplyReferrerChanges(IList { // updatedReferrers is a list to store the updated referrers var updatedReferrers = new List(); - // referrerIndexMap is a Dictionary to store referrer as the key - // and index(int) in the updatedReferrers list as the value - var referrerIndexMap = new Dictionary(); + // updatedReferrersSet is a HashSet to store unique referrers + var updatedReferrersSet = new HashSet(); var updateRequired = false; foreach (var oldReferrer in oldReferrers) @@ -67,69 +67,45 @@ internal static (IList, bool) ApplyReferrerChanges(IList continue; } var basicDesc = oldReferrer.BasicDescriptor; - if (referrerIndexMap.ContainsKey(basicDesc)) + if (updatedReferrersSet.Contains(basicDesc)) { // Skip any duplicate referrers updateRequired = true; continue; } // Update the updatedReferrers list - // Add referrer index in the referrerIndexMap - - // delete - // ...... - if (referrerChange.ReferrerOperation == ReferrerOperation.ReferrerDelete) + // Add referrer index in the updatedReferrersSet + if (referrerChange.ReferrerOperation == ReferrerOperation.ReferrerDelete && Descriptor.Equals(basicDesc, referrerChange.Referrer.BasicDescriptor)) { - var toBeDeletedBasicDesc = referrerChange.Referrer.BasicDescriptor; - if (basicDesc == toBeDeletedBasicDesc) - { - updateRequired = true; - continue; - } + updateRequired = true; + continue; } updatedReferrers.Add(oldReferrer); - referrerIndexMap[basicDesc] = updatedReferrers.Count - 1; + updatedReferrersSet.Add(basicDesc); } - // old => 1, 1 - // new => nil if (!Descriptor.IsEmptyOrNull(referrerChange.Referrer)) { var basicDesc = referrerChange.Referrer.BasicDescriptor; - switch (referrerChange.ReferrerOperation) + if (referrerChange.ReferrerOperation == ReferrerOperation.ReferrerAdd) { - case ReferrerOperation.ReferrerAdd: - if (!referrerIndexMap.ContainsKey(basicDesc)) - { - // Add the new referrer only when it has not already existed in the referrerIndexMap - updatedReferrers.Add(referrerChange.Referrer); - referrerIndexMap[basicDesc] = updatedReferrers.Count - 1; - } - - break; - case ReferrerOperation.ReferrerDelete: - if (referrerIndexMap.TryGetValue(basicDesc, out var index)) - { - // Delete the referrer only when it existed in the referrerIndexMap - // updatedReferrers.Remove(basicDesc); - - updatedReferrers[index] = Descriptor.EmptyDescriptor(); - referrerIndexMap.Remove(basicDesc); - } - break; - default: - break; + if (!updatedReferrersSet.Contains(basicDesc)) + { + // Add the new referrer only when it has not already existed in the updatedReferrersSet + updatedReferrers.Add(referrerChange.Referrer); + updatedReferrersSet.Add(basicDesc); + } } } // Skip unnecessary update - if (!updateRequired && referrerIndexMap.Count == oldReferrers.Count) + if (!updateRequired && updatedReferrersSet.Count == oldReferrers.Count) { - // Check for any new referrers in the referrerIndexMap that are not present in the oldReferrers list + // Check for any new referrers in the updatedReferrersSet that are not present in the oldReferrers list foreach (var oldReferrer in oldReferrers) { var basicDesc = oldReferrer.BasicDescriptor; - if (!referrerIndexMap.ContainsKey(basicDesc)) + if (!updatedReferrersSet.Contains(basicDesc)) { updateRequired = true; break; @@ -141,37 +117,6 @@ internal static (IList, bool) ApplyReferrerChanges(IList return (updatedReferrers, false); } } - - RemoveEmptyDescriptors(updatedReferrers, referrerIndexMap.Count); return (updatedReferrers, true); } - - /// - /// RemoveEmptyDescriptors removes any empty or null descriptors from the provided list of descriptors, - /// ensuring that only non-empty descriptors remain in the list. - /// It optimizes the list by shifting valid descriptors forward and trimming the remaining elements at the end. - /// The list is truncated to only contain non-empty descriptors up to the specified count. - /// - /// - /// - internal static void RemoveEmptyDescriptors(List descriptors, int numNonEmptyDescriptors) - { - var lastEmptyIndex = 0; - for (var i = 0; i < descriptors.Count; ++i) - { - if (Descriptor.IsEmptyOrNull(descriptors[i])) continue; - if (i > lastEmptyIndex) - { - // Move the descriptor at index i to lastEmptyIndex - descriptors[lastEmptyIndex] = descriptors[i]; - } - ++lastEmptyIndex; - if (lastEmptyIndex == numNonEmptyDescriptors) - { - // Break the loop when lastEmptyIndex reaches the number of Non-Empty descriptors - break; - } - } - descriptors.RemoveRange(lastEmptyIndex, descriptors.Count - lastEmptyIndex); - } } diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs index 215d5a8c..5db44fd0 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -71,6 +71,108 @@ public void ApplyReferrerChanges_ShouldAddNewReferrers() Assert.True(updateRequired); } + [Fact] + public void ApplyReferrerChanges_ShouldDeleteReferrers() + { + var oldDescriptor1 = RandomDescriptor(); + var oldDescriptor2 = RandomDescriptor(); + var oldDescriptor3 = RandomDescriptor(); + + var oldReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + oldDescriptor3 + }; + + var expectedReferrers = new List + { + oldDescriptor1, + oldDescriptor3 + }; + var referrerChange = new Referrers.ReferrerChange( + oldDescriptor2, + Referrers.ReferrerOperation.ReferrerDelete + ); + + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Equal(2, updatedReferrers.Count); + for (var i = 0; i < updatedReferrers.Count; ++i) + { + Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); + } + Assert.True(updateRequired); + } + + + [Fact] + public void ApplyReferrerChanges_ShouldDeleteReferrersWithDuplicates() + { + var oldDescriptor1 = RandomDescriptor(); + var oldDescriptor2 = RandomDescriptor(); + var oldDescriptor3 = RandomDescriptor(); + + var oldReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + oldDescriptor3, + oldDescriptor2, + oldDescriptor2, + oldDescriptor3, + }; + + var expectedReferrers = new List + { + oldDescriptor1, + oldDescriptor2 + }; + var referrerChange = new Referrers.ReferrerChange( + oldDescriptor3, + Referrers.ReferrerOperation.ReferrerDelete + ); + + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Equal(2, updatedReferrers.Count); + for (var i = 0; i < updatedReferrers.Count; ++i) + { + Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); + } + Assert.True(updateRequired); + } + + [Fact] + public void ApplyReferrerChanges_ShouldNotDeleteReferrersWhenNoUpdateRequired() + { + var oldDescriptor1 = RandomDescriptor(); + var oldDescriptor2 = RandomDescriptor(); + var oldDescriptor3 = RandomDescriptor(); + + var oldReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + }; + + var expectedReferrers = new List + { + oldDescriptor1, + oldDescriptor2, + }; + var referrerChange = new Referrers.ReferrerChange( + oldDescriptor3, + Referrers.ReferrerOperation.ReferrerDelete + ); + + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); + Assert.Equal(2, updatedReferrers.Count); + for (var i = 0; i < updatedReferrers.Count; ++i) + { + Assert.True(AreDescriptorsEqual(updatedReferrers[i], expectedReferrers[i])); + } + Assert.False(updateRequired); + } + [Fact] public void ApplyReferrerChanges_ShouldDiscardDuplicateReferrers() { @@ -156,7 +258,7 @@ public void ApplyReferrerChanges_ShouldNotKeepOldEmptyReferrers() } [Fact] - public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreEmpty() + public void ApplyReferrerChanges_NoUpdateWhenOldAndNewReferrersAreEmpty() { var oldReferrers = new List(); var referrerChange = new Referrers.ReferrerChange(Descriptor.EmptyDescriptor(), Referrers.ReferrerOperation.ReferrerAdd); @@ -165,85 +267,4 @@ public void ApplyReferrerChanges_ThrowsWhenOldAndNewReferrersAreEmpty() Assert.Empty(updatedReferrers); Assert.False(updateRequired); } - - [Fact] - public void RemoveEmptyDescriptors_ShouldRemoveEmptyDescriptors() - { - var randomDescriptor1 = RandomDescriptor(); - var randomDescriptor2 = RandomDescriptor(); - var randomDescriptor3 = RandomDescriptor(); - var randomDescriptor4 = RandomDescriptor(); - var descriptors = new List - { - Descriptor.EmptyDescriptor(), - randomDescriptor1, - Descriptor.EmptyDescriptor(), - randomDescriptor2, - Descriptor.EmptyDescriptor(), - Descriptor.EmptyDescriptor(), - randomDescriptor3, - randomDescriptor4, - }; - - var expectedDescriptors = new List - { - randomDescriptor1, - randomDescriptor2, - randomDescriptor3, - randomDescriptor4 - }; - Referrers.RemoveEmptyDescriptors(descriptors, 4); - - Assert.Equal(4, descriptors.Count); - Assert.DoesNotContain(Descriptor.EmptyDescriptor(), descriptors); - for (var i = 0; i < descriptors.Count; ++i) - { - Assert.True(AreDescriptorsEqual(descriptors[i], expectedDescriptors[i])); - } - } - - [Fact] - public void RemoveEmptyDescriptors_ShouldReturnAllNonEmptyDescriptors() - { - var randomDescriptor1 = RandomDescriptor(); - var randomDescriptor2 = RandomDescriptor(); - var randomDescriptor3 = RandomDescriptor(); - var randomDescriptor4 = RandomDescriptor(); - var descriptors = new List - { - randomDescriptor1, - randomDescriptor2, - randomDescriptor3, - randomDescriptor4, - }; - - var expectedDescriptors = new List - { - randomDescriptor1, - randomDescriptor2, - randomDescriptor3, - randomDescriptor4 - }; - Referrers.RemoveEmptyDescriptors(descriptors, 4); - Assert.Equal(4, descriptors.Count); - for (var i = 0; i < descriptors.Count; ++i) - { - Assert.True(AreDescriptorsEqual(descriptors[i], expectedDescriptors[i])); - } - } - - [Fact] - public void RemoveEmptyDescriptors_ShouldRemoveAllEmptyDescriptors() - { - var descriptors = new List - { - Descriptor.EmptyDescriptor(), - Descriptor.EmptyDescriptor(), - Descriptor.EmptyDescriptor(), - Descriptor.EmptyDescriptor(), - }; - - Referrers.RemoveEmptyDescriptors(descriptors, 0); - Assert.Empty(descriptors); - } } From 2b24e071f95ff3c0f741dce50cb51b517d949f7a Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 3 Dec 2024 13:26:53 +1100 Subject: [PATCH 16/36] resolve comments Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Oci/Descriptor.cs | 2 +- .../Remote/HttpResponseMessageExtensions.cs | 2 +- .../Registry/Remote/ManifestStore.cs | 14 +++++------ .../Registry/Remote/Referrers.cs | 24 +++++++++++-------- .../Remote/ManifestStoreTest.cs | 2 +- 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/OrasProject.Oras/Oci/Descriptor.cs b/src/OrasProject.Oras/Oci/Descriptor.cs index 8e552c62..8370aa5e 100644 --- a/src/OrasProject.Oras/Oci/Descriptor.cs +++ b/src/OrasProject.Oras/Oci/Descriptor.cs @@ -72,7 +72,7 @@ public static Descriptor Create(Span data, string mediaType) internal BasicDescriptor BasicDescriptor => new BasicDescriptor(MediaType, Digest, Size); - internal static bool IsEmptyOrNull(Descriptor? descriptor) + internal static bool IsEmptyOrInvalid(Descriptor? descriptor) { return descriptor == null || descriptor.Size == 0 || string.IsNullOrEmpty(descriptor.Digest) || string.IsNullOrEmpty(descriptor.MediaType); } diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 0e4585ce..af00c487 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -107,7 +107,7 @@ public static void VerifyContentDigest(this HttpResponseMessage response, string /// /// /// - public static void CheckOCISubjectHeader(this HttpResponseMessage response, Repository repository) + internal static void CheckOCISubjectHeader(this HttpResponseMessage response, Repository repository) { if (response.Headers.TryGetValues("OCI-Subject", out var values)) { diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index c4675c13..d74f30fd 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -184,7 +184,7 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, Re return; } - var contentBytes = await content.ReadAllAsync(expected, cancellationToken); + var contentBytes = await content.ReadAllAsync(expected, cancellationToken).ConfigureAwait(false); using (var contentDuplicate = new MemoryStream(contentBytes)) { // Push the manifest when ReferrerState is Unknown or NotSupported @@ -202,11 +202,11 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, Re // 1. Index the referrers list using referrers tag schema when manifest contains a subject field // And the ReferrerState is not supported // 2. Or do nothing when the manifest does not contain a subject field when ReferrerState is not supported/unknown - await ProcessReferrersAndPushIndex(expected, contentDuplicate, cancellationToken); + await ProcessReferrersAndPushIndex(expected, contentDuplicate, cancellationToken).ConfigureAwait(false); } break; default: - await DoPushAsync(expected, content, reference, cancellationToken); + await DoPushAsync(expected, content, reference, cancellationToken).ConfigureAwait(false); break; } } @@ -245,7 +245,7 @@ private async Task ProcessReferrersAndPushIndex(Descriptor desc, Stream content, } Repository.SetReferrersState(Referrers.ReferrersState.ReferrersNotSupported); - await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd), cancellationToken); + await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(desc, Referrers.ReferrerOperation.ReferrerAdd), cancellationToken).ConfigureAwait(false); } /// @@ -265,7 +265,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, { // 1. pull the original referrers index list using referrers tag schema var referrersTag = Referrers.BuildReferrersTag(subject); - var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag, cancellationToken); + var (oldDesc, oldReferrers) = await PullReferrersIndexList(referrersTag, cancellationToken).ConfigureAwait(false); // 2. apply the referrer change to referrers list var (updatedReferrers, updateRequired) = @@ -287,7 +287,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, } } - if (repository.Options.SkipReferrersGC || Descriptor.IsEmptyOrNull(oldDesc)) + if (repository.Options.SkipReferrersGC || Descriptor.IsEmptyOrInvalid(oldDesc)) { // Skip the delete process if SkipReferrersGC is set to true or the old Descriptor is empty or null return; @@ -310,7 +310,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, { try { - var (desc, content) = await FetchAsync(referrersTag, cancellationToken); + var (desc, content) = await FetchAsync(referrersTag, cancellationToken).ConfigureAwait(false); var index = JsonSerializer.Deserialize(content); if (index == null) { diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 6d15a268..0e1fbe63 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -52,6 +52,11 @@ internal static string BuildReferrersTag(Descriptor descriptor) /// The updated referrers list, updateRequired internal static (IList, bool) ApplyReferrerChanges(IList oldReferrers, ReferrerChange referrerChange) { + if (Descriptor.IsEmptyOrInvalid(referrerChange.Referrer)) + { + return (oldReferrers, false); + } + // updatedReferrers is a list to store the updated referrers var updatedReferrers = new List(); // updatedReferrersSet is a HashSet to store unique referrers @@ -60,7 +65,7 @@ internal static (IList, bool) ApplyReferrerChanges(IList var updateRequired = false; foreach (var oldReferrer in oldReferrers) { - if (Descriptor.IsEmptyOrNull(oldReferrer)) + if (Descriptor.IsEmptyOrInvalid(oldReferrer)) { // Skip any empty or null referrers updateRequired = true; @@ -84,19 +89,18 @@ internal static (IList, bool) ApplyReferrerChanges(IList updatedReferrersSet.Add(basicDesc); } - if (!Descriptor.IsEmptyOrNull(referrerChange.Referrer)) + + var basicReferrerDesc = referrerChange.Referrer.BasicDescriptor; + if (referrerChange.ReferrerOperation == ReferrerOperation.ReferrerAdd) { - var basicDesc = referrerChange.Referrer.BasicDescriptor; - if (referrerChange.ReferrerOperation == ReferrerOperation.ReferrerAdd) + if (!updatedReferrersSet.Contains(basicReferrerDesc)) { - if (!updatedReferrersSet.Contains(basicDesc)) - { - // Add the new referrer only when it has not already existed in the updatedReferrersSet - updatedReferrers.Add(referrerChange.Referrer); - updatedReferrersSet.Add(basicDesc); - } + // Add the new referrer only when it has not already existed in the updatedReferrersSet + updatedReferrers.Add(referrerChange.Referrer); + updatedReferrersSet.Add(basicReferrerDesc); } } + // Skip unnecessary update if (!updateRequired && updatedReferrersSet.Count == oldReferrers.Count) diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index e0721e99..e8fd70b8 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -100,7 +100,7 @@ public async Task ManifestStore_PullReferrersIndexListNotFound() var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); var (receivedDesc, receivedManifests) = await store.PullReferrersIndexList("test", cancellationToken); - Assert.True(Descriptor.IsEmptyOrNull(receivedDesc)); + Assert.True(Descriptor.IsEmptyOrInvalid(receivedDesc)); Assert.Empty(receivedManifests); } From 36a428f4bd991c8e22165630edaa79e1c00fe9ad Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 10 Dec 2024 16:08:30 +1100 Subject: [PATCH 17/36] delete manifest with subject Signed-off-by: Patrick Pan --- .../Remote/HttpResponseMessageExtensions.cs | 3 +- .../Registry/Remote/ManifestStore.cs | 74 +++- .../Registry/Remote/Referrers.cs | 4 + .../Registry/Remote/Repository.cs | 51 +++ .../Registry/Remote/ResponseException.cs | 63 ++-- .../Registry/Remote/UriFactory.cs | 11 + .../Remote/ManifestStoreTest.cs | 334 ++++++++++++++++++ .../Remote/RepositoryTest.cs | 168 ++++++++- .../Remote/Util/RandomDataGenerator.cs | 2 - .../Remote/Util/Util.cs | 2 +- 10 files changed, 668 insertions(+), 44 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index af00c487..55b34e47 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -97,7 +97,7 @@ public static void VerifyContentDigest(this HttpResponseMessage response, string } if (contentDigest != expected) { - throw new HttpIOException(HttpRequestError.InvalidResponse, $"{response.RequestMessage!.Method} {response.RequestMessage.RequestUri}: invalid response; digest mismatch in Docker-Content-Digest: received {contentDigest} when expecting {digestStr}"); + throw new HttpIOException(HttpRequestError.InvalidResponse, $"{response.RequestMessage!.Method} {response.RequestMessage.RequestUri}: invalid response; digest mismatch in Docker-Content-Digest: received {contentDigest} when expecting {expected}"); } } @@ -109,6 +109,7 @@ public static void VerifyContentDigest(this HttpResponseMessage response, string /// internal static void CheckOCISubjectHeader(this HttpResponseMessage response, Repository repository) { + // response.Content.Headers.TryGetValues ?? if (response.Headers.TryGetValues("OCI-Subject", out var values)) { // Set it to ReferrerSupported when the response header contains OCI-Subject diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index d74f30fd..0708ea1d 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -385,5 +385,77 @@ public async Task TagAsync(Descriptor descriptor, string reference, Cancellation /// /// public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) - => await Repository.DeleteAsync(target, true, cancellationToken).ConfigureAwait(false); + => await DeleteWithIndexing(target, cancellationToken).ConfigureAwait(false); + + /// + /// DeleteWithIndexing deletes the specified target (Descriptor) from the repository, + /// handling referrer indexing if necessary. + /// + /// The target descriptor to delete. + /// A cancellation token to cancel the operation if needed. Defaults to default. + /// + internal async Task DeleteWithIndexing(Descriptor target, CancellationToken cancellationToken = default) + { + switch (target.MediaType) + { + case MediaType.ImageManifest: + case MediaType.ImageIndex: + if (Repository.ReferrersState == Referrers.ReferrersState.ReferrersSupported) + { + // referrers API is available, no client-side indexing needed + await Repository.DeleteAsync(target, true, cancellationToken).ConfigureAwait(false); + return; + } + var manifest = await FetchAsync(target, cancellationToken).ConfigureAwait(false); + await IndexReferrersForDelete(target, manifest, cancellationToken).ConfigureAwait(false); + break; + } + await Repository.DeleteAsync(target, true, cancellationToken).ConfigureAwait(false); // isManifest true?? + + } + /// + /// IndexReferrersForDelete indexes referrers for manifests with a subject field on manifest delete. + /// References: + /// - Latest spec: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#deleting-manifests + /// - Compatible spec: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#deleting-manifests + /// + /// + /// + /// + private async Task IndexReferrersForDelete(Descriptor target, Stream manifestContent, CancellationToken cancellationToken = default) + { + Descriptor subject; + switch (target.MediaType) + { + case MediaType.ImageManifest: + var imageManifest = JsonSerializer.Deserialize(manifestContent); + if (imageManifest?.Subject == null) + { + // no subject, no indexing needed + return; + } + subject = imageManifest.Subject; + break; + case MediaType.ImageIndex: + var imageIndex = JsonSerializer.Deserialize(manifestContent); + if (imageIndex?.Subject == null) + { + // no subject, no indexing needed + return; + } + subject = imageIndex.Subject; + break; + default: + return; + } + + var isReferrersSupported = Repository.PingReferrers(cancellationToken); + if (isReferrersSupported) + { + // referrers API is available, no client-side indexing needed + return; + } + await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(target, Referrers.ReferrerOperation.ReferrerDelete), cancellationToken) + .ConfigureAwait(false); + } } diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 0e1fbe63..58173a05 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -36,6 +36,10 @@ internal enum ReferrerOperation ReferrerDelete, } + // zeroDigest represents a digest that consists of zeros. zeroDigest is used + // for pinging Referrers API. + internal static readonly string ZeroDigest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + internal static string BuildReferrersTag(Descriptor descriptor) { var validatedDigest = Digest.Validate(descriptor.Digest); diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 613400f5..7b318737 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -65,6 +65,8 @@ internal Referrers.ReferrersState ReferrersState private RepositoryOptions _opts; + private static readonly Object _referrersPingLock = new(); + /// /// Creates a client to the remote repository identified by a reference /// Example: localhost:5000/hello-world @@ -381,4 +383,53 @@ internal Reference ParseReferenceFromContentReference(string reference) /// public async Task MountAsync(Descriptor descriptor, string fromRepository, Func>? getContent = null, CancellationToken cancellationToken = default) => await ((IMounter)Blobs).MountAsync(descriptor, fromRepository, getContent, cancellationToken).ConfigureAwait(false); + + /// + /// PingReferrers returns true if the Referrers API is available for the repository, + /// otherwise returns false + /// + /// + /// + /// + /// + internal bool PingReferrers(CancellationToken cancellationToken = default) + { + if (ReferrersState == Referrers.ReferrersState.ReferrersSupported) return true; + if (ReferrersState == Referrers.ReferrersState.ReferrersNotSupported) return false; + + lock (_referrersPingLock) + { + // referrers state is unknown + // lock to limit the rate of pinging referrers API + if (ReferrersState == Referrers.ReferrersState.ReferrersSupported) return true; + if (ReferrersState == Referrers.ReferrersState.ReferrersNotSupported) return false; + + // var reference = Options.Reference.ContentReference; // ??? + Options.Reference.ContentReference = Referrers.ZeroDigest; + var url = new UriFactory(Options.Reference, Options.PlainHttp).BuildReferrersUrl(); + var request = new HttpRequestMessage(HttpMethod.Get, url); + var response = Options.HttpClient.SendAsync(request, cancellationToken).ConfigureAwait(true).GetAwaiter().GetResult(); + // Options.Reference.ContentReference = reference; + + switch (response.StatusCode) + { + case HttpStatusCode.OK: + var supported = response.Content.Headers.ContentType?.MediaType == MediaType.ImageIndex; + SetReferrersState(supported ? Referrers.ReferrersState.ReferrersSupported : Referrers.ReferrersState.ReferrersNotSupported); + return supported; + case HttpStatusCode.NotFound: + var err = (ResponseException) response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(true).GetAwaiter().GetResult(); + if (err.Errors?.First().Code == ResponseException.ErrorCodeNameUnknown) + { + // referrer state is unknown because the repository is not found + throw err; + } + + SetReferrersState(Referrers.ReferrersState.ReferrersNotSupported); + return false; + default: + throw response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(true).GetAwaiter().GetResult(); + } + } + } } diff --git a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs b/src/OrasProject.Oras/Registry/Remote/ResponseException.cs index 05bf4e0e..72392b3d 100644 --- a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs +++ b/src/OrasProject.Oras/Registry/Remote/ResponseException.cs @@ -23,7 +23,8 @@ namespace OrasProject.Oras.Registry.Remote; public class ResponseException : HttpRequestException -{ +{ + public static readonly string ErrorCodeNameUnknown = "NAME_UNKNOWN"; public class Error { [JsonPropertyName("code")] @@ -34,45 +35,45 @@ public class Error [JsonPropertyName("detail")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public JsonElement? Detail { get; set; } + public JsonElement? Detail { get; set; } } - public class ErrorResponse - { + public class ErrorResponse + { [JsonPropertyName("errors")] - public required IList Errors { get; set; } + public required IList Errors { get; set; } } public HttpMethod? Method { get; } public Uri? RequestUri { get; } - public IList? Errors { get; } - - public ResponseException(HttpResponseMessage response, string? responseBody = null) - : this(response, responseBody, null) - { - } - - public ResponseException(HttpResponseMessage response, string? responseBody, string? message) - : this(response, responseBody, response.StatusCode == HttpStatusCode.Unauthorized ? HttpRequestError.UserAuthenticationError : HttpRequestError.Unknown, message, null) - { - } - - public ResponseException(HttpResponseMessage response, string? responseBody, HttpRequestError httpRequestError, string? message, Exception? inner) - : base(httpRequestError, message, inner, response.StatusCode) - { - var request = response.RequestMessage; - Method = request?.Method; - RequestUri = request?.RequestUri; - if (responseBody != null) - { - try - { - var errorResponse = JsonSerializer.Deserialize(responseBody); - Errors = errorResponse?.Errors; - } - catch { } + public IList? Errors { get; } + + public ResponseException(HttpResponseMessage response, string? responseBody = null) + : this(response, responseBody, null) + { + } + + public ResponseException(HttpResponseMessage response, string? responseBody, string? message) + : this(response, responseBody, response.StatusCode == HttpStatusCode.Unauthorized ? HttpRequestError.UserAuthenticationError : HttpRequestError.Unknown, message, null) + { + } + + public ResponseException(HttpResponseMessage response, string? responseBody, HttpRequestError httpRequestError, string? message, Exception? inner) + : base(httpRequestError, message, inner, response.StatusCode) + { + var request = response.RequestMessage; + Method = request?.Method; + RequestUri = request?.RequestUri; + if (responseBody != null) + { + try + { + var errorResponse = JsonSerializer.Deserialize(responseBody); + Errors = errorResponse?.Errors; + } + catch { } } } } diff --git a/src/OrasProject.Oras/Registry/Remote/UriFactory.cs b/src/OrasProject.Oras/Registry/Remote/UriFactory.cs index a4b37a45..5d2ad96d 100644 --- a/src/OrasProject.Oras/Registry/Remote/UriFactory.cs +++ b/src/OrasProject.Oras/Registry/Remote/UriFactory.cs @@ -102,6 +102,15 @@ public Uri BuildRepositoryBlobUpload() return builder.Uri; } + public Uri BuildReferrersUrl(string? artifactType = null) + { + var query = string.IsNullOrEmpty(artifactType) ? "" : $"?artifactType={artifactType}"; + var builder = NewRepositoryBaseBuilder(); + builder.Path += $"/referrers/{_reference.ContentReference}{query}"; + return builder.Uri; + } + + /// /// Generates a UriBuilder with the base endpoint of the remote repository. /// Format: :///v2/ @@ -119,4 +128,6 @@ protected UriBuilder NewRepositoryBaseBuilder() }; return builder; } + + } diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index e8fd70b8..3330bd81 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -22,6 +22,7 @@ using static OrasProject.Oras.Content.Digest; using Index = OrasProject.Oras.Oci.Index; using Xunit; +using Xunit.Abstractions; namespace OrasProject.Oras.Tests.Remote; @@ -29,6 +30,13 @@ public class ManifestStoreTest { private const string _dockerContentDigestHeader = "Docker-Content-Digest"; + private readonly ITestOutputHelper _output; + + public ManifestStoreTest(ITestOutputHelper output) + { + _output = output; + } + [Fact] public async Task ManifestStore_PullReferrersIndexListSuccessfully() { @@ -302,6 +310,7 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupported() }; var secondExpectedReferrersList = new List(oldIndex.Manifests); secondExpectedReferrersList.Add(secondExpectedManifestDesc); + var (secondExpectedIndexReferrersDesc, secondExpectedIndexReferrersBytes) = Index.GenerateIndex(secondExpectedReferrersList); byte[]? receivedManifestContent = null; @@ -343,12 +352,37 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupported() { return new HttpResponseMessage(HttpStatusCode.BadRequest); } + response.Content = new ByteArrayContent(oldIndexBytes); response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); if (oldIndexDeleted) response.Headers.Add(_dockerContentDigestHeader, new string[] { firstExpectedIndexReferrersDesc.Digest }); else response.Headers.Add(_dockerContentDigestHeader, new string[] { oldIndexDesc.Digest }); response.StatusCode = HttpStatusCode.OK; return response; + } + else if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{oldIndexDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(oldIndexBytes); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + response.Headers.Add(_dockerContentDigestHeader, new string[] { oldIndexDesc.Digest }); + response.StatusCode = HttpStatusCode.OK; + return response; + } + else if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{firstExpectedIndexReferrersDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(oldIndexBytes); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + response.Headers.Add(_dockerContentDigestHeader, new string[] { firstExpectedIndexReferrersDesc.Digest }); + response.StatusCode = HttpStatusCode.OK; + return response; } else if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{oldIndexDesc.Digest}") { response.Headers.Add(_dockerContentDigestHeader, new[] { oldIndexDesc.Digest }); @@ -552,4 +586,304 @@ public async Task ManifestStore_PushAsyncWithSubjectAndNoUpdateRequired() Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); Assert.Equal(expectedManifestBytes, receivedManifestContent); } + + + [Fact] + public async Task ManifestStore_DeleteWithSubjectWhenReferrersAPISupported() + { + var (_, manifestBytes) = RandomManifestWithSubject(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifestBytes), + Size = manifestBytes.Length + }; + var manifestDeleted = false; + var httpHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + manifestDeleted = true; + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(httpHandler), + PlainHttp = true, + }); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + repo.SetReferrersState(Referrers.ReferrersState.ReferrersSupported); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + await store.DeleteAsync(manifestDesc, cancellationToken); + Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); + Assert.True(manifestDeleted); + } + + [Fact] + public async Task ManifestStore_DeleteWithoutSubjectWhenReferrersAPIUnknown() + { + var (_, manifestBytes) = RandomManifest(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifestBytes), + Size = manifestBytes.Length + }; + var manifestDeleted = false; + var httpHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + manifestDeleted = true; + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(manifestBytes); + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(httpHandler), + PlainHttp = true, + }); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + await store.DeleteAsync(manifestDesc, cancellationToken); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + Assert.True(manifestDeleted); + } + + [Fact] + public async Task ManifestStore_DeleteWithSubjectWhenReferrersAPINotSupported() + { + // first delete image manifest + var (manifestToDelete, manifestToDeleteBytes) = RandomManifestWithSubject(); + var manifestToDeleteDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifestToDeleteBytes), + Size = manifestToDeleteBytes.Length + }; + + // then delete image index + var indexToDelete = RandomIndex(); + indexToDelete.Subject = manifestToDelete.Subject; + var indexToDeleteBytes = JsonSerializer.SerializeToUtf8Bytes(indexToDelete); + var indexToDeleteDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(indexToDeleteBytes), + Size = indexToDeleteBytes.Length + }; + + // original referrers list + var oldReferrersList = RandomIndex(); + oldReferrersList.Manifests.Add(manifestToDeleteDesc); + oldReferrersList.Manifests.Add(indexToDeleteDesc); + var oldReferrersBytes = JsonSerializer.SerializeToUtf8Bytes(oldReferrersList); + var oldReferrersDesc = new Descriptor() + { + Digest = ComputeSHA256(oldReferrersBytes), + MediaType = MediaType.ImageIndex, + Size = oldReferrersBytes.Length + }; + + // referrers list after deleting the image manifest + var firstUpdatedReferrersList = new List(oldReferrersList.Manifests); + firstUpdatedReferrersList.Remove(manifestToDeleteDesc); + var (firstUpdatedIndexReferrersDesc, firstUpdatedIndexReferrersBytes) = Index.GenerateIndex(firstUpdatedReferrersList); + + // referrers list after deleting the index manifest + var secondUpdatedReferrersList = new List(firstUpdatedReferrersList); + secondUpdatedReferrersList.Remove(indexToDeleteDesc); + var (secondUpdatedIndexReferrersDesc, secondUpdatedIndexReferrersBytes) = Index.GenerateIndex(secondUpdatedReferrersList); + + + var manifestDeleted = false; + var oldIndexDeleted = false; + var firstUpdatedIndexDeleted = false; + var imageIndexDeleted = false; + var referrersTag = Referrers.BuildReferrersTag(manifestToDelete.Subject); + byte[]? receivedIndexContent = null; + var httpHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var response = new HttpResponseMessage(); + response.RequestMessage = req; + if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get && req.Method != HttpMethod.Put) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{referrersTag}") + { + if (req.Content?.Headers?.ContentLength != null) + { + var buffer = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buffer, 0); + receivedIndexContent = buffer; + } + + if (oldIndexDeleted) + { + response.Headers.Add(_dockerContentDigestHeader, new[] { secondUpdatedIndexReferrersDesc.Digest }); + } + else + { + response.Headers.Add(_dockerContentDigestHeader, new[] { firstUpdatedIndexReferrersDesc.Digest }); + } + response.StatusCode = HttpStatusCode.Created; + return response; + } + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{referrersTag}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + if (oldIndexDeleted) + { + response.Content = new ByteArrayContent(firstUpdatedIndexReferrersBytes); + response.Headers.Add(_dockerContentDigestHeader, new string[] { firstUpdatedIndexReferrersDesc.Digest }); + } + else + { + response.Content = new ByteArrayContent(oldReferrersBytes); + response.Headers.Add(_dockerContentDigestHeader, new string[] { oldReferrersDesc.Digest }); + } + + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + response.StatusCode = HttpStatusCode.OK; + return response; + } + + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestToDeleteDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(manifestToDeleteBytes); + response.Headers.Add(_dockerContentDigestHeader, new string[] { manifestToDeleteDesc.Digest }); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); + return response; + } + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexToDeleteDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(indexToDeleteBytes); + response.Headers.Add(_dockerContentDigestHeader, new string[] { indexToDeleteDesc.Digest }); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + return response; + } + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{oldReferrersDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(oldReferrersBytes); + response.Headers.Add(_dockerContentDigestHeader, new string[] { oldReferrersDesc.Digest }); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + return response; + } + + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{firstUpdatedIndexReferrersDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(firstUpdatedIndexReferrersBytes); + response.Headers.Add(_dockerContentDigestHeader, new string[] { firstUpdatedIndexReferrersDesc.Digest }); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + return response; + } + if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{oldReferrersDesc.Digest}") + { + response.Headers.Add(_dockerContentDigestHeader, new[] { oldReferrersDesc.Digest }); + response.StatusCode = HttpStatusCode.Accepted; + oldIndexDeleted = true; + return response; + } + if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{firstUpdatedIndexReferrersDesc.Digest}") + { + response.Headers.Add(_dockerContentDigestHeader, new[] { firstUpdatedIndexReferrersDesc.Digest }); + response.StatusCode = HttpStatusCode.Accepted; + firstUpdatedIndexDeleted = true; + return response; + } + + if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestToDeleteDesc.Digest}") + { + manifestDeleted = true; + response.StatusCode = HttpStatusCode.Accepted; + return response; + } + if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexToDeleteDesc.Digest}") + { + imageIndexDeleted = true; + response.StatusCode = HttpStatusCode.Accepted; + return response; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(httpHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + // first delete the image manifest + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + await store.DeleteAsync(manifestToDeleteDesc, cancellationToken); + Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); + Assert.True(manifestDeleted); + Assert.True(oldIndexDeleted); + Assert.Equal(firstUpdatedIndexReferrersBytes, receivedIndexContent); + + // then delete the image index + Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); + await store.DeleteAsync(indexToDeleteDesc, cancellationToken); + Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); + Assert.True(imageIndexDeleted); + Assert.True(firstUpdatedIndexDeleted); + Assert.Equal(secondUpdatedIndexReferrersBytes, receivedIndexContent); + } } diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index 6aae7e15..c0a118de 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -20,13 +20,15 @@ using System.Net; using System.Net.Http.Headers; using System.Text; -using System.Text.Json; using System.Text.RegularExpressions; using System.Web; +using Newtonsoft.Json; using Xunit; +using Xunit.Abstractions; using static OrasProject.Oras.Content.Digest; using static OrasProject.Oras.Tests.Remote.Util.Util; using static OrasProject.Oras.Tests.Remote.Util.RandomDataGenerator; +using JsonSerializer = System.Text.Json.JsonSerializer; namespace OrasProject.Oras.Tests.Remote; @@ -45,6 +47,14 @@ public struct TestIOStruct private const string _theAmazingBanDigest = "b526a4f2be963a2f9b0990c001255669eab8a254ab1a6e3f84f1820212ac7078"; private const string _dockerContentDigestHeader = "Docker-Content-Digest"; + + private readonly ITestOutputHelper _output; + + public RepositoryTest(ITestOutputHelper output) + { + _output = output; + } + // The following truth table aims to cover the expected GET/HEAD request outcome // for all possible permutations of the client/server "containing a digest", for @@ -391,12 +401,12 @@ public async Task Repository_DeleteAsync() { var res = new HttpResponseMessage(); res.RequestMessage = req; - if (req.Method != HttpMethod.Delete) + if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } - if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) + if (req.Method == HttpMethod.Delete && req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) { blobDeleted = true; res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); @@ -404,13 +414,25 @@ public async Task Repository_DeleteAsync() return res; } - if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + if (req.Method == HttpMethod.Delete && req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { indexDeleted = true; // no dockerContentDigestHeader header for manifest deletion res.StatusCode = HttpStatusCode.Accepted; return res; } + + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(index); + res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + return res; + } return new HttpResponseMessage(HttpStatusCode.NotFound); }; @@ -1745,12 +1767,12 @@ public async Task ManifestStore_ExistAsync() [Fact] public async Task ManifestStore_DeleteAsync() { - var manifest = """{"layers":[]}"""u8.ToArray(); + var (_, manifestBytes) = RandomManifest(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifest), - Size = manifest.Length + Digest = ComputeSHA256(manifestBytes), + Size = manifestBytes.Length }; var manifestDeleted = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => @@ -1773,7 +1795,7 @@ public async Task ManifestStore_DeleteAsync() { return new HttpResponseMessage(HttpStatusCode.BadRequest); } - res.Content = new ByteArrayContent(manifest); + res.Content = new ByteArrayContent(manifestBytes); res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); return res; @@ -2580,4 +2602,134 @@ public void SetReferrersState_ShouldNotThrowException_WhenSettingSameValue() var exception = Record.Exception(() => repo.SetReferrersState(Referrers.ReferrersState.ReferrersSupported)); Assert.Null(exception); } + + [Fact] + public void PingReferrers_ShouldReturnTrueWhenReferrersAPISupported() + { + var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/referrers/{Referrers.ZeroDigest}") + { + res.Content.Headers.Add("Content-Type", MediaType.ImageIndex); + res.StatusCode = HttpStatusCode.OK; + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + Assert.True(repo.PingReferrers(cancellationToken)); + Assert.Equal(Referrers.ReferrersState.ReferrersSupported, repo.ReferrersState); + } + + [Fact] + public void PingReferrers_ShouldReturnFalseWhenReferrersAPINotSupportedNoContentTypeHeader() + { + var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/referrers/{Referrers.ZeroDigest}") + { + res.StatusCode = HttpStatusCode.OK; + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + Assert.False(repo.PingReferrers(cancellationToken)); + Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo.ReferrersState); + } + + [Fact] + public void PingReferrers_ShouldFailWhenReturnNotFound() + { + var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + res.StatusCode = HttpStatusCode.NotFound; + + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/referrers/{Referrers.ZeroDigest}") + { + return res; + } + + var errors = new + { + errors = new[] + { + new + { + message = "The repository could not be found.", + code = ResponseException.ErrorCodeNameUnknown + } + } + }; + res.Content = new StringContent(JsonSerializer.Serialize(errors), Encoding.UTF8, "application/json"); + return res; + }; + var cancellationToken = new CancellationToken(); + + // repo abc is not found + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/abc"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + Assert.Throws(() => repo.PingReferrers(cancellationToken)); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + + // referrer API is not supported + var repo1 = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo1.ReferrersState); + Assert.False(repo1.PingReferrers(cancellationToken)); + Assert.Equal(Referrers.ReferrersState.ReferrersNotSupported, repo1.ReferrersState); + } + + [Fact] + public void PingReferrers_ShouldFailWhenBadRequestReturns() + { + var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + return new HttpResponseMessage(HttpStatusCode.BadRequest); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + Assert.Throws(() => repo.PingReferrers(cancellationToken)); + Assert.Equal(Referrers.ReferrersState.ReferrersUnknown, repo.ReferrersState); + } } diff --git a/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs b/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs index f34df44f..df5f5d29 100644 --- a/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs +++ b/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs @@ -84,6 +84,4 @@ public static Index RandomIndex() MediaType = MediaType.ImageIndex, }; } - - } diff --git a/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs b/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs index 6da68cab..1c4a6bf8 100644 --- a/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs +++ b/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs @@ -40,7 +40,7 @@ public static HttpClient CustomClient(Func> func) { var moqHandler = new Mock(); From 181386994a9d5b763bbd8f19a624e8704e5afeb8 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 24 Dec 2024 09:24:27 +1100 Subject: [PATCH 18/36] add reference clone Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Registry/Reference.cs | 5 +++ .../Registry/Remote/ManifestStore.cs | 4 +-- .../Registry/Remote/Repository.cs | 31 +++++++++++++------ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Reference.cs b/src/OrasProject.Oras/Registry/Reference.cs index 54eca43b..f6443429 100644 --- a/src/OrasProject.Oras/Registry/Reference.cs +++ b/src/OrasProject.Oras/Registry/Reference.cs @@ -201,6 +201,11 @@ public static bool TryParse(string reference, [NotNullWhen(true)] out Reference? } } + public Reference Clone() + { + return new Reference(Registry, Repository, ContentReference); + } + public Reference(string registry) => _registry = ValidateRegistry(registry); public Reference(string registry, string? repository) : this(registry) diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 0708ea1d..d5468e74 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -410,9 +410,9 @@ internal async Task DeleteWithIndexing(Descriptor target, CancellationToken canc await IndexReferrersForDelete(target, manifest, cancellationToken).ConfigureAwait(false); break; } - await Repository.DeleteAsync(target, true, cancellationToken).ConfigureAwait(false); // isManifest true?? - + await Repository.DeleteAsync(target, true, cancellationToken).ConfigureAwait(false); } + /// /// IndexReferrersForDelete indexes referrers for manifests with a subject field on manifest delete. /// References: diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 7b318737..9193f0c0 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -394,22 +394,35 @@ public async Task MountAsync(Descriptor descriptor, string fromRepository, Func< /// internal bool PingReferrers(CancellationToken cancellationToken = default) { - if (ReferrersState == Referrers.ReferrersState.ReferrersSupported) return true; - if (ReferrersState == Referrers.ReferrersState.ReferrersNotSupported) return false; + if (ReferrersState == Referrers.ReferrersState.ReferrersSupported) + { + return true; + } + + if (ReferrersState == Referrers.ReferrersState.ReferrersNotSupported) + { + return false; + } lock (_referrersPingLock) { // referrers state is unknown // lock to limit the rate of pinging referrers API - if (ReferrersState == Referrers.ReferrersState.ReferrersSupported) return true; - if (ReferrersState == Referrers.ReferrersState.ReferrersNotSupported) return false; - - // var reference = Options.Reference.ContentReference; // ??? - Options.Reference.ContentReference = Referrers.ZeroDigest; - var url = new UriFactory(Options.Reference, Options.PlainHttp).BuildReferrersUrl(); + if (ReferrersState == Referrers.ReferrersState.ReferrersSupported) + { + return true; + } + + if (ReferrersState == Referrers.ReferrersState.ReferrersNotSupported) + { + return false; + } + + var reference = Options.Reference.Clone(); + reference.ContentReference = Referrers.ZeroDigest; + var url = new UriFactory(reference, Options.PlainHttp).BuildReferrersUrl(); var request = new HttpRequestMessage(HttpMethod.Get, url); var response = Options.HttpClient.SendAsync(request, cancellationToken).ConfigureAwait(true).GetAwaiter().GetResult(); - // Options.Reference.ContentReference = reference; switch (response.StatusCode) { From 878535cc5aafbe333cf3896bde564aa8129f38a8 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Fri, 27 Dec 2024 09:57:50 +1100 Subject: [PATCH 19/36] change variable naming Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Registry/Remote/ManifestStore.cs | 6 +++--- src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index d5468e74..78b8d9e0 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -273,7 +273,7 @@ private async Task UpdateReferrersIndex(Descriptor subject, if (!updateRequired) return; // 3. push the updated referrers list using referrers tag schema - if (updatedReferrers.Count > 0 || repository.Options.SkipReferrersGC) + if (updatedReferrers.Count > 0 || repository.Options.SkipReferrersGc) { // push a new index in either case: // 1. the referrers list has been updated with a non-zero size @@ -287,9 +287,9 @@ private async Task UpdateReferrersIndex(Descriptor subject, } } - if (repository.Options.SkipReferrersGC || Descriptor.IsEmptyOrInvalid(oldDesc)) + if (repository.Options.SkipReferrersGc || Descriptor.IsEmptyOrInvalid(oldDesc)) { - // Skip the delete process if SkipReferrersGC is set to true or the old Descriptor is empty or null + // Skip the delete process if SkipReferrersGc is set to true or the old Descriptor is empty or null return; } diff --git a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs index 4742f9d8..77f82596 100644 --- a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs +++ b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs @@ -51,7 +51,7 @@ public struct RepositoryOptions /// public int TagListPageSize { get; set; } - // SkipReferrersGC specifies whether to delete the dangling referrers + // SkipReferrersGc specifies whether to delete the dangling referrers // index when referrers tag schema is utilized. // - If false, the old referrers index will be deleted after the new one is successfully uploaded. // - If true, the old referrers index is kept. @@ -59,5 +59,5 @@ public struct RepositoryOptions // - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#referrers-tag-schema // - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#pushing-manifests-with-subject // - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#deleting-manifests - public bool SkipReferrersGC { get; set; } + public bool SkipReferrersGc { get; set; } } From c848ed2e98fe141127c8e0f654ea348b6707eb4e Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 7 Jan 2025 10:03:43 +1100 Subject: [PATCH 20/36] polish pr Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Oci/Descriptor.cs | 2 +- .../Registry/Remote/HttpResponseMessageExtensions.cs | 2 +- src/OrasProject.Oras/Registry/Remote/ManifestStore.cs | 3 --- src/OrasProject.Oras/Registry/Remote/Referrers.cs | 3 --- src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs | 1 - src/OrasProject.Oras/Registry/Remote/UriFactory.cs | 2 -- tests/OrasProject.Oras.Tests/Remote/Util/Util.cs | 4 ++-- 7 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/OrasProject.Oras/Oci/Descriptor.cs b/src/OrasProject.Oras/Oci/Descriptor.cs index 73bb3cf6..f7d738b1 100644 --- a/src/OrasProject.Oras/Oci/Descriptor.cs +++ b/src/OrasProject.Oras/Oci/Descriptor.cs @@ -68,7 +68,7 @@ public static Descriptor Create(Span data, string mediaType) }; internal BasicDescriptor BasicDescriptor => new BasicDescriptor(MediaType, Digest, Size); - + internal static bool IsNullOrInvalid(Descriptor? descriptor) { return descriptor == null || string.IsNullOrEmpty(descriptor.Digest) || string.IsNullOrEmpty(descriptor.MediaType); diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index ee589d4f..7c036b7a 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -26,7 +26,7 @@ namespace OrasProject.Oras.Registry.Remote; internal static class HttpResponseMessageExtensions { private const string _dockerContentDigestHeader = "Docker-Content-Digest"; - + /// /// Parses the error returned by the remote registry. /// diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index a11d9850..14e04a4e 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -178,7 +178,6 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, Re { case MediaType.ImageManifest: case MediaType.ImageIndex: - if (Repository.ReferrersState == Referrers.ReferrersState.Supported) { // Push the manifest straightaway when the registry supports referrers API @@ -192,7 +191,6 @@ private async Task PushWithIndexingAsync(Descriptor expected, Stream content, Re // Push the manifest when ReferrerState is Unknown or NotSupported await DoPushAsync(expected, contentDuplicate, reference, cancellationToken).ConfigureAwait(false); } - if (Repository.ReferrersState == Referrers.ReferrersState.Supported) { // Early exit when the registry supports Referrers API @@ -334,7 +332,6 @@ private async Task UpdateReferrersIndex(Descriptor subject, } catch (NotFoundException) { - return (null, ImmutableArray.Empty); } } diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 5163dbbb..139e76b7 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -17,7 +17,6 @@ namespace OrasProject.Oras.Registry.Remote; - internal static class Referrers { internal enum ReferrersState @@ -65,7 +64,6 @@ internal static (IList, bool) ApplyReferrerChanges(IList var updateRequired = false; foreach (var oldReferrer in oldReferrers) { - if (Descriptor.IsNullOrInvalid(oldReferrer)) { // Skip any empty or null referrers @@ -101,7 +99,6 @@ internal static (IList, bool) ApplyReferrerChanges(IList // Add the new referrer only when it has not already existed in the updatedReferrersSet updatedReferrers.Add(referrerChange.Referrer); updatedReferrersSet.Add(basicReferrerDesc); - updateRequired = true; } } diff --git a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs index b9b1752d..63e9a5bd 100644 --- a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs +++ b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs @@ -51,7 +51,6 @@ public struct RepositoryOptions /// public int TagListPageSize { get; set; } - /// /// SkipReferrersGc specifies whether to delete the dangling referrers /// index when referrers tag schema is utilized. diff --git a/src/OrasProject.Oras/Registry/Remote/UriFactory.cs b/src/OrasProject.Oras/Registry/Remote/UriFactory.cs index 5d2ad96d..d4b552df 100644 --- a/src/OrasProject.Oras/Registry/Remote/UriFactory.cs +++ b/src/OrasProject.Oras/Registry/Remote/UriFactory.cs @@ -128,6 +128,4 @@ protected UriBuilder NewRepositoryBaseBuilder() }; return builder; } - - } diff --git a/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs b/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs index db7f2013..4d24bf88 100644 --- a/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs +++ b/tests/OrasProject.Oras.Tests/Remote/Util/Util.cs @@ -40,7 +40,7 @@ public static HttpClient CustomClient(Func> func) { var moqHandler = new Mock(); @@ -51,7 +51,7 @@ public static HttpClient CustomClient(Func new() { MediaType = "", From a7460f35a5a775cb37aebad545f96b08f69def71 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Tue, 7 Jan 2025 11:23:28 +1100 Subject: [PATCH 21/36] fix responseException style Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Registry/Remote/ResponseException.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs b/src/OrasProject.Oras/Registry/Remote/ResponseException.cs index 72392b3d..0123e9c0 100644 --- a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs +++ b/src/OrasProject.Oras/Registry/Remote/ResponseException.cs @@ -15,9 +15,7 @@ using System.Collections.Generic; using System.Net; using System.Net.Http; -using System.Text; using System.Text.Json; -using System.Text.Json.Nodes; using System.Text.Json.Serialization; namespace OrasProject.Oras.Registry.Remote; From d7eb9b5146039b69ed88a4728dba5f403b708ca2 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Mon, 13 Jan 2025 15:22:53 +1100 Subject: [PATCH 22/36] add unit tests Signed-off-by: Patrick Pan --- .../Registry/Remote/ManifestStore.cs | 2 +- .../Registry/Remote/UriFactory.cs | 18 +++++-- .../Remote/RepositoryTest.cs | 8 --- .../Remote/UriFactoryTest.cs | 50 +++++++++++++++++++ .../Remote/Util/RandomDataGenerator.cs | 2 + 5 files changed, 68 insertions(+), 12 deletions(-) create mode 100644 tests/OrasProject.Oras.Tests/Remote/UriFactoryTest.cs diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 14e04a4e..2807b1cb 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -15,7 +15,6 @@ using OrasProject.Oras.Oci; using System; using System.Collections.Generic; -using OrasProject.Oras.Content; using System.Collections.Immutable; using System.IO; using System.Net; @@ -23,6 +22,7 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using OrasProject.Oras.Content; using Index = OrasProject.Oras.Oci.Index; namespace OrasProject.Oras.Registry.Remote; diff --git a/src/OrasProject.Oras/Registry/Remote/UriFactory.cs b/src/OrasProject.Oras/Registry/Remote/UriFactory.cs index d4b552df..88c678e9 100644 --- a/src/OrasProject.Oras/Registry/Remote/UriFactory.cs +++ b/src/OrasProject.Oras/Registry/Remote/UriFactory.cs @@ -13,6 +13,7 @@ using OrasProject.Oras.Exceptions; using System; +using System.Web; namespace OrasProject.Oras.Registry.Remote; @@ -102,15 +103,26 @@ public Uri BuildRepositoryBlobUpload() return builder.Uri; } + /// + /// Builds the URL for accessing the Referrers API + /// Format: :///v2//referrers/?artifactType= + /// + /// + /// public Uri BuildReferrersUrl(string? artifactType = null) { - var query = string.IsNullOrEmpty(artifactType) ? "" : $"?artifactType={artifactType}"; var builder = NewRepositoryBaseBuilder(); - builder.Path += $"/referrers/{_reference.ContentReference}{query}"; + builder.Path += $"/referrers/{_reference.ContentReference}"; + if (!string.IsNullOrEmpty(artifactType)) + { + var query = HttpUtility.ParseQueryString(builder.Query); + query.Add("artifactType", artifactType); + builder.Query = query.ToString(); + } + return builder.Uri; } - /// /// Generates a UriBuilder with the base endpoint of the remote repository. /// Format: :///v2/ diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index 66acdb1d..ebf3dc81 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -46,14 +46,6 @@ public struct TestIOStruct private const string _theAmazingBanDigest = "b526a4f2be963a2f9b0990c001255669eab8a254ab1a6e3f84f1820212ac7078"; private const string _dockerContentDigestHeader = "Docker-Content-Digest"; - - private readonly ITestOutputHelper _output; - - public RepositoryTest(ITestOutputHelper output) - { - _output = output; - } - // The following truth table aims to cover the expected GET/HEAD request outcome // for all possible permutations of the client/server "containing a digest", for diff --git a/tests/OrasProject.Oras.Tests/Remote/UriFactoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/UriFactoryTest.cs new file mode 100644 index 00000000..1fc7dd3b --- /dev/null +++ b/tests/OrasProject.Oras.Tests/Remote/UriFactoryTest.cs @@ -0,0 +1,50 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using OrasProject.Oras.Registry; +using OrasProject.Oras.Registry.Remote; +using Xunit; +using static OrasProject.Oras.Tests.Remote.Util.RandomDataGenerator; + +namespace OrasProject.Oras.Tests.Remote; + +public class UriFactoryTest +{ + [Fact] + public void BuildReferrersUrl_WithArtifactType_ShouldAddArtifactTypeToQueryString() + { + var desc = RandomDescriptor(); + + var reference = Reference.Parse("localhost:5000/test"); + reference.ContentReference = desc.Digest; + + const string artifactType = "doc/example"; + var expectedPath = $"referrers/{reference.ContentReference}"; + const string expectedQuery = "artifactType=doc%2fexample"; + var result = new UriFactory(reference).BuildReferrersUrl(artifactType); + Assert.Equal($"https://localhost:5000/v2/test/{expectedPath}?{expectedQuery}", result.ToString()); + } + + [Fact] + public void BuildReferrersUrl_WithoutArtifactType() + { + var desc = RandomDescriptor(); + var reference = Reference.Parse("localhost:5000/test"); + reference.ContentReference = desc.Digest; + + + var expectedPath = $"referrers/{reference.ContentReference}"; + var result = new UriFactory(reference).BuildReferrersUrl(); + Assert.Equal($"https://localhost:5000/v2/test/{expectedPath}", result.ToString()); + } +} diff --git a/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs b/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs index df5f5d29..f34df44f 100644 --- a/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs +++ b/tests/OrasProject.Oras.Tests/Remote/Util/RandomDataGenerator.cs @@ -84,4 +84,6 @@ public static Index RandomIndex() MediaType = MediaType.ImageIndex, }; } + + } From dfc538dec8d9cbeae38447a22268aee7fe86769e Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Mon, 13 Jan 2025 15:25:15 +1100 Subject: [PATCH 23/36] remove unnecessary changes Signed-off-by: Patrick Pan --- tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs | 1 + tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs index e9fcbfb9..d2c8088c 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -262,6 +262,7 @@ public void ApplyReferrerChanges_NoUpdateWhenOldAndNewReferrersAreEmpty() { var oldReferrers = new List(); var referrerChange = new Referrers.ReferrerChange(ZeroDescriptor(), Referrers.ReferrerOperation.Add); + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); Assert.Empty(updatedReferrers); Assert.False(updateRequired); diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index ebf3dc81..19b9bd03 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -23,7 +23,6 @@ using System.Text.RegularExpressions; using System.Web; using Xunit; -using Xunit.Abstractions; using static OrasProject.Oras.Content.Digest; using static OrasProject.Oras.Tests.Remote.Util.Util; using static OrasProject.Oras.Tests.Remote.Util.RandomDataGenerator; From ff334ca20109bce5aebd3e1101fa11f229d61def Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Mon, 13 Jan 2025 15:29:53 +1100 Subject: [PATCH 24/36] remove unnecessary changes Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Registry/Remote/ResponseException.cs | 4 +++- tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs b/src/OrasProject.Oras/Registry/Remote/ResponseException.cs index 0123e9c0..a447af1c 100644 --- a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs +++ b/src/OrasProject.Oras/Registry/Remote/ResponseException.cs @@ -15,7 +15,9 @@ using System.Collections.Generic; using System.Net; using System.Net.Http; +using System.Text; using System.Text.Json; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; namespace OrasProject.Oras.Registry.Remote; @@ -74,4 +76,4 @@ public ResponseException(HttpResponseMessage response, string? responseBody, Htt catch { } } } -} +} \ No newline at end of file diff --git a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs index d2c8088c..ed01505a 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ReferrersTest.cs @@ -262,7 +262,7 @@ public void ApplyReferrerChanges_NoUpdateWhenOldAndNewReferrersAreEmpty() { var oldReferrers = new List(); var referrerChange = new Referrers.ReferrerChange(ZeroDescriptor(), Referrers.ReferrerOperation.Add); - + var (updatedReferrers, updateRequired) = Referrers.ApplyReferrerChanges(oldReferrers, referrerChange); Assert.Empty(updatedReferrers); Assert.False(updateRequired); From 53f8e12abbb20020266690eee0256bde8f739f3f Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Mon, 13 Jan 2025 15:31:54 +1100 Subject: [PATCH 25/36] move ResponseException to Exceptions folder Signed-off-by: Patrick Pan --- .../{Registry/Remote => Exceptions}/ResponseException.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) rename src/OrasProject.Oras/{Registry/Remote => Exceptions}/ResponseException.cs (98%) diff --git a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs b/src/OrasProject.Oras/Exceptions/ResponseException.cs similarity index 98% rename from src/OrasProject.Oras/Registry/Remote/ResponseException.cs rename to src/OrasProject.Oras/Exceptions/ResponseException.cs index a447af1c..0123e9c0 100644 --- a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs +++ b/src/OrasProject.Oras/Exceptions/ResponseException.cs @@ -15,9 +15,7 @@ using System.Collections.Generic; using System.Net; using System.Net.Http; -using System.Text; using System.Text.Json; -using System.Text.Json.Nodes; using System.Text.Json.Serialization; namespace OrasProject.Oras.Registry.Remote; @@ -76,4 +74,4 @@ public ResponseException(HttpResponseMessage response, string? responseBody, Htt catch { } } } -} \ No newline at end of file +} From b13de614535d2f3d5e66eba855c816dc1ab0a6cb Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Wed, 15 Jan 2025 14:50:49 +1100 Subject: [PATCH 26/36] resolve comments Signed-off-by: Patrick Pan --- .../Registry/Remote/ManifestStore.cs | 1 - src/OrasProject.Oras/Registry/Remote/Referrers.cs | 2 +- src/OrasProject.Oras/Registry/Remote/Repository.cs | 12 +++++------- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 2807b1cb..91ea4e67 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -429,7 +429,6 @@ internal async Task DeleteWithIndexing(Descriptor target, CancellationToken canc /// IndexReferrersForDelete indexes referrers for manifests with a subject field on manifest delete. /// References: /// - Latest spec: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#deleting-manifests - /// - Compatible spec: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#deleting-manifests /// /// /// diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 139e76b7..951650e4 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -28,7 +28,7 @@ internal enum ReferrersState internal record ReferrerChange(Descriptor Referrer, ReferrerOperation ReferrerOperation); - internal static readonly string ZeroDigest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + internal const string ZeroDigest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; internal enum ReferrerOperation { diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index b2b6c2f0..c89ab442 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -381,14 +381,12 @@ public async Task MountAsync(Descriptor descriptor, string fromRepository, Func< /// internal bool PingReferrers(CancellationToken cancellationToken = default) { - if (ReferrersState == Referrers.ReferrersState.Supported) + switch (ReferrersState) { - return true; - } - - if (ReferrersState == Referrers.ReferrersState.NotSupported) - { - return false; + case Referrers.ReferrersState.Supported: + return true; + case Referrers.ReferrersState.NotSupported: + return false; } lock (_referrersPingLock) From 21c714a5834a29f91c53f577977958d3fe449c1c Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Wed, 5 Feb 2025 16:58:15 +1100 Subject: [PATCH 27/36] move response exception back to remote Signed-off-by: Patrick Pan --- .../{Exceptions => Registry/Remote}/ResponseException.cs | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/OrasProject.Oras/{Exceptions => Registry/Remote}/ResponseException.cs (100%) diff --git a/src/OrasProject.Oras/Exceptions/ResponseException.cs b/src/OrasProject.Oras/Registry/Remote/ResponseException.cs similarity index 100% rename from src/OrasProject.Oras/Exceptions/ResponseException.cs rename to src/OrasProject.Oras/Registry/Remote/ResponseException.cs From a23d7ebce971ce5b02494397e2ff5f850ee23e02 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Wed, 5 Feb 2025 17:27:32 +1100 Subject: [PATCH 28/36] change to copy constructor Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Registry/Reference.cs | 11 +++++++++-- src/OrasProject.Oras/Registry/Remote/ManifestStore.cs | 2 +- src/OrasProject.Oras/Registry/Remote/Repository.cs | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Reference.cs b/src/OrasProject.Oras/Registry/Reference.cs index f6443429..f52ca3a0 100644 --- a/src/OrasProject.Oras/Registry/Reference.cs +++ b/src/OrasProject.Oras/Registry/Reference.cs @@ -201,9 +201,16 @@ public static bool TryParse(string reference, [NotNullWhen(true)] out Reference? } } - public Reference Clone() + public Reference(Reference other) { - return new Reference(Registry, Repository, ContentReference); + if (other == null) + { + throw new ArgumentNullException(nameof(other)); + } + + _registry = other.Registry; + _repository = other.Repository; + ContentReference = other.ContentReference; } public Reference(string registry) => _registry = ValidateRegistry(registry); diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 91ea4e67..d1530077 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -406,7 +406,7 @@ public async Task DeleteAsync(Descriptor target, CancellationToken cancellationT /// The target descriptor to delete. /// A cancellation token to cancel the operation if needed. Defaults to default. /// - internal async Task DeleteWithIndexing(Descriptor target, CancellationToken cancellationToken = default) + private async Task DeleteWithIndexing(Descriptor target, CancellationToken cancellationToken = default) { switch (target.MediaType) { diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index c89ab442..d7a5a966 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -403,7 +403,7 @@ internal bool PingReferrers(CancellationToken cancellationToken = default) return false; } - var reference = Options.Reference.Clone(); + var reference = new Reference(Options.Reference); reference.ContentReference = Referrers.ZeroDigest; var url = new UriFactory(reference, Options.PlainHttp).BuildReferrersUrl(); var request = new HttpRequestMessage(HttpMethod.Get, url); From d5c38b9394e10ed34b25d8adfa80a5803096bf2d Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Thu, 6 Feb 2025 13:26:27 +1100 Subject: [PATCH 29/36] dispose resource Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Registry/Remote/ManifestStore.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index d1530077..7c1430bd 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -418,8 +418,11 @@ private async Task DeleteWithIndexing(Descriptor target, CancellationToken cance await Repository.DeleteAsync(target, true, cancellationToken).ConfigureAwait(false); return; } - var manifest = await FetchAsync(target, cancellationToken).ConfigureAwait(false); - await IndexReferrersForDelete(target, manifest, cancellationToken).ConfigureAwait(false); + + await using (var manifest = await FetchAsync(target, cancellationToken).ConfigureAwait(false)) + { + await IndexReferrersForDelete(target, manifest, cancellationToken).ConfigureAwait(false); + } break; } await Repository.DeleteAsync(target, true, cancellationToken).ConfigureAwait(false); From 3b78217760baedfc0757f0eff85a8bf07ae53a57 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Mon, 10 Feb 2025 11:17:18 +1100 Subject: [PATCH 30/36] switch to use semaphore Signed-off-by: Patrick Pan --- .../Registry/Remote/ManifestStore.cs | 2 +- .../Registry/Remote/Repository.cs | 23 +++-- .../Remote/RepositoryTest.cs | 89 +++++++++++++++++-- 3 files changed, 95 insertions(+), 19 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 7c1430bd..9df22714 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -463,7 +463,7 @@ private async Task IndexReferrersForDelete(Descriptor target, Stream manifestCon return; } - var isReferrersSupported = Repository.PingReferrers(cancellationToken); + var isReferrersSupported = await Repository.PingReferrers(cancellationToken).ConfigureAwait(false); if (isReferrersSupported) { // referrers API is available, no client-side indexing needed diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index d7a5a966..ed43cb93 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -77,6 +77,8 @@ internal Referrers.ReferrersState ReferrersState private RepositoryOptions _opts; private static readonly Object _referrersPingLock = new(); + + private readonly SemaphoreSlim _referrersPingSemaphore = new SemaphoreSlim(1, 1); /// /// Creates a client to the remote repository identified by a reference @@ -379,7 +381,7 @@ public async Task MountAsync(Descriptor descriptor, string fromRepository, Func< /// /// /// - internal bool PingReferrers(CancellationToken cancellationToken = default) + internal async Task PingReferrers(CancellationToken cancellationToken = default) { switch (ReferrersState) { @@ -388,8 +390,9 @@ internal bool PingReferrers(CancellationToken cancellationToken = default) case Referrers.ReferrersState.NotSupported: return false; } - - lock (_referrersPingLock) + + await _referrersPingSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try { // referrers state is unknown // lock to limit the rate of pinging referrers API @@ -407,8 +410,7 @@ internal bool PingReferrers(CancellationToken cancellationToken = default) reference.ContentReference = Referrers.ZeroDigest; var url = new UriFactory(reference, Options.PlainHttp).BuildReferrersUrl(); var request = new HttpRequestMessage(HttpMethod.Get, url); - var response = Options.HttpClient.SendAsync(request, cancellationToken).ConfigureAwait(true).GetAwaiter() - .GetResult(); + var response = await Options.HttpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); switch (response.StatusCode) { @@ -417,8 +419,8 @@ internal bool PingReferrers(CancellationToken cancellationToken = default) SetReferrersState(supported); return supported; case HttpStatusCode.NotFound: - var err = (ResponseException)response.ParseErrorResponseAsync(cancellationToken) - .ConfigureAwait(true).GetAwaiter().GetResult(); + var err = (ResponseException) await response.ParseErrorResponseAsync(cancellationToken) + .ConfigureAwait(false); if (err.Errors?.First().Code == ResponseException.ErrorCodeNameUnknown) { // referrer state is unknown because the repository is not found @@ -428,10 +430,13 @@ internal bool PingReferrers(CancellationToken cancellationToken = default) SetReferrersState(false); return false; default: - throw response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(true).GetAwaiter() - .GetResult(); + throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); } } + finally + { + _referrersPingSemaphore.Release(); + } } /// diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index 19b9bd03..7c676a5b 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -2594,7 +2594,7 @@ public void SetReferrersState_ShouldNotThrowException_WhenSettingSameValue() } [Fact] - public void PingReferrers_ShouldReturnTrueWhenReferrersAPISupported() + public async Task PingReferrers_ShouldReturnTrueWhenReferrersAPISupported() { var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => { @@ -2617,12 +2617,81 @@ public void PingReferrers_ShouldReturnTrueWhenReferrersAPISupported() }); var cancellationToken = new CancellationToken(); Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); - Assert.True(repo.PingReferrers(cancellationToken)); + var result = await repo.PingReferrers(cancellationToken); + Assert.True(result); Assert.Equal(Referrers.ReferrersState.Supported, repo.ReferrersState); } [Fact] - public void PingReferrers_ShouldReturnFalseWhenReferrersAPINotSupportedNoContentTypeHeader() + public async Task PingReferrers_WaitsForSemaphoreRelease() + { + var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/referrers/{Referrers.ZeroDigest}") + { + res.Content.Headers.Add("Content-Type", MediaType.ImageIndex); + res.StatusCode = HttpStatusCode.OK; + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + var ping1 = repo.PingReferrers(cancellationToken); + await Task.Delay(50, cancellationToken); + var ping2 = repo.PingReferrers(cancellationToken); + Assert.True(ping1.IsCompletedSuccessfully); + Assert.True(ping2.IsCompletedSuccessfully); + Assert.Equal(Referrers.ReferrersState.Supported, repo.ReferrersState); + } + + [Fact] + public async Task PingReferrers_LimitsConcurrency() + { + var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/referrers/{Referrers.ZeroDigest}") + { + res.Content.Headers.Add("Content-Type", MediaType.ImageIndex); + res.StatusCode = HttpStatusCode.OK; + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + + var tasks = new List>(); + for (int i = 0; i < 5; ++i) + { + tasks.Add(repo.PingReferrers(cancellationToken)); + } + + var results = await Task.WhenAll(tasks); + Assert.All(results, result => Assert.True(result)); + Assert.Equal(Referrers.ReferrersState.Supported, repo.ReferrersState); + } + + [Fact] + public async Task PingReferrers_ShouldReturnFalseWhenReferrersAPINotSupportedNoContentTypeHeader() { var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => { @@ -2644,12 +2713,13 @@ public void PingReferrers_ShouldReturnFalseWhenReferrersAPINotSupportedNoContent }); var cancellationToken = new CancellationToken(); Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); - Assert.False(repo.PingReferrers(cancellationToken)); + var result = await repo.PingReferrers(cancellationToken); + Assert.False(result); Assert.Equal(Referrers.ReferrersState.NotSupported, repo.ReferrersState); } [Fact] - public void PingReferrers_ShouldFailWhenReturnNotFound() + public async Task PingReferrers_ShouldFailWhenReturnNotFound() { var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => { @@ -2686,7 +2756,7 @@ public void PingReferrers_ShouldFailWhenReturnNotFound() PlainHttp = true, }); Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); - Assert.Throws(() => repo.PingReferrers(cancellationToken)); + await Assert.ThrowsAsync(async () => await repo.PingReferrers(cancellationToken)); Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); // referrer API is not supported @@ -2697,12 +2767,13 @@ public void PingReferrers_ShouldFailWhenReturnNotFound() PlainHttp = true, }); Assert.Equal(Referrers.ReferrersState.Unknown, repo1.ReferrersState); - Assert.False(repo1.PingReferrers(cancellationToken)); + var result = await repo1.PingReferrers(cancellationToken); + Assert.False(result); Assert.Equal(Referrers.ReferrersState.NotSupported, repo1.ReferrersState); } [Fact] - public void PingReferrers_ShouldFailWhenBadRequestReturns() + public async Task PingReferrers_ShouldFailWhenBadRequestReturns() { var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => { @@ -2719,7 +2790,7 @@ public void PingReferrers_ShouldFailWhenBadRequestReturns() }); var cancellationToken = new CancellationToken(); Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); - Assert.Throws(() => repo.PingReferrers(cancellationToken)); + await Assert.ThrowsAsync(async () => await repo.PingReferrers(cancellationToken)); Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); } } From fc2c590d8fc0402963d784c415b4453c91d59d64 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Mon, 10 Feb 2025 11:53:22 +1100 Subject: [PATCH 31/36] change to enum Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Registry/Remote/Repository.cs | 2 +- src/OrasProject.Oras/Registry/Remote/ResponseException.cs | 5 ++++- tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index ed43cb93..ba0dfc36 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -421,7 +421,7 @@ internal async Task PingReferrers(CancellationToken cancellationToken = de case HttpStatusCode.NotFound: var err = (ResponseException) await response.ParseErrorResponseAsync(cancellationToken) .ConfigureAwait(false); - if (err.Errors?.First().Code == ResponseException.ErrorCodeNameUnknown) + if (err.Errors?.First().Code == nameof(ResponseException.ErrorCode.NAME_UNKNOWN)) { // referrer state is unknown because the repository is not found throw err; diff --git a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs b/src/OrasProject.Oras/Registry/Remote/ResponseException.cs index 0123e9c0..18aed7f4 100644 --- a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs +++ b/src/OrasProject.Oras/Registry/Remote/ResponseException.cs @@ -22,7 +22,10 @@ namespace OrasProject.Oras.Registry.Remote; public class ResponseException : HttpRequestException { - public static readonly string ErrorCodeNameUnknown = "NAME_UNKNOWN"; + public enum ErrorCode + { + NAME_UNKNOWN + } public class Error { [JsonPropertyName("code")] diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index 7c676a5b..260eda3c 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -2739,7 +2739,7 @@ public async Task PingReferrers_ShouldFailWhenReturnNotFound() new { message = "The repository could not be found.", - code = ResponseException.ErrorCodeNameUnknown + code = nameof(ResponseException.ErrorCode.NAME_UNKNOWN) } } }; From e7ae40411529414fd2bd1187f8574e5ed41185b0 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Wed, 19 Feb 2025 13:21:55 +1100 Subject: [PATCH 32/36] add maxmetadatabytes and address commments Signed-off-by: Patrick Pan --- .../Exceptions/SizeLimitExceededException.cs | 33 +++++++++++++++++ .../Registry/Remote/ManifestStore.cs | 1 + .../Registry/Remote/Repository.cs | 33 +++++++++++------ .../Registry/Remote/RepositoryOptions.cs | 21 +++++++++++ .../Registry/Remote/ResponseException.cs | 2 +- .../Exceptions/ExceptionTest.cs | 8 +++++ .../Remote/ManifestStoreTest.cs | 36 +++++++++++++++++++ .../Remote/RepositoryTest.cs | 22 ++++++++++++ 8 files changed, 144 insertions(+), 12 deletions(-) create mode 100644 src/OrasProject.Oras/Exceptions/SizeLimitExceededException.cs diff --git a/src/OrasProject.Oras/Exceptions/SizeLimitExceededException.cs b/src/OrasProject.Oras/Exceptions/SizeLimitExceededException.cs new file mode 100644 index 00000000..c434e454 --- /dev/null +++ b/src/OrasProject.Oras/Exceptions/SizeLimitExceededException.cs @@ -0,0 +1,33 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; + +namespace OrasProject.Oras.Exceptions; + +public class SizeLimitExceededException : Exception +{ + public SizeLimitExceededException() + { + } + + public SizeLimitExceededException(string? message) + : base(message) + { + } + + public SizeLimitExceededException(string? message, Exception? inner) + : base(message, inner) + { + } +} diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 9df22714..de095e86 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -419,6 +419,7 @@ private async Task DeleteWithIndexing(Descriptor target, CancellationToken cance return; } + Repository.LimitSize(target, Repository.Options.MaxMetadataBytes); await using (var manifest = await FetchAsync(target, cancellationToken).ConfigureAwait(false)) { await IndexReferrersForDelete(target, manifest, cancellationToken).ConfigureAwait(false); diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index ba0dfc36..64f17504 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -75,8 +75,6 @@ internal Referrers.ReferrersState ReferrersState ]; private RepositoryOptions _opts; - - private static readonly Object _referrersPingLock = new(); private readonly SemaphoreSlim _referrersPingSemaphore = new SemaphoreSlim(1, 1); @@ -394,17 +392,15 @@ internal async Task PingReferrers(CancellationToken cancellationToken = de await _referrersPingSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); try { - // referrers state is unknown - // lock to limit the rate of pinging referrers API - if (ReferrersState == Referrers.ReferrersState.Supported) - { - return true; - } - - if (ReferrersState == Referrers.ReferrersState.NotSupported) + switch (ReferrersState) { - return false; + case Referrers.ReferrersState.Supported: + return true; + case Referrers.ReferrersState.NotSupported: + return false; } + // referrers state is unknown + // lock to limit the rate of pinging referrers API var reference = new Reference(Options.Reference); reference.ContentReference = Referrers.ZeroDigest; @@ -455,4 +451,19 @@ public void SetReferrersState(bool isSupported) { ReferrersState = isSupported ? Referrers.ReferrersState.Supported : Referrers.ReferrersState.NotSupported; } + + + /// + /// LimitSize throws SizeLimitExceededException if the size of desc exceeds the limit limitSize. + /// If limitSize is less than or equal to zero, _defaultMaxMetadataBytes is used. + /// + /// + /// + /// + internal static void LimitSize(Descriptor desc, long limitSize) { + if (desc.Size > limitSize) + { + throw new SizeLimitExceededException($"content size {desc.Size} exceeds MaxMetadataBytes {limitSize}"); + } + } } diff --git a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs index 63e9a5bd..c6efdcf8 100644 --- a/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs +++ b/src/OrasProject.Oras/Registry/Remote/RepositoryOptions.cs @@ -62,4 +62,25 @@ public struct RepositoryOptions /// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#deleting-manifests /// public bool SkipReferrersGc { get; set; } + + /// + /// MaxMetadataBytes specifies a limit on how many response bytes are allowed + /// in the server's response to the metadata APIs, such as catalog list, tag + /// list, and referrers list. + /// If less than or equal to zero, a default (currently 4MiB) is used. + /// + public long MaxMetadataBytes + { + get => _maxMetadataBytes > 0 ? _maxMetadataBytes : _defaultMaxMetadataBytes; + set => _maxMetadataBytes = value; + } + + private long _maxMetadataBytes; + + /// + /// _defaultMaxMetadataBytes specifies the default limit on how many response + /// bytes are allowed in the server's response to the metadata APIs. + /// See also: Repository.MaxMetadataBytes + /// + private const long _defaultMaxMetadataBytes = 4 * 1024 * 1024; // 4 MiB } diff --git a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs b/src/OrasProject.Oras/Registry/Remote/ResponseException.cs index 18aed7f4..b4a7ae4b 100644 --- a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs +++ b/src/OrasProject.Oras/Registry/Remote/ResponseException.cs @@ -20,7 +20,7 @@ namespace OrasProject.Oras.Registry.Remote; -public class ResponseException : HttpRequestException +internal class ResponseException : HttpRequestException { public enum ErrorCode { diff --git a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs index d9fb8a5c..c00ce67d 100644 --- a/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs +++ b/tests/OrasProject.Oras.Tests/Exceptions/ExceptionTest.cs @@ -57,4 +57,12 @@ public async Task ReferrersSupportLevelAlreadySetException() await Assert.ThrowsAsync(() => throw new ReferrersStateAlreadySetException("Referrers state has already been set")); await Assert.ThrowsAsync(() => throw new ReferrersStateAlreadySetException("Referrers state has already been set", null)); } + + [Fact] + public async Task SizeLimitExceededException() + { + await Assert.ThrowsAsync(() => throw new SizeLimitExceededException()); + await Assert.ThrowsAsync(() => throw new SizeLimitExceededException("Size limit exceeded")); + await Assert.ThrowsAsync(() => throw new SizeLimitExceededException("Size limit exceeded", null)); + } } diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index 05dc6e94..f7559d0f 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -14,6 +14,7 @@ using System.Net; using System.Text; using System.Text.Json; +using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; using OrasProject.Oras.Registry; using OrasProject.Oras.Registry.Remote; @@ -676,6 +677,41 @@ public async Task ManifestStore_DeleteWithoutSubjectWhenReferrersAPIUnknown() Assert.True(manifestDeleted); } + [Fact] + public async Task ManifestStore_DeleteWhenTargetSizeExceedsLimit() + { + var (_, manifestBytes) = RandomManifest(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifestBytes), + Size = manifestBytes.Length + }; + var httpHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(httpHandler), + PlainHttp = true, + MaxMetadataBytes = manifestDesc.Size - 1, + }); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + var exception = await Assert.ThrowsAsync(async () => await store.DeleteAsync(manifestDesc, cancellationToken)); + Assert.Equal($"content size {manifestDesc.Size} exceeds MaxMetadataBytes {repo.Options.MaxMetadataBytes}", exception.Message); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + } + [Fact] public async Task ManifestStore_DeleteWithSubjectWhenReferrersAPINotSupported() { diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index 260eda3c..e39e9649 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -2793,4 +2793,26 @@ public async Task PingReferrers_ShouldFailWhenBadRequestReturns() await Assert.ThrowsAsync(async () => await repo.PingReferrers(cancellationToken)); Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); } + + [Fact] + public void LimitSize_ShouldThrowException_WhenSizeExceedsLimit() + { + var desc = RandomDescriptor(); + desc.Size = 150; + long limitSize = 100; + + var exception = Assert.Throws(() => Repository.LimitSize(desc, limitSize)); + Assert.Equal("content size 150 exceeds MaxMetadataBytes 100", exception.Message); + } + + [Fact] + public void LimitSize_ShouldNotThrowException_WhenSizeIsWithinLimit() + { + var desc = RandomDescriptor(); + desc.Size = 50; + long limitSize = 100; + + var exception = Record.Exception(() => Repository.LimitSize(desc, limitSize)); + Assert.Null(exception); // No exception should be thrown + } } From d17ceb67356a55574a4257b44b7f7448f3ccc6b4 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Thu, 20 Feb 2025 10:19:13 +1100 Subject: [PATCH 33/36] verify the manifest response Signed-off-by: Patrick Pan --- .../Registry/Remote/HttpResponseMessageExtensions.cs | 2 +- src/OrasProject.Oras/Registry/Remote/ManifestStore.cs | 5 +++-- src/OrasProject.Oras/Registry/Remote/Repository.cs | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 7c036b7a..bbd481bb 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -32,7 +32,7 @@ internal static class HttpResponseMessageExtensions /// /// /// - public static async Task ParseErrorResponseAsync(this HttpResponseMessage response, CancellationToken cancellationToken) + public static async Task ParseErrorResponseAsync(this HttpResponseMessage response, CancellationToken cancellationToken) { var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); return new ResponseException(response, body); diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index de095e86..89f932d5 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -420,9 +420,10 @@ private async Task DeleteWithIndexing(Descriptor target, CancellationToken cance } Repository.LimitSize(target, Repository.Options.MaxMetadataBytes); - await using (var manifest = await FetchAsync(target, cancellationToken).ConfigureAwait(false)) + var manifest = await Repository.FetchAllAsync(target, cancellationToken).ConfigureAwait(false); + using (var manifestStream = new MemoryStream(manifest)) { - await IndexReferrersForDelete(target, manifest, cancellationToken).ConfigureAwait(false); + await IndexReferrersForDelete(target, manifestStream, cancellationToken).ConfigureAwait(false); } break; } diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 64f17504..6a210260 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -415,7 +415,7 @@ internal async Task PingReferrers(CancellationToken cancellationToken = de SetReferrersState(supported); return supported; case HttpStatusCode.NotFound: - var err = (ResponseException) await response.ParseErrorResponseAsync(cancellationToken) + var err = await response.ParseErrorResponseAsync(cancellationToken) .ConfigureAwait(false); if (err.Errors?.First().Code == nameof(ResponseException.ErrorCode.NAME_UNKNOWN)) { From 6851df2cf809420c92f4e33bdbe25f5e8bd256f2 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Fri, 21 Feb 2025 13:54:51 +1100 Subject: [PATCH 34/36] fix test Signed-off-by: Patrick Pan --- tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index f7559d0f..b1511426 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -303,7 +303,7 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupported() Size = secondExpectedManifestBytes.Length, ArtifactType = MediaType.ImageConfig, }; - var secondExpectedReferrersList = new List(oldIndex.Manifests); + var secondExpectedReferrersList = new List(firstExpectedReferrersList); secondExpectedReferrersList.Add(secondExpectedManifestDesc); var (secondExpectedIndexReferrersDesc, secondExpectedIndexReferrersBytes) = Index.GenerateIndex(secondExpectedReferrersList); @@ -347,7 +347,8 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupported() return new HttpResponseMessage(HttpStatusCode.BadRequest); } - response.Content = new ByteArrayContent(oldIndexBytes); + if (oldIndexDeleted) response.Content = new ByteArrayContent(firstExpectedIndexReferrersBytes); + else response.Content = new ByteArrayContent(oldIndexBytes); response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); if (oldIndexDeleted) response.Headers.Add(_dockerContentDigestHeader, new string[] { firstExpectedIndexReferrersDesc.Digest }); else response.Headers.Add(_dockerContentDigestHeader, new string[] { oldIndexDesc.Digest }); @@ -372,7 +373,7 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupported() { return new HttpResponseMessage(HttpStatusCode.BadRequest); } - response.Content = new ByteArrayContent(oldIndexBytes); + response.Content = new ByteArrayContent(firstExpectedIndexReferrersBytes); response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); response.Headers.Add(_dockerContentDigestHeader, new string[] { firstExpectedIndexReferrersDesc.Digest }); response.StatusCode = HttpStatusCode.OK; From 7d0a8efea404d6b6ba789fba17daaa678b24e4c5 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Mon, 24 Feb 2025 10:50:46 +1100 Subject: [PATCH 35/36] address comments Signed-off-by: Patrick Pan --- .../Registry/Remote/ManifestStore.cs | 2 +- .../Registry/Remote/Repository.cs | 5 ++--- .../Remote/RepositoryTest.cs | 16 ++++++++-------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index 89f932d5..efae9e9f 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -465,7 +465,7 @@ private async Task IndexReferrersForDelete(Descriptor target, Stream manifestCon return; } - var isReferrersSupported = await Repository.PingReferrers(cancellationToken).ConfigureAwait(false); + var isReferrersSupported = await Repository.PingReferrersAsync(cancellationToken).ConfigureAwait(false); if (isReferrersSupported) { // referrers API is available, no client-side indexing needed diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 6a210260..a3a59c18 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -372,14 +372,14 @@ public async Task MountAsync(Descriptor descriptor, string fromRepository, Func< => await ((IMounter)Blobs).MountAsync(descriptor, fromRepository, getContent, cancellationToken).ConfigureAwait(false); /// - /// PingReferrers returns true if the Referrers API is available for the repository, + /// PingReferrersAsync returns true if the Referrers API is available for the repository, /// otherwise returns false /// /// /// /// /// - internal async Task PingReferrers(CancellationToken cancellationToken = default) + internal async Task PingReferrersAsync(CancellationToken cancellationToken = default) { switch (ReferrersState) { @@ -455,7 +455,6 @@ public void SetReferrersState(bool isSupported) /// /// LimitSize throws SizeLimitExceededException if the size of desc exceeds the limit limitSize. - /// If limitSize is less than or equal to zero, _defaultMaxMetadataBytes is used. /// /// /// diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index e39e9649..89684963 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -2617,7 +2617,7 @@ public async Task PingReferrers_ShouldReturnTrueWhenReferrersAPISupported() }); var cancellationToken = new CancellationToken(); Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); - var result = await repo.PingReferrers(cancellationToken); + var result = await repo.PingReferrersAsync(cancellationToken); Assert.True(result); Assert.Equal(Referrers.ReferrersState.Supported, repo.ReferrersState); } @@ -2646,9 +2646,9 @@ public async Task PingReferrers_WaitsForSemaphoreRelease() }); var cancellationToken = new CancellationToken(); Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); - var ping1 = repo.PingReferrers(cancellationToken); + var ping1 = repo.PingReferrersAsync(cancellationToken); await Task.Delay(50, cancellationToken); - var ping2 = repo.PingReferrers(cancellationToken); + var ping2 = repo.PingReferrersAsync(cancellationToken); Assert.True(ping1.IsCompletedSuccessfully); Assert.True(ping2.IsCompletedSuccessfully); Assert.Equal(Referrers.ReferrersState.Supported, repo.ReferrersState); @@ -2682,7 +2682,7 @@ public async Task PingReferrers_LimitsConcurrency() var tasks = new List>(); for (int i = 0; i < 5; ++i) { - tasks.Add(repo.PingReferrers(cancellationToken)); + tasks.Add(repo.PingReferrersAsync(cancellationToken)); } var results = await Task.WhenAll(tasks); @@ -2713,7 +2713,7 @@ public async Task PingReferrers_ShouldReturnFalseWhenReferrersAPINotSupportedNoC }); var cancellationToken = new CancellationToken(); Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); - var result = await repo.PingReferrers(cancellationToken); + var result = await repo.PingReferrersAsync(cancellationToken); Assert.False(result); Assert.Equal(Referrers.ReferrersState.NotSupported, repo.ReferrersState); } @@ -2756,7 +2756,7 @@ public async Task PingReferrers_ShouldFailWhenReturnNotFound() PlainHttp = true, }); Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); - await Assert.ThrowsAsync(async () => await repo.PingReferrers(cancellationToken)); + await Assert.ThrowsAsync(async () => await repo.PingReferrersAsync(cancellationToken)); Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); // referrer API is not supported @@ -2767,7 +2767,7 @@ public async Task PingReferrers_ShouldFailWhenReturnNotFound() PlainHttp = true, }); Assert.Equal(Referrers.ReferrersState.Unknown, repo1.ReferrersState); - var result = await repo1.PingReferrers(cancellationToken); + var result = await repo1.PingReferrersAsync(cancellationToken); Assert.False(result); Assert.Equal(Referrers.ReferrersState.NotSupported, repo1.ReferrersState); } @@ -2790,7 +2790,7 @@ public async Task PingReferrers_ShouldFailWhenBadRequestReturns() }); var cancellationToken = new CancellationToken(); Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); - await Assert.ThrowsAsync(async () => await repo.PingReferrers(cancellationToken)); + await Assert.ThrowsAsync(async () => await repo.PingReferrersAsync(cancellationToken)); Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); } From 90cb14196f898c251ee95bab67a53c9f41d1dff1 Mon Sep 17 00:00:00 2001 From: Patrick Pan Date: Mon, 24 Feb 2025 15:47:34 +1100 Subject: [PATCH 36/36] change to public for responsexception Signed-off-by: Patrick Pan --- src/OrasProject.Oras/Registry/Remote/ResponseException.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs b/src/OrasProject.Oras/Registry/Remote/ResponseException.cs index b4a7ae4b..5160f1ee 100644 --- a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs +++ b/src/OrasProject.Oras/Registry/Remote/ResponseException.cs @@ -20,7 +20,7 @@ namespace OrasProject.Oras.Registry.Remote; -internal class ResponseException : HttpRequestException +public class ResponseException : HttpRequestException { public enum ErrorCode { @@ -39,7 +39,7 @@ public class Error public JsonElement? Detail { get; set; } } - public class ErrorResponse + private class ErrorResponse { [JsonPropertyName("errors")] public required IList Errors { get; set; }