From 10c08ac4f56e615965e1b1cc496635774e546525 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 15 Sep 2025 11:59:26 -0700 Subject: [PATCH 01/38] Create ImageBuilder manifest library --- Microsoft.DotNet.DockerTools.slnx | 1 + src/ImageBuilder.Manifest/Class1.cs | 6 ++++++ .../Microsoft.DotNet.ImageBuilder.Manifest.csproj | 9 +++++++++ 3 files changed, 16 insertions(+) create mode 100644 src/ImageBuilder.Manifest/Class1.cs create mode 100644 src/ImageBuilder.Manifest/Microsoft.DotNet.ImageBuilder.Manifest.csproj diff --git a/Microsoft.DotNet.DockerTools.slnx b/Microsoft.DotNet.DockerTools.slnx index 13ca555a..5eecc2ee 100644 --- a/Microsoft.DotNet.DockerTools.slnx +++ b/Microsoft.DotNet.DockerTools.slnx @@ -7,5 +7,6 @@ + diff --git a/src/ImageBuilder.Manifest/Class1.cs b/src/ImageBuilder.Manifest/Class1.cs new file mode 100644 index 00000000..3be3ab70 --- /dev/null +++ b/src/ImageBuilder.Manifest/Class1.cs @@ -0,0 +1,6 @@ +namespace Microsoft.DotNet.ImageBuilder.Manifest; + +public class Class1 +{ + +} diff --git a/src/ImageBuilder.Manifest/Microsoft.DotNet.ImageBuilder.Manifest.csproj b/src/ImageBuilder.Manifest/Microsoft.DotNet.ImageBuilder.Manifest.csproj new file mode 100644 index 00000000..bbb3fa64 --- /dev/null +++ b/src/ImageBuilder.Manifest/Microsoft.DotNet.ImageBuilder.Manifest.csproj @@ -0,0 +1,9 @@ + + + + net9.0 + enable + Microsoft.DotNet.ImageBuilder.Manifest + + + From 123d9715c82f30404b4d4ee334c98b417ed81c97 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 15 Sep 2025 14:06:29 -0700 Subject: [PATCH 02/38] Rename project to ImageBuilder.Models and move Manifest model out of ImageBuilder tool --- Microsoft.DotNet.DockerTools.slnx | 2 +- src/Dockerfile.linux | 1 + src/Dockerfile.windows | 1 + src/ImageBuilder.Manifest/Class1.cs | 6 ------ .../Models => ImageBuilder.Models}/Manifest/Architecture.cs | 0 .../Manifest/CustomBuildLegDependencyType.cs | 0 .../Manifest/CustomBuildLegGroup.cs | 0 .../Models => ImageBuilder.Models}/Manifest/Image.cs | 0 .../Models => ImageBuilder.Models}/Manifest/Manifest.cs | 2 +- .../Models => ImageBuilder.Models}/Manifest/OS.cs | 0 .../Manifest/PackageQueryInfo.cs | 0 .../Models => ImageBuilder.Models}/Manifest/Platform.cs | 0 .../Models => ImageBuilder.Models}/Manifest/Readme.cs | 0 .../Models => ImageBuilder.Models}/Manifest/Repo.cs | 0 .../Models => ImageBuilder.Models}/Manifest/Tag.cs | 0 .../Manifest/TagDocumentationType.cs | 0 .../Manifest/TagSyndication.cs | 0 .../Microsoft.DotNet.ImageBuilder.Models.csproj} | 6 +++++- src/ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj | 4 ++++ 19 files changed, 13 insertions(+), 9 deletions(-) delete mode 100644 src/ImageBuilder.Manifest/Class1.cs rename src/{ImageBuilder/Models => ImageBuilder.Models}/Manifest/Architecture.cs (100%) rename src/{ImageBuilder/Models => ImageBuilder.Models}/Manifest/CustomBuildLegDependencyType.cs (100%) rename src/{ImageBuilder/Models => ImageBuilder.Models}/Manifest/CustomBuildLegGroup.cs (100%) rename src/{ImageBuilder/Models => ImageBuilder.Models}/Manifest/Image.cs (100%) rename src/{ImageBuilder/Models => ImageBuilder.Models}/Manifest/Manifest.cs (99%) rename src/{ImageBuilder/Models => ImageBuilder.Models}/Manifest/OS.cs (100%) rename src/{ImageBuilder/Models => ImageBuilder.Models}/Manifest/PackageQueryInfo.cs (100%) rename src/{ImageBuilder/Models => ImageBuilder.Models}/Manifest/Platform.cs (100%) rename src/{ImageBuilder/Models => ImageBuilder.Models}/Manifest/Readme.cs (100%) rename src/{ImageBuilder/Models => ImageBuilder.Models}/Manifest/Repo.cs (100%) rename src/{ImageBuilder/Models => ImageBuilder.Models}/Manifest/Tag.cs (100%) rename src/{ImageBuilder/Models => ImageBuilder.Models}/Manifest/TagDocumentationType.cs (100%) rename src/{ImageBuilder/Models => ImageBuilder.Models}/Manifest/TagSyndication.cs (100%) rename src/{ImageBuilder.Manifest/Microsoft.DotNet.ImageBuilder.Manifest.csproj => ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj} (50%) diff --git a/Microsoft.DotNet.DockerTools.slnx b/Microsoft.DotNet.DockerTools.slnx index 5eecc2ee..0c8d9669 100644 --- a/Microsoft.DotNet.DockerTools.slnx +++ b/Microsoft.DotNet.DockerTools.slnx @@ -7,6 +7,6 @@ - + diff --git a/src/Dockerfile.linux b/src/Dockerfile.linux index 85135e07..98d9a35a 100644 --- a/src/Dockerfile.linux +++ b/src/Dockerfile.linux @@ -19,6 +19,7 @@ WORKDIR /image-builder # restore packages before copying entire source - provides optimizations when rebuilding COPY NuGet.config ./ COPY ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj ./ImageBuilder/ +COPY ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj ./ImageBuilder.Models/ RUN dotnet restore -r linux-$TARGETARCH ./ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj # copy everything else and publish diff --git a/src/Dockerfile.windows b/src/Dockerfile.windows index 4ef231a5..31b98729 100644 --- a/src/Dockerfile.windows +++ b/src/Dockerfile.windows @@ -10,6 +10,7 @@ WORKDIR /image-builder # restore packages before copying entire source - provides optimizations when rebuilding COPY NuGet.config ./ COPY ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj ./ImageBuilder/ +COPY ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj ./ImageBuilder.Models/ RUN dotnet restore -r win-x64 ./ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj # copy everything else and publish diff --git a/src/ImageBuilder.Manifest/Class1.cs b/src/ImageBuilder.Manifest/Class1.cs deleted file mode 100644 index 3be3ab70..00000000 --- a/src/ImageBuilder.Manifest/Class1.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Microsoft.DotNet.ImageBuilder.Manifest; - -public class Class1 -{ - -} diff --git a/src/ImageBuilder/Models/Manifest/Architecture.cs b/src/ImageBuilder.Models/Manifest/Architecture.cs similarity index 100% rename from src/ImageBuilder/Models/Manifest/Architecture.cs rename to src/ImageBuilder.Models/Manifest/Architecture.cs diff --git a/src/ImageBuilder/Models/Manifest/CustomBuildLegDependencyType.cs b/src/ImageBuilder.Models/Manifest/CustomBuildLegDependencyType.cs similarity index 100% rename from src/ImageBuilder/Models/Manifest/CustomBuildLegDependencyType.cs rename to src/ImageBuilder.Models/Manifest/CustomBuildLegDependencyType.cs diff --git a/src/ImageBuilder/Models/Manifest/CustomBuildLegGroup.cs b/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs similarity index 100% rename from src/ImageBuilder/Models/Manifest/CustomBuildLegGroup.cs rename to src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs diff --git a/src/ImageBuilder/Models/Manifest/Image.cs b/src/ImageBuilder.Models/Manifest/Image.cs similarity index 100% rename from src/ImageBuilder/Models/Manifest/Image.cs rename to src/ImageBuilder.Models/Manifest/Image.cs diff --git a/src/ImageBuilder/Models/Manifest/Manifest.cs b/src/ImageBuilder.Models/Manifest/Manifest.cs similarity index 99% rename from src/ImageBuilder/Models/Manifest/Manifest.cs rename to src/ImageBuilder.Models/Manifest/Manifest.cs index d3e5cb09..142f3d47 100644 --- a/src/ImageBuilder/Models/Manifest/Manifest.cs +++ b/src/ImageBuilder.Models/Manifest/Manifest.cs @@ -41,7 +41,7 @@ public class Manifest [Description( "A set of custom variables that can be referenced in various parts of the " + "manifest. This provides a few benefits: 1) allows a commmonly used value " + - "to be defined only once and referenced by its variable name many times" + + "to be defined only once and referenced by its variable name many times" + "2) allows tools that consume the manifest file to provide a mechanism to " + "dynamically override the value of these variables. Variables may be " + "referenced in other parts of the manifest by using the following syntax: " + diff --git a/src/ImageBuilder/Models/Manifest/OS.cs b/src/ImageBuilder.Models/Manifest/OS.cs similarity index 100% rename from src/ImageBuilder/Models/Manifest/OS.cs rename to src/ImageBuilder.Models/Manifest/OS.cs diff --git a/src/ImageBuilder/Models/Manifest/PackageQueryInfo.cs b/src/ImageBuilder.Models/Manifest/PackageQueryInfo.cs similarity index 100% rename from src/ImageBuilder/Models/Manifest/PackageQueryInfo.cs rename to src/ImageBuilder.Models/Manifest/PackageQueryInfo.cs diff --git a/src/ImageBuilder/Models/Manifest/Platform.cs b/src/ImageBuilder.Models/Manifest/Platform.cs similarity index 100% rename from src/ImageBuilder/Models/Manifest/Platform.cs rename to src/ImageBuilder.Models/Manifest/Platform.cs diff --git a/src/ImageBuilder/Models/Manifest/Readme.cs b/src/ImageBuilder.Models/Manifest/Readme.cs similarity index 100% rename from src/ImageBuilder/Models/Manifest/Readme.cs rename to src/ImageBuilder.Models/Manifest/Readme.cs diff --git a/src/ImageBuilder/Models/Manifest/Repo.cs b/src/ImageBuilder.Models/Manifest/Repo.cs similarity index 100% rename from src/ImageBuilder/Models/Manifest/Repo.cs rename to src/ImageBuilder.Models/Manifest/Repo.cs diff --git a/src/ImageBuilder/Models/Manifest/Tag.cs b/src/ImageBuilder.Models/Manifest/Tag.cs similarity index 100% rename from src/ImageBuilder/Models/Manifest/Tag.cs rename to src/ImageBuilder.Models/Manifest/Tag.cs diff --git a/src/ImageBuilder/Models/Manifest/TagDocumentationType.cs b/src/ImageBuilder.Models/Manifest/TagDocumentationType.cs similarity index 100% rename from src/ImageBuilder/Models/Manifest/TagDocumentationType.cs rename to src/ImageBuilder.Models/Manifest/TagDocumentationType.cs diff --git a/src/ImageBuilder/Models/Manifest/TagSyndication.cs b/src/ImageBuilder.Models/Manifest/TagSyndication.cs similarity index 100% rename from src/ImageBuilder/Models/Manifest/TagSyndication.cs rename to src/ImageBuilder.Models/Manifest/TagSyndication.cs diff --git a/src/ImageBuilder.Manifest/Microsoft.DotNet.ImageBuilder.Manifest.csproj b/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj similarity index 50% rename from src/ImageBuilder.Manifest/Microsoft.DotNet.ImageBuilder.Manifest.csproj rename to src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj index bbb3fa64..cb91e8b0 100644 --- a/src/ImageBuilder.Manifest/Microsoft.DotNet.ImageBuilder.Manifest.csproj +++ b/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj @@ -3,7 +3,11 @@ net9.0 enable - Microsoft.DotNet.ImageBuilder.Manifest + Microsoft.DotNet.ImageBuilder.Models + + + + diff --git a/src/ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj b/src/ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj index a23aa042..098d0dce 100644 --- a/src/ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj +++ b/src/ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj @@ -9,6 +9,10 @@ true + + + + From 4880c1e958c9a051d2d76d5cbc79feeca73a727c Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 15 Sep 2025 14:18:42 -0700 Subject: [PATCH 03/38] Remove unused imports --- src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs | 1 - src/ImageBuilder.Models/Manifest/Image.cs | 1 - src/ImageBuilder.Models/Manifest/Manifest.cs | 2 -- src/ImageBuilder.Models/Manifest/Platform.cs | 2 -- src/ImageBuilder.Models/Manifest/Repo.cs | 1 - 5 files changed, 7 deletions(-) diff --git a/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs b/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs index 5c3a856e..02398410 100644 --- a/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs +++ b/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; using System.ComponentModel; using Newtonsoft.Json; diff --git a/src/ImageBuilder.Models/Manifest/Image.cs b/src/ImageBuilder.Models/Manifest/Image.cs index 68124f0c..68fb60ad 100644 --- a/src/ImageBuilder.Models/Manifest/Image.cs +++ b/src/ImageBuilder.Models/Manifest/Image.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System.Collections.Generic; using System.ComponentModel; using Newtonsoft.Json; diff --git a/src/ImageBuilder.Models/Manifest/Manifest.cs b/src/ImageBuilder.Models/Manifest/Manifest.cs index 142f3d47..dda951f0 100644 --- a/src/ImageBuilder.Models/Manifest/Manifest.cs +++ b/src/ImageBuilder.Models/Manifest/Manifest.cs @@ -2,8 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; -using System.Collections.Generic; using System.ComponentModel; namespace Microsoft.DotNet.ImageBuilder.Models.Manifest diff --git a/src/ImageBuilder.Models/Manifest/Platform.cs b/src/ImageBuilder.Models/Manifest/Platform.cs index ccac058a..1a5214ef 100644 --- a/src/ImageBuilder.Models/Manifest/Platform.cs +++ b/src/ImageBuilder.Models/Manifest/Platform.cs @@ -2,8 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; -using System.Collections.Generic; using System.ComponentModel; using Newtonsoft.Json; using Newtonsoft.Json.Converters; diff --git a/src/ImageBuilder.Models/Manifest/Repo.cs b/src/ImageBuilder.Models/Manifest/Repo.cs index 3cbd0a57..5c80ed95 100644 --- a/src/ImageBuilder.Models/Manifest/Repo.cs +++ b/src/ImageBuilder.Models/Manifest/Repo.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; using System.ComponentModel; using Newtonsoft.Json; From bc4c1ba6fa3db15bd7612eec837666c137891121 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 15 Sep 2025 14:20:56 -0700 Subject: [PATCH 04/38] Use file scoped namespaces --- .../Manifest/Architecture.cs | 15 ++- .../Manifest/CustomBuildLegDependencyType.cs | 45 ++++--- .../Manifest/CustomBuildLegGroup.cs | 49 ++++---- src/ImageBuilder.Models/Manifest/Image.cs | 39 +++--- src/ImageBuilder.Models/Manifest/Manifest.cs | 83 +++++++------ src/ImageBuilder.Models/Manifest/OS.cs | 11 +- .../Manifest/PackageQueryInfo.cs | 27 ++--- src/ImageBuilder.Models/Manifest/Platform.cs | 111 +++++++++--------- src/ImageBuilder.Models/Manifest/Repo.cs | 73 ++++++------ src/ImageBuilder.Models/Manifest/Tag.cs | 47 ++++---- .../Manifest/TagDocumentationType.cs | 31 +++-- .../Manifest/TagSyndication.cs | 27 ++--- 12 files changed, 273 insertions(+), 285 deletions(-) diff --git a/src/ImageBuilder.Models/Manifest/Architecture.cs b/src/ImageBuilder.Models/Manifest/Architecture.cs index cdcbb328..b39ed37f 100644 --- a/src/ImageBuilder.Models/Manifest/Architecture.cs +++ b/src/ImageBuilder.Models/Manifest/Architecture.cs @@ -2,13 +2,12 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +// Enum values must align with the $GOARCH values specified at https://golang.org/doc/install/source#environment +public enum Architecture { - // Enum values must align with the $GOARCH values specified at https://golang.org/doc/install/source#environment - public enum Architecture - { - ARM, - ARM64, - AMD64, - } + ARM, + ARM64, + AMD64, } diff --git a/src/ImageBuilder.Models/Manifest/CustomBuildLegDependencyType.cs b/src/ImageBuilder.Models/Manifest/CustomBuildLegDependencyType.cs index 67fb4308..b1027eab 100644 --- a/src/ImageBuilder.Models/Manifest/CustomBuildLegDependencyType.cs +++ b/src/ImageBuilder.Models/Manifest/CustomBuildLegDependencyType.cs @@ -4,31 +4,30 @@ using System.ComponentModel; -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description( + "The type of dependency an image has for a specific scenario." + )] +public enum CustomBuildLegDependencyType { [Description( - "The type of dependency an image has for a specific scenario." + "Indicates the dependency is considered to be integral to the depending image." + + "This means the dependent image will not have its own dependency graph considered for build leg " + + "generation. An example of this is when a custom build leg dependency is defined from sdk to " + + "aspnet; in that case, aspnet and sdk will be included in a leg together but the sdk will not " + + "have its own leg generated." )] - public enum CustomBuildLegDependencyType - { - [Description( - "Indicates the dependency is considered to be integral to the depending image." + - "This means the dependent image will not have its own dependency graph considered for build leg " + - "generation. An example of this is when a custom build leg dependency is defined from sdk to " + - "aspnet; in that case, aspnet and sdk will be included in a leg together but the sdk will not " + - "have its own leg generated." - )] - Integral, + Integral, - [Description( - "Indicates the dependency is considered to be a supplemental companion to the depending image." + - "This means the dependent image will have its own dependency graph considered for build leg " + - "generation. An example of this is when a custom build leg dependency is defined to " + - "include an SDK image supported on a particular architecture in order to test a runtime OS " + - "that doesn't its own SDK on that architecture (Trixie ARM SDK to test Alpine ARM runtime); " + - "in that case, the SDK will be included in a leg together with the runtime and the SDK will " + - "still have have its own leg." - )] - Supplemental - } + [Description( + "Indicates the dependency is considered to be a supplemental companion to the depending image." + + "This means the dependent image will have its own dependency graph considered for build leg " + + "generation. An example of this is when a custom build leg dependency is defined to " + + "include an SDK image supported on a particular architecture in order to test a runtime OS " + + "that doesn't its own SDK on that architecture (Trixie ARM SDK to test Alpine ARM runtime); " + + "in that case, the SDK will be included in a leg together with the runtime and the SDK will " + + "still have have its own leg." + )] + Supplemental } diff --git a/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs b/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs index 02398410..97928849 100644 --- a/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs +++ b/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs @@ -5,34 +5,33 @@ using System.ComponentModel; using Newtonsoft.Json; -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description( + "This object describes the tag dependencies of the image for a specific named scenario. This is " + + "for advanced cases only. It allows tooling to modify the build matrix that would normally be " + + "generated for the image by including the customizations described in this metadata. An example " + + "usage of this is in PR builds where it is necessary to build and test in the same job. In such " + + "a scenario, some images are part of a test matrix that require images to be available on the " + + "build machine that aren't part of that images dependency graph in normal scenarios. By " + + "specifying a customBuildLegGroup for this scenario, those additional image dependencies can " + + "be specified and the build pipeline can make use of them when constructing its build graph when " + + "specified to do so." + )] +public class CustomBuildLegGroup { [Description( - "This object describes the tag dependencies of the image for a specific named scenario. This is " + - "for advanced cases only. It allows tooling to modify the build matrix that would normally be " + - "generated for the image by including the customizations described in this metadata. An example " + - "usage of this is in PR builds where it is necessary to build and test in the same job. In such " + - "a scenario, some images are part of a test matrix that require images to be available on the " + - "build machine that aren't part of that images dependency graph in normal scenarios. By " + - "specifying a customBuildLegGroup for this scenario, those additional image dependencies can " + - "be specified and the build pipeline can make use of them when constructing its build graph when " + - "specified to do so." + "Name of the group describing the scenario in which it's relevant. This is just a " + + " custom label that can then be used by tooling to lookup the group when necessary." )] - public class CustomBuildLegGroup - { - [Description( - "Name of the group describing the scenario in which it's relevant. This is just a " + - " custom label that can then be used by tooling to lookup the group when necessary." - )] - [JsonProperty(Required = Required.Always)] - public string Name { get; set; } + [JsonProperty(Required = Required.Always)] + public string Name { get; set; } - [Description("The type of the dependency which impacts how it's used during the build.")] - [JsonProperty(Required = Required.Always)] - public CustomBuildLegDependencyType Type { get; set; } + [Description("The type of the dependency which impacts how it's used during the build.")] + [JsonProperty(Required = Required.Always)] + public CustomBuildLegDependencyType Type { get; set; } - [Description("The set of dependencies the image has for this scenario.")] - [JsonProperty(Required = Required.Always)] - public string[] Dependencies { get; set; } = Array.Empty(); - } + [Description("The set of dependencies the image has for this scenario.")] + [JsonProperty(Required = Required.Always)] + public string[] Dependencies { get; set; } = Array.Empty(); } diff --git a/src/ImageBuilder.Models/Manifest/Image.cs b/src/ImageBuilder.Models/Manifest/Image.cs index 68fb60ad..224c3c6f 100644 --- a/src/ImageBuilder.Models/Manifest/Image.cs +++ b/src/ImageBuilder.Models/Manifest/Image.cs @@ -5,29 +5,28 @@ using System.ComponentModel; using Newtonsoft.Json; -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description("An image object contains metadata about a specific Docker image.")] +public class Image { - [Description("An image object contains metadata about a specific Docker image.")] - public class Image - { - [Description( - "The set of platforms that describe the platform-specific variations of the Docker image.")] - [JsonProperty(Required = Required.Always)] - public Platform[] Platforms { get; set; } + [Description( + "The set of platforms that describe the platform-specific variations of the Docker image.")] + [JsonProperty(Required = Required.Always)] + public Platform[] Platforms { get; set; } - [Description( - "The set of tags that are shared amongst all platform-specific versions of the image. An " + - "example of a shared tag, including its repo name, is dotnet/core/runtime:2.2; running " + - "`docker pull mcr.microsoft.com/dotnet/core/runtime:2.2` on Windows will get the " + - "default Windows-based tag whereas running it on Linux will get the default " + - "Linux-based tag.")] - public IDictionary SharedTags { get; set; } + [Description( + "The set of tags that are shared amongst all platform-specific versions of the image. An " + + "example of a shared tag, including its repo name, is dotnet/core/runtime:2.2; running " + + "`docker pull mcr.microsoft.com/dotnet/core/runtime:2.2` on Windows will get the " + + "default Windows-based tag whereas running it on Linux will get the default " + + "Linux-based tag.")] + public IDictionary SharedTags { get; set; } - [Description("The full version of the product that the Docker image contains.")] - public string ProductVersion { get; set; } + [Description("The full version of the product that the Docker image contains.")] + public string ProductVersion { get; set; } - public Image() - { - } + public Image() + { } } diff --git a/src/ImageBuilder.Models/Manifest/Manifest.cs b/src/ImageBuilder.Models/Manifest/Manifest.cs index dda951f0..d31e2e6b 100644 --- a/src/ImageBuilder.Models/Manifest/Manifest.cs +++ b/src/ImageBuilder.Models/Manifest/Manifest.cs @@ -4,50 +4,49 @@ using System.ComponentModel; -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description( + "The manifest file is the primary source of metadata that drives the production " + + "of all .NET Docker images. It describes various attributes of the Docker images " + + "that are to be produced by a given GitHub repo. .NET Docker's engineering system " + + "consumes this file in various ways as part of the automated build pipelines and " + + "other tools. It's intended to be product-agnostic meaning that it could be used " + + "to describe metadata for Docker image production of any product, not just .NET.")] +public class Manifest { [Description( - "The manifest file is the primary source of metadata that drives the production " + - "of all .NET Docker images. It describes various attributes of the Docker images " + - "that are to be produced by a given GitHub repo. .NET Docker's engineering system " + - "consumes this file in various ways as part of the automated build pipelines and " + - "other tools. It's intended to be product-agnostic meaning that it could be used " + - "to describe metadata for Docker image production of any product, not just .NET.")] - public class Manifest + "Additional json files to be loaded with this manifest. This is a convienent" + + "way to split the manifest apart into logical parts." + )] + public string[] Includes { get; set; } + + [Description( + "Info about the readme that documents the product family." + )] + public Readme Readme { get; set; } + + [Description( + "The location of the Docker registry where the images are to be published." + )] + public string Registry { get; set; } + + [Description( + "The set of Docker repositories described by this manifest." + )] + public Repo[] Repos { get; set; } = Array.Empty(); + + [Description( + "A set of custom variables that can be referenced in various parts of the " + + "manifest. This provides a few benefits: 1) allows a commmonly used value " + + "to be defined only once and referenced by its variable name many times" + + "2) allows tools that consume the manifest file to provide a mechanism to " + + "dynamically override the value of these variables. Variables may be " + + "referenced in other parts of the manifest by using the following syntax: " + + "$(_VariableName_).")] + public IDictionary Variables { get; set; } = new Dictionary(); + + public Manifest() { - [Description( - "Additional json files to be loaded with this manifest. This is a convienent" + - "way to split the manifest apart into logical parts." - )] - public string[] Includes { get; set; } - - [Description( - "Info about the readme that documents the product family." - )] - public Readme Readme { get; set; } - - [Description( - "The location of the Docker registry where the images are to be published." - )] - public string Registry { get; set; } - - [Description( - "The set of Docker repositories described by this manifest." - )] - public Repo[] Repos { get; set; } = Array.Empty(); - - [Description( - "A set of custom variables that can be referenced in various parts of the " + - "manifest. This provides a few benefits: 1) allows a commmonly used value " + - "to be defined only once and referenced by its variable name many times" + - "2) allows tools that consume the manifest file to provide a mechanism to " + - "dynamically override the value of these variables. Variables may be " + - "referenced in other parts of the manifest by using the following syntax: " + - "$(_VariableName_).")] - public IDictionary Variables { get; set; } = new Dictionary(); - - public Manifest() - { - } } } diff --git a/src/ImageBuilder.Models/Manifest/OS.cs b/src/ImageBuilder.Models/Manifest/OS.cs index ca0984a6..8690bfaf 100644 --- a/src/ImageBuilder.Models/Manifest/OS.cs +++ b/src/ImageBuilder.Models/Manifest/OS.cs @@ -2,11 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +public enum OS { - public enum OS - { - Linux, - Windows, - } + Linux, + Windows, } diff --git a/src/ImageBuilder.Models/Manifest/PackageQueryInfo.cs b/src/ImageBuilder.Models/Manifest/PackageQueryInfo.cs index 0d04f92c..5031ca7f 100644 --- a/src/ImageBuilder.Models/Manifest/PackageQueryInfo.cs +++ b/src/ImageBuilder.Models/Manifest/PackageQueryInfo.cs @@ -4,22 +4,21 @@ #nullable enable using System.ComponentModel; -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description( + "Relative path to the template the Dockerfile is generated from." +)] +public class PackageQueryInfo { [Description( - "Relative path to the template the Dockerfile is generated from." - )] - public class PackageQueryInfo - { - [Description( - "Relative path from the manifest file to the script which queries the packages installed for a platform Dockerfile." - )] - public string? GetInstalledPackagesPath { get; set; } + "Relative path from the manifest file to the script which queries the packages installed for a platform Dockerfile." + )] + public string? GetInstalledPackagesPath { get; set; } - [Description( - "Relative path from the manifest file to the script which queries the packages available for upgrade for a platform Dockerfile." - )] - public string? GetUpgradablePackagesPath { get; set; } - } + [Description( + "Relative path from the manifest file to the script which queries the packages available for upgrade for a platform Dockerfile." + )] + public string? GetUpgradablePackagesPath { get; set; } } #nullable disable diff --git a/src/ImageBuilder.Models/Manifest/Platform.cs b/src/ImageBuilder.Models/Manifest/Platform.cs index 1a5214ef..66a929cf 100644 --- a/src/ImageBuilder.Models/Manifest/Platform.cs +++ b/src/ImageBuilder.Models/Manifest/Platform.cs @@ -7,73 +7,72 @@ using Newtonsoft.Json.Converters; #nullable enable -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description( + "A platform object contains metadata about a platform-specific version of an " + + "image and refers to the actual Dockerfile used to build the image.")] +public class Platform { [Description( - "A platform object contains metadata about a platform-specific version of an " + - "image and refers to the actual Dockerfile used to build the image.")] - public class Platform - { - [Description( - "The processor architecture associated with the image." - )] - [DefaultValue(Architecture.AMD64)] - [JsonConverter(typeof(StringEnumConverter))] - [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] - public Architecture Architecture { get; set; } = Architecture.AMD64; + "The processor architecture associated with the image." + )] + [DefaultValue(Architecture.AMD64)] + [JsonConverter(typeof(StringEnumConverter))] + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public Architecture Architecture { get; set; } = Architecture.AMD64; - [Description( - "A set of values that will passed to the `docker build` command " + - "to override variables defined in the Dockerfile.")] - public IDictionary BuildArgs { get; set; } = new Dictionary(); + [Description( + "A set of values that will passed to the `docker build` command " + + "to override variables defined in the Dockerfile.")] + public IDictionary BuildArgs { get; set; } = new Dictionary(); - [Description( - "Relative path to the associated Dockerfile. This can be a file or a " + - "directory. If it is a directory, the file name defaults to Dockerfile." - )] - [JsonProperty(Required = Required.Always)] - public string Dockerfile { get; set; } = string.Empty; + [Description( + "Relative path to the associated Dockerfile. This can be a file or a " + + "directory. If it is a directory, the file name defaults to Dockerfile." + )] + [JsonProperty(Required = Required.Always)] + public string Dockerfile { get; set; } = string.Empty; - [Description( - "Relative path to the template the Dockerfile is generated from." - )] - public string? DockerfileTemplate { get; set; } + [Description( + "Relative path to the template the Dockerfile is generated from." + )] + public string? DockerfileTemplate { get; set; } - [Description( - "The generic name of the operating system associated with the image." - )] - [JsonConverter(typeof(StringEnumConverter))] - [JsonProperty(Required = Required.Always)] - public OS OS { get; set; } + [Description( + "The generic name of the operating system associated with the image." + )] + [JsonConverter(typeof(StringEnumConverter))] + [JsonProperty(Required = Required.Always)] + public OS OS { get; set; } - [Description( - "The specific version of the operating system associated with the image. " + - "Examples: alpine3.9, bionic, nanoserver-1903." - )] - [JsonProperty(Required = Required.Always)] - public string OsVersion { get; set; } = string.Empty; + [Description( + "The specific version of the operating system associated with the image. " + + "Examples: alpine3.9, bionic, nanoserver-1903." + )] + [JsonProperty(Required = Required.Always)] + public string OsVersion { get; set; } = string.Empty; - [Description( - "The set of platform-specific tags associated with the image." - )] - [JsonProperty(Required = Required.Always)] - public IDictionary Tags { get; set; } = new Dictionary(); + [Description( + "The set of platform-specific tags associated with the image." + )] + [JsonProperty(Required = Required.Always)] + public IDictionary Tags { get; set; } = new Dictionary(); - [Description( - "The custom build leg groups associated with the platform." - )] - public CustomBuildLegGroup[] CustomBuildLegGroups { get; set; } = Array.Empty(); + [Description( + "The custom build leg groups associated with the platform." + )] + public CustomBuildLegGroup[] CustomBuildLegGroups { get; set; } = Array.Empty(); - [Description( - "A label which further distinguishes the architecture when it " + - "contains variants. For example, the ARM architecture has variants " + - "named v6, v7, etc." - )] - public string? Variant { get; set; } + [Description( + "A label which further distinguishes the architecture when it " + + "contains variants. For example, the ARM architecture has variants " + + "named v6, v7, etc." + )] + public string? Variant { get; set; } - public Platform() - { - } + public Platform() + { } } #nullable disable diff --git a/src/ImageBuilder.Models/Manifest/Repo.cs b/src/ImageBuilder.Models/Manifest/Repo.cs index 5c80ed95..bc899af0 100644 --- a/src/ImageBuilder.Models/Manifest/Repo.cs +++ b/src/ImageBuilder.Models/Manifest/Repo.cs @@ -5,46 +5,45 @@ using System.ComponentModel; using Newtonsoft.Json; -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description( + "A repository object contains metadata about a target Docker repository " + + "and the images to be contained in it." + )] +public class Repo { [Description( - "A repository object contains metadata about a target Docker repository " + - "and the images to be contained in it." + "A unique identifier of the repo. This is purely within the context " + + "of the manifest and not exposed to Docker in any way." + )] + public string Id { get; set; } + + [Description( + "The set of images contained in this repository." + )] + [JsonProperty(Required = Required.Always)] + public Image[] Images { get; set; } + + [Description( + "Relative path to the MCR tags template YAML file that is used by " + + "tooling to generate the tags section of the readme file." )] - public class Repo + public string McrTagsMetadataTemplate { get; set; } + + [Description( + "The name of the Docker repository where the described images are to " + + "be published (example: dotnet/core/runtime)." + )] + [JsonProperty(Required = Required.Always)] + public string Name { get; set; } + + [Description( + "Info about the readme that documents the repo." + )] + public Readme[] Readmes { get; set; } = Array.Empty(); + + public Repo() { - [Description( - "A unique identifier of the repo. This is purely within the context " + - "of the manifest and not exposed to Docker in any way." - )] - public string Id { get; set; } - - [Description( - "The set of images contained in this repository." - )] - [JsonProperty(Required = Required.Always)] - public Image[] Images { get; set; } - - [Description( - "Relative path to the MCR tags template YAML file that is used by " + - "tooling to generate the tags section of the readme file." - )] - public string McrTagsMetadataTemplate { get; set; } - - [Description( - "The name of the Docker repository where the described images are to " + - "be published (example: dotnet/core/runtime)." - )] - [JsonProperty(Required = Required.Always)] - public string Name { get; set; } - - [Description( - "Info about the readme that documents the repo." - )] - public Readme[] Readmes { get; set; } = Array.Empty(); - - public Repo() - { - } } } diff --git a/src/ImageBuilder.Models/Manifest/Tag.cs b/src/ImageBuilder.Models/Manifest/Tag.cs index 61567e3d..74f221a6 100644 --- a/src/ImageBuilder.Models/Manifest/Tag.cs +++ b/src/ImageBuilder.Models/Manifest/Tag.cs @@ -4,35 +4,34 @@ using System.ComponentModel; -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description( + "A tag object contains metadata about a Docker tag. It is a JSON object " + + "with its tag name used as the attribute name." + )] +public class Tag { [Description( - "A tag object contains metadata about a Docker tag. It is a JSON object " + - "with its tag name used as the attribute name." + "An identifier used to conceptually group related tags in the readme " + + "documentation." )] - public class Tag - { - [Description( - "An identifier used to conceptually group related tags in the readme " + - "documentation." - )] - public string DocumentationGroup { get; set; } + public string DocumentationGroup { get; set; } - [Description( - "Indicates how this tag should not be documented in the readme file. Regardless of the " + - "setting, the image will still be tagged with this tag and will still be published. " + - "This is useful when deprecating a tag that still needs to be kept up-to-date " + - "but not wanting it documented." - )] - [DefaultValue(TagDocumentationType.Documented)] - public TagDocumentationType DocType { get; set; } + [Description( + "Indicates how this tag should not be documented in the readme file. Regardless of the " + + "setting, the image will still be tagged with this tag and will still be published. " + + "This is useful when deprecating a tag that still needs to be kept up-to-date " + + "but not wanting it documented." + )] + [DefaultValue(TagDocumentationType.Documented)] + public TagDocumentationType DocType { get; set; } - [Description( - "Description of where the tag should be syndicated to.")] - public TagSyndication Syndication { get; set; } + [Description( + "Description of where the tag should be syndicated to.")] + public TagSyndication Syndication { get; set; } - public Tag() - { - } + public Tag() + { } } diff --git a/src/ImageBuilder.Models/Manifest/TagDocumentationType.cs b/src/ImageBuilder.Models/Manifest/TagDocumentationType.cs index 8946dbe4..e15f1946 100644 --- a/src/ImageBuilder.Models/Manifest/TagDocumentationType.cs +++ b/src/ImageBuilder.Models/Manifest/TagDocumentationType.cs @@ -2,23 +2,22 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +public enum TagDocumentationType { - public enum TagDocumentationType - { - /// - /// The tag is always documented. - /// - Documented, + /// + /// The tag is always documented. + /// + Documented, - /// - /// The tag is never documented. - /// - Undocumented, + /// + /// The tag is never documented. + /// + Undocumented, - /// - /// The tag is only documented if there are corresponding platform tags that are documented. - /// - PlatformDocumented - } + /// + /// The tag is only documented if there are corresponding platform tags that are documented. + /// + PlatformDocumented } diff --git a/src/ImageBuilder.Models/Manifest/TagSyndication.cs b/src/ImageBuilder.Models/Manifest/TagSyndication.cs index 835fd58d..813120df 100644 --- a/src/ImageBuilder.Models/Manifest/TagSyndication.cs +++ b/src/ImageBuilder.Models/Manifest/TagSyndication.cs @@ -4,21 +4,20 @@ using System.ComponentModel; -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description( + "A description of where a tag should be syndicated to." + )] +public class TagSyndication { [Description( - "A description of where a tag should be syndicated to." - )] - public class TagSyndication - { - [Description( - "Name of the repo to syndicate the tag to." - )] - public string Repo { get; set; } + "Name of the repo to syndicate the tag to." + )] + public string Repo { get; set; } - [Description( - "List of destination tag names to syndicate the tag to." - )] - public string[] DestinationTags { get; set; } - } + [Description( + "List of destination tag names to syndicate the tag to." + )] + public string[] DestinationTags { get; set; } } From 94a3993a6e5bcbcf80acdc91cbad431008567bfe Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 15 Sep 2025 14:23:25 -0700 Subject: [PATCH 05/38] Remove unused class --- .../Manifest/PackageQueryInfo.cs | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 src/ImageBuilder.Models/Manifest/PackageQueryInfo.cs diff --git a/src/ImageBuilder.Models/Manifest/PackageQueryInfo.cs b/src/ImageBuilder.Models/Manifest/PackageQueryInfo.cs deleted file mode 100644 index 5031ca7f..00000000 --- a/src/ImageBuilder.Models/Manifest/PackageQueryInfo.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#nullable enable -using System.ComponentModel; - -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; - -[Description( - "Relative path to the template the Dockerfile is generated from." -)] -public class PackageQueryInfo -{ - [Description( - "Relative path from the manifest file to the script which queries the packages installed for a platform Dockerfile." - )] - public string? GetInstalledPackagesPath { get; set; } - - [Description( - "Relative path from the manifest file to the script which queries the packages available for upgrade for a platform Dockerfile." - )] - public string? GetUpgradablePackagesPath { get; set; } -} -#nullable disable From 5e920f245e92ffb3c3b0fa83b3fc179bfdb4c7da Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 15 Sep 2025 15:13:06 -0700 Subject: [PATCH 06/38] Enable nullable annotations in Models project --- .../Manifest/CustomBuildLegGroup.cs | 6 +++--- src/ImageBuilder.Models/Manifest/Image.cs | 6 +++--- src/ImageBuilder.Models/Manifest/Platform.cs | 2 -- src/ImageBuilder.Models/Manifest/Readme.cs | 2 -- src/ImageBuilder.Models/Manifest/Repo.cs | 10 +++++----- src/ImageBuilder.Models/Manifest/Tag.cs | 6 +++--- src/ImageBuilder.Models/Manifest/TagSyndication.cs | 4 ++-- .../Microsoft.DotNet.ImageBuilder.Models.csproj | 1 + src/ImageBuilder/ViewModel/ImageInfo.cs | 6 +++++- src/ImageBuilder/ViewModel/PlatformInfo.cs | 4 ++-- src/ImageBuilder/ViewModel/VariableHelper.cs | 13 ++++--------- 11 files changed, 28 insertions(+), 32 deletions(-) diff --git a/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs b/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs index 97928849..c0aaf3ab 100644 --- a/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs +++ b/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs @@ -25,13 +25,13 @@ public class CustomBuildLegGroup " custom label that can then be used by tooling to lookup the group when necessary." )] [JsonProperty(Required = Required.Always)] - public string Name { get; set; } + public required string Name { get; set; } [Description("The type of the dependency which impacts how it's used during the build.")] [JsonProperty(Required = Required.Always)] - public CustomBuildLegDependencyType Type { get; set; } + public required CustomBuildLegDependencyType Type { get; set; } [Description("The set of dependencies the image has for this scenario.")] [JsonProperty(Required = Required.Always)] - public string[] Dependencies { get; set; } = Array.Empty(); + public required string[] Dependencies { get; set; } } diff --git a/src/ImageBuilder.Models/Manifest/Image.cs b/src/ImageBuilder.Models/Manifest/Image.cs index 224c3c6f..1ed3eee3 100644 --- a/src/ImageBuilder.Models/Manifest/Image.cs +++ b/src/ImageBuilder.Models/Manifest/Image.cs @@ -13,7 +13,7 @@ public class Image [Description( "The set of platforms that describe the platform-specific variations of the Docker image.")] [JsonProperty(Required = Required.Always)] - public Platform[] Platforms { get; set; } + public Platform[] Platforms { get; set; } = []; [Description( "The set of tags that are shared amongst all platform-specific versions of the image. An " + @@ -21,10 +21,10 @@ public class Image "`docker pull mcr.microsoft.com/dotnet/core/runtime:2.2` on Windows will get the " + "default Windows-based tag whereas running it on Linux will get the default " + "Linux-based tag.")] - public IDictionary SharedTags { get; set; } + public IDictionary? SharedTags { get; set; } [Description("The full version of the product that the Docker image contains.")] - public string ProductVersion { get; set; } + public string? ProductVersion { get; set; } public Image() { diff --git a/src/ImageBuilder.Models/Manifest/Platform.cs b/src/ImageBuilder.Models/Manifest/Platform.cs index 66a929cf..14fd9cc9 100644 --- a/src/ImageBuilder.Models/Manifest/Platform.cs +++ b/src/ImageBuilder.Models/Manifest/Platform.cs @@ -6,7 +6,6 @@ using Newtonsoft.Json; using Newtonsoft.Json.Converters; -#nullable enable namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; [Description( @@ -75,4 +74,3 @@ public Platform() { } } -#nullable disable diff --git a/src/ImageBuilder.Models/Manifest/Readme.cs b/src/ImageBuilder.Models/Manifest/Readme.cs index e3e4f1b4..5b12a325 100644 --- a/src/ImageBuilder.Models/Manifest/Readme.cs +++ b/src/ImageBuilder.Models/Manifest/Readme.cs @@ -6,7 +6,6 @@ namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; -#nullable enable public class Readme { [Description( @@ -32,4 +31,3 @@ public Readme(string path, string? templatePath) TemplatePath = templatePath; } } -#nullable disable diff --git a/src/ImageBuilder.Models/Manifest/Repo.cs b/src/ImageBuilder.Models/Manifest/Repo.cs index bc899af0..87965047 100644 --- a/src/ImageBuilder.Models/Manifest/Repo.cs +++ b/src/ImageBuilder.Models/Manifest/Repo.cs @@ -17,31 +17,31 @@ public class Repo "A unique identifier of the repo. This is purely within the context " + "of the manifest and not exposed to Docker in any way." )] - public string Id { get; set; } + public string? Id { get; set; } [Description( "The set of images contained in this repository." )] [JsonProperty(Required = Required.Always)] - public Image[] Images { get; set; } + public Image[] Images { get; set; } = []; [Description( "Relative path to the MCR tags template YAML file that is used by " + "tooling to generate the tags section of the readme file." )] - public string McrTagsMetadataTemplate { get; set; } + public string? McrTagsMetadataTemplate { get; set; } [Description( "The name of the Docker repository where the described images are to " + "be published (example: dotnet/core/runtime)." )] [JsonProperty(Required = Required.Always)] - public string Name { get; set; } + public required string Name { get; set; } [Description( "Info about the readme that documents the repo." )] - public Readme[] Readmes { get; set; } = Array.Empty(); + public Readme[] Readmes { get; set; } = []; public Repo() { diff --git a/src/ImageBuilder.Models/Manifest/Tag.cs b/src/ImageBuilder.Models/Manifest/Tag.cs index 74f221a6..a241b3b0 100644 --- a/src/ImageBuilder.Models/Manifest/Tag.cs +++ b/src/ImageBuilder.Models/Manifest/Tag.cs @@ -16,7 +16,7 @@ public class Tag "An identifier used to conceptually group related tags in the readme " + "documentation." )] - public string DocumentationGroup { get; set; } + public string? DocumentationGroup { get; set; } [Description( "Indicates how this tag should not be documented in the readme file. Regardless of the " + @@ -25,11 +25,11 @@ public class Tag "but not wanting it documented." )] [DefaultValue(TagDocumentationType.Documented)] - public TagDocumentationType DocType { get; set; } + public TagDocumentationType DocType { get; set; } = TagDocumentationType.Documented; [Description( "Description of where the tag should be syndicated to.")] - public TagSyndication Syndication { get; set; } + public TagSyndication? Syndication { get; set; } public Tag() { diff --git a/src/ImageBuilder.Models/Manifest/TagSyndication.cs b/src/ImageBuilder.Models/Manifest/TagSyndication.cs index 813120df..e2c4b4a6 100644 --- a/src/ImageBuilder.Models/Manifest/TagSyndication.cs +++ b/src/ImageBuilder.Models/Manifest/TagSyndication.cs @@ -14,10 +14,10 @@ public class TagSyndication [Description( "Name of the repo to syndicate the tag to." )] - public string Repo { get; set; } + public required string Repo { get; set; } [Description( "List of destination tag names to syndicate the tag to." )] - public string[] DestinationTags { get; set; } + public string[] DestinationTags { get; set; } = []; } diff --git a/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj b/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj index cb91e8b0..ea7bf34f 100644 --- a/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj +++ b/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj @@ -3,6 +3,7 @@ net9.0 enable + enable Microsoft.DotNet.ImageBuilder.Models diff --git a/src/ImageBuilder/ViewModel/ImageInfo.cs b/src/ImageBuilder/ViewModel/ImageInfo.cs index 219bd7ab..46e533ae 100644 --- a/src/ImageBuilder/ViewModel/ImageInfo.cs +++ b/src/ImageBuilder/ViewModel/ImageInfo.cs @@ -54,7 +54,11 @@ public static ImageInfo Create( .Select(platform => PlatformInfo.Create(platform, fullRepoModelName, repoName, variableHelper, baseDirectory)) .ToArray(); - string? productVersion = variableHelper.SubstituteValues(model.ProductVersion); + string? productVersion = model.ProductVersion; + if (productVersion is not null) + { + productVersion = variableHelper.SubstituteValues(productVersion); + } IEnumerable filteredPlatformModels = manifestFilter.FilterPlatforms(model.Platforms, productVersion); IEnumerable filteredPlatforms = allPlatforms diff --git a/src/ImageBuilder/ViewModel/PlatformInfo.cs b/src/ImageBuilder/ViewModel/PlatformInfo.cs index cdfe8417..ee3922cb 100644 --- a/src/ImageBuilder/ViewModel/PlatformInfo.cs +++ b/src/ImageBuilder/ViewModel/PlatformInfo.cs @@ -28,7 +28,7 @@ public class PlatformInfo private IEnumerable _internalRepos = Enumerable.Empty(); public string BaseOsVersion { get; private set; } - public IDictionary BuildArgs { get; private set; } = ImmutableDictionary.Empty; + public IDictionary BuildArgs { get; private set; } = ImmutableDictionary.Empty; public string BuildContextPath { get; private set; } public string DockerfilePath { get; private set; } public string DockerfilePathRelativeToManifest { get; private set; } @@ -101,7 +101,7 @@ public void Initialize(IEnumerable internalRepos, string registry) Name = group.Name, Type = group.Type, Dependencies = group.Dependencies - .Select(dependency => VariableHelper.SubstituteValues(dependency)) + .Select(dependency => VariableHelper.SubstituteValues(dependency)!) .ToArray() }) .ToDictionary(info => info.Name) diff --git a/src/ImageBuilder/ViewModel/VariableHelper.cs b/src/ImageBuilder/ViewModel/VariableHelper.cs index de58a817..8ccdcba5 100644 --- a/src/ImageBuilder/ViewModel/VariableHelper.cs +++ b/src/ImageBuilder/ViewModel/VariableHelper.cs @@ -34,7 +34,7 @@ public VariableHelper(Manifest manifest, IManifestOptionsInfo options, Func kvp in Manifest.Variables) + foreach (KeyValuePair kvp in Manifest.Variables) { string? variableValue; if (Options.Variables is not null && Options.Variables.TryGetValue(kvp.Key, out string? overridenValue)) @@ -56,22 +56,17 @@ public VariableHelper(Manifest manifest, IManifestOptionsInfo options, Func kvp in Options.Variables) { - if (!ResolvedVariables.ContainsKey(kvp.Key)) + if (!ResolvedVariables.ContainsKey(kvp.Key) && kvp.Value is not null) { - string? value = SubstituteValues(kvp.Value); + string value = SubstituteValues(kvp.Value); ResolvedVariables.Add(kvp.Key, value); } } } } - public string? SubstituteValues(string? expression, Func? getContextBasedSystemValue = null) + public string SubstituteValues(string expression, Func? getContextBasedSystemValue = null) { - if (expression == null) - { - return null; - } - foreach (Match match in Regex.Matches(expression, s_tagVariablePattern)) { string? variableValue; From 417d9aff5e6bb77c8af220ade89497282712134a Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 15 Sep 2025 18:28:23 -0700 Subject: [PATCH 07/38] Upgrade ImageBuilder to .NET 10 --- .devcontainer/devcontainer.json | 2 +- eng/common/Install-DotNetSdk.ps1 | 2 +- eng/common/templates/jobs/cg-build-projects.yml | 2 +- src/Dockerfile.linux | 4 ++-- src/Dockerfile.windows | 2 +- .../Microsoft.DotNet.ImageBuilder.Models.csproj | 2 +- .../Microsoft.DotNet.ImageBuilder.Tests.csproj | 2 +- src/ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj | 5 +---- 8 files changed, 9 insertions(+), 12 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 515d699f..eaf40475 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "image": "mcr.microsoft.com/dotnet/sdk:9.0-noble", + "image": "mcr.microsoft.com/dotnet/sdk:10.0-noble", "features": { "ghcr.io/devcontainers/features/common-utils": { "username": "app", diff --git a/eng/common/Install-DotNetSdk.ps1 b/eng/common/Install-DotNetSdk.ps1 index 114faa26..ce2696f8 100644 --- a/eng/common/Install-DotNetSdk.ps1 +++ b/eng/common/Install-DotNetSdk.ps1 @@ -20,7 +20,7 @@ param( [string] $InstallPath, [string] - $Channel = "9.0" + $Channel = "10.0" ) Set-StrictMode -Version Latest diff --git a/eng/common/templates/jobs/cg-build-projects.yml b/eng/common/templates/jobs/cg-build-projects.yml index de372c4c..71280498 100644 --- a/eng/common/templates/jobs/cg-build-projects.yml +++ b/eng/common/templates/jobs/cg-build-projects.yml @@ -10,7 +10,7 @@ parameters: # See https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script#options for possible Channel values - name: dotnetVersionChannel type: string - default: '9.0' + default: '10.0' displayName: .NET Version jobs: diff --git a/src/Dockerfile.linux b/src/Dockerfile.linux index 98d9a35a..1f2f0848 100644 --- a/src/Dockerfile.linux +++ b/src/Dockerfile.linux @@ -3,7 +3,7 @@ # docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v :/repo -w /repo image-builder # build Microsoft.DotNet.ImageBuilder -FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0-azurelinux3.0 AS build-env +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:10.0-azurelinux3.0 AS build-env ARG TARGETARCH # download oras package tarball @@ -28,7 +28,7 @@ RUN dotnet publish -r linux-$TARGETARCH ./ImageBuilder/Microsoft.DotNet.ImageBui # build runtime image -FROM mcr.microsoft.com/dotnet/runtime-deps:9.0-azurelinux3.0 +FROM mcr.microsoft.com/dotnet/runtime-deps:10.0-azurelinux3.0 # install tooling RUN tdnf install -y \ diff --git a/src/Dockerfile.windows b/src/Dockerfile.windows index 31b98729..d3b8f074 100644 --- a/src/Dockerfile.windows +++ b/src/Dockerfile.windows @@ -4,7 +4,7 @@ ARG WINDOWS_BASE ARG WINDOWS_SDK # build Microsoft.DotNet.ImageBuilder -FROM mcr.microsoft.com/dotnet/sdk:9.0-$WINDOWS_SDK AS build-env +FROM mcr.microsoft.com/dotnet/sdk:10.0-$WINDOWS_SDK AS build-env WORKDIR /image-builder # restore packages before copying entire source - provides optimizations when rebuilding diff --git a/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj b/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj index ea7bf34f..5c95f5c4 100644 --- a/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj +++ b/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj @@ -1,7 +1,7 @@  - net9.0 + net10.0 enable enable Microsoft.DotNet.ImageBuilder.Models diff --git a/src/ImageBuilder.Tests/Microsoft.DotNet.ImageBuilder.Tests.csproj b/src/ImageBuilder.Tests/Microsoft.DotNet.ImageBuilder.Tests.csproj index c629747c..81e78355 100644 --- a/src/ImageBuilder.Tests/Microsoft.DotNet.ImageBuilder.Tests.csproj +++ b/src/ImageBuilder.Tests/Microsoft.DotNet.ImageBuilder.Tests.csproj @@ -1,7 +1,7 @@ - net9.0 + net10.0 false true true diff --git a/src/ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj b/src/ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj index 098d0dce..55841cc2 100644 --- a/src/ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj +++ b/src/ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj @@ -3,7 +3,7 @@ Exe False - net9.0 + net10.0 Microsoft.DotNet.ImageBuilder true true @@ -30,12 +30,9 @@ - - - From d98761c6fdf7873ce8c9d6da048fa13be9f9a208 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Mon, 15 Sep 2025 22:07:46 -0700 Subject: [PATCH 08/38] WIP add templating library and CLI --- Microsoft.DotNet.DockerTools.slnx | 2 ++ ...otNet.ImageBuilder.TemplateGenerator.csproj | 18 ++++++++++++++++++ src/ImageBuilder.TemplateGenerator/Program.cs | 9 +++++++++ .../TemplateGeneratorCli.cs | 15 +++++++++++++++ src/ImageBuilder.Templating/Class1.cs | 5 +++++ ...osoft.DotNet.ImageBuilder.Templating.csproj | 13 +++++++++++++ 6 files changed, 62 insertions(+) create mode 100644 src/ImageBuilder.TemplateGenerator/Microsoft.DotNet.ImageBuilder.TemplateGenerator.csproj create mode 100644 src/ImageBuilder.TemplateGenerator/Program.cs create mode 100644 src/ImageBuilder.TemplateGenerator/TemplateGeneratorCli.cs create mode 100644 src/ImageBuilder.Templating/Class1.cs create mode 100644 src/ImageBuilder.Templating/Microsoft.DotNet.ImageBuilder.Templating.csproj diff --git a/Microsoft.DotNet.DockerTools.slnx b/Microsoft.DotNet.DockerTools.slnx index 0c8d9669..e4b786e1 100644 --- a/Microsoft.DotNet.DockerTools.slnx +++ b/Microsoft.DotNet.DockerTools.slnx @@ -9,4 +9,6 @@ + + diff --git a/src/ImageBuilder.TemplateGenerator/Microsoft.DotNet.ImageBuilder.TemplateGenerator.csproj b/src/ImageBuilder.TemplateGenerator/Microsoft.DotNet.ImageBuilder.TemplateGenerator.csproj new file mode 100644 index 00000000..f5d1fab8 --- /dev/null +++ b/src/ImageBuilder.TemplateGenerator/Microsoft.DotNet.ImageBuilder.TemplateGenerator.csproj @@ -0,0 +1,18 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/ImageBuilder.TemplateGenerator/Program.cs b/src/ImageBuilder.TemplateGenerator/Program.cs new file mode 100644 index 00000000..e77de69c --- /dev/null +++ b/src/ImageBuilder.TemplateGenerator/Program.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.ImageBuilder.TemplateGenerator; +using ConsoleAppFramework; + +var app = ConsoleApp.Create(); +app.Add(); +app.Run(args); diff --git a/src/ImageBuilder.TemplateGenerator/TemplateGeneratorCli.cs b/src/ImageBuilder.TemplateGenerator/TemplateGeneratorCli.cs new file mode 100644 index 00000000..9152d09e --- /dev/null +++ b/src/ImageBuilder.TemplateGenerator/TemplateGeneratorCli.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using ConsoleAppFramework; + +namespace Microsoft.DotNet.ImageBuilder.TemplateGenerator; + +internal sealed class TemplateGeneratorCli +{ + [Command("generate-dockerfiles")] + public async Task GenerateDockerfiles(string manifestPath) + { + var manifestJson = await File.ReadAllTextAsync(manifestPath); + } +} diff --git a/src/ImageBuilder.Templating/Class1.cs b/src/ImageBuilder.Templating/Class1.cs new file mode 100644 index 00000000..0a63fdcf --- /dev/null +++ b/src/ImageBuilder.Templating/Class1.cs @@ -0,0 +1,5 @@ +namespace Microsoft.DotNet.ImageBuilder.Templating; + +public sealed class DockerfileGenerator +{ +} diff --git a/src/ImageBuilder.Templating/Microsoft.DotNet.ImageBuilder.Templating.csproj b/src/ImageBuilder.Templating/Microsoft.DotNet.ImageBuilder.Templating.csproj new file mode 100644 index 00000000..4a96f5a9 --- /dev/null +++ b/src/ImageBuilder.Templating/Microsoft.DotNet.ImageBuilder.Templating.csproj @@ -0,0 +1,13 @@ + + + + net10.0 + enable + enable + + + + + + + From 1bed2d06aa01ab8319ec630160a2495aaeaa2b6b Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Tue, 16 Sep 2025 16:43:10 -0700 Subject: [PATCH 09/38] WIP: Add read model / view model to models library --- ...icrosoft.DotNet.ImageBuilder.Models.csproj | 6 +- .../ReadModel/ManifestInfo.cs | 89 +++++++++++++++++++ .../ReadModel/VariableHelper.cs | 77 ++++++++++++++++ 3 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 src/ImageBuilder.Models/ReadModel/ManifestInfo.cs create mode 100644 src/ImageBuilder.Models/ReadModel/VariableHelper.cs diff --git a/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj b/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj index 5c95f5c4..c36ea73d 100644 --- a/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj +++ b/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj @@ -4,11 +4,15 @@ net10.0 enable enable - Microsoft.DotNet.ImageBuilder.Models + Microsoft.DotNet.ImageBuilder + + + + diff --git a/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs new file mode 100644 index 00000000..b4004f06 --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs @@ -0,0 +1,89 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using System.Text.Json; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +public sealed record ManifestInfo(Manifest Model, ImmutableList Repos) +{ + public static ManifestInfo Create(Manifest model) + { + var repoInfos = model.Repos.Select(RepoInfo.Create).ToImmutableList(); + return new ManifestInfo(model, repoInfos); + } +} + +public sealed record RepoInfo(Repo Model, ImmutableList Images) +{ + public static RepoInfo Create(Repo model) + { + var imageInfos = model.Images.Select(ImageInfo.Create).ToImmutableList(); + return new RepoInfo(model, imageInfos); + } +} + +public sealed record ImageInfo(Image Model, ImmutableList Platforms) +{ + public static ImageInfo Create(Image model) + { + var platformInfos = model.Platforms.Select(PlatformInfo.Create).ToImmutableList(); + return new ImageInfo(model, platformInfos); + } +} + +public sealed record PlatformInfo(Platform Model) +{ + public static PlatformInfo Create(Platform model) + { + return new PlatformInfo(model); + } +} + +internal static class ManifestJsonHelper +{ + public static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true + }; +} + +public static class ManifestInfoExtensions +{ + extension(ManifestInfo manifestInfo) + { + public static async Task LoadAsync(string manifestPath) + { + Manifest manifest = await Manifest.LoadFromFileAsync(manifestPath); + return ManifestInfo.Create(manifest); + } + + public static ManifestInfo Deserialize(string manifestPath) + { + Manifest manifest = Manifest.Deserialize(manifestPath); + return ManifestInfo.Create(manifest); + } + } +} + +public static class ManifestReadModelExtensions +{ + extension(Manifest manifest) + { + public static async Task LoadFromFileAsync(string manifestPath) + { + var json = await File.ReadAllTextAsync(manifestPath); + var manifestObject = Manifest.Deserialize(json); + return manifestObject; + } + + public static Manifest Deserialize(string json) + { + return JsonSerializer.Deserialize(json, ManifestJsonHelper.JsonOptions) + ?? throw new InvalidOperationException($"Failed to deserialize manifest from content: '{json}'"); + } + } +} diff --git a/src/ImageBuilder.Models/ReadModel/VariableHelper.cs b/src/ImageBuilder.Models/ReadModel/VariableHelper.cs new file mode 100644 index 00000000..9d9a8a57 --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/VariableHelper.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.RegularExpressions; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +internal sealed partial class VariableHelper +{ + private const char BuiltInDelimiter = ':'; + private const string RepoVariableTypeId = "Repo"; + private const string VariableGroupName = "variable"; + private const string TagVariablePattern = $"\\$\\((?<{VariableGroupName}>[\\w:\\-.| ]+)\\)"; + + private Manifest Manifest { get; } + + public IDictionary ResolvedVariables { get; } = new Dictionary(); + + public VariableHelper(Manifest manifest, IEnumerable repos) + { + Manifest = manifest; + + if (Manifest.Variables is not null) + { + foreach (var (key, value) in Manifest.Variables) + { + var variableValue = SubstituteValues(value); + ResolvedVariables.Add(key, variableValue); + } + } + } + + private string SubstituteValues(string expression, Func? getContextBasedSystemValue = null) + { + foreach (Match match in TagVariableRegex.Matches(expression)) + { + string variableName = match.Groups[VariableGroupName].Value; + string? variableValue = variableName.Contains(BuiltInDelimiter) + ? GetBuiltInValue(variableName, getContextBasedSystemValue) + : GetResolvedValue(variableName); + + if (variableValue is null) + { + throw new InvalidOperationException($"A value was not found for the variable '{match.Value}'"); + } + + expression = expression.Replace(match.Value, variableValue); + } + return expression; + } + + private string? GetBuiltInValue(string variableName) + { + string[] parts = variableName.Split(BuiltInDelimiter, 2); + string variableType = parts[0]; + string remainder = parts[1]; + + if (string.Equals(variableType, RepoVariableTypeId, StringComparison.Ordinal)) + { + // Optional fallback: match by name if Ids are sparse + var byName = Repos.FirstOrDefault(r => r.Model.Name == remainder); + return byName?.QualifiedName; + } + + return getContextBasedSystemValue?.Invoke(variableType, remainder); + } + + private string? GetResolvedValue(string variableName) + { + ResolvedVariables.TryGetValue(variableName, out string? variableValue); + return variableValue; + } + + [GeneratedRegex(TagVariablePattern)] + private partial Regex TagVariableRegex { get; } +} From 6f63594ebd391ad7735ebdaa6a1888c98e3eff5d Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Wed, 17 Sep 2025 07:54:38 -0700 Subject: [PATCH 10/38] Add some backreferences to RepoInfo --- .../ReadModel/ManifestInfo.cs | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs index b4004f06..57da0c51 100644 --- a/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs +++ b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs @@ -11,34 +11,43 @@ public sealed record ManifestInfo(Manifest Model, ImmutableList Repos) { public static ManifestInfo Create(Manifest model) { - var repoInfos = model.Repos.Select(RepoInfo.Create).ToImmutableList(); + var repoInfos = model.Repos + .Select(repo => RepoInfo.Create(repo, model)) + .ToImmutableList(); + return new ManifestInfo(model, repoInfos); } } -public sealed record RepoInfo(Repo Model, ImmutableList Images) +public sealed record RepoInfo(Repo Model, Manifest Manifest, ImmutableList Images) { - public static RepoInfo Create(Repo model) + public static RepoInfo Create(Repo model, Manifest manifest) { - var imageInfos = model.Images.Select(ImageInfo.Create).ToImmutableList(); - return new RepoInfo(model, imageInfos); + var imageInfos = model.Images + .Select(image => ImageInfo.Create(image, model)) + .ToImmutableList(); + + return new RepoInfo(model, manifest, imageInfos); } } public sealed record ImageInfo(Image Model, ImmutableList Platforms) { - public static ImageInfo Create(Image model) + public static ImageInfo Create(Image model, Repo repo) { - var platformInfos = model.Platforms.Select(PlatformInfo.Create).ToImmutableList(); + var platformInfos = model.Platforms + .Select(platform => PlatformInfo.Create(platform, model)) + .ToImmutableList(); + return new ImageInfo(model, platformInfos); } } -public sealed record PlatformInfo(Platform Model) +public sealed record PlatformInfo(Platform Model, Image Image) { - public static PlatformInfo Create(Platform model) + public static PlatformInfo Create(Platform model, Image image) { - return new PlatformInfo(model); + return new PlatformInfo(model, image); } } From 5b12450b9b36c6d82f02ad98edd8637ca27c4e48 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Wed, 17 Sep 2025 08:49:02 -0700 Subject: [PATCH 11/38] WIP on variable store --- src/ImageBuilder.Models/Manifest/Manifest.cs | 2 +- ...icrosoft.DotNet.ImageBuilder.Models.csproj | 4 - .../ReadModel/ManifestInfo.cs | 10 +++ .../ReadModel/VariableHelper.cs | 77 ------------------- .../ReadModel/VariableStore.cs | 65 ++++++++++++++++ 5 files changed, 76 insertions(+), 82 deletions(-) delete mode 100644 src/ImageBuilder.Models/ReadModel/VariableHelper.cs create mode 100644 src/ImageBuilder.Models/ReadModel/VariableStore.cs diff --git a/src/ImageBuilder.Models/Manifest/Manifest.cs b/src/ImageBuilder.Models/Manifest/Manifest.cs index d31e2e6b..db7a78a8 100644 --- a/src/ImageBuilder.Models/Manifest/Manifest.cs +++ b/src/ImageBuilder.Models/Manifest/Manifest.cs @@ -34,7 +34,7 @@ public class Manifest [Description( "The set of Docker repositories described by this manifest." )] - public Repo[] Repos { get; set; } = Array.Empty(); + public Repo[] Repos { get; set; } = []; [Description( "A set of custom variables that can be referenced in various parts of the " + diff --git a/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj b/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj index c36ea73d..1b5d9485 100644 --- a/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj +++ b/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj @@ -11,8 +11,4 @@ - - - - diff --git a/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs index 57da0c51..1790bf53 100644 --- a/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs +++ b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs @@ -9,6 +9,16 @@ namespace Microsoft.DotNet.ImageBuilder.ReadModel; public sealed record ManifestInfo(Manifest Model, ImmutableList Repos) { + private readonly ImmutableDictionary _reposById = + Repos.Where(repo => repo.Model.Id is not null) + .ToImmutableDictionary(repo => repo.Model.Id!); + + private readonly ImmutableDictionary _reposByName = + Repos.ToImmutableDictionary(repo => repo.Model.Name); + + public RepoInfo? GetRepoById(string id) => _reposById.GetValueOrDefault(id); + public RepoInfo? GetRepoByName(string name) => _reposByName.GetValueOrDefault(name); + public static ManifestInfo Create(Manifest model) { var repoInfos = model.Repos diff --git a/src/ImageBuilder.Models/ReadModel/VariableHelper.cs b/src/ImageBuilder.Models/ReadModel/VariableHelper.cs deleted file mode 100644 index 9d9a8a57..00000000 --- a/src/ImageBuilder.Models/ReadModel/VariableHelper.cs +++ /dev/null @@ -1,77 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Text.RegularExpressions; -using Microsoft.DotNet.ImageBuilder.Models.Manifest; - -namespace Microsoft.DotNet.ImageBuilder.ReadModel; - -internal sealed partial class VariableHelper -{ - private const char BuiltInDelimiter = ':'; - private const string RepoVariableTypeId = "Repo"; - private const string VariableGroupName = "variable"; - private const string TagVariablePattern = $"\\$\\((?<{VariableGroupName}>[\\w:\\-.| ]+)\\)"; - - private Manifest Manifest { get; } - - public IDictionary ResolvedVariables { get; } = new Dictionary(); - - public VariableHelper(Manifest manifest, IEnumerable repos) - { - Manifest = manifest; - - if (Manifest.Variables is not null) - { - foreach (var (key, value) in Manifest.Variables) - { - var variableValue = SubstituteValues(value); - ResolvedVariables.Add(key, variableValue); - } - } - } - - private string SubstituteValues(string expression, Func? getContextBasedSystemValue = null) - { - foreach (Match match in TagVariableRegex.Matches(expression)) - { - string variableName = match.Groups[VariableGroupName].Value; - string? variableValue = variableName.Contains(BuiltInDelimiter) - ? GetBuiltInValue(variableName, getContextBasedSystemValue) - : GetResolvedValue(variableName); - - if (variableValue is null) - { - throw new InvalidOperationException($"A value was not found for the variable '{match.Value}'"); - } - - expression = expression.Replace(match.Value, variableValue); - } - return expression; - } - - private string? GetBuiltInValue(string variableName) - { - string[] parts = variableName.Split(BuiltInDelimiter, 2); - string variableType = parts[0]; - string remainder = parts[1]; - - if (string.Equals(variableType, RepoVariableTypeId, StringComparison.Ordinal)) - { - // Optional fallback: match by name if Ids are sparse - var byName = Repos.FirstOrDefault(r => r.Model.Name == remainder); - return byName?.QualifiedName; - } - - return getContextBasedSystemValue?.Invoke(variableType, remainder); - } - - private string? GetResolvedValue(string variableName) - { - ResolvedVariables.TryGetValue(variableName, out string? variableValue); - return variableValue; - } - - [GeneratedRegex(TagVariablePattern)] - private partial Regex TagVariableRegex { get; } -} diff --git a/src/ImageBuilder.Models/ReadModel/VariableStore.cs b/src/ImageBuilder.Models/ReadModel/VariableStore.cs new file mode 100644 index 00000000..ef0c2118 --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/VariableStore.cs @@ -0,0 +1,65 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using System.Text.RegularExpressions; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +internal sealed partial class VariableStore +{ + private const string VariableGroupName = "variable"; + + private ImmutableDictionary _resolvedVariables { get; } + + public VariableStore(IDictionary variables) + { + var resolvedVariables = new Dictionary(); + foreach (var (variable, unresolvedValue) in variables) + { + string resolvedValue = ResolveInnerVariables(unresolvedValue); + resolvedVariables.Add(variable, resolvedValue); + } + + _resolvedVariables = resolvedVariables.ToImmutableDictionary(); + } + + public string? Get(string variableName) => GetResolvedValue(variableName); + + /// + /// Evaluates an expression and replaces any variables inside with their + /// fully-resolved values. + /// + /// + /// Variable references inside this expression will be replaced. + /// + private string ResolveInnerVariables(string expression) + { + var subVariableMatches = TagVariableRegex.Matches(expression); + foreach (Match match in subVariableMatches) + { + string variableName = match.Groups[VariableGroupName].Value; + string? variableValue = GetResolvedValue(variableName) + ?? throw new InvalidOperationException($"A value was not found for the variable '{match.Value}'"); + expression = expression.Replace(match.Value, variableValue); + } + + return expression; + } + + /// + /// Get the resolved value for a variable name. Returns null if the + /// variable is not found. + /// + /// + /// The variable value, or null if the variable is not found + /// + private string? GetResolvedValue(string variableName) + { + _resolvedVariables.TryGetValue(variableName, out string? variableValue); + return variableValue; + } + + [GeneratedRegex($"\\$\\((?<{VariableGroupName}>[\\w:\\-.| ]+)\\)")] + private static partial Regex TagVariableRegex { get; } +} From 644de115ee5fdf644829a96fa5d49409d1d5c519 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Wed, 17 Sep 2025 16:06:45 -0700 Subject: [PATCH 12/38] Fix warnings from manifest model --- src/ImageBuilder.Models/Manifest/Manifest.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ImageBuilder.Models/Manifest/Manifest.cs b/src/ImageBuilder.Models/Manifest/Manifest.cs index db7a78a8..2262b8c6 100644 --- a/src/ImageBuilder.Models/Manifest/Manifest.cs +++ b/src/ImageBuilder.Models/Manifest/Manifest.cs @@ -19,17 +19,17 @@ public class Manifest "Additional json files to be loaded with this manifest. This is a convienent" + "way to split the manifest apart into logical parts." )] - public string[] Includes { get; set; } + public string[] Includes { get; set; } = []; [Description( "Info about the readme that documents the product family." )] - public Readme Readme { get; set; } + public Readme? Readme { get; set; } [Description( "The location of the Docker registry where the images are to be published." )] - public string Registry { get; set; } + public string? Registry { get; set; } [Description( "The set of Docker repositories described by this manifest." From e56bdbe581a6832c1e97d53acf33a2feb84f4bd2 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Wed, 17 Sep 2025 16:13:19 -0700 Subject: [PATCH 13/38] Read manifest from file and correctly resolve variables and includes --- .../ReadModel/EmptyVariableStore.cs | 9 ++ .../ReadModel/IVariableStore.cs | 9 ++ .../ReadModel/JsonHelper.cs | 28 ++++++ .../ReadModel/JsonNodeExtensions.cs | 73 +++++++++++++++ .../ReadModel/ManifestInfo.cs | 56 ----------- .../ReadModel/ManifestInfoExtensions.cs | 66 +++++++++++++ .../ReadModel/ManifestPreprocessor.cs | 93 +++++++++++++++++++ .../ReadModel/VariableStore.cs | 22 +++-- .../TemplateGeneratorCli.cs | 11 ++- 9 files changed, 299 insertions(+), 68 deletions(-) create mode 100644 src/ImageBuilder.Models/ReadModel/EmptyVariableStore.cs create mode 100644 src/ImageBuilder.Models/ReadModel/IVariableStore.cs create mode 100644 src/ImageBuilder.Models/ReadModel/JsonHelper.cs create mode 100644 src/ImageBuilder.Models/ReadModel/JsonNodeExtensions.cs create mode 100644 src/ImageBuilder.Models/ReadModel/ManifestInfoExtensions.cs create mode 100644 src/ImageBuilder.Models/ReadModel/ManifestPreprocessor.cs diff --git a/src/ImageBuilder.Models/ReadModel/EmptyVariableStore.cs b/src/ImageBuilder.Models/ReadModel/EmptyVariableStore.cs new file mode 100644 index 00000000..c8932683 --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/EmptyVariableStore.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +internal sealed class EmptyVariableStore : IVariableStore +{ + public string ResolveInnerVariables(string expression) => expression; +} diff --git a/src/ImageBuilder.Models/ReadModel/IVariableStore.cs b/src/ImageBuilder.Models/ReadModel/IVariableStore.cs new file mode 100644 index 00000000..307c8b50 --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/IVariableStore.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +internal interface IVariableStore +{ + string ResolveInnerVariables(string expression); +} diff --git a/src/ImageBuilder.Models/ReadModel/JsonHelper.cs b/src/ImageBuilder.Models/ReadModel/JsonHelper.cs new file mode 100644 index 00000000..a1595043 --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/JsonHelper.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +internal static class JsonHelper +{ + private static readonly JsonSerializerOptions s_jsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + Converters = + { + new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, allowIntegerValues: false), + }, + NumberHandling = JsonNumberHandling.AllowReadingFromString, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault + }; + + public static T Deserialize(JsonNode jsonNode) => JsonSerializer.Deserialize(jsonNode, s_jsonOptions) + ?? throw new Exception($"Failed to deserialize JSON object to {typeof(T)}."); + + public static string Serialize(T model) => JsonSerializer.Serialize(model, s_jsonOptions); +} diff --git a/src/ImageBuilder.Models/ReadModel/JsonNodeExtensions.cs b/src/ImageBuilder.Models/ReadModel/JsonNodeExtensions.cs new file mode 100644 index 00000000..1d2fe388 --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/JsonNodeExtensions.cs @@ -0,0 +1,73 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Nodes; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +internal static class JsonNodeExtensions +{ + extension(JsonNode baseNode) + { + // Based on https://gist.github.com/cajuncoding/bf78bdcf790782090d231590cbc2438f + public JsonNode Merge(JsonNode incomingNode) + { + switch (baseNode) + { + case JsonObject baseObject when incomingNode is JsonObject incomingObject: + MergeObjects(baseObject, incomingObject); + break; + case JsonArray baseArray when incomingNode is JsonArray incomingArray: + MergeArrays(baseArray, incomingArray); + break; + default: + throw new ArgumentException( + $"The JsonNode type [{baseNode.GetType().Name}] is incompatible for" + + $" merging with the target/base type {incomingNode.GetType().Name}." + + " Merging requires the types to be the same." + ); + } + + return baseNode; + } + } + + private static void MergeObjects(JsonObject baseObject, JsonObject incomingObject) + { + // Clear object so that the child elements no longer have a parent + var incomingObjectSnapshot = incomingObject.ToArray(); + incomingObject.Clear(); + + foreach (KeyValuePair incomingProperty in incomingObjectSnapshot) + { + var baseObjectValue = baseObject[incomingProperty.Key]; + baseObject[incomingProperty.Key] = baseObjectValue switch + { + // If both are JsonObjects, merge them recursively + JsonObject baseChildObject when incomingProperty.Value is JsonObject incomingChildObject => + baseChildObject.Merge(incomingChildObject), + + // If both are JsonArrays, merge them recursively + JsonArray baseChildArray when incomingProperty.Value is JsonArray incomingChildArray => + baseChildArray.Merge(incomingChildArray), + + // If the base property and incoming property are of different + // types, or if the base property does not exist, overwrite + // with the incoming property. + _ => incomingProperty.Value + }; + } + } + + private static void MergeArrays(JsonArray baseArray, JsonArray incomingArray) + { + // Clear array so that the child elements no longer have a parent + var incomingArraySnapshot = incomingArray.ToArray(); + incomingArray.Clear(); + + foreach (JsonNode? incomingElement in incomingArraySnapshot) + { + baseArray.Add(incomingElement); + } + } +} diff --git a/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs index 1790bf53..4e6564e8 100644 --- a/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs +++ b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Immutable; -using System.Text.Json; using Microsoft.DotNet.ImageBuilder.Models.Manifest; namespace Microsoft.DotNet.ImageBuilder.ReadModel; @@ -18,15 +17,6 @@ public sealed record ManifestInfo(Manifest Model, ImmutableList Repos) public RepoInfo? GetRepoById(string id) => _reposById.GetValueOrDefault(id); public RepoInfo? GetRepoByName(string name) => _reposByName.GetValueOrDefault(name); - - public static ManifestInfo Create(Manifest model) - { - var repoInfos = model.Repos - .Select(repo => RepoInfo.Create(repo, model)) - .ToImmutableList(); - - return new ManifestInfo(model, repoInfos); - } } public sealed record RepoInfo(Repo Model, Manifest Manifest, ImmutableList Images) @@ -60,49 +50,3 @@ public static PlatformInfo Create(Platform model, Image image) return new PlatformInfo(model, image); } } - -internal static class ManifestJsonHelper -{ - public static readonly JsonSerializerOptions JsonOptions = new() - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - WriteIndented = true - }; -} - -public static class ManifestInfoExtensions -{ - extension(ManifestInfo manifestInfo) - { - public static async Task LoadAsync(string manifestPath) - { - Manifest manifest = await Manifest.LoadFromFileAsync(manifestPath); - return ManifestInfo.Create(manifest); - } - - public static ManifestInfo Deserialize(string manifestPath) - { - Manifest manifest = Manifest.Deserialize(manifestPath); - return ManifestInfo.Create(manifest); - } - } -} - -public static class ManifestReadModelExtensions -{ - extension(Manifest manifest) - { - public static async Task LoadFromFileAsync(string manifestPath) - { - var json = await File.ReadAllTextAsync(manifestPath); - var manifestObject = Manifest.Deserialize(json); - return manifestObject; - } - - public static Manifest Deserialize(string json) - { - return JsonSerializer.Deserialize(json, ManifestJsonHelper.JsonOptions) - ?? throw new InvalidOperationException($"Failed to deserialize manifest from content: '{json}'"); - } - } -} diff --git a/src/ImageBuilder.Models/ReadModel/ManifestInfoExtensions.cs b/src/ImageBuilder.Models/ReadModel/ManifestInfoExtensions.cs new file mode 100644 index 00000000..7b2d5e7b --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/ManifestInfoExtensions.cs @@ -0,0 +1,66 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using System.Text.Json.Nodes; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; +using static Microsoft.DotNet.ImageBuilder.ReadModel.JsonHelper; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +public static class ManifestInfoExtensions +{ + extension(ManifestInfo manifestInfo) + { + public static async Task LoadAsync(string manifestJsonPath) + { + var manifestJsonObject = await LoadModelFromFileAsync(manifestJsonPath); + var manifestDir = Path.GetDirectoryName(manifestJsonPath) ?? ""; + + // Load and deserialize included files + IEnumerable includesJsonNodes = []; + var includesNode = manifestJsonObject["includes"]; + if (includesNode is not null) + { + IEnumerable includesFiles = Deserialize(includesNode); + includesJsonNodes = await Task.WhenAll( + includesFiles + // Make includes paths relative to the manifest file + .Select(includesFile => Path.Combine(manifestDir, includesFile)) + .Select(LoadModelFromFileAsync)); + } + + var preprocessor = new ManifestPreprocessor(); + var processedRootJsonNode = preprocessor.Process(manifestJsonObject, includesJsonNodes); + var processedModel = Deserialize(processedRootJsonNode); + + return ManifestInfo.Create(processedModel); + } + + public string ToJsonString() => Serialize(manifestInfo.Model); + + internal static ManifestInfo Create(Manifest model) + { + var repoInfos = model.Repos + .Select(repo => RepoInfo.Create(repo, model)) + .ToImmutableList(); + + return new ManifestInfo(model, repoInfos); + } + } + + private static async Task LoadModelFromFileAsync(string manifestJsonPath) + { + var jsonStream = File.OpenRead(manifestJsonPath); + var rootJsonNode = await JsonNode.ParseAsync(jsonStream) + ?? throw new Exception( + $"Failed to parse manifest JSON from file: {manifestJsonPath}"); + + if (rootJsonNode is not JsonObject rootJsonObject) + { + throw new InvalidDataException($"Manifest root must be a JSON object."); + } + + return rootJsonObject; + } +} diff --git a/src/ImageBuilder.Models/ReadModel/ManifestPreprocessor.cs b/src/ImageBuilder.Models/ReadModel/ManifestPreprocessor.cs new file mode 100644 index 00000000..7e20b424 --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/ManifestPreprocessor.cs @@ -0,0 +1,93 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Nodes; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +internal sealed class ManifestPreprocessor +{ + private IVariableStore _variableStore = new EmptyVariableStore(); + + public JsonNode Process(JsonObject root, IEnumerable includesNodes) + { + // Process includes first so variables in included files can be processed + ProcessIncludes(root, includesNodes); + + var rawManifest = JsonHelper.Deserialize(root); + var variables = rawManifest.Variables ?? new Dictionary(); + + // Add variables for each repo name (e.g. "Repo:dotnet" -> "mcr.microsoft.com/dotnet") + foreach (var kvp in rawManifest.RepoVariables) + { + variables.Add(kvp); + } + + _variableStore = new VariableStore(variables); + + ProcessVariables(root); + return root; + } + + /// + /// Replace keys and string values in JSON that reference variables defined + /// in the variable store. + /// + private void ProcessVariables(JsonNode? node) + { + switch (node) + { + case JsonObject jsonObject: + var jsonObjectSnapshot = jsonObject.ToList(); + foreach ((string oldKey, JsonNode? value) in jsonObjectSnapshot) + { + var newKey = _variableStore.ResolveInnerVariables(oldKey); + if (newKey != oldKey) + { + jsonObject.Remove(oldKey); + jsonObject[newKey] = value; + } + + ProcessVariables(value); + } + break; + + case JsonArray jsonArray: + for (int i = 0; i < jsonArray.Count; i++) + { + ProcessVariables(jsonArray[i]); + } + break; + + case JsonValue jsonValue: + if (jsonValue.TryGetValue(out string? stringValue)) + { + jsonValue.ReplaceWith(_variableStore.ResolveInnerVariables(stringValue)); + } + break; + + case null: + break; + } + } + + private static void ProcessIncludes(JsonObject jsonObject, IEnumerable includes) + { + foreach (JsonObject includeObject in includes) + { + jsonObject.Merge(includeObject); + } + } +} + +internal static class ManifestRepoVariableExtensions +{ + extension(Manifest manifest) + { + public IEnumerable> RepoVariables => + manifest.Repos + .Where(repo => !string.IsNullOrWhiteSpace(repo.Id)) + .Select(repo => new KeyValuePair($"Repo:{repo.Id}", repo.Name)); + } +} diff --git a/src/ImageBuilder.Models/ReadModel/VariableStore.cs b/src/ImageBuilder.Models/ReadModel/VariableStore.cs index ef0c2118..20b2ea46 100644 --- a/src/ImageBuilder.Models/ReadModel/VariableStore.cs +++ b/src/ImageBuilder.Models/ReadModel/VariableStore.cs @@ -1,31 +1,33 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections.Immutable; using System.Text.RegularExpressions; namespace Microsoft.DotNet.ImageBuilder.ReadModel; -internal sealed partial class VariableStore +internal sealed partial class VariableStore : IVariableStore { private const string VariableGroupName = "variable"; - private ImmutableDictionary _resolvedVariables { get; } + private readonly Dictionary _resolvedVariables; + /// + /// Creates a new . + /// + /// + /// The order of variable definitions determines which order they are + /// resolved in. Variable references must come after their definition. + /// public VariableStore(IDictionary variables) { - var resolvedVariables = new Dictionary(); + _resolvedVariables = new Dictionary(); foreach (var (variable, unresolvedValue) in variables) { string resolvedValue = ResolveInnerVariables(unresolvedValue); - resolvedVariables.Add(variable, resolvedValue); + _resolvedVariables.Add(variable, resolvedValue); } - - _resolvedVariables = resolvedVariables.ToImmutableDictionary(); } - public string? Get(string variableName) => GetResolvedValue(variableName); - /// /// Evaluates an expression and replaces any variables inside with their /// fully-resolved values. @@ -33,7 +35,7 @@ public VariableStore(IDictionary variables) /// /// Variable references inside this expression will be replaced. /// - private string ResolveInnerVariables(string expression) + public string ResolveInnerVariables(string expression) { var subVariableMatches = TagVariableRegex.Matches(expression); foreach (Match match in subVariableMatches) diff --git a/src/ImageBuilder.TemplateGenerator/TemplateGeneratorCli.cs b/src/ImageBuilder.TemplateGenerator/TemplateGeneratorCli.cs index 9152d09e..446de833 100644 --- a/src/ImageBuilder.TemplateGenerator/TemplateGeneratorCli.cs +++ b/src/ImageBuilder.TemplateGenerator/TemplateGeneratorCli.cs @@ -2,14 +2,21 @@ // The .NET Foundation licenses this file to you under the MIT license. using ConsoleAppFramework; +using Microsoft.DotNet.ImageBuilder.ReadModel; namespace Microsoft.DotNet.ImageBuilder.TemplateGenerator; internal sealed class TemplateGeneratorCli { + /// + /// Generates Dockerfiles from a manifest file. + /// + /// Path to manifest JSON file [Command("generate-dockerfiles")] - public async Task GenerateDockerfiles(string manifestPath) + public async Task GenerateDockerfiles([Argument] string manifestPath) { - var manifestJson = await File.ReadAllTextAsync(manifestPath); + ManifestInfo manifest = await ManifestInfo.LoadAsync(manifestPath); + var manifestString = manifest.ToJsonString(); + await File.WriteAllTextAsync("manifest.processed.json", manifestString); } } From 25d20a2b06f02fe6f91c03c5cbb898f73b35ce4d Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Wed, 17 Sep 2025 18:56:53 -0700 Subject: [PATCH 14/38] Disassociate Templating projects from ImageBuilder namespace --- Microsoft.DotNet.DockerTools.slnx | 9 +++++++-- src/ImageBuilder.Templating/Class1.cs | 5 ----- .../Program.cs | 2 +- .../TemplateGenerator.csproj} | 3 ++- .../TemplateGeneratorCli.cs | 2 +- src/Templating/Class1.cs | 8 ++++++++ .../Templating.csproj} | 1 + 7 files changed, 20 insertions(+), 10 deletions(-) delete mode 100644 src/ImageBuilder.Templating/Class1.cs rename src/{ImageBuilder.TemplateGenerator => TemplateGenerator}/Program.cs (81%) rename src/{ImageBuilder.TemplateGenerator/Microsoft.DotNet.ImageBuilder.TemplateGenerator.csproj => TemplateGenerator/TemplateGenerator.csproj} (71%) rename src/{ImageBuilder.TemplateGenerator => TemplateGenerator}/TemplateGeneratorCli.cs (92%) create mode 100644 src/Templating/Class1.cs rename src/{ImageBuilder.Templating/Microsoft.DotNet.ImageBuilder.Templating.csproj => Templating/Templating.csproj} (82%) diff --git a/Microsoft.DotNet.DockerTools.slnx b/Microsoft.DotNet.DockerTools.slnx index e4b786e1..a927cc9f 100644 --- a/Microsoft.DotNet.DockerTools.slnx +++ b/Microsoft.DotNet.DockerTools.slnx @@ -1,14 +1,19 @@ + + + - - + + + + diff --git a/src/ImageBuilder.Templating/Class1.cs b/src/ImageBuilder.Templating/Class1.cs deleted file mode 100644 index 0a63fdcf..00000000 --- a/src/ImageBuilder.Templating/Class1.cs +++ /dev/null @@ -1,5 +0,0 @@ -namespace Microsoft.DotNet.ImageBuilder.Templating; - -public sealed class DockerfileGenerator -{ -} diff --git a/src/ImageBuilder.TemplateGenerator/Program.cs b/src/TemplateGenerator/Program.cs similarity index 81% rename from src/ImageBuilder.TemplateGenerator/Program.cs rename to src/TemplateGenerator/Program.cs index e77de69c..6b96b365 100644 --- a/src/ImageBuilder.TemplateGenerator/Program.cs +++ b/src/TemplateGenerator/Program.cs @@ -1,7 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.DotNet.ImageBuilder.TemplateGenerator; +using Microsoft.DotNet.DockerTools.TemplateGenerator; using ConsoleAppFramework; var app = ConsoleApp.Create(); diff --git a/src/ImageBuilder.TemplateGenerator/Microsoft.DotNet.ImageBuilder.TemplateGenerator.csproj b/src/TemplateGenerator/TemplateGenerator.csproj similarity index 71% rename from src/ImageBuilder.TemplateGenerator/Microsoft.DotNet.ImageBuilder.TemplateGenerator.csproj rename to src/TemplateGenerator/TemplateGenerator.csproj index f5d1fab8..214cb54b 100644 --- a/src/ImageBuilder.TemplateGenerator/Microsoft.DotNet.ImageBuilder.TemplateGenerator.csproj +++ b/src/TemplateGenerator/TemplateGenerator.csproj @@ -5,6 +5,7 @@ net10.0 enable enable + Microsoft.DotNet.DockerTools.TemplateGenerator @@ -12,7 +13,7 @@ - + diff --git a/src/ImageBuilder.TemplateGenerator/TemplateGeneratorCli.cs b/src/TemplateGenerator/TemplateGeneratorCli.cs similarity index 92% rename from src/ImageBuilder.TemplateGenerator/TemplateGeneratorCli.cs rename to src/TemplateGenerator/TemplateGeneratorCli.cs index 446de833..36b0ff99 100644 --- a/src/ImageBuilder.TemplateGenerator/TemplateGeneratorCli.cs +++ b/src/TemplateGenerator/TemplateGeneratorCli.cs @@ -4,7 +4,7 @@ using ConsoleAppFramework; using Microsoft.DotNet.ImageBuilder.ReadModel; -namespace Microsoft.DotNet.ImageBuilder.TemplateGenerator; +namespace Microsoft.DotNet.DockerTools.TemplateGenerator; internal sealed class TemplateGeneratorCli { diff --git a/src/Templating/Class1.cs b/src/Templating/Class1.cs new file mode 100644 index 00000000..d39fa887 --- /dev/null +++ b/src/Templating/Class1.cs @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.DockerTools.Templating; + +public sealed class DockerfileGenerator +{ +} diff --git a/src/ImageBuilder.Templating/Microsoft.DotNet.ImageBuilder.Templating.csproj b/src/Templating/Templating.csproj similarity index 82% rename from src/ImageBuilder.Templating/Microsoft.DotNet.ImageBuilder.Templating.csproj rename to src/Templating/Templating.csproj index 4a96f5a9..3041c794 100644 --- a/src/ImageBuilder.Templating/Microsoft.DotNet.ImageBuilder.Templating.csproj +++ b/src/Templating/Templating.csproj @@ -4,6 +4,7 @@ net10.0 enable enable + Microsoft.DotNet.DockerTools.Templating From da4d45a26c9e8d0adfb20124938f7a6c436a7f15 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Wed, 17 Sep 2025 22:03:08 -0700 Subject: [PATCH 15/38] Add benchmark for round-trip serialization --- .gitignore | 3 +++ Microsoft.DotNet.DockerTools.slnx | 1 + src/TemplateGenerator.Benchmarks/Program.cs | 24 +++++++++++++++++++ .../TemplateGenerator.Benchmarks.csproj | 19 +++++++++++++++ 4 files changed, 47 insertions(+) create mode 100644 src/TemplateGenerator.Benchmarks/Program.cs create mode 100644 src/TemplateGenerator.Benchmarks/TemplateGenerator.Benchmarks.csproj diff --git a/.gitignore b/.gitignore index 2dfc1016..0b71563a 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ # Test files *.trx + +# BenchmarkDotNet Artifacts +BenchmarkDotNet.Artifacts/ diff --git a/Microsoft.DotNet.DockerTools.slnx b/Microsoft.DotNet.DockerTools.slnx index a927cc9f..753147ab 100644 --- a/Microsoft.DotNet.DockerTools.slnx +++ b/Microsoft.DotNet.DockerTools.slnx @@ -15,5 +15,6 @@ + diff --git a/src/TemplateGenerator.Benchmarks/Program.cs b/src/TemplateGenerator.Benchmarks/Program.cs new file mode 100644 index 00000000..baa79948 --- /dev/null +++ b/src/TemplateGenerator.Benchmarks/Program.cs @@ -0,0 +1,24 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Running; +using Microsoft.DotNet.ImageBuilder.ReadModel; + +namespace Microsoft.DotNet.DockerTools.TemplateGenerator; + +[MemoryDiagnoser] +public class Benchmarks +{ + private static readonly string s_manifestPath = + Environment.GetEnvironmentVariable("MANIFEST_PATH") ?? "manifest.json"; + + [Benchmark] + public async Task RoundTripManifestSerialization() + { + ManifestInfo manifest = await ManifestInfo.LoadAsync(s_manifestPath); + return manifest.ToJsonString(); + } + + public static void Main(string[] args) + { + var summary = BenchmarkRunner.Run(); + } +} diff --git a/src/TemplateGenerator.Benchmarks/TemplateGenerator.Benchmarks.csproj b/src/TemplateGenerator.Benchmarks/TemplateGenerator.Benchmarks.csproj new file mode 100644 index 00000000..d168def0 --- /dev/null +++ b/src/TemplateGenerator.Benchmarks/TemplateGenerator.Benchmarks.csproj @@ -0,0 +1,19 @@ + + + + Exe + net10.0 + enable + enable + Microsoft.DotNet.DockerTools.TemplateGenerator.Benchmarks + + + + + + + + + + + From 156394a968df32a64803d5b090abaa026b29eeef Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 18 Sep 2025 10:21:46 -0700 Subject: [PATCH 16/38] Add simple templating abstractions and cottle implementation --- .../ManifestBenchmarks.cs | 21 ++++++++ src/TemplateGenerator.Benchmarks/Program.cs | 27 ++-------- .../SimpleTemplateBenchmarks.cs | 16 ++++++ src/TemplateGenerator/TemplateGeneratorCli.cs | 51 ++++++++++++++++++- .../Abstractions/ICompiledTemplate.cs | 9 ++++ .../Abstractions/ITemplateEngine.cs | 9 ++++ src/Templating/Class1.cs | 8 --- src/Templating/Cottle/CottleTemplate.cs | 17 +++++++ src/Templating/Cottle/CottleTemplateEngine.cs | 27 ++++++++++ src/Templating/Templating.csproj | 5 ++ 10 files changed, 159 insertions(+), 31 deletions(-) create mode 100644 src/TemplateGenerator.Benchmarks/ManifestBenchmarks.cs create mode 100644 src/TemplateGenerator.Benchmarks/SimpleTemplateBenchmarks.cs create mode 100644 src/Templating/Abstractions/ICompiledTemplate.cs create mode 100644 src/Templating/Abstractions/ITemplateEngine.cs delete mode 100644 src/Templating/Class1.cs create mode 100644 src/Templating/Cottle/CottleTemplate.cs create mode 100644 src/Templating/Cottle/CottleTemplateEngine.cs diff --git a/src/TemplateGenerator.Benchmarks/ManifestBenchmarks.cs b/src/TemplateGenerator.Benchmarks/ManifestBenchmarks.cs new file mode 100644 index 00000000..263ea23e --- /dev/null +++ b/src/TemplateGenerator.Benchmarks/ManifestBenchmarks.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using BenchmarkDotNet.Attributes; +using Microsoft.DotNet.ImageBuilder.ReadModel; + +namespace Microsoft.DotNet.DockerTools.TemplateGenerator.Benchmarks; + +[MemoryDiagnoser] +public class ManifestBenchmarks +{ + private static readonly string s_manifestPath = + Environment.GetEnvironmentVariable("MANIFEST_PATH") ?? "manifest.json"; + + [Benchmark] + public async Task RoundTripManifestSerialization() + { + ManifestInfo manifest = await ManifestInfo.LoadAsync(s_manifestPath); + return manifest.ToJsonString(); + } +} diff --git a/src/TemplateGenerator.Benchmarks/Program.cs b/src/TemplateGenerator.Benchmarks/Program.cs index baa79948..08918f76 100644 --- a/src/TemplateGenerator.Benchmarks/Program.cs +++ b/src/TemplateGenerator.Benchmarks/Program.cs @@ -1,24 +1,7 @@ -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Running; -using Microsoft.DotNet.ImageBuilder.ReadModel; - -namespace Microsoft.DotNet.DockerTools.TemplateGenerator; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. -[MemoryDiagnoser] -public class Benchmarks -{ - private static readonly string s_manifestPath = - Environment.GetEnvironmentVariable("MANIFEST_PATH") ?? "manifest.json"; - - [Benchmark] - public async Task RoundTripManifestSerialization() - { - ManifestInfo manifest = await ManifestInfo.LoadAsync(s_manifestPath); - return manifest.ToJsonString(); - } +using BenchmarkDotNet.Running; +using Microsoft.DotNet.DockerTools.TemplateGenerator.Benchmarks; - public static void Main(string[] args) - { - var summary = BenchmarkRunner.Run(); - } -} +var summary = BenchmarkRunner.Run(); diff --git a/src/TemplateGenerator.Benchmarks/SimpleTemplateBenchmarks.cs b/src/TemplateGenerator.Benchmarks/SimpleTemplateBenchmarks.cs new file mode 100644 index 00000000..2cebd5ff --- /dev/null +++ b/src/TemplateGenerator.Benchmarks/SimpleTemplateBenchmarks.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using BenchmarkDotNet.Attributes; + +namespace Microsoft.DotNet.DockerTools.TemplateGenerator.Benchmarks; + +[MemoryDiagnoser] +public class SimpleTemplateBenchmarks +{ + [Benchmark] + public string GenerateSimpleTemplate() + { + return TemplateGenerator.GenerateCottleTemplate(); + } +} diff --git a/src/TemplateGenerator/TemplateGeneratorCli.cs b/src/TemplateGenerator/TemplateGeneratorCli.cs index 36b0ff99..9cc6b306 100644 --- a/src/TemplateGenerator/TemplateGeneratorCli.cs +++ b/src/TemplateGenerator/TemplateGeneratorCli.cs @@ -3,10 +3,12 @@ using ConsoleAppFramework; using Microsoft.DotNet.ImageBuilder.ReadModel; +using Microsoft.DotNet.DockerTools.Templating; +using Cottle; namespace Microsoft.DotNet.DockerTools.TemplateGenerator; -internal sealed class TemplateGeneratorCli +public sealed class TemplateGeneratorCli { /// /// Generates Dockerfiles from a manifest file. @@ -19,4 +21,51 @@ public async Task GenerateDockerfiles([Argument] string manifestPath) var manifestString = manifest.ToJsonString(); await File.WriteAllTextAsync("manifest.processed.json", manifestString); } + + [Command("generate-template")] + public void GenerateTemplate() + { + var result = TemplateGenerator.GenerateCottleTemplate(); + Console.WriteLine(result); + } +} + +public static class TemplateGenerator +{ + public static string GenerateCottleTemplate() + { + const string TemplateString = + """ + FROM Repo:2.1-{{OS_VERSION_BASE}} + ENV TEST1 {{if OS_VERSION = "trixie-slim":IfWorks}} + ENV TEST2 {{VARIABLES["Variable1"]}} + """; + + const string ExpectedOutput = + """ + FROM Repo:2.1-trixie + ENV TEST1 IfWorks + ENV TEST2 Value1 + """; + + ITemplateEngine engine = new CottleTemplateEngine(); + + var predefinedVariables = new Dictionary + { + { "OS_VERSION", "trixie-slim" }, + { "OS_VERSION_BASE", "trixie" }, + { + "VARIABLES", + new Dictionary + { + { "Variable1", "Value1" } + } + } + }; + + var context = Context.CreateBuiltin(predefinedVariables); + var template = engine.Compile(TemplateString); + string result = template.Render(context); + return result; + } } diff --git a/src/Templating/Abstractions/ICompiledTemplate.cs b/src/Templating/Abstractions/ICompiledTemplate.cs new file mode 100644 index 00000000..2a3cef0a --- /dev/null +++ b/src/Templating/Abstractions/ICompiledTemplate.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.DockerTools.Templating.Abstractions; + +public interface ICompiledTemplate +{ + string Render(TContext context); +} diff --git a/src/Templating/Abstractions/ITemplateEngine.cs b/src/Templating/Abstractions/ITemplateEngine.cs new file mode 100644 index 00000000..f830c974 --- /dev/null +++ b/src/Templating/Abstractions/ITemplateEngine.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.DockerTools.Templating.Abstractions; + +public interface ITemplateEngine +{ + ICompiledTemplate Compile(string template); +} diff --git a/src/Templating/Class1.cs b/src/Templating/Class1.cs deleted file mode 100644 index d39fa887..00000000 --- a/src/Templating/Class1.cs +++ /dev/null @@ -1,8 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace Microsoft.DotNet.DockerTools.Templating; - -public sealed class DockerfileGenerator -{ -} diff --git a/src/Templating/Cottle/CottleTemplate.cs b/src/Templating/Cottle/CottleTemplate.cs new file mode 100644 index 00000000..d03fdbde --- /dev/null +++ b/src/Templating/Cottle/CottleTemplate.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Cottle; +using Microsoft.DotNet.DockerTools.Templating.Abstractions; + +namespace Microsoft.DotNet.DockerTools.Templating.Cottle; + +public sealed class CottleTemplate(IDocument document) : ICompiledTemplate +{ + private readonly IDocument _document = document; + + public string Render(IContext context) + { + return _document.Render(context); + } +} diff --git a/src/Templating/Cottle/CottleTemplateEngine.cs b/src/Templating/Cottle/CottleTemplateEngine.cs new file mode 100644 index 00000000..ecf2c5a3 --- /dev/null +++ b/src/Templating/Cottle/CottleTemplateEngine.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Cottle; +using Microsoft.DotNet.DockerTools.Templating.Abstractions; + +namespace Microsoft.DotNet.DockerTools.Templating.Cottle; + +public sealed class CottleTemplateEngine : ITemplateEngine +{ + private static readonly DocumentConfiguration s_config = new() + { + BlockBegin = "{{", + BlockContinue = "^", + BlockEnd = "}}", + Escape = '@', + Trimmer = DocumentConfiguration.TrimNothing + }; + + public ICompiledTemplate Compile(string template) + { + var documentResult = Document.CreateDefault(template, s_config); + var document = documentResult.DocumentOrThrow; + var compiledTemplate = new CottleTemplate(document); + return compiledTemplate; + } +} diff --git a/src/Templating/Templating.csproj b/src/Templating/Templating.csproj index 3041c794..8171c295 100644 --- a/src/Templating/Templating.csproj +++ b/src/Templating/Templating.csproj @@ -11,4 +11,9 @@ + + + + + From 07f4243c9a1293ad9f0a71a7fbeba3f656f5d141 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 18 Sep 2025 10:52:03 -0700 Subject: [PATCH 17/38] Separate context based variables and predefined variables --- src/TemplateGenerator/TemplateGeneratorCli.cs | 35 +++++++++---------- .../Abstractions/ITemplateEngine.cs | 1 + .../Cottle/CottleContextExtensions.cs | 30 ++++++++++++++++ src/Templating/Cottle/CottleTemplateEngine.cs | 17 +++++++++ 4 files changed, 65 insertions(+), 18 deletions(-) create mode 100644 src/Templating/Cottle/CottleContextExtensions.cs diff --git a/src/TemplateGenerator/TemplateGeneratorCli.cs b/src/TemplateGenerator/TemplateGeneratorCli.cs index 9cc6b306..b9a1bd71 100644 --- a/src/TemplateGenerator/TemplateGeneratorCli.cs +++ b/src/TemplateGenerator/TemplateGeneratorCli.cs @@ -3,8 +3,9 @@ using ConsoleAppFramework; using Microsoft.DotNet.ImageBuilder.ReadModel; -using Microsoft.DotNet.DockerTools.Templating; +using Microsoft.DotNet.DockerTools.Templating.Abstractions; using Cottle; +using Microsoft.DotNet.DockerTools.Templating.Cottle; namespace Microsoft.DotNet.DockerTools.TemplateGenerator; @@ -41,31 +42,29 @@ public static string GenerateCottleTemplate() ENV TEST2 {{VARIABLES["Variable1"]}} """; - const string ExpectedOutput = - """ - FROM Repo:2.1-trixie - ENV TEST1 IfWorks - ENV TEST2 Value1 - """; + // Expected output: + // FROM Repo:2.1-trixie + // ENV TEST1 IfWorks + // ENV TEST2 Value1 + + var engine = new CottleTemplateEngine(); - ITemplateEngine engine = new CottleTemplateEngine(); + var predefinedVariables = new Dictionary() + { + { "Variable1", "Value1" } + }; - var predefinedVariables = new Dictionary + var contextBasedVariables = new Dictionary() { { "OS_VERSION", "trixie-slim" }, { "OS_VERSION_BASE", "trixie" }, - { - "VARIABLES", - new Dictionary - { - { "Variable1", "Value1" } - } - } }; - var context = Context.CreateBuiltin(predefinedVariables); + var templateContext = engine.CreateVariableContext(predefinedVariables); + templateContext.Add(contextBasedVariables); + var template = engine.Compile(TemplateString); - string result = template.Render(context); + string result = template.Render(templateContext); return result; } } diff --git a/src/Templating/Abstractions/ITemplateEngine.cs b/src/Templating/Abstractions/ITemplateEngine.cs index f830c974..312f6f6a 100644 --- a/src/Templating/Abstractions/ITemplateEngine.cs +++ b/src/Templating/Abstractions/ITemplateEngine.cs @@ -6,4 +6,5 @@ namespace Microsoft.DotNet.DockerTools.Templating.Abstractions; public interface ITemplateEngine { ICompiledTemplate Compile(string template); + TContext CreateContext(IReadOnlyDictionary variables); } diff --git a/src/Templating/Cottle/CottleContextExtensions.cs b/src/Templating/Cottle/CottleContextExtensions.cs new file mode 100644 index 00000000..611a8d34 --- /dev/null +++ b/src/Templating/Cottle/CottleContextExtensions.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Cottle; + +namespace Microsoft.DotNet.DockerTools.Templating.Cottle; + +public static class CottleContextExtensions +{ + extension(IContext context) + { + public IContext Add(Dictionary variables) + { + var variablesDictionary = variables.ToCottleDictionary(); + var newContext = Context.CreateBuiltin(variablesDictionary); + return Context.CreateCascade(newContext, context); + } + } + + extension(IReadOnlyDictionary stringDictionary) + { + public Dictionary ToCottleDictionary() + { + return stringDictionary.ToDictionary( + kv => (Value)kv.Key, + kv => (Value)kv.Value + ); + } + } +} diff --git a/src/Templating/Cottle/CottleTemplateEngine.cs b/src/Templating/Cottle/CottleTemplateEngine.cs index ecf2c5a3..27140cc2 100644 --- a/src/Templating/Cottle/CottleTemplateEngine.cs +++ b/src/Templating/Cottle/CottleTemplateEngine.cs @@ -24,4 +24,21 @@ public ICompiledTemplate Compile(string template) var compiledTemplate = new CottleTemplate(document); return compiledTemplate; } + + public IContext CreateContext(IReadOnlyDictionary variables) + { + var symbols = variables.ToCottleDictionary(); + return Context.CreateBuiltin(symbols); + } + + public IContext CreateVariableContext(Dictionary variables) + { + var variableValues = variables.ToCottleDictionary(); + var symbols = new Dictionary + { + { "VARIABLES", variableValues } + }; + + return Context.CreateBuiltin(symbols); + } } From 6d03594eb4f87a9e7a849bba467674cd7630d33e Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 18 Sep 2025 11:01:58 -0700 Subject: [PATCH 18/38] Move ManifestInfo serialization to separate namespace --- .../ManifestInfoSerializationExtensions.cs} | 4 ++-- src/TemplateGenerator.Benchmarks/ManifestBenchmarks.cs | 1 + src/TemplateGenerator/TemplateGeneratorCli.cs | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) rename src/ImageBuilder.Models/ReadModel/{ManifestInfoExtensions.cs => Serialization/ManifestInfoSerializationExtensions.cs} (95%) diff --git a/src/ImageBuilder.Models/ReadModel/ManifestInfoExtensions.cs b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs similarity index 95% rename from src/ImageBuilder.Models/ReadModel/ManifestInfoExtensions.cs rename to src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs index 7b2d5e7b..d8470cf7 100644 --- a/src/ImageBuilder.Models/ReadModel/ManifestInfoExtensions.cs +++ b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs @@ -6,9 +6,9 @@ using Microsoft.DotNet.ImageBuilder.Models.Manifest; using static Microsoft.DotNet.ImageBuilder.ReadModel.JsonHelper; -namespace Microsoft.DotNet.ImageBuilder.ReadModel; +namespace Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; -public static class ManifestInfoExtensions +public static class ManifestInfoSerializationExtensions { extension(ManifestInfo manifestInfo) { diff --git a/src/TemplateGenerator.Benchmarks/ManifestBenchmarks.cs b/src/TemplateGenerator.Benchmarks/ManifestBenchmarks.cs index 263ea23e..1d91a3ff 100644 --- a/src/TemplateGenerator.Benchmarks/ManifestBenchmarks.cs +++ b/src/TemplateGenerator.Benchmarks/ManifestBenchmarks.cs @@ -3,6 +3,7 @@ using BenchmarkDotNet.Attributes; using Microsoft.DotNet.ImageBuilder.ReadModel; +using Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; namespace Microsoft.DotNet.DockerTools.TemplateGenerator.Benchmarks; diff --git a/src/TemplateGenerator/TemplateGeneratorCli.cs b/src/TemplateGenerator/TemplateGeneratorCli.cs index b9a1bd71..75e1b338 100644 --- a/src/TemplateGenerator/TemplateGeneratorCli.cs +++ b/src/TemplateGenerator/TemplateGeneratorCli.cs @@ -3,9 +3,11 @@ using ConsoleAppFramework; using Microsoft.DotNet.ImageBuilder.ReadModel; +using Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; using Microsoft.DotNet.DockerTools.Templating.Abstractions; using Cottle; using Microsoft.DotNet.DockerTools.Templating.Cottle; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; namespace Microsoft.DotNet.DockerTools.TemplateGenerator; @@ -19,8 +21,6 @@ public sealed class TemplateGeneratorCli public async Task GenerateDockerfiles([Argument] string manifestPath) { ManifestInfo manifest = await ManifestInfo.LoadAsync(manifestPath); - var manifestString = manifest.ToJsonString(); - await File.WriteAllTextAsync("manifest.processed.json", manifestString); } [Command("generate-template")] From 004a1bc92f38e5013acc090da3bb63932503a3c4 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 18 Sep 2025 11:02:17 -0700 Subject: [PATCH 19/38] Remove unused usings --- src/TemplateGenerator/TemplateGeneratorCli.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/TemplateGenerator/TemplateGeneratorCli.cs b/src/TemplateGenerator/TemplateGeneratorCli.cs index 75e1b338..cc6a2e4a 100644 --- a/src/TemplateGenerator/TemplateGeneratorCli.cs +++ b/src/TemplateGenerator/TemplateGeneratorCli.cs @@ -4,10 +4,7 @@ using ConsoleAppFramework; using Microsoft.DotNet.ImageBuilder.ReadModel; using Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; -using Microsoft.DotNet.DockerTools.Templating.Abstractions; -using Cottle; using Microsoft.DotNet.DockerTools.Templating.Cottle; -using Microsoft.DotNet.ImageBuilder.Models.Manifest; namespace Microsoft.DotNet.DockerTools.TemplateGenerator; From f8d3e4733de3b86e90be77d0f7c796647e6ebced Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 18 Sep 2025 14:36:30 -0700 Subject: [PATCH 20/38] Get template generation working --- .../ReadModel/ManifestInfo.cs | 34 ++- .../ReadModel/ManifestInfoExtensions.cs | 13 ++ .../ManifestInfoSerializationExtensions.cs | 11 +- src/TemplateGenerator/TemplateGeneratorCli.cs | 26 ++- .../Cottle/CottleContextExtensions.cs | 217 +++++++++++++++++- 5 files changed, 276 insertions(+), 25 deletions(-) create mode 100644 src/ImageBuilder.Models/ReadModel/ManifestInfoExtensions.cs diff --git a/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs index 4e6564e8..eeea09b9 100644 --- a/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs +++ b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs @@ -6,7 +6,7 @@ namespace Microsoft.DotNet.ImageBuilder.ReadModel; -public sealed record ManifestInfo(Manifest Model, ImmutableList Repos) +public sealed record ManifestInfo(Manifest Model, string FilePath, ImmutableList Repos) { private readonly ImmutableDictionary _reposById = Repos.Where(repo => repo.Model.Id is not null) @@ -17,14 +17,24 @@ public sealed record ManifestInfo(Manifest Model, ImmutableList Repos) public RepoInfo? GetRepoById(string id) => _reposById.GetValueOrDefault(id); public RepoInfo? GetRepoByName(string name) => _reposByName.GetValueOrDefault(name); + + internal static ManifestInfo Create(Manifest model, string manifestFilePath) + { + var manifestDir = Path.GetDirectoryName(manifestFilePath) ?? ""; + var repoInfos = model.Repos + .Select(repo => RepoInfo.Create(repo, model, manifestDir)) + .ToImmutableList(); + + return new ManifestInfo(model, manifestFilePath, repoInfos); + } } public sealed record RepoInfo(Repo Model, Manifest Manifest, ImmutableList Images) { - public static RepoInfo Create(Repo model, Manifest manifest) + internal static RepoInfo Create(Repo model, Manifest manifest, string manifestDir) { var imageInfos = model.Images - .Select(image => ImageInfo.Create(image, model)) + .Select(image => ImageInfo.Create(image, model, manifestDir)) .ToImmutableList(); return new RepoInfo(model, manifest, imageInfos); @@ -33,20 +43,28 @@ public static RepoInfo Create(Repo model, Manifest manifest) public sealed record ImageInfo(Image Model, ImmutableList Platforms) { - public static ImageInfo Create(Image model, Repo repo) + internal static ImageInfo Create(Image model, Repo repo, string manifestDir) { var platformInfos = model.Platforms - .Select(platform => PlatformInfo.Create(platform, model)) + .Select(platform => PlatformInfo.Create(platform, model, manifestDir)) .ToImmutableList(); return new ImageInfo(model, platformInfos); } } -public sealed record PlatformInfo(Platform Model, Image Image) +public sealed record PlatformInfo( + Platform Model, + Image Image, + string DockerfilePath, + string? DockerfileTemplatePath = null) { - public static PlatformInfo Create(Platform model, Image image) + internal static PlatformInfo Create(Platform model, Image image, string manifestDir) { - return new PlatformInfo(model, image); + var dockerfilePath = Path.Combine(manifestDir, model.Dockerfile, "Dockerfile"); + var dockerfileTemplatePath = model.DockerfileTemplate is not null + ? Path.Combine(manifestDir, model.DockerfileTemplate) : null; + + return new PlatformInfo(model, image, dockerfilePath, dockerfileTemplatePath); } } diff --git a/src/ImageBuilder.Models/ReadModel/ManifestInfoExtensions.cs b/src/ImageBuilder.Models/ReadModel/ManifestInfoExtensions.cs new file mode 100644 index 00000000..67db1a6d --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/ManifestInfoExtensions.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +public static class ManifestInfoExtensions +{ + extension(ManifestInfo manifest) + { + public IEnumerable AllImages => manifest.Repos.SelectMany(repo => repo.Images); + public IEnumerable AllPlatforms => manifest.AllImages.SelectMany(image => image.Platforms); + } +} diff --git a/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs index d8470cf7..8018e9ec 100644 --- a/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs +++ b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs @@ -34,19 +34,10 @@ public static async Task LoadAsync(string manifestJsonPath) var processedRootJsonNode = preprocessor.Process(manifestJsonObject, includesJsonNodes); var processedModel = Deserialize(processedRootJsonNode); - return ManifestInfo.Create(processedModel); + return ManifestInfo.Create(processedModel, manifestJsonPath); } public string ToJsonString() => Serialize(manifestInfo.Model); - - internal static ManifestInfo Create(Manifest model) - { - var repoInfos = model.Repos - .Select(repo => RepoInfo.Create(repo, model)) - .ToImmutableList(); - - return new ManifestInfo(model, repoInfos); - } } private static async Task LoadModelFromFileAsync(string manifestJsonPath) diff --git a/src/TemplateGenerator/TemplateGeneratorCli.cs b/src/TemplateGenerator/TemplateGeneratorCli.cs index cc6a2e4a..7a6483b3 100644 --- a/src/TemplateGenerator/TemplateGeneratorCli.cs +++ b/src/TemplateGenerator/TemplateGeneratorCli.cs @@ -18,13 +18,27 @@ public sealed class TemplateGeneratorCli public async Task GenerateDockerfiles([Argument] string manifestPath) { ManifestInfo manifest = await ManifestInfo.LoadAsync(manifestPath); - } - [Command("generate-template")] - public void GenerateTemplate() - { - var result = TemplateGenerator.GenerateCottleTemplate(); - Console.WriteLine(result); + var engine = new CottleTemplateEngine(); + var globalContext = engine.CreateContext(new Dictionary()); + + var platformsWithTemplates = manifest.AllPlatforms + .Where(platform => platform.DockerfileTemplatePath is not null); + + var compiledTemplates = platformsWithTemplates + .Select(platform => platform.DockerfileTemplatePath!) + .Select(File.ReadAllText) + .Select(engine.Compile); + + var compiledTemplateInfos = platformsWithTemplates + .Zip(compiledTemplates); + + foreach (var (platform, compiledTemplate) in compiledTemplateInfos) + { + var platformSpecificContext = globalContext.Add(platform.PlatformSpecificTemplateVariables); + var output = compiledTemplate.Render(platformSpecificContext); + File.WriteAllText(platform.DockerfilePath, output); + } } } diff --git a/src/Templating/Cottle/CottleContextExtensions.cs b/src/Templating/Cottle/CottleContextExtensions.cs index 611a8d34..41e92f7c 100644 --- a/src/Templating/Cottle/CottleContextExtensions.cs +++ b/src/Templating/Cottle/CottleContextExtensions.cs @@ -2,6 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. using Cottle; +using Microsoft.DotNet.ImageBuilder.ReadModel; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; +using System.Text.RegularExpressions; +using System.Diagnostics.CodeAnalysis; namespace Microsoft.DotNet.DockerTools.Templating.Cottle; @@ -9,7 +13,7 @@ public static class CottleContextExtensions { extension(IContext context) { - public IContext Add(Dictionary variables) + public IContext Add(IReadOnlyDictionary variables) { var variablesDictionary = variables.ToCottleDictionary(); var newContext = Context.CreateBuiltin(variablesDictionary); @@ -28,3 +32,214 @@ public Dictionary ToCottleDictionary() } } } + +public static class PlatformInfoVariableExtensions +{ + extension(PlatformInfo platform) + { + public IReadOnlyDictionary PlatformSpecificTemplateVariables => + new Dictionary() + { + { "ARCH_SHORT", platform.Model.Architecture.ShortName }, + { "ARCH_NUPKG", platform.Model.Architecture.NupkgName }, + { "ARCH_VERSIONED", platform.ArchWithVariant }, + { "ARCH_TAG_SUFFIX", $"-{platform.ArchWithVariant}" }, + { "PRODUCT_VERSION", platform.Image.ProductVersion ?? "" }, + { "OS_VERSION", platform.Model.OsVersion }, + { "OS_VERSION_BASE", "" }, + { "OS_VERSION_NUMBER", platform.GetOsVersionNumber() }, + { "OS_ARCH_HYPHENATED", platform.GetOsArchHyphenatedName() }, + }; + } +} + +internal static partial class PlatformInfoExtensions +{ + extension(PlatformInfo platform) + { + public string ArchWithVariant => platform.Model.Architecture.LongName + platform.ArchVariant; + public string ArchVariant => platform.Model.Variant?.ToLowerInvariant() ?? ""; + public string BaseOsVersion => platform.Model.OsVersion.TrimEndString("-slim"); + + public string GetOsVersionNumber() + { + const string PrefixGroup = "Prefix"; + const string VersionGroup = "Version"; + const string LtscPrefix = "ltsc"; + Match match = OsVersionRegex.Match(platform.Model.OsVersion); + + string versionNumber = string.Empty; + if (match.Groups[PrefixGroup].Success && match.Groups[PrefixGroup].Value == LtscPrefix) + { + versionNumber = LtscPrefix; + } + + versionNumber += match.Groups[VersionGroup].Value; + return versionNumber; + } + + public string GetOsArchHyphenatedName() + { + string osName; + if (platform.BaseOsVersion.Contains("nanoserver")) + { + string version = platform.BaseOsVersion.Split('-')[1]; + osName = $"NanoServer-{version}"; + } + else if (platform.BaseOsVersion.Contains("windowsservercore")) + { + string version = platform.BaseOsVersion.Split('-')[1]; + osName = $"WindowsServerCore-{version}"; + } + else + { + osName = platform.OSDisplayName.Replace(' ', '-'); + } + + string archName = platform.Model.Architecture != Architecture.AMD64 + ? $"-{platform.Model.Architecture.GetDisplayName()}" + : string.Empty; + + return osName + archName; + } + + private string OSDisplayName => platform.Model.OS switch + { + OS.Windows => GetWindowsOSDisplayName(platform.BaseOsVersion), + _ => GetLinuxOSDisplayName(platform.BaseOsVersion) + }; + } + + extension(Architecture architecture) + { + public string ShortName => architecture switch + { + Architecture.AMD64 => "x64", + _ => architecture.ToString().ToLowerInvariant(), + }; + + public string NupkgName => architecture switch + { + Architecture.AMD64 => "x64", + Architecture.ARM => "arm32", + _ => architecture.ToString().ToLowerInvariant(), + }; + + public string LongName => architecture switch + { + Architecture.ARM => "arm32", + _ => architecture.ToString().ToLowerInvariant(), + }; + + private string GetDisplayName(string? variant = null) + { + string displayName = architecture switch + { + Architecture.ARM => "arm32", + _ => architecture.ToString().ToLowerInvariant(), + }; + + if (variant != null) + { + displayName += variant.ToLowerInvariant(); + } + + return displayName; + } + + public string DockerName => architecture.ToString().ToLowerInvariant(); + } + + extension(OS os) + { + public string DockerName => os.ToString().ToLowerInvariant(); + } + + private static string GetWindowsOSDisplayName(string osName) + { + string version = osName.Split('-')[1]; + return osName switch + { + var s when s.StartsWith("nanoserver") => + GetWindowsVersionDisplayName("Nano Server", version), + var s when s.StartsWith("windowsservercore") => + GetWindowsVersionDisplayName("Windows Server Core", version), + _ => throw new NotSupportedException($"The OS version '{osName}' is not supported.") + }; + } + + private static string GetLinuxOSDisplayName(string osName) => osName switch + { + string s when s.Contains("debian") => "Debian", + string s when s.Contains("bookworm") => "Debian 12", + string s when s.Contains("trixie") => "Debian 13", + string s when s.Contains("forky") => "Debian 14", + string s when s.Contains("duke") => "Debian 15", + string s when s.Contains("jammy") => "Ubuntu 22.04", + string s when s.Contains("noble") => "Ubuntu 24.04", + string s when s.Contains("azurelinux") => FormatVersionableOsName(osName, name => "Azure Linux"), + string s when s.Contains("cbl-mariner") => FormatVersionableOsName(osName, name => "CBL-Mariner"), + string s when s.Contains("leap") => FormatVersionableOsName(osName, name => "openSUSE Leap"), + string s when s.Contains("ubuntu") => FormatVersionableOsName(osName, name => "Ubuntu"), + string s when s.Contains("alpine") + || s.Contains("centos") + || s.Contains("fedora") => FormatVersionableOsName(osName, name => name.FirstCharToUpper()), + _ => throw new NotSupportedException($"The OS version '{osName}' is not supported.") + }; + + private static string GetWindowsVersionDisplayName(string windowsName, string version) => + version.StartsWith("ltsc") switch + { + true => $"{windowsName} {version.TrimStartString("ltsc")}", + false => $"{windowsName}, version {version}" + }; + + private static string FormatVersionableOsName(string os, Func formatName) + { + (string osName, string osVersion) = GetOsVersionInfo(os); + if (string.IsNullOrEmpty(osVersion)) + { + return formatName(osName); + } + else + { + return $"{formatName(osName)} {osVersion}"; + } + } + + private static (string Name, string Version) GetOsVersionInfo(string os) + { + // Regex matches an os name ending in a non-numeric or decimal character and up to + // a 3 part version number. Any additional characters are dropped (e.g. -distroless). + Regex versionRegex = new Regex(@"(?.+[^0-9\.])(?\d+(\.\d*){0,2})"); + Match match = versionRegex.Match(os); + + if (match.Success) + { + return (match.Groups["name"].Value, match.Groups["version"].Value); + } + else + { + return (os, string.Empty); + } + } + + public static string FirstCharToUpper(this string source) => char.ToUpper(source[0]) + source.Substring(1); + + [return: NotNullIfNotNull(nameof(source))] + private static string? TrimEndString(this string? source, string trim) => source switch + { + string s when s.EndsWith(trim) => s.Substring(0, s.Length - trim.Length).TrimEndString(trim), + _ => source, + }; + + [return: NotNullIfNotNull(nameof(source))] + private static string? TrimStartString(this string? source, string trim) => source switch + { + string s when s.StartsWith(trim) => s.Substring(trim.Length).TrimStartString(trim), + _ => source, + }; + + [GeneratedRegex(@"(-(?[a-zA-Z_]*))?(?\d+.\d+)")] + private static partial Regex OsVersionRegex { get; } +} From c4d2ecb89ef0ca8c82e6de2ea0ad164a7c124388 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 18 Sep 2025 20:19:18 -0700 Subject: [PATCH 21/38] Get sub-templates working without crashing --- src/TemplateGenerator.Benchmarks/Program.cs | 2 +- .../SimpleTemplateBenchmarks.cs | 16 ------ src/TemplateGenerator/TemplateGeneratorCli.cs | 53 ++++--------------- src/Templating/Abstractions/IFileSystem.cs | 10 ++++ .../Abstractions/ITemplateEngine.cs | 1 - .../Cottle/CottleContextExtensions.cs | 8 ++- src/Templating/Cottle/CottleTemplateEngine.cs | 52 +++++++++++++++--- src/Templating/FileSystem.cs | 41 ++++++++++++++ 8 files changed, 112 insertions(+), 71 deletions(-) delete mode 100644 src/TemplateGenerator.Benchmarks/SimpleTemplateBenchmarks.cs create mode 100644 src/Templating/Abstractions/IFileSystem.cs create mode 100644 src/Templating/FileSystem.cs diff --git a/src/TemplateGenerator.Benchmarks/Program.cs b/src/TemplateGenerator.Benchmarks/Program.cs index 08918f76..ab2defc6 100644 --- a/src/TemplateGenerator.Benchmarks/Program.cs +++ b/src/TemplateGenerator.Benchmarks/Program.cs @@ -4,4 +4,4 @@ using BenchmarkDotNet.Running; using Microsoft.DotNet.DockerTools.TemplateGenerator.Benchmarks; -var summary = BenchmarkRunner.Run(); +var summary = BenchmarkRunner.Run(); diff --git a/src/TemplateGenerator.Benchmarks/SimpleTemplateBenchmarks.cs b/src/TemplateGenerator.Benchmarks/SimpleTemplateBenchmarks.cs deleted file mode 100644 index 2cebd5ff..00000000 --- a/src/TemplateGenerator.Benchmarks/SimpleTemplateBenchmarks.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using BenchmarkDotNet.Attributes; - -namespace Microsoft.DotNet.DockerTools.TemplateGenerator.Benchmarks; - -[MemoryDiagnoser] -public class SimpleTemplateBenchmarks -{ - [Benchmark] - public string GenerateSimpleTemplate() - { - return TemplateGenerator.GenerateCottleTemplate(); - } -} diff --git a/src/TemplateGenerator/TemplateGeneratorCli.cs b/src/TemplateGenerator/TemplateGeneratorCli.cs index 7a6483b3..64419ceb 100644 --- a/src/TemplateGenerator/TemplateGeneratorCli.cs +++ b/src/TemplateGenerator/TemplateGeneratorCli.cs @@ -5,6 +5,7 @@ using Microsoft.DotNet.ImageBuilder.ReadModel; using Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; using Microsoft.DotNet.DockerTools.Templating.Cottle; +using Microsoft.DotNet.DockerTools.Templating; namespace Microsoft.DotNet.DockerTools.TemplateGenerator; @@ -19,63 +20,27 @@ public async Task GenerateDockerfiles([Argument] string manifestPath) { ManifestInfo manifest = await ManifestInfo.LoadAsync(manifestPath); - var engine = new CottleTemplateEngine(); - var globalContext = engine.CreateContext(new Dictionary()); + var fileSystem = new FileSystem(); + var engine = new CottleTemplateEngine(fileSystem); var platformsWithTemplates = manifest.AllPlatforms .Where(platform => platform.DockerfileTemplatePath is not null); var compiledTemplates = platformsWithTemplates .Select(platform => platform.DockerfileTemplatePath!) - .Select(File.ReadAllText) - .Select(engine.Compile); + .Select(engine.ReadAndCompile); var compiledTemplateInfos = platformsWithTemplates .Zip(compiledTemplates); foreach (var (platform, compiledTemplate) in compiledTemplateInfos) { - var platformSpecificContext = globalContext.Add(platform.PlatformSpecificTemplateVariables); - var output = compiledTemplate.Render(platformSpecificContext); - File.WriteAllText(platform.DockerfilePath, output); + var platformContext = engine.CreatePlatformContext(platform); + var output = compiledTemplate.Render(platformContext); + fileSystem.WriteAllText(platform.DockerfilePath, output); } - } -} - -public static class TemplateGenerator -{ - public static string GenerateCottleTemplate() - { - const string TemplateString = - """ - FROM Repo:2.1-{{OS_VERSION_BASE}} - ENV TEST1 {{if OS_VERSION = "trixie-slim":IfWorks}} - ENV TEST2 {{VARIABLES["Variable1"]}} - """; - - // Expected output: - // FROM Repo:2.1-trixie - // ENV TEST1 IfWorks - // ENV TEST2 Value1 - - var engine = new CottleTemplateEngine(); - - var predefinedVariables = new Dictionary() - { - { "Variable1", "Value1" } - }; - - var contextBasedVariables = new Dictionary() - { - { "OS_VERSION", "trixie-slim" }, - { "OS_VERSION_BASE", "trixie" }, - }; - - var templateContext = engine.CreateVariableContext(predefinedVariables); - templateContext.Add(contextBasedVariables); - var template = engine.Compile(TemplateString); - string result = template.Render(templateContext); - return result; + Console.WriteLine($"Read {fileSystem.FilesRead} files ({fileSystem.BytesRead} bytes)"); + Console.WriteLine($"Wrote {fileSystem.FilesWritten} files ({fileSystem.BytesWritten} bytes)"); } } diff --git a/src/Templating/Abstractions/IFileSystem.cs b/src/Templating/Abstractions/IFileSystem.cs new file mode 100644 index 00000000..f3f6332a --- /dev/null +++ b/src/Templating/Abstractions/IFileSystem.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.DockerTools.Templating.Abstractions; + +public interface IFileSystem +{ + string ReadAllText(string path); + void WriteAllText(string path, string content); +} diff --git a/src/Templating/Abstractions/ITemplateEngine.cs b/src/Templating/Abstractions/ITemplateEngine.cs index 312f6f6a..f830c974 100644 --- a/src/Templating/Abstractions/ITemplateEngine.cs +++ b/src/Templating/Abstractions/ITemplateEngine.cs @@ -6,5 +6,4 @@ namespace Microsoft.DotNet.DockerTools.Templating.Abstractions; public interface ITemplateEngine { ICompiledTemplate Compile(string template); - TContext CreateContext(IReadOnlyDictionary variables); } diff --git a/src/Templating/Cottle/CottleContextExtensions.cs b/src/Templating/Cottle/CottleContextExtensions.cs index 41e92f7c..f2bd7781 100644 --- a/src/Templating/Cottle/CottleContextExtensions.cs +++ b/src/Templating/Cottle/CottleContextExtensions.cs @@ -13,10 +13,16 @@ public static class CottleContextExtensions { extension(IContext context) { + public IContext Add(Value key, Value value) + { + var newContext = Context.CreateCustom(new Dictionary { { key, value } }); + return Context.CreateCascade(primary: newContext, fallback: context); + } + public IContext Add(IReadOnlyDictionary variables) { var variablesDictionary = variables.ToCottleDictionary(); - var newContext = Context.CreateBuiltin(variablesDictionary); + var newContext = Context.CreateCustom(variablesDictionary); return Context.CreateCascade(newContext, context); } } diff --git a/src/Templating/Cottle/CottleTemplateEngine.cs b/src/Templating/Cottle/CottleTemplateEngine.cs index 27140cc2..440f301f 100644 --- a/src/Templating/Cottle/CottleTemplateEngine.cs +++ b/src/Templating/Cottle/CottleTemplateEngine.cs @@ -3,10 +3,11 @@ using Cottle; using Microsoft.DotNet.DockerTools.Templating.Abstractions; +using Microsoft.DotNet.ImageBuilder.ReadModel; namespace Microsoft.DotNet.DockerTools.Templating.Cottle; -public sealed class CottleTemplateEngine : ITemplateEngine +public sealed class CottleTemplateEngine(IFileSystem fileSystem) : ITemplateEngine { private static readonly DocumentConfiguration s_config = new() { @@ -17,6 +18,10 @@ public sealed class CottleTemplateEngine : ITemplateEngine Trimmer = DocumentConfiguration.TrimNothing }; + private static readonly IContext s_globalContext = Context.CreateBuiltin(new Dictionary()); + + private readonly IFileSystem _fileSystem = fileSystem; + public ICompiledTemplate Compile(string template) { var documentResult = Document.CreateDefault(template, s_config); @@ -25,20 +30,51 @@ public ICompiledTemplate Compile(string template) return compiledTemplate; } - public IContext CreateContext(IReadOnlyDictionary variables) + public ICompiledTemplate ReadAndCompile(string path) { - var symbols = variables.ToCottleDictionary(); - return Context.CreateBuiltin(symbols); + string content = _fileSystem.ReadAllText(path); + return Compile(content); } - public IContext CreateVariableContext(Dictionary variables) + public IContext CreatePlatformContext(PlatformInfo platform) { - var variableValues = variables.ToCottleDictionary(); + var variables = platform.PlatformSpecificTemplateVariables.ToCottleDictionary(); var symbols = new Dictionary { - { "VARIABLES", variableValues } + { "VARIABLES", variables } }; - return Context.CreateBuiltin(symbols); + var variableContext = Context.CreateCustom(symbols); + var platformContext = Context.CreateCascade(primary: variableContext, fallback: s_globalContext); + + // It's OK for the insert template function not to have a reference to itself. Any sub-templates will have + // their own InsertTemplate function created for them when they are rendered. + var insertTemplateFunction = CreateInsertTemplateFunction(platformContext, platform.DockerfileTemplatePath!); + var fullContext = platformContext.Add("InsertTemplate", insertTemplateFunction); + + return fullContext; + } + + private Value CreateInsertTemplateFunction(IContext platformContext, string currentTemplatePath) + { + var function = Function.CreatePure( + (state, args) => + { + var templateRelativePath = args[0].AsString; + var templateArgs = args.Count > 1 ? args[1] : Value.EmptyMap; + var indent = args.Count > 2 ? args[2].AsString : ""; + + var parentTemplateDir = Path.GetDirectoryName(currentTemplatePath) ?? string.Empty; + var newTemplatePath = Path.Combine(parentTemplateDir, templateRelativePath); + var compiledTemplate = ReadAndCompile(newTemplatePath); + + var newInsertTemplateFunction = CreateInsertTemplateFunction(platformContext, newTemplatePath); + var newContext = platformContext.Add("InsertTemplate", newInsertTemplateFunction); + + return compiledTemplate.Render(newContext); + } + ); + + return Value.FromFunction(function); } } diff --git a/src/Templating/FileSystem.cs b/src/Templating/FileSystem.cs new file mode 100644 index 00000000..cc51570d --- /dev/null +++ b/src/Templating/FileSystem.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using Microsoft.DotNet.DockerTools.Templating.Abstractions; + +namespace Microsoft.DotNet.DockerTools.Templating; + +public sealed class FileSystem : IFileSystem +{ + private int _reads = 0; + private int _bytesRead = 0; + private int _writes = 0; + private int _bytesWritten = 0; + + public int FilesRead => _reads; + public int BytesRead => _bytesRead; + public int FilesWritten => _writes; + public int BytesWritten => _bytesWritten; + + public string ReadAllText(string path) + { + string content = File.ReadAllText(path); + + _bytesRead += GetBytes(content); + _reads += 1; + + return content; + } + + public void WriteAllText(string path, string content) + { + File.WriteAllText(path, content); + + _bytesWritten += GetBytes(content); + _writes += 1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int GetBytes(string content) => System.Text.Encoding.UTF8.GetByteCount(content); +} From 4e18e9a36dde1a772c7b4806c0687b4aa8bab235 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 19 Sep 2025 09:39:40 -0700 Subject: [PATCH 22/38] Fix ARGS and VARIABLES in templates --- src/TemplateGenerator/TemplateGeneratorCli.cs | 1 + .../Cottle/CottleContextExtensions.cs | 12 ++++++-- src/Templating/Cottle/CottleTemplateEngine.cs | 30 +++++++++++++------ 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/TemplateGenerator/TemplateGeneratorCli.cs b/src/TemplateGenerator/TemplateGeneratorCli.cs index 64419ceb..2a97d5a6 100644 --- a/src/TemplateGenerator/TemplateGeneratorCli.cs +++ b/src/TemplateGenerator/TemplateGeneratorCli.cs @@ -22,6 +22,7 @@ public async Task GenerateDockerfiles([Argument] string manifestPath) var fileSystem = new FileSystem(); var engine = new CottleTemplateEngine(fileSystem); + engine.AddGlobalVariables(manifest.Model.Variables); var platformsWithTemplates = manifest.AllPlatforms .Where(platform => platform.DockerfileTemplatePath is not null); diff --git a/src/Templating/Cottle/CottleContextExtensions.cs b/src/Templating/Cottle/CottleContextExtensions.cs index f2bd7781..ac242334 100644 --- a/src/Templating/Cottle/CottleContextExtensions.cs +++ b/src/Templating/Cottle/CottleContextExtensions.cs @@ -19,7 +19,13 @@ public IContext Add(Value key, Value value) return Context.CreateCascade(primary: newContext, fallback: context); } - public IContext Add(IReadOnlyDictionary variables) + public IContext Add(Dictionary symbols) + { + var newContext = Context.CreateCustom(symbols); + return Context.CreateCascade(primary: newContext, fallback: context); + } + + public IContext Add(IDictionary variables) { var variablesDictionary = variables.ToCottleDictionary(); var newContext = Context.CreateCustom(variablesDictionary); @@ -27,7 +33,7 @@ public IContext Add(IReadOnlyDictionary variables) } } - extension(IReadOnlyDictionary stringDictionary) + extension(IDictionary stringDictionary) { public Dictionary ToCottleDictionary() { @@ -43,7 +49,7 @@ public static class PlatformInfoVariableExtensions { extension(PlatformInfo platform) { - public IReadOnlyDictionary PlatformSpecificTemplateVariables => + public Dictionary PlatformSpecificTemplateVariables => new Dictionary() { { "ARCH_SHORT", platform.Model.Architecture.ShortName }, diff --git a/src/Templating/Cottle/CottleTemplateEngine.cs b/src/Templating/Cottle/CottleTemplateEngine.cs index 440f301f..19840bcc 100644 --- a/src/Templating/Cottle/CottleTemplateEngine.cs +++ b/src/Templating/Cottle/CottleTemplateEngine.cs @@ -18,7 +18,7 @@ public sealed class CottleTemplateEngine(IFileSystem fileSystem) : ITemplateEngi Trimmer = DocumentConfiguration.TrimNothing }; - private static readonly IContext s_globalContext = Context.CreateBuiltin(new Dictionary()); + private IContext _globalContext = Context.CreateBuiltin(new Dictionary()); private readonly IFileSystem _fileSystem = fileSystem; @@ -36,16 +36,22 @@ public ICompiledTemplate ReadAndCompile(string path) return Compile(content); } - public IContext CreatePlatformContext(PlatformInfo platform) + public void AddGlobalVariables(IDictionary variables) { - var variables = platform.PlatformSpecificTemplateVariables.ToCottleDictionary(); - var symbols = new Dictionary + var variableSymbols = new Dictionary { - { "VARIABLES", variables } + { "VARIABLES", variables.ToCottleDictionary() } }; - var variableContext = Context.CreateCustom(symbols); - var platformContext = Context.CreateCascade(primary: variableContext, fallback: s_globalContext); + _globalContext = _globalContext.Add(variableSymbols); + } + + public IContext CreatePlatformContext(PlatformInfo platform) + { + var platformVariables = platform.PlatformSpecificTemplateVariables.ToCottleDictionary(); + + var variableContext = Context.CreateCustom(platformVariables); + var platformContext = Context.CreateCascade(primary: variableContext, fallback: _globalContext); // It's OK for the insert template function not to have a reference to itself. Any sub-templates will have // their own InsertTemplate function created for them when they are rendered. @@ -60,17 +66,23 @@ private Value CreateInsertTemplateFunction(IContext platformContext, string curr var function = Function.CreatePure( (state, args) => { + // Resolve arguments to InsertTemplate var templateRelativePath = args[0].AsString; var templateArgs = args.Count > 1 ? args[1] : Value.EmptyMap; var indent = args.Count > 2 ? args[2].AsString : ""; + // Resolve the path of the sub-template to be inserted, relative to the current template var parentTemplateDir = Path.GetDirectoryName(currentTemplatePath) ?? string.Empty; var newTemplatePath = Path.Combine(parentTemplateDir, templateRelativePath); var compiledTemplate = ReadAndCompile(newTemplatePath); - var newInsertTemplateFunction = CreateInsertTemplateFunction(platformContext, newTemplatePath); - var newContext = platformContext.Add("InsertTemplate", newInsertTemplateFunction); + var newSymbols = new Dictionary + { + { "InsertTemplate", CreateInsertTemplateFunction(platformContext, newTemplatePath) }, + { "ARGS", new Dictionary(templateArgs.Fields) }, + }; + var newContext = platformContext.Add(newSymbols); return compiledTemplate.Render(newContext); } ); From 5137b443b9ca60e043a807dc64ed84bb3c53411c Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 19 Sep 2025 09:52:43 -0700 Subject: [PATCH 23/38] Add sub-template trimming --- src/TemplateGenerator/TemplateGeneratorCli.cs | 2 +- src/Templating/Cottle/CottleTemplateEngine.cs | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/TemplateGenerator/TemplateGeneratorCli.cs b/src/TemplateGenerator/TemplateGeneratorCli.cs index 2a97d5a6..bcfe3e5b 100644 --- a/src/TemplateGenerator/TemplateGeneratorCli.cs +++ b/src/TemplateGenerator/TemplateGeneratorCli.cs @@ -29,7 +29,7 @@ public async Task GenerateDockerfiles([Argument] string manifestPath) var compiledTemplates = platformsWithTemplates .Select(platform => platform.DockerfileTemplatePath!) - .Select(engine.ReadAndCompile); + .Select(templatePath => engine.ReadAndCompile(templatePath, trim: false)); var compiledTemplateInfos = platformsWithTemplates .Zip(compiledTemplates); diff --git a/src/Templating/Cottle/CottleTemplateEngine.cs b/src/Templating/Cottle/CottleTemplateEngine.cs index 19840bcc..64adec6d 100644 --- a/src/Templating/Cottle/CottleTemplateEngine.cs +++ b/src/Templating/Cottle/CottleTemplateEngine.cs @@ -30,9 +30,15 @@ public ICompiledTemplate Compile(string template) return compiledTemplate; } - public ICompiledTemplate ReadAndCompile(string path) + public ICompiledTemplate ReadAndCompile(string path, bool trim = true) { string content = _fileSystem.ReadAllText(path); + + if (trim) + { + content = content.Trim(); + } + return Compile(content); } @@ -74,7 +80,7 @@ private Value CreateInsertTemplateFunction(IContext platformContext, string curr // Resolve the path of the sub-template to be inserted, relative to the current template var parentTemplateDir = Path.GetDirectoryName(currentTemplatePath) ?? string.Empty; var newTemplatePath = Path.Combine(parentTemplateDir, templateRelativePath); - var compiledTemplate = ReadAndCompile(newTemplatePath); + var compiledTemplate = ReadAndCompile(newTemplatePath, trim: true); var newSymbols = new Dictionary { From 055ec80c5ae5762abf388294d29a672a3fd3403b Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 19 Sep 2025 09:57:09 -0700 Subject: [PATCH 24/38] Reorder fields --- src/Templating/Cottle/CottleTemplateEngine.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Templating/Cottle/CottleTemplateEngine.cs b/src/Templating/Cottle/CottleTemplateEngine.cs index 64adec6d..ae4d4f25 100644 --- a/src/Templating/Cottle/CottleTemplateEngine.cs +++ b/src/Templating/Cottle/CottleTemplateEngine.cs @@ -18,10 +18,10 @@ public sealed class CottleTemplateEngine(IFileSystem fileSystem) : ITemplateEngi Trimmer = DocumentConfiguration.TrimNothing }; - private IContext _globalContext = Context.CreateBuiltin(new Dictionary()); - private readonly IFileSystem _fileSystem = fileSystem; + private IContext _globalContext = Context.CreateBuiltin(new Dictionary()); + public ICompiledTemplate Compile(string template) { var documentResult = Document.CreateDefault(template, s_config); From 1c143f0cddb5c6f5d9e35869f6ce0f7447539806 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 19 Sep 2025 10:04:54 -0700 Subject: [PATCH 25/38] Apply trimming and indent after document render --- src/TemplateGenerator/TemplateGeneratorCli.cs | 2 +- src/Templating/Abstractions/ICompiledTemplate.cs | 2 +- src/Templating/Cottle/CottleTemplate.cs | 16 ++++++++++++++-- src/Templating/Cottle/CottleTemplateEngine.cs | 12 +++--------- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/TemplateGenerator/TemplateGeneratorCli.cs b/src/TemplateGenerator/TemplateGeneratorCli.cs index bcfe3e5b..2a97d5a6 100644 --- a/src/TemplateGenerator/TemplateGeneratorCli.cs +++ b/src/TemplateGenerator/TemplateGeneratorCli.cs @@ -29,7 +29,7 @@ public async Task GenerateDockerfiles([Argument] string manifestPath) var compiledTemplates = platformsWithTemplates .Select(platform => platform.DockerfileTemplatePath!) - .Select(templatePath => engine.ReadAndCompile(templatePath, trim: false)); + .Select(engine.ReadAndCompile); var compiledTemplateInfos = platformsWithTemplates .Zip(compiledTemplates); diff --git a/src/Templating/Abstractions/ICompiledTemplate.cs b/src/Templating/Abstractions/ICompiledTemplate.cs index 2a3cef0a..18251be5 100644 --- a/src/Templating/Abstractions/ICompiledTemplate.cs +++ b/src/Templating/Abstractions/ICompiledTemplate.cs @@ -5,5 +5,5 @@ namespace Microsoft.DotNet.DockerTools.Templating.Abstractions; public interface ICompiledTemplate { - string Render(TContext context); + string Render(TContext context, bool trim = false, string indent = ""); } diff --git a/src/Templating/Cottle/CottleTemplate.cs b/src/Templating/Cottle/CottleTemplate.cs index d03fdbde..109c355d 100644 --- a/src/Templating/Cottle/CottleTemplate.cs +++ b/src/Templating/Cottle/CottleTemplate.cs @@ -10,8 +10,20 @@ public sealed class CottleTemplate(IDocument document) : ICompiledTemplate Compile(string template) return compiledTemplate; } - public ICompiledTemplate ReadAndCompile(string path, bool trim = true) + public ICompiledTemplate ReadAndCompile(string path) { string content = _fileSystem.ReadAllText(path); - - if (trim) - { - content = content.Trim(); - } - return Compile(content); } @@ -80,7 +74,7 @@ private Value CreateInsertTemplateFunction(IContext platformContext, string curr // Resolve the path of the sub-template to be inserted, relative to the current template var parentTemplateDir = Path.GetDirectoryName(currentTemplatePath) ?? string.Empty; var newTemplatePath = Path.Combine(parentTemplateDir, templateRelativePath); - var compiledTemplate = ReadAndCompile(newTemplatePath, trim: true); + var compiledTemplate = ReadAndCompile(newTemplatePath); var newSymbols = new Dictionary { @@ -89,7 +83,7 @@ private Value CreateInsertTemplateFunction(IContext platformContext, string curr }; var newContext = platformContext.Add(newSymbols); - return compiledTemplate.Render(newContext); + return compiledTemplate.Render(newContext, trim: true, indent: indent); } ); From e43c3262e2e0cbc9786503f5fa275df75ace238d Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 19 Sep 2025 10:09:38 -0700 Subject: [PATCH 26/38] Add replace function --- src/Templating/Cottle/CottleTemplateEngine.cs | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/Templating/Cottle/CottleTemplateEngine.cs b/src/Templating/Cottle/CottleTemplateEngine.cs index 69e85432..668a11db 100644 --- a/src/Templating/Cottle/CottleTemplateEngine.cs +++ b/src/Templating/Cottle/CottleTemplateEngine.cs @@ -20,7 +20,12 @@ public sealed class CottleTemplateEngine(IFileSystem fileSystem) : ITemplateEngi private readonly IFileSystem _fileSystem = fileSystem; - private IContext _globalContext = Context.CreateBuiltin(new Dictionary()); + private IContext _globalContext = Context.CreateBuiltin( + new Dictionary() + { + { "replace", ReplaceFunction } + } + ); public ICompiledTemplate Compile(string template) { @@ -89,4 +94,18 @@ private Value CreateInsertTemplateFunction(IContext platformContext, string curr return Value.FromFunction(function); } + + private static Value ReplaceFunction = Value.FromFunction( + Function.CreatePure( + (state, args) => + { + string source = args[0].AsString; + string oldValue = args[1].AsString; + string newValue = args[2].AsString; + return Value.FromString(source.Replace(oldValue, newValue)); + }, + min: 3, + max: 3 + ) + ); } From 2dd1d3e740f397fb002a2adfb35dd92244dbafbf Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 19 Sep 2025 10:13:58 -0700 Subject: [PATCH 27/38] Fix OS_VERSION_BASE variable --- src/Templating/Cottle/CottleContextExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Templating/Cottle/CottleContextExtensions.cs b/src/Templating/Cottle/CottleContextExtensions.cs index ac242334..dfbe5d28 100644 --- a/src/Templating/Cottle/CottleContextExtensions.cs +++ b/src/Templating/Cottle/CottleContextExtensions.cs @@ -58,7 +58,7 @@ public static class PlatformInfoVariableExtensions { "ARCH_TAG_SUFFIX", $"-{platform.ArchWithVariant}" }, { "PRODUCT_VERSION", platform.Image.ProductVersion ?? "" }, { "OS_VERSION", platform.Model.OsVersion }, - { "OS_VERSION_BASE", "" }, + { "OS_VERSION_BASE", platform.BaseOsVersion }, { "OS_VERSION_NUMBER", platform.GetOsVersionNumber() }, { "OS_ARCH_HYPHENATED", platform.GetOsArchHyphenatedName() }, }; From 0f09cb47a669486d480a9cdc2963f694195458a8 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 19 Sep 2025 10:51:35 -0700 Subject: [PATCH 28/38] Add file system and template caches --- src/TemplateGenerator/TemplateGeneratorCli.cs | 5 ++- src/Templating/Cottle/CottleTemplateEngine.cs | 19 ++++++++- src/Templating/FileSystem.cs | 4 ++ src/Templating/FileSystemCache.cs | 32 +++++++++++++++ src/Templating/ForeverCache.cs | 39 +++++++++++++++++++ src/Templating/ICache.cs | 12 ++++++ 6 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 src/Templating/FileSystemCache.cs create mode 100644 src/Templating/ForeverCache.cs create mode 100644 src/Templating/ICache.cs diff --git a/src/TemplateGenerator/TemplateGeneratorCli.cs b/src/TemplateGenerator/TemplateGeneratorCli.cs index 2a97d5a6..6a1a96ef 100644 --- a/src/TemplateGenerator/TemplateGeneratorCli.cs +++ b/src/TemplateGenerator/TemplateGeneratorCli.cs @@ -21,7 +21,8 @@ public async Task GenerateDockerfiles([Argument] string manifestPath) ManifestInfo manifest = await ManifestInfo.LoadAsync(manifestPath); var fileSystem = new FileSystem(); - var engine = new CottleTemplateEngine(fileSystem); + var fileSystemCache = new FileSystemCache(fileSystem); + var engine = new CottleTemplateEngine(fileSystemCache); engine.AddGlobalVariables(manifest.Model.Variables); var platformsWithTemplates = manifest.AllPlatforms @@ -43,5 +44,7 @@ public async Task GenerateDockerfiles([Argument] string manifestPath) Console.WriteLine($"Read {fileSystem.FilesRead} files ({fileSystem.BytesRead} bytes)"); Console.WriteLine($"Wrote {fileSystem.FilesWritten} files ({fileSystem.BytesWritten} bytes)"); + Console.WriteLine($"File system cache hits: {fileSystemCache.CacheHits}, misses: {fileSystemCache.CacheMisses}"); + Console.WriteLine($"Compiled template cache hits: {engine.CompiledTemplateCacheHits}, misses: {engine.CompiledTemplateCacheMisses}"); } } diff --git a/src/Templating/Cottle/CottleTemplateEngine.cs b/src/Templating/Cottle/CottleTemplateEngine.cs index 668a11db..5a4dc2a8 100644 --- a/src/Templating/Cottle/CottleTemplateEngine.cs +++ b/src/Templating/Cottle/CottleTemplateEngine.cs @@ -7,7 +7,7 @@ namespace Microsoft.DotNet.DockerTools.Templating.Cottle; -public sealed class CottleTemplateEngine(IFileSystem fileSystem) : ITemplateEngine +public sealed class CottleTemplateEngine : ITemplateEngine { private static readonly DocumentConfiguration s_config = new() { @@ -18,7 +18,8 @@ public sealed class CottleTemplateEngine(IFileSystem fileSystem) : ITemplateEngi Trimmer = DocumentConfiguration.TrimNothing }; - private readonly IFileSystem _fileSystem = fileSystem; + private readonly IFileSystem _fileSystem; + private readonly ForeverCache> _templateCache; private IContext _globalContext = Context.CreateBuiltin( new Dictionary() @@ -27,6 +28,15 @@ public sealed class CottleTemplateEngine(IFileSystem fileSystem) : ITemplateEngi } ); + public CottleTemplateEngine(IFileSystem fileSystem) + { + _fileSystem = fileSystem; + _templateCache = new ForeverCache>(ReadAndCompileWithNoCache); + } + + public int CompiledTemplateCacheHits => _templateCache.Hits; + public int CompiledTemplateCacheMisses => _templateCache.Misses; + public ICompiledTemplate Compile(string template) { var documentResult = Document.CreateDefault(template, s_config); @@ -36,6 +46,11 @@ public ICompiledTemplate Compile(string template) } public ICompiledTemplate ReadAndCompile(string path) + { + return _templateCache.GetOrAdd(path); + } + + private ICompiledTemplate ReadAndCompileWithNoCache(string path) { string content = _fileSystem.ReadAllText(path); return Compile(content); diff --git a/src/Templating/FileSystem.cs b/src/Templating/FileSystem.cs index cc51570d..1559e9a4 100644 --- a/src/Templating/FileSystem.cs +++ b/src/Templating/FileSystem.cs @@ -6,6 +6,10 @@ namespace Microsoft.DotNet.DockerTools.Templating; +/// +/// General purpose synchronous file system implementation that tracks simple +/// metrics about reads and writes. +/// public sealed class FileSystem : IFileSystem { private int _reads = 0; diff --git a/src/Templating/FileSystemCache.cs b/src/Templating/FileSystemCache.cs new file mode 100644 index 00000000..fd11b360 --- /dev/null +++ b/src/Templating/FileSystemCache.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.DockerTools.Templating.Abstractions; + +namespace Microsoft.DotNet.DockerTools.Templating; + +public sealed class FileSystemCache : IFileSystem +{ + private readonly IFileSystem _fileSystem; + private readonly ICache _cache; + + public int CacheHits => _cache.Hits; + public int CacheMisses => _cache.Misses; + + public FileSystemCache(IFileSystem fileSystem) + { + _fileSystem = fileSystem; + _cache = new ForeverCache(key => _fileSystem.ReadAllText(key)); + } + + public string ReadAllText(string path) + { + var content = _cache.GetOrAdd(path); + return content; + } + + public void WriteAllText(string path, string content) + { + _fileSystem.WriteAllText(path, content); + } +} diff --git a/src/Templating/ForeverCache.cs b/src/Templating/ForeverCache.cs new file mode 100644 index 00000000..4bb9d604 --- /dev/null +++ b/src/Templating/ForeverCache.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.DockerTools.Templating; + +/// +/// A cache that retains values for the lifetime of the object. +/// +public sealed class ForeverCache(Func valueFactory) : ICache +{ + private readonly Func _valueFactory = valueFactory; + private readonly Dictionary _cache = []; + + /// + /// Number of times a cached value was returned. + /// + public int Hits { get; private set; } = 0; + + /// + /// Number of times a new value was created and added to the cache. + /// + public int Misses { get; private set; } = 0; + + public T GetOrAdd(string key) + { + if (!_cache.TryGetValue(key, out T? value)) + { + value = _valueFactory(key); + _cache[key] = value; + Misses += 1; + } + else + { + Hits += 1; + } + + return value; + } +} diff --git a/src/Templating/ICache.cs b/src/Templating/ICache.cs new file mode 100644 index 00000000..e487fd0a --- /dev/null +++ b/src/Templating/ICache.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.DockerTools.Templating; + +public interface ICache +{ + int Hits { get; } + int Misses { get; } + + T GetOrAdd(string key); +} From 6dc10e7ad84a8ec146ce3c4126a20b3bc5461c44 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 19 Sep 2025 11:25:42 -0700 Subject: [PATCH 29/38] Use source generated JSON serialization --- .../ReadModel/JsonHelper.cs | 22 +++++-------------- .../ReadModel/ManifestPreprocessor.cs | 3 ++- .../ManifestInfoSerializationExtensions.cs | 9 ++++---- .../ManifestSerializationContext.cs | 19 ++++++++++++++++ 4 files changed, 31 insertions(+), 22 deletions(-) create mode 100644 src/ImageBuilder.Models/ReadModel/Serialization/ManifestSerializationContext.cs diff --git a/src/ImageBuilder.Models/ReadModel/JsonHelper.cs b/src/ImageBuilder.Models/ReadModel/JsonHelper.cs index a1595043..688beaaa 100644 --- a/src/ImageBuilder.Models/ReadModel/JsonHelper.cs +++ b/src/ImageBuilder.Models/ReadModel/JsonHelper.cs @@ -3,26 +3,16 @@ using System.Text.Json; using System.Text.Json.Nodes; -using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; namespace Microsoft.DotNet.ImageBuilder.ReadModel; internal static class JsonHelper { - private static readonly JsonSerializerOptions s_jsonOptions = new() - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - WriteIndented = true, - Converters = - { - new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, allowIntegerValues: false), - }, - NumberHandling = JsonNumberHandling.AllowReadingFromString, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault - }; + public static T Deserialize(JsonNode jsonNode, JsonTypeInfo typeInfo) => + JsonSerializer.Deserialize(jsonNode, typeInfo) + ?? throw new Exception($"Failed to deserialize JSON object to {typeof(T)}."); - public static T Deserialize(JsonNode jsonNode) => JsonSerializer.Deserialize(jsonNode, s_jsonOptions) - ?? throw new Exception($"Failed to deserialize JSON object to {typeof(T)}."); - - public static string Serialize(T model) => JsonSerializer.Serialize(model, s_jsonOptions); + public static string Serialize(T model, JsonTypeInfo typeInfo) => + JsonSerializer.Serialize(model, typeInfo); } diff --git a/src/ImageBuilder.Models/ReadModel/ManifestPreprocessor.cs b/src/ImageBuilder.Models/ReadModel/ManifestPreprocessor.cs index 7e20b424..c3732dd8 100644 --- a/src/ImageBuilder.Models/ReadModel/ManifestPreprocessor.cs +++ b/src/ImageBuilder.Models/ReadModel/ManifestPreprocessor.cs @@ -3,6 +3,7 @@ using System.Text.Json.Nodes; using Microsoft.DotNet.ImageBuilder.Models.Manifest; +using Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; namespace Microsoft.DotNet.ImageBuilder.ReadModel; @@ -15,7 +16,7 @@ public JsonNode Process(JsonObject root, IEnumerable includesNodes) // Process includes first so variables in included files can be processed ProcessIncludes(root, includesNodes); - var rawManifest = JsonHelper.Deserialize(root); + var rawManifest = JsonHelper.Deserialize(root, ManifestSerializationContext.Default.Manifest); var variables = rawManifest.Variables ?? new Dictionary(); // Add variables for each repo name (e.g. "Repo:dotnet" -> "mcr.microsoft.com/dotnet") diff --git a/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs index 8018e9ec..46785936 100644 --- a/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs +++ b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs @@ -1,9 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections.Immutable; using System.Text.Json.Nodes; -using Microsoft.DotNet.ImageBuilder.Models.Manifest; using static Microsoft.DotNet.ImageBuilder.ReadModel.JsonHelper; namespace Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; @@ -22,7 +20,8 @@ public static async Task LoadAsync(string manifestJsonPath) var includesNode = manifestJsonObject["includes"]; if (includesNode is not null) { - IEnumerable includesFiles = Deserialize(includesNode); + var includesFiles = Deserialize(includesNode, ManifestSerializationContext.Default.StringArray); + includesJsonNodes = await Task.WhenAll( includesFiles // Make includes paths relative to the manifest file @@ -32,12 +31,12 @@ public static async Task LoadAsync(string manifestJsonPath) var preprocessor = new ManifestPreprocessor(); var processedRootJsonNode = preprocessor.Process(manifestJsonObject, includesJsonNodes); - var processedModel = Deserialize(processedRootJsonNode); + var processedModel = Deserialize(processedRootJsonNode, ManifestSerializationContext.Default.Manifest); return ManifestInfo.Create(processedModel, manifestJsonPath); } - public string ToJsonString() => Serialize(manifestInfo.Model); + public string ToJsonString() => Serialize(manifestInfo.Model, ManifestSerializationContext.Default.Manifest); } private static async Task LoadModelFromFileAsync(string manifestJsonPath) diff --git a/src/ImageBuilder.Models/ReadModel/Serialization/ManifestSerializationContext.cs b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestSerializationContext.cs new file mode 100644 index 00000000..20453758 --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestSerializationContext.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Serialization; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; + +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + WriteIndented = true, + UseStringEnumConverter = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault +)] +[JsonSerializable(typeof(Manifest))] +[JsonSerializable(typeof(string[]))] +public partial class ManifestSerializationContext : JsonSerializerContext +{ +} From d389a533964a63feece3edc4a2358d9df77b9246 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 19 Sep 2025 11:35:46 -0700 Subject: [PATCH 30/38] Move JsonHelper to Serialization namespace --- .../ReadModel/{ => Serialization}/JsonHelper.cs | 2 +- .../Serialization/ManifestInfoSerializationExtensions.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename src/ImageBuilder.Models/ReadModel/{ => Serialization}/JsonHelper.cs (90%) diff --git a/src/ImageBuilder.Models/ReadModel/JsonHelper.cs b/src/ImageBuilder.Models/ReadModel/Serialization/JsonHelper.cs similarity index 90% rename from src/ImageBuilder.Models/ReadModel/JsonHelper.cs rename to src/ImageBuilder.Models/ReadModel/Serialization/JsonHelper.cs index 688beaaa..d9ff0709 100644 --- a/src/ImageBuilder.Models/ReadModel/JsonHelper.cs +++ b/src/ImageBuilder.Models/ReadModel/Serialization/JsonHelper.cs @@ -5,7 +5,7 @@ using System.Text.Json.Nodes; using System.Text.Json.Serialization.Metadata; -namespace Microsoft.DotNet.ImageBuilder.ReadModel; +namespace Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; internal static class JsonHelper { diff --git a/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs index 46785936..689b6a51 100644 --- a/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs +++ b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs @@ -2,7 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Text.Json.Nodes; -using static Microsoft.DotNet.ImageBuilder.ReadModel.JsonHelper; +using static Microsoft.DotNet.ImageBuilder.ReadModel.Serialization.JsonHelper; namespace Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; From 900c9ba9d665cbd8736a7a2403db6e200d79c1c6 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 19 Sep 2025 11:58:35 -0700 Subject: [PATCH 31/38] Add GenerateDockerfiles benchmark --- .../GenerateDockerfilesBenchmarks.cs | 20 +++++++++++++++++++ src/TemplateGenerator.Benchmarks/Program.cs | 3 ++- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 src/TemplateGenerator.Benchmarks/GenerateDockerfilesBenchmarks.cs diff --git a/src/TemplateGenerator.Benchmarks/GenerateDockerfilesBenchmarks.cs b/src/TemplateGenerator.Benchmarks/GenerateDockerfilesBenchmarks.cs new file mode 100644 index 00000000..ed2b9200 --- /dev/null +++ b/src/TemplateGenerator.Benchmarks/GenerateDockerfilesBenchmarks.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using BenchmarkDotNet.Attributes; + +namespace Microsoft.DotNet.DockerTools.TemplateGenerator.Benchmarks; + +[MemoryDiagnoser] +public class GenerateDockerfilesBenchmarks +{ + private static readonly string s_manifestPath = + Environment.GetEnvironmentVariable("MANIFEST_PATH") ?? "manifest.json"; + + [Benchmark] + public async Task GenerateDockerfiles() + { + await new TemplateGeneratorCli().GenerateDockerfiles(s_manifestPath); + return "Completed"; + } +} diff --git a/src/TemplateGenerator.Benchmarks/Program.cs b/src/TemplateGenerator.Benchmarks/Program.cs index ab2defc6..f4223ff6 100644 --- a/src/TemplateGenerator.Benchmarks/Program.cs +++ b/src/TemplateGenerator.Benchmarks/Program.cs @@ -4,4 +4,5 @@ using BenchmarkDotNet.Running; using Microsoft.DotNet.DockerTools.TemplateGenerator.Benchmarks; -var summary = BenchmarkRunner.Run(); +BenchmarkRunner.Run(); +BenchmarkRunner.Run(); From c62c23d0b7cced72cfac3d7422c5d9a63182752b Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 19 Sep 2025 14:39:22 -0700 Subject: [PATCH 32/38] Enable Native AOT --- .../Microsoft.DotNet.ImageBuilder.Models.csproj | 1 + .../ReadModel/ManifestPreprocessor.cs | 14 +++++++++++++- .../Serialization/ManifestSerializationContext.cs | 1 + src/TemplateGenerator/TemplateGenerator.csproj | 1 + 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj b/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj index 1b5d9485..c19d9f99 100644 --- a/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj +++ b/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj @@ -5,6 +5,7 @@ enable enable Microsoft.DotNet.ImageBuilder + true diff --git a/src/ImageBuilder.Models/ReadModel/ManifestPreprocessor.cs b/src/ImageBuilder.Models/ReadModel/ManifestPreprocessor.cs index c3732dd8..150372e4 100644 --- a/src/ImageBuilder.Models/ReadModel/ManifestPreprocessor.cs +++ b/src/ImageBuilder.Models/ReadModel/ManifestPreprocessor.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text.Json; using System.Text.Json.Nodes; using Microsoft.DotNet.ImageBuilder.Models.Manifest; using Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; @@ -64,7 +65,18 @@ private void ProcessVariables(JsonNode? node) case JsonValue jsonValue: if (jsonValue.TryGetValue(out string? stringValue)) { - jsonValue.ReplaceWith(_variableStore.ResolveInnerVariables(stringValue)); + var newValue = _variableStore.ResolveInnerVariables(stringValue); + + #pragma warning disable IL2026 + #pragma warning disable IL3050 + // JsonValue.ReplaceWith is annotated with 'RequiresDynamicCodeAttribute', but it doesn't use + // dynamic code in all cases. Since we're giving it a JsonNode, it'll short-circuit before + // resorting to dynamic code for serialization. + jsonValue.ReplaceWith( + JsonSerializer.SerializeToNode(newValue, ManifestSerializationContext.Default.String) + ); + #pragma warning restore IL2026 + #pragma warning restore IL3050 } break; diff --git a/src/ImageBuilder.Models/ReadModel/Serialization/ManifestSerializationContext.cs b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestSerializationContext.cs index 20453758..a23e93c4 100644 --- a/src/ImageBuilder.Models/ReadModel/Serialization/ManifestSerializationContext.cs +++ b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestSerializationContext.cs @@ -14,6 +14,7 @@ namespace Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; )] [JsonSerializable(typeof(Manifest))] [JsonSerializable(typeof(string[]))] +[JsonSerializable(typeof(string))] public partial class ManifestSerializationContext : JsonSerializerContext { } diff --git a/src/TemplateGenerator/TemplateGenerator.csproj b/src/TemplateGenerator/TemplateGenerator.csproj index 214cb54b..dac18643 100644 --- a/src/TemplateGenerator/TemplateGenerator.csproj +++ b/src/TemplateGenerator/TemplateGenerator.csproj @@ -6,6 +6,7 @@ enable enable Microsoft.DotNet.DockerTools.TemplateGenerator + true From f92de74e7714ac71394f8ae69a2694f27fb93969 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 19 Sep 2025 15:03:24 -0700 Subject: [PATCH 33/38] Use fully synchronous manifest loading --- .../ManifestInfoSerializationExtensions.cs | 38 +++++++++++++++++++ src/TemplateGenerator/TemplateGeneratorCli.cs | 4 +- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs index 689b6a51..549ef4a9 100644 --- a/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs +++ b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs @@ -36,6 +36,29 @@ public static async Task LoadAsync(string manifestJsonPath) return ManifestInfo.Create(processedModel, manifestJsonPath); } + public static ManifestInfo Load(string manifestJsonPath) + { + var manifestJsonObject = LoadModelFromFile(manifestJsonPath); + var manifestDir = Path.GetDirectoryName(manifestJsonPath) ?? ""; + + // Load and deserialize included files + IEnumerable includesJsonNodes = []; + var includesNode = manifestJsonObject["includes"]; + if (includesNode is not null) + { + var includesFiles = Deserialize(includesNode, ManifestSerializationContext.Default.StringArray); + includesJsonNodes = includesFiles + .Select(includesFile => Path.Combine(manifestDir, includesFile)) + .Select(LoadModelFromFile); + } + + var preprocessor = new ManifestPreprocessor(); + var processedRootJsonNode = preprocessor.Process(manifestJsonObject, includesJsonNodes); + var processedModel = Deserialize(processedRootJsonNode, ManifestSerializationContext.Default.Manifest); + + return ManifestInfo.Create(processedModel, manifestJsonPath); + } + public string ToJsonString() => Serialize(manifestInfo.Model, ManifestSerializationContext.Default.Manifest); } @@ -53,4 +76,19 @@ private static async Task LoadModelFromFileAsync(string manifestJson return rootJsonObject; } + + private static JsonObject LoadModelFromFile(string manifestJsonPath) + { + var jsonStream = File.OpenRead(manifestJsonPath); + var rootJsonNode = JsonNode.Parse(jsonStream) + ?? throw new Exception( + $"Failed to parse manifest JSON from file: {manifestJsonPath}"); + + if (rootJsonNode is not JsonObject rootJsonObject) + { + throw new InvalidDataException($"Manifest root must be a JSON object."); + } + + return rootJsonObject; + } } diff --git a/src/TemplateGenerator/TemplateGeneratorCli.cs b/src/TemplateGenerator/TemplateGeneratorCli.cs index 6a1a96ef..ee05f205 100644 --- a/src/TemplateGenerator/TemplateGeneratorCli.cs +++ b/src/TemplateGenerator/TemplateGeneratorCli.cs @@ -16,9 +16,9 @@ public sealed class TemplateGeneratorCli /// /// Path to manifest JSON file [Command("generate-dockerfiles")] - public async Task GenerateDockerfiles([Argument] string manifestPath) + public void GenerateDockerfiles([Argument] string manifestPath) { - ManifestInfo manifest = await ManifestInfo.LoadAsync(manifestPath); + ManifestInfo manifest = ManifestInfo.Load(manifestPath); var fileSystem = new FileSystem(); var fileSystemCache = new FileSystemCache(fileSystem); From 3bf11f538cc00e97b66773c350093fcab7176715 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Sat, 20 Sep 2025 19:30:37 -0700 Subject: [PATCH 34/38] Get readme generation working --- .../ReadModel/ManifestInfo.cs | 53 ++++++++-- .../ReadModel/PathHelper.cs | 18 ++++ src/TemplateGenerator/LoggingExtensions.cs | 35 ++++++ src/TemplateGenerator/TemplateGeneratorCli.cs | 100 ++++++++++++++++-- .../Cottle/CottleContextExtensions.cs | 70 +++++++++--- src/Templating/Cottle/CottleTemplateEngine.cs | 27 +++-- 6 files changed, 268 insertions(+), 35 deletions(-) create mode 100644 src/ImageBuilder.Models/ReadModel/PathHelper.cs create mode 100644 src/TemplateGenerator/LoggingExtensions.cs diff --git a/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs index eeea09b9..b9bb3246 100644 --- a/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs +++ b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs @@ -6,7 +6,11 @@ namespace Microsoft.DotNet.ImageBuilder.ReadModel; -public sealed record ManifestInfo(Manifest Model, string FilePath, ImmutableList Repos) +public sealed record ManifestInfo( + Manifest Model, + string FilePath, + ManifestReadmeInfo? Readme, + ImmutableList Repos) { private readonly ImmutableDictionary _reposById = Repos.Where(repo => repo.Model.Id is not null) @@ -25,11 +29,42 @@ internal static ManifestInfo Create(Manifest model, string manifestFilePath) .Select(repo => RepoInfo.Create(repo, model, manifestDir)) .ToImmutableList(); - return new ManifestInfo(model, manifestFilePath, repoInfos); + var readmeInfo = model.Readme is not null + ? ManifestReadmeInfo.Create(model.Readme, model, manifestDir) + : null; + + return new ManifestInfo(model, manifestFilePath, readmeInfo, repoInfos); + } +} + +public sealed record ManifestReadmeInfo(Readme Model, Manifest Manifest, string FilePath, string? TemplatePath) +{ + internal static ManifestReadmeInfo Create(Readme model, Manifest manifest, string manifestDir) + { + string path = Path.Combine(manifestDir, model.Path); + string? templatePath = PathHelper.MaybeCombine(manifestDir, model.TemplatePath); + + return new ManifestReadmeInfo(model, manifest, path, templatePath); + } +} + +public sealed record RepoReadmeInfo(Readme Model, Repo Repo, string FilePath, string? TemplatePath) +{ + internal static RepoReadmeInfo Create(Readme model, Repo repo, string manifestDir) + { + string path = Path.Combine(manifestDir, model.Path); + string? templatePath = PathHelper.MaybeCombine(manifestDir, model.TemplatePath); + + return new RepoReadmeInfo(model, repo, path, templatePath); } } -public sealed record RepoInfo(Repo Model, Manifest Manifest, ImmutableList Images) +public sealed record RepoInfo( + Repo Model, + Manifest Manifest, + string FullName, + ImmutableList Images, + ImmutableList Readmes) { internal static RepoInfo Create(Repo model, Manifest manifest, string manifestDir) { @@ -37,11 +72,17 @@ internal static RepoInfo Create(Repo model, Manifest manifest, string manifestDi .Select(image => ImageInfo.Create(image, model, manifestDir)) .ToImmutableList(); - return new RepoInfo(model, manifest, imageInfos); + var readmeInfos = model.Readmes + .Select(readme => RepoReadmeInfo.Create(readme, model, manifestDir)) + .ToImmutableList(); + + var fullName = manifest.Registry + "/" + model.Name; + + return new RepoInfo(model, manifest, fullName, imageInfos, readmeInfos); } } -public sealed record ImageInfo(Image Model, ImmutableList Platforms) +public sealed record ImageInfo(Image Model, Repo repo, ImmutableList Platforms) { internal static ImageInfo Create(Image model, Repo repo, string manifestDir) { @@ -49,7 +90,7 @@ internal static ImageInfo Create(Image model, Repo repo, string manifestDir) .Select(platform => PlatformInfo.Create(platform, model, manifestDir)) .ToImmutableList(); - return new ImageInfo(model, platformInfos); + return new ImageInfo(model, repo, platformInfos); } } diff --git a/src/ImageBuilder.Models/ReadModel/PathHelper.cs b/src/ImageBuilder.Models/ReadModel/PathHelper.cs new file mode 100644 index 00000000..542a29ec --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/PathHelper.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +internal static class PathHelper +{ + [return: NotNullIfNotNull(nameof(optionalFilePath))] + public static string? MaybeCombine(string basePath, string? optionalFilePath) => + (basePath, optionalFilePath) switch + { + // If the optionalFilePath is null, then we want to maintain that null value. + (_, null) => null, + _ => Path.Combine(basePath, optionalFilePath) + }; +} diff --git a/src/TemplateGenerator/LoggingExtensions.cs b/src/TemplateGenerator/LoggingExtensions.cs new file mode 100644 index 00000000..55719c55 --- /dev/null +++ b/src/TemplateGenerator/LoggingExtensions.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.DockerTools.Templating.Cottle; +using Microsoft.DotNet.DockerTools.Templating; + +namespace Microsoft.DotNet.DockerTools.TemplateGenerator; + +internal static class LoggingExtensions +{ + extension(FileSystem fs) + { + public void LogStatistics() => Console.WriteLine( + $""" + Read {fs.FilesRead} files ({fs.BytesRead} bytes) + Wrote {fs.FilesWritten} files ({fs.BytesWritten} bytes) + """ + ); + } + + extension(FileSystemCache fsCache) + { + public void LogStatistics() => Console.WriteLine( + $"File system cache hits: {fsCache.CacheHits}, misses: {fsCache.CacheMisses}" + ); + } + + extension(CottleTemplateEngine engine) + { + public void LogStatistics() => Console.WriteLine( + $"Compiled template cache hits: {engine.CompiledTemplateCacheHits}," + + $" misses: {engine.CompiledTemplateCacheMisses}" + ); + } +} diff --git a/src/TemplateGenerator/TemplateGeneratorCli.cs b/src/TemplateGenerator/TemplateGeneratorCli.cs index ee05f205..62192910 100644 --- a/src/TemplateGenerator/TemplateGeneratorCli.cs +++ b/src/TemplateGenerator/TemplateGeneratorCli.cs @@ -6,6 +6,7 @@ using Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; using Microsoft.DotNet.DockerTools.Templating.Cottle; using Microsoft.DotNet.DockerTools.Templating; +using System.Diagnostics; namespace Microsoft.DotNet.DockerTools.TemplateGenerator; @@ -15,10 +16,9 @@ public sealed class TemplateGeneratorCli /// Generates Dockerfiles from a manifest file. /// /// Path to manifest JSON file - [Command("generate-dockerfiles")] public void GenerateDockerfiles([Argument] string manifestPath) { - ManifestInfo manifest = ManifestInfo.Load(manifestPath); + var manifest = ManifestInfo.Load(manifestPath); var fileSystem = new FileSystem(); var fileSystemCache = new FileSystemCache(fileSystem); @@ -37,14 +37,100 @@ public void GenerateDockerfiles([Argument] string manifestPath) foreach (var (platform, compiledTemplate) in compiledTemplateInfos) { - var platformContext = engine.CreatePlatformContext(platform); + var platformContext = engine.CreateContext( + variables: platform.PlatformSpecificTemplateVariables, + // Null-forgiving operator is safe here because we filtered out + // platforms without templates above. + templatePath: platform.DockerfileTemplatePath!); + var output = compiledTemplate.Render(platformContext); fileSystem.WriteAllText(platform.DockerfilePath, output); } - Console.WriteLine($"Read {fileSystem.FilesRead} files ({fileSystem.BytesRead} bytes)"); - Console.WriteLine($"Wrote {fileSystem.FilesWritten} files ({fileSystem.BytesWritten} bytes)"); - Console.WriteLine($"File system cache hits: {fileSystemCache.CacheHits}, misses: {fileSystemCache.CacheMisses}"); - Console.WriteLine($"Compiled template cache hits: {engine.CompiledTemplateCacheHits}, misses: {engine.CompiledTemplateCacheMisses}"); + fileSystem.LogStatistics(); + fileSystemCache.LogStatistics(); + engine.LogStatistics(); + } + + /// + /// Generates README.md files from a manifest file. + /// + /// Path to manifest JSON file + public void GenerateReadmes([Argument] string manifestPath) + { + var manifest = ManifestInfo.Load(manifestPath); + + var fileSystem = new FileSystem(); + var fileSystemCache = new FileSystemCache(fileSystem); + var engine = new CottleTemplateEngine(fileSystemCache); + engine.AddGlobalVariables(manifest.Model.Variables); + + var templatedRepoReadmes = + manifest.Repos + .SelectMany(repo => repo.Readmes + .Where(readme => readme.TemplatePath is not null) + .Select(readme => (Repo: repo, Readme: readme))); + + foreach (var (repo, readme) in templatedRepoReadmes) + { + // Null-forgiving operator is safe here because we filtered out + // readmes without templates above. + var readmeTemplatePath = readme.TemplatePath!; + var compiledTemplate = engine.ReadAndCompile(readmeTemplatePath); + var repoContext = engine.CreateContext(repo.TemplateVariables, readmeTemplatePath); + var output = compiledTemplate.Render(repoContext); + fileSystem.WriteAllText(readme.FilePath, output); + } + + var manifestReadme = manifest.Readme; + if (manifestReadme is not null && manifestReadme.TemplatePath is not null) + { + var compiledTemplate = engine.ReadAndCompile(manifestReadme.TemplatePath); + var manifestContext = engine.CreateContext(manifest.TemplateVariables, manifestReadme.TemplatePath); + var output = compiledTemplate.Render(manifestContext); + fileSystem.WriteAllText(manifestReadme.FilePath, output); + } + + fileSystem.LogStatistics(); + fileSystemCache.LogStatistics(); + engine.LogStatistics(); + } + + /// + /// Generates both Dockerfiles and READMEs from a manifest file. + /// + /// Path to manifest JSON file + public void GenerateAll([Argument] string manifestPath) + { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + Console.WriteLine( + """ + + --- Generating Dockerfiles --- + """); + GenerateDockerfiles(manifestPath); + + stopwatch.Stop(); + + Console.WriteLine( + $""" + ({stopwatch.ElapsedMilliseconds} ms) + + --- Generating READMEs --- + """); + + stopwatch.Reset(); + stopwatch.Start(); + + GenerateReadmes(manifestPath); + + Console.WriteLine( + $""" + ({stopwatch.ElapsedMilliseconds} ms) + + """ + ); } } diff --git a/src/Templating/Cottle/CottleContextExtensions.cs b/src/Templating/Cottle/CottleContextExtensions.cs index dfbe5d28..41d55f06 100644 --- a/src/Templating/Cottle/CottleContextExtensions.cs +++ b/src/Templating/Cottle/CottleContextExtensions.cs @@ -49,19 +49,63 @@ public static class PlatformInfoVariableExtensions { extension(PlatformInfo platform) { - public Dictionary PlatformSpecificTemplateVariables => - new Dictionary() + public Dictionary PlatformSpecificTemplateVariables => new() + { + { "ARCH_SHORT", platform.Model.Architecture.ShortName }, + { "ARCH_NUPKG", platform.Model.Architecture.NupkgName }, + { "ARCH_VERSIONED", platform.ArchWithVariant }, + { "ARCH_TAG_SUFFIX", $"-{platform.ArchWithVariant}" }, + { "PRODUCT_VERSION", platform.Image.ProductVersion ?? "" }, + { "OS_VERSION", platform.Model.OsVersion }, + { "OS_VERSION_BASE", platform.BaseOsVersion }, + { "OS_VERSION_NUMBER", platform.GetOsVersionNumber() }, + { "OS_ARCH_HYPHENATED", platform.GetOsArchHyphenatedName() }, + }; + } +} + +public static class ManifestInfoVariableExtensions +{ + extension(ManifestInfo manifest) + { + public Dictionary TemplateVariables => new() + { + { "IS_PRODUCT_FAMILY", true.ToString() }, + }; + } +} + +public static class RepoInfoVariableExtensions +{ + extension(RepoInfo repo) + { + public Dictionary TemplateVariables => new() + { + { "REPO", repo.Model.Name }, + { "FULL_REPO", repo.FullName }, + { "PARENT_REPO", repo.GetParentRepoName() }, + { "SHORT_REPO", repo.ShortName }, + }; + + private string ShortName => + // LastIndexOf returns -1 when not found, so in the case the repo + // name doesn't have any slashes, (-1 + 1) becomes 0 which selects + // the whole string. + repo.Model.Name[(repo.Model.Name.LastIndexOf('/') + 1)..]; + + private string GetParentRepoName() + { + // Avoid using string.Split(...) to prevent array allocation. + var name = repo.Model.Name; + int last = name.LastIndexOf('/'); + if (last <= 0) { - { "ARCH_SHORT", platform.Model.Architecture.ShortName }, - { "ARCH_NUPKG", platform.Model.Architecture.NupkgName }, - { "ARCH_VERSIONED", platform.ArchWithVariant }, - { "ARCH_TAG_SUFFIX", $"-{platform.ArchWithVariant}" }, - { "PRODUCT_VERSION", platform.Image.ProductVersion ?? "" }, - { "OS_VERSION", platform.Model.OsVersion }, - { "OS_VERSION_BASE", platform.BaseOsVersion }, - { "OS_VERSION_NUMBER", platform.GetOsVersionNumber() }, - { "OS_ARCH_HYPHENATED", platform.GetOsArchHyphenatedName() }, - }; + return string.Empty; + } + + int prev = name.LastIndexOf('/', last - 1); + return name[(prev + 1)..last]; + } } } @@ -236,7 +280,7 @@ private static (string Name, string Version) GetOsVersionInfo(string os) } } - public static string FirstCharToUpper(this string source) => char.ToUpper(source[0]) + source.Substring(1); + private static string FirstCharToUpper(this string source) => char.ToUpper(source[0]) + source.Substring(1); [return: NotNullIfNotNull(nameof(source))] private static string? TrimEndString(this string? source, string trim) => source switch diff --git a/src/Templating/Cottle/CottleTemplateEngine.cs b/src/Templating/Cottle/CottleTemplateEngine.cs index 5a4dc2a8..bbe50e05 100644 --- a/src/Templating/Cottle/CottleTemplateEngine.cs +++ b/src/Templating/Cottle/CottleTemplateEngine.cs @@ -3,7 +3,6 @@ using Cottle; using Microsoft.DotNet.DockerTools.Templating.Abstractions; -using Microsoft.DotNet.ImageBuilder.ReadModel; namespace Microsoft.DotNet.DockerTools.Templating.Cottle; @@ -66,19 +65,29 @@ public void AddGlobalVariables(IDictionary variables) _globalContext = _globalContext.Add(variableSymbols); } - public IContext CreatePlatformContext(PlatformInfo platform) + /// + /// Create a new context for rendering a template. + /// + /// + /// Dictionary of variables to add to context. These variables will take + /// precedence over any global variables already set in the engine. + /// + /// + /// The path to the current template is needed to correctly resolve paths + /// to sub-templates + /// + public IContext CreateContext(IDictionary variables, string templatePath) { - var platformVariables = platform.PlatformSpecificTemplateVariables.ToCottleDictionary(); - - var variableContext = Context.CreateCustom(platformVariables); - var platformContext = Context.CreateCascade(primary: variableContext, fallback: _globalContext); + var variableSymbols = variables.ToCottleDictionary(); + var variableContext = Context.CreateCustom(variableSymbols); + var newContext = Context.CreateCascade(primary: variableContext, fallback: _globalContext); // It's OK for the insert template function not to have a reference to itself. Any sub-templates will have // their own InsertTemplate function created for them when they are rendered. - var insertTemplateFunction = CreateInsertTemplateFunction(platformContext, platform.DockerfileTemplatePath!); - var fullContext = platformContext.Add("InsertTemplate", insertTemplateFunction); + var insertTemplateFunction = CreateInsertTemplateFunction(newContext, templatePath); + newContext = newContext.Add("InsertTemplate", insertTemplateFunction); - return fullContext; + return newContext; } private Value CreateInsertTemplateFunction(IContext platformContext, string currentTemplatePath) From 4bf5c40aeec9cd82a72d217d3490f8b9b542ed0b Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Sun, 21 Sep 2025 09:37:16 -0700 Subject: [PATCH 35/38] Fix nullable warnings in ImageBuilder --- src/ImageBuilder/Commands/CopyBaseImagesCommand.cs | 3 ++- src/ImageBuilder/ImageNameResolver.cs | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/ImageBuilder/Commands/CopyBaseImagesCommand.cs b/src/ImageBuilder/Commands/CopyBaseImagesCommand.cs index afae1033..9b6ad76a 100644 --- a/src/ImageBuilder/Commands/CopyBaseImagesCommand.cs +++ b/src/ImageBuilder/Commands/CopyBaseImagesCommand.cs @@ -77,7 +77,8 @@ public override async Task ExecuteAsync() private IEnumerable GetFromImages(ManifestInfo manifest) => manifest.GetExternalFromImages() .Select(fromImage => Options.BaseImageOverrideOptions.ApplyBaseImageOverride(fromImage)) - .Where(fromImage => !fromImage.StartsWith(manifest.Model.Registry)); + .Where(fromImage => string.IsNullOrEmpty(manifest.Model.Registry) + || !fromImage.StartsWith(manifest.Model.Registry)); private Task CopyImageAsync(string fromImage, string destinationRegistryName) { diff --git a/src/ImageBuilder/ImageNameResolver.cs b/src/ImageBuilder/ImageNameResolver.cs index 27104a98..4e0467ec 100644 --- a/src/ImageBuilder/ImageNameResolver.cs +++ b/src/ImageBuilder/ImageNameResolver.cs @@ -79,8 +79,8 @@ private string GetFromImageTag(string fromImage, string? registry) { fromImage = _baseImageOverrideOptions.ApplyBaseImageOverride(fromImage); - if ((registry is not null && DockerHelper.IsInRegistry(fromImage, registry)) || - DockerHelper.IsInRegistry(fromImage, Manifest.Model.Registry) + if ((registry is not null && DockerHelper.IsInRegistry(fromImage, registry)) + || (Manifest.Model.Registry is not null && DockerHelper.IsInRegistry(fromImage, Manifest.Model.Registry)) || _sourceRepoPrefix is null) { return fromImage; @@ -97,7 +97,7 @@ protected string TrimInternallyOwnedRegistryAndRepoPrefix(string imageTag) => private bool IsInInternallyOwnedRegistry(string imageTag) => DockerHelper.IsInRegistry(imageTag, Manifest.Registry) || - DockerHelper.IsInRegistry(imageTag, Manifest.Model.Registry); + (Manifest.Model.Registry is not null && DockerHelper.IsInRegistry(imageTag, Manifest.Model.Registry)); } public class ImageNameResolverForBuild : ImageNameResolver From b3408086b2bfd0f721a4266f9c953559455eba37 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Sun, 21 Sep 2025 09:37:34 -0700 Subject: [PATCH 36/38] Add benchmark for GenerateReadmes and GenerateAll --- .../GenerateDockerfilesBenchmarks.cs | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/TemplateGenerator.Benchmarks/GenerateDockerfilesBenchmarks.cs b/src/TemplateGenerator.Benchmarks/GenerateDockerfilesBenchmarks.cs index ed2b9200..127961b1 100644 --- a/src/TemplateGenerator.Benchmarks/GenerateDockerfilesBenchmarks.cs +++ b/src/TemplateGenerator.Benchmarks/GenerateDockerfilesBenchmarks.cs @@ -12,9 +12,23 @@ public class GenerateDockerfilesBenchmarks Environment.GetEnvironmentVariable("MANIFEST_PATH") ?? "manifest.json"; [Benchmark] - public async Task GenerateDockerfiles() + public void GenerateDockerfiles() { - await new TemplateGeneratorCli().GenerateDockerfiles(s_manifestPath); - return "Completed"; + var generator = new TemplateGeneratorCli(); + generator.GenerateDockerfiles(s_manifestPath); + } + + [Benchmark] + public void GenerateReadmes() + { + var generator = new TemplateGeneratorCli(); + generator.GenerateReadmes(s_manifestPath); + } + + [Benchmark] + public void GenerateAll() + { + var generator = new TemplateGeneratorCli(); + generator.GenerateAll(s_manifestPath); } } From 473c72da011734a624f7afddea173193276d5080 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Sun, 21 Sep 2025 11:37:05 -0700 Subject: [PATCH 37/38] Implement rudimentary tags table generation --- .../ReadModel/ManifestInfo.cs | 31 ++++- src/Templating/Abstractions/ITableBuilder.cs | 11 ++ .../Cottle/CottleContextExtensions.cs | 111 +----------------- .../Readmes/MarkdownTableBuilder.cs | 51 ++++++++ src/Templating/Readmes/TagExtensions.cs | 18 +++ src/Templating/Readmes/TagsTableGenerator.cs | 60 ++++++++++ .../Shared/ArchDisplayExtensions.cs | 29 +++++ src/Templating/Shared/OsHelper.cs | 78 ++++++++++++ .../Shared/PlatformInfoSharedExtensions.cs | 20 ++++ src/Templating/Shared/StringExtensions.cs | 34 ++++++ 10 files changed, 332 insertions(+), 111 deletions(-) create mode 100644 src/Templating/Abstractions/ITableBuilder.cs create mode 100644 src/Templating/Readmes/MarkdownTableBuilder.cs create mode 100644 src/Templating/Readmes/TagExtensions.cs create mode 100644 src/Templating/Readmes/TagsTableGenerator.cs create mode 100644 src/Templating/Shared/ArchDisplayExtensions.cs create mode 100644 src/Templating/Shared/OsHelper.cs create mode 100644 src/Templating/Shared/PlatformInfoSharedExtensions.cs create mode 100644 src/Templating/Shared/StringExtensions.cs diff --git a/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs index b9bb3246..4e299343 100644 --- a/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs +++ b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs @@ -97,15 +97,42 @@ internal static ImageInfo Create(Image model, Repo repo, string manifestDir) public sealed record PlatformInfo( Platform Model, Image Image, + ImmutableList Tags, string DockerfilePath, + string RelativeDockerfilePath, string? DockerfileTemplatePath = null) { internal static PlatformInfo Create(Platform model, Image image, string manifestDir) { - var dockerfilePath = Path.Combine(manifestDir, model.Dockerfile, "Dockerfile"); + var relativeDockerfilePath = Path.Combine(model.Dockerfile, "Dockerfile"); + var fullDockerfilePath = Path.Combine(manifestDir, relativeDockerfilePath); var dockerfileTemplatePath = model.DockerfileTemplate is not null ? Path.Combine(manifestDir, model.DockerfileTemplate) : null; - return new PlatformInfo(model, image, dockerfilePath, dockerfileTemplatePath); + var tagInfos = model.Tags + .Select(tag => TagInfo.Create(tag.Value, model, tag.Key)) + .ToImmutableList(); + + return new PlatformInfo( + model, + image, + tagInfos, + fullDockerfilePath, + relativeDockerfilePath, + dockerfileTemplatePath); + } +} + +public sealed record TagInfo(Tag Model, Platform Platform, string Tag, bool IsDocumented) +{ + internal static TagInfo Create(Tag model, Platform platform, string tag) + { + bool isDocumented = model.DocType switch + { + TagDocumentationType.Undocumented => false, + _ => true + }; + + return new TagInfo(model, platform, tag, isDocumented); } } diff --git a/src/Templating/Abstractions/ITableBuilder.cs b/src/Templating/Abstractions/ITableBuilder.cs new file mode 100644 index 00000000..9bb5cc02 --- /dev/null +++ b/src/Templating/Abstractions/ITableBuilder.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.DockerTools.Templating.Abstractions; + +internal interface ITableBuilder +{ + ITableBuilder WithColumnHeadings(params IEnumerable headings); + void AddRow(params IEnumerable row); + string ToString(); +} diff --git a/src/Templating/Cottle/CottleContextExtensions.cs b/src/Templating/Cottle/CottleContextExtensions.cs index 41d55f06..efec5662 100644 --- a/src/Templating/Cottle/CottleContextExtensions.cs +++ b/src/Templating/Cottle/CottleContextExtensions.cs @@ -2,10 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. using Cottle; -using Microsoft.DotNet.ImageBuilder.ReadModel; +using Microsoft.DotNet.DockerTools.Templating.Shared; using Microsoft.DotNet.ImageBuilder.Models.Manifest; +using Microsoft.DotNet.ImageBuilder.ReadModel; using System.Text.RegularExpressions; -using System.Diagnostics.CodeAnalysis; namespace Microsoft.DotNet.DockerTools.Templating.Cottle; @@ -158,12 +158,6 @@ public string GetOsArchHyphenatedName() return osName + archName; } - - private string OSDisplayName => platform.Model.OS switch - { - OS.Windows => GetWindowsOSDisplayName(platform.BaseOsVersion), - _ => GetLinuxOSDisplayName(platform.BaseOsVersion) - }; } extension(Architecture architecture) @@ -187,22 +181,6 @@ public string GetOsArchHyphenatedName() _ => architecture.ToString().ToLowerInvariant(), }; - private string GetDisplayName(string? variant = null) - { - string displayName = architecture switch - { - Architecture.ARM => "arm32", - _ => architecture.ToString().ToLowerInvariant(), - }; - - if (variant != null) - { - displayName += variant.ToLowerInvariant(); - } - - return displayName; - } - public string DockerName => architecture.ToString().ToLowerInvariant(); } @@ -211,91 +189,6 @@ private string GetDisplayName(string? variant = null) public string DockerName => os.ToString().ToLowerInvariant(); } - private static string GetWindowsOSDisplayName(string osName) - { - string version = osName.Split('-')[1]; - return osName switch - { - var s when s.StartsWith("nanoserver") => - GetWindowsVersionDisplayName("Nano Server", version), - var s when s.StartsWith("windowsservercore") => - GetWindowsVersionDisplayName("Windows Server Core", version), - _ => throw new NotSupportedException($"The OS version '{osName}' is not supported.") - }; - } - - private static string GetLinuxOSDisplayName(string osName) => osName switch - { - string s when s.Contains("debian") => "Debian", - string s when s.Contains("bookworm") => "Debian 12", - string s when s.Contains("trixie") => "Debian 13", - string s when s.Contains("forky") => "Debian 14", - string s when s.Contains("duke") => "Debian 15", - string s when s.Contains("jammy") => "Ubuntu 22.04", - string s when s.Contains("noble") => "Ubuntu 24.04", - string s when s.Contains("azurelinux") => FormatVersionableOsName(osName, name => "Azure Linux"), - string s when s.Contains("cbl-mariner") => FormatVersionableOsName(osName, name => "CBL-Mariner"), - string s when s.Contains("leap") => FormatVersionableOsName(osName, name => "openSUSE Leap"), - string s when s.Contains("ubuntu") => FormatVersionableOsName(osName, name => "Ubuntu"), - string s when s.Contains("alpine") - || s.Contains("centos") - || s.Contains("fedora") => FormatVersionableOsName(osName, name => name.FirstCharToUpper()), - _ => throw new NotSupportedException($"The OS version '{osName}' is not supported.") - }; - - private static string GetWindowsVersionDisplayName(string windowsName, string version) => - version.StartsWith("ltsc") switch - { - true => $"{windowsName} {version.TrimStartString("ltsc")}", - false => $"{windowsName}, version {version}" - }; - - private static string FormatVersionableOsName(string os, Func formatName) - { - (string osName, string osVersion) = GetOsVersionInfo(os); - if (string.IsNullOrEmpty(osVersion)) - { - return formatName(osName); - } - else - { - return $"{formatName(osName)} {osVersion}"; - } - } - - private static (string Name, string Version) GetOsVersionInfo(string os) - { - // Regex matches an os name ending in a non-numeric or decimal character and up to - // a 3 part version number. Any additional characters are dropped (e.g. -distroless). - Regex versionRegex = new Regex(@"(?.+[^0-9\.])(?\d+(\.\d*){0,2})"); - Match match = versionRegex.Match(os); - - if (match.Success) - { - return (match.Groups["name"].Value, match.Groups["version"].Value); - } - else - { - return (os, string.Empty); - } - } - - private static string FirstCharToUpper(this string source) => char.ToUpper(source[0]) + source.Substring(1); - - [return: NotNullIfNotNull(nameof(source))] - private static string? TrimEndString(this string? source, string trim) => source switch - { - string s when s.EndsWith(trim) => s.Substring(0, s.Length - trim.Length).TrimEndString(trim), - _ => source, - }; - - [return: NotNullIfNotNull(nameof(source))] - private static string? TrimStartString(this string? source, string trim) => source switch - { - string s when s.StartsWith(trim) => s.Substring(trim.Length).TrimStartString(trim), - _ => source, - }; - [GeneratedRegex(@"(-(?[a-zA-Z_]*))?(?\d+.\d+)")] private static partial Regex OsVersionRegex { get; } } diff --git a/src/Templating/Readmes/MarkdownTableBuilder.cs b/src/Templating/Readmes/MarkdownTableBuilder.cs new file mode 100644 index 00000000..ef6ca1ca --- /dev/null +++ b/src/Templating/Readmes/MarkdownTableBuilder.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; +using Microsoft.DotNet.DockerTools.Templating.Abstractions; +using Microsoft.DotNet.DockerTools.Templating.Shared; +using Microsoft.DotNet.ImageBuilder.ReadModel; + +namespace Microsoft.DotNet.DockerTools.Templating.Readmes; + +internal sealed class MarkdownTableBuilder : ITableBuilder +{ + private readonly List _headings = []; + private readonly List> _rows = []; + + public ITableBuilder WithColumnHeadings(params IEnumerable headings) + { + _headings.Clear(); + _headings.AddRange(headings); + return this; + } + + public void AddRow(params IEnumerable row) => _rows.Add(row); + + public override string ToString() + { + var table = new StringBuilder(); + + if (_headings.Count > 0) + { + AppendRow(table, _headings); + table.AppendLine(); + + AppendRow(table, _headings.Select(_ => "---")); + table.AppendLine(); + } + + foreach (var row in _rows.WithIndex()) + { + AppendRow(table, row.Item); + + // Put lines between rows but not after the last row + if (row.Index != _rows.Count - 1) table.AppendLine(); + } + + return table.ToString(); + } + + private static StringBuilder AppendRow(StringBuilder stringBuilder, IEnumerable cells) => + stringBuilder.Append("| ").AppendJoin(" | ", cells).Append(" |"); +} diff --git a/src/Templating/Readmes/TagExtensions.cs b/src/Templating/Readmes/TagExtensions.cs new file mode 100644 index 00000000..1445f75e --- /dev/null +++ b/src/Templating/Readmes/TagExtensions.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.ImageBuilder.Models.Manifest; + +namespace Microsoft.DotNet.DockerTools.Templating.Readmes; + +internal static class TagExtensions +{ + extension(Tag tag) + { + public bool IsDocumented => tag.DocType switch + { + TagDocumentationType.Undocumented => false, + _ => true + }; + } +} diff --git a/src/Templating/Readmes/TagsTableGenerator.cs b/src/Templating/Readmes/TagsTableGenerator.cs new file mode 100644 index 00000000..5d8938b5 --- /dev/null +++ b/src/Templating/Readmes/TagsTableGenerator.cs @@ -0,0 +1,60 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; +using Microsoft.DotNet.DockerTools.Templating.Shared; +using Microsoft.DotNet.ImageBuilder.ReadModel; + +namespace Microsoft.DotNet.DockerTools.Templating.Readmes; + +public static class TagsTableGenerator +{ + public static string GenerateTagsTables(RepoInfo repo) + { + var output = new StringBuilder(); + + var documentedPlatforms = repo.Images + .SelectMany(image => image.Platforms) + .Where(platform => platform.Tags + .Any(tag => tag.IsDocumented)); + + var platformsByOsArch = documentedPlatforms + .GroupBy(platform => (platform.Model.OS, platform.Model.Architecture)); + + foreach (var archGroup in platformsByOsArch) + { + var os = archGroup.Key.OS.ToString(); + var arch = archGroup.Key.Architecture.GetDisplayName(); + + output.AppendLine($""" + + ### {os} {arch} Tags + + """); + + output.AppendLine(GeneratePlatformsTable(archGroup)); + } + + return output.ToString(); + } + + private static string GeneratePlatformsTable(IEnumerable platforms) + { + var table = new MarkdownTableBuilder() + .WithColumnHeadings("Tags", "Dockerfile", "OS Version"); + + foreach (var platform in platforms) + { + var tags = platform.Tags + .Where(tag => tag.IsDocumented) + .Select(tag => tag.Tag); + + table.AddRow( + string.Join(", ", tags), + platform.RelativeDockerfilePath, + platform.OSDisplayName); + } + + return table.ToString(); + } +} diff --git a/src/Templating/Shared/ArchDisplayExtensions.cs b/src/Templating/Shared/ArchDisplayExtensions.cs new file mode 100644 index 00000000..436fa4d4 --- /dev/null +++ b/src/Templating/Shared/ArchDisplayExtensions.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.ImageBuilder.Models.Manifest; + +namespace Microsoft.DotNet.DockerTools.Templating.Shared; + +internal static class ArchDisplayExtensions +{ + extension(Architecture architecture) + { + public string GetDisplayName(string? variant = null) + { + string displayName = architecture switch + { + Architecture.ARM => "arm32", + _ => architecture.ToString().ToLowerInvariant(), + }; + + if (variant != null) + { + displayName += variant.ToLowerInvariant(); + } + + return displayName; + } + + } +} diff --git a/src/Templating/Shared/OsHelper.cs b/src/Templating/Shared/OsHelper.cs new file mode 100644 index 00000000..7fa07c22 --- /dev/null +++ b/src/Templating/Shared/OsHelper.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.RegularExpressions; + +namespace Microsoft.DotNet.DockerTools.Templating.Shared; + +internal static class OsHelper +{ + public static string GetWindowsOSDisplayName(string osName) + { + string version = osName.Split('-')[1]; + return osName switch + { + var s when s.StartsWith("nanoserver") => + GetWindowsVersionDisplayName("Nano Server", version), + var s when s.StartsWith("windowsservercore") => + GetWindowsVersionDisplayName("Windows Server Core", version), + _ => throw new NotSupportedException($"The OS version '{osName}' is not supported.") + }; + } + + public static string GetLinuxOSDisplayName(string osName) => osName switch + { + string s when s.Contains("debian") => "Debian", + string s when s.Contains("bookworm") => "Debian 12", + string s when s.Contains("trixie") => "Debian 13", + string s when s.Contains("forky") => "Debian 14", + string s when s.Contains("duke") => "Debian 15", + string s when s.Contains("jammy") => "Ubuntu 22.04", + string s when s.Contains("noble") => "Ubuntu 24.04", + string s when s.Contains("azurelinux") => FormatVersionableOsName(osName, name => "Azure Linux"), + string s when s.Contains("cbl-mariner") => FormatVersionableOsName(osName, name => "CBL-Mariner"), + string s when s.Contains("leap") => FormatVersionableOsName(osName, name => "openSUSE Leap"), + string s when s.Contains("ubuntu") => FormatVersionableOsName(osName, name => "Ubuntu"), + string s when s.Contains("alpine") + || s.Contains("centos") + || s.Contains("fedora") => FormatVersionableOsName(osName, name => name.FirstCharToUpper()), + _ => throw new NotSupportedException($"The OS version '{osName}' is not supported.") + }; + + private static string GetWindowsVersionDisplayName(string windowsName, string version) => + version.StartsWith("ltsc") switch + { + true => $"{windowsName} {version.TrimStartString("ltsc")}", + false => $"{windowsName}, version {version}" + }; + + private static string FormatVersionableOsName(string os, Func formatName) + { + (string osName, string osVersion) = GetOsVersionInfo(os); + if (string.IsNullOrEmpty(osVersion)) + { + return formatName(osName); + } + else + { + return $"{formatName(osName)} {osVersion}"; + } + } + + private static (string Name, string Version) GetOsVersionInfo(string os) + { + // Regex matches an os name ending in a non-numeric or decimal character and up to + // a 3 part version number. Any additional characters are dropped (e.g. -distroless). + Regex versionRegex = new Regex(@"(?.+[^0-9\.])(?\d+(\.\d*){0,2})"); + Match match = versionRegex.Match(os); + + if (match.Success) + { + return (match.Groups["name"].Value, match.Groups["version"].Value); + } + else + { + return (os, string.Empty); + } + } +} diff --git a/src/Templating/Shared/PlatformInfoSharedExtensions.cs b/src/Templating/Shared/PlatformInfoSharedExtensions.cs new file mode 100644 index 00000000..6bd3ddbf --- /dev/null +++ b/src/Templating/Shared/PlatformInfoSharedExtensions.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.ImageBuilder.ReadModel; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; +using Microsoft.DotNet.DockerTools.Templating.Cottle; + +namespace Microsoft.DotNet.DockerTools.Templating.Shared; + +internal static class PlatformInfoSharedExtensions +{ + extension(PlatformInfo platform) + { + public string OSDisplayName => platform.Model.OS switch + { + OS.Windows => OsHelper.GetWindowsOSDisplayName(platform.BaseOsVersion), + _ => OsHelper.GetLinuxOSDisplayName(platform.BaseOsVersion) + }; + } +} diff --git a/src/Templating/Shared/StringExtensions.cs b/src/Templating/Shared/StringExtensions.cs new file mode 100644 index 00000000..1a84bdd0 --- /dev/null +++ b/src/Templating/Shared/StringExtensions.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.DotNet.DockerTools.Templating.Shared; + +internal static class StringExtensions +{ + public static string FirstCharToUpper(this string source) => char.ToUpper(source[0]) + source.Substring(1); + + [return: NotNullIfNotNull(nameof(source))] + public static string? TrimStartString(this string? source, string trim) => source switch + { + string s when s.StartsWith(trim) => s.Substring(trim.Length).TrimStartString(trim), + _ => source, + }; + + [return: NotNullIfNotNull(nameof(source))] + public static string? TrimEndString(this string? source, string trim) => source switch + { + string s when s.EndsWith(trim) => s.Substring(0, s.Length - trim.Length).TrimEndString(trim), + _ => source, + }; +} + +internal static class EnumerableExtensions +{ + extension(IEnumerable source) + { + public IEnumerable<(T Item, int Index)> WithIndex() => + source.Select((item, index) => (item, index)); + } +} From cd4fba9204ba4ec73039ccf6dfb79386ebfd313a Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Sun, 21 Sep 2025 11:48:00 -0700 Subject: [PATCH 38/38] Improve template engine abstraction --- .../Abstractions/ITemplateEngine.cs | 24 +++++++++++- src/Templating/Cottle/CottleTemplateEngine.cs | 39 ++++++------------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/src/Templating/Abstractions/ITemplateEngine.cs b/src/Templating/Abstractions/ITemplateEngine.cs index f830c974..def63a41 100644 --- a/src/Templating/Abstractions/ITemplateEngine.cs +++ b/src/Templating/Abstractions/ITemplateEngine.cs @@ -5,5 +5,27 @@ namespace Microsoft.DotNet.DockerTools.Templating.Abstractions; public interface ITemplateEngine { - ICompiledTemplate Compile(string template); + /// + /// Add global variables that will be available in all newly created + /// template contexts. + /// + void AddGlobalVariables(IDictionary variables); + + /// + /// Create a new context for rendering a template. + /// + /// + /// Dictionary of variables to add to context. These variables will take + /// precedence over any global variables already set in the engine. + /// + /// + /// The path to the current template is needed to correctly resolve paths + /// to sub-templates + /// + TContext CreateContext(IDictionary variables, string templatePath); + + /// + /// Read a template from a file and compile it. + /// + ICompiledTemplate ReadAndCompile(string path); } diff --git a/src/Templating/Cottle/CottleTemplateEngine.cs b/src/Templating/Cottle/CottleTemplateEngine.cs index bbe50e05..395cc2f5 100644 --- a/src/Templating/Cottle/CottleTemplateEngine.cs +++ b/src/Templating/Cottle/CottleTemplateEngine.cs @@ -18,7 +18,7 @@ public sealed class CottleTemplateEngine : ITemplateEngine }; private readonly IFileSystem _fileSystem; - private readonly ForeverCache> _templateCache; + private readonly ForeverCache _templateCache; private IContext _globalContext = Context.CreateBuiltin( new Dictionary() @@ -30,31 +30,17 @@ public sealed class CottleTemplateEngine : ITemplateEngine public CottleTemplateEngine(IFileSystem fileSystem) { _fileSystem = fileSystem; - _templateCache = new ForeverCache>(ReadAndCompileWithNoCache); + _templateCache = new ForeverCache(valueFactory: ReadAndCompileWithNoCache); } public int CompiledTemplateCacheHits => _templateCache.Hits; public int CompiledTemplateCacheMisses => _templateCache.Misses; - public ICompiledTemplate Compile(string template) - { - var documentResult = Document.CreateDefault(template, s_config); - var document = documentResult.DocumentOrThrow; - var compiledTemplate = new CottleTemplate(document); - return compiledTemplate; - } - public ICompiledTemplate ReadAndCompile(string path) { return _templateCache.GetOrAdd(path); } - private ICompiledTemplate ReadAndCompileWithNoCache(string path) - { - string content = _fileSystem.ReadAllText(path); - return Compile(content); - } - public void AddGlobalVariables(IDictionary variables) { var variableSymbols = new Dictionary @@ -65,17 +51,7 @@ public void AddGlobalVariables(IDictionary variables) _globalContext = _globalContext.Add(variableSymbols); } - /// - /// Create a new context for rendering a template. - /// - /// - /// Dictionary of variables to add to context. These variables will take - /// precedence over any global variables already set in the engine. - /// - /// - /// The path to the current template is needed to correctly resolve paths - /// to sub-templates - /// + /// public IContext CreateContext(IDictionary variables, string templatePath) { var variableSymbols = variables.ToCottleDictionary(); @@ -119,6 +95,15 @@ private Value CreateInsertTemplateFunction(IContext platformContext, string curr return Value.FromFunction(function); } + private CottleTemplate ReadAndCompileWithNoCache(string path) + { + string content = _fileSystem.ReadAllText(path); + var documentResult = Document.CreateDefault(content, s_config); + var document = documentResult.DocumentOrThrow; + var compiledTemplate = new CottleTemplate(document); + return compiledTemplate; + } + private static Value ReplaceFunction = Value.FromFunction( Function.CreatePure( (state, args) =>