Don't index packages from maintenance-packages repo (#584)

Author: gewarren

PackageIndexer/CsvUtils.cs

@@ -256,19 +256,10 @@ string targetFramework
         {
             bool includeXml = true;
 
-            if (PlatformPackageDefinition.packagesWithoutDocs.Contains(packageEntry.Name))
+            if (PlatformPackageDefinition.runtimePackagesWithoutDocs.Contains(packageEntry.Name) ||
+                PlatformPackageDefinition.otherPackagesWithoutDocs.Contains(packageEntry.Name))
                 includeXml = false;
 
-            // And don't include XML file for Microsoft.Extensions.Diagnostics.ResourceMonitoring
-            // (see https://github.com/dotnet/dotnet-api-docs/pull/10395#discussion_r1758128787).
-            if (string.Equals(
-                packageEntry.Name,
-                "Microsoft.Extensions.Diagnostics.ResourceMonitoring",
-                StringComparison.InvariantCultureIgnoreCase))
-            {
-                includeXml = false;
-            }
-
             // And don't include XMl files for Microsoft.Bcl.* packages.
             if (packageEntry.Name.StartsWith("Microsoft.Bcl.", StringComparison.InvariantCultureIgnoreCase))
                 includeXml = false;

PackageIndexer/DotnetPackageIndex.cs

@@ -86,7 +86,7 @@ params string[] feedUrls
 
         var connectionString = new KustoConnectionStringBuilder(cluster).WithAadUserPromptAuthentication();
         using var queryProvider = KustoClientFactory.CreateCslQueryProvider(connectionString);
-        using var reader = queryProvider.ExecuteQuery(databaseName, query, null);
+        using System.Data.IDataReader reader = queryProvider.ExecuteQuery(databaseName, query, null);
 
         List<string> packageIds = [];
 

PackageIndexer/PackageFilter.cs

@@ -64,8 +64,8 @@ public bool IsMatch(string packageId)
 
 public sealed class PackageFilter(IEnumerable<PackageFilterExpression> includes, IEnumerable<PackageFilterExpression> excludes)
 {
-    public ImmutableArray<PackageFilterExpression> Includes { get; } = includes.ToImmutableArray();
-    public ImmutableArray<PackageFilterExpression> Excludes { get; } = excludes.ToImmutableArray();
+    public ImmutableArray<PackageFilterExpression> Includes { get; } = [.. includes];
+    public ImmutableArray<PackageFilterExpression> Excludes { get; } = [.. excludes];
 
     public bool IsMatch(string packageId)
     {

PackageIndexer/PlatformPackageDefinition.cs

@@ -6,14 +6,25 @@ internal static class PlatformPackageDefinition
 {
     private static FrozenSet<string> s_packageIds;
 
-    public static readonly List<string> packagesWithoutDocs =
+    public static readonly List<string> otherPackagesWithoutDocs =
+    [
+        // See https://github.com/dotnet/dotnet-api-docs/pull/10395#discussion_r1758128787.
+        "Microsoft.Extensions.Diagnostics.ResourceMonitoring",
+        // From WinForms and doesn't use compiler-generated XML docs.
+        "System.Drawing.Common",
+        // WCF
+        "System.ServiceModel.Federation"
+    ];
+
+    public static readonly List<string> runtimePackagesWithoutDocs =
     [
         "Microsoft.Extensions.DependencyModel",
         "Microsoft.Extensions.FileSystemGlobbing",
         "Microsoft.NETCore.Platforms",
         "Microsoft.Win32.Registry.AccessControl",
         "Microsoft.Win32.SystemEvents",
         "System.CodeDom",
+        "System.Collections.Immutable",
         "System.ComponentModel.Composition",
         "System.ComponentModel.Composition.Registration",
         "System.Composition.AttributedModel",
@@ -24,21 +35,26 @@ internal static class PlatformPackageDefinition
         "System.Configuration.ConfigurationManager",
         "System.Data.Odbc",
         "System.Data.OleDb",
+        "System.Diagnostics.DiagnosticSource",
         "System.Diagnostics.EventLog",
         "System.Diagnostics.PerformanceCounter",
-        "System.DirectoryServices",
         "System.DirectoryServices.AccountManagement",
+        "System.DirectoryServices",
         "System.DirectoryServices.Protocols",
         "System.IO.Hashing",
         "System.IO.Packaging",
+        "System.IO.Pipelines",
         "System.IO.Ports",
         "System.Management",
         "System.Memory.Data",
+        "System.Net.Http.Json",
         "System.Net.Http.WinHttpHandler",
         "System.Reflection.Context",
+        "System.Reflection.Metadata",
         "System.Reflection.MetadataLoadContext",
         "System.Resources.Extensions",
         "System.Runtime.Caching",
+        "System.Runtime.Serialization.Formatters",
         "System.Security.Cryptography.Cose",
         "System.Security.Cryptography.Pkcs",
         "System.Security.Cryptography.ProtectedData",
@@ -47,6 +63,12 @@ internal static class PlatformPackageDefinition
         "System.ServiceModel.Syndication",
         "System.ServiceProcess.ServiceController",
         "System.Speech",
+        "System.Text.Encoding.CodePages",
+        "System.Text.Encodings.Web",
+        "System.Text.Json",
+        "System.Threading.AccessControl",
+        "System.Threading.Channels",
+        "System.Threading.Tasks.Dataflow",
         "System.Windows.Extensions"
     ];
 
@@ -56,6 +78,12 @@ internal static class PlatformPackageDefinition
         "dotnetframework"
     ], StringComparer.OrdinalIgnoreCase);
 
+    // Repository URLs to exclude
+    public static readonly FrozenSet<string> ExcludedRepositories = FrozenSet.ToFrozenSet(
+    [
+        "https://github.com/dotnet/maintenance-packages"
+    ], StringComparer.OrdinalIgnoreCase);
+
     public static PackageFilter Filter { get; } = new(
         includes:
         [
@@ -100,6 +128,26 @@ internal static class PlatformPackageDefinition
             PackageFilterExpression.Parse("System.Cloud.Messaging.Abstractions"),
             // Test APIs.
             PackageFilterExpression.Parse("Microsoft.Extensions.DependencyInjection.Specification.Tests"),
+            // No longer built in runtime repo.
+            PackageFilterExpression.Parse("System.Reflection.Emit"),
+            PackageFilterExpression.Parse("System.ComponentModel.Annotations"),
+            PackageFilterExpression.Parse("System.Data.DataSetExtensions"),
+            PackageFilterExpression.Parse("System.Security.AccessControl"),
+            PackageFilterExpression.Parse("System.Security.Cryptography.Cng"),
+            PackageFilterExpression.Parse("System.Security.Principal.Windows"),
+            // Maintenance packages.
+            PackageFilterExpression.Parse("Microsoft.IO.Redist"),
+            PackageFilterExpression.Parse("System.Buffers"),
+            PackageFilterExpression.Parse("System.Data.SqlClient"),
+            PackageFilterExpression.Parse("System.Json"),
+            PackageFilterExpression.Parse("System.Memory"),
+            PackageFilterExpression.Parse("System.Net.WebSockets.WebSocketProtocol"),
+            PackageFilterExpression.Parse("System.Numerics.Vectors"),
+            PackageFilterExpression.Parse("System.Reflection.DispatchProxy"),
+            PackageFilterExpression.Parse("System.Runtime.CompilerServices.Unsafe"),
+            PackageFilterExpression.Parse("System.Threading.Tasks.Extensions"),
+            PackageFilterExpression.Parse("System.ValueTuple"),
+            PackageFilterExpression.Parse("System.Xml.XPath.XmlDocument"),
             // Suffixes.
             PackageFilterExpression.Parse("*.cs"),
             PackageFilterExpression.Parse("*.de"),
@@ -117,13 +165,11 @@ internal static class PlatformPackageDefinition
         ]
     );
 
-    // For ASP.NET, include these packages:
-    //        PackageFilterExpression.Parse("Microsoft.AspNetCore*"),
-    //        PackageFilterExpression.Parse("Microsoft.Authentication.WebAssembly.Msal"),
-    //        PackageFilterExpression.Parse("Microsoft.JSInterop*"),
-    //        PackageFilterExpression.Parse("Microsoft.Net.Http.Headers"),
-    //        PackageFilterExpression.Parse("Microsoft.Extensions.ApiDescription.Server"),
-    //        PackageFilterExpression.Parse("Microsoft.Extensions.Features"),
-    //        PackageFilterExpression.Parse("Microsoft.Extensions.Identity.Core"),
-    //        PackageFilterExpression.Parse("Microsoft.Extensions.Identity.Stores")
+    public static bool IsRepositoryExcluded(string? repositoryUrl)
+    {
+        // Exclude package if the repository URL is missing,
+        // or if the repo is in the excluded list.
+        return (string.IsNullOrWhiteSpace(repositoryUrl) ||
+               ExcludedRepositories.Contains(repositoryUrl));
+    }
 }

PackageIndexer/Program.cs

@@ -8,7 +8,7 @@ internal static class Program
     private static async Task<int> Main(string[] args)
     {
 #if DEBUG
-        args = [@"c:\users\gewarren\desktop\Package Index 0227", "preview"];
+        args = [@"c:\users\gewarren\desktop\Package Index 1015", "preview"];
 #endif
 
         if ((args.Length == 0) || (args.Length > 2))
@@ -126,6 +126,12 @@ string indexPackagesPath
                         File.WriteAllText(disabledPath, string.Empty);
                         nugetStore.DeleteFromCache(id, version);
                     }
+                    else if (PlatformPackageDefinition.IsRepositoryExcluded(packageEntry.Repository))
+                    {
+                        Console.WriteLine($"Excluding due to repository: {packageEntry.Repository}");
+                        File.WriteAllText(disabledPath, $"Excluded repository: {packageEntry.Repository}");
+                        nugetStore.DeleteFromCache(id, version);
+                    }
                     else
                     {
                         using (FileStream stream = File.Create(path))