Merge pull request #613 from NuGet/dev

[ReleasePrep][2018.11.02]RI of dev into master
NuGet · Nov 2, 2018 · 82843ac · 82843ac
2 parents 0d376eb + d0e0f42
commit 82843ac
Show file tree

Hide file tree

Showing 11 changed files with 287 additions and 7 deletions.
diff --git a/build.ps1 b/build.ps1
@@ -9,7 +9,7 @@ param (
     [string]$SemanticVersion = '1.0.0-zlocal',
     [string]$Branch = 'zlocal',
     [string]$CommitSHA,
-    [string]$BuildBranch = '795fed66b8bae2d248237ee5ec82e688e7174a42'
+    [string]$BuildBranch = '001c5deac1f1e2e194c1d0d5eec14183bb78e4cf'
 )
 
 $msBuildVersion = 15;

diff --git a/src/NuGet.Services.Revalidate/Configuration/RevalidationQueueConfiguration.cs b/src/NuGet.Services.Revalidate/Configuration/RevalidationQueueConfiguration.cs
@@ -7,6 +7,11 @@ namespace NuGet.Services.Revalidate
 {
     public class RevalidationQueueConfiguration
     {
+        /// <summary>
+        /// If non-null, this skips revalidations of packages with more than this many versions.
+        /// </summary>
+        public int? MaximumPackageVersions { get; set; }
+
         /// <summary>
         /// The maximum times that the <see cref="RevalidationQueue"/> should look for a revalidation
         /// before giving up.

diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationQueue.cs b/src/NuGet.Services.Revalidate/Services/RevalidationQueue.cs
@@ -45,7 +45,22 @@ public async Task<PackageRevalidation> NextOrNullAsync()
                     i + 1,
                     _config.MaximumAttempts);
 
-                var next = await _validationContext.PackageRevalidations
+                // Find the next package to revalidate. We will skip packages if:
+                //   1. The package has more than "MaximumPackageVersions" versions
+                //   2. The package has already been enqueued for revalidation
+                //   3. The package's revalidation was completed by an external factory (like manual admin revalidation)
+                IQueryable<PackageRevalidation> query = _validationContext.PackageRevalidations;
+
+                if (_config.MaximumPackageVersions.HasValue)
+                {
+                    query = query.Where(
+                        r =>
+                        !_validationContext.PackageRevalidations.GroupBy(r2 => r2.PackageId)
+                        .Where(g => g.Count() > _config.MaximumPackageVersions)
+                        .Any(g => g.Key == r.PackageId));
+                }
+
+                var next = await query
                     .Where(r => r.Enqueued == null)
                     .Where(r => r.Completed == false)
                     .OrderBy(r => r.Key)

diff --git a/src/NuGet.Services.Revalidate/Settings/dev.json b/src/NuGet.Services.Revalidate/Settings/dev.json
@@ -27,6 +27,7 @@
     },
 
     "Queue": {
+      "MaximumPackageVersions": #{Jobs.nuget.services.revalidate.MaximumPackageVersions},
       "MaximumAttempts": 5,
       "SleepBetweenAttempts": "00:00:30"
     }

diff --git a/src/NuGet.Services.Revalidate/Settings/int.json b/src/NuGet.Services.Revalidate/Settings/int.json
@@ -27,6 +27,7 @@
     },
 
     "Queue": {
+      "MaximumPackageVersions": #{Jobs.nuget.services.revalidate.MaximumPackageVersions},
       "MaximumAttempts": 5,
       "SleepBetweenAttempts": "00:00:30"
     }

diff --git a/src/NuGet.Services.Revalidate/Settings/prod.json b/src/NuGet.Services.Revalidate/Settings/prod.json
@@ -27,6 +27,7 @@
     },
 
     "Queue": {
+      "MaximumPackageVersions": #{Jobs.nuget.services.revalidate.MaximumPackageVersions},
       "MaximumAttempts": 5,
       "SleepBetweenAttempts": "00:00:30"
     }

diff --git a/src/NuGet.Services.Validation.Orchestrator/Services/SymbolEntityService.cs b/src/NuGet.Services.Validation.Orchestrator/Services/SymbolEntityService.cs
@@ -2,6 +2,7 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
+using System.Data.Entity;
 using System.Linq;
 using System.Threading.Tasks;
 using NuGetGallery;
@@ -42,7 +43,9 @@ public IValidatingEntity<SymbolPackage> FindPackageByKey(int key)
         {
             var symbolPackage = _symbolsPackageRepository
                    .GetAll()
-                   .SingleOrDefault(p => p.Key == key);
+                   .Include(sp => sp.Package)
+                   .Include(sp => sp.Package.PackageRegistration)
+                   .SingleOrDefault(sp => sp.Key == key);
             return symbolPackage == null ? null : new SymbolPackageValidatingEntity(symbolPackage);
         }
 

diff --git a/src/NuGet.Services.Validation.Orchestrator/Services/SymbolsMessageService.cs b/src/NuGet.Services.Validation.Orchestrator/Services/SymbolsMessageService.cs
@@ -43,6 +43,10 @@ public async Task SendPublishedMessageAsync(SymbolPackage symbolPackage)
                                     packageSupportUrl,
                                     _serviceConfiguration.EmailConfiguration.EmailSettingsUrl,
                                     Array.Empty<string>());
+
+            _logger.LogInformation("The publish email will be sent for the symbol {SymbolId} {SymbolVersion}",
+                   symbolPackage.Id,
+                   symbolPackage.Version);
             await _messageService.SendMessageAsync(symbolPackageAddedMessage);
         }
 
@@ -66,6 +70,10 @@ public async Task  SendValidationFailedMessageAsync(SymbolPackage symbolPackage,
                                    _serviceConfiguration.EmailConfiguration.AnnouncementsUrl,
                                    _serviceConfiguration.EmailConfiguration.TwitterUrl);
 
+            _logger.LogInformation("The validation failed  email will be sent for the symbol {SymbolId} {SymbolVersion} and ValidationSetKey {ValidationSetKey}",
+                   symbolPackage.Id,
+                   symbolPackage.Version,
+                   validationSet.Key);
             await _messageService.SendMessageAsync(symbolPackageValidationFailedMessage);
         }
 
@@ -80,6 +88,9 @@ public async Task SendValidationTakingTooLongMessageAsync(SymbolPackage symbolPa
                                    symbolPackage,
                                    _serviceConfiguration.GalleryPackageUrl(symbolPackage.Package.PackageRegistration.Id, symbolPackage.Package.NormalizedVersion));
 
+            _logger.LogInformation("The validation failed  email will be sent for the symbol {SymbolId} {SymbolVersion}.",
+                   symbolPackage.Id,
+                   symbolPackage.Version);
             await _messageService.SendMessageAsync(symbolPackageValidationTakingTooLongMessage);
         }
     }

diff --git a/src/Validation.Common.Job/Storage/ValidatorStateService.cs b/src/Validation.Common.Job/Storage/ValidatorStateService.cs
@@ -142,6 +142,7 @@ public async Task<ValidatorStatus> TryAddValidatorStatusAsync(IValidationRequest
                 _logger.LogWarning(
                     Error.ValidatorStateServiceFailedToAddStatus,
                     "Failed to add validation status for {ValidationId} ({PackageId} {PackageVersion}) as a record already exists",
+                    request.ValidationId,
                     request.PackageId,
                     request.PackageVersion);
 
@@ -169,6 +170,7 @@ public async Task<ValidatorStatus> TryUpdateValidationStatusAsync(IValidationReq
                 _logger.LogWarning(
                     Error.ValidatorStateServiceFailedToUpdateStatus,
                     "Failed to save validation status for {ValidationId} ({PackageId} {PackageVersion}) as the current status is stale",
+                    request.ValidationId,
                     request.PackageId,
                     request.PackageVersion);
 

diff --git a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationQueueFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationQueueFacts.cs
@@ -2,6 +2,7 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
+using System.Collections.Generic;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Moq;
@@ -132,6 +133,121 @@ public async Task SkipsRepositorySignedPackages()
                 _telemetryService.Verify(t => t.TrackPackageRevalidationMarkedAsCompleted("Repository.Signed.Package", "1.0.0"), Times.Once);
             }
 
+            [Theory]
+            [MemberData(nameof(SkipsPackagesWithTooManyVersionsData))]
+            public async Task SkipsPackagesWithTooManyVersions(int? maximumPackageVersions, bool skipsPackageWithManyVersions)
+            {
+                _config.MaximumPackageVersions = maximumPackageVersions;
+
+                // Arrange
+                _validationContext.Mock(packageRevalidations: new[]
+                {
+                    new PackageRevalidation
+                    {
+                        Key = 1,
+                        PackageId = "Package.With.Many.Versions",
+                        PackageNormalizedVersion = "1.0.0",
+                        Enqueued = null,
+                        Completed = false,
+                    },
+                    new PackageRevalidation
+                    {
+                        Key = 2,
+                        PackageId = "Package.With.Many.Versions",
+                        PackageNormalizedVersion = "2.0.0",
+                        Enqueued = null,
+                        Completed = false,
+                    },
+                    new PackageRevalidation
+                    {
+                        Key = 3,
+                        PackageId = "Package.With.Many.Versions",
+                        PackageNormalizedVersion = "3.0.0",
+                        Enqueued = null,
+                        Completed = false,
+                    },
+                    new PackageRevalidation
+                    {
+                        Key = 4,
+                        PackageId = "Package",
+                        PackageNormalizedVersion = "1.0.0",
+                        Enqueued = null,
+                        Completed = false,
+                    },
+                });
+
+                _galleryContext.Mock(packages: new[]
+                {
+                    new Package
+                    {
+                        PackageRegistration = new PackageRegistration { Id = "Package.With.Many.Versions" },
+                        NormalizedVersion = "1.0.0",
+                        PackageStatusKey = PackageStatus.Available,
+                    },
+                    new Package
+                    {
+                        PackageRegistration = new PackageRegistration { Id = "Package.With.Many.Versions" },
+                        NormalizedVersion = "2.0.0",
+                        PackageStatusKey = PackageStatus.Available,
+                    },
+                    new Package
+                    {
+                        PackageRegistration = new PackageRegistration { Id = "Package.With.Many.Versions" },
+                        NormalizedVersion = "3.0.0",
+                        PackageStatusKey = PackageStatus.Available,
+                    },
+                    new Package
+                    {
+                        PackageRegistration = new PackageRegistration { Id = "Package" },
+                        NormalizedVersion = "1.0.0",
+                        PackageStatusKey = PackageStatus.Available,
+                    },
+                });
+
+                // Act
+                var next = await _target.NextOrNullAsync();
+
+                // Assert
+                if (skipsPackageWithManyVersions)
+                {
+                    Assert.Equal("Package", next.PackageId);
+                    Assert.Equal("1.0.0", next.PackageNormalizedVersion);
+
+                    _telemetryService.Verify(t => t.TrackPackageRevalidationMarkedAsCompleted("Package.With.Many.Versions", "1.0.0"), Times.Never);
+                    _telemetryService.Verify(t => t.TrackPackageRevalidationMarkedAsCompleted("Package.With.Many.Versions", "2.0.0"), Times.Never);
+                    _telemetryService.Verify(t => t.TrackPackageRevalidationMarkedAsCompleted("Package.With.Many.Versions", "3.0.0"), Times.Never);
+                }
+                else
+                {
+                    Assert.Equal("Package.With.Many.Versions", next.PackageId);
+                    Assert.Equal("1.0.0", next.PackageNormalizedVersion);
+                }
+            }
+
+            public static IEnumerable<object[]> SkipsPackagesWithTooManyVersionsData()
+            {
+                // If the "MaximumPackageVersions" is null, no packages should be skipped for having too many versions.
+                yield return new object[]
+                {
+                    null,
+                    false
+                };
+
+                // If "MaximumPackageVersions" is set, packages with less versions than the value should not be skipped.
+                yield return new object[]
+                {
+                    100,
+                    false
+                };
+
+                // If "MaximumPackageVersions" is set, packages with more versions than the value should be skipped.
+                yield return new object[]
+                {
+                    2,
+                    true
+                };
+            }
+
             [Fact]
             public async Task SkipsDeletedPackages()
             {