Merge branch 'dev' into merge-master

.gitignore

@@ -207,3 +207,5 @@ Results.*.xml
 AssemblyInfo.*.cs
 /tests/Validation.Helper.Tests/Validation.Helper.Tests.nuget.props
 /tests/Validation.Common.Tests/Validation.Common.Tests.nuget.props
+/tests/Validation.Common.Tests/Validation.Common.Tests.nuget.targets
+/tests/Validation.Helper.Tests/Validation.Helper.Tests.nuget.targets

src/NuGet.Services.Validation.Orchestrator/IValidationSetProvider.cs

@@ -17,7 +17,10 @@ public interface IValidationSetProvider
         /// </summary>
         /// <param name="validationTrackingId">Validation tracking id</param>
         /// <param name="package">Package details from Gallery DB</param>
-        /// <returns><see cref="PackageValidationSet"/> object with information about requested <paramref name="validationTrackingId"/></returns>
-        Task<PackageValidationSet> GetOrCreateValidationSetAsync(Guid validationTrackingId, Package package);
+        /// <returns><see cref="PackageValidationSet"/> object with information about
+        /// requested <paramref name="validationTrackingId"/>. Null if no further processing
+        /// should be made (e.g. duplicate validation request was detected).
+        /// </returns>
+        Task<PackageValidationSet> TryGetOrCreateValidationSetAsync(Guid validationTrackingId, Package package);
     }
 }

src/NuGet.Services.Validation.Orchestrator/IValidationStorageService.cs

@@ -25,6 +25,13 @@ public interface IValidationStorageService
         /// <returns>Validation set instance if found, null otherwise.</returns>
         Task<PackageValidationSet> GetValidationSetAsync(Guid validationTrackingId);
 
+        /// <summary>
+        /// Gets the number of validation sets that the provided package key has.
+        /// </summary>
+        /// <param name="packageKey">The package key.</param>
+        /// <returns>The count.</returns>
+        Task<int> GetValidationSetCountAsync(int packageKey);
+
         /// <summary>
         /// Updates the passed <see cref="PackageValidation"/> with the validation result's status,
         /// updates the <see cref="PackageValidation.ValidationStatusTimestamp"/> to current timestamp,
@@ -50,5 +57,20 @@ public interface IValidationStorageService
         /// <param name="validationResult">The result of the validation.</param>
         /// <returns>Task object tracking the async operation status.</returns>
         Task UpdateValidationStatusAsync(PackageValidation packageValidation, IValidationResult validationResult);
+
+        /// <summary>
+        /// Checks whether a validation set was created within the time range specified
+        /// by <paramref name="recentDuration"/> argument with tracking id that is different
+        /// from the supplied as a <paramref name="currentValidationSetTrackingId"/> argument.
+        /// </summary>
+        /// <param name="packageId">Package ID for which recent validation sets are to be looked up.</param>
+        /// <param name="normalizedVersion">Normalized version of the package.</param>
+        /// <param name="recentDuration">Max amount of time to look back.</param>
+        /// <param name="currentValidationSetTrackingId">Validation set tracking for the currently processed request.</param>
+        /// <returns>True if validation set exists, false otherwise.</returns>
+        Task<bool> OtherRecentValidationSetForPackageExists(
+            int packageKey,
+            TimeSpan recentDuration,
+            Guid currentValidationSetTrackingId);
     }
 }

src/NuGet.Services.Validation.Orchestrator/Job.cs

@@ -10,6 +10,7 @@
 using Autofac;
 using Autofac.Core;
 using Autofac.Extensions.DependencyInjection;
+using Microsoft.ApplicationInsights;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
@@ -22,7 +23,9 @@
 using NuGet.Jobs.Validation.PackageSigning.Storage;
 using NuGet.Services.Configuration;
 using NuGet.Services.KeyVault;
+using NuGet.Services.Logging;
 using NuGet.Services.ServiceBus;
+using NuGet.Services.Validation.Orchestrator.Telemetry;
 using NuGet.Services.Validation.PackageCertificates;
 using NuGet.Services.Validation.PackageSigning;
 using NuGet.Services.Validation.Vcs;
@@ -207,6 +210,8 @@ private void ConfigureJobServices(IServiceCollection services, IConfigurationRoo
             services.AddTransient<ICoreMessageServiceConfiguration, CoreMessageServiceConfiguration>();
             services.AddTransient<ICoreMessageService, CoreMessageService>();
             services.AddTransient<IMessageService, MessageService>();
+            services.AddTransient<ITelemetryService, TelemetryService>();
+            services.AddSingleton(new TelemetryClient());
         }
 
         private static IServiceProvider CreateProvider(IServiceCollection services)

src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj

@@ -72,6 +72,8 @@
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Properties\AssemblyInfo.*.cs" />
     <Compile Include="SmtpConfiguration.cs" />
+    <Compile Include="Telemetry\ITelemetryService.cs" />
+    <Compile Include="Telemetry\TelemetryService.cs" />
     <Compile Include="ValidationConfiguration.cs" />
     <Compile Include="ValidationConfigurationItem.cs" />
     <Compile Include="ValidationFailureBehavior.cs" />

src/NuGet.Services.Validation.Orchestrator/Telemetry/ITelemetryService.cs

@@ -0,0 +1,82 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using NuGetGallery;
+
+namespace NuGet.Services.Validation.Orchestrator.Telemetry
+{
+    /// <summary>
+    /// The interface used for emitting telemetry from the validation orchestrator.
+    /// </summary>
+    public interface ITelemetryService
+    {
+        /// <summary>
+        /// The duration from when the package was created to when the first validation set was created. This metric
+        /// is not emitted for revalidation requests.
+        /// </summary>
+        void TrackDurationToValidationSetCreation(TimeSpan duration);
+
+        /// <summary>
+        /// A counter metric emitted when a package changes package status. This metric is not emitted if package status
+        /// does not change. This metric is emitted for revalidation if the terminal state changes.
+        /// </summary>
+        /// <param name="fromStatus">The status that the package moved from.</param>
+        /// <param name="toStatus">The status that the package moved tp.</param>
+        void TrackPackageStatusChange(PackageStatus fromStatus, PackageStatus toStatus);
+
+        /// <summary>
+        /// The total duration of all validators. This is the time that the validation set is first created until all of
+        /// the validators have completed. This metric is also emitted for revalidations.
+        /// </summary>
+        /// <param name="duration">The duration.</param>
+        /// <param name="isSuccess">Whether or not all of the validations succeeded.</param>
+        void TrackTotalValidationDuration(TimeSpan duration, bool isSuccess);
+
+        /// <summary>
+        /// A counter metric emitted when a validator fails due to the <see cref="ValidationConfigurationItem.FailAfter"/>
+        /// configuration.
+        /// </summary>
+        /// <param name="validatorType">The validator type (name).</param>
+        void TrackValidatorTimeout(string validatorType);
+
+        /// <summary>
+        /// The total duration of a single validator. This is the time from when the validation is first started until
+        /// when the validation either completes or times out.
+        /// </summary>
+        /// <param name="duration">The duration.</param>
+        /// <param name="validatorType">The validator type (name).</param>
+        /// <param name="isSuccess">Whether or not the validation succeeded.</param>
+        void TrackValidatorDuration(TimeSpan duration, string validatorType, bool isSuccess);
+
+        /// <summary>
+        /// A counter metric emitted when a validator is started.
+        /// </summary>
+        /// <param name="validatorType">The validator type (name).</param>
+        void TrackValidatorStarted(string validatorType);
+
+        /// <summary>
+        /// A counter metric emmitted when a validator reaches a terminal state and potentially persists validation
+        /// issues. A count of zero is emitted if the validator does not produce any issues. This metric is not emitted
+        /// if the validation is still at a non-terminal state.
+        /// </summary>
+        /// <param name="count">The number of issues.</param>
+        /// <param name="validatorType">The validator type (name) that returned the issue list.</param>
+        /// <param name="isSuccess">Whether or not the validation succeeded.</param>
+        void TrackValidationIssueCount(int count, string validatorType, bool isSuccess);
+
+        /// <summary>
+        /// A counter metric emitted when a validation issue is created.
+        /// </summary>
+        /// <param name="validatorType">The validator type (name) the produced the issue.</param>
+        /// <param name="code">The issue code.</param>
+        void TrackValidationIssue(string validatorType, ValidationIssueCode code);
+
+        /// <summary>
+        /// A counter metric emitted when a client-mastered validation issue is emitted.
+        /// </summary>
+        /// <param name="validatorType">The validator type (name) the produced the issue.</param>
+        /// <param name="clientCode">The client code.</param>
+        void TrackClientValidationIssue(string validatorType, string clientCode);
+    }
+}

src/NuGet.Services.Validation.Orchestrator/Telemetry/TelemetryService.cs

@@ -0,0 +1,139 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.ApplicationInsights;
+using NuGetGallery;
+
+namespace NuGet.Services.Validation.Orchestrator.Telemetry
+{
+    public class TelemetryService : ITelemetryService
+    {
+        private const string Prefix = "Orchestrator.";
+
+        private const string DurationToValidationSetCreationSeconds = Prefix + "DurationToValidationSetCreationSeconds";
+        private const string PackageStatusChange = Prefix + "PackageStatusChange";
+        private const string TotalValidationDurationSeconds = Prefix + "TotalValidationDurationSeconds";
+        private const string ValidationIssue = Prefix + "ValidationIssue";
+        private const string ValidationIssueCount = Prefix + "ValidationIssueCount";
+        private const string ValidatorTimeout = Prefix + "ValidatorTimeout";
+        private const string ValidatorDurationSeconds = Prefix + "ValidatorDurationSeconds";
+        private const string ValidatorStarted = Prefix + "ValidatorStarted";
+        private const string ClientValidationIssue = Prefix + "ClientValidationIssue";
+
+        private const string FromStatus = "FromStatus";
+        private const string ToStatus = "ToStatus";
+        private const string IsSuccess = "IsSuccess";
+        private const string ValidatorType = "ValidatorType";
+        private const string IssueCode = "IssueCode";
+        private const string ClientCode = "ClientCode";
+
+        private readonly TelemetryClient _telemetryClient;
+
+        public TelemetryService(TelemetryClient telemetryClient)
+        {
+            _telemetryClient = telemetryClient ?? throw new ArgumentNullException(nameof(telemetryClient));
+        }
+
+        public void TrackDurationToValidationSetCreation(TimeSpan duration)
+        {
+            _telemetryClient.TrackMetric(
+                DurationToValidationSetCreationSeconds,
+                duration.TotalSeconds);
+        }
+
+        public void TrackPackageStatusChange(PackageStatus fromStatus, PackageStatus toStatus)
+        {
+            _telemetryClient.TrackMetric(
+                PackageStatusChange,
+                1,
+                new Dictionary<string, string>
+                {
+                    { FromStatus, fromStatus.ToString() },
+                    { ToStatus, toStatus.ToString() },
+                });
+        }
+
+        public void TrackTotalValidationDuration(TimeSpan duration, bool isSuccess)
+        {
+            _telemetryClient.TrackMetric(
+                TotalValidationDurationSeconds,
+                duration.TotalSeconds,
+                new Dictionary<string, string>
+                {
+                    { IsSuccess, isSuccess.ToString() },
+                });
+        }
+
+        public void TrackValidationIssue(string validatorType, ValidationIssueCode code)
+        {
+            _telemetryClient.TrackMetric(
+                ValidationIssue,
+                1,
+                new Dictionary<string, string>
+                {
+                    { ValidatorType, validatorType },
+                    { IssueCode, code.ToString() },
+                });
+        }
+
+        public void TrackValidationIssueCount(int count, string validatorType, bool isSuccess)
+        {
+            _telemetryClient.TrackMetric(
+                ValidationIssueCount,
+                count,
+                new Dictionary<string, string>
+                {
+                    { ValidatorType, validatorType },
+                    { IsSuccess, isSuccess.ToString() },
+                });
+        }
+
+        public void TrackValidatorTimeout(string validatorType)
+        {
+            _telemetryClient.TrackMetric(
+                ValidatorTimeout,
+                1,
+                new Dictionary<string, string>
+                {
+                    { ValidatorType, validatorType },
+                });
+        }
+
+        public void TrackValidatorDuration(TimeSpan duration, string validatorType, bool isSuccess)
+        {
+            _telemetryClient.TrackMetric(
+                ValidatorDurationSeconds,
+                duration.TotalSeconds,
+                new Dictionary<string, string>
+                {
+                    { ValidatorType, validatorType },
+                    { IsSuccess, isSuccess.ToString() },
+                });
+        }
+
+        public void TrackValidatorStarted(string validatorType)
+        {
+            _telemetryClient.TrackMetric(
+                ValidatorStarted,
+                1,
+                new Dictionary<string, string>
+                {
+                    { ValidatorType, validatorType },
+                });
+        }
+
+        public void TrackClientValidationIssue(string validatorType, string clientCode)
+        {
+            _telemetryClient.TrackMetric(
+                ClientValidationIssue,
+                1,
+                new Dictionary<string, string>
+                {
+                    { ValidatorType, validatorType },
+                    { ClientCode, clientCode },
+                });
+        }
+    }
+}

src/NuGet.Services.Validation.Orchestrator/ValidationConfiguration.cs

@@ -25,5 +25,13 @@ public class ValidationConfiguration
         /// Time to wait between checking the state of a certain validation.
         /// </summary>
         public TimeSpan ValidationMessageRecheckPeriod { get; set; }
+
+        /// <summary>
+        /// The duplication detection window used for new validation requests.
+        /// Validation requests will be ignored if there exists another validation
+        /// request for the same package id and version that was created within
+        /// this window.
+        /// </summary>
+        public TimeSpan NewValidationRequestDeduplicationWindow { get; set; }
     }
 }

src/NuGet.Services.Validation.Orchestrator/ValidationFailureBehavior.cs

@@ -4,7 +4,7 @@
 namespace NuGet.Services.Validation.Orchestrator
 {
     /// <summary>
-    /// Options for orchestrator for how to handle valiation failure
+    /// Options for orchestrator for how to handle validation failure
     /// </summary>
     public enum ValidationFailureBehavior
     {

src/NuGet.Services.Validation.Orchestrator/ValidationMessageHandler.cs

@@ -46,7 +46,16 @@ public async Task<bool> HandleAsync(PackageValidationMessageData message)
 
             using (_logger.BeginScope("Handling message for {PackageId} {PackageVersion} validation set {ValidationSetId}", message.PackageId, message.PackageVersion, message.ValidationTrackingId))
             {
-                var validationSet = await _validationSetProvider.GetOrCreateValidationSetAsync(message.ValidationTrackingId, package);
+                var validationSet = await _validationSetProvider.TryGetOrCreateValidationSetAsync(message.ValidationTrackingId, package);
+
+                if (validationSet == null)
+                {
+                    _logger.LogInformation("The validation request for {PackageId} {PackageVersion} validation set {ValidationSetId} is a duplicate. Discarding.",
+                        message.PackageId,
+                        message.PackageVersion,
+                        message.ValidationTrackingId);
+                    return true;
+                }
 
                 await _validationSetProcessor.ProcessValidationsAsync(validationSet, package);
                 await _validationOutcomeProcessor.ProcessValidationOutcomeAsync(validationSet, package);