Merge pull request #412 from NuGet/dev

[ReleasePrep][2018.04.25]RI of dev into master

src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs

@@ -14,6 +14,7 @@ public static class JobArgumentNames
         public const string Once = "Once";
         public const string Sleep = "Sleep";
         public const string Interval = "Interval";
+        public const string ReinitializeAfterSeconds = "ReinitializeAfterSeconds";
 
         public const string WhatIf = "WhatIf";
 
@@ -72,7 +73,7 @@ public static class JobArgumentNames
         public const string AzureCdnAccountNumber = "AzureCdnAccountNumber";
         public const string AzureCdnPlatform = "AzureCdnPlatform";
 
-        //Arguments shared by CollectAzureCdnLogs and ParseAzureCdnLogs
+        //Arguments shared by CollectAzureCdnLogs, ParseAzureCdnLogs, Search.GenerateAuxiliaryData
         public const string AzureCdnCloudStorageAccount = "AzureCdnCloudStorageAccount";
         public const string AzureCdnCloudStorageContainerName = "AzureCdnCloudStorageContainerName";
 

src/NuGet.Jobs.Common/JobRunner.cs

@@ -72,7 +72,9 @@ public static async Task Run(JobBase job, string[] commandLineArgs)
                 loggerFactory = ConfigureLogging(job);
 
                 var runContinuously = !JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.Once);
+                var reinitializeAfterSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.ReinitializeAfterSeconds);
                 var sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Sleep); // sleep is in milliseconds
+
                 if (!sleepDuration.HasValue)
                 {
                     sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Interval);
@@ -88,12 +90,19 @@ public static async Task Run(JobBase job, string[] commandLineArgs)
                     sleepDuration = 5000;
                 }
 
+                if (!reinitializeAfterSeconds.HasValue)
+                {
+                    _logger.LogInformation(
+                        $"{JobArgumentNames.ReinitializeAfterSeconds} command line argument is not provided or is not a valid integer. " +
+                        "The job will reinitialize on every iteration");
+                }
+
                 // Ensure that SSLv3 is disabled and that Tls v1.2 is enabled.
                 ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
                 ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
 
                 // Run the job loop
-                await JobLoop(job, runContinuously, sleepDuration.Value, jobArgsDictionary);
+                await JobLoop(job, runContinuously, sleepDuration.Value, reinitializeAfterSeconds, jobArgsDictionary);
             }
             catch (Exception ex)
             {
@@ -123,22 +132,34 @@ private static string PrettyPrintTime(double milliSeconds)
                 $"'{milliSeconds:F3}' ms (or '{seconds:F3}' seconds or '{minutes:F3}' mins)";
         }
 
-        private static async Task JobLoop(JobBase job, bool runContinuously, int sleepDuration, IDictionary<string, string> jobArgsDictionary)
+        private static async Task JobLoop(
+            JobBase job,
+            bool runContinuously,
+            int sleepDuration,
+            int? reinitializeAfterSeconds,
+            IDictionary<string, string> jobArgsDictionary)
         {
             // Run the job now
             var stopWatch = new Stopwatch();
+            Stopwatch timeSinceInitialization = null;
 
             while (true)
             {
                 _logger.LogInformation("Running {RunType}", (runContinuously ? " continuously..." : " once..."));
                 _logger.LogInformation("SleepDuration is {SleepDuration}", PrettyPrintTime(sleepDuration));
                 _logger.LogInformation("Job run started...");
-                
+
                 var initialized = false;
                 stopWatch.Restart();
+
                 try
                 {
-                    job.Init(jobArgsDictionary);
+                    if (ShouldInitialize(reinitializeAfterSeconds, timeSinceInitialization))
+                    {
+                        job.Init(jobArgsDictionary);
+                        timeSinceInitialization = Stopwatch.StartNew();
+                    }
+
                     initialized = true;
 
                     await job.Run();
@@ -169,5 +190,23 @@ private static async Task JobLoop(JobBase job, bool runContinuously, int sleepDu
                 await Task.Delay(sleepDuration);
             }
         }
+
+        private static bool ShouldInitialize(int? reinitializeAfterSeconds, Stopwatch timeSinceInitialization)
+        {
+            // If there is no wait time between reinitializations, always reinitialize.
+            if (!reinitializeAfterSeconds.HasValue)
+            {
+                return true;
+            }
+
+            // A null time since last initialization indicates that the job hasn't been initialized yet.
+            if (timeSinceInitialization == null)
+            {
+                return true;
+            }
+
+            // Otherwise, only reinitialize if the reinitialization threshold has been reached.
+            return (timeSinceInitialization.Elapsed.TotalSeconds > reinitializeAfterSeconds.Value);
+        }
     }
 }

src/NuGet.Services.Validation.Orchestrator/Job.cs

@@ -337,8 +337,8 @@ private static void ConfigurePackageSigningValidator(ContainerBuilder builder)
             builder
                 .RegisterType<ValidatorStateService>()
                 .WithParameter(
-                    (pi, ctx) => pi.ParameterType == typeof(Type),
-                    (pi, ctx) => typeof(PackageSigningValidator))
+                    (pi, ctx) => pi.ParameterType == typeof(string),
+                    (pi, ctx) => ValidatorName.PackageSigning)
                 .Keyed<IValidatorStateService>(PackageSigningBindingKey);
 
             // Configure the package signature verification enqueuer.
@@ -369,8 +369,8 @@ private static void ConfigurePackageCertificatesValidator(ContainerBuilder build
             builder
                 .RegisterType<ValidatorStateService>()
                 .WithParameter(
-                    (pi, ctx) => pi.ParameterType == typeof(Type),
-                    (pi, ctx) => typeof(PackageCertificatesValidator))
+                    (pi, ctx) => pi.ParameterType == typeof(string),
+                    (pi, ctx) => ValidatorName.PackageCertificate)
                 .Keyed<IValidatorStateService>(PackageCertificatesBindingKey);
 
             // Configure the certificate verification enqueuer.

src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj

@@ -58,7 +58,6 @@
     <Compile Include="IValidationSetProcessor.cs" />
     <Compile Include="IValidationSetProvider.cs" />
     <Compile Include="IValidationStorageService.cs" />
-    <Compile Include="IValidatorProvider.cs" />
     <Compile Include="Job.cs" />
     <Compile Include="MessageService.cs" />
     <Compile Include="OrchestrationRunner.cs" />
@@ -108,7 +107,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="NuGet.Services.Validation.Issues">
-      <Version>2.22.0</Version>
+      <Version>2.23.0</Version>
     </PackageReference>
   </ItemGroup>
   <ItemGroup>
@@ -136,4 +135,4 @@
     <SignPath Condition="'$(NuGetBuildPath)' != ''">$(NuGetBuildPath)</SignPath>
   </PropertyGroup>
   <Import Project="$(SignPath)\sign.targets" Condition="Exists('$(SignPath)\sign.targets')" />
-</Project>
+</Project>

src/NuGet.Services.Validation.Orchestrator/PackageSigning/ProcessSignature/PackageSigningValidator.cs

@@ -5,12 +5,14 @@
 using System.Diagnostics;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
+using NuGet.Jobs.Validation;
 using NuGet.Jobs.Validation.PackageSigning.Storage;
 using NuGet.Jobs.Validation.Storage;
 using NuGet.Services.Validation.Orchestrator.Telemetry;
 
 namespace NuGet.Services.Validation.PackageSigning.ProcessSignature
 {
+    [ValidatorName(ValidatorName.PackageSigning)]
     public class PackageSigningValidator : IProcessor
     {
         private readonly IValidatorStateService _validatorStateService;

src/NuGet.Services.Validation.Orchestrator/PackageSigning/ValidateCertificate/PackageCertificatesValidator.cs

@@ -8,12 +8,15 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
+using NuGet.Jobs.Validation;
 using NuGet.Jobs.Validation.PackageSigning.Storage;
 using NuGet.Services.Validation.Orchestrator;
 using NuGet.Services.Validation.Orchestrator.Telemetry;
+using Error = NuGet.Services.Validation.Orchestrator.Error;
 
 namespace NuGet.Services.Validation.PackageSigning.ValidateCertificate
 {
+    [ValidatorName(ValidatorName.PackageCertificate)]
     public class PackageCertificatesValidator : BaseValidator, IValidator
     {
         private static readonly TimeSpan DefaultCertificateRevalidationThresholdTime = TimeSpan.FromDays(1);
@@ -233,6 +236,7 @@ private Task<PackageSignature> FindSignatureAsync(IValidationRequest request)
                         .PackageSignatures
                         .Include(s => s.EndCertificate)
                         .Include(s => s.TrustedTimestamps.Select(t => t.EndCertificate))
+                        .Where(s => s.Type == PackageSignatureType.Author)
                         .SingleAsync(s => s.PackageKey == request.PackageKey);
         }
 

src/NuGet.Services.Validation.Orchestrator/ValidatorProvider.cs

@@ -7,6 +7,7 @@
 using System.Reflection;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using NuGet.Jobs.Validation;
 
 namespace NuGet.Services.Validation.Orchestrator
 {
@@ -55,13 +56,14 @@ private void InitializeEvaluatedTypes(Assembly callingAssembly)
                     var validatorTypes = candidateTypes
                         .Where(type => typeof(IValidator).IsAssignableFrom(type)
                                && type != typeof(IValidator)
-                               && type != typeof(IProcessor))
-                        .ToDictionary(type => type.Name);
+                               && type != typeof(IProcessor)
+                               && ValidatorUtility.HasValidatorNameAttribute(type))
+                        .ToDictionary(ValidatorUtility.GetValidatorName);
 
                     var processorTypes = validatorTypes
                         .Values
                         .Where(IsProcessorType)
-                        .ToDictionary(type => type.Name);
+                        .ToDictionary(ValidatorUtility.GetValidatorName);
 
                     _logger.LogTrace("After enumeration, got {NumImplementations} implementations: {TypeNames}",
                         validatorTypes.Count,

src/NuGet.Services.Validation.Orchestrator/Vcs/VcsValidator.cs

@@ -8,16 +8,19 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Microsoft.WindowsAzure.Storage;
+using NuGet.Jobs.Validation;
 using NuGet.Jobs.Validation.Common;
 using NuGet.Services.Validation.Orchestrator;
 using NuGet.Versioning;
 using NuGetGallery;
+using Error = NuGet.Services.Validation.Orchestrator.Error;
 
 namespace NuGet.Services.Validation.Vcs
 {
+    [ValidatorName(ValidatorName.Vcs)]
     public class VcsValidator : BaseValidator, IValidator
     {
-        private const string ValidatorName = Jobs.Validation.Common.Validators.Vcs.VcsValidator.ValidatorName;
+        private const string InternalValidatorName = Jobs.Validation.Common.Validators.Vcs.VcsValidator.ValidatorName;
 
         private readonly IPackageValidationService _validationService;
         private readonly IPackageValidationAuditor _validationAuditor;
@@ -61,7 +64,7 @@ public async Task<IValidationResult> GetResultAsync(IValidationRequest request)
 
             var validationStatusList = audit
                 .Entries
-                .Where(x => x.ValidatorName == ValidatorName)
+                .Where(x => x.ValidatorName == InternalValidatorName)
                 .Select(x => GetValidationStatus(request, x.EventId))
                 .ToList();
 
@@ -127,8 +130,8 @@ await _validationService.StartValidationProcessAsync(
                         Version = normalizedPackageVerison,
                         DownloadUrl = new Uri(request.NupkgUrl),
                     },
-                    new[] { ValidatorName },
-                    request.ValidationId);
+                    validators: new[] { InternalValidatorName },
+                    validationId: request.ValidationId);
             }
             catch (StorageException e) when (e.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.Conflict
                                              || e.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.PreconditionFailed)

src/NuGetCDNRedirect/Controllers/StatusController.cs

@@ -8,6 +8,7 @@ namespace NuGet.Services.CDNRedirect.Controllers
     public class StatusController : Controller
     {
         // GET: Status
+        [HttpGet]
         public ActionResult Index()
         {
             return View();

src/NuGetCDNRedirect/Views/Web.config

@@ -34,6 +34,7 @@
   </system.webServer>
 
   <system.web>
+    <httpCookies requireSSL="true" httpOnlyCookies="true" />
     <compilation>
       <assemblies>
         <add assembly="System.Web.Mvc, Version=5.2.3.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />

src/NuGetCDNRedirect/Web.config

@@ -11,6 +11,7 @@
     <add key="UnobtrusiveJavaScriptEnabled" value="true" />
   </appSettings>
   <system.web>
+    <httpCookies requireSSL="true" httpOnlyCookies="true" />
     <compilation debug="true" targetFramework="4.6.1" />
     <httpRuntime targetFramework="4.6.1" />
     <httpModules>

src/PackageHash/Settings/dev.json

@@ -18,7 +18,7 @@
   "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
   "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
   "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
-  "KeyVault_ValidateCertificate": false,
+  "KeyVault_ValidateCertificate": true,
   "KeyVault_StoreName": "My",
   "KeyVault_StoreLocation": "LocalMachine"
 }

src/PackageHash/Settings/int.json

@@ -18,7 +18,7 @@
   "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
   "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
   "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
-  "KeyVault_ValidateCertificate": false,
+  "KeyVault_ValidateCertificate": true,
   "KeyVault_StoreName": "My",
   "KeyVault_StoreLocation": "LocalMachine"
 }

src/PackageHash/Settings/prod.json

@@ -18,7 +18,7 @@
   "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
   "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
   "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
-  "KeyVault_ValidateCertificate": false,
+  "KeyVault_ValidateCertificate": true,
   "KeyVault_StoreName": "My",
   "KeyVault_StoreLocation": "LocalMachine"
 }

src/Search.GenerateAuxiliaryData/BlobStorageExporter.cs

@@ -0,0 +1,63 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Blob;
+
+namespace Search.GenerateAuxiliaryData
+{
+    /// <summary>
+    /// Used for data that needs to be copied from a blob storage
+    /// </summary>
+    public class BlobStorageExporter : Exporter
+    {
+        private static readonly TimeSpan MaxCopyDuration = TimeSpan.FromMinutes(10);
+        private static readonly TimeSpan CopyPollFrequency = TimeSpan.FromMilliseconds(500);
+
+        private CloudBlobContainer _sourceContainer;
+        private string _sourceName;
+
+        public BlobStorageExporter(ILogger<Exporter> logger, CloudBlobContainer sourceContainer, string sourceName, CloudBlobContainer destinationContainer, string destinationName):
+            base(logger, destinationContainer, destinationName)
+        {
+            _sourceContainer = sourceContainer ?? throw new ArgumentNullException(nameof(sourceContainer));
+            _sourceName = sourceName;
+        }
+
+        public override async Task ExportAsync()
+        {
+            _logger.LogInformation("Copying {ReportName} report from {ConnectionString}/{SourceName}", _name, _sourceContainer.Uri, _sourceName);
+
+            await _destinationContainer.CreateIfNotExistsAsync();
+
+            var sourceCloudBlob = _sourceContainer.GetBlockBlobReference(_sourceName);
+            var destinationCloudBlob = _destinationContainer.GetBlockBlobReference(_name);
+
+
+            await destinationCloudBlob.StartCopyAsync(sourceCloudBlob);
+
+            var stopwatch = Stopwatch.StartNew();
+            while (destinationCloudBlob.CopyState.Status == CopyStatus.Pending
+                   && stopwatch.Elapsed < MaxCopyDuration)
+            {
+                await destinationCloudBlob.FetchAttributesAsync();
+                await Task.Delay(CopyPollFrequency);
+            }
+
+            if (destinationCloudBlob.CopyState.Status == CopyStatus.Pending)
+            {
+                throw new TimeoutException($"Waiting for the blob copy operation to complete timed out after {MaxCopyDuration.TotalSeconds} seconds.");
+            }
+            else if (destinationCloudBlob.CopyState.Status != CopyStatus.Success)
+            {
+                throw new StorageException($"The blob copy operation had copy status {destinationCloudBlob.CopyState.Status} ({destinationCloudBlob.CopyState.StatusDescription}).");
+            }
+
+            _logger.LogInformation("Copy of {ReportName} completed. Took: {Seconds} seconds.", _name, stopwatch.Elapsed.TotalSeconds);
+        }
+    }
+}