Merge pull request #448 from NuGet/dev

[ReleasePrep][2018.05.14]RI of dev into master
NuGet · May 14, 2018 · e8d3b10 · e8d3b10
2 parents ac0e8b6 + ec58a5b
commit e8d3b10
Show file tree

Hide file tree

Showing 175 changed files with 3,729 additions and 2,427 deletions.
diff --git a/NuGet.Jobs.sln b/NuGet.Jobs.sln
@@ -119,8 +119,16 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.R
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.RevalidateCertificate.Tests", "tests\Validation.PackageSigning.RevalidateCertificate.Tests\Validation.PackageSigning.RevalidateCertificate.Tests.csproj", "{64095857-E9E3-4D9C-8769-7E558CD757CB}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.ScanAndSign.Core", "src\Validation.ScanAndSign.Core\Validation.ScanAndSign.Core.csproj", "{DFAC2769-4B67-4FBC-AD60-D93A39DD45AE}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests.Stats.AggregateCdnDownloadsInGallery", "tests\Tests.Stats.AggregateCdnDownloadsInGallery\Tests.Stats.AggregateCdnDownloadsInGallery.csproj", "{136411AF-B9FA-438D-B790-9FB78A5F7F54}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.ScanAndSign.Tests", "tests\Validation.PackageSigning.ScanAndSign.Tests\Validation.PackageSigning.ScanAndSign.Tests.csproj", "{CAE45AC9-F11B-4215-9D1A-C98BC0F1F687}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Monitoring", "Monitoring", "{814F9B31-4AF3-46CC-AD61-CEB40F47083A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Monitoring.PackageLag", "src\PackageLagMonitor\Monitoring.PackageLag.csproj", "{B5147169-E941-4CF8-9FCD-1C123ACD3149}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -309,10 +317,22 @@ Global
 		{64095857-E9E3-4D9C-8769-7E558CD757CB}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{64095857-E9E3-4D9C-8769-7E558CD757CB}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{64095857-E9E3-4D9C-8769-7E558CD757CB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DFAC2769-4B67-4FBC-AD60-D93A39DD45AE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DFAC2769-4B67-4FBC-AD60-D93A39DD45AE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DFAC2769-4B67-4FBC-AD60-D93A39DD45AE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DFAC2769-4B67-4FBC-AD60-D93A39DD45AE}.Release|Any CPU.Build.0 = Release|Any CPU
 		{136411AF-B9FA-438D-B790-9FB78A5F7F54}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{136411AF-B9FA-438D-B790-9FB78A5F7F54}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{136411AF-B9FA-438D-B790-9FB78A5F7F54}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{136411AF-B9FA-438D-B790-9FB78A5F7F54}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CAE45AC9-F11B-4215-9D1A-C98BC0F1F687}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CAE45AC9-F11B-4215-9D1A-C98BC0F1F687}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CAE45AC9-F11B-4215-9D1A-C98BC0F1F687}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CAE45AC9-F11B-4215-9D1A-C98BC0F1F687}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B5147169-E941-4CF8-9FCD-1C123ACD3149}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B5147169-E941-4CF8-9FCD-1C123ACD3149}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B5147169-E941-4CF8-9FCD-1C123ACD3149}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B5147169-E941-4CF8-9FCD-1C123ACD3149}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -363,7 +383,10 @@ Global
 		{26435822-8938-48C9-96FD-0DCCF8F7CE00} = {6A776396-02B1-475D-A104-26940ADB04AB}
 		{EA32E1E5-7E7D-44E6-B496-43E1FEDE9400} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
 		{64095857-E9E3-4D9C-8769-7E558CD757CB} = {6A776396-02B1-475D-A104-26940ADB04AB}
+		{DFAC2769-4B67-4FBC-AD60-D93A39DD45AE} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
 		{136411AF-B9FA-438D-B790-9FB78A5F7F54} = {6A776396-02B1-475D-A104-26940ADB04AB}
+		{CAE45AC9-F11B-4215-9D1A-C98BC0F1F687} = {6A776396-02B1-475D-A104-26940ADB04AB}
+		{B5147169-E941-4CF8-9FCD-1C123ACD3149} = {814F9B31-4AF3-46CC-AD61-CEB40F47083A}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B}

diff --git a/build.ps1 b/build.ps1
@@ -111,7 +111,9 @@ Invoke-BuildStep 'Set version metadata in AssemblyInfo.cs' { `
             "$PSScriptRoot\src\Validation.PackageSigning.ValidateCertificate\Properties\AssemblyInfo.g.cs",
             "$PSScriptRoot\src\Validation.PackageSigning.RevalidateCertificate\Properties\AssemblyInfo.g.cs",
             "$PSScriptRoot\src\NuGet.Jobs.Common\Properties\AssemblyInfo.g.cs",
-            "$PSScriptRoot\src\Validation.Common.Job\Properties\AssemblyInfo.g.cs"
+            "$PSScriptRoot\src\Validation.Common.Job\Properties\AssemblyInfo.g.cs",
+            "$PSScriptRoot\src\Validation.ScanAndSign.Core\Properties\AssemblyInfo.g.cs",
+            "$PSScriptRoot\src\PackageLagMonitor\Properties\AssemblyInfo.g.cs"
 
         $versionMetadata | ForEach-Object {
             Set-VersionInfo -Path $_ -Version $SimpleVersion -Branch $Branch -Commit $CommitSHA
@@ -143,7 +145,8 @@ Invoke-BuildStep 'Creating artifacts' {
         # don't need to be shared, hence no need for symbols for them
         $ProjectsWithSymbols =
             "src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj",
-            "src/Validation.Common.Job/Validation.Common.Job.csproj"
+            "src/Validation.Common.Job/Validation.Common.Job.csproj",
+            "src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj"
 
         $Projects = `
             "src/Stats.CollectAzureCdnLogs/Stats.CollectAzureCdnLogs.csproj", `
@@ -167,7 +170,8 @@ Invoke-BuildStep 'Creating artifacts' {
             "src/Stats.CollectAzureChinaCDNLogs/Stats.CollectAzureChinaCDNLogs.csproj", `
             "src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj", `
             "src/Validation.PackageSigning.ValidateCertificate/Validation.PackageSigning.ValidateCertificate.csproj", `
-            "src/Validation.PackageSigning.RevalidateCertificate/Validation.PackageSigning.RevalidateCertificate.csproj" `
+            "src/Validation.PackageSigning.RevalidateCertificate/Validation.PackageSigning.RevalidateCertificate.csproj", `
+            "src/PackageLagMonitor/Monitoring.PackageLag.csproj" `
             + $ProjectsWithSymbols
 
         Foreach ($Project in $Projects) {

diff --git a/...ZipLib.0.86.0/ICSharpCode.SharpZipLib.dll → ...ZipLib.0.86.0/ICSharpCode.SharpZipLib.dll b/...ZipLib.0.86.0/ICSharpCode.SharpZipLib.dll → ...ZipLib.0.86.0/ICSharpCode.SharpZipLib.dll
diff --git a/...ICSharpCode.SharpZipLib.0.86.0/license.md → ...ICSharpCode.SharpZipLib.0.86.0/license.md b/...ICSharpCode.SharpZipLib.0.86.0/license.md → ...ICSharpCode.SharpZipLib.0.86.0/license.md
diff --git a/...CollectAzureCdnLogs/References/Readme.txt → external/Readme.txt b/...CollectAzureCdnLogs/References/Readme.txt → external/Readme.txt
diff --git a/src/ArchivePackages/App.config b/src/ArchivePackages/App.config
@@ -1,45 +1,33 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <configuration>
     <startup> 
-        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.2" />
+        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.2"/>
     </startup>
   <runtime>
     <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
       <dependentAssembly>
-        <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-9.0.0.0" newVersion="9.0.0.0" />
+        <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral"/>
+        <bindingRedirect oldVersion="0.0.0.0-9.0.0.0" newVersion="9.0.0.0"/>
       </dependentAssembly>
       <dependentAssembly>
-        <assemblyIdentity name="Microsoft.Azure.KeyVault" publicKeyToken="31bf3856ad364e35" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-1.0.0.0" newVersion="1.0.0.0" />
+        <assemblyIdentity name="Microsoft.Azure.KeyVault" publicKeyToken="31bf3856ad364e35" culture="neutral"/>
+        <bindingRedirect oldVersion="0.0.0.0-1.0.0.0" newVersion="1.0.0.0"/>
       </dependentAssembly>
       <dependentAssembly>
-        <assemblyIdentity name="Microsoft.IdentityModel.Clients.ActiveDirectory.Platform" publicKeyToken="31bf3856ad364e35" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-3.13.4.878" newVersion="3.13.4.878" />
+        <assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral"/>
+        <bindingRedirect oldVersion="0.0.0.0-5.7.0.0" newVersion="5.7.0.0"/>
       </dependentAssembly>
       <dependentAssembly>
-        <assemblyIdentity name="Microsoft.IdentityModel.Clients.ActiveDirectory" publicKeyToken="31bf3856ad364e35" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-3.13.4.878" newVersion="3.13.4.878" />
+        <assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral"/>
+        <bindingRedirect oldVersion="0.0.0.0-5.7.0.0" newVersion="5.7.0.0"/>
       </dependentAssembly>
       <dependentAssembly>
-        <assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-5.7.0.0" newVersion="5.7.0.0" />
+        <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral"/>
+        <bindingRedirect oldVersion="0.0.0.0-5.7.0.0" newVersion="5.7.0.0"/>
       </dependentAssembly>
       <dependentAssembly>
-        <assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-5.7.0.0" newVersion="5.7.0.0" />
-      </dependentAssembly>
-      <dependentAssembly>
-        <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-5.7.0.0" newVersion="5.7.0.0" />
-      </dependentAssembly>
-      <dependentAssembly>
-        <assemblyIdentity name="NuGet.Services.KeyVault" publicKeyToken="31bf3856ad364e35" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-2.2.3.0" newVersion="2.2.3.0" />
-      </dependentAssembly>
-      <dependentAssembly>
-        <assemblyIdentity name="Microsoft.ApplicationInsights" publicKeyToken="31bf3856ad364e35" culture="neutral" />
-        <bindingRedirect oldVersion="0.0.0.0-2.1.0.0" newVersion="2.1.0.0" />
+        <assemblyIdentity name="Microsoft.ApplicationInsights" publicKeyToken="31bf3856ad364e35" culture="neutral"/>
+        <bindingRedirect oldVersion="0.0.0.0-2.1.0.0" newVersion="2.1.0.0"/>
       </dependentAssembly>
     </assemblyBinding>
   </runtime>

diff --git a/src/ArchivePackages/ArchivePackages.Job.cs b/src/ArchivePackages/ArchivePackages.Job.cs
@@ -1,19 +1,19 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-using Dapper;
-using Microsoft.WindowsAzure.Storage;
-using Microsoft.WindowsAzure.Storage.Blob;
-using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
-using System.Data.SqlClient;
-using System.Diagnostics;
+using System.ComponentModel.Design;
 using System.Diagnostics.Tracing;
 using System.Linq;
 using System.Threading.Tasks;
+using Dapper;
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Blob;
+using Newtonsoft.Json.Linq;
 using NuGet.Jobs;
-using Microsoft.Extensions.Logging;
+using NuGet.Services.KeyVault;
+using NuGet.Services.Sql;
 
 namespace ArchivePackages
 {
@@ -31,6 +31,7 @@ public class Job : JobBase
         /// Gets or sets an Azure Storage Uri referring to a container to use as the source for package blobs
         /// </summary>
         public CloudStorageAccount Source { get; set; }
+
         public string SourceContainerName { get; set; }
 
         /// <summary>
@@ -52,22 +53,22 @@ public class Job : JobBase
         /// Blob containing the cursor data. Cursor data comprises of cursorDateTime
         /// </summary>
         public string CursorBlobName { get; set; }
-
-        /// <summary>
-        /// Gets or sets a connection string to the database containing package data.
-        /// </summary>
-        public SqlConnectionStringBuilder PackageDatabase { get; set; }
+
+        private ISqlConnectionFactory _packageDbConnectionFactory;
 
         protected CloudBlobContainer SourceContainer { get; private set; }
+
         protected CloudBlobContainer PrimaryDestinationContainer { get; private set; }
+
         protected CloudBlobContainer SecondaryDestinationContainer { get; private set; }
 
         public Job() : base(JobEventSource.Log) { }
 
-        public override void Init(IDictionary<string, string> jobArgsDictionary)
+        public override void Init(IServiceContainer serviceContainer, IDictionary<string, string> jobArgsDictionary)
         {
-            PackageDatabase = new SqlConnectionStringBuilder(
-                        JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase));
+            var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector));
+            var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase);
+            _packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector);
 
             Source = CloudStorageAccount.Parse(
                         JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.Source));
@@ -91,9 +92,10 @@ public override void Init(IDictionary<string, string> jobArgsDictionary)
 
         public override async Task Run()
         {
-            JobEventSourceLog.PreparingToArchive(Source.Credentials.AccountName, SourceContainer.Name, PrimaryDestination.Credentials.AccountName, PrimaryDestinationContainer.Name, PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
+            JobEventSourceLog.PreparingToArchive(Source.Credentials.AccountName, SourceContainer.Name, PrimaryDestination.Credentials.AccountName, PrimaryDestinationContainer.Name, _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog);
             await Archive(PrimaryDestinationContainer);
 
+            // todo: consider reusing package query for primary and secondary archives
             if (SecondaryDestinationContainer != null)
             {
                 JobEventSourceLog.PreparingToArchive2(SecondaryDestination.Credentials.AccountName, SecondaryDestinationContainer.Name);
@@ -122,9 +124,9 @@ private async Task Archive(CloudBlobContainer destinationContainer)
 
             JobEventSourceLog.CursorData(cursorDateTime.ToString(DateTimeFormatSpecifier));
 
-            JobEventSourceLog.GatheringPackagesToArchiveFromDb(PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
+            JobEventSourceLog.GatheringPackagesToArchiveFromDb(_packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog);
             List<PackageRef> packages;
-            using (var connection = await PackageDatabase.ConnectTo())
+            using (var connection = await _packageDbConnectionFactory.CreateAsync())
             {
                 packages = (await connection.QueryAsync<PackageRef>(@"
 			    SELECT pr.Id, p.NormalizedVersion AS Version, p.Hash, p.LastEdited, p.Published
@@ -133,7 +135,7 @@ FROM Packages p
 			    WHERE Published > @cursorDateTime OR LastEdited > @cursorDateTime", new { cursorDateTime = cursorDateTime }))
                     .ToList();
             }
-            JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
+            JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog);
 
             var archiveSet = packages
                 .AsParallel()