Merge pull request #533 from NuGet/dev

[ReleasePrep][2018.08.13]RI of dev into master

NuGet.Jobs.sln

@@ -151,6 +151,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Symbols.Tests",
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Symbols.Core.Tests", "tests\Validation.Symbols.Core.Tests\Validation.Symbols.Core.Tests.csproj", "{9ED642DF-4623-4EB2-8B72-52C6489BF9D6}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Common.Tests", "tests\NuGet.Jobs.Common.Tests\NuGet.Jobs.Common.Tests.csproj", "{CE96428B-8138-4914-9999-2B391797FFF8}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -399,6 +401,10 @@ Global
 		{9ED642DF-4623-4EB2-8B72-52C6489BF9D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{9ED642DF-4623-4EB2-8B72-52C6489BF9D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{9ED642DF-4623-4EB2-8B72-52C6489BF9D6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CE96428B-8138-4914-9999-2B391797FFF8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CE96428B-8138-4914-9999-2B391797FFF8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CE96428B-8138-4914-9999-2B391797FFF8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CE96428B-8138-4914-9999-2B391797FFF8}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -464,6 +470,7 @@ Global
 		{2DD07A73-8C88-4429-BB24-C2813586EF92} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
 		{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503} = {6A776396-02B1-475D-A104-26940ADB04AB}
 		{9ED642DF-4623-4EB2-8B72-52C6489BF9D6} = {6A776396-02B1-475D-A104-26940ADB04AB}
+		{CE96428B-8138-4914-9999-2B391797FFF8} = {6A776396-02B1-475D-A104-26940ADB04AB}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B}

src/ArchivePackages/ArchivePackages.Job.cs

@@ -4,20 +4,23 @@
 using System;
 using System.Collections.Generic;
 using System.ComponentModel.Design;
-using System.Diagnostics.Tracing;
+using System.Data.SqlClient;
 using System.Linq;
 using System.Threading.Tasks;
+using Autofac;
 using Dapper;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
 using Microsoft.WindowsAzure.Storage;
 using Microsoft.WindowsAzure.Storage.Blob;
 using Newtonsoft.Json.Linq;
 using NuGet.Jobs;
-using NuGet.Services.KeyVault;
-using NuGet.Services.Sql;
+using NuGet.Jobs.Configuration;
 
 namespace ArchivePackages
 {
-    public class Job : JobBase
+    public class Job : JsonConfigurationJob
     {
         private readonly JobEventSource JobEventSourceLog = JobEventSource.Log;
         private const string ContentTypeJson = "application/json";
@@ -27,6 +30,8 @@ public class Job : JobBase
         private const string DefaultPackagesArchiveContainerName = "ng-backups";
         private const string DefaultCursorBlobName = "cursor.json";
 
+        private InitializationConfiguration Configuration { get; set; }
+
         /// <summary>
         /// Gets or sets an Azure Storage Uri referring to a container to use as the source for package blobs
         /// </summary>
@@ -44,6 +49,7 @@ public class Job : JobBase
         /// DestinationContainerName should be same as the primary destination
         /// </summary>
         public CloudStorageAccount SecondaryDestination { get; set; }
+
         /// <summary>
         /// Destination Container name for both Primary and Secondary destinations. Also, for the cursor blob
         /// </summary>
@@ -53,8 +59,11 @@ public class Job : JobBase
         /// Blob containing the cursor data. Cursor data comprises of cursorDateTime
         /// </summary>
         public string CursorBlobName { get; set; }
-
-        private ISqlConnectionFactory _packageDbConnectionFactory;
+
+        /// <summary>
+        /// Gallery database registration, for diagnostics.
+        /// </summary>
+        private SqlConnectionStringBuilder GalleryDatabase { get; set; }
 
         protected CloudBlobContainer SourceContainer { get; private set; }
 
@@ -66,33 +75,34 @@ public Job() : base(JobEventSource.Log) { }
 
         public override void Init(IServiceContainer serviceContainer, IDictionary<string, string> jobArgsDictionary)
         {
-            var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector));
-            var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase);
-            _packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector);
+            base.Init(serviceContainer, jobArgsDictionary);
 
-            Source = CloudStorageAccount.Parse(
-                        JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.Source));
+            Configuration = _serviceProvider.GetRequiredService<IOptionsSnapshot<InitializationConfiguration>>().Value;
 
-            PrimaryDestination = CloudStorageAccount.Parse(
-                                    JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PrimaryDestination));
+            GalleryDatabase = GetDatabaseRegistration<GalleryDbConfiguration>();
 
-            var secondaryDestinationCstr = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.SecondaryDestination);
-            SecondaryDestination = string.IsNullOrEmpty(secondaryDestinationCstr) ? null : CloudStorageAccount.Parse(secondaryDestinationCstr);
+            Source = CloudStorageAccount.Parse(Configuration.Source);
 
-            SourceContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.SourceContainerName) ?? DefaultPackagesContainerName;
+            PrimaryDestination = CloudStorageAccount.Parse(Configuration.PrimaryDestination);
 
-            DestinationContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.DestinationContainerName) ?? DefaultPackagesArchiveContainerName;
+            if (!string.IsNullOrEmpty(Configuration.SecondaryDestination))
+            {
+                SecondaryDestination = CloudStorageAccount.Parse(Configuration.SecondaryDestination);
+            }
+
+            SourceContainerName = Configuration.SourceContainerName ?? DefaultPackagesContainerName;
+            DestinationContainerName = Configuration.DestinationContainerName ?? DefaultPackagesArchiveContainerName;
 
             SourceContainer = Source.CreateCloudBlobClient().GetContainerReference(SourceContainerName);
             PrimaryDestinationContainer = PrimaryDestination.CreateCloudBlobClient().GetContainerReference(DestinationContainerName);
             SecondaryDestinationContainer = SecondaryDestination?.CreateCloudBlobClient().GetContainerReference(DestinationContainerName);
 
-            CursorBlobName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.CursorBlob) ?? DefaultCursorBlobName;
+            CursorBlobName = Configuration.CursorBlob ?? DefaultCursorBlobName;
         }
 
         public override async Task Run()
         {
-            JobEventSourceLog.PreparingToArchive(Source.Credentials.AccountName, SourceContainer.Name, PrimaryDestination.Credentials.AccountName, PrimaryDestinationContainer.Name, _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog);
+            JobEventSourceLog.PreparingToArchive(Source.Credentials.AccountName, SourceContainer.Name, PrimaryDestination.Credentials.AccountName, PrimaryDestinationContainer.Name, GalleryDatabase.DataSource, GalleryDatabase.InitialCatalog);
             await Archive(PrimaryDestinationContainer);
 
             // todo: consider reusing package query for primary and secondary archives
@@ -124,9 +134,9 @@ private async Task Archive(CloudBlobContainer destinationContainer)
 
             JobEventSourceLog.CursorData(cursorDateTime.ToString(DateTimeFormatSpecifier));
 
-            JobEventSourceLog.GatheringPackagesToArchiveFromDb(_packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog);
+            JobEventSourceLog.GatheringPackagesToArchiveFromDb(GalleryDatabase.DataSource, GalleryDatabase.InitialCatalog);
             List<PackageRef> packages;
-            using (var connection = await _packageDbConnectionFactory.CreateAsync())
+            using (var connection = await OpenSqlConnectionAsync<GalleryDbConfiguration>())
             {
                 packages = (await connection.QueryAsync<PackageRef>(@"
 			    SELECT pr.Id, p.NormalizedVersion AS Version, p.Hash, p.LastEdited, p.Published
@@ -135,7 +145,7 @@ FROM Packages p
 			    WHERE Published > @cursorDateTime OR LastEdited > @cursorDateTime", new { cursorDateTime = cursorDateTime }))
                     .ToList();
             }
-            JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog);
+            JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, GalleryDatabase.DataSource, GalleryDatabase.InitialCatalog);
 
             var archiveSet = packages
                 .AsParallel()
@@ -193,129 +203,14 @@ private async Task ArchivePackage(string sourceBlobName, string destinationBlobN
                 JobEventSourceLog.StartedCopy(sourceBlob.Name, destBlob.Name);
             }
         }
-    }
 
-    [EventSource(Name = "Outercurve-NuGet-Jobs-ArchivePackages")]
-    public class JobEventSource : EventSource
-    {
-        public static readonly JobEventSource Log = new JobEventSource();
-
-        private JobEventSource() { }
-
-        [Event(
-            eventId: 1,
-            Level = EventLevel.Informational,
-            Message = "Preparing to archive packages from {0}/{1} to primary destination {2}/{3} using package data from {4}/{5}")]
-        public void PreparingToArchive(string sourceAccount, string sourceContainer, string destAccount, string destContainer, string dbServer, string dbName) { WriteEvent(1, sourceAccount, sourceContainer, destAccount, destContainer, dbServer, dbName); }
-
-        [Event(
-            eventId: 2,
-            Level = EventLevel.Informational,
-            Message = "Preparing to archive packages to secondary destination {0}/{1}")]
-        public void PreparingToArchive2(string destAccount, string destContainer) { WriteEvent(2, destAccount, destContainer); }
-
-        [Event(
-            eventId: 3,
-            Level = EventLevel.Informational,
-            Message = "Cursor data: CursorDateTime is {0}")]
-        public void CursorData(string cursorDateTime) { WriteEvent(3, cursorDateTime); }
-
-        [Event(
-            eventId: 4,
-            Level = EventLevel.Informational,
-            Task = Tasks.GatheringDbPackages,
-            Opcode = EventOpcode.Start,
-            Message = "Gathering list of packages to archive from {0}/{1}")]
-        public void GatheringPackagesToArchiveFromDb(string dbServer, string dbName) { WriteEvent(4, dbServer, dbName); }
-
-        [Event(
-            eventId: 5,
-            Level = EventLevel.Informational,
-            Task = Tasks.GatheringDbPackages,
-            Opcode = EventOpcode.Stop,
-            Message = "Gathered {0} packages to archive from {1}/{2}")]
-        public void GatheredPackagesToArchiveFromDb(int gathered, string dbServer, string dbName) { WriteEvent(5, gathered, dbServer, dbName); }
-
-        [Event(
-            eventId: 6,
-            Level = EventLevel.Informational,
-            Task = Tasks.ArchivingPackages,
-            Opcode = EventOpcode.Start,
-            Message = "Starting archive of {0} packages.")]
-        public void StartingArchive(int count) { WriteEvent(6, count); }
-
-        [Event(
-            eventId: 7,
-            Level = EventLevel.Informational,
-            Task = Tasks.ArchivingPackages,
-            Opcode = EventOpcode.Stop,
-            Message = "Started archive.")]
-        public void StartedArchive() { WriteEvent(7); }
-
-        [Event(
-            eventId: 8,
-            Level = EventLevel.Informational,
-            Message = "Archive already exists: {0}")]
-        public void ArchiveExists(string blobName) { WriteEvent(8, blobName); }
-
-        [Event(
-            eventId: 9,
-            Level = EventLevel.Warning,
-            Message = "Source Blob does not exist: {0}")]
-        public void SourceBlobMissing(string blobName) { WriteEvent(9, blobName); }
-
-        [Event(
-            eventId: 12,
-            Level = EventLevel.Informational,
-            Task = Tasks.StartingPackageCopy,
-            Opcode = EventOpcode.Start,
-            Message = "Starting copy of {0} to {1}.")]
-        public void StartingCopy(string source, string dest) { WriteEvent(12, source, dest); }
-
-        [Event(
-            eventId: 13,
-            Level = EventLevel.Informational,
-            Task = Tasks.StartingPackageCopy,
-            Opcode = EventOpcode.Stop,
-            Message = "Started copy of {0} to {1}.")]
-        public void StartedCopy(string source, string dest) { WriteEvent(13, source, dest); }
-
-        [Event(
-            eventId: 14,
-            Level = EventLevel.Informational,
-            Message = "NewCursor data: CursorDateTime is {0}")]
-        public void NewCursorData(string cursorDateTime) { WriteEvent(14, cursorDateTime); }
-    }
-
-    public static class Tasks
-    {
-        public const EventTask GatheringDbPackages = (EventTask)0x1;
-        public const EventTask ArchivingPackages = (EventTask)0x2;
-        public const EventTask StartingPackageCopy = (EventTask)0x3;
-    }
-
-    public class PackageRef
-    {
-        public PackageRef(string id, string version, string hash)
-        {
-            Id = id;
-            Version = version;
-            Hash = hash;
-        }
-        public PackageRef(string id, string version, string hash, DateTime lastEdited)
-            : this(id, version, hash)
+        protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder)
         {
-            LastEdited = lastEdited;
         }
-        public PackageRef(string id, string version, string hash, DateTime lastEdited, DateTime published)
-            : this(id, version, hash, lastEdited)
+
+        protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
         {
-            Published = published;
+            ConfigureInitializationSection<InitializationConfiguration>(services, configurationRoot);
         }
-        public string Id { get; set; }
-        public string Version { get; set; }
-        public string Hash { get; set; }
-        public DateTime? LastEdited { get; set; }
-        public DateTime? Published { get; set; }
     }
 }

src/ArchivePackages/ArchivePackages.csproj

@@ -44,14 +44,19 @@
   <ItemGroup>
     <Compile Include="ArchivePackages.Job.cs" />
     <Compile Include="ArchivePackages.Program.cs" />
+    <Compile Include="Configuration\InitializationConfiguration.cs" />
+    <Compile Include="JobEventSource.cs" />
+    <Compile Include="JobTasks.cs" />
+    <Compile Include="PackageRef.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
-    <None Include="App.config">
-      <SubType>Designer</SubType>
-    </None>
+    <None Include="App.config" />
     <None Include="Scripts\*" />
     <None Include="ArchivePackages.nuspec" />
+    <None Include="Settings\dev.json" />
+    <None Include="Settings\int.json" />
+    <None Include="Settings\prod.json" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\NuGet.Jobs.Common\NuGet.Jobs.Common.csproj">
@@ -75,16 +80,14 @@
     <PackageReference Include="Newtonsoft.Json">
       <Version>9.0.1</Version>
     </PackageReference>
-    <PackageReference Include="NuGet.Services.Sql">
-      <Version>2.25.0-master-30453</Version>
-    </PackageReference>
     <PackageReference Include="System.Net.Http">
       <Version>4.3.3</Version>
     </PackageReference>
     <PackageReference Include="WindowsAzure.Storage">
       <Version>7.1.2</Version>
     </PackageReference>
   </ItemGroup>
+  <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <Import Project="..\..\build\sign.targets" Condition="Exists('..\..\build\sign.targets')" />
 </Project>

src/ArchivePackages/ArchivePackages.nuspec

@@ -11,12 +11,14 @@
   </metadata>
   <files>
     <file src="bin\$configuration$\*.*" target="bin"/>
-	
+
     <file src="Scripts\ArchivePackages.cmd" />
-	
+
     <file src="Scripts\Functions.ps1" />
     <file src="Scripts\PreDeploy.ps1" />
     <file src="Scripts\PostDeploy.ps1" />
     <file src="Scripts\nssm.exe" />
+
+    <file src="Settings\*.json" target="bin" />
   </files>
 </package>

src/ArchivePackages/Configuration/InitializationConfiguration.cs

@@ -0,0 +1,38 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace ArchivePackages
+{
+    public class InitializationConfiguration
+    {
+        /// <summary>
+        /// Source storage account.
+        /// </summary>
+        public string Source { get; set; }
+
+        /// <summary>
+        /// Source storage container name (defaults to "packages").
+        /// </summary>
+        public string SourceContainerName { get; set; }
+
+        /// <summary>
+        /// Primary archive destination.
+        /// </summary>
+        public string PrimaryDestination { get; set; }
+
+        /// <summary>
+        /// Secondary archive destination (optional).
+        /// </summary>
+        public string SecondaryDestination { get; set; }
+
+        /// <summary>
+        /// Destination storage container name (defaults to "ng-backups").
+        /// </summary>
+        public string DestinationContainerName { get; set; }
+
+        /// <summary>
+        /// Cursor blob name (defaults to "cursor.json").
+        /// </summary>
+        public string CursorBlob { get; set; }
+    }
+}