Re-enable leak detection parameters. (#1867)

* Re-enable leak detection parameters.

* Use our copy of SourceLink to build.

* Remove signatures from poisoned packages.

* Port updates from release/2.1 branch.

* Remove reference to obsolete task project.

* Exclude some more files we don't care about.

* Address code review comment.

Author: crummel

.vsts.pipelines/builds/matrix.yml

@@ -16,7 +16,7 @@ stages:
     name: centos71
     jobParameters:
       imageName: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-359e48e-20200313130914
-      reportPrebuiltLeaks: false
+      reportPrebuiltLeaks: true
     matrix:
 # Temporarily disable unit tests until entire product builds.
 # See https://github.com/dotnet/source-build/issues/1549
@@ -43,7 +43,7 @@ stages:
     name: centos8
     jobParameters:
       imageName: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-8-source-build-20200402192642-9e679d4
-      reportPrebuiltLeaks: false
+      reportPrebuiltLeaks: true
       systemLibunwind: false
     matrix:
       Production: {}

eng/SourceBuild.Tarball.targets

@@ -320,12 +320,16 @@
     <MSBuild
       Projects="$(LeakDetectionProjectFile)"
       Targets="Restore"
-      Properties="RestoreSources=$(SourceBuiltPackagesPath)%3B$(TarballRootDir)packages/prebuilt%3bhttps://api.nuget.org/v3/index.json"/>
+      Properties="
+        RestoreSources=$(SourceBuiltPackagesPath)%3B$(TarballRootDir)packages/prebuilt%3bhttps://api.nuget.org/v3/index.json;
+        DotNetPackageVersionPropsPath=$(PackageVersionPropsPath)"/>
 
     <MSBuild
       Projects="$(LeakDetectionProjectFile)"
       Targets="Publish"
-      Properties="OutputPath=$(TarballRootDir)tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/"/>
+      Properties="
+        OutputPath=$(TarballRootDir)tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/;
+        DotNetPackageVersionPropsPath=$(PackageVersionPropsPath)"/>
   </Target>
 
   <Target Name="CopyTarballFiles">

tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs

@@ -51,6 +51,12 @@ public class CheckForPoison : Task
         /// </summary>
         public bool FailOnPoisonFound { get; set; }
 
+        /// <summary>
+        /// Use this directory instead of the system temp directory for staging.
+        /// Intended for Linux systems with limited /tmp space, like Azure VMs.
+        /// </summary>
+        public string OverrideTempPath { get; set; }
+
         private static readonly string[] ZipFileExtensions =
         {
             ".zip",
@@ -78,6 +84,7 @@ public class CheckForPoison : Task
             ".gitkeep",
             ".rels",
             "LICENSE",
+            "prefercliruntime",
             "RunCsc",
             "RunVbc",
         };
@@ -103,6 +110,7 @@ public class CheckForPoison : Task
             ".png",
             ".props",
             ".psmdcp",
+            ".rtf",
             ".scss",
             ".svg",
             ".targets",
@@ -140,7 +148,7 @@ public override bool Execute()
             }
             else
             {
-                //Log.LogError($"No leaked files found in output.  Either something is broken or it is the future and we have fixed all leaks - please verify and remove this error if so (and default {nameof(FailOnPoisonFound)} to true).");
+                Log.LogError($"No leaked files found in output.  Either something is broken or it is the future and we have fixed all leaks - please verify and remove this error if so (and default {nameof(FailOnPoisonFound)} to true).");
                 return false;
             }
 
@@ -154,15 +162,19 @@ public override bool Execute()
         /// <param name="catalogedPackagesFilePath">File path to the file hash catalog</param>
         /// <param name="markerFileName">Marker file name to check for in poisoned nupkgs</param>
         /// <returns>List of poisoned packages and files found and reasons for each</returns>
-        internal static IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<string> initialCandidates, string catalogedPackagesFilePath, string markerFileName)
+        internal IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<string> initialCandidates, string catalogedPackagesFilePath, string markerFileName)
         {
-            var tempDirName = Path.GetRandomFileName();
-            var tempDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), tempDirName));
             IEnumerable<CatalogPackageEntry> catalogedPackages = ReadCatalog(catalogedPackagesFilePath);
             var poisons = new List<PoisonedFileEntry>();
             var candidateQueue = new Queue<string>(initialCandidates);
             // avoid collisions between nupkgs with the same name
             var dirCounter = 0;
+            if (!string.IsNullOrWhiteSpace(OverrideTempPath))
+            {
+                Directory.CreateDirectory(OverrideTempPath);
+            }
+            var tempDirName = Path.GetRandomFileName();
+            var tempDir = Directory.CreateDirectory(Path.Combine(OverrideTempPath ?? Path.GetTempPath(), tempDirName));
 
             while (candidateQueue.Any())
             {
@@ -172,7 +184,7 @@ internal static IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<stri
                 // add its contents to the list to be checked.
                 if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => checking.ToLowerInvariant().EndsWith(e)))
                 {
-                    var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(checking) + "." + (++dirCounter).ToString());
+                    var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetRandomFileName(), Path.GetFileNameWithoutExtension(checking) + "." + (++dirCounter).ToString());
                     PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, checking, markerFileName, tempCheckingDir, candidateQueue);
                     if (result != null)
                     {
@@ -190,6 +202,7 @@ internal static IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<stri
             }
 
             tempDir.Delete(true);
+
             return poisons;
         }
 
@@ -295,7 +308,7 @@ private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable<CatalogP
             // now extract and look for the marker file
             if (ZipFileExtensions.Any(e => zipToCheck.ToLowerInvariant().EndsWith(e)))
             {
-                ZipFile.ExtractToDirectory(zipToCheck, tempDir);
+                ZipFile.ExtractToDirectory(zipToCheck, tempDir, true);
             }
             else if (TarFileExtensions.Any(e => zipToCheck.ToLowerInvariant().EndsWith(e)))
             {

tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/MarkAndCatalogPackages.cs

@@ -53,10 +53,20 @@ public class MarkAndCatalogPackages : Task
         [Required]
         public ITaskItem[] PackagesToMark { get; set; }
 
+        /// <summary>
+        /// Use this directory instead of the system temp directory for staging.
+        /// Intended for Linux systems with limited /tmp space, like Azure VMs.
+        /// </summary>
+        public string OverrideTempPath { get; set; }
+
         public override bool Execute()
         {
             var tempDirName = Path.GetRandomFileName();
-            var tempDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), tempDirName));
+            if (!string.IsNullOrWhiteSpace(OverrideTempPath))
+            {
+                Directory.CreateDirectory(OverrideTempPath);
+            }
+            var tempDir = Directory.CreateDirectory(Path.Combine(OverrideTempPath ?? Path.GetTempPath(), tempDirName));
 
             var packageEntries = new List<CatalogPackageEntry>();
 
@@ -79,6 +89,13 @@ public override bool Execute()
 
                     foreach (string f in Directory.EnumerateFiles(packageTempPath, "*", SearchOption.AllDirectories))
                     {
+                        // remove signatures so we don't later fail validation
+                        if (Path.GetFileName(f) == ".signature.p7s")
+                        {
+                            File.Delete(f);
+                            continue;
+                        }
+
                         var catalogFileEntry = new CatalogFileEntry();
                         packageEntry.Files.Add(catalogFileEntry);
                         catalogFileEntry.Path = Utility.MakeRelativePath(f, packageTempPath);

tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj

@@ -25,7 +25,6 @@
       <Project>{D68133BD-1E63-496E-9EDE-4FBDBF77B486}</Project>
       <Name>Mono.Cecil</Name>
     </ProjectReference>
-    <ProjectReference Include="../Microsoft.DotNet.SourceBuild.Tasks/Microsoft.DotNet.SourceBuild.Tasks.csproj" />
     <ProjectReference Include="../Microsoft.DotNet.SourceBuild.Tasks.XPlat/Microsoft.DotNet.SourceBuild.Tasks.XPlat.csproj" />
   </ItemGroup>