From 3632aa6c28308328f3e30fd8e06dcca1e78646ad Mon Sep 17 00:00:00 2001 From: aabhinav Date: Fri, 28 Apr 2023 23:12:26 +0530 Subject: [PATCH 1/2] Added a new detector MvnPomCliComponentDetector. This detector just finds all the .pom files and parses the dependencies (which are present similar to maven structure like ) in those files. It does a simple text parsing and does not create the dependency graph. --- .../maven/MvnPomCliComponentDetector.cs | 112 ++++++++++++++++++ .../Extensions/ServiceCollectionExtensions.cs | 3 +- 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.ComponentDetection.Detectors/maven/MvnPomCliComponentDetector.cs diff --git a/src/Microsoft.ComponentDetection.Detectors/maven/MvnPomCliComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/maven/MvnPomCliComponentDetector.cs new file mode 100644 index 000000000..05f3fcad4 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/maven/MvnPomCliComponentDetector.cs @@ -0,0 +1,112 @@ +namespace Microsoft.ComponentDetection.Detectors.Maven; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using System.Xml; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; + +public class MvnPomCliComponentDetector : FileComponentDetector +{ + public MvnPomCliComponentDetector( + IComponentStreamEnumerableFactory componentStreamEnumerableFactory, + IObservableDirectoryWalkerFactory walkerFactory, + ILogger logger) + { + this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; + this.Scanner = walkerFactory; + this.Logger = logger; + } + + public override string Id => "MvnPomCli"; + + public override IList SearchPatterns => new List() { "*.pom" }; + + public override IEnumerable Categories => new[] { Enum.GetName(typeof(DetectorClass), DetectorClass.Maven) }; + + public override IEnumerable SupportedComponentTypes => new[] { ComponentType.Maven }; + + public override int Version => 2; + + protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs) + { + await this.ProcessFileAsync(processRequest); + } + + private async Task ProcessFileAsync(ProcessRequest processRequest) + { + var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; + var stream = processRequest.ComponentStream; + + try + { + byte[] pomBytes = null; + + if ("*.pom".Equals(stream.Pattern, StringComparison.OrdinalIgnoreCase)) + { + using (var contentStream = File.Open(stream.Location, FileMode.Open)) + { + pomBytes = new byte[contentStream.Length]; + await contentStream.ReadAsync(pomBytes.AsMemory(0, (int)contentStream.Length)); + + using var pomStream = new MemoryStream(pomBytes, false); + var doc = new XmlDocument(); + doc.Load(pomStream); + + var nsmgr = new XmlNamespaceManager(doc.NameTable); + nsmgr.AddNamespace("ns", "http://maven.apache.org/POM/4.0.0"); + + var dependencies = doc.SelectSingleNode("//ns:project/ns:dependencies", nsmgr); + if (dependencies == null) + { + return; + } + + foreach (XmlNode node in dependencies.ChildNodes) + { + this.RegisterComponent(node, nsmgr, singleFileComponentRecorder); + } + } + } + else + { + return; + } + } + catch (Exception e) + { + // If something went wrong, just ignore the component + this.Logger.LogError(e, "Error parsing pom maven component from {PomLocation}", stream.Location); + singleFileComponentRecorder.RegisterPackageParseFailure(stream.Location); + } + } + + private void RegisterComponent(XmlNode node, XmlNamespaceManager nsmgr, ISingleFileComponentRecorder singleFileComponentRecorder) + { + var groupIdNode = node.SelectSingleNode("ns:groupId", nsmgr); + var artifactIdNode = node.SelectSingleNode("ns:artifactId", nsmgr); + var versionNode = node.SelectSingleNode("ns:version", nsmgr); + + if (groupIdNode == null || artifactIdNode == null || versionNode == null) + { + return; + } + + var groupId = groupIdNode.InnerText; + var artifactId = artifactIdNode.InnerText; + var version = versionNode.InnerText; + var dependencyScope = DependencyScope.MavenCompile; + + var component = new MavenComponent(groupId, artifactId, version); + + singleFileComponentRecorder.RegisterUsage( + new DetectedComponent(component), + isDevelopmentDependency: null, + dependencyScope: dependencyScope); + } +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index 9ee129ee3..075885e4f 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -1,4 +1,4 @@ -namespace Microsoft.ComponentDetection.Orchestrator.Extensions; +namespace Microsoft.ComponentDetection.Orchestrator.Extensions; using Microsoft.ComponentDetection.Common; using Microsoft.ComponentDetection.Common.Telemetry; @@ -97,6 +97,7 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); // npm services.AddSingleton(); From f575118887748e0874c55c4555a5de77c8a8534e Mon Sep 17 00:00:00 2001 From: aabhinav Date: Mon, 15 May 2023 20:44:15 +0530 Subject: [PATCH 2/2] Created a new interface (IMavenFileParserService) for parsing pom files (Just the deps in the pom files will be read as components) along with one implementation (MavenFileParserService) that is being used in MavenPomCliComponentDetector. Renamed MavenPomCliComponentDetector to MavenPomComponentDetector Added test case (MavenPomComponentDetectorTest.cs) for the new detector class MavenPomComponentDetector. --- .../maven/IMavenFileParserService.cs | 8 ++ .../maven/MavenFileParserService.cs | 74 +++++++++++++++++++ .../maven/MavenPomComponentDetector.cs | 48 ++++++++++++ .../Extensions/ServiceCollectionExtensions.cs | 3 +- .../MvnPomComponentDetectorTest.cs | 71 ++++++++++++++++++ 5 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.ComponentDetection.Detectors/maven/IMavenFileParserService.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/maven/MavenFileParserService.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/maven/MavenPomComponentDetector.cs create mode 100644 test/Microsoft.ComponentDetection.Detectors.Tests/MvnPomComponentDetectorTest.cs diff --git a/src/Microsoft.ComponentDetection.Detectors/maven/IMavenFileParserService.cs b/src/Microsoft.ComponentDetection.Detectors/maven/IMavenFileParserService.cs new file mode 100644 index 000000000..0378abcb5 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/maven/IMavenFileParserService.cs @@ -0,0 +1,8 @@ +namespace Microsoft.ComponentDetection.Detectors.Maven; + +using Microsoft.ComponentDetection.Contracts.Internal; + +public interface IMavenFileParserService +{ + void ParseDependenciesFile(ProcessRequest processRequest); +} diff --git a/src/Microsoft.ComponentDetection.Detectors/maven/MavenFileParserService.cs b/src/Microsoft.ComponentDetection.Detectors/maven/MavenFileParserService.cs new file mode 100644 index 000000000..4cd34dbd6 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/maven/MavenFileParserService.cs @@ -0,0 +1,74 @@ +namespace Microsoft.ComponentDetection.Detectors.Maven; + +using System; +using System.Xml; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; + +public class MavenFileParserService : IMavenFileParserService +{ + private readonly ILogger logger; + + public MavenFileParserService( + ILogger logger) => this.logger = logger; + + public void ParseDependenciesFile(ProcessRequest processRequest) + { + var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; + var stream = processRequest.ComponentStream; + + try + { + var doc = new XmlDocument(); + doc.Load(stream.Location); + + var nsmgr = new XmlNamespaceManager(doc.NameTable); + nsmgr.AddNamespace("ns", "http://maven.apache.org/POM/4.0.0"); + + var dependencies = doc.SelectSingleNode("//ns:project/ns:dependencies", nsmgr); + if (dependencies == null) + { + return; + } + + foreach (XmlNode node in dependencies.ChildNodes) + { + this.RegisterComponent(node, nsmgr, singleFileComponentRecorder); + } + } + catch (Exception e) + { + // If something went wrong, just ignore the component + this.logger.LogError(e, "Error parsing pom maven component from {PomLocation}", stream.Location); + singleFileComponentRecorder.RegisterPackageParseFailure(stream.Location); + } + } + + private void RegisterComponent(XmlNode node, XmlNamespaceManager nsmgr, ISingleFileComponentRecorder singleFileComponentRecorder) + { + var groupIdNode = node.SelectSingleNode("ns:groupId", nsmgr); + var artifactIdNode = node.SelectSingleNode("ns:artifactId", nsmgr); + var versionNode = node.SelectSingleNode("ns:version", nsmgr); + + if (groupIdNode == null || artifactIdNode == null || versionNode == null) + { + this.logger.LogInformation("{XmlNode} doesn't have groupId, artifactId or version information", node.InnerText); + return; + } + + var groupId = groupIdNode.InnerText; + var artifactId = artifactIdNode.InnerText; + var version = versionNode.InnerText; + var dependencyScope = DependencyScope.MavenCompile; + + var component = new MavenComponent(groupId, artifactId, version); + + singleFileComponentRecorder.RegisterUsage( + new DetectedComponent(component), + isDevelopmentDependency: null, + dependencyScope: dependencyScope); + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/maven/MavenPomComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/maven/MavenPomComponentDetector.cs new file mode 100644 index 000000000..820f521e5 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/maven/MavenPomComponentDetector.cs @@ -0,0 +1,48 @@ +namespace Microsoft.ComponentDetection.Detectors.Maven; + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; + +public class MavenPomComponentDetector : FileComponentDetector, IDefaultOffComponentDetector +{ + private readonly IMavenFileParserService mavenFileParserService; + + public MavenPomComponentDetector( + IComponentStreamEnumerableFactory componentStreamEnumerableFactory, + IObservableDirectoryWalkerFactory walkerFactory, + IMavenFileParserService mavenFileParserService, + ILogger logger) + { + this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; + this.Scanner = walkerFactory; + this.mavenFileParserService = mavenFileParserService; + this.Logger = logger; + } + + public override string Id => "MvnPom"; + + public override IList SearchPatterns => new List() { "*.pom" }; + + public override IEnumerable Categories => new[] { Enum.GetName(typeof(DetectorClass), DetectorClass.Maven) }; + + public override IEnumerable SupportedComponentTypes => new[] { ComponentType.Maven }; + + public override int Version => 2; + + protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs) + { + await this.ProcessFileAsync(processRequest); + } + + private async Task ProcessFileAsync(ProcessRequest processRequest) + { + this.mavenFileParserService.ParseDependenciesFile(processRequest); + + await Task.CompletedTask; + } +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index 075885e4f..67aa60085 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -97,7 +97,8 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); // npm services.AddSingleton(); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/MvnPomComponentDetectorTest.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/MvnPomComponentDetectorTest.cs new file mode 100644 index 000000000..75d54d72d --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/MvnPomComponentDetectorTest.cs @@ -0,0 +1,71 @@ +namespace Microsoft.ComponentDetection.Detectors.Tests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.Maven; +using Microsoft.ComponentDetection.Detectors.Tests.Utilities; +using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class MvnPomComponentDetectorTest : BaseDetectorTest +{ + private readonly Mock mavenFileParserServiceMock; + + public MvnPomComponentDetectorTest() + { + this.mavenFileParserServiceMock = new Mock(); + this.DetectorTestUtility.AddServiceMock(this.mavenFileParserServiceMock); + } + + [TestMethod] + public async Task MavenRootsAsync() + { + const string componentString = "org.apache.maven:maven-compat:jar:3.6.1-SNAPSHOT"; + const string childComponentString = "org.apache.maven:maven-compat-child:jar:3.6.1-SNAPSHOT"; + var content = $@"com.bcde.test:top-level:jar:1.0.0{Environment.NewLine}\- {componentString}{Environment.NewLine} \- {childComponentString}"; + this.DetectorTestUtility.WithFile("pom.xml", content) + .WithFile("pom.xml", content, searchPatterns: new[] { "pom.xml" }); + + this.mavenFileParserServiceMock.Setup(x => x.ParseDependenciesFile(It.IsAny())) + .Callback((ProcessRequest pr) => + { + pr.SingleFileComponentRecorder.RegisterUsage( + new DetectedComponent( + new MavenComponent("com.bcde.test", "top-levelt", "1.0.0")), + isExplicitReferencedDependency: true); + pr.SingleFileComponentRecorder.RegisterUsage( + new DetectedComponent( + new MavenComponent("org.apache.maven", "maven-compat", "3.6.1-SNAPSHOT")), + isExplicitReferencedDependency: true); + pr.SingleFileComponentRecorder.RegisterUsage( + new DetectedComponent( + new MavenComponent("org.apache.maven", "maven-compat-child", "3.6.1-SNAPSHOT")), + isExplicitReferencedDependency: false, + parentComponentId: "org.apache.maven maven-compat 3.6.1-SNAPSHOT - Maven"); + }); + + var (detectorResult, componentRecorder) = await this.DetectorTestUtility.ExecuteDetectorAsync(); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + Assert.AreEqual(detectedComponents.Count(), 3); + Assert.AreEqual(detectorResult.ResultCode, ProcessingResultCode.Success); + + var splitComponent = componentString.Split(':'); + var splitChildComponent = childComponentString.Split(':'); + + var mavenComponent = detectedComponents.FirstOrDefault(x => (x.Component as MavenComponent).ArtifactId == splitChildComponent[1]); + Assert.IsNotNull(mavenComponent); + + componentRecorder.AssertAllExplicitlyReferencedComponents( + mavenComponent.Component.Id, + parentComponent => parentComponent.ArtifactId == splitComponent[1]); + } +}