From c66f08726b0ac79311f44b0f06aec571d210d6f1 Mon Sep 17 00:00:00 2001 From: Andrian Sevastyanov Date: Thu, 20 Mar 2025 10:54:40 -0600 Subject: [PATCH 1/2] Do not fail extraction if inspector cannot find the root project and minor doc update --- .../pip/inspector/PipInspectorExtractor.java | 2 +- .../parser/PipInspectorTreeParser.java | 7 +----- .../unit/PipInspectorTreeParserTest.java | 24 ++++--------------- .../src/main/markdown/packagemgrs/python.md | 2 +- 4 files changed, 8 insertions(+), 27 deletions(-) diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pip/inspector/PipInspectorExtractor.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pip/inspector/PipInspectorExtractor.java index edfa50ada6..c3ad9d5d97 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pip/inspector/PipInspectorExtractor.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pip/inspector/PipInspectorExtractor.java @@ -63,7 +63,7 @@ public Extraction extract( for (Path requirementFilePath : requirementsPaths) { List inspectorOutput = runInspector(directory, pythonExe, pipInspector, projectName, requirementFilePath); - Optional result = pipInspectorTreeParser.parse(inspectorOutput, directory.toString(), StringUtils.isNotEmpty(projectName)); + Optional result = pipInspectorTreeParser.parse(inspectorOutput, directory.toString()); if (result.isPresent()) { codeLocations.add(result.get().getCodeLocation()); String potentialProjectVersion = result.get().getProjectVersion(); diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pip/inspector/parser/PipInspectorTreeParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pip/inspector/parser/PipInspectorTreeParser.java index d6c33e6822..9ff0aeb91f 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pip/inspector/parser/PipInspectorTreeParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pip/inspector/parser/PipInspectorTreeParser.java @@ -34,7 +34,7 @@ public PipInspectorTreeParser(ExternalIdFactory externalIdFactory) { this.externalIdFactory = externalIdFactory; } - public Optional parse(List pipInspectorOutputAsList, String sourcePath, boolean projectNameWasGiven) { + public Optional parse(List pipInspectorOutputAsList, String sourcePath) { NameVersionCodeLocation parseResult = null; DependencyGraph graph = new BasicDependencyGraph(); @@ -48,7 +48,6 @@ public Optional parse(List pipInspectorOutputAs || trimmedLine.startsWith(UNKNOWN_REQUIREMENTS_PREFIX) || trimmedLine.startsWith(UNPARSEABLE_REQUIREMENTS_PREFIX) || trimmedLine.startsWith(UNKNOWN_PACKAGE_PREFIX) - || trimmedLine.startsWith(UNKNOWN_PROJECT_NAME) && projectNameWasGiven ) { boolean wasUnresolved = parseErrorsFromLine(trimmedLine); if (wasUnresolved) { @@ -118,10 +117,6 @@ private boolean parseErrorsFromLine(String trimmedLine) { unResolvedPackage = true; } - if (trimmedLine.startsWith(UNKNOWN_PROJECT_NAME)) { - logger.error("Pip inspector could not resolve the project"); - unResolvedPackage = true; - } return unResolvedPackage; } diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/pip/inspector/unit/PipInspectorTreeParserTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/pip/inspector/unit/PipInspectorTreeParserTest.java index e56ad0d458..8cd1c9e8e2 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/pip/inspector/unit/PipInspectorTreeParserTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/pip/inspector/unit/PipInspectorTreeParserTest.java @@ -35,7 +35,7 @@ public void validTest() { " test==4.0.0" ); - Optional validParse = parser.parse(pipInspectorOutput, "", true); + Optional validParse = parser.parse(pipInspectorOutput, ""); Assertions.assertTrue(validParse.isPresent()); Assertions.assertEquals("projectName", validParse.get().getProjectName()); Assertions.assertEquals("projectVersionName", validParse.get().getProjectVersion()); @@ -59,7 +59,7 @@ void projectNotFoundWhenNameNotGivenTest() { " test==4.0.0" ); - Optional validParse = parser.parse(pipInspectorOutput, "src/path", false); + Optional validParse = parser.parse(pipInspectorOutput, "src/path"); Assertions.assertTrue(validParse.isPresent()); Assertions.assertEquals("", validParse.get().getProjectName()); Assertions.assertEquals("", validParse.get().getProjectVersion()); @@ -73,20 +73,6 @@ void projectNotFoundWhenNameNotGivenTest() { graphAssert.hasRootSize(3); } - @Test - void projectNotFoundWhenNameGivenTest() { - List pipInspectorOutput = Arrays.asList( - "n?==v?", - " with-dashes==1.0.0", - " Uppercase==2.0.0", - " child==3.0.0", - " test==4.0.0" - ); - - Optional invalidParse = parser.parse(pipInspectorOutput, "src/path", true); - Assertions.assertFalse(invalidParse.isPresent()); - } - @Test public void packageNotFoundTest() { List pipInspectorOutput = Arrays.asList( @@ -98,7 +84,7 @@ public void packageNotFoundTest() { " test==4.0.0" ); - Optional result = parser.parse(pipInspectorOutput, "src/path", true); + Optional result = parser.parse(pipInspectorOutput, "src/path"); Assertions.assertFalse(result.isPresent()); } @@ -107,7 +93,7 @@ public void invalidParseTest() { List invalidText = new ArrayList<>(); invalidText.add("i am not a valid file"); invalidText.add("the status should be optional.empty()"); - Optional invalidParse = parser.parse(invalidText, "", true); + Optional invalidParse = parser.parse(invalidText, ""); Assertions.assertFalse(invalidParse.isPresent()); } @@ -117,7 +103,7 @@ public void errorTest() { invalidText.add(PipInspectorTreeParser.UNKNOWN_PACKAGE_PREFIX + "probably_an_internal_dependency_PY"); invalidText.add(PipInspectorTreeParser.UNPARSEABLE_REQUIREMENTS_PREFIX + "/not/a/real/path/encrypted/requirements.txt"); invalidText.add(PipInspectorTreeParser.UNKNOWN_REQUIREMENTS_PREFIX + "/not/a/real/path/requirements.txt"); - Optional invalidParse = parser.parse(invalidText, "", true); + Optional invalidParse = parser.parse(invalidText, ""); Assertions.assertFalse(invalidParse.isPresent()); } } diff --git a/documentation/src/main/markdown/packagemgrs/python.md b/documentation/src/main/markdown/packagemgrs/python.md index bbb598d6ef..1bcd9573e3 100644 --- a/documentation/src/main/markdown/packagemgrs/python.md +++ b/documentation/src/main/markdown/packagemgrs/python.md @@ -84,7 +84,7 @@ Pip Native Inspector runs the [pip-inspector.py script](https://github.com/black python setup.py install pip install -r requirements.txt ```` -* Pip detector derives your project name using your setup.py file if you have one. If you do not have a setup.py file, you must provide the correct project name using the propety --detect.pip.project.name. +* Pip detector attempts to derive the project name using your setup.py file if you have one. If you do not have a setup.py file, you can provide the correct project name using the propety `--detect.pip.project.name`. * If there are any dependencies specified in requirements.txt that are not specified in setup.py, then provide the requirements.txt file using the [detect_product_short] property. If you are using a virtual environment, be sure to switch to that virtual environment when you run [detect_product_short]. This also applies when you are using a tool such as Poetry that sets up a Python virtual environment. From 998ed05f7a2252516114445d344c5d75072e9e66 Mon Sep 17 00:00:00 2001 From: Andrian Sevastyanov Date: Thu, 20 Mar 2025 20:41:42 -0600 Subject: [PATCH 2/2] Log info message when no package matching project name can be found --- .../pip/inspector/parser/PipInspectorTreeParser.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pip/inspector/parser/PipInspectorTreeParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pip/inspector/parser/PipInspectorTreeParser.java index 9ff0aeb91f..57e91e860c 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pip/inspector/parser/PipInspectorTreeParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pip/inspector/parser/PipInspectorTreeParser.java @@ -55,6 +55,9 @@ public Optional parse(List pipInspectorOutputAs } continue; } + if (trimmedLine.startsWith(UNKNOWN_PROJECT_NAME)) { + logger.info("Pip inspector did not find a package matching project name"); + } Dependency currentDependency = parseDependencyFromLine(trimmedLine, sourcePath); adjustForIndentLevel(history, line); project = addDependencyToGraph(graph, history, project, currentDependency);