Improve the CycloneDX SBOM pre-validation fixes #1230

Signed-off-by: tdruez <[email protected]>

Author: tdruez

CHANGELOG.rst

@@ -17,7 +17,10 @@ v34.5.0 (unreleased)
 
 - Workaround an issue with the cyclonedx-python-lib that does not allow to load
   SBOMs that contains properties with no values.
+  Also, a few fixes pre-validation are applyed before deserializing thr SBOM for
+  maximum compatibility.
   https://github.com/nexB/scancode.io/issues/1185
+  https://github.com/nexB/scancode.io/issues/1230
 
 - Add a new `CollectTreeSitterSymbolsAndStrings` pipeline (addon) for collecting source
   symbol and string using tree-sitter.

scanpipe/pipes/cyclonedx.py

@@ -171,43 +171,73 @@ def cyclonedx_component_to_package_data(cdx_component):
 
 
 def get_components(bom):
-    """Return list of components from CycloneDX BOM."""
-    return list(bom._get_all_components())
+    """Return components from CycloneDX BOM except for the metadata.component."""
+    for component in bom.components:
+        yield from component.get_all_nested_components(include_self=True)
 
 
-def delete_tools(cyclonedx_document_json):
+def delete_ignored_root_properties(cyclonedx_document_json):
     """
-    Remove the ``tools`` section, if defined, from the SBOM as it can
-    be in the way of loading a SBOM that is valid regarding the spec, but fails the
-    deserialization.
+    Remove root properties from the CycloneDX document that are irrelevant
+    when loading SBOM component data as packages.
 
-    The ``metadata.tools`` as an array was deprecated in 1.5 and replaced by an
-    object structure where you can define a list of ``components`` and ``services``.
+    This function aims to maximize compatibility by excluding unsupported SPEC
+    definitions while utilizing the cyclonedx-python-lib library.
 
-    The new structure is not yet supported by the cyclonedx-python-lib, neither for
-    serialization (output) nor deserialization (input).
-    https://github.com/CycloneDX/cyclonedx-python-lib/issues/578
+    The data contained in these properties is unnecessary for loading components
+    from the SBOM and can be safely disregarded.
 
-    The tools are not used anyway in the context of loading the SBOM component data as
-    packages.
+    https://github.com/CycloneDX/cyclonedx-python-lib/issues/578
     """
-    if "tools" in cyclonedx_document_json.get("metadata", {}):
-        del cyclonedx_document_json["metadata"]["tools"]
+    ignored_root_properties = [
+        "metadata",
+        "services",
+        "externalReferences",
+        "compositions",
+        "vulnerabilities",
+        "annotations",
+        "formulation",
+        "declarations",
+        "definitions",
+        "properties",
+    ]
+
+    cleaned_document = {
+        key: value
+        for key, value in cyclonedx_document_json.items()
+        if key not in ignored_root_properties
+    }
 
-    return cyclonedx_document_json
+    return cleaned_document
 
 
-def delete_empty_properties(cyclonedx_document_json):
+def cleanup_components_properties(cyclonedx_document_json):
     """
     Remove entries for which no values are set, such as ``{"name": ""}`` or
     ``"licenses":[{}]``.
 
+    Also remove the properties that are not used in the context of loading packages
+    from SBOM and that  may be unsupported by the cyclonedx-python-lib library.
+
     Class like cyclonedx.model.contact.OrganizationalEntity raise a
     NoPropertiesProvidedException while it is not enforced in the spec.
 
     See https://github.com/CycloneDX/cyclonedx-python-lib/issues/600
     """
     entries_to_delete = []
+    ignored_properties = [
+        "evidence",
+        "omniborId",
+        "swhid",
+        "swid",
+        "modified",
+        "pedigree",
+        "releaseNotes",
+        "modelCard",
+        "data",
+        "cryptoProperties",
+        "signature",
+    ]
 
     def is_empty(value):
         if isinstance(value, dict) and not any(value.values()):
@@ -217,7 +247,7 @@ def is_empty(value):
 
     for component in cyclonedx_document_json["components"]:
         for property_name, property_value in component.items():
-            if is_empty(property_value):
+            if is_empty(property_value) or property_name in ignored_properties:
                 entries_to_delete.append((component, property_name))
 
     # Delete the keys outside the main check loop
@@ -240,8 +270,8 @@ def resolve_cyclonedx_packages(input_location):
         cyclonedx_document = json.loads(document_data)
 
         # Apply a few fixes pre-validation for maximum compatibility
-        cyclonedx_document = delete_tools(cyclonedx_document)
-        cyclonedx_document = delete_empty_properties(cyclonedx_document)
+        cyclonedx_document = delete_ignored_root_properties(cyclonedx_document)
+        cyclonedx_document = cleanup_components_properties(cyclonedx_document)
 
         if errors := validate_document(cyclonedx_document):
             error_msg = (

scanpipe/tests/data/cyclonedx/broken_sbom.json

@@ -0,0 +1,35 @@
+{
+  "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json",
+  "bomFormat": "CycloneDX",
+  "specVersion": "1.6",
+  "serialNumber": "urn:uuid:b74fe5df-e965-415e-ba65-f38421a0695d",
+  "version": 1,
+  "metadata": {
+    "lifecycles": [
+      {
+        "phase": "build"
+      }
+    ]
+  },
+  "components": [
+    {
+      "bom-ref": "pkg:pypi/[email protected]",
+      "name": "asgiref",
+      "type": "library",
+      "supplier": {
+        "name": ""
+      },
+      "licenses": [
+        {}
+      ],
+      "evidence": {
+        "identity": [
+          {
+            "field": "purl",
+            "confidence": 1
+          }
+        ]
+      }
+    }
+  ]
+}

scanpipe/tests/pipes/test_cyclonedx.py

@@ -62,9 +62,9 @@ def test_scanpipe_cyclonedx_is_cyclonedx_bom(self):
 
     def test_scanpipe_cyclonedx_get_components(self):
         empty_bom = Bom()
-        self.assertEqual([], cyclonedx.get_components(empty_bom))
+        self.assertEqual([], list(cyclonedx.get_components(empty_bom)))
 
-        components = cyclonedx.get_components(self.bom)
+        components = list(cyclonedx.get_components(self.bom))
         self.assertEqual(3, len(components))
 
         purls = [component.bom_ref.value for component in components]
@@ -198,29 +198,36 @@ def test_scanpipe_cyclonedx_resolve_cyclonedx_packages(self):
         # JSON v1.2
         input_location = self.data_location / "laravel-7.12.0" / "bom.1.2.json"
         packages = cyclonedx.resolve_cyclonedx_packages(input_location)
-        self.assertEqual(63, len(packages))
+        self.assertEqual(62, len(packages))
 
         # JSON v1.3
         input_location = self.data_location / "laravel-7.12.0" / "bom.1.3.json"
         packages = cyclonedx.resolve_cyclonedx_packages(input_location)
-        self.assertEqual(63, len(packages))
+        self.assertEqual(62, len(packages))
 
         # JSON v1.4
         input_location = self.data_location / "laravel-7.12.0" / "bom.1.4.json"
         packages = cyclonedx.resolve_cyclonedx_packages(input_location)
-        self.assertEqual(63, len(packages))
+        self.assertEqual(62, len(packages))
 
         # JSON v1.5 (this file is generated by the to_cyclonedx)
         input_location = self.data_location / "asgiref-3.3.0.cdx.json"
         packages = cyclonedx.resolve_cyclonedx_packages(input_location)
-        self.assertEqual(3, len(packages))
+        self.assertEqual(1, len(packages))
 
         # XML v1.4
         input_location = self.data_location / "laravel-7.12.0" / "bom.1.4.xml"
         packages = cyclonedx.resolve_cyclonedx_packages(input_location)
-        self.assertEqual(63, len(packages))
+        self.assertEqual(62, len(packages))
+
+    def test_scanpipe_cyclonedx_resolve_cyclonedx_packages_pre_validation(self):
+        # This SBOM includes multiple deserialization issues that are "fixed"
+        # by the pre-validation cleanup.
+        input_location = self.data_location / "broken_sbom.json"
+        package_data = cyclonedx.resolve_cyclonedx_packages(input_location)
+        self.assertEqual([{"name": "asgiref"}], package_data)
 
-    def test_scanpipe_cyclonedx_delete_empty_properties(self):
+    def test_scanpipe_cyclonedx_cleanup_components_properties(self):
         cyclonedx_document_json = {
             "components": [
                 {
@@ -231,6 +238,24 @@ def test_scanpipe_cyclonedx_delete_empty_properties(self):
                 }
             ]
         }
-        results = cyclonedx.delete_empty_properties(cyclonedx_document_json)
+        results = cyclonedx.cleanup_components_properties(cyclonedx_document_json)
         expected = {"components": [{"bom-ref": "pkg:type/name"}]}
         self.assertEqual(expected, results)
+
+    def test_scanpipe_cyclonedx_delete_ignored_root_properties(self):
+        cyclonedx_document_json = {
+            "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json",
+            "bomFormat": "CycloneDX",
+            "specVersion": "1.6",
+            "serialNumber": "urn:uuid:b74fe5df-e965-415e-ba65-f38421a0695d",
+            "version": 1,
+            "metadata": {
+                "component": {
+                    "bom-ref": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e",
+                },
+            },
+            "components": [{"bom-ref": "pkg:type/name"}],
+        }
+        results = cyclonedx.delete_ignored_root_properties(cyclonedx_document_json)
+        self.assertIn("components", results)
+        self.assertNotIn("metadata", results)

setup.cfg

@@ -52,15 +52,15 @@ install_requires =
     importlib-metadata==7.1.0
     setuptools==69.5.1
     # Django related
-    Django==5.0.4
+    Django==5.0.6
     django-environ==0.11.2
     django-crispy-forms==2.1
     crispy-bootstrap3==2024.1
     django-filter==24.2
     djangorestframework==3.15.1
     django-taggit==5.0.1
     # Database
-    psycopg[binary]==3.1.18
+    psycopg[binary]==3.1.19
     # wait_for_database Django management command
     django-probes==1.7.0
     # Task queue
@@ -91,7 +91,7 @@ install_requires =
     # Profiling
     pyinstrument==4.6.2
     # CycloneDX
-    cyclonedx-python-lib==7.3.2
+    cyclonedx-python-lib==7.3.4
     jsonschema==4.22.0
     # Font Awesome
     fontawesomefree==6.5.1