Merge pull request #12 from blackducksoftware/dev

v1.0.12 - Added recipe_report option

README.md

@@ -1,4 +1,4 @@
-# Synopsys Scan Yocto Script - bd_scan_yocto_via_sbom.py v1.0.11
+# Synopsys Scan Yocto Script - bd_scan_yocto_via_sbom.py v1.0.12
 
 # PROVISION OF THIS SCRIPT
 This script is provided under the MIT license (see LICENSE file).
@@ -54,7 +54,7 @@ Alternatively, if you have installed the repository locally:
 2. Invoke virtualenv where utility was installed
 3. Run `python3 run.py OPTIONS`
 
-## RECOMMENDATIONS
+## BEST PRACTICE RECOMMENDATIONS
 
 For optimal Yocto scan results, consider the following:
 
@@ -65,6 +65,7 @@ For optimal Yocto scan results, consider the following:
 2 projects and compare the results with and without this option.
 5. If you wish to add the Linux kernel and other packages specified in the image manifest only, 
 consider using the `--process_image_manifest` option and optionally specifying the image manifest license file path (--image_license_manifest FILEPATH) where it does not exist in the same folder and the license.manifest file.
+6. Use the `--recipe_report REPFILE` option to create a report of matched and unmatched recipes in the BOM. In particular check the recipes in the `RECIPES NOT IN BOM - MATCHED IN OE DATA` section.
 
 ## OPTIONAL BEHAVIOUR
 
@@ -152,6 +153,8 @@ There are several additional options to modify the behaviour of this utility inc
                            Create custom components when uploading SBOM (default False)
      --debug               Debug logging mode
      --logfile LOGFILE     Logging output file
+     --recipe_report REPFILE
+                           Output specified file with a list of recipes including those not matched in the BOM
 
 ### MINIMUM REQUIRED OPTIONS
 
@@ -254,15 +257,17 @@ OSS components as manual additions in the Project Version.
 
 - If you are looking for a specific package which appears to be missing from the project, confirm that you are looking for the recipe name not the package name. See the FAQs for an explanation of Yocto recipes versus packages. Check that the package file was included in the Signature scan (within the Source tab).
 
-## REPORTING ISSUES
+## HOW TO REPORT ISSUES WITH THIS SCRIPT
 
 If you encounter issues using the script, then please create an issue in GitHub, ensuring that you include the following information:
 
+- Your Organisation name (to validate you have a Black Duck license)
 - Description of the issue
 - Yocto version in use (use the command `bitbake -e | grep DISTRO_CODENAME`)
 - The `license.manifest` file for the build
 - The output of the command `bitbake-layers show-recipes`
 - The log output from this script with `--debug` enabled (use the `--logfile LOGFILE` option to output to a file )
+- The recipe report file (use `--recipe_report REPFILE`)
 
 # ADDITIONAL SCAN OPTIONS
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "bd_scan_yocto_via_sbom"
-version = "1.0.11"
+version = "1.0.12"
 authors = [
   { name="Matthew Brady", email="[email protected]" },
 ]

yocto_import_sbom/ConfigClass.py

@@ -89,6 +89,8 @@ def __init__(self):
 
         parser.add_argument("--debug", help="Debug logging mode", action='store_true')
         parser.add_argument("--logfile", help="Logging output file", default="")
+        parser.add_argument("--recipe_report", help="Output recipe report to file", default="")
+
 
         args = parser.parse_args()
 
@@ -125,6 +127,7 @@ def __init__(self):
         self.sbom_custom_components = args.sbom_create_custom_components
         self.cve_check_dir = ''
         self.license_dir = ''
+        self.recipe_report = ''
 
         terminate = False
         if args.debug:
@@ -141,7 +144,7 @@ def __init__(self):
         else:
             logging.basicConfig(level=loglevel)
 
-        logging.info("Black Duck Yocto scan via SBOM utility - v1.0.11")
+        logging.info("Black Duck Yocto scan via SBOM utility - v1.0.12")
         logging.info("SUPPLIED ARGUMENTS:")
         for arg in vars(args):
             logging.info(f"--{arg}={getattr(args, arg)}")
@@ -276,6 +279,13 @@ def __init__(self):
                 logging.error("Option --skip_bitbake set but --bitbake_layers_file not supplied")
                 terminate = True
 
+        if args.recipe_report != '':
+            if os.path.exists(args.recipe_report):
+                logging.error(f"Output recipe report file {args.recipe_report} already exists - terminating")
+                terminate = True
+            else:
+                self.recipe_report = args.recipe_report
+
         if terminate:
             sys.exit(2)
         return

yocto_import_sbom/RecipeClass.py

@@ -36,7 +36,10 @@ def get_epoch_and_version(version):
         return '', Recipe.filter_version_string(version)
 
     def print_recipe(self):
-        logging.info(f"Processed Recipe '{self.name}': {self.layer}/{self.name}/{self.version}")
+        logging.info(f"Processed Recipe '{self.name}': {self.full_id()}")
 
     def check_in_bom(self, bom):
-        return bom.check_recipe_in_bom(self.name, self.version)
+        return bom.check_recipe_in_bom(self.name, self.version)
+
+    def full_id(self):
+        return(f"{self.layer}/{self.name}/{self.version}")

yocto_import_sbom/RecipeListClass.py

@@ -143,33 +143,53 @@ def copy_files(files):
         else:
             return ''
 
-    def check_recipes_in_bom(self, bom):
-        not_in_bom = 0
-        not_matched_oe = 0
-        matched_oe_not_in_bom = 0
+    def check_recipes_in_bom(self, conf, bom):
+
+        in_bom = []
+        not_in_bom = []
+        matched_oe_not_in_bom = []
+        not_matched_oe_not_in_bom = []
+        not_matched_oe_in_bom = []
+        logging.info(f"Missing Recipes:")
         for recipe in self.recipes:
+            fullid = recipe.full_id()
 
             if not recipe.check_in_bom(bom):
-                not_in_bom += 1
+                not_in_bom.append(fullid)
                 if recipe.matched_oe:
-                    matched_oe_not_in_bom += 1
-                    logging.info(f"- Recipe {recipe.name}/{recipe.version}: Matched in OE data but not in BOM")
+                    matched_oe_not_in_bom.append(fullid)
+                    logging.info(f"- Recipe {fullid}: Matched in OE data but NOT found in BOM")
                 else:
-                    logging.debug(f"- Recipe {recipe.name}/{recipe.version}: Not matched in OE data and not in BOM")
-                    not_matched_oe += 1
-                    recipe.matched_in_bom
-            elif not recipe.matched_oe:
-                not_matched_oe += 1
-                logging.debug(f"- Recipe {recipe.name}/{recipe.version}: Not matched in OE data but in BOM")
-                recipe.matched_in_bom = True
+                    not_matched_oe_not_in_bom.append(fullid)
+                    logging.debug(f"- Recipe {fullid}: NOT matched in OE data and NOT found in BOM")
             else:
+                in_bom.append(fullid)
                 recipe.matched_in_bom = True
+                if not recipe.matched_oe:
+                    not_matched_oe_in_bom.append(fullid)
+                    # logging.debug(f"- Recipe {fullid}: Not matched in OE data but found in BOM")
             #     logging.info(f"- Recipe {recipe.name}/{recipe.version}: Found in BOM")
 
         logging.info("")
-        logging.info(" Summary of Components Mapped in BOM:")
-        logging.info(f"- Total components in BOM - {bom.count_comps()}")
+        logging.info(" Summary of Components Mapped in Black Duck BOM:")
         logging.info(f"- Total recipes in Yocto project - {self.count()}")
-        logging.info(f"- Recipes not in BOM - {not_in_bom}")
-        logging.info(f"- Recipes not matched from OE data - {not_matched_oe}")
-        logging.info(f"- Recipes matched in OE data but not in BOM - {matched_oe_not_in_bom}")
+        logging.info(f"- Total recipes matched in BOM - {bom.count_comps()}")
+        logging.info(f"    - Of which {len(not_matched_oe_in_bom)} not matched in OE data")
+        logging.info(f"- Recipes NOT in BOM - {len(not_in_bom)}")
+        logging.info(f"    - Of which {len(matched_oe_not_in_bom)} matched in OE data")
+        logging.info(f"    - Of which {len(not_matched_oe_not_in_bom)} not matched in OE data")
+
+        if conf.recipe_report != '':
+            try:
+                with open(conf.recipe_report, "w") as repfile:
+                    repfile.write(f"RECIPES IN BOM ({len(in_bom)}):\n")
+                    repfile.write("\n".join(in_bom))
+                    repfile.write(f"\n\nRECIPES IN BOM - NOT MATCHED IN OE DATA ({len(not_matched_oe_in_bom)}):\n")
+                    repfile.write("\n".join(not_matched_oe_in_bom))
+                    repfile.write(f"\n\nRECIPES NOT IN BOM - NOT MATCHED IN OE DATA ({len(not_matched_oe_not_in_bom)}):\n")
+                    repfile.write("\n".join(not_matched_oe_not_in_bom))
+                    repfile.write(f"\n\nRECIPES NOT IN BOM - MATCHED IN OE DATA ({len(matched_oe_not_in_bom)}):\n")
+                    repfile.write("\n".join(matched_oe_not_in_bom))
+                logging.info(f"Output full recipe report to '{conf.recipe_report}'")
+            except IOError as error:
+                logging.error(f"Unable to write recipe report file {conf.recipe_report} - {error}")

yocto_import_sbom/main.py

@@ -61,8 +61,7 @@ def main():
         logging.error("Error waiting for project scan completion")
         sys.exit(2)
     bom.get_data()
-    reclist.check_recipes_in_bom(bom)
-
+    reclist.check_recipes_in_bom(conf, bom)
 
     logging.info("")
     logging.info("--- PHASE 6 - SIGNATURE SCAN PACKAGES ------------------------------------")