fix: pollish evidence gathering (#1312)

followup of #1309

- fixed some false-positives for license evidences.
- refactored some functionality, so that it is much easier to add #1310
later

Signed-off-by: Jan Kowalleck <[email protected]>

Author: jkowalleck

HISTORY.md

@@ -7,7 +7,7 @@ All notable changes to this project will be documented in this file.
 <!-- unreleased changes go here -->
 
 * Added
-  * Feature for collecting (license) evidence ([#676] via [#1309])  
+  * Feature for collecting (license) evidence ([#676] via [#1309], [#1312])  
     Controlled with option `collectEvidence`, disabled by default.
 * Build
   * Use _TypeScript_ `v5.6.2` now, was `v5.5.3` (via [#1302], [#1306])
@@ -16,6 +16,7 @@ All notable changes to this project will be documented in this file.
 [#1302]: https://github.com/CycloneDX/cyclonedx-webpack-plugin/pull/1302
 [#1306]: https://github.com/CycloneDX/cyclonedx-webpack-plugin/pull/1306
 [#1309]: https://github.com/CycloneDX/cyclonedx-webpack-plugin/pull/1309
+[#1312]: https://github.com/CycloneDX/cyclonedx-webpack-plugin/pull/1312
 
 ## 3.13.0 - 2024-07-21
 

README.md

@@ -49,14 +49,14 @@ new CycloneDxWebpackPlugin(options?: object)
 | **`specVersion`** | `{string}`<br/>one of: `"1.2"`, `"1.3"`, `"1.4"`, `"1.5"`, `"1.6"` | `"1.4"` |  Which version of [CycloneDX-spec] to use.<br/> Supported values depend on the installed dependency [CycloneDX-javascript-library]. |
 | **`reproducibleResults`** | `{boolean}` | `false` | Whether to go the extra mile and make the output reproducible.<br/> Reproducibility might result in loss of time- and random-based-values. |
 | **`validateResults`** | `{boolean}` | `true` | Whether to validate the BOM result.<br/>Validation is skipped, if requirements not met. Requires [transitive optional dependencies](https://github.com/CycloneDX/cyclonedx-javascript-library#optional-dependencies). |
-| **`collectEvidence`** | `{boolean}` | `false` | Look for common files that may provide licenses and attach them to the component as evidence. |
 | **`outputLocation`** | `{string}` | `"./cyclonedx"` | Path to write the output to. The path is relative to _webpack_'s overall output path. |
 | **`includeWellknown`** | `{boolean}` | `true` | Whether to write the Wellknowns. |
 | **`wellknownLocation`** | `{string}` | `"./.well-known"` | Path to write the Wellknowns to. The path is relative to _webpack_'s overall output path. | 
 | **`rootComponentAutodetect`** | `{boolean}` | `true` | Whether to try auto-detection of the RootComponent.<br/> Tries to find the nearest `package.json` and build a CycloneDX component from it, so it can be assigned to `bom.metadata.component`. |
 | **`rootComponentType`** | `{string}` | `"application"` | Set the RootComponent's type.<br/>See [the list of valid values](https://cyclonedx.org/docs/1.4/json/#metadata_component_type). Supported values depend on [CycloneDX-javascript-library]'s enum `ComponentType`. |
 | **`rootComponentName`** | optional `{string}` | `undefined` | If `rootComponentAutodetect` is disabled, then this value is assumed as the "name" of the `package.json`. |
 | **`rootComponentVersion`** | optional `{string}` | `undefined` | If `rootComponentAutodetect` is disabled, then this value is assumed as the "version" of the `package.json`. |
+| **`collectEvidence`** | `{boolean}` | `false` | Whether to collect (license) evidence and attach them to the resulting SBOM. |
 
 ### Example
 

src/_helpers.ts

@@ -17,7 +17,7 @@ SPDX-License-Identifier: Apache-2.0
 Copyright (c) OWASP Foundation. All Rights Reserved.
 */
 
-import { existsSync, readdirSync, readFileSync } from 'fs'
+import { existsSync, readFileSync } from 'fs'
 import { dirname, extname, isAbsolute, join, sep } from 'path'
 
 export function isNonNullable<T> (value: T): value is NonNullable<T> {
@@ -88,50 +88,20 @@ export function loadJsonFile (path: string): any {
   // see https://github.com/tc39/proposal-import-attributes
 }
 
-const LICENSE_FILENAME_PATTERN = /^(?:UN)?LICEN[CS]E|NOTICE/i
-/**
- * Searches typical files in the package path which have typical a license notice text inside
- *
- * @param {string} searchFolder folder to look for common filenames
- *
- * @yields {{ filepath: string, contentType: string}} Next matching file containing path and MIME type
- */
-export function * searchEvidenceSources (searchFolder: string): Generator<{
-  filepath: string
-  contentType: string
-}> {
-  for (const dirent of readdirSync(searchFolder, { withFileTypes: true })) {
-    if (
-      !dirent.isFile() ||
-      !LICENSE_FILENAME_PATTERN.test(dirent.name)
-    ) {
-      continue
-    }
+// region MIME
 
-    const contentType = determineContentType(dirent.name)
-    if (contentType === undefined) {
-      continue
-    }
+export type MimeType = string
 
-    yield {
-      filepath: `${dirent.parentPath}/${dirent.name}`,
-      contentType
-    }
-  }
-}
-
-// common file endings that are used for notice/license files
-const CONTENT_TYPE_MAP: Record<string, string> = {
+const MAP_TEXT_EXTENSION_MIME: Readonly<Record<string, MimeType>> = {
   '': 'text/plain',
-  '.txt': 'text/plain',
   '.md': 'text/markdown',
-  '.xml': 'text/xml'
+  '.rst': 'text/prs.fallenstein.rst',
+  '.txt': 'text/plain',
+  '.xml': 'text/xml' // not `application/xml` -- our scope is text!
 } as const
 
-/**
- * Returns the MIME type for the file or undefined if nothing was matched
- * @param {string} filename filename or complete filepath
- */
-export function determineContentType (filename: string): string | undefined {
-  return CONTENT_TYPE_MAP[extname(filename)]
+export function getMimeForTextFile (filename: string): MimeType | undefined {
+  return MAP_TEXT_EXTENSION_MIME[extname(filename)]
 }
+
+// endregion MIME

src/extractor.ts

@@ -18,12 +18,12 @@ Copyright (c) OWASP Foundation. All Rights Reserved.
 */
 
 import * as CDX from '@cyclonedx/cyclonedx-library'
-import { readFileSync } from 'fs'
+import { readdirSync, readFileSync } from 'fs'
 import * as normalizePackageJson from 'normalize-package-data'
-import { basename, dirname } from 'path'
+import { dirname, join } from 'path'
 import type { Compilation, Module } from 'webpack'
 
-import { getPackageDescription, isNonNullable, type PackageDescription, searchEvidenceSources, structuredClonePolyfill } from './_helpers'
+import { getMimeForTextFile, getPackageDescription, isNonNullable, type PackageDescription, structuredClonePolyfill } from './_helpers'
 
 type WebpackLogger = Compilation['logger']
 
@@ -42,7 +42,7 @@ export class Extractor {
     this.#purlFactory = purlFactory
   }
 
-  generateComponents (modules: Iterable<Module>, collectEvidence?: boolean, logger?: WebpackLogger): Iterable<CDX.Models.Component> {
+  generateComponents (modules: Iterable<Module>, collectEvidence: boolean, logger?: WebpackLogger): Iterable<CDX.Models.Component> {
     const pkgs: Record<string, CDX.Models.Component | undefined> = {}
     const components = new Map<Module, CDX.Models.Component>()
 
@@ -83,7 +83,7 @@ export class Extractor {
   /**
    * @throws {Error} when no component could be fetched
    */
-  makeComponent (pkg: PackageDescription, collectEvidence?: boolean, logger?: WebpackLogger): CDX.Models.Component {
+  makeComponent (pkg: PackageDescription, collectEvidence: boolean, logger?: WebpackLogger): CDX.Models.Component {
     try {
       const _packageJson = structuredClonePolyfill(pkg.packageJson)
       normalizePackageJson(_packageJson as object /* add debug for warnings? */)
@@ -107,17 +107,15 @@ export class Extractor {
       l.acknowledgement = CDX.Enums.LicenseAcknowledgement.Declared
     })
 
+    if (collectEvidence) {
+      component.evidence = new CDX.Models.ComponentEvidence({
+        licenses: new CDX.Models.LicenseRepository(this.getLicenseEvidence(dirname(pkg.path), logger))
+      })
+    }
+
     component.purl = this.#purlFactory.makeFromComponent(component)
     component.bomRef.value = component.purl?.toString()
 
-    if (collectEvidence === true) {
-      try {
-        component.evidence = this.makeComponentEvidence(pkg)
-      } catch (e) {
-        logger?.warn('collecting Evidence from PkgPath', pkg.path, 'failed:', e)
-      }
-    }
-
     return component
   }
 
@@ -132,29 +130,45 @@ export class Extractor {
     }
   }
 
-  /**
-   * Look for common files that may provide licenses and attach them to the component as evidence
-   * @param pkg
-   */
-  makeComponentEvidence (pkg: PackageDescription): CDX.Models.ComponentEvidence {
-    const cdxComponentEvidence = new CDX.Models.ComponentEvidence()
-
-    // Add license evidence
-    for (const { contentType, filepath } of searchEvidenceSources(dirname(pkg.path))) {
-      cdxComponentEvidence.licenses.add(new CDX.Models.NamedLicense(
-        `file: ${basename(filepath)}`,
-        {
-          text: new CDX.Models.Attachment(
-            readFileSync(filepath).toString('base64'),
-            {
-              contentType,
-              encoding: CDX.Enums.AttachmentEncoding.Base64
-            }
-          )
-        }
-      ))
+  readonly #LICENSE_FILENAME_PATTERN = /^(?:UN)?LICEN[CS]E|^NOTICE$/i
+
+  public * getLicenseEvidence (packageDir: string, logger?: WebpackLogger): Generator<CDX.Models.License> {
+    let pcis
+    try {
+      pcis = readdirSync(packageDir, { withFileTypes: true })
+    } catch (e) {
+      logger?.warn('collecting license evidence in', packageDir, 'failed:', e)
+      return
     }
+    for (const pci of pcis) {
+      if (
+        !pci.isFile() ||
+        !this.#LICENSE_FILENAME_PATTERN.test(pci.name)
+      ) {
+        continue
+      }
+
+      const contentType = getMimeForTextFile(pci.name)
+      if (contentType === undefined) {
+        continue
+      }
 
-    return cdxComponentEvidence
+      const fp = join(packageDir, pci.name)
+      try {
+        yield new CDX.Models.NamedLicense(
+          `file: ${pci.name}`,
+          {
+            text: new CDX.Models.Attachment(
+              readFileSync(fp).toString('base64'),
+              {
+                contentType,
+                encoding: CDX.Enums.AttachmentEncoding.Base64
+              }
+            )
+          })
+      } catch (e) { // may throw if `readFileSync()` fails
+        logger?.warn('collecting license evidence from', fp, 'failed:', e)
+      }
+    }
   }
 }

src/plugin.ts

@@ -53,13 +53,6 @@ export interface CycloneDxWebpackPluginOptions {
    */
   validateResults?: CycloneDxWebpackPlugin['validateResults']
 
-  /**
-   * Look for common files that may provide licenses and attach them to the component as evidence
-   *
-   * @default false
-   */
-  collectEvidence?: boolean
-
   /**
    * Path to write the output to.
    * The path is relative to webpack's overall output path.
@@ -111,6 +104,13 @@ export interface CycloneDxWebpackPluginOptions {
    * @default undefined
    */
   rootComponentVersion?: CycloneDxWebpackPlugin['rootComponentVersion']
+
+  /**
+   * Whether to collect (license) evidence and attach them to the resulting SBOM.
+   *
+   * @default false
+   */
+  collectEvidence?: boolean
 }
 
 class ValidationError extends Error {
@@ -126,7 +126,6 @@ export class CycloneDxWebpackPlugin {
   specVersion: CDX.Spec.Version
   reproducibleResults: boolean
   validateResults: boolean
-  collectEvidence: boolean
 
   resultXml: string
   resultJson: string
@@ -137,23 +136,24 @@ export class CycloneDxWebpackPlugin {
   rootComponentName: CDX.Models.Component['name'] | undefined
   rootComponentVersion: CDX.Models.Component['version'] | undefined
 
+  collectEvidence: boolean
+
   constructor ({
     specVersion = CDX.Spec.Version.v1dot4,
     reproducibleResults = false,
     validateResults = true,
-    collectEvidence = false,
     outputLocation = './cyclonedx',
     includeWellknown = true,
     wellknownLocation = './.well-known',
     rootComponentAutodetect = true,
     rootComponentType = CDX.Enums.ComponentType.Application,
     rootComponentName = undefined,
-    rootComponentVersion = undefined
+    rootComponentVersion = undefined,
+    collectEvidence = false
   }: CycloneDxWebpackPluginOptions = {}) {
     this.specVersion = specVersion
     this.reproducibleResults = reproducibleResults
     this.validateResults = validateResults
-    this.collectEvidence = collectEvidence
     this.resultXml = joinPath(outputLocation, './bom.xml')
     this.resultJson = joinPath(outputLocation, './bom.json')
     this.resultWellknown = includeWellknown
@@ -163,6 +163,7 @@ export class CycloneDxWebpackPlugin {
     this.rootComponentType = rootComponentType
     this.rootComponentName = rootComponentName
     this.rootComponentVersion = rootComponentVersion
+    this.collectEvidence = collectEvidence
   }
 
   apply (compiler: Compiler): void {

tests/integration/__snapshots__/index.test.js.snap

@@ -4,8 +4,9 @@ const { CycloneDxWebpackPlugin } = require('@cyclonedx/webpack-plugin')
 const cycloneDxWebpackPluginOptions = new CycloneDxWebpackPlugin({
   specVersion: '1.4',
   outputLocation: '.bom',
-  collectEvidence: true,
-  reproducibleResults: true
+  reproducibleResults: true,
+  validateResults: true,
+  collectEvidence: true
 })
 
 module.exports = {