Refactor run_scancode to handle errors along success #1018 (#1019)

Signed-off-by: tdruez <[email protected]>

Author: tdruez

CHANGELOG.rst

@@ -1,6 +1,16 @@
 Changelog
 =========
 
+v32.8.0 (unreleased)
+--------------------
+
+- Refactor run_scancode to not fail on scan errors happening at the resource level,
+  such as a timeout. Project error message are created instead.
+  https://github.com/nexB/scancode.io/issues/1018
+
+- Add support for the SCANCODEIO_SCAN_FILE_TIMEOUT setting in the scan_package pipeline.
+  https://github.com/nexB/scancode.io/issues/1018
+
 v32.7.0 (2023-10-25)
 --------------------
 

docker-compose.dev.yml

@@ -0,0 +1,17 @@
+version: "3"
+
+# Mount the local scanpipe/ directory in the containers
+
+# This can be used to refresh fixtures from the docker container:
+# $ docker compose -f docker-compose.yml -f docker-compose.dev.yml up
+# $ docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm web bash
+# $ SCANCODEIO_TEST_FIXTURES_REGEN=1 ./manage.py test
+
+services:
+  web:
+    volumes:
+      - ./scanpipe:/opt/scancodeio/scanpipe
+
+  worker:
+    volumes:
+      - ./scanpipe:/opt/scancodeio/scanpipe

docs/application-settings.rst

@@ -132,19 +132,6 @@ synchronously in the web server process.
 
         export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
 
-SCANCODE_TOOLKIT_RUN_SCAN_ARGS
-------------------------------
-
-Use this setting to provide any default options for running ScanCode-toolkit.
-
-.. note::
-    Refer to `ScanCode-toolkit Available Options <https://scancode-toolkit.readthedocs.io/en/latest/cli-reference/list-options.html>`_
-    for the full list of available options.
-
-The following example explicitly defines a timeout value of 60::
-
-    SCANCODE_TOOLKIT_RUN_SCAN_ARGS="timeout=60,"
-
 SCANCODEIO_TASK_TIMEOUT
 -----------------------
 

scancodeio/settings.py

@@ -78,8 +78,6 @@
     "SCANCODEIO_CONFIG_FILE", default="scancode-config.yml"
 )
 
-SCANCODE_TOOLKIT_RUN_SCAN_ARGS = env.dict("SCANCODE_TOOLKIT_RUN_SCAN_ARGS", default={})
-
 SCANCODEIO_LOG_LEVEL = env.str("SCANCODEIO_LOG_LEVEL", "INFO")
 
 # Set the number of parallel processes to use for ScanCode related scan execution.

scanpipe/pipelines/scan_package.py

@@ -25,7 +25,6 @@
 from django.core.serializers.json import DjangoJSONEncoder
 
 from commoncode.hash import multi_checksums
-from scancode import ScancodeError
 
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import input
@@ -99,14 +98,18 @@ def run_scancode(self):
         if license_score := self.project.get_env("scancode_license_score"):
             run_scan_args["license_score"] = license_score
 
-        errors = scancode.run_scan(
+        scanning_errors = scancode.run_scan(
             location=str(self.project.codebase_path),
             output_file=self.scan_output_location,
             run_scan_args=run_scan_args,
         )
 
-        if errors:
-            raise ScancodeError(errors)
+        for resource_path, errors in scanning_errors.items():
+            self.project.add_error(
+                description="\n".join(errors),
+                model=self.pipeline_name,
+                details={"path": resource_path},
+            )
 
         if not scan_output_path.exists():
             raise FileNotFoundError("ScanCode output not available.")

scanpipe/pipes/scancode.py

@@ -466,38 +466,31 @@ def get_pretty_params(args):
 
 
 def run_scan(location, output_file, run_scan_args):
-    """
-    Scan the `location` content and write the results into an `output_file`.
-    If `raise_on_error` is enabled, a ScancodeError will be raised if an error occurs
-    during the scan.
-    """
-    run_args = settings.SCANCODE_TOOLKIT_RUN_SCAN_ARGS.copy()
-    # The run_scan_args should override any values provided in the settings
-    run_args.update(run_scan_args)
-
-    if "timeout" in run_args:
-        run_args["timeout"] = int(run_args.get("timeout"))
-
-    success, results = scancode_run_scan(
+    """Scan the `location` content and write the results into an `output_file`."""
+    _success, results = scancode_run_scan(
         input=shlex.quote(location),
         processes=get_max_workers(keep_available=1),
         quiet=True,
         verbose=False,
         return_results=True,
         echo_func=None,
-        pretty_params=get_pretty_params(run_args),
-        **run_args,
+        pretty_params=get_pretty_params(run_scan_args),
+        timeout=settings.SCANCODEIO_SCAN_FILE_TIMEOUT,
+        **run_scan_args,
     )
 
-    if success:
+    # ``_success`` will be False if any scanning errors occur, but we still want
+    # to generate the results output in that case.
+    if results:
         Path(output_file).write_text(json.dumps(results, indent=2))
-        return
 
-    errors = {}
+    # Capture scan errors logged at the files level.
+    scanning_errors = {}
     for file in results.get("files", []):
-        if scan_errors := file.get("scan_errors"):
-            errors[file.get("path")] = scan_errors
-    return errors
+        if errors := file.get("scan_errors"):
+            scanning_errors[file.get("path")] = errors
+
+    return scanning_errors
 
 
 def get_virtual_codebase(project, input_location):