Upgrade Django to version 5.0.0 #1020 (#1021)

* Upgrade multiple dependencies to latest version #1020

Signed-off-by: tdruez <[email protected]>

* Upgrade Django to version 5.0.0 #1020

Signed-off-by: tdruez <[email protected]>

* Upgrade the virtualenv package #1020

Signed-off-by: tdruez <[email protected]>

* Fix some DeprecationWarning #1020

Signed-off-by: tdruez <[email protected]>

* Remove support for Python 3.8 and 3.9 #1020

Signed-off-by: tdruez <[email protected]>

* Migrate to POST-only logout view #1020

Signed-off-by: tdruez <[email protected]>

* Silent the warning about DB access during app initialization #1020

Signed-off-by: tdruez <[email protected]>

* Fix failing test #1020

Signed-off-by: tdruez <[email protected]>

* Add changelog entry #1020

Signed-off-by: tdruez <[email protected]>

* Fix the python_inspector import reference #1020

Signed-off-by: tdruez <[email protected]>

* Improve unit test support across OSes #1020

Signed-off-by: tdruez <[email protected]>

---------

Signed-off-by: tdruez <[email protected]>

Author: tdruez

.github/workflows/ci.yml

@@ -31,7 +31,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11"]
 
     steps:
       - name: Checkout code

CHANGELOG.rst

@@ -1,9 +1,12 @@
 Changelog
 =========
 
-v32.8.0 (unreleased)
+v33.0.0 (unreleased)
 --------------------
 
+- Upgrade Django to version 5.0 and drop support for Python 3.8 and 3.9
+  https://github.com/nexB/scancode.io/issues/1020
+
 - Refactor run_scancode to not fail on scan errors happening at the resource level,
   such as a timeout. Project error message are created instead.
   https://github.com/nexB/scancode.io/issues/1018

docs/custom-pipelines.rst

@@ -240,7 +240,7 @@ the entry point to the pipeline under the ``[options.entry_points]`` section.
     packages=find:
     include_package_data = true
     zip_safe = false
-    python_requires = >=3.8
+    python_requires = >=3.10
     setup_requires = setuptools_scm[toml] >= 4
 
     [options.packages.find]

docs/installation.rst

@@ -224,7 +224,7 @@ Pre-installation Checklist
 
 Before you install ScanCode.io, make sure you have the following prerequisites:
 
- * **Python: versions 3.8 to 3.11** found at https://www.python.org/downloads/
+ * **Python: versions 3.10 to 3.11** found at https://www.python.org/downloads/
  * **Git**: most recent release available at https://git-scm.com/
  * **PostgreSQL**: release 11 or later found at https://www.postgresql.org/ or
    https://postgresapp.com/ on macOS

etc/thirdparty/virtualenv.pyz

@@ -1,7 +1,7 @@
 about_resource: virtualenv.pyz
 name: get-virtualenv
-version: 20.24.6
-download_url: https://github.com/pypa/get-virtualenv/raw/20.24.6/public/virtualenv.pyz
+version: 20.25.0
+download_url: https://github.com/pypa/get-virtualenv/raw/20.25.0/public/virtualenv.pyz
 description: virtualenv is a tool to create isolated Python environments.
 homepage_url: https://github.com/pypa/virtualenv
 license_expression: lgpl-2.1-plus AND (bsd-new OR apache-2.0) AND mit AND python AND bsd-new
@@ -10,4 +10,4 @@ copyright: Copyright (c) The Python Software Foundation and others
 redistribute: yes
 attribute: yes
 track_changes: yes
-package_url: pkg:github/pypa/get-virtualenv@20.24.6#public/virtualenv.pyz
+package_url: pkg:github/pypa/get-virtualenv@20.25.0#public/virtualenv.pyz

etc/thirdparty/virtualenv.pyz.ABOUT

@@ -23,6 +23,7 @@
 import inspect
 import logging
 import sys
+import warnings
 from importlib.machinery import SourceFileLoader
 from pathlib import Path
 
@@ -74,6 +75,12 @@ def ready(self):
         # before its running process death.
         # In ASYNC mode, the cleanup is handled by the "ScanCodeIOWorker" worker.
         if not settings.SCANCODEIO_ASYNC and "runserver" in sys.argv:
+            warnings.filterwarnings(
+                "ignore",
+                message="Accessing the database during app initialization",
+                category=RuntimeWarning,
+                module="django",
+            )
             self.sync_runs_and_jobs()
 
     def load_pipelines(self):
@@ -82,9 +89,7 @@ def load_pipelines(self):
         pipelines Python files found at `SCANCODEIO_PIPELINES_DIRS` locations.
         """
         entry_points = importlib_metadata.entry_points()
-
-        # Ignore duplicated entries caused by duplicated paths in `sys.path`.
-        pipeline_entry_points = set(entry_points.get("scancodeio_pipelines"))
+        pipeline_entry_points = set(entry_points.select(group="scancodeio_pipelines"))
 
         for entry_point in sorted(pipeline_entry_points):
             self.register_pipeline(name=entry_point.name, cls=entry_point.load())

scanpipe/apps.py

@@ -302,19 +302,6 @@ def get_bin_executable(filename):
     return str(Path(sys.executable).parent / filename)
 
 
-def remove_prefix(text, prefix):
-    """
-    Remove the `prefix` from `text`.
-    Note that build-in `removeprefix` was added in Python3.9 but we need to keep
-    this one for Python3.8 support.
-    https://docs.python.org/3.9/library/stdtypes.html#str.removeprefix
-    """
-    if text.startswith(prefix):
-        prefix_len = len(prefix)
-        return text[prefix_len:]
-    return text
-
-
 class LoopProgress:
     """
     A context manager for logging progress in loops.

scanpipe/pipes/__init__.py

@@ -1402,7 +1402,8 @@ def flag_whitespace_files(project):
     whitespace_set = set(b" \n\r\t\f\b")
 
     for resource in resources:
-        binary_data = open(resource.location, "rb").read()
+        with open(resource.location, "rb") as f:
+            binary_data = f.read()
         binary_set = set(binary_data)
         non_whitespace_bytes = binary_set - whitespace_set
 

scanpipe/pipes/d2d.py

@@ -30,7 +30,7 @@
 from packagedcode import APPLICATION_PACKAGE_DATAFILE_HANDLERS
 from packagedcode.licensing import get_license_detections_and_expression
 from packageurl import PackageURL
-from python_inspector.resolve_cli import resolver_api
+from python_inspector.api import resolve_dependencies
 from scancode.api import get_package_data
 
 from scanpipe.models import DiscoveredPackage
@@ -64,7 +64,7 @@ def resolve_pypi_packages(input_location):
     python_version = f"{sys.version_info.major}{sys.version_info.minor}"
     operating_system = "linux"
 
-    inspector_output = resolver_api(
+    inspector_output = resolve_dependencies(
         requirement_files=[input_location],
         python_version=python_version,
         operating_system=operating_system,

scanpipe/pipes/resolve.py

@@ -39,6 +39,7 @@
       .is-black-link {color: #363636;}
       .is-grey-link {color: #7a7a7a;}
       .is-black-link:hover, .is-grey-link:hover {color: #3273dc; text-decoration: underline;}
+      .navbar button.navbar-item {font-size: 1em;}
       #inputs-panel .panel-block.dropdown:hover {background-color: #f5f5f5;}
       #inputs-panel .dropdown-menu {width: 85%;}
       a.panel-block {word-break: break-all;}

scanpipe/templates/scanpipe/base.html

@@ -27,9 +27,12 @@
           <a class="navbar-item" href="{% url 'account_profile' %}">
             Profile settings
           </a>
-          <a class="navbar-item" href="{% url 'logout' %}">
-            Sign out
-          </a>
+          <form id="logout-form" method="post" action="{% url 'logout' %}">
+            {% csrf_token %}
+            <button class="navbar-item button is-white has-text-grey-dark is-fullwidth is-justify-content-flex-start" type="submit">
+              Sign out
+            </button>
+          </form>
           <hr class="navbar-divider">
           <div class="navbar-item">
             <i>ScanCode.io {{ SCANCODEIO_VERSION }}</i>

scanpipe/templates/scanpipe/includes/navbar_header.html

@@ -963,23 +963,23 @@ def test_scanpipe_pipes_d2d_get_project_resources_qs(self):
             "directory1/foo.txt",
         ]
         expected_qs = self.project1.codebaseresources.filter(path__in=expected_paths)
-        self.assertQuerysetEqual(expected_qs, resources_qs)
+        self.assertQuerySetEqual(expected_qs, resources_qs)
 
     def test_scanpipe_pipes_d2d_get_from_files_related_with_not_in_package_to_files(
         self,
     ):
         from_resource1 = make_resource_file(self.project1, "from/foo.java")
         to_resource1 = make_resource_file(self.project1, "to/foo.class")
         qs = d2d.get_from_files_related_with_not_in_package_to_files(self.project1)
-        self.assertQuerysetEqual([], qs)
+        self.assertQuerySetEqual([], qs)
 
         pipes.make_relation(from_resource1, to_resource1, "java_to_class")
         qs = d2d.get_from_files_related_with_not_in_package_to_files(self.project1)
-        self.assertQuerysetEqual([], qs)
+        self.assertQuerySetEqual([], qs)
 
         from_resource1.update(detected_license_expression="mit")
         qs = d2d.get_from_files_related_with_not_in_package_to_files(self.project1)
-        self.assertQuerysetEqual([from_resource1], qs)
+        self.assertQuerySetEqual([from_resource1], qs)
 
     def test_scanpipe_pipes_d2d_create_local_files_packages(self):
         from_resource1 = make_resource_file(

scanpipe/tests/pipes/test_d2d.py

@@ -133,9 +133,11 @@ def test_scanpipe_pipes_js_get_map_sources_content(self):
                 "resources/adaptive_media/js/main.js.map"
             )
         )
-        expected = open(expected_location, "r").read()
-        result = js.get_map_sources_content(to_resource)
 
+        with open(expected_location, "r") as f:
+            expected = f.read()
+
+        result = js.get_map_sources_content(to_resource)
         self.assertEqual([expected], result)
 
     def test_scanpipe_pipes_js_get_minified_resource(self):

scanpipe/tests/pipes/test_js.py

@@ -458,10 +458,10 @@ def test_scanpipe_pipes_scancode_make_results_summary(self, regen=FIXTURES_REGEN
         uuid = "ba110d49-b6f2-4c86-8d89-a6fd34838ca8"
         package.update(package_uid=f"pkg:npm/[email protected]?uuid={uuid}")
 
-        # Patching the ``file_type`` values as those OS dependant.
+        # Patching the ``file_type`` and ``mime_typea` values as those are OS dependant.
         # Note that we cannot use proper ``mock`` as the ``scan_package`` pipeline
         # uses a subprocess call to run the ``scancode`` command.
-        project1.codebaseresources.all().update(file_type="")
+        project1.codebaseresources.all().update(file_type="", mime_type="text/plain")
 
         scan_output_location = self.data_location / "is-npm-1.0.0_scan_package.json"
         summary = scancode.make_results_summary(project1, scan_output_location)

scanpipe/tests/pipes/test_scancode.py

@@ -95,15 +95,18 @@ def test_scancodeio_auth_logged_in_navbar_header(self):
         self.assertContains(response, expected, html=True)
         expected = f'<a class="navbar-item" href="{profile_url}">Profile settings</a>'
         self.assertContains(response, expected, html=True)
-        expected = f'<a class="navbar-item" href="{logout_url}">Sign out</a>'
-        self.assertContains(response, expected, html=True)
+        expected = f'<form id="logout-form" method="post" action="{logout_url}">'
+        self.assertContains(response, expected)
 
     def test_scancodeio_auth_logout_view(self):
         response = self.client.get(logout_url)
+        self.assertEqual(405, response.status_code)
+
+        response = self.client.post(logout_url)
         self.assertRedirects(response, login_url)
 
         self.client.login(username=self.basic_user.username, password=TEST_PASSWORD)
-        response = self.client.get(logout_url)
+        response = self.client.post(logout_url)
         self.assertRedirects(response, login_url)
 
     def test_scancodeio_account_profile_view(self):

scanpipe/tests/test_auth.py

@@ -27,6 +27,7 @@
 import uuid
 from contextlib import redirect_stdout
 from datetime import datetime
+from datetime import timezone as tz
 from pathlib import Path
 from unittest import mock
 from unittest import skipIf
@@ -754,11 +755,11 @@ def test_scanpipe_run_model_task_execution_time_property(self):
 
         self.assertIsNone(run1.execution_time)
 
-        run1.task_start_date = datetime(1984, 10, 10, 10, 10, 10, tzinfo=timezone.utc)
+        run1.task_start_date = datetime(1984, 10, 10, 10, 10, 10, tzinfo=tz.utc)
         run1.save()
         self.assertIsNone(run1.execution_time)
 
-        run1.task_end_date = datetime(1984, 10, 10, 10, 10, 35, tzinfo=timezone.utc)
+        run1.task_end_date = datetime(1984, 10, 10, 10, 10, 35, tzinfo=tz.utc)
         run1.save()
         self.assertEqual(25.0, run1.execution_time)
 
@@ -771,19 +772,19 @@ def test_scanpipe_run_model_execution_time_for_display_property(self):
 
         self.assertIsNone(run1.execution_time_for_display)
 
-        run1.task_start_date = datetime(1984, 10, 10, 10, 10, 10, tzinfo=timezone.utc)
+        run1.task_start_date = datetime(1984, 10, 10, 10, 10, 10, tzinfo=tz.utc)
         run1.save()
         self.assertIsNone(run1.execution_time_for_display)
 
-        run1.task_end_date = datetime(1984, 10, 10, 10, 10, 35, tzinfo=timezone.utc)
+        run1.task_end_date = datetime(1984, 10, 10, 10, 10, 35, tzinfo=tz.utc)
         run1.save()
         self.assertEqual("25 seconds", run1.execution_time_for_display)
 
-        run1.task_end_date = datetime(1984, 10, 10, 10, 12, 35, tzinfo=timezone.utc)
+        run1.task_end_date = datetime(1984, 10, 10, 10, 12, 35, tzinfo=tz.utc)
         run1.save()
         self.assertEqual("145 seconds (2.4 minutes)", run1.execution_time_for_display)
 
-        run1.task_end_date = datetime(1984, 10, 10, 11, 12, 35, tzinfo=timezone.utc)
+        run1.task_end_date = datetime(1984, 10, 10, 11, 12, 35, tzinfo=tz.utc)
         run1.save()
         self.assertEqual("3745 seconds (1.0 hours)", run1.execution_time_for_display)
 

scanpipe/tests/test_models.py

@@ -253,13 +253,16 @@ def test_scanpipe_rootfs_pipeline_extract_input_files_errors(self):
         self.assertEqual("Error\nError", error.description)
 
 
-def sort_scanned_files_by_path(scan_data):
+def sort_for_os_compatibility(scan_data):
     """
-    Sort the ``scan_data`` files in place. Return ``scan_data``.
+    Sort the ``scan_data`` files and relations in place. Return ``scan_data``.
     """
-    files = scan_data.get("files")
-    if files:
+    if files := scan_data.get("files"):
         files.sort(key=lambda x: x["path"])
+
+    if relations := scan_data.get("relations"):
+        relations.sort(key=lambda x: x["to_resource"])
+
     return scan_data
 
 
@@ -292,8 +295,9 @@ class PipelinesIntegrationTest(TestCase):
         # system_environment differs between systems
         "system_environment",
         "file_type",
-        # mime type is inconsistent across systems
+        # mime type and is_script are inconsistent across systems
         "mime_type",
+        "is_script",
         "notes",
         "settings",
         "description",
@@ -381,15 +385,15 @@ def assertPipelineResultEqual(
         result_json = json.loads(Path(result_file).read_text())
         result_json = self._normalize_package_uids(result_json)
         result_data = self._without_keys(result_json, self.exclude_from_diff)
-        result_data = sort_scanned_files_by_path(result_data)
+        result_data = sort_for_os_compatibility(result_data)
 
         if regen:
             expected_file.write_text(json.dumps(result_data, indent=2))
 
         expected_json = json.loads(expected_file.read_text())
         expected_json = self._normalize_package_uids(expected_json)
         expected_data = self._without_keys(expected_json, self.exclude_from_diff)
-        expected_data = sort_scanned_files_by_path(expected_data)
+        expected_data = sort_for_os_compatibility(expected_data)
 
         self.assertEqual(expected_data, result_data)
 
@@ -766,23 +770,23 @@ def test_scanpipe_inspect_manifest_pipeline_integration_test(self):
         self.assertEqual(1, exitcode, msg=out)
         self.assertIn("No package type found for", out)
 
-    @mock.patch("scanpipe.pipes.resolve.resolver_api")
+    @mock.patch("scanpipe.pipes.resolve.resolve_dependencies")
     def test_scanpipe_inspect_manifest_pipeline_pypi_integration_test(
-        self, resolver_api
+        self, resolve_dependencies
     ):
         pipeline_name = "inspect_manifest"
         project1 = Project.objects.create(name="Analysis")
 
         run = project1.add_pipeline(pipeline_name)
         pipeline = run.make_pipeline_instance()
 
-        resolver_api.return_value = mock.Mock(packages=[])
+        resolve_dependencies.return_value = mock.Mock(packages=[])
         project1.move_input_from(tempfile.mkstemp(suffix="requirements.txt")[1])
         exitcode, out = pipeline.execute()
         self.assertEqual(1, exitcode, msg=out)
         self.assertIn("No packages could be resolved", out)
 
-        resolver_api.return_value = mock.Mock(packages=[package_data1])
+        resolve_dependencies.return_value = mock.Mock(packages=[package_data1])
         exitcode, out = pipeline.execute()
         self.assertEqual(0, exitcode, msg=out)