diff --git a/aboutcode/federated/__init__.py b/aboutcode/federated/__init__.py index 275f42cf8..373d655f1 100644 --- a/aboutcode/federated/__init__.py +++ b/aboutcode/federated/__init__.py @@ -1300,7 +1300,6 @@ def from_hashids( purl_type: str, hashids: list[str], ) -> "DataRepository": - """ Return a new DataRepository to store ``data_kind`` of ``purl_type`` for a list of ``hashids``. diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index 850587ca7..4b0f13fd1 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -456,7 +456,7 @@ def clean_summary(self, summary): # https://github.com/cms-dev/cms/issues/888#issuecomment-516977572 summary = summary.strip() if summary: - summary = summary.replace("\x00", "\uFFFD") + summary = summary.replace("\x00", "\ufffd") return summary def to_dict(self): @@ -506,9 +506,9 @@ def from_dict(cls, advisory_data): affected_package_cls.from_dict(pkg) for pkg in affected_packages if pkg is not None ], "references": [Reference.from_dict(ref) for ref in advisory_data["references"]], - "date_published": datetime.datetime.fromisoformat(date_published) - if date_published - else None, + "date_published": ( + datetime.datetime.fromisoformat(date_published) if date_published else None + ), "weaknesses": advisory_data["weaknesses"], "url": advisory_data.get("url") or None, } @@ -548,7 +548,7 @@ def clean_summary(self, summary): # https://github.com/cms-dev/cms/issues/888#issuecomment-516977572 summary = summary.strip() if summary: - summary = summary.replace("\x00", "\uFFFD") + summary = summary.replace("\x00", "\ufffd") return summary def to_dict(self): @@ -574,9 +574,9 @@ def from_dict(cls, advisory_data): if pkg is not None ], "references": [Reference.from_dict(ref) for ref in advisory_data["references"]], - "date_published": datetime.datetime.fromisoformat(date_published) - if date_published - else None, + "date_published": ( + datetime.datetime.fromisoformat(date_published) if date_published else None + ), "weaknesses": advisory_data["weaknesses"], "url": advisory_data.get("url") or None, } diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 82ee4525a..507148e19 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -50,6 +50,7 @@ from vulnerabilities.pipelines.v2_importers import github_osv_importer as github_osv_importer_v2 from vulnerabilities.pipelines.v2_importers import gitlab_importer as gitlab_importer_v2 from vulnerabilities.pipelines.v2_importers import istio_importer as istio_importer_v2 +from vulnerabilities.pipelines.v2_importers import liferay_importer from vulnerabilities.pipelines.v2_importers import mozilla_importer as mozilla_importer_v2 from vulnerabilities.pipelines.v2_importers import npm_importer as npm_importer_v2 from vulnerabilities.pipelines.v2_importers import nvd_importer as nvd_importer_v2 @@ -115,5 +116,6 @@ ubuntu_usn.UbuntuUSNImporter, fireeye.FireyeImporter, oss_fuzz.OSSFuzzImporter, + liferay_importer.LiferayImporterPipeline, ] ) diff --git a/vulnerabilities/importers/gentoo.py b/vulnerabilities/importers/gentoo.py index 2f569cdf1..3716ddaa4 100644 --- a/vulnerabilities/importers/gentoo.py +++ b/vulnerabilities/importers/gentoo.py @@ -76,9 +76,11 @@ def process_file(self, file): summary=summary, references=vuln_references, affected_packages=affected_packages, - url=f"https://security.gentoo.org/glsa/{id}" - if id - else "https://security.gentoo.org/glsa", + url=( + f"https://security.gentoo.org/glsa/{id}" + if id + else "https://security.gentoo.org/glsa" + ), ) @staticmethod diff --git a/vulnerabilities/importers/mattermost.py b/vulnerabilities/importers/mattermost.py index a422ea32a..a9581a813 100644 --- a/vulnerabilities/importers/mattermost.py +++ b/vulnerabilities/importers/mattermost.py @@ -103,13 +103,15 @@ def to_advisories(self, data): Reference( reference_id=ref_col.text, url=SECURITY_UPDATES_URL, - severities=[ - VulnerabilitySeverity( - system=severity_systems.CVSS31_QUALITY, value=severity_col.text - ) - ] - if severity_col.text.lower() != "na" - else [], + severities=( + [ + VulnerabilitySeverity( + system=severity_systems.CVSS31_QUALITY, value=severity_col.text + ) + ] + if severity_col.text.lower() != "na" + else [] + ), ) ] diff --git a/vulnerabilities/importers/postgresql.py b/vulnerabilities/importers/postgresql.py index 70ab1bfe9..6044dddaa 100644 --- a/vulnerabilities/importers/postgresql.py +++ b/vulnerabilities/importers/postgresql.py @@ -74,11 +74,11 @@ def to_advisories(data): type="generic", qualifiers=pkg_qualifiers, ), - affected_version_range=GenericVersionRange.from_versions( - affected_version_list - ) - if affected_version_list - else None, + affected_version_range=( + GenericVersionRange.from_versions(affected_version_list) + if affected_version_list + else None + ), fixed_version=GenericVersion(fixed_version) if fixed_version else None, ) ) diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index 68e3d5062..4e4682f18 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -156,7 +156,9 @@ def to_advisory(advisory_data): affected_packages=affected_packages, references=references, weaknesses=cwe_list, - url=resource_url - if resource_url - else "https://access.redhat.com/hydra/rest/securitydata/cve.json", + url=( + resource_url + if resource_url + else "https://access.redhat.com/hydra/rest/securitydata/cve.json" + ), ) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index e1c4ddc6b..86e8b0584 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -1128,9 +1128,9 @@ def get_affecting_vulnerabilities(self): next_fixed_package_vulns = list(fixed_by_pkg.affected_by) fixed_by_package_details["fixed_by_purl"] = fixed_by_purl - fixed_by_package_details[ - "fixed_by_purl_vulnerabilities" - ] = next_fixed_package_vulns + fixed_by_package_details["fixed_by_purl_vulnerabilities"] = ( + next_fixed_package_vulns + ) fixed_by_pkgs.append(fixed_by_package_details) vuln_details["fixed_by_package_details"] = fixed_by_pkgs diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py index c9fc21a84..50859f074 100644 --- a/vulnerabilities/pipelines/enhance_with_kev.py +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -88,9 +88,9 @@ def add_vulnerability_exploit(kev_vul, logger): "required_action": kev_vul["requiredAction"], "due_date": kev_vul["dueDate"], "notes": kev_vul["notes"], - "known_ransomware_campaign_use": True - if kev_vul["knownRansomwareCampaignUse"] == "Known" - else False, + "known_ransomware_campaign_use": ( + True if kev_vul["knownRansomwareCampaignUse"] == "Known" else False + ), }, ) return 1 diff --git a/vulnerabilities/pipelines/v2_importers/liferay_importer.py b/vulnerabilities/pipelines/v2_importers/liferay_importer.py new file mode 100644 index 000000000..4a5d53030 --- /dev/null +++ b/vulnerabilities/pipelines/v2_importers/liferay_importer.py @@ -0,0 +1,204 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +import re +from typing import Iterable +from urllib.parse import urljoin + +import requests +from bs4 import BeautifulSoup +from packageurl import PackageURL +from univers.version_range import MavenVersionRange + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackageV2 +from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 +from vulnerabilities.severity_systems import CVSSV31 + +logger = logging.getLogger(__name__) + + +class LiferayImporterPipeline(VulnerableCodeBaseImporterPipelineV2): + spdx_license_expression = "Apache-2.0" + license_url = "https://www.apache.org/licenses/LICENSE-2.0" + pipeline_id = "liferay_importer_v2" + importer_name = "Liferay Importer" + + release_links = [] + + @classmethod + def steps(cls): + return (cls.collect_and_store_advisories,) + + def advisories_count(self) -> int: + if not self.release_links: + self.release_links = self.fetch_release_links() + return len(self.release_links) + + def collect_advisories(self) -> Iterable[AdvisoryData]: + if not self.release_links: + self.release_links = self.fetch_release_links() + + for release_url in self.release_links: + yield from self.process_release_page(release_url) + + def fetch_release_links(self): + base_url = "https://liferay.dev/portal/security/known-vulnerabilities" + try: + main_page = requests.get(base_url) + main_page.raise_for_status() + except requests.RequestException as e: + logger.error(f"Failed to fetch Liferay main page: {e}") + return [] + + soup = BeautifulSoup(main_page.content, "lxml") + links = set() + for a in soup.find_all("a", href=True): + href = a["href"] + if "/categories/" in href and "known-vulnerabilities" in href: + links.add(urljoin(base_url, href)) + return list(links) + + def process_release_page(self, release_url): + try: + page = requests.get(release_url) + page.raise_for_status() + except requests.RequestException as e: + logger.error(f"Failed to fetch release page {release_url}: {e}") + return + + soup = BeautifulSoup(page.content, "lxml") + + # 3. Find Vulnerability Links + vuln_links = set() + for a in soup.find_all("a", href=True): + href = a["href"] + if "/asset_publisher/" in href and "cve-" in href.lower(): + vuln_links.add(urljoin(release_url, href)) + + for vuln_url in vuln_links: + yield from self.process_vulnerability_page(vuln_url) + + def process_vulnerability_page(self, vuln_url): + try: + page = requests.get(vuln_url) + page.raise_for_status() + except requests.RequestException as e: + logger.error(f"Failed to fetch vulnerability page {vuln_url}: {e}") + return + + soup = BeautifulSoup(page.content, "lxml") + + # Extract Details + title = soup.find("h1") + title_text = title.get_text(strip=True) if title else "" + + # CVE ID + cve_match = re.search(r"(CVE-\d{4}-\d{4,})", title_text) + if not cve_match: + cve_match = re.search(r"(CVE-\d{4}-\d{4,})", soup.get_text()) + + cve_id = cve_match.group(1) if cve_match else "" + if not cve_id: + return + + # Description + description_header = soup.find(string=re.compile("Description")) + description = "" + if description_header: + header_elem = description_header.parent + if header_elem.name.startswith("h"): + desc_elem = header_elem.find_next_sibling() + if desc_elem: + description = desc_elem.get_text(strip=True) + + # Severity + severity_header = soup.find(string=re.compile("Severity")) + severities = [] + if severity_header: + header_elem = severity_header.parent + if header_elem.name.startswith("h"): + sev_elem = header_elem.find_next_sibling() + if sev_elem: + sev_text = sev_elem.get_text(strip=True) + cvss_match = re.search(r"\(CVSS:3\.1/(.*?)\)", sev_text) + if cvss_match: + vector = cvss_match.group(1) + score_match = re.match(r"([\d\.]+)", sev_text) + score = score_match.group(1) if score_match else None + + severities.append( + VulnerabilitySeverity( + system=CVSSV31, value=score, scoring_elements=f"CVSS:3.1/{vector}" + ) + ) + + # Affected Versions + affected_header = soup.find(string=re.compile("Affected Version")) + affected_packages = [] + if affected_header: + header_elem = affected_header.parent + if header_elem.name.startswith("h"): + next_elem = header_elem.find_next_sibling() + if next_elem: + if next_elem.name == "ul": + items = next_elem.find_all("li") + for item in items: + pkg = self.parse_version_text(item.get_text(strip=True)) + if pkg: + affected_packages.append(pkg) + else: + lines = next_elem.get_text("\n").split("\n") + for line in lines: + pkg = self.parse_version_text(line.strip()) + if pkg: + affected_packages.append(pkg) + + # Clean URL + # Example: https://liferay.dev/portal/security/known-vulnerabilities/-/asset_publisher/jekt/content/cve-2025-1234?_com_liferay_asset_publisher_web_portlet_AssetPublisherPortlet_INSTANCE_jekt_redirect=... + if "?" in vuln_url: + vuln_url = vuln_url.split("?")[0] + + yield AdvisoryData( + advisory_id=cve_id, + aliases=[], + summary=description, + affected_packages=affected_packages, + references_v2=[ReferenceV2(url=vuln_url)], + url=vuln_url, + severities=severities, + ) + + def parse_version_text(self, text): + if not text: + return None + + if "DXP" in text: + name = "liferay-dxp" + elif "Portal" in text: + name = "liferay-portal" + else: + name = "liferay-portal" + + purl = PackageURL(type="generic", name=name) + + version_match = re.search(r"(\d+\.\d+(\.\d+)?)", text) + if version_match: + version = version_match.group(1) + try: + affected_range = MavenVersionRange.from_versions([version]) + return AffectedPackageV2(package=purl, affected_version_range=affected_range) + except Exception as e: + logger.error(f"Failed to parse version {version}: {e}") + return None + + return None diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index 413b260b6..5b8e8048d 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -194,12 +194,16 @@ def insert_advisory_v2( impact = ImpactedPackage.objects.create( advisory=advisory_obj, base_purl=str(affected_pkg.package), - affecting_vers=str(affected_pkg.affected_version_range) - if affected_pkg.affected_version_range - else None, - fixed_vers=str(affected_pkg.fixed_version_range) - if affected_pkg.fixed_version_range - else None, + affecting_vers=( + str(affected_pkg.affected_version_range) + if affected_pkg.affected_version_range + else None + ), + fixed_vers=( + str(affected_pkg.fixed_version_range) + if affected_pkg.fixed_version_range + else None + ), ) package_affected_purls, package_fixed_purls = get_exact_purls_v2( affected_package=affected_pkg, diff --git a/vulnerabilities/rpm_utils.py b/vulnerabilities/rpm_utils.py index 84d440c9f..a37bcf2c7 100644 --- a/vulnerabilities/rpm_utils.py +++ b/vulnerabilities/rpm_utils.py @@ -17,6 +17,7 @@ # This code has been vendored from scancode. + # https://github.com/nexB/scancode-toolkit/blob/16ae20a343c5332114edac34c7b6fcf2fb6bca74/src/packagedcode/rpm.py#L91 class EVR(namedtuple("EVR", "epoch version release")): """ diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_apache_httpd_importer_pipeline_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_apache_httpd_importer_pipeline_v2.py index 94454c473..85d734ac7 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_apache_httpd_importer_pipeline_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_apache_httpd_importer_pipeline_v2.py @@ -50,6 +50,7 @@ def test_fetch_links_filters_and_resolves(monkeypatch): """ base_url = "https://example.com/base/" + # Monkeypatch HTTP GET for HTML def fake_get(url): assert url == base_url @@ -126,6 +127,7 @@ def test_collect_advisories_and_to_advisory(monkeypatch, pipeline): "affects": {"vendor": {"vendor_data": []}}, "timeline": [], } + # Monkeypatch requests.get to return JSON def fake_get(u): if u == "u1": diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index 9ed647099..31f2b7774 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -75,9 +75,9 @@ def cleaned_response(response): reference["scores"] = sorted( reference["scores"], key=lambda x: (x["value"], x["scoring_system"]) ) - package_data["resolved_vulnerabilities"][index]["references"][index2][ - "scores" - ] = reference["scores"] + package_data["resolved_vulnerabilities"][index]["references"][index2]["scores"] = ( + reference["scores"] + ) cleaned_response.append(package_data) diff --git a/vulnerabilities/tests/test_liferay.py b/vulnerabilities/tests/test_liferay.py new file mode 100644 index 000000000..0bc01083c --- /dev/null +++ b/vulnerabilities/tests/test_liferay.py @@ -0,0 +1,83 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os +from unittest import TestCase +from unittest.mock import MagicMock +from unittest.mock import patch + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.pipelines.v2_importers.liferay_importer import LiferayImporterPipeline + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data") + + +class TestLiferayImporterPipeline(TestCase): + @patch("vulnerabilities.pipelines.v2_importers.liferay_importer.requests.get") + def test_collect_advisories(self, mock_get): + importer = LiferayImporterPipeline() + + # Mock responses + mock_main_page = MagicMock() + mock_main_page.content = b""" + + + Liferay Portal 7.4 + + + """ + + mock_release_page = MagicMock() + mock_release_page.content = b""" + + + CVE-2023-1234 Title + + + """ + + mock_vuln_page = MagicMock() + mock_vuln_page.content = b""" + + +

CVE-2023-1234 Title

+

Description

+

This is a test vulnerability description.

+

Severity

+

4.8 (CVSS:3.1/AV:N/AC:L/PR:H/UI:R/S:C/C:L/I:L/A:N)

+

Affected Version(s)

+ + + + """ + + # Configure side_effect to return different mocks based on URL + def side_effect(url): + if url == "https://liferay.dev/portal/security/known-vulnerabilities": + return mock_main_page + elif "categories" in url: + return mock_release_page + elif "asset_publisher" in url: + return mock_vuln_page + return MagicMock() + + mock_get.side_effect = side_effect + + advisories = list(importer.collect_advisories()) + + self.assertEqual(len(advisories), 1) + advisory = advisories[0] + self.assertIsInstance(advisory, AdvisoryData) + self.assertEqual(advisory.aliases, []) + self.assertEqual(advisory.summary, "This is a test vulnerability description.") + self.assertEqual(len(advisory.affected_packages), 1) + self.assertEqual(advisory.affected_packages[0].package.name, "liferay-portal") diff --git a/vulnerablecode/__init__.py b/vulnerablecode/__init__.py index 66e48f76a..13c70b495 100644 --- a/vulnerablecode/__init__.py +++ b/vulnerablecode/__init__.py @@ -45,7 +45,7 @@ def get_git_commit_from_version_file(): if not commit_line.startswith("commit=") or commit_line.startswith("commit=$Format"): return return commit_line.replace("commit=", "") - except (UnicodeDecodeError): + except UnicodeDecodeError: return @@ -61,7 +61,7 @@ def get_git_tag_from_version_file(): if "tag:" in ref_line: if vcio_tag := ref_line.split("tag:")[-1].strip(): return vcio_tag - except (UnicodeDecodeError): + except UnicodeDecodeError: return