Add Liferay Importer (Fixes #1410)

aboutcode/federated/__init__.py

@@ -1300,7 +1300,6 @@ def from_hashids(
         purl_type: str,
         hashids: list[str],
     ) -> "DataRepository":
-
         """
         Return a new DataRepository to store ``data_kind`` of ``purl_type`` for
         a list of ``hashids``.

vulnerabilities/importer.py

@@ -456,7 +456,7 @@ def clean_summary(self, summary):
         # https://github.com/cms-dev/cms/issues/888#issuecomment-516977572
         summary = summary.strip()
         if summary:
-            summary = summary.replace("\x00", "\uFFFD")
+            summary = summary.replace("\x00", "\ufffd")
         return summary
 
     def to_dict(self):
@@ -506,9 +506,9 @@ def from_dict(cls, advisory_data):
                 affected_package_cls.from_dict(pkg) for pkg in affected_packages if pkg is not None
             ],
             "references": [Reference.from_dict(ref) for ref in advisory_data["references"]],
-            "date_published": datetime.datetime.fromisoformat(date_published)
-            if date_published
-            else None,
+            "date_published": (
+                datetime.datetime.fromisoformat(date_published) if date_published else None
+            ),
             "weaknesses": advisory_data["weaknesses"],
             "url": advisory_data.get("url") or None,
         }
@@ -548,7 +548,7 @@ def clean_summary(self, summary):
         # https://github.com/cms-dev/cms/issues/888#issuecomment-516977572
         summary = summary.strip()
         if summary:
-            summary = summary.replace("\x00", "\uFFFD")
+            summary = summary.replace("\x00", "\ufffd")
         return summary
 
     def to_dict(self):
@@ -574,9 +574,9 @@ def from_dict(cls, advisory_data):
                 if pkg is not None
             ],
             "references": [Reference.from_dict(ref) for ref in advisory_data["references"]],
-            "date_published": datetime.datetime.fromisoformat(date_published)
-            if date_published
-            else None,
+            "date_published": (
+                datetime.datetime.fromisoformat(date_published) if date_published else None
+            ),
             "weaknesses": advisory_data["weaknesses"],
             "url": advisory_data.get("url") or None,
         }

vulnerabilities/importers/__init__.py

@@ -50,6 +50,7 @@
 from vulnerabilities.pipelines.v2_importers import github_osv_importer as github_osv_importer_v2
 from vulnerabilities.pipelines.v2_importers import gitlab_importer as gitlab_importer_v2
 from vulnerabilities.pipelines.v2_importers import istio_importer as istio_importer_v2
+from vulnerabilities.pipelines.v2_importers import liferay_importer
 from vulnerabilities.pipelines.v2_importers import mozilla_importer as mozilla_importer_v2
 from vulnerabilities.pipelines.v2_importers import npm_importer as npm_importer_v2
 from vulnerabilities.pipelines.v2_importers import nvd_importer as nvd_importer_v2
@@ -115,5 +116,6 @@
         ubuntu_usn.UbuntuUSNImporter,
         fireeye.FireyeImporter,
         oss_fuzz.OSSFuzzImporter,
+        liferay_importer.LiferayImporterPipeline,
     ]
 )

vulnerabilities/importers/gentoo.py

@@ -76,9 +76,11 @@ def process_file(self, file):
                 summary=summary,
                 references=vuln_references,
                 affected_packages=affected_packages,
-                url=f"https://security.gentoo.org/glsa/{id}"
-                if id
-                else "https://security.gentoo.org/glsa",
+                url=(
+                    f"https://security.gentoo.org/glsa/{id}"
+                    if id
+                    else "https://security.gentoo.org/glsa"
+                ),
             )
 
     @staticmethod

vulnerabilities/importers/mattermost.py

@@ -103,13 +103,15 @@ def to_advisories(self, data):
                 Reference(
                     reference_id=ref_col.text,
                     url=SECURITY_UPDATES_URL,
-                    severities=[
-                        VulnerabilitySeverity(
-                            system=severity_systems.CVSS31_QUALITY, value=severity_col.text
-                        )
-                    ]
-                    if severity_col.text.lower() != "na"
-                    else [],
+                    severities=(
+                        [
+                            VulnerabilitySeverity(
+                                system=severity_systems.CVSS31_QUALITY, value=severity_col.text
+                            )
+                        ]
+                        if severity_col.text.lower() != "na"
+                        else []
+                    ),
                 )
             ]
 

vulnerabilities/importers/postgresql.py

@@ -74,11 +74,11 @@ def to_advisories(data):
                             type="generic",
                             qualifiers=pkg_qualifiers,
                         ),
-                        affected_version_range=GenericVersionRange.from_versions(
-                            affected_version_list
-                        )
-                        if affected_version_list
-                        else None,
+                        affected_version_range=(
+                            GenericVersionRange.from_versions(affected_version_list)
+                            if affected_version_list
+                            else None
+                        ),
                         fixed_version=GenericVersion(fixed_version) if fixed_version else None,
                     )
                 )

vulnerabilities/importers/redhat.py

@@ -156,7 +156,9 @@ def to_advisory(advisory_data):
         affected_packages=affected_packages,
         references=references,
         weaknesses=cwe_list,
-        url=resource_url
-        if resource_url
-        else "https://access.redhat.com/hydra/rest/securitydata/cve.json",
+        url=(
+            resource_url
+            if resource_url
+            else "https://access.redhat.com/hydra/rest/securitydata/cve.json"
+        ),
     )

vulnerabilities/models.py

@@ -1128,9 +1128,9 @@ def get_affecting_vulnerabilities(self):
                     next_fixed_package_vulns = list(fixed_by_pkg.affected_by)
 
                     fixed_by_package_details["fixed_by_purl"] = fixed_by_purl
-                    fixed_by_package_details[
-                        "fixed_by_purl_vulnerabilities"
-                    ] = next_fixed_package_vulns
+                    fixed_by_package_details["fixed_by_purl_vulnerabilities"] = (
+                        next_fixed_package_vulns
+                    )
                     fixed_by_pkgs.append(fixed_by_package_details)
 
                     vuln_details["fixed_by_package_details"] = fixed_by_pkgs

vulnerabilities/pipelines/enhance_with_kev.py

@@ -88,9 +88,9 @@ def add_vulnerability_exploit(kev_vul, logger):
             "required_action": kev_vul["requiredAction"],
             "due_date": kev_vul["dueDate"],
             "notes": kev_vul["notes"],
-            "known_ransomware_campaign_use": True
-            if kev_vul["knownRansomwareCampaignUse"] == "Known"
-            else False,
+            "known_ransomware_campaign_use": (
+                True if kev_vul["knownRansomwareCampaignUse"] == "Known" else False
+            ),
         },
     )
     return 1

vulnerabilities/pipelines/v2_importers/liferay_importer.py

@@ -0,0 +1,204 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+import re
+from typing import Iterable
+from urllib.parse import urljoin
+
+import requests
+from bs4 import BeautifulSoup
+from packageurl import PackageURL
+from univers.version_range import MavenVersionRange
+
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AffectedPackageV2
+from vulnerabilities.importer import ReferenceV2
+from vulnerabilities.importer import VulnerabilitySeverity
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.severity_systems import CVSSV31
+
+logger = logging.getLogger(__name__)
+
+
+class LiferayImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
+    spdx_license_expression = "Apache-2.0"
+    license_url = "https://www.apache.org/licenses/LICENSE-2.0"
+    pipeline_id = "liferay_importer_v2"
+    importer_name = "Liferay Importer"
+
+    release_links = []
+
+    @classmethod
+    def steps(cls):
+        return (cls.collect_and_store_advisories,)
+
+    def advisories_count(self) -> int:
+        if not self.release_links:
+            self.release_links = self.fetch_release_links()
+        return len(self.release_links)
+
+    def collect_advisories(self) -> Iterable[AdvisoryData]:
+        if not self.release_links:
+            self.release_links = self.fetch_release_links()
+
+        for release_url in self.release_links:
+            yield from self.process_release_page(release_url)
+
+    def fetch_release_links(self):
+        base_url = "https://liferay.dev/portal/security/known-vulnerabilities"
+        try:
+            main_page = requests.get(base_url)
+            main_page.raise_for_status()
+        except requests.RequestException as e:
+            logger.error(f"Failed to fetch Liferay main page: {e}")
+            return []
+
+        soup = BeautifulSoup(main_page.content, "lxml")
+        links = set()
+        for a in soup.find_all("a", href=True):
+            href = a["href"]
+            if "/categories/" in href and "known-vulnerabilities" in href:
+                links.add(urljoin(base_url, href))
+        return list(links)
+
+    def process_release_page(self, release_url):
+        try:
+            page = requests.get(release_url)
+            page.raise_for_status()
+        except requests.RequestException as e:
+            logger.error(f"Failed to fetch release page {release_url}: {e}")
+            return
+
+        soup = BeautifulSoup(page.content, "lxml")
+
+        # 3. Find Vulnerability Links
+        vuln_links = set()
+        for a in soup.find_all("a", href=True):
+            href = a["href"]
+            if "/asset_publisher/" in href and "cve-" in href.lower():
+                vuln_links.add(urljoin(release_url, href))
+
+        for vuln_url in vuln_links:
+            yield from self.process_vulnerability_page(vuln_url)
+
+    def process_vulnerability_page(self, vuln_url):
+        try:
+            page = requests.get(vuln_url)
+            page.raise_for_status()
+        except requests.RequestException as e:
+            logger.error(f"Failed to fetch vulnerability page {vuln_url}: {e}")
+            return
+
+        soup = BeautifulSoup(page.content, "lxml")
+
+        # Extract Details
+        title = soup.find("h1")
+        title_text = title.get_text(strip=True) if title else ""
+
+        # CVE ID
+        cve_match = re.search(r"(CVE-\d{4}-\d{4,})", title_text)
+        if not cve_match:
+            cve_match = re.search(r"(CVE-\d{4}-\d{4,})", soup.get_text())
+
+        cve_id = cve_match.group(1) if cve_match else ""
+        if not cve_id:
+            return
+
+        # Description
+        description_header = soup.find(string=re.compile("Description"))
+        description = ""
+        if description_header:
+            header_elem = description_header.parent
+            if header_elem.name.startswith("h"):
+                desc_elem = header_elem.find_next_sibling()
+                if desc_elem:
+                    description = desc_elem.get_text(strip=True)
+
+        # Severity
+        severity_header = soup.find(string=re.compile("Severity"))
+        severities = []
+        if severity_header:
+            header_elem = severity_header.parent
+            if header_elem.name.startswith("h"):
+                sev_elem = header_elem.find_next_sibling()
+                if sev_elem:
+                    sev_text = sev_elem.get_text(strip=True)
+                    cvss_match = re.search(r"\(CVSS:3\.1/(.*?)\)", sev_text)
+                    if cvss_match:
+                        vector = cvss_match.group(1)
+                        score_match = re.match(r"([\d\.]+)", sev_text)
+                        score = score_match.group(1) if score_match else None
+
+                        severities.append(
+                            VulnerabilitySeverity(
+                                system=CVSSV31, value=score, scoring_elements=f"CVSS:3.1/{vector}"
+                            )
+                        )
+
+        # Affected Versions
+        affected_header = soup.find(string=re.compile("Affected Version"))
+        affected_packages = []
+        if affected_header:
+            header_elem = affected_header.parent
+            if header_elem.name.startswith("h"):
+                next_elem = header_elem.find_next_sibling()
+                if next_elem:
+                    if next_elem.name == "ul":
+                        items = next_elem.find_all("li")
+                        for item in items:
+                            pkg = self.parse_version_text(item.get_text(strip=True))
+                            if pkg:
+                                affected_packages.append(pkg)
+                    else:
+                        lines = next_elem.get_text("\n").split("\n")
+                        for line in lines:
+                            pkg = self.parse_version_text(line.strip())
+                            if pkg:
+                                affected_packages.append(pkg)
+
+        # Clean URL
+        # Example: https://liferay.dev/portal/security/known-vulnerabilities/-/asset_publisher/jekt/content/cve-2025-1234?_com_liferay_asset_publisher_web_portlet_AssetPublisherPortlet_INSTANCE_jekt_redirect=...
+        if "?" in vuln_url:
+            vuln_url = vuln_url.split("?")[0]
+
+        yield AdvisoryData(
+            advisory_id=cve_id,
+            aliases=[],
+            summary=description,
+            affected_packages=affected_packages,
+            references_v2=[ReferenceV2(url=vuln_url)],
+            url=vuln_url,
+            severities=severities,
+        )
+
+    def parse_version_text(self, text):
+        if not text:
+            return None
+
+        if "DXP" in text:
+            name = "liferay-dxp"
+        elif "Portal" in text:
+            name = "liferay-portal"
+        else:
+            name = "liferay-portal"
+
+        purl = PackageURL(type="generic", name=name)
+
+        version_match = re.search(r"(\d+\.\d+(\.\d+)?)", text)
+        if version_match:
+            version = version_match.group(1)
+            try:
+                affected_range = MavenVersionRange.from_versions([version])
+                return AffectedPackageV2(package=purl, affected_version_range=affected_range)
+            except Exception as e:
+                logger.error(f"Failed to parse version {version}: {e}")
+                return None
+
+        return None

vulnerabilities/pipes/advisory.py

@@ -194,12 +194,16 @@ def insert_advisory_v2(
             impact = ImpactedPackage.objects.create(
                 advisory=advisory_obj,
                 base_purl=str(affected_pkg.package),
-                affecting_vers=str(affected_pkg.affected_version_range)
-                if affected_pkg.affected_version_range
-                else None,
-                fixed_vers=str(affected_pkg.fixed_version_range)
-                if affected_pkg.fixed_version_range
-                else None,
+                affecting_vers=(
+                    str(affected_pkg.affected_version_range)
+                    if affected_pkg.affected_version_range
+                    else None
+                ),
+                fixed_vers=(
+                    str(affected_pkg.fixed_version_range)
+                    if affected_pkg.fixed_version_range
+                    else None
+                ),
             )
             package_affected_purls, package_fixed_purls = get_exact_purls_v2(
                 affected_package=affected_pkg,

vulnerabilities/rpm_utils.py

@@ -17,6 +17,7 @@
 
 # This code has been vendored from scancode.
 
+
 # https://github.com/nexB/scancode-toolkit/blob/16ae20a343c5332114edac34c7b6fcf2fb6bca74/src/packagedcode/rpm.py#L91
 class EVR(namedtuple("EVR", "epoch version release")):
     """