diff --git a/.cqfdrc b/.cqfdrc index 4a0b9b5f..2a95c87b 100644 --- a/.cqfdrc +++ b/.cqfdrc @@ -18,4 +18,8 @@ command="make -C tests test_frontend" [test_ci] command="make -C tests test_ci" -docker_run_args="-v /run/docker.sock:/run/docker.sock" \ No newline at end of file +docker_run_args="-v /run/docker.sock:/run/docker.sock" + +[build_nvd] +command="python3 -m src.bin.nvd_db_builder" +docker_run_args="-v .vulnscout/cache:/cache/vulnscout:Z -e NVD_VERBOSE_LOGGING=true" \ No newline at end of file diff --git a/.gitignore b/.gitignore index f2041c09..f87664ce 100644 --- a/.gitignore +++ b/.gitignore @@ -88,5 +88,6 @@ venv.bak/ .vulnscout/example/output/ .vulnscout/example-spdx3/output/ .vulnscout/cache/ +!.vulnscout/cache/nvd.db.*.xz frontend/package-lock.json .vulnscout-npm.pid diff --git a/.vulnscout/example-spdx3/docker-example-spdx3.yml b/.vulnscout/example-spdx3/docker-example-spdx3.yml index 0774b4bc..74b933b5 100644 --- a/.vulnscout/example-spdx3/docker-example-spdx3.yml +++ b/.vulnscout/example-spdx3/docker-example-spdx3.yml @@ -20,6 +20,8 @@ services: - FLASK_RUN_HOST=0.0.0.0 - IGNORE_PARSING_ERRORS=false - GENERATE_DOCUMENTS=summary.adoc, time_estimates.csv + - NVD_VERBOSE_LOGGING=true + - NVD_LOGFILE=/cache/vulnscout/nvd.log # - NVD_API_KEY=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx # - DEBUG_SKIP_SCAN=false # Enable to skip scan and re-use last scan result instead # - PRODUCT_NAME="" diff --git a/.vulnscout/example/docker-example.yml b/.vulnscout/example/docker-example.yml index 400f94b7..7173b38c 100644 --- a/.vulnscout/example/docker-example.yml +++ b/.vulnscout/example/docker-example.yml @@ -21,6 +21,8 @@ services: - FLASK_RUN_HOST=0.0.0.0 - IGNORE_PARSING_ERRORS=false - GENERATE_DOCUMENTS=summary.adoc, time_estimates.csv + - NVD_VERBOSE_LOGGING=true + - NVD_LOGFILE=/cache/vulnscout/nvd.log # - NVD_API_KEY=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx # - DEBUG_SKIP_SCAN=false # Enable to skip scan and re-use last scan result instead # - PRODUCT_NAME="" diff --git a/ci/release_tag.sh b/ci/release_tag.sh index 7df8bbdc..5f5157a2 100755 --- a/ci/release_tag.sh +++ b/ci/release_tag.sh @@ -33,6 +33,27 @@ sed -Ei "3s/^v[0-9]+(\.[0-9]+){0,2}/v${semversion}/" WRITING_CI_CONDITIONS.adoc sed -i "s/LABEL org.opencontainers.image.version=\".*\"/LABEL org.opencontainers.image.version=\"${version}\"/i" Dockerfile sed -i "s/^VULNSCOUT_VERSION=\".*\"$/VULNSCOUT_VERSION=\"${version}\"/i" bin/vulnscout.sh +# Check if nvd.db exists and compress it +find .vulnscout/cache -maxdepth 1 -type f -name "nvd.db.*.xz" -exec rm -f {} + +mkdir -p .vulnscout/cache +nvd_db_path=".vulnscout/cache/nvd.db" +cqfd init +cqfd -b build_nvd +if [ -f "$nvd_db_path" ]; then + echo "Found nvd.db, compressing with xz..." + + # xz compresses file in place, so we need to copy it then compress + cp "$nvd_db_path" "${nvd_db_path}.tmp" + xz "${nvd_db_path}.tmp" + mv "${nvd_db_path}.tmp.xz" "${nvd_db_path}.${version}.xz" + echo "nvd.db compressed to ${nvd_db_path}.${version}.xz" + + # Add the compressed file to git + git add -f "${nvd_db_path}.${version}.xz" +else + echo "nvd.db not found at $nvd_db_path, skipping compression" +fi + # Commit the changes git add frontend/package.json git add README.adoc diff --git a/src/bin/nvd_db_builder.py b/src/bin/nvd_db_builder.py index 67d6913b..007f6f38 100644 --- a/src/bin/nvd_db_builder.py +++ b/src/bin/nvd_db_builder.py @@ -4,26 +4,69 @@ # SPDX-License-Identifier: GPL-3.0-only from ..controllers.nvd_db import NVD_DB +from ..helpers.nvd_logging import setup_logging, log_and_print import os +import glob +import lzma +import shutil + + +def decompress_nvd_db(nvd_db_path, verbose_logging=True): + db_dir, db_name = os.path.dirname(nvd_db_path), os.path.basename(nvd_db_path) + matches = sorted(glob.glob(f"{db_dir}/{db_name}.*.xz"), key=os.path.getmtime, reverse=True) + if not matches: + log_and_print("No compressed database files found", verbose_logging) + return + src = matches[0] + log_and_print(f"Decompressing database from {src}", verbose_logging) + try: + with lzma.open(src, "rb") as f_in, open(nvd_db_path, "wb") as f_out: + shutil.copyfileobj(f_in, f_out) + os.remove(src) + log_and_print(f"Successfully decompressed database to {nvd_db_path}", verbose_logging) + except Exception as e: + log_and_print(f"Error decompressing database: {e}", verbose_logging, force_print=True) + if os.path.exists(nvd_db_path): + os.remove(nvd_db_path) + raise def fetch_db_updates(): + verbose_logging = setup_logging() nvd_db_path = os.getenv("NVD_DB_PATH", "/cache/vulnscout/nvd.db") + log_and_print("Starting NVD database update", verbose_logging) + if not verbose_logging: + log_and_print("NVD DB is syncing, it may take a few minutes", verbose_logging=False, force_print=True) + decompress_nvd_db(nvd_db_path, verbose_logging) + log_and_print("Initializing NVD database connection", verbose_logging) nvd_db = NVD_DB(nvd_db_path) nvd_api_key = os.getenv("NVD_API_KEY") if not nvd_api_key: - print("NVD API key not found, this may slow down db update. See NVD_API_KEY configuration") + log_and_print( + "NVD API key not found, this may slow down db update. See NVD_API_KEY configuration", + verbose_logging + ) else: + log_and_print("NVD API key found, using authenticated requests", verbose_logging) nvd_db.nvd_api_key = nvd_api_key nvd_db.set_writing_flag(True) for step, total in nvd_db.build_initial_db(): - print(f"NVD update: {step} / {total} [{round((step / total) * 100)}%]", flush=True) + percent = round((step / total) * 100) if total else 100 + message = f"NVD update: {step} / {total} [{percent}%]" + log_and_print(message, verbose_logging) for txt in nvd_db.update_db(): - print(f"NVD update: {txt}", flush=True) + message = f"NVD update: {txt}" + log_and_print(message, verbose_logging) nvd_db.in_sync = True nvd_db.set_writing_flag(False) + log_and_print("NVD DB sync completed successfully", verbose_logging, force_print=True) if __name__ == "__main__": - fetch_db_updates() - print("DB is now synced") + try: + fetch_db_updates() + except Exception as e: + verbose_logging = os.getenv("NVD_VERBOSE_LOGGING", "false").lower() == "true" + error_msg = f"Error during NVD database update: {e}" + log_and_print(error_msg, verbose_logging, force_print=True) + raise diff --git a/src/controllers/nvd_db.py b/src/controllers/nvd_db.py index 0fd1f239..4962c1e8 100644 --- a/src/controllers/nvd_db.py +++ b/src/controllers/nvd_db.py @@ -9,6 +9,7 @@ import urllib.parse from datetime import datetime, timezone, timedelta from ..helpers.fixs_scrapper import FixsScrapper +from ..helpers.nvd_logging import setup_logging, log_and_print from typing import Optional, Generator, Tuple import time import sys @@ -25,6 +26,7 @@ class NVD_DB: """ def __init__(self, db_path: str, nvd_api_key: Optional[str] = None): + self.verbose_logging = setup_logging() self.conn = sqlite3.connect(db_path) self.cursor = self.conn.cursor() @@ -60,7 +62,7 @@ def _load_metadata(self): self.conn.commit() elif res[0] != DB_MODEL_VERSION: # incompatible database version - print("DB version mismatch, please update or reset the DB") + log_and_print("DB version mismatch, please update or reset the DB", self.verbose_logging, force_print=True) raise Exception(f"DB version mismatch, expected {DB_MODEL_VERSION}, got {res[0]}") else: # database was existing before and with correct version, restore metadata @@ -78,11 +80,9 @@ def _load_metadata(self): except Exception: self.last_index = 0 - print( - "Restored DB from cache, last_index =", - self.last_index, - ", last_modified =", - self.last_modified + log_and_print( + f"Restored DB from cache, last_index = {self.last_index}, last_modified = {self.last_modified}", + self.verbose_logging ) def set_writing_flag(self, flag: bool): @@ -121,8 +121,11 @@ def _call_nvd_api(self, params: dict = {}) -> Tuple[int, dict]: try: resp_json = json.loads(resp.read().decode()) except json.decoder.JSONDecodeError: - print("NVD API responded with invalid JSON. Adding an free NVD API key " - + f"can help to avoid this error. (status: {resp_status})", flush=True) + log_and_print( + f"NVD API responded with invalid JSON. Adding an free NVD API key can help to avoid this error. " + f"(status: {resp_status})", + self.verbose_logging + ) resp_json = {} self.client.close() @@ -130,7 +133,7 @@ def _call_nvd_api(self, params: dict = {}) -> Tuple[int, dict]: return resp_status, resp_json except Exception as e: - print(f"Error calling NVD API: {e}", flush=True) + log_and_print(f"Error calling NVD API: {e}", self.verbose_logging, force_print=True) self.client.close() raise e @@ -236,7 +239,7 @@ def write_result_to_db(self, data: dict) -> bool: self.conn.commit() return True except Exception as e: - print(f"Error writing to DB: {e}") + log_and_print(f"Error writing to DB: {e}", self.verbose_logging, force_print=True) raise e return False diff --git a/src/helpers/nvd_logging.py b/src/helpers/nvd_logging.py new file mode 100644 index 00000000..a89ab4dc --- /dev/null +++ b/src/helpers/nvd_logging.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2024 Savoir-faire Linux, Inc. +# SPDX-License-Identifier: GPL-3.0-only + +import os +import logging + + +def setup_logging(): + """Setup logging configuration for NVD operations.""" + log_file = os.getenv("NVD_LOGFILE", "/cache/vulnscout/nvd.log") + verbose_logging = os.getenv("NVD_VERBOSE_LOGGING", "false").lower() == "true" + + log_dir = os.path.dirname(log_file) + if not os.path.exists(log_dir): + print("[Patch-Finder] Warning: Log directory does not exist, falling back to console logging.") + handlers = [logging.StreamHandler()] + else: + handlers = [logging.FileHandler(log_file)] + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + handlers=handlers + ) + + return verbose_logging + + +def log_and_print(message, verbose_logging=True, force_print=False): + """Log a message and optionally print it to console.""" + logging.info(message) + if verbose_logging or force_print: + print(f"[Patch-Finder] {message}", flush=True)