Add caching for nvd.db

.cqfdrc

@@ -18,4 +18,8 @@ command="make -C tests test_frontend"
 
 [test_ci]
 command="make -C tests test_ci"
-docker_run_args="-v /run/docker.sock:/run/docker.sock"
+docker_run_args="-v /run/docker.sock:/run/docker.sock"
+
+[build_nvd]
+command="python3 -m src.bin.nvd_db_builder"
+docker_run_args="-v .vulnscout/cache:/cache/vulnscout:Z -e NVD_VERBOSE_LOGGING=true"

.gitignore

@@ -88,5 +88,6 @@ venv.bak/
 .vulnscout/example/output/
 .vulnscout/example-spdx3/output/
 .vulnscout/cache/
+!.vulnscout/cache/nvd.db.*.xz
 frontend/package-lock.json
 .vulnscout-npm.pid

.vulnscout/example-spdx3/docker-example-spdx3.yml

@@ -20,6 +20,8 @@ services:
       - FLASK_RUN_HOST=0.0.0.0
       - IGNORE_PARSING_ERRORS=false
       - GENERATE_DOCUMENTS=summary.adoc, time_estimates.csv
+      - NVD_VERBOSE_LOGGING=true
+      - NVD_LOGFILE=/cache/vulnscout/nvd.log
       # - NVD_API_KEY=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
       # - DEBUG_SKIP_SCAN=false # Enable to skip scan and re-use last scan result instead
       # - PRODUCT_NAME=""

.vulnscout/example/docker-example.yml

@@ -21,6 +21,8 @@ services:
       - FLASK_RUN_HOST=0.0.0.0
       - IGNORE_PARSING_ERRORS=false
       - GENERATE_DOCUMENTS=summary.adoc, time_estimates.csv
+      - NVD_VERBOSE_LOGGING=true
+      - NVD_LOGFILE=/cache/vulnscout/nvd.log
       # - NVD_API_KEY=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
       # - DEBUG_SKIP_SCAN=false # Enable to skip scan and re-use last scan result instead
       # - PRODUCT_NAME=""

ci/release_tag.sh

@@ -33,6 +33,27 @@ sed -Ei "3s/^v[0-9]+(\.[0-9]+){0,2}/v${semversion}/" WRITING_CI_CONDITIONS.adoc
 sed -i "s/LABEL org.opencontainers.image.version=\".*\"/LABEL org.opencontainers.image.version=\"${version}\"/i" Dockerfile
 sed -i "s/^VULNSCOUT_VERSION=\".*\"$/VULNSCOUT_VERSION=\"${version}\"/i" bin/vulnscout.sh
 
+# Check if nvd.db exists and compress it
+find .vulnscout/cache -maxdepth 1 -type f -name "nvd.db.*.xz" -exec rm -f {} +
+mkdir -p .vulnscout/cache
+nvd_db_path=".vulnscout/cache/nvd.db"
+cqfd init
+cqfd -b build_nvd
+if [ -f "$nvd_db_path" ]; then
+    echo "Found nvd.db, compressing with xz..."
+
+    # xz compresses file in place, so we need to copy it then compress
+    cp "$nvd_db_path" "${nvd_db_path}.tmp"
+    xz "${nvd_db_path}.tmp"
+    mv "${nvd_db_path}.tmp.xz" "${nvd_db_path}.${version}.xz"
+    echo "nvd.db compressed to ${nvd_db_path}.${version}.xz"
+
+    # Add the compressed file to git
+    git add -f "${nvd_db_path}.${version}.xz"
+else
+    echo "nvd.db not found at $nvd_db_path, skipping compression"
+fi
+
 # Commit the changes
 git add frontend/package.json
 git add README.adoc

src/bin/nvd_db_builder.py

@@ -4,26 +4,69 @@
 # SPDX-License-Identifier: GPL-3.0-only
 
 from ..controllers.nvd_db import NVD_DB
+from ..helpers.nvd_logging import setup_logging, log_and_print
 import os
+import glob
+import lzma
+import shutil
+
+
+def decompress_nvd_db(nvd_db_path, verbose_logging=True):
+    db_dir, db_name = os.path.dirname(nvd_db_path), os.path.basename(nvd_db_path)
+    matches = sorted(glob.glob(f"{db_dir}/{db_name}.*.xz"), key=os.path.getmtime, reverse=True)
+    if not matches:
+        log_and_print("No compressed database files found", verbose_logging)
+        return
+    src = matches[0]
+    log_and_print(f"Decompressing database from {src}", verbose_logging)
+    try:
+        with lzma.open(src, "rb") as f_in, open(nvd_db_path, "wb") as f_out:
+            shutil.copyfileobj(f_in, f_out)
+        os.remove(src)
+        log_and_print(f"Successfully decompressed database to {nvd_db_path}", verbose_logging)
+    except Exception as e:
+        log_and_print(f"Error decompressing database: {e}", verbose_logging, force_print=True)
+        if os.path.exists(nvd_db_path):
+            os.remove(nvd_db_path)
+        raise
 
 
 def fetch_db_updates():
+    verbose_logging = setup_logging()
     nvd_db_path = os.getenv("NVD_DB_PATH", "/cache/vulnscout/nvd.db")
+    log_and_print("Starting NVD database update", verbose_logging)
+    if not verbose_logging:
+        log_and_print("NVD DB is syncing, it may take a few minutes", verbose_logging=False, force_print=True)
+    decompress_nvd_db(nvd_db_path, verbose_logging)
+    log_and_print("Initializing NVD database connection", verbose_logging)
     nvd_db = NVD_DB(nvd_db_path)
     nvd_api_key = os.getenv("NVD_API_KEY")
     if not nvd_api_key:
-        print("NVD API key not found, this may slow down db update. See NVD_API_KEY configuration")
+        log_and_print(
+            "NVD API key not found, this may slow down db update. See NVD_API_KEY configuration",
+            verbose_logging
+        )
     else:
+        log_and_print("NVD API key found, using authenticated requests", verbose_logging)
         nvd_db.nvd_api_key = nvd_api_key
     nvd_db.set_writing_flag(True)
     for step, total in nvd_db.build_initial_db():
-        print(f"NVD update: {step} / {total} [{round((step / total) * 100)}%]", flush=True)
+        percent = round((step / total) * 100) if total else 100
+        message = f"NVD update: {step} / {total} [{percent}%]"
+        log_and_print(message, verbose_logging)
     for txt in nvd_db.update_db():
-        print(f"NVD update: {txt}", flush=True)
+        message = f"NVD update: {txt}"
+        log_and_print(message, verbose_logging)
     nvd_db.in_sync = True
     nvd_db.set_writing_flag(False)
+    log_and_print("NVD DB sync completed successfully", verbose_logging, force_print=True)
 
 
 if __name__ == "__main__":
-    fetch_db_updates()
-    print("DB is now synced")
+    try:
+        fetch_db_updates()
+    except Exception as e:
+        verbose_logging = os.getenv("NVD_VERBOSE_LOGGING", "false").lower() == "true"
+        error_msg = f"Error during NVD database update: {e}"
+        log_and_print(error_msg, verbose_logging, force_print=True)
+        raise

src/controllers/nvd_db.py

@@ -9,6 +9,7 @@
 import urllib.parse
 from datetime import datetime, timezone, timedelta
 from ..helpers.fixs_scrapper import FixsScrapper
+from ..helpers.nvd_logging import setup_logging, log_and_print
 from typing import Optional, Generator, Tuple
 import time
 import sys
@@ -25,6 +26,7 @@ class NVD_DB:
     """
 
     def __init__(self, db_path: str, nvd_api_key: Optional[str] = None):
+        self.verbose_logging = setup_logging()
         self.conn = sqlite3.connect(db_path)
         self.cursor = self.conn.cursor()
 
@@ -60,7 +62,7 @@ def _load_metadata(self):
             self.conn.commit()
         elif res[0] != DB_MODEL_VERSION:
             # incompatible database version
-            print("DB version mismatch, please update or reset the DB")
+            log_and_print("DB version mismatch, please update or reset the DB", self.verbose_logging, force_print=True)
             raise Exception(f"DB version mismatch, expected {DB_MODEL_VERSION}, got {res[0]}")
         else:
             # database was existing before and with correct version, restore metadata
@@ -78,11 +80,9 @@ def _load_metadata(self):
             except Exception:
                 self.last_index = 0
 
-            print(
-                "Restored DB from cache, last_index =",
-                self.last_index,
-                ", last_modified =",
-                self.last_modified
+            log_and_print(
+                f"Restored DB from cache, last_index = {self.last_index}, last_modified = {self.last_modified}",
+                self.verbose_logging
             )
 
     def set_writing_flag(self, flag: bool):
@@ -121,16 +121,19 @@ def _call_nvd_api(self, params: dict = {}) -> Tuple[int, dict]:
             try:
                 resp_json = json.loads(resp.read().decode())
             except json.decoder.JSONDecodeError:
-                print("NVD API responded with invalid JSON. Adding an free NVD API key "
-                      + f"can help to avoid this error. (status: {resp_status})", flush=True)
+                log_and_print(
+                    f"NVD API responded with invalid JSON. Adding an free NVD API key can help to avoid this error. "
+                    f"(status: {resp_status})",
+                    self.verbose_logging
+                )
                 resp_json = {}
 
             self.client.close()
 
             return resp_status, resp_json
 
         except Exception as e:
-            print(f"Error calling NVD API: {e}", flush=True)
+            log_and_print(f"Error calling NVD API: {e}", self.verbose_logging, force_print=True)
             self.client.close()
             raise e
 
@@ -236,7 +239,7 @@ def write_result_to_db(self, data: dict) -> bool:
             self.conn.commit()
             return True
         except Exception as e:
-            print(f"Error writing to DB: {e}")
+            log_and_print(f"Error writing to DB: {e}", self.verbose_logging, force_print=True)
             raise e
         return False
 

src/helpers/nvd_logging.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 Savoir-faire Linux, Inc.
+# SPDX-License-Identifier: GPL-3.0-only
+
+import os
+import logging
+
+
+def setup_logging():
+    """Setup logging configuration for NVD operations."""
+    log_file = os.getenv("NVD_LOGFILE", "/cache/vulnscout/nvd.log")
+    verbose_logging = os.getenv("NVD_VERBOSE_LOGGING", "false").lower() == "true"
+
+    log_dir = os.path.dirname(log_file)
+    if not os.path.exists(log_dir):
+        print("[Patch-Finder] Warning: Log directory does not exist, falling back to console logging.")
+        handlers = [logging.StreamHandler()]
+    else:
+        handlers = [logging.FileHandler(log_file)]
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s - %(levelname)s - %(message)s",
+        handlers=handlers
+    )
+
+    return verbose_logging
+
+
+def log_and_print(message, verbose_logging=True, force_print=False):
+    """Log a message and optionally print it to console."""
+    logging.info(message)
+    if verbose_logging or force_print:
+        print(f"[Patch-Finder] {message}", flush=True)