From 41ba3b872108fa33f12269d2b8d1e4eb0d247c8d Mon Sep 17 00:00:00 2001
From: emmanuelgjr <emmanuelgjr@gmail.com>
Date: Sat, 18 Jul 2026 16:28:38 -0400
Subject: [PATCH] fix(scanner): decode rg/git output as utf-8 with replacement

The benchmark surfaced a real cross-platform crash: subprocess.run(text=True) decodes rg/git output with the platform default encoding (cp1252 on Windows, strict utf-8 on Linux), so the CLI raised UnicodeDecodeError on any repo containing a file with non-decodable bytes (hit on assafelovic/gpt-researcher). The clean-ASCII fixture never triggered it.

Fix: every output-capturing subprocess.run now passes encoding='utf-8', errors='replace' (8 call sites). Added a regression test that scans a file with raw non-UTF-8 bytes and asserts no crash. Fixture unchanged (36); 27 tests pass.

Surfaced during PR-16 benchmark prep.
---
 dsgai_scanner_tool/cli/dsgai_scan.py    | 16 ++++++++--------
 dsgai_scanner_tool/tests/test_runner.py | 17 +++++++++++++++++
 2 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/dsgai_scanner_tool/cli/dsgai_scan.py b/dsgai_scanner_tool/cli/dsgai_scan.py
index 12571e9..cd25870 100644
--- a/dsgai_scanner_tool/cli/dsgai_scan.py
+++ b/dsgai_scanner_tool/cli/dsgai_scan.py
@@ -90,7 +90,7 @@ def find_rg():
 
 def rg_supports_pcre2(rg):
     try:
-        out = subprocess.run([rg, "--pcre2-version"], capture_output=True, text=True)
+        out = subprocess.run([rg, "--pcre2-version"], capture_output=True, text=True, encoding="utf-8", errors="replace")
         return out.returncode == 0
     except Exception:
         return False
@@ -156,13 +156,13 @@ def _sensitive_walk_files(target):
 def _git_files(target):
     try:
         top = subprocess.run(["git", "-C", target, "rev-parse", "--show-toplevel"],
-                             capture_output=True, text=True)
+                             capture_output=True, text=True, encoding="utf-8", errors="replace")
         if top.returncode != 0:
             return None
         res = subprocess.run(
             ["git", "-C", target, "ls-files", "--cached", "--others",
              "--exclude-standard", "-z", "--", "."],
-            capture_output=True, text=True)
+            capture_output=True, text=True, encoding="utf-8", errors="replace")
         if res.returncode != 0:
             return None
         rels = [p for p in res.stdout.split("\0") if p]
@@ -225,7 +225,7 @@ def _run_rg(rg, base_cmd, files, scan_root, rid):
     for batch in _batches(files):
         cmd = base_cmd + [f[0] for f in batch]
         try:
-            proc = subprocess.run(cmd, capture_output=True, text=True, cwd=scan_root)
+            proc = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", cwd=scan_root)
         except OSError as exc:  # e.g. arg list too long on a pathological path
             raise RuntimeError(f"rg invocation failed for {rid}: {exc}")
         if proc.returncode >= 2:
@@ -384,7 +384,7 @@ def now_iso():
 def git_commit(target):
     try:
         r = subprocess.run(["git", "-C", os.path.abspath(target), "rev-parse", "HEAD"],
-                           capture_output=True, text=True)
+                           capture_output=True, text=True, encoding="utf-8", errors="replace")
         return r.stdout.strip() if r.returncode == 0 else None
     except Exception:
         return None
@@ -437,7 +437,7 @@ def checkpoint_is_valid(cp, target, current_ruleset_version):
         return False
     try:
         r = subprocess.run(["git", "-C", os.path.abspath(target), "status", "--porcelain"],
-                           capture_output=True, text=True)
+                           capture_output=True, text=True, encoding="utf-8", errors="replace")
         if r.returncode != 0 or r.stdout.strip():
             return False  # dirty working tree
     except Exception:
@@ -565,7 +565,7 @@ def diff_files(target, ref):
     """Return the set of rel paths changed vs `ref` (git). None if unavailable."""
     try:
         r = subprocess.run(["git", "-C", os.path.abspath(target), "diff",
-                            "--name-only", ref], capture_output=True, text=True)
+                            "--name-only", ref], capture_output=True, text=True, encoding="utf-8", errors="replace")
         if r.returncode != 0:
             return None
         return {p.strip().replace("/", os.sep) for p in r.stdout.splitlines() if p.strip()}
@@ -709,7 +709,7 @@ def cmd_doctor(args):
     rg = find_rg()
     print(f"ripgrep: {'found at ' + rg if rg else 'NOT FOUND'}")
     if rg:
-        v = subprocess.run([rg, "--version"], capture_output=True, text=True)
+        v = subprocess.run([rg, "--version"], capture_output=True, text=True, encoding="utf-8", errors="replace")
         print("  " + v.stdout.splitlines()[0])
         pcre = rg_supports_pcre2(rg)
         print(f"  PCRE2 support: {'yes' if pcre else 'NO'}")
diff --git a/dsgai_scanner_tool/tests/test_runner.py b/dsgai_scanner_tool/tests/test_runner.py
index 8e60645..47de37c 100644
--- a/dsgai_scanner_tool/tests/test_runner.py
+++ b/dsgai_scanner_tool/tests/test_runner.py
@@ -408,6 +408,23 @@ def test_report_no_path_leak_on_filemap_miss(tmp_path):
     assert "<script>" not in rendered               # line escaped
 
 
+@requires_rg
+def test_scan_survives_non_utf8_bytes(tmp_path):
+    """A repo file with bytes not decodable as the platform default must not
+    crash the scanner (found by the benchmark; the ASCII fixture never hit it)."""
+    repo = tmp_path / "r"
+    repo.mkdir()
+    # P13.3 (host ... 0.0.0.0) matches this line; -o will emit the 0x8f byte.
+    (repo / "x.py").write_bytes(b'host = "\x8f\x9f-bad-bytes" 0.0.0.0\n')
+    out = tmp_path / "s.json"
+    proc = subprocess.run([sys.executable, CLI, "scan", str(repo), "--no-cve",
+                           "--json-out", str(out), "--format", "none"],
+                          capture_output=True, text=True, encoding="utf-8",
+                          errors="replace")
+    assert proc.returncode in (0, 1), f"scanner crashed on non-UTF-8 input: {proc.stderr}"
+    assert out.exists()
+
+
 def test_rules_json_in_sync():
     from_yaml = yaml.safe_load(open(RULES_YAML, encoding="utf-8"))
     rebuilt = json.dumps(from_yaml, indent=2, sort_keys=True, ensure_ascii=False) + "\n"