From e428ecef7e10c8b515b4b4d68968cfabb0739724 Mon Sep 17 00:00:00 2001 From: emmanuelgjr Date: Sat, 18 Jul 2026 13:52:16 -0400 Subject: [PATCH] fix(scanner): scan gitignored .env files (audit C1, critical) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovery used 'git ls-files --cached --others --exclude-standard', which drops gitignored+untracked files. In a real repo .env is gitignored, so the flagship value-bearing credential detection (hardcoded key in .env) silently never ran — the report showed a clean bill over a live key. The committed fixture .env (tracked) masked this in tests. Fix: union git discovery with a targeted walk for ALWAYS_SCAN_GLOBS (.env, *.env, .env.*, *.envrc) so credential files are scanned even when gitignored. Fixture scan unchanged (32 findings, exact); added a regression test with a real-world gitignored-.env repo layout (would have caught this). --- dsgai_scanner_tool/cli/dsgai_scan.py | 33 ++++++++++++++++++++++--- dsgai_scanner_tool/tests/test_runner.py | 25 +++++++++++++++++++ 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/dsgai_scanner_tool/cli/dsgai_scan.py b/dsgai_scanner_tool/cli/dsgai_scan.py index 668d43e..240bd41 100644 --- a/dsgai_scanner_tool/cli/dsgai_scan.py +++ b/dsgai_scanner_tool/cli/dsgai_scan.py @@ -40,6 +40,10 @@ } SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv", "env", "dist", "build", ".mypy_cache", ".pytest_cache", ".tox", ".idea"} +# Credential-bearing files that MUST be scanned even when gitignored — a secret +# scanner's whole job is to catch keys in exactly these, and they are almost +# always gitignored (so `git ls-files` would drop them). Matched by basename. +ALWAYS_SCAN_GLOBS = ["*.env", "*.env.*", ".env", ".env.*", "*.envrc"] # requires_nearby resolution: status when the requirement is violated (required # rule absent) vs satisfied (present). Read the require list / window from the @@ -109,16 +113,25 @@ def read_version(): # File discovery # --------------------------------------------------------------------------- # def discover_files(target, excludes): - """Return files under target as (abs_path, rel_path) honoring .gitignore. + """Return files under target as (abs_path, rel_path). - Uses `git ls-files` (tracked + untracked-not-ignored) when target is inside - a repo — this respects .gitignore yet still includes tracked files such as a - committed fixture .env. Falls back to a filtered os.walk otherwise. + Uses `git ls-files` (tracked + untracked-not-ignored) when target is inside a + repo, so general noise/build output respects .gitignore. BUT credential + files matching ALWAYS_SCAN_GLOBS (e.g. `.env`) are ALWAYS included even when + gitignored — a secret scanner must scan exactly those. Falls back to a + filtered os.walk (which already sees everything) outside a repo. """ target = os.path.abspath(target) files = _git_files(target) if files is None: files = _walk_files(target) + else: + # Union in gitignored credential files git would otherwise drop. + seen = set(files) + for ap in _sensitive_walk_files(target): + if ap not in seen: + seen.add(ap) + files.append(ap) out = [] for ap in files: rel = os.path.relpath(ap, target).replace(os.sep, "/") @@ -129,6 +142,18 @@ def discover_files(target, excludes): return out +def _sensitive_walk_files(target): + """Files under target whose basename matches ALWAYS_SCAN_GLOBS (skips noise + dirs). Used to re-include gitignored credential files like .env.""" + hits = [] + for dp, dirs, fns in os.walk(target): + dirs[:] = [d for d in dirs if d not in SKIP_DIRS] + for fn in fns: + if any(fnmatch.fnmatch(fn, g) for g in ALWAYS_SCAN_GLOBS): + hits.append(os.path.join(dp, fn)) + return hits + + def _git_files(target): try: top = subprocess.run(["git", "-C", target, "rev-parse", "--show-toplevel"], diff --git a/dsgai_scanner_tool/tests/test_runner.py b/dsgai_scanner_tool/tests/test_runner.py index 16e5b50..a5a2d19 100644 --- a/dsgai_scanner_tool/tests/test_runner.py +++ b/dsgai_scanner_tool/tests/test_runner.py @@ -314,6 +314,31 @@ def test_atlas_map_valid(): assert all(ctrl_re.match(c) for c in t["controls"]) +@requires_rg +def test_gitignored_env_is_still_scanned(tmp_path): + """Real-world layout: a `.env` is gitignored + untracked. A secret scanner + MUST still scan it — regression test for the discovery bug where git + ls-files silently dropped it (the committed fixture .env masked this).""" + repo = tmp_path / "realrepo" + repo.mkdir() + subprocess.run(["git", "init", "-q"], cwd=repo, check=True) + (repo / ".gitignore").write_text(".env\n", encoding="utf-8") + (repo / ".env").write_text( + "OPENAI_API_KEY=sk-proj-FAKE00000000000000000000000000\n", encoding="utf-8") + (repo / "app.py").write_text("x = 1\n", encoding="utf-8") + env = dict(os.environ) + subprocess.run(["git", "add", ".gitignore", "app.py"], cwd=repo, check=True) + subprocess.run(["git", "-c", "user.email=a@b.c", "-c", "user.name=t", + "commit", "-qm", "init"], cwd=repo, check=True) + out = tmp_path / "s.json" + subprocess.run([sys.executable, CLI, "scan", str(repo), "--no-cve", + "--json-out", str(out), "--format", "none"], env=env) + cp = json.loads(out.read_text(encoding="utf-8")) + env_findings = [f for f in cp["findings"] if ".env" in f["path"]] + assert env_findings, "gitignored .env credential was not scanned" + assert cp["controls"]["DSGAI02"] == "FAIL" + + def test_rules_json_in_sync(): from_yaml = yaml.safe_load(open(RULES_YAML, encoding="utf-8")) rebuilt = json.dumps(from_yaml, indent=2, sort_keys=True, ensure_ascii=False) + "\n"