diff --git a/dsgai_scanner_tool/build/export_semgrep.py b/dsgai_scanner_tool/build/export_semgrep.py index 7824f91..0e1bd57 100644 --- a/dsgai_scanner_tool/build/export_semgrep.py +++ b/dsgai_scanner_tool/build/export_semgrep.py @@ -37,13 +37,19 @@ def render(): continue sev = SEVERITY.get(r["signal"], "INFO") includes = ", ".join(yq(g) for g in r["file_globs"]) + # Semgrep pattern-regex runs against whole-file text; force multiline so a + # line-anchored rule (e.g. P04.4's `^`) matches on any line, not just + # file start (audit L7). + pcre = r["pcre"] + if ("^" in pcre or "$" in pcre) and not pcre.startswith("(?m)"): + pcre = "(?m)" + pcre out += [ f" - id: dsgai-{r['id']}", f" languages: [generic]", f" severity: {sev}", f" message: {yq(r['control'] + ' ' + r['id'] + ': ' + r['description'])}", f" patterns:", - f" - pattern-regex: {yq(r['pcre'])}", + f" - pattern-regex: {yq(pcre)}", f" paths:", f" include: [{includes}]", f" metadata:", diff --git a/dsgai_scanner_tool/dist/dsgai.semgrep.yaml b/dsgai_scanner_tool/dist/dsgai.semgrep.yaml index 2d86d46..3177a77 100644 --- a/dsgai_scanner_tool/dist/dsgai.semgrep.yaml +++ b/dsgai_scanner_tool/dist/dsgai.semgrep.yaml @@ -124,7 +124,7 @@ rules: severity: INFO message: 'DSGAI04 P04.2: Safe pickle (PASS counter)' patterns: - - pattern-regex: 'torch\.load\s*\([^)]*weights_only\s*=\s*True' + - pattern-regex: '(?m)torch\.load\s*\([^)]*weights_only\s*=\s*True' paths: include: ['*.py', '*.sh', '*.yml', '*.yaml', 'Dockerfile', 'requirements*.txt', 'pyproject.toml', 'setup.py'] metadata: @@ -150,7 +150,7 @@ rules: severity: WARNING message: 'DSGAI04 P04.4: Unpinned ML deps (WARN)' patterns: - - pattern-regex: '^(torch|transformers|tensorflow|langchain|openai|anthropic|llama-index)\s*(>=|~=|\^|>|<|<=|\*|latest)' + - pattern-regex: '(?m)^(torch|transformers|tensorflow|langchain|openai|anthropic|llama-index)\s*(>=|~=|\^|>|<|<=|\*|latest)' paths: include: ['*.py', '*.sh', '*.yml', '*.yaml', 'Dockerfile', 'requirements*.txt', 'pyproject.toml', 'setup.py'] metadata: @@ -228,7 +228,7 @@ rules: severity: INFO message: 'DSGAI05 P05.3: Tenant filter on search (PASS)' patterns: - - pattern-regex: '(similarity_search|vector_search)[^)]{0,100}(filter|namespace|where)[^)]{0,40}tenant' + - pattern-regex: '(?m)(similarity_search|vector_search)[^)]{0,100}(filter|namespace|where)[^)]{0,40}tenant' paths: include: ['*.py', '*.ts', '*.java', '*.kt'] metadata: @@ -241,7 +241,7 @@ rules: severity: INFO message: 'DSGAI05 P05.4: Integrity check on docs (PASS)' patterns: - - pattern-regex: '(hashlib|sha256[^\n]{0,40}doc|integrity[._-]check|verify[._-]document)' + - pattern-regex: '(?m)(hashlib|sha256[^\n]{0,40}doc|integrity[._-]check|verify[._-]document)' paths: include: ['*.py', '*.ts', '*.java', '*.kt'] metadata: @@ -280,7 +280,7 @@ rules: severity: ERROR message: 'DSGAI06 P06.1: Insecure MCP transport (FAIL)' patterns: - - pattern-regex: '("url"\s*:\s*"http://|transport[^\n]{0,20}http://|"command"[^}]{0,200}--http(?!s))' + - pattern-regex: '(?m)("url"\s*:\s*"http://|transport[^\n]{0,20}http://|"command"[^}]{0,200}--http(?!s))' paths: include: ['*.json', '*.yaml', '*.toml', '*.py', '*.ts', 'mcp.json', 'claude_desktop_config.json'] metadata: @@ -293,7 +293,7 @@ rules: severity: INFO message: 'DSGAI06 P06.2: MCP auth (PASS)' patterns: - - pattern-regex: '(mcp[^\n]{0,30}(api_key|auth|bearer)|x-api-key[^\n]{0,20}mcp)' + - pattern-regex: '(?m)(mcp[^\n]{0,30}(api_key|auth|bearer)|x-api-key[^\n]{0,20}mcp)' paths: include: ['*.json', '*.yaml', '*.toml', '*.py', '*.ts', 'mcp.json', 'claude_desktop_config.json'] metadata: @@ -306,7 +306,7 @@ rules: severity: INFO message: 'DSGAI06 P06.3: Tool schema validation (PASS)' patterns: - - pattern-regex: '(jsonschema|pydantic[^\n]{0,30}validate|zod|schema[^\n]{0,20}tool|input_schema)' + - pattern-regex: '(?m)(jsonschema|pydantic[^\n]{0,30}validate|zod|schema[^\n]{0,20}tool|input_schema)' paths: include: ['*.json', '*.yaml', '*.toml', '*.py', '*.ts', 'mcp.json', 'claude_desktop_config.json'] metadata: @@ -332,7 +332,7 @@ rules: severity: INFO message: 'DSGAI06 P06.5: uvicorn bind-all + no auth' patterns: - - pattern-regex: 'uvicorn\.run\([^)]*host\s*=\s*["'']0\.0\.0\.0' + - pattern-regex: '(?m)uvicorn\.run\([^)]*host\s*=\s*["'']0\.0\.0\.0' paths: include: ['*.json', '*.yaml', '*.toml', '*.py', '*.ts', 'mcp.json', 'claude_desktop_config.json'] metadata: @@ -618,7 +618,7 @@ rules: severity: INFO message: 'DSGAI11 P11.2: Tenant filter present (PASS)' patterns: - - pattern-regex: '(namespace\s*=[^,)]{0,40}tenant|collection[^=]{0,20}=\s*f?["''][^"'']*\{?tenant|filter\s*=\s*\{[^}]*tenant)' + - pattern-regex: '(?m)(namespace\s*=[^,)]{0,40}tenant|collection[^=]{0,20}=\s*f?["''][^"'']*\{?tenant|filter\s*=\s*\{[^}]*tenant)' paths: include: ['*.py', '*.ts', '*.java', '*.kt', '*.go'] metadata: @@ -657,7 +657,7 @@ rules: severity: ERROR message: 'DSGAI12 P12.1: Raw LLM-output execution (FAIL)' patterns: - - pattern-regex: '(?:\.execute|\.run_query|executemany)\(\s*(?:f["'']|[^),]*\b(llm_\w*|completion\w*|generated_sql|generated_query|\w*\.content)\b)' + - pattern-regex: '(?m)(?:\.execute|\.run_query|executemany)\(\s*(?:f["'']|[^),]*\b(llm_\w*|completion\w*|generated_sql|generated_query|\w*\.content)\b)' paths: include: ['*.py', '*.ts', '*.java', '*.kt', '*.go'] metadata: @@ -683,7 +683,7 @@ rules: severity: INFO message: 'DSGAI12 P12.3: Parameterized query (PASS)' patterns: - - pattern-regex: '(cursor\.execute\([^)]+,\s*[\[\(]|prepareStatement|bindValue|:param)' + - pattern-regex: '(?m)(cursor\.execute\([^)]+,\s*[\[\(]|prepareStatement|bindValue|:param)' paths: include: ['*.py', '*.ts', '*.java', '*.kt', '*.go'] metadata: @@ -774,7 +774,7 @@ rules: severity: INFO message: 'DSGAI16 P16.3: Telemetry off (PASS)' patterns: - - pattern-regex: '(telemetry[^\n]{0,10}(off|false|disabled)|share_data\s*[:=]\s*false)' + - pattern-regex: '(?m)(telemetry[^\n]{0,10}(off|false|disabled)|share_data\s*[:=]\s*false)' paths: include: ['.copilotignore', '.aiignore', '.cursorignore', '.continueignore', '.codeiumignore', '.vscode/settings.json', 'cursor.json', 'continue.json'] metadata: @@ -826,7 +826,7 @@ rules: severity: INFO message: 'DSGAI17 P17.3: Timeout on LLM call (PASS)' patterns: - - pattern-regex: '(timeout\s*=\s*\d|request_timeout\s*=|httpx\.[^\n]{0,30}timeout)' + - pattern-regex: '(?m)(timeout\s*=\s*\d|request_timeout\s*=|httpx\.[^\n]{0,30}timeout)' paths: include: ['*.py', '*.ts', '*.java', '*.kt', '*.go'] metadata: @@ -865,7 +865,7 @@ rules: severity: ERROR message: 'DSGAI17 P17.6: Unbounded retry (FAIL)' patterns: - - pattern-regex: 'while\s+True[^}]{0,200}(generate|complete|chat)|retry\s*\(\s*\)' + - pattern-regex: '(?m)while\s+True[^}]{0,200}(generate|complete|chat)|retry\s*\(\s*\)' paths: include: ['*.py', '*.ts', '*.java', '*.kt', '*.go'] metadata: @@ -891,7 +891,7 @@ rules: severity: INFO message: 'DSGAI18 P18.2: PII detection on output' patterns: - - pattern-regex: '(detect_pii[^\n]{0,20}output|output[^\n]{0,20}detect_pii|presidio[^\n]{0,20}output|scan_output)' + - pattern-regex: '(?m)(detect_pii[^\n]{0,20}output|output[^\n]{0,20}detect_pii|presidio[^\n]{0,20}output|scan_output)' paths: include: ['*.py', '*.ts', '*.java', '*.kt', '*.go'] metadata: @@ -956,7 +956,7 @@ rules: severity: INFO message: 'DSGAI19 P19.2: Labeling SDK auth from vault' patterns: - - pattern-regex: '(label_studio[^\n]{0,30}vault|labelbox[^\n]{0,30}secret|scale_api_key[^\n]{0,30}(vault|env))' + - pattern-regex: '(?m)(label_studio[^\n]{0,30}vault|labelbox[^\n]{0,30}secret|scale_api_key[^\n]{0,30}(vault|env))' paths: include: ['*.py', '*.ipynb', '*.ts', '*.yaml'] metadata: @@ -995,7 +995,7 @@ rules: severity: INFO message: 'DSGAI20 P20.1: API auth required (PASS)' patterns: - - pattern-regex: '(api_key[^\n]{0,15}(verify|validate)|bearer_token[^\n]{0,15}validate|Authorization[^\n]{0,15}required|@require_auth|auth_middleware|Depends\(.*auth)' + - pattern-regex: '(?m)(api_key[^\n]{0,15}(verify|validate)|bearer_token[^\n]{0,15}validate|Authorization[^\n]{0,15}required|@require_auth|auth_middleware|Depends\(.*auth)' paths: include: ['*.py', '*.ts', '*.java', '*.kt', '*.go'] metadata: @@ -1047,7 +1047,7 @@ rules: severity: INFO message: 'DSGAI20 P20.5: Inference endpoint (count)' patterns: - - pattern-regex: '(@app\.(post|get)\(["''][^"'']*(chat|generate|completion|infer|predict)|@router\.(post|get)\(["''][^"'']*(chat|generate|completion|infer|predict)|app\.(post|get)\(["''][^"'']*(chat|generate|completion|infer|predict))' + - pattern-regex: '(?m)(@app\.(post|get)\(["''][^"'']*(chat|generate|completion|infer|predict)|@router\.(post|get)\(["''][^"'']*(chat|generate|completion|infer|predict)|app\.(post|get)\(["''][^"'']*(chat|generate|completion|infer|predict))' paths: include: ['*.py', '*.ts', '*.java', '*.kt', '*.go'] metadata: @@ -1060,7 +1060,7 @@ rules: severity: INFO message: 'DSGAI21 P21.1: Read-only retrieval (PASS)' patterns: - - pattern-regex: '(read_only\s*=\s*True|readonly\s*=\s*True|HttpMethod\.GET|http_method[^\n]{0,10}get)' + - pattern-regex: '(?m)(read_only\s*=\s*True|readonly\s*=\s*True|HttpMethod\.GET|http_method[^\n]{0,10}get)' paths: include: ['*.py', '*.ts', '*.java', '*.kt'] metadata: @@ -1073,7 +1073,7 @@ rules: severity: WARNING message: 'DSGAI21 P21.2: KB write from agent (WARN)' patterns: - - pattern-regex: '(chromadb|qdrant|pinecone|weaviate|knowledge_base|kb_client)[^\n]{0,40}\.(upsert|insert|update|delete|add_documents|write)\(' + - pattern-regex: '(?m)(chromadb|qdrant|pinecone|weaviate|knowledge_base|kb_client)[^\n]{0,40}\.(upsert|insert|update|delete|add_documents|write)\(' paths: include: ['*.py', '*.ts', '*.java', '*.kt'] metadata: diff --git a/dsgai_scanner_tool/dsgai_scanner_prompt.md b/dsgai_scanner_tool/dsgai_scanner_prompt.md index 7705823..32825a5 100644 --- a/dsgai_scanner_tool/dsgai_scanner_prompt.md +++ b/dsgai_scanner_tool/dsgai_scanner_prompt.md @@ -38,8 +38,7 @@ **Outputs:** `dsgai-reports/DSGAI-report-.html` (the deliverable) + `DSGAI-scan.json` (checkpoint at repo root). STRICT mode is designed to minimize disclosure, but a findings report still reveals that controls fail and where — handle it like any security assessment (see *Residual risk* below). -**Time:** 2–5 minutes for a typical repo in Claude Code (parallel scans). Longer outside Claude Code. - +**Time:** a few minutes for a typical repo. --- ## Argument Handling @@ -648,7 +647,7 @@ Run value-bearing patterns exclusively via Bash with ripgrep's `--replace ''`, w rg -n -o --replace '' --pcre2 'PATTERN' ``` -The output stream is `path:line:` only — there is no matched content to leak, because rg never emitted it. Do **not** use the Grep tool's `output_mode="content"` on value-bearing patterns; do **not** pipe rg output through a second process that could re-expose the line. The guarantee is now *structural* (rg erased it), not *behavioral* (you remembered not to write it). +The output stream is `path:line:` only — there is no matched content to leak, because rg never emitted it. Do **not** pipe rg output through a second process that could re-expose the line.The guarantee is now *structural* (rg erased it), not *behavioral* (you remembered not to write it). **V2 — Render evidence:** ``` diff --git a/dsgai_scanner_tool/dsgai_scanner_tool.md b/dsgai_scanner_tool/dsgai_scanner_tool.md index efccb19..f5e5dc8 100644 --- a/dsgai_scanner_tool/dsgai_scanner_tool.md +++ b/dsgai_scanner_tool/dsgai_scanner_tool.md @@ -35,7 +35,7 @@ compatible_cli: '>=0.3,<0.5' **Outputs:** `dsgai-reports/DSGAI-report-.html` (the deliverable) + `DSGAI-scan.json` (checkpoint at repo root). STRICT mode is designed to minimize disclosure, but a findings report still reveals that controls fail and where — handle it like any security assessment (see *Residual risk* below). -**Time:** 2–5 minutes for a typical repo in Claude Code (parallel scans). Longer outside Claude Code. +**Time:** a few minutes for a typical repo. In Claude Code the 21 scans run in parallel, so it's faster (2–5 minutes); longer outside Claude Code. --- @@ -646,7 +646,7 @@ Run value-bearing patterns exclusively via Bash with ripgrep's `--replace ''`, w rg -n -o --replace '' --pcre2 'PATTERN' ``` -The output stream is `path:line:` only — there is no matched content to leak, because rg never emitted it. Do **not** use the Grep tool's `output_mode="content"` on value-bearing patterns; do **not** pipe rg output through a second process that could re-expose the line. The guarantee is now *structural* (rg erased it), not *behavioral* (you remembered not to write it). +The output stream is `path:line:` only — there is no matched content to leak, because rg never emitted it. Do **not** pipe rg output through a second process that could re-expose the line. In Claude Code specifically, do **not** use the Grep tool's `output_mode="content"` on value-bearing patterns. The guarantee is now *structural* (rg erased it), not *behavioral* (you remembered not to write it). **V2 — Render evidence:** ``` diff --git a/dsgai_scanner_tool/integrations/dsgai-secret-scan.sh b/dsgai_scanner_tool/integrations/dsgai-secret-scan.sh index 2857c5e..1efb2ca 100644 --- a/dsgai_scanner_tool/integrations/dsgai-secret-scan.sh +++ b/dsgai_scanner_tool/integrations/dsgai-secret-scan.sh @@ -17,6 +17,15 @@ if ! command -v rg >/dev/null 2>&1; then exit 0 fi +# The patterns need PCRE2. Without it, `rg --pcre2` errors on every file and the +# check would silently pass (fail-open on a security gate). Probe once and skip +# loudly instead. +if ! printf 'x' | rg --pcre2 -q 'x' 2>/dev/null; then + echo "warning: this ripgrep lacks PCRE2 support; skipping DSGAI02 pre-commit check" + echo " install a PCRE2-enabled ripgrep, or use the gitleaks pack instead" + exit 0 +fi + # Collect staged files with a NUL-delimited read loop (bash-3.2 safe — no # mapfile). Filter extensions with a case statement (no `grep -z`, which BSD # grep lacks). Paths with spaces or newlines are handled correctly. diff --git a/dsgai_scanner_tool/integrations/gitleaks/dsgai.toml b/dsgai_scanner_tool/integrations/gitleaks/dsgai.toml index 9a82cbb..b541a85 100644 --- a/dsgai_scanner_tool/integrations/gitleaks/dsgai.toml +++ b/dsgai_scanner_tool/integrations/gitleaks/dsgai.toml @@ -28,7 +28,9 @@ keywords = ["anthropic", "sk-ant-"] id = "dsgai-named-credential" description = "DSGAI02 — named LLM/cloud/vector-store credential assignment, quote-optional" regex = '''(?i)(COHERE_API_KEY|GOOGLE_API_KEY|HF_TOKEN|HUGGINGFACE_TOKEN|AWS_ACCESS_KEY_ID|AZURE_OPENAI_KEY|GCP_SERVICE_ACCOUNT_KEY|QDRANT_API_KEY|PINECONE_API_KEY|WEAVIATE_API_KEY|MILVUS_TOKEN|LANGSMITH_API_KEY|LANGFUSE_PUBLIC_KEY|LANGFUSE_SECRET_KEY)\s*[:=]\s*["']?[A-Za-z0-9_\-]{16,}''' -keywords = ["api_key", "token", "qdrant", "pinecone", "weaviate", "langfuse", "langsmith"] +# "key" (case-insensitive) covers AZURE_OPENAI_KEY / GCP_SERVICE_ACCOUNT_KEY / +# AWS_ACCESS_KEY_ID etc.; the keyword prefilter previously dropped those three. +keywords = ["key", "token", "qdrant", "pinecone", "weaviate", "langfuse", "langsmith"] [[rules]] id = "dsgai-token-prefix" diff --git a/dsgai_scanner_tool/rules/README.md b/dsgai_scanner_tool/rules/README.md index 7e9ba49..62301f7 100644 --- a/dsgai_scanner_tool/rules/README.md +++ b/dsgai_scanner_tool/rules/README.md @@ -43,7 +43,7 @@ deterministic CLI loads with the standard library only — no PyYAML at runtime) | `remediation` | no | fix guidance | | `references` | no | CVE IDs / links | | `subtract` | no | rule IDs whose match on the same line cancels this hit (e.g. `torch.load(` minus `weights_only=True`) | -| `requires_nearby` | no | compound proximity logic: `{rule\|rules, lines\|scope, absent}` | +| `requires_nearby` | no | compound proximity logic: `{rule\|rules\|pattern, lines\|scope, absent}` — `pattern` is a raw corroborating regex (e.g. P12.1 requires an LLM call within 30 lines) | | `gated_on` | no | only evaluated when the stack is detected: `multimodal`, `synthetic_data`, `labeling` | | `notes` | no | control-level prose that isn't yet fully formalized |