Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion dsgai_scanner_tool/build/export_semgrep.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,19 @@ def render():
continue
sev = SEVERITY.get(r["signal"], "INFO")
includes = ", ".join(yq(g) for g in r["file_globs"])
# Semgrep pattern-regex runs against whole-file text; force multiline so a
# line-anchored rule (e.g. P04.4's `^`) matches on any line, not just
# file start (audit L7).
pcre = r["pcre"]
if ("^" in pcre or "$" in pcre) and not pcre.startswith("(?m)"):
pcre = "(?m)" + pcre
out += [
f" - id: dsgai-{r['id']}",
f" languages: [generic]",
f" severity: {sev}",
f" message: {yq(r['control'] + ' ' + r['id'] + ': ' + r['description'])}",
f" patterns:",
f" - pattern-regex: {yq(r['pcre'])}",
f" - pattern-regex: {yq(pcre)}",
f" paths:",
f" include: [{includes}]",
f" metadata:",
Expand Down
40 changes: 20 additions & 20 deletions dsgai_scanner_tool/dist/dsgai.semgrep.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ rules:
severity: INFO
message: 'DSGAI04 P04.2: Safe pickle (PASS counter)'
patterns:
- pattern-regex: 'torch\.load\s*\([^)]*weights_only\s*=\s*True'
- pattern-regex: '(?m)torch\.load\s*\([^)]*weights_only\s*=\s*True'
paths:
include: ['*.py', '*.sh', '*.yml', '*.yaml', 'Dockerfile', 'requirements*.txt', 'pyproject.toml', 'setup.py']
metadata:
Expand All @@ -150,7 +150,7 @@ rules:
severity: WARNING
message: 'DSGAI04 P04.4: Unpinned ML deps (WARN)'
patterns:
- pattern-regex: '^(torch|transformers|tensorflow|langchain|openai|anthropic|llama-index)\s*(>=|~=|\^|>|<|<=|\*|latest)'
- pattern-regex: '(?m)^(torch|transformers|tensorflow|langchain|openai|anthropic|llama-index)\s*(>=|~=|\^|>|<|<=|\*|latest)'
paths:
include: ['*.py', '*.sh', '*.yml', '*.yaml', 'Dockerfile', 'requirements*.txt', 'pyproject.toml', 'setup.py']
metadata:
Expand Down Expand Up @@ -228,7 +228,7 @@ rules:
severity: INFO
message: 'DSGAI05 P05.3: Tenant filter on search (PASS)'
patterns:
- pattern-regex: '(similarity_search|vector_search)[^)]{0,100}(filter|namespace|where)[^)]{0,40}tenant'
- pattern-regex: '(?m)(similarity_search|vector_search)[^)]{0,100}(filter|namespace|where)[^)]{0,40}tenant'
paths:
include: ['*.py', '*.ts', '*.java', '*.kt']
metadata:
Expand All @@ -241,7 +241,7 @@ rules:
severity: INFO
message: 'DSGAI05 P05.4: Integrity check on docs (PASS)'
patterns:
- pattern-regex: '(hashlib|sha256[^\n]{0,40}doc|integrity[._-]check|verify[._-]document)'
- pattern-regex: '(?m)(hashlib|sha256[^\n]{0,40}doc|integrity[._-]check|verify[._-]document)'
paths:
include: ['*.py', '*.ts', '*.java', '*.kt']
metadata:
Expand Down Expand Up @@ -280,7 +280,7 @@ rules:
severity: ERROR
message: 'DSGAI06 P06.1: Insecure MCP transport (FAIL)'
patterns:
- pattern-regex: '("url"\s*:\s*"http://|transport[^\n]{0,20}http://|"command"[^}]{0,200}--http(?!s))'
- pattern-regex: '(?m)("url"\s*:\s*"http://|transport[^\n]{0,20}http://|"command"[^}]{0,200}--http(?!s))'
paths:
include: ['*.json', '*.yaml', '*.toml', '*.py', '*.ts', 'mcp.json', 'claude_desktop_config.json']
metadata:
Expand All @@ -293,7 +293,7 @@ rules:
severity: INFO
message: 'DSGAI06 P06.2: MCP auth (PASS)'
patterns:
- pattern-regex: '(mcp[^\n]{0,30}(api_key|auth|bearer)|x-api-key[^\n]{0,20}mcp)'
- pattern-regex: '(?m)(mcp[^\n]{0,30}(api_key|auth|bearer)|x-api-key[^\n]{0,20}mcp)'
paths:
include: ['*.json', '*.yaml', '*.toml', '*.py', '*.ts', 'mcp.json', 'claude_desktop_config.json']
metadata:
Expand All @@ -306,7 +306,7 @@ rules:
severity: INFO
message: 'DSGAI06 P06.3: Tool schema validation (PASS)'
patterns:
- pattern-regex: '(jsonschema|pydantic[^\n]{0,30}validate|zod|schema[^\n]{0,20}tool|input_schema)'
- pattern-regex: '(?m)(jsonschema|pydantic[^\n]{0,30}validate|zod|schema[^\n]{0,20}tool|input_schema)'
paths:
include: ['*.json', '*.yaml', '*.toml', '*.py', '*.ts', 'mcp.json', 'claude_desktop_config.json']
metadata:
Expand All @@ -332,7 +332,7 @@ rules:
severity: INFO
message: 'DSGAI06 P06.5: uvicorn bind-all + no auth'
patterns:
- pattern-regex: 'uvicorn\.run\([^)]*host\s*=\s*["'']0\.0\.0\.0'
- pattern-regex: '(?m)uvicorn\.run\([^)]*host\s*=\s*["'']0\.0\.0\.0'
paths:
include: ['*.json', '*.yaml', '*.toml', '*.py', '*.ts', 'mcp.json', 'claude_desktop_config.json']
metadata:
Expand Down Expand Up @@ -618,7 +618,7 @@ rules:
severity: INFO
message: 'DSGAI11 P11.2: Tenant filter present (PASS)'
patterns:
- pattern-regex: '(namespace\s*=[^,)]{0,40}tenant|collection[^=]{0,20}=\s*f?["''][^"'']*\{?tenant|filter\s*=\s*\{[^}]*tenant)'
- pattern-regex: '(?m)(namespace\s*=[^,)]{0,40}tenant|collection[^=]{0,20}=\s*f?["''][^"'']*\{?tenant|filter\s*=\s*\{[^}]*tenant)'
paths:
include: ['*.py', '*.ts', '*.java', '*.kt', '*.go']
metadata:
Expand Down Expand Up @@ -657,7 +657,7 @@ rules:
severity: ERROR
message: 'DSGAI12 P12.1: Raw LLM-output execution (FAIL)'
patterns:
- pattern-regex: '(?:\.execute|\.run_query|executemany)\(\s*(?:f["'']|[^),]*\b(llm_\w*|completion\w*|generated_sql|generated_query|\w*\.content)\b)'
- pattern-regex: '(?m)(?:\.execute|\.run_query|executemany)\(\s*(?:f["'']|[^),]*\b(llm_\w*|completion\w*|generated_sql|generated_query|\w*\.content)\b)'
paths:
include: ['*.py', '*.ts', '*.java', '*.kt', '*.go']
metadata:
Expand All @@ -683,7 +683,7 @@ rules:
severity: INFO
message: 'DSGAI12 P12.3: Parameterized query (PASS)'
patterns:
- pattern-regex: '(cursor\.execute\([^)]+,\s*[\[\(]|prepareStatement|bindValue|:param)'
- pattern-regex: '(?m)(cursor\.execute\([^)]+,\s*[\[\(]|prepareStatement|bindValue|:param)'
paths:
include: ['*.py', '*.ts', '*.java', '*.kt', '*.go']
metadata:
Expand Down Expand Up @@ -774,7 +774,7 @@ rules:
severity: INFO
message: 'DSGAI16 P16.3: Telemetry off (PASS)'
patterns:
- pattern-regex: '(telemetry[^\n]{0,10}(off|false|disabled)|share_data\s*[:=]\s*false)'
- pattern-regex: '(?m)(telemetry[^\n]{0,10}(off|false|disabled)|share_data\s*[:=]\s*false)'
paths:
include: ['.copilotignore', '.aiignore', '.cursorignore', '.continueignore', '.codeiumignore', '.vscode/settings.json', 'cursor.json', 'continue.json']
metadata:
Expand Down Expand Up @@ -826,7 +826,7 @@ rules:
severity: INFO
message: 'DSGAI17 P17.3: Timeout on LLM call (PASS)'
patterns:
- pattern-regex: '(timeout\s*=\s*\d|request_timeout\s*=|httpx\.[^\n]{0,30}timeout)'
- pattern-regex: '(?m)(timeout\s*=\s*\d|request_timeout\s*=|httpx\.[^\n]{0,30}timeout)'
paths:
include: ['*.py', '*.ts', '*.java', '*.kt', '*.go']
metadata:
Expand Down Expand Up @@ -865,7 +865,7 @@ rules:
severity: ERROR
message: 'DSGAI17 P17.6: Unbounded retry (FAIL)'
patterns:
- pattern-regex: 'while\s+True[^}]{0,200}(generate|complete|chat)|retry\s*\(\s*\)'
- pattern-regex: '(?m)while\s+True[^}]{0,200}(generate|complete|chat)|retry\s*\(\s*\)'
paths:
include: ['*.py', '*.ts', '*.java', '*.kt', '*.go']
metadata:
Expand All @@ -891,7 +891,7 @@ rules:
severity: INFO
message: 'DSGAI18 P18.2: PII detection on output'
patterns:
- pattern-regex: '(detect_pii[^\n]{0,20}output|output[^\n]{0,20}detect_pii|presidio[^\n]{0,20}output|scan_output)'
- pattern-regex: '(?m)(detect_pii[^\n]{0,20}output|output[^\n]{0,20}detect_pii|presidio[^\n]{0,20}output|scan_output)'
paths:
include: ['*.py', '*.ts', '*.java', '*.kt', '*.go']
metadata:
Expand Down Expand Up @@ -956,7 +956,7 @@ rules:
severity: INFO
message: 'DSGAI19 P19.2: Labeling SDK auth from vault'
patterns:
- pattern-regex: '(label_studio[^\n]{0,30}vault|labelbox[^\n]{0,30}secret|scale_api_key[^\n]{0,30}(vault|env))'
- pattern-regex: '(?m)(label_studio[^\n]{0,30}vault|labelbox[^\n]{0,30}secret|scale_api_key[^\n]{0,30}(vault|env))'
paths:
include: ['*.py', '*.ipynb', '*.ts', '*.yaml']
metadata:
Expand Down Expand Up @@ -995,7 +995,7 @@ rules:
severity: INFO
message: 'DSGAI20 P20.1: API auth required (PASS)'
patterns:
- pattern-regex: '(api_key[^\n]{0,15}(verify|validate)|bearer_token[^\n]{0,15}validate|Authorization[^\n]{0,15}required|@require_auth|auth_middleware|Depends\(.*auth)'
- pattern-regex: '(?m)(api_key[^\n]{0,15}(verify|validate)|bearer_token[^\n]{0,15}validate|Authorization[^\n]{0,15}required|@require_auth|auth_middleware|Depends\(.*auth)'
paths:
include: ['*.py', '*.ts', '*.java', '*.kt', '*.go']
metadata:
Expand Down Expand Up @@ -1047,7 +1047,7 @@ rules:
severity: INFO
message: 'DSGAI20 P20.5: Inference endpoint (count)'
patterns:
- pattern-regex: '(@app\.(post|get)\(["''][^"'']*(chat|generate|completion|infer|predict)|@router\.(post|get)\(["''][^"'']*(chat|generate|completion|infer|predict)|app\.(post|get)\(["''][^"'']*(chat|generate|completion|infer|predict))'
- pattern-regex: '(?m)(@app\.(post|get)\(["''][^"'']*(chat|generate|completion|infer|predict)|@router\.(post|get)\(["''][^"'']*(chat|generate|completion|infer|predict)|app\.(post|get)\(["''][^"'']*(chat|generate|completion|infer|predict))'
paths:
include: ['*.py', '*.ts', '*.java', '*.kt', '*.go']
metadata:
Expand All @@ -1060,7 +1060,7 @@ rules:
severity: INFO
message: 'DSGAI21 P21.1: Read-only retrieval (PASS)'
patterns:
- pattern-regex: '(read_only\s*=\s*True|readonly\s*=\s*True|HttpMethod\.GET|http_method[^\n]{0,10}get)'
- pattern-regex: '(?m)(read_only\s*=\s*True|readonly\s*=\s*True|HttpMethod\.GET|http_method[^\n]{0,10}get)'
paths:
include: ['*.py', '*.ts', '*.java', '*.kt']
metadata:
Expand All @@ -1073,7 +1073,7 @@ rules:
severity: WARNING
message: 'DSGAI21 P21.2: KB write from agent (WARN)'
patterns:
- pattern-regex: '(chromadb|qdrant|pinecone|weaviate|knowledge_base|kb_client)[^\n]{0,40}\.(upsert|insert|update|delete|add_documents|write)\('
- pattern-regex: '(?m)(chromadb|qdrant|pinecone|weaviate|knowledge_base|kb_client)[^\n]{0,40}\.(upsert|insert|update|delete|add_documents|write)\('
paths:
include: ['*.py', '*.ts', '*.java', '*.kt']
metadata:
Expand Down
5 changes: 2 additions & 3 deletions dsgai_scanner_tool/dsgai_scanner_prompt.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,7 @@

**Outputs:** `dsgai-reports/DSGAI-report-<timestamp>.html` (the deliverable) + `DSGAI-scan.json` (checkpoint at repo root). STRICT mode is designed to minimize disclosure, but a findings report still reveals that controls fail and where — handle it like any security assessment (see *Residual risk* below).

**Time:** 2–5 minutes for a typical repo in Claude Code (parallel scans). Longer outside Claude Code.

**Time:** a few minutes for a typical repo.
---

## Argument Handling
Expand Down Expand Up @@ -648,7 +647,7 @@ Run value-bearing patterns exclusively via Bash with ripgrep's `--replace ''`, w
rg -n -o --replace '' --pcre2 'PATTERN' <files>
```

The output stream is `path:line:` only — there is no matched content to leak, because rg never emitted it. Do **not** use the Grep tool's `output_mode="content"` on value-bearing patterns; do **not** pipe rg output through a second process that could re-expose the line. The guarantee is now *structural* (rg erased it), not *behavioral* (you remembered not to write it).
The output stream is `path:line:` only — there is no matched content to leak, because rg never emitted it. Do **not** pipe rg output through a second process that could re-expose the line.The guarantee is now *structural* (rg erased it), not *behavioral* (you remembered not to write it).

**V2 — Render evidence:**
```
Expand Down
4 changes: 2 additions & 2 deletions dsgai_scanner_tool/dsgai_scanner_tool.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ compatible_cli: '>=0.3,<0.5'

**Outputs:** `dsgai-reports/DSGAI-report-<timestamp>.html` (the deliverable) + `DSGAI-scan.json` (checkpoint at repo root). STRICT mode is designed to minimize disclosure, but a findings report still reveals that controls fail and where — handle it like any security assessment (see *Residual risk* below).

**Time:** 2–5 minutes for a typical repo in Claude Code (parallel scans). Longer outside Claude Code.
**Time:** a few minutes for a typical repo.<!-- cc-only:start --> In Claude Code the 21 scans run in parallel, so it's faster (2–5 minutes); longer outside Claude Code.<!-- cc-only:end -->

---

Expand Down Expand Up @@ -646,7 +646,7 @@ Run value-bearing patterns exclusively via Bash with ripgrep's `--replace ''`, w
rg -n -o --replace '' --pcre2 'PATTERN' <files>
```

The output stream is `path:line:` only — there is no matched content to leak, because rg never emitted it. Do **not** use the Grep tool's `output_mode="content"` on value-bearing patterns; do **not** pipe rg output through a second process that could re-expose the line. The guarantee is now *structural* (rg erased it), not *behavioral* (you remembered not to write it).
The output stream is `path:line:` only — there is no matched content to leak, because rg never emitted it. Do **not** pipe rg output through a second process that could re-expose the line.<!-- cc-only:start --> In Claude Code specifically, do **not** use the Grep tool's `output_mode="content"` on value-bearing patterns.<!-- cc-only:end --> The guarantee is now *structural* (rg erased it), not *behavioral* (you remembered not to write it).

**V2 — Render evidence:**
```
Expand Down
9 changes: 9 additions & 0 deletions dsgai_scanner_tool/integrations/dsgai-secret-scan.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,15 @@ if ! command -v rg >/dev/null 2>&1; then
exit 0
fi

# The patterns need PCRE2. Without it, `rg --pcre2` errors on every file and the
# check would silently pass (fail-open on a security gate). Probe once and skip
# loudly instead.
if ! printf 'x' | rg --pcre2 -q 'x' 2>/dev/null; then
echo "warning: this ripgrep lacks PCRE2 support; skipping DSGAI02 pre-commit check"
echo " install a PCRE2-enabled ripgrep, or use the gitleaks pack instead"
exit 0
fi

# Collect staged files with a NUL-delimited read loop (bash-3.2 safe — no
# mapfile). Filter extensions with a case statement (no `grep -z`, which BSD
# grep lacks). Paths with spaces or newlines are handled correctly.
Expand Down
4 changes: 3 additions & 1 deletion dsgai_scanner_tool/integrations/gitleaks/dsgai.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ keywords = ["anthropic", "sk-ant-"]
id = "dsgai-named-credential"
description = "DSGAI02 — named LLM/cloud/vector-store credential assignment, quote-optional"
regex = '''(?i)(COHERE_API_KEY|GOOGLE_API_KEY|HF_TOKEN|HUGGINGFACE_TOKEN|AWS_ACCESS_KEY_ID|AZURE_OPENAI_KEY|GCP_SERVICE_ACCOUNT_KEY|QDRANT_API_KEY|PINECONE_API_KEY|WEAVIATE_API_KEY|MILVUS_TOKEN|LANGSMITH_API_KEY|LANGFUSE_PUBLIC_KEY|LANGFUSE_SECRET_KEY)\s*[:=]\s*["']?[A-Za-z0-9_\-]{16,}'''
keywords = ["api_key", "token", "qdrant", "pinecone", "weaviate", "langfuse", "langsmith"]
# "key" (case-insensitive) covers AZURE_OPENAI_KEY / GCP_SERVICE_ACCOUNT_KEY /
# AWS_ACCESS_KEY_ID etc.; the keyword prefilter previously dropped those three.
keywords = ["key", "token", "qdrant", "pinecone", "weaviate", "langfuse", "langsmith"]

[[rules]]
id = "dsgai-token-prefix"
Expand Down
2 changes: 1 addition & 1 deletion dsgai_scanner_tool/rules/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ deterministic CLI loads with the standard library only — no PyYAML at runtime)
| `remediation` | no | fix guidance |
| `references` | no | CVE IDs / links |
| `subtract` | no | rule IDs whose match on the same line cancels this hit (e.g. `torch.load(` minus `weights_only=True`) |
| `requires_nearby` | no | compound proximity logic: `{rule\|rules, lines\|scope, absent}` |
| `requires_nearby` | no | compound proximity logic: `{rule\|rules\|pattern, lines\|scope, absent}` — `pattern` is a raw corroborating regex (e.g. P12.1 requires an LLM call within 30 lines) |
| `gated_on` | no | only evaluated when the stack is detected: `multimodal`, `synthetic_data`, `labeling` |
| `notes` | no | control-level prose that isn't yet fully formalized |

Expand Down
Loading