Skip to content

Commit 4079250

Browse files
babenekbabenek
authored
sql_password markup (#187)
* sql_password markup * "SQL Password" rule requires 4 chars minimum * upd * BM empty scores * add markup in new files * BM zero markup scores
1 parent 8a2e470 commit 4079250

26 files changed

+103
-34
lines changed

‎.ci/benchmark.txt

+22-21
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
META MD5 d0a224099b6f47bb8948c372b8fc6144
2-
DATA MD5 0c336686c107937f85997d599484de04
3-
DATA: 16329853 interested lines. MARKUP: 59550 items
1+
META MD5 ea6bfd4ab65bd77b9e769ffdfcb5dc2a
2+
DATA MD5 7249009ff1484bfef2e45b9fc62c8d6b
3+
DATA: 16334259 interested lines. MARKUP: 59607 items
44
FileType FileNumber ValidLines Positives Negatives Templates
55
--------------- ------------ ------------ ----------- ----------- -----------
6-
193 28288 69 415 90
6+
194 28342 71 415 90
77
.1 2 641 2 5
88
.admx 1 26 1
99
.adoc 1 158 13 5 1
@@ -15,7 +15,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
1515
.bat 5 248 2 14 2
1616
.bats 15 2804 14 49 9
1717
.bazel 3 424 8
18-
.build 2 40 3
18+
.build 2 40 1 2
1919
.bundle 4 1512 476
2020
.bzl 3 2503 11
2121
.c 179 284009 8 774 5
@@ -35,7 +35,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
3535
.creds 1 10 1 1
3636
.crlf 1 27 1
3737
.crt 2 4979 211
38-
.cs 266 79474 153 872 95
38+
.cs 266 79474 156 872 95
3939
.cshtml 5 180 12
4040
.csp 3 379 9
4141
.csproj 1 14 1
@@ -55,15 +55,15 @@ FileType FileNumber ValidLines Positives Negatives Templat
5555
.erb 13 323 26
5656
.erl 4 96 7
5757
.ex 25 4968 5 98 5
58-
.example 17 1838 74 36 51
58+
.example 18 1863 74 37 51
5959
.exs 24 4842 8 187 4
6060
.ext 5 211 1 4 2
6161
.fsproj 1 75 1 2
6262
.g4 2 201 2
6363
.gd 1 37 1
6464
.gml 3 3075 16
6565
.gni 3 5017 19
66-
.go 1075 565673 652 4090 748
66+
.go 1081 568661 654 4097 748
6767
.golden 5 1168 1 13 29
6868
.gradle 45 3265 2 90 100
6969
.graphql 7 420 13
@@ -85,7 +85,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
8585
.java 613 133184 345 1325 171
8686
.jenkinsfile 1 58 2 6
8787
.jinja2 1 64 2
88-
.js 653 532652 527 2450 316
88+
.js 653 532652 528 2450 316
8989
.json 843 13045846 1076 10012 139
9090
.jsp 13 3202 1 37
9191
.jsx 7 857 19
@@ -113,7 +113,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
113113
.markdown 3 139 3 1
114114
.markerb 3 12 3
115115
.marko 1 21 2
116-
.md 671 148304 719 2340 614
116+
.md 674 148660 733 2340 614
117117
.mdx 3 549 7
118118
.mjml 1 18 1
119119
.mjs 22 4424 71 333
@@ -135,7 +135,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
135135
.patch 4 109405 4 27
136136
.pbxproj 1 941 1
137137
.pem 48 1169 47 8
138-
.php 370 75658 114 1451 71
138+
.php 371 75676 114 1452 71
139139
.pl 16 14727 7 33
140140
.pm 3 744 7
141141
.po 3 2994 15
@@ -153,7 +153,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
153153
.pug 2 193 2
154154
.purs 1 69 4
155155
.pxd 1 150 4 2
156-
.py 886 290215 665 3236 726
156+
.py 886 290215 667 3236 726
157157
.pyi 4 1361 9
158158
.pyp 1 167 1
159159
.pyx 2 1094 23
@@ -172,20 +172,20 @@ FileType FileNumber ValidLines Positives Negatives Templat
172172
.rs 31 9855 2 234 11
173173
.rsc 1 691 1
174174
.rsp 16 7101 20 10 27
175-
.rst 86 33980 71 305 68
175+
.rst 87 34002 72 305 68
176176
.rules 1 6 2
177177
.sample 2 25 3 4 4
178178
.sbt 3 570 5 2
179179
.scala 39 5028 22 99
180180
.scss 16 8553 32 1
181181
.secrets 1 11 1
182-
.sh 142 21518 58 464 23
182+
.sh 147 22139 64 464 23
183183
.slim 1 153 1 2
184184
.smali 1 775 18
185185
.snap 3 1708 9 29 2
186186
.spec 2 332 2
187187
.spin 1 565 1
188-
.sql 25 6456 124 56 3
188+
.sql 26 6468 129 57 3
189189
.storyboard 20 1802 339
190190
.strings 20 1240 137
191191
.stub 3 84 6
@@ -194,7 +194,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
194194
.svg 1 638 12
195195
.t 9 1767 15 44 13
196196
.td 2 14002 6
197-
.template 19 1633 4 35 11
197+
.template 19 1633 4 36 11
198198
.test 2 24 22 1 4
199199
.testsettings 1 21 1 10
200200
.tf 21 1377 2 30 2
@@ -209,7 +209,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
209209
.ts 581 106648 137 1774 203
210210
.tsx 54 7914 1 113 5
211211
.ttar 1 452 1
212-
.txt 266 76254 5144 5094 46
212+
.txt 268 76325 5148 5094 46
213213
.utf8 1 77 2
214214
.vsixmanifest 1 36 1
215215
.vsmdi 1 6 2
@@ -219,10 +219,10 @@ FileType FileNumber ValidLines Positives Negatives Templat
219219
.xib 11 503 164
220220
.xsl 1 311 1
221221
.yaml 136 18591 123 341 42
222-
.yml 418 36057 523 910 375
222+
.yml 420 36296 545 910 375
223223
.zsh 6 872 12
224224
.zsh-theme 1 97 1
225-
TOTAL: 10003 16329853 11874 46613 5067
225+
TOTAL: 10026 16334259 11937 46623 5067
226226
credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0
227227
Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1
228228
------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ----
@@ -261,7 +261,8 @@ Key 3912 15714 485
261261
Nonce 93 49 0 0 0 49 93 0.000000 1.000000 0.345070 0.000000
262262
Other 9 7447 5 0 0 7452 9 0.000000 1.000000 0.998794 0.000000
263263
PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000
264-
Password 1887 7535 2663 0 0 10198 1887 0.000000 1.000000 0.843856 0.000000
264+
Password 1906 7532 2663 0 0 10195 1906 0.000000 1.000000 0.842492 0.000000
265+
SQL Password 44 13 0 0 0 13 44 0.000000 1.000000 0.228070 0.000000
265266
Salesforce Credentials 2 0 0 0 0 0 2 1.000000 0.000000 0.000000
266267
Salt 47 76 1 0 0 77 47 0.000000 1.000000 0.620968 0.000000
267268
Secret 1297 1576 802 0 0 2378 1297 0.000000 1.000000 0.647075 0.000000
@@ -273,4 +274,4 @@ Token 643 4171 454
273274
Twilio Credentials 30 39 0 0 0 39 30 0.000000 1.000000 0.565217 0.000000
274275
URL Credentials 210 157 215 0 0 372 210 0.000000 1.000000 0.639175 0.000000
275276
UUID 1075 265 0 0 0 265 1075 0.000000 1.000000 0.197761 0.000000
276-
11874 46613 5067 0 0 0 46613 11874 0.000000 1.000000 0.796981 0.000000
277+
11937 46623 5067 0 0 0 46623 11937 0.000000 1.000000 0.796158 0.000000

‎.github/workflows/review.yml

+2
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,8 @@ jobs:
7373
# skip extra hints from git
7474
git config --global init.defaultBranch work
7575
python download_data.py --data_dir data --jobs $(nproc)
76+
# quick review with check only to detect errors in console
77+
python review_data.py --check
7678
python review_data.py &>review_head.txt
7779
ansi2html --style 'pre {font-family: monospace; font-size: large}' <review_head.txt >review_head.html
7880

‎meta/0436af4a.csv

+3
Original file line numberDiff line numberDiff line change
@@ -181,3 +181,6 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
181181
1371381,6d6c3e78,GitHub,0436af4a,data/0436af4a/src/6d6c3e78.cs,151,151,F,F,51,61,F,F,,,,,0.0,0,F,F,F,Auth
182182
1381055,8a1f6cd5,GitHub,0436af4a,data/0436af4a/test/8a1f6cd5.cs,269,269,F,F,,,F,F,,,,,0.0,0,F,F,F,Password
183183
1479604,e4cc8527,GitHub,0436af4a,data/0436af4a/src/e4cc8527.cs,26,26,F,F,59,77,F,F,,,,,0.0,0,F,F,F,Auth
184+
1480722,92b8ee86,GitHub,0436af4a,data/0436af4a/test/92b8ee86.cs,115,115,T,F,41,50,F,F,,,,,0.0,0,F,F,F,Password
185+
1480723,92b8ee86,GitHub,0436af4a,data/0436af4a/test/92b8ee86.cs,150,150,T,F,41,50,F,F,,,,,0.0,0,F,F,F,Password
186+
1480724,92b8ee86,GitHub,0436af4a,data/0436af4a/test/92b8ee86.cs,312,312,T,F,41,50,F,F,,,,,0.0,0,F,F,F,Password

‎meta/0a0d22aa.csv

+1
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
2424
31800,5f40d16d,GitHub,0a0d22aa,data/0a0d22aa/test/5f40d16d.go,45,45,Template,F,17,73,F,F,,,,,0.0,56,,F,F,Password
2525
1113091,c50315a3,GitHub,0a0d22aa,data/0a0d22aa/test/c50315a3.go,15,15,F,F,17,33,F,F,,,,,0.0,0,F,F,F,Key
2626
1031800,5f40d16d,GitHub,0a0d22aa,data/0a0d22aa/test/5f40d16d.go,15,15,Template,F,17,61,F,F,,,,,0.0,56,,F,F,Password
27+
1480725,fea85211,GitHub,0a0d22aa,data/0a0d22aa/test/fea85211.go,34,34,T,F,16,31,F,F,,,,,0.0,0,F,F,F,Password

‎meta/255bae6f.csv

+1
Original file line numberDiff line numberDiff line change
@@ -334,3 +334,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
334334
135000,873d5f9d,GitHub,255bae6f,data/255bae6f/src/873d5f9d.rb,16,16,T,F,25,33,F,F,Any,,,Secret,2.5,8,F,F,F,Password
335335
137872,cc66396f,GitHub,255bae6f,data/255bae6f/src/cc66396f.rb,34,34,F,F,46,50,F,F,,,,,0.0,0,F,F,F,Password
336336
137873,cc66396f,GitHub,255bae6f,data/255bae6f/src/cc66396f.rb,85,85,F,F,46,50,F,F,,,,,0.0,0,F,F,F,Password
337+
1480677,a5f87782,GitHub,255bae6f,data/255bae6f/other/a5f87782.md,444,444,T,F,76,88,F,F,,,,,0.0,0,F,F,F,SQL Password

‎meta/5f62aae4.csv

+1
Original file line numberDiff line numberDiff line change
@@ -171,3 +171,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
171171
130419,6aafcdf1,GitHub,5f62aae4,data/5f62aae4/test/6aafcdf1.py,147,147,F,F,,,F,F,,,,,0.00,,F,F,F,Password
172172
130420,fe884fa6,GitHub,5f62aae4,data/5f62aae4/src/fe884fa6.less,239,239,F,F,,,F,F,,,,,0.00,,F,F,F,Password
173173
132795,2b8eecd7,GitHub,5f62aae4,data/5f62aae4/test/2b8eecd7.py,71,71,F,F,,,F,F,,,,,0,0,F,F,F,Password
174+
1480656,0aad1ebd,GitHub,5f62aae4,data/5f62aae4/test/0aad1ebd.rst,24,24,T,F,48,63,F,F,,,,,0.0,0,F,F,F,SQL Password

‎meta/638eb0c6.csv

+2
Original file line numberDiff line numberDiff line change
@@ -131,3 +131,5 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
131131
1339642,c2e4c1dd,GitHub,638eb0c6,data/638eb0c6/test/c2e4c1dd.snap,67,67,T,F,18,54,F,F,,,,,0.0,0,F,F,F,UUID
132132
1479564,d4451bab,GitHub,638eb0c6,data/638eb0c6/test/d4451bab.md,802,802,T,F,15,30,F,F,,,,,0.0,0,F,F,F,CMD Secret
133133
1480600,51e4f558,GitHub,638eb0c6,data/638eb0c6/src/51e4f558.lock,7816,7816,F,F,30,62,F,F,,,,,0.0,0,F,F,F,Bitbucket Client ID:Bitbucket Client Secret
134+
1480657,04b06f40,GitHub,638eb0c6,data/638eb0c6/other/04b06f40.md,112,112,T,F,42,52,F,F,,,,,0.0,0,F,F,F,SQL Password
135+
1480658,eca12c0a,GitHub,638eb0c6,data/638eb0c6/other/eca12c0a.md,108,108,T,F,52,62,F,F,,,,,0.0,0,F,F,F,SQL Password

‎meta/7c7558cf.csv

+1
Original file line numberDiff line numberDiff line change
@@ -60,3 +60,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
6060
1339749,eec6ccc1,GitHub,7c7558cf,data/7c7558cf/test/eec6ccc1.py,86,86,T,F,18,54,F,F,,,,,0.0,0,F,F,F,UUID
6161
1339750,eec6ccc1,GitHub,7c7558cf,data/7c7558cf/test/eec6ccc1.py,88,88,T,F,18,54,F,F,,,,,0.0,0,F,F,F,UUID
6262
1339751,eec6ccc1,GitHub,7c7558cf,data/7c7558cf/test/eec6ccc1.py,89,89,T,F,18,54,F,F,,,,,0.0,0,F,F,F,UUID
63+
1480682,1ea02434,GitHub,7c7558cf,data/7c7558cf/other/1ea02434,29,29,T,F,50,54,F,F,,,,,0.0,0,F,F,F,SQL Password

‎meta/848ad017.csv

+1
Original file line numberDiff line numberDiff line change
@@ -158,3 +158,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
158158
133801,4d714c23,GitHub,848ad017,data/848ad017/other/4d714c23,450,450,F,F,,,F,F,,,,,0,0,F,F,F,Auth
159159
133802,b3356305,GitHub,848ad017,data/848ad017/other/b3356305.md,673,673,F,F,,,F,F,,,,,0,0,F,F,F,Auth
160160
133803,5f2dd11c,GitHub,848ad017,data/848ad017/src/5f2dd11c.yml,1041,1041,F,F,,,F,F,,,,,0,0,F,F,F,Certificate
161+
1480659,b3356305,GitHub,848ad017,data/848ad017/other/b3356305.md,250,250,T,F,49,57,F,F,,,,,0.0,0,F,F,F,SQL Password

‎meta/850c2319.csv

+4
Original file line numberDiff line numberDiff line change
@@ -111,4 +111,8 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
111111
133804,a746be65,GitHub,850c2319,data/850c2319/src/a746be65.go,642,642,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key
112112
133805,a746be65,GitHub,850c2319,data/850c2319/src/a746be65.go,643,643,F,F,-1,-1,F,F,,,,,0.0,0,F,F,F,Key
113113
1479603,5ac4f856,GitHub,850c2319,data/850c2319/other/5ac4f856.md,28,28,T,F,15,25,F,F,,,,,0.0,0,F,F,F,Password
114+
1480685,3245bad3,GitHub,850c2319,data/850c2319/other/3245bad3.md,32,32,T,F,43,53,F,F,,,,,0.0,0,F,F,F,SQL Password
115+
1480686,ece869a3,GitHub,850c2319,data/850c2319/other/ece869a3.md,88,88,T,F,43,53,F,F,,,,,0.0,0,F,F,F,SQL Password
116+
1480687,f6959391,GitHub,850c2319,data/850c2319/test/f6959391.go,616,616,T,F,76,82,F,F,,,,,0.0,0,F,F,F,SQL Password
114117

118+
1480715,f6959391,GitHub,850c2319,data/850c2319/test/f6959391.go,616,616,F,F,106,112,F,F,,,,,0.0,0,F,F,F,SQL Password

‎meta/8cda00f3.csv

+2-1
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
241241
22093,f41b6cf3,GitHub,8cda00f3,data/8cda00f3/test/f41b6cf3.go,147,147,F,F,,,F,F,,,,,0,0,F,F,F,Key
242242
22099,619e6100,GitHub,8cda00f3,data/8cda00f3/src/619e6100.go,91,91,F,F,,,F,F,,,,,0,0,F,F,F,Auth
243243
22252,4a104eb3,GitHub,8cda00f3,data/8cda00f3/other/4a104eb3.md,129,129,F,F,,,F,F,,,,,0,0,F,F,F,Auth
244-
22981,4f06c21f,GitHub,8cda00f3,data/8cda00f3/src/4f06c21f.template,1,1,F,F,,,F,F,,,,,0,0,F,F,F,Password
244+
22981,4f06c21f,GitHub,8cda00f3,data/8cda00f3/src/4f06c21f.template,1,1,F,F,,,F,F,,,,,0,0,F,F,F,Password:SQL Password
245245
23061,df343c91,GitHub,8cda00f3,data/8cda00f3/src/df343c91.go,51,51,F,F,,,F,F,,,,,0,0,F,F,F,API
246246
23125,e41364c6,GitHub,8cda00f3,data/8cda00f3/src/e41364c6.go,83,83,F,F,,,F,F,,,,,0,0,F,F,F,Password
247247
23126,e41364c6,GitHub,8cda00f3,data/8cda00f3/src/e41364c6.go,78,78,F,F,,,F,F,,,,,0,0,F,F,F,Password
@@ -914,3 +914,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
914914
1340770,4cf2897e,GitHub,8cda00f3,data/8cda00f3/src/4cf2897e.go,16,16,T,F,22,58,F,F,,,,,0.0,0,F,F,F,UUID
915915
1479363,0a7921b3,GitHub,8cda00f3,data/8cda00f3/test/0a7921b3.go,155,155,F,F,,,F,F,,,,,0.0,0,F,F,F,Auth
916916
1479364,8bb83972,GitHub,8cda00f3,data/8cda00f3/src/8bb83972.go,146,146,F,F,,,F,F,,,,,0.0,0,F,F,F,Auth
917+
1480661,07026345,GitHub,8cda00f3,data/8cda00f3/other/07026345.md,45,45,T,F,42,50,F,F,,,,,0.0,0,F,F,F,SQL Password

0 commit comments

Comments
 (0)