testfix

Samsung · Nov 20, 2023 · 4c63169 · 4c63169
1 parent 8fb28e0
commit 4c63169
Show file tree

Hide file tree

Showing 11 changed files with 52 additions and 643 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -22,7 +22,8 @@ jobs:
     - name: Checkout CredData
       uses: actions/checkout@v3
       with:
-        repository: Samsung/CredData
+        ref: ecpkey
+        repository: babenek/CredData
 
     - name: Cache data
       id: cache-data

diff --git a/credsweeper/app.py b/credsweeper/app.py
@@ -374,7 +374,6 @@ def export_results(self) -> None:
 
         if self.sort_output:
             credentials.sort(key=lambda x: (  #
-                x.line_data_list[0].line,  #
                 x.line_data_list[0].path,  #
                 x.line_data_list[0].line_num,  #
                 x.severity,  #

diff --git a/credsweeper/rules/config.yaml b/credsweeper/rules/config.yaml
@@ -383,7 +383,7 @@
   min_line_len: 72
 
 - name: PEM Private Key
-  severity: critical
+  severity: high
   type: pem_key
   values:
     - (?P<value>-----BEGIN\s(?!ENCRYPTED)[^-]*PRIVATE[^-]*KEY[^-]*-----(.+-----END[^-]+-----)?)

diff --git a/credsweeper/scanner/scanner.py b/credsweeper/scanner/scanner.py
@@ -149,8 +149,8 @@ def scan(self, provider: ContentProvider) -> List[Candidate]:
 
             if not (matched_keyword or matched_pem_key or matched_pattern or matched_multi):
                 # target may be skipped only with length because not all rules have required_substrings
-                # logger.debug("Skip too short (%d) line %s:%d", target_line_stripped_len, target.file_path,
-                #              target.line_num)
+                logger.debug("Skip too short (%d) line %s:%d", target_line_stripped_len, target.file_path,
+                             target.line_num)
                 continue
 
             # use lower case for required substring

diff --git a/tests/__init__.py b/tests/__init__.py
@@ -4,19 +4,19 @@
 SAMPLES_FILES_COUNT: int = 126
 
 # credentials count after scan
-SAMPLES_CRED_COUNT: int = 384
-SAMPLES_CRED_LINE_COUNT: int = 405
+SAMPLES_CRED_COUNT: int = 383
+SAMPLES_CRED_LINE_COUNT: int = 402
 
 # credentials count after post-processing
-SAMPLES_POST_CRED_COUNT: int = 298
+SAMPLES_POST_CRED_COUNT: int = 297
 
 # with option --doc
-SAMPLES_IN_DOC = 432
+SAMPLES_IN_DOC = 431
 
 # archived credentials that are not found without --depth
 SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 24
 SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 16
-SAMPLES_IN_DEEP_3 = SAMPLES_IN_DEEP_2 + 3
+SAMPLES_IN_DEEP_3 = SAMPLES_IN_DEEP_2 + 2
 
 # well known string with all latin letters
 AZ_DATA = b"The quick brown fox jumps over the lazy dog"

diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json
@@ -6763,280 +6763,22 @@
         "severity": "high",
         "line_data_list": [
             {
-                "line": "char pk[] = \"-----BEGIN RSA PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----\\n\";",
+                "line": "char pk[] = \"-----BEGIN OPENSSH PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----\\n\";",
                 "line_num": 12,
                 "path": "tests/samples/pem_key",
                 "info": "tests/samples/pem_key|RAW",
-                "value": "-----BEGIN RSA PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----",
+                "value": "-----BEGIN OPENSSH PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----",
                 "value_start": 13,
-                "value_end": 209,
+                "value_end": 213,
                 "variable": null,
                 "entropy_validation": {
                     "iterator": "BASE64_CHARS",
-                    "entropy": 5.029849490561501,
+                    "entropy": 5.054323782040971,
                     "valid": true
                 }
             }
         ]
     },
-    {
-        "api_validation": "NOT_AVAILABLE",
-        "ml_validation": "NOT_AVAILABLE",
-        "ml_probability": null,
-        "rule": "PEM Private Key",
-        "severity": "high",
-        "line_data_list": [
-            {
-                "line": "-----BEGIN EC PRIVATE KEY-----",
-                "line_num": 15,
-                "path": "tests/samples/pem_key",
-                "info": "tests/samples/pem_key|RAW",
-                "value": "-----BEGIN EC PRIVATE KEY-----",
-                "value_start": 0,
-                "value_end": 30,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 2.447238004178161,
-                    "valid": false
-                }
-            },
-            {
-                "line": "MHcCAQEEINGSM49oAoGCCqAEi9Hdw6KvZcWxfg2IDhA7UkpDtzzt6mNAlLUqjShUsUBBngG0u2fZqJXSsFdLd+Kx4S3Sx4cVO+AwEHoUQDQgGWx0zo6fhJ/0EAfrPzVFyFC9s18lBt3cRoEDhS3ARo6/ZOXRnPqEqA==",
-                "line_num": 16,
-                "path": "tests/samples/pem_key",
-                "info": "tests/samples/pem_key|RAW",
-                "value": "MHcCAQEEINGSM49oAoGCCqAEi9Hdw6KvZcWxfg2IDhA7UkpDtzzt6mNAlLUqjShUsUBBngG0u2fZqJXSsFdLd+Kx4S3Sx4cVO+AwEHoUQDQgGWx0zo6fhJ/0EAfrPzVFyFC9s18lBt3cRoEDhS3ARo6/ZOXRnPqEqA==",
-                "value_start": 0,
-                "value_end": 164,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 5.668600385988814,
-                    "valid": true
-                }
-            },
-            {
-                "line": "-----END EC PRIVATE KEY-----",
-                "line_num": 17,
-                "path": "tests/samples/pem_key",
-                "info": "tests/samples/pem_key|RAW",
-                "value": "-----END EC PRIVATE KEY-----",
-                "value_start": 0,
-                "value_end": 28,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 2.2896544225308593,
-                    "valid": false
-                }
-            }
-        ]
-    },
-    {
-        "api_validation": "NOT_AVAILABLE",
-        "ml_validation": "NOT_AVAILABLE",
-        "ml_probability": null,
-        "rule": "PEM Private Key",
-        "severity": "high",
-        "line_data_list": [
-            {
-                "line": "-----BEGIN RSA PRIVATE KEY-----",
-                "line_num": 1,
-                "path": "tests/samples/pem_key.apk",
-                "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW",
-                "value": "-----BEGIN RSA PRIVATE KEY-----",
-                "value_start": 0,
-                "value_end": 31,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 2.529698260800009,
-                    "valid": false
-                }
-            },
-            {
-                "line": "MIICjdsnc34fdsjkgfdjvnkjcnvenr8vjsdjvsjhdfkshkehr34hr98whfuskhdfjhqllladhsjd",
-                "line_num": 2,
-                "path": "tests/samples/pem_key.apk",
-                "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW",
-                "value": "MIICjdsnc34fdsjkgfdjvnkjcnvenr8vjsdjvsjhdfkshkehr34hr98whfuskhdfjhqllladhsjd",
-                "value_start": 0,
-                "value_end": 76,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE36_CHARS",
-                    "entropy": 3.880801520359916,
-                    "valid": true
-                }
-            },
-            {
-                "line": "wmJG8wVQZKjeGcjDOL5UlsuusFncCzWBQ7RKNUSesmQRMSGkVb1/3j+skZ6UtW+5u09lHNsj6tQ5",
-                "line_num": 3,
-                "path": "tests/samples/pem_key.apk",
-                "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW",
-                "value": "wmJG8wVQZKjeGcjDOL5UlsuusFncCzWBQ7RKNUSesmQRMSGkVb1/3j+skZ6UtW+5u09lHNsj6tQ5",
-                "value_start": 0,
-                "value_end": 76,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 5.213332901823973,
-                    "valid": true
-                }
-            },
-            {
-                "line": "1s1SPrCBkedbNf0Tp0GbMJDyR4e9T04ZZwIDAQABAoGAFijko56+qGyN8M0RVyaRAXz++xTqHBLh",
-                "line_num": 4,
-                "path": "tests/samples/pem_key.apk",
-                "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW",
-                "value": "1s1SPrCBkedbNf0Tp0GbMJDyR4e9T04ZZwIDAQABAoGAFijko56+qGyN8M0RVyaRAXz++xTqHBLh",
-                "value_start": 0,
-                "value_end": 76,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 5.325046388609145,
-                    "valid": true
-                }
-            },
-            {
-                "line": "3tx4VgMtrQ+WEgCjhoTwo23KMBAuJGSYnRmoBZM3lMfTKevIkAidPExvYCdm5dYq3XToLkkLv5L2",
-                "line_num": 5,
-                "path": "tests/samples/pem_key.apk",
-                "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW",
-                "value": "3tx4VgMtrQ+WEgCjhoTwo23KMBAuJGSYnRmoBZM3lMfTKevIkAidPExvYCdm5dYq3XToLkkLv5L2",
-                "value_start": 0,
-                "value_end": 76,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 5.267278500114894,
-                    "valid": true
-                }
-            },
-            {
-                "line": "pIIVOFMDG+KESnAFV7l2c+cnzRMW0+b6f8mR1CJzZuxVLL6Q02fvLi55/mbSYxECQQDeAw6fiIQX",
-                "line_num": 6,
-                "path": "tests/samples/pem_key.apk",
-                "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW",
-                "value": "pIIVOFMDG+KESnAFV7l2c+cnzRMW0+b6f8mR1CJzZuxVLL6Q02fvLi55/mbSYxECQQDeAw6fiIQX",
-                "value_start": 0,
-                "value_end": 76,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 5.293594289588578,
-                    "valid": true
-                }
-            },
-            {
-                "line": "GukBI4eMZZt4nscy2o12KyYner3VpoeE+Np2q+Z3pvAMd/aNzQ/W9WaI+NRfcxUJrmfPwIGm63il",
-                "line_num": 7,
-                "path": "tests/samples/pem_key.apk",
-                "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW",
-                "value": "GukBI4eMZZt4nscy2o12KyYner3VpoeE+Np2q+Z3pvAMd/aNzQ/W9WaI+NRfcxUJrmfPwIGm63il",
-                "value_start": 0,
-                "value_end": 76,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 5.405307776373748,
-                    "valid": true
-                }
-            },
-            {
-                "line": "AkEAxCL5HQb2bQr4ByorcMWm/hEP2MZzROV73yF41hPsRC9m66KrheO9HPTJuo3/9s5p+sqGxOlF",
-                "line_num": 8,
-                "path": "tests/samples/pem_key.apk",
-                "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW",
-                "value": "AkEAxCL5HQb2bQr4ByorcMWm/hEP2MZzROV73yF41hPsRC9m66KrheO9HPTJuo3/9s5p+sqGxOlF",
-                "value_start": 0,
-                "value_end": 76,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 5.372541658009631,
-                    "valid": true
-                }
-            },
-            {
-                "line": "L0NDt4SkosjgGwJAFklyR1uZ/wPJjj611cdBcztlPdqoxssQGnh85BzCj/u3WqBpE2vjvyyvyI5k",
-                "line_num": 9,
-                "path": "tests/samples/pem_key.apk",
-                "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW",
-                "value": "L0NDt4SkosjgGwJAFklyR1uZ/wPJjj611cdBcztlPdqoxssQGnh85BzCj/u3WqBpE2vjvyyvyI5k",
-                "value_start": 0,
-                "value_end": 76,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 5.308663329427085,
-                    "valid": true
-                }
-            },
-            {
-                "line": "X6zk7S0ljKtt2jny2+00VsBerQJBAJGC1Mg5Oydo5NwD6BiROrPxGo2bpTbu/fhrT8ebHkTz2epl",
-                "line_num": 10,
-                "path": "tests/samples/pem_key.apk",
-                "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW",
-                "value": "X6zk7S0ljKtt2jny2+00VsBerQJBAJGC1Mg5Oydo5NwD6BiROrPxGo2bpTbu/fhrT8ebHkTz2epl",
-                "value_start": 0,
-                "value_end": 76,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 5.425173236956998,
-                    "valid": true
-                }
-            },
-            {
-                "line": "U9VQQSQzY1oZMVX8i1m5WUTLPz2yLJIBQVdXqhMCQBGoiuSoSjafUhV7i1cEGpb88h5NBYZzWXGZ",
-                "line_num": 11,
-                "path": "tests/samples/pem_key.apk",
-                "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW",
-                "value": "U9VQQSQzY1oZMVX8i1m5WUTLPz2yLJIBQVdXqhMCQBGoiuSoSjafUhV7i1cEGpb88h5NBYZzWXGZ",
-                "value_start": 0,
-                "value_end": 76,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 5.107555270017291,
-                    "valid": true
-                }
-            },
-            {
-                "line": "37sJ5QsW+sJyoNde3xH8vdXhzU7eT82D6X/scw9RZz+/6rCJ4p0=",
-                "line_num": 12,
-                "path": "tests/samples/pem_key.apk",
-                "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW",
-                "value": "37sJ5QsW+sJyoNde3xH8vdXhzU7eT82D6X/scw9RZz+/6rCJ4p0=",
-                "value_start": 0,
-                "value_end": 52,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 5.0705380354071785,
-                    "valid": true
-                }
-            },
-            {
-                "line": "-----END RSA PRIVATE KEY-----",
-                "line_num": 13,
-                "path": "tests/samples/pem_key.apk",
-                "info": "tests/samples/pem_key.apk|ZIP|3.zip|ZIP|2.zip|ZIP|2|RAW",
-                "value": "-----END RSA PRIVATE KEY-----",
-                "value_start": 0,
-                "value_end": 29,
-                "variable": null,
-                "entropy_validation": {
-                    "iterator": "BASE64_CHARS",
-                    "entropy": 2.3783727041337137,
-                    "valid": false
-                }
-            }
-        ]
-    },
     {
         "api_validation": "NOT_AVAILABLE",
         "ml_validation": "NOT_AVAILABLE",