[skip actions] [ecpkey] 2023-11-20T20:04:49+02:00

credsweeper/app.py

@@ -374,6 +374,7 @@ def export_results(self) -> None:
 
         if self.sort_output:
             credentials.sort(key=lambda x: (  #
+                x.line_data_list[0].line,  #
                 x.line_data_list[0].path,  #
                 x.line_data_list[0].line_num,  #
                 x.severity,  #

credsweeper/filters/__init__.py

@@ -29,7 +29,6 @@
 from credsweeper.filters.value_number_check import ValueNumberCheck
 from credsweeper.filters.value_pattern_check import ValuePatternCheck
 from credsweeper.filters.value_pattern_length_check import ValuePatternLengthCheck
-from credsweeper.filters.value_pem_pattern_check import ValuePemPatternCheck
 from credsweeper.filters.value_pii_check import ValuePIICheck
 from credsweeper.filters.value_phone_check import ValuePhoneCheck
 from credsweeper.filters.value_similarity_check import ValueSimilarityCheck

credsweeper/rules/config.yaml

@@ -383,7 +383,7 @@
   min_line_len: 72
 
 - name: PEM Private Key
-  severity: high
+  severity: critical
   type: pem_key
   values:
     - (?P<value>-----BEGIN\s(?!ENCRYPTED)[^-]*PRIVATE[^-]*KEY[^-]*-----(.+-----END[^-]+-----)?)

credsweeper/scanner/scan_type/pem_key_pattern.py

@@ -1,15 +1,16 @@
+import contextlib
 import logging
 import re
 import string
-from typing import Optional, List
+from typing import List
 
-from credsweeper.common.constants import Chars, PEM_BEGIN_PATTERN, PEM_END_PATTERN, RuleType
+from credsweeper.common.constants import PEM_BEGIN_PATTERN, PEM_END_PATTERN, RuleType, Chars
 from credsweeper.config import Config
 from credsweeper.credentials import Candidate, LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import ValuePatternCheck, ValuePemPatternCheck
 from credsweeper.rules import Rule
 from credsweeper.scanner.scan_type import ScanType
+from credsweeper.utils import Util
 from credsweeper.utils.entropy_validator import EntropyValidator
 
 logger = logging.getLogger(__name__)
@@ -29,7 +30,6 @@ class PemKeyPattern(ScanType):
     wrap_characters = "\\'\";,[]#*"
     remove_characters = string.whitespace + wrap_characters
     remove_characters_plus = remove_characters + '+'
-    pem_pattern_check: Optional[ValuePatternCheck] = None
     # last line contains 4 symbols, at least
     re_value_pem = re.compile(r"(?P<value>([^-]*" + PEM_END_PATTERN +
                               r"[^-]+-----)|(([a-zA-Z0-9/+=]{64}.*)?[a-zA-Z0-9/+=]{4})+)")
@@ -50,8 +50,6 @@ def run(cls, config: Config, rule: Rule, target: AnalysisTarget) -> List[Candida
         """
         assert rule.rule_type == RuleType.PEM_KEY, \
             "Rules provided to PemKeyPattern.run should have pattern_type equal to PEM_KEY_PATTERN"
-        if not cls.pem_pattern_check:
-            cls.pem_pattern_check = ValuePemPatternCheck(config)
         if candidates := cls._get_candidates(config, rule, target):
             candidate = candidates[0]
             if pem_lines := cls.detect_pem_key(config, rule, target):
@@ -95,17 +93,20 @@ def detect_pem_key(cls, config: Config, rule: Rule, target: AnalysisTarget) -> L
                 if cls.is_leading_config_line(subline):
                     continue
                 elif PEM_END_PATTERN in subline:
-                    # Check if entropy is high enough for base64 set with padding sign
-                    entropy_validator = EntropyValidator(key_data, Chars.BASE64_CHARS)
-                    if not entropy_validator.valid:
+                    if "OPENSSH" in target.line_strip or "PGP" in target.line_strip:
+                        # Check if entropy is high enough for base64 set with padding sign
+                        entropy_validator = EntropyValidator(key_data, Chars.BASE64_CHARS)
+                        if entropy_validator.valid:
+                            return line_data
                         logger.debug("Filtered with entropy %f '%s'", entropy_validator.entropy, key_data)
-                        return []
-                    # OPENSSH format has multiple AAAAA pattern
-                    if "OPENSSH" not in target.line_strip and cls.pem_pattern_check.equal_pattern_check(key_data):
-                        logger.debug("Filtered with ValuePemPatternCheck %s", target)
-                        return []
-                    # all OK - return line data with all lines which include PEM
-                    return line_data
+                    else:
+                        with contextlib.suppress(Exception):
+                            decoded = Util.decode_base64(key_data, urlsafe_detect=True)
+                            if Util.is_asn1(decoded):
+                                # all OK - the key is not encrypted in this top level
+                                return line_data
+                        logger.debug("Filtered with non asn1 '%s'", key_data)
+                    return []
                 else:
                     sanitized_line = cls.sanitize_line(subline)
                     # PEM key line should not contain spaces or . (and especially not ...)

credsweeper/scanner/scanner.py

@@ -149,8 +149,8 @@ def scan(self, provider: ContentProvider) -> List[Candidate]:
 
             if not (matched_keyword or matched_pem_key or matched_pattern or matched_multi):
                 # target may be skipped only with length because not all rules have required_substrings
-                logger.debug("Skip too short (%d) line %s:%d", target_line_stripped_len, target.file_path,
-                             target.line_num)
+                # logger.debug("Skip too short (%d) line %s:%d", target_line_stripped_len, target.file_path,
+                #              target.line_num)
                 continue
 
             # use lower case for required substring

credsweeper/utils/util.py

@@ -414,13 +414,15 @@ def is_asn1(data: bytes) -> bool:
                 byte_len = (0x7F & length)
                 if 0x80 == length and data.endswith(b"\x00\x00"):
                     return True
-                elif 0x80 < length and byte_len < data_length:  # additional check
+                elif 0x80 < length and 1 < byte_len < data_length:  # additional check
                     len_bytes = data[2:2 + byte_len]
                     try:
                         long_size = struct.unpack(">h", len_bytes)
                     except struct.error:
                         long_size = (-1,)  # yapf: disable
                     length = long_size[0]
+                elif 0x80 < length and 1 == byte_len:  # small size
+                    length = data[2]
                 else:
                     byte_len = 0
                 return data_length == length + 2 + byte_len

tests/data/output.json

@@ -6319,7 +6319,7 @@
         "ml_validation": "NOT_AVAILABLE",
         "ml_probability": null,
         "rule": "PEM Private Key",
-        "severity": "high",
+        "severity": "critical",
         "line_data_list": [
             {
                 "line": "value = \"-----BEGIN OPENSSH PRIVATE KEY-----\" \\",
@@ -6448,20 +6448,20 @@
         "ml_validation": "NOT_AVAILABLE",
         "ml_probability": null,
         "rule": "PEM Private Key",
-        "severity": "high",
+        "severity": "critical",
         "line_data_list": [
             {
-                "line": "char pk[] = \"-----BEGIN RSA PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----\\n\";",
+                "line": "char pk[] = \"-----BEGIN OPENSSH PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----\\n\";",
                 "line_num": 12,
                 "path": "tests/samples/pem_key",
                 "info": "",
-                "value": "-----BEGIN RSA PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----",
+                "value": "-----BEGIN OPENSSH PRIVATE KEY-----\\n\\ni7aHavqQ9T2f2drU4N5WsM7aqQ97kbB/K4RUPdit+tIpqSb1GgY44yg6lckfoLMH\\najimpQ6sr9BuseERqELGE1U+Vll3izwuqr1UzCZ61gZnwPDBsD02jF038wYU6mZT\\n-----END RSA PRIVATE KEY-----",
                 "value_start": 13,
-                "value_end": 209,
+                "value_end": 213,
                 "variable": null,
                 "entropy_validation": {
                     "iterator": "BASE64_CHARS",
-                    "entropy": 5.029849490561501,
+                    "entropy": 5.054323782040971,
                     "valid": true
                 }
             }
@@ -6472,7 +6472,7 @@
         "ml_validation": "NOT_AVAILABLE",
         "ml_probability": null,
         "rule": "PEM Private Key",
-        "severity": "high",
+        "severity": "critical",
         "line_data_list": [
             {
                 "line": "-----BEGIN EC PRIVATE KEY-----",
@@ -6521,6 +6521,195 @@
             }
         ]
     },
+    {
+        "api_validation": "NOT_AVAILABLE",
+        "ml_validation": "NOT_AVAILABLE",
+        "ml_probability": null,
+        "rule": "PEM Private Key",
+        "severity": "critical",
+        "line_data_list": [
+            {
+                "line": "-----BEGIN PRIVATE KEY-----",
+                "line_num": 43,
+                "path": "tests/samples/pem_key",
+                "info": "",
+                "value": "-----BEGIN PRIVATE KEY-----",
+                "value_start": 0,
+                "value_end": 27,
+                "variable": null,
+                "entropy_validation": {
+                    "iterator": "BASE64_CHARS",
+                    "entropy": 2.391423149269947,
+                    "valid": false
+                }
+            },
+            {
+                "line": "MIIByQIBADCCAZsGCSqGSIb3DQEDATCCAYwCggGBAP//////////rfhUWKK7Spqv",
+                "line_num": 44,
+                "path": "tests/samples/pem_key",
+                "info": "",
+                "value": "MIIByQIBADCCAZsGCSqGSIb3DQEDATCCAYwCggGBAP//////////rfhUWKK7Spqv",
+                "value_start": 0,
+                "value_end": 64,
+                "variable": null,
+                "entropy_validation": {
+                    "iterator": "BASE64_CHARS",
+                    "entropy": 4.510027399432949,
+                    "valid": true
+                }
+            },
+            {
+                "line": "3FYgJz088di5xYPOLTaVqeE2QRRkM/vMk53OJJs++X0v42NjDHXY9oGyAq7EYXrT",
+                "line_num": 45,
+                "path": "tests/samples/pem_key",
+                "info": "",
+                "value": "3FYgJz088di5xYPOLTaVqeE2QRRkM/vMk53OJJs++X0v42NjDHXY9oGyAq7EYXrT",
+                "value_start": 0,
+                "value_end": 64,
+                "variable": null,
+                "entropy_validation": {
+                    "iterator": "BASE64_CHARS",
+                    "entropy": 5.288909765557392,
+                    "valid": true
+                }
+            },
+            {
+                "line": "3x7V1f1lYSQz9R9fBm7QhWNlVT3tGvO1VxNef1fJNZhPDHDg5ot34qaJ2vPv6HId",
+                "line_num": 46,
+                "path": "tests/samples/pem_key",
+                "info": "",
+                "value": "3x7V1f1lYSQz9R9fBm7QhWNlVT3tGvO1VxNef1fJNZhPDHDg5ot34qaJ2vPv6HId",
+                "value_start": 0,
+                "value_end": 64,
+                "variable": null,
+                "entropy_validation": {
+                    "iterator": "BASE64_CHARS",
+                    "entropy": 5.109069531114784,
+                    "valid": true
+                }
+            },
+            {
+                "line": "8VihNq3nNTCsyk9IOnl6vAqxgrMk+2HRCKlLssjj+7lq2rdg1/RoHU9Co945TfSu",
+                "line_num": 47,
+                "path": "tests/samples/pem_key",
+                "info": "",
+                "value": "8VihNq3nNTCsyk9IOnl6vAqxgrMk+2HRCKlLssjj+7lq2rdg1/RoHU9Co945TfSu",
+                "value_start": 0,
+                "value_end": 64,
+                "variable": null,
+                "entropy_validation": {
+                    "iterator": "BASE64_CHARS",
+                    "entropy": 5.25352441389348,
+                    "valid": true
+                }
+            },
+            {
+                "line": "Vu3nY3K7GQsHp8juCm1wngL84c334uzANATNKDQvYZFy/pzphYP/jk8SMu7ygYPD",
+                "line_num": 48,
+                "path": "tests/samples/pem_key",
+                "info": "",
+                "value": "Vu3nY3K7GQsHp8juCm1wngL84c334uzANATNKDQvYZFy/pzphYP/jk8SMu7ygYPD",
+                "value_start": 0,
+                "value_end": 64,
+                "variable": null,
+                "entropy_validation": {
+                    "iterator": "BASE64_CHARS",
+                    "entropy": 5.038909765557392,
+                    "valid": true
+                }
+            },
+            {
+                "line": "/jsbTG+tczu1/LwuwiAFxY7xg30Wg7LG80omwbLv+ohrQjhhH8/c3jVbO2UZA1u8",
+                "line_num": 49,
+                "path": "tests/samples/pem_key",
+                "info": "",
+                "value": "/jsbTG+tczu1/LwuwiAFxY7xg30Wg7LG80omwbLv+ohrQjhhH8/c3jVbO2UZA1u8",
+                "value_start": 0,
+                "value_end": 64,
+                "variable": null,
+                "entropy_validation": {
+                    "iterator": "BASE64_CHARS",
+                    "entropy": 5.061889062229568,
+                    "valid": true
+                }
+            },
+            {
+                "line": "NPTe+ZwCOGG0b8nW5skHetkdJpH39+5ZjLD6wYbZHK7+EwmFE5JwtBMMk7xDeUT0",
+                "line_num": 50,
+                "path": "tests/samples/pem_key",
+                "info": "",
+                "value": "NPTe+ZwCOGG0b8nW5skHetkdJpH39+5ZjLD6wYbZHK7+EwmFE5JwtBMMk7xDeUT0",
+                "value_start": 0,
+                "value_end": 64,
+                "variable": null,
+                "entropy_validation": {
+                    "iterator": "BASE64_CHARS",
+                    "entropy": 5.116729296672176,
+                    "valid": true
+                }
+            },
+            {
+                "line": "/URS4tdN02Ty4h5x9Uv/XK6Cq5yd9p7obSvFIjY6DavFIZebDeraHb+aQtXESE4K",
+                "line_num": 51,
+                "path": "tests/samples/pem_key",
+                "info": "",
+                "value": "/URS4tdN02Ty4h5x9Uv/XK6Cq5yd9p7obSvFIjY6DavFIZebDeraHb+aQtXESE4K",
+                "value_start": 0,
+                "value_end": 64,
+                "variable": null,
+                "entropy_validation": {
+                    "iterator": "BASE64_CHARS",
+                    "entropy": 5.15977441389348,
+                    "valid": true
+                }
+            },
+            {
+                "line": "vNBr+lPd7zwbIO4/1Z18JeQdK2bGLjf//////////wIBAgICARMEJQIjB8TRLx6q",
+                "line_num": 52,
+                "path": "tests/samples/pem_key",
+                "info": "",
+                "value": "vNBr+lPd7zwbIO4/1Z18JeQdK2bGLjf//////////wIBAgICARMEJQIjB8TRLx6q",
+                "value_start": 0,
+                "value_end": 64,
+                "variable": null,
+                "entropy_validation": {
+                    "iterator": "BASE64_CHARS",
+                    "entropy": 4.86236507332541,
+                    "valid": true
+                }
+            },
+            {
+                "line": "XYQJ0RAM+5ztVLhy9EXNdjY0EYODS7TFi5RZLE4=",
+                "line_num": 53,
+                "path": "tests/samples/pem_key",
+                "info": "",
+                "value": "XYQJ0RAM+5ztVLhy9EXNdjY0EYODS7TFi5RZLE4=",
+                "value_start": 0,
+                "value_end": 40,
+                "variable": null,
+                "entropy_validation": {
+                    "iterator": "BASE64_CHARS",
+                    "entropy": 4.834183719779189,
+                    "valid": true
+                }
+            },
+            {
+                "line": "-----END PRIVATE KEY-----",
+                "line_num": 54,
+                "path": "tests/samples/pem_key",
+                "info": "",
+                "value": "-----END PRIVATE KEY-----",
+                "value_start": 0,
+                "value_end": 25,
+                "variable": null,
+                "entropy_validation": {
+                    "iterator": "BASE64_CHARS",
+                    "entropy": 2.224609718596318,
+                    "valid": false
+                }
+            }
+        ]
+    },
     {
         "api_validation": "NOT_AVAILABLE",
         "ml_validation": "NOT_AVAILABLE",