JWT filter check improve

credsweeper/filters/value_json_web_token_check.py

@@ -6,6 +6,7 @@
 from credsweeper.credentials import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
 from credsweeper.filters import Filter
+from credsweeper.utils import Util
 
 
 class ValueJsonWebTokenCheck(Filter):
@@ -33,9 +34,13 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
             return True
         with contextlib.suppress(Exception):
             delimiter_pos = line_data.value.find(".")
-            # jwt token. '.' must be always in given data, according regex in rule
-            value = line_data.value[:delimiter_pos]
-            decoded = base64.b64decode(value)
+            # JWT token. '.' MAY be always in given data
+            if 0 <= delimiter_pos:
+                value = line_data.value[:delimiter_pos]
+            else:
+                value = line_data.value
+            # https://www.rfc-editor.org/rfc/rfc7515.txt - padding is optional
+            decoded = Util.decode_base64(value, padding_safe=True, urlsafe_detect=True)
             if header := json.loads(decoded):
                 if "alg" in header or "typ" in header:
                     return False

credsweeper/rules/config.yaml

@@ -268,7 +268,7 @@
   severity: medium
   type: pattern
   values:
-    - (^|[^.0-9A-Za-z_/+-])(?P<value>eyJ[A-Za-z0-9=_-]{13,}(\.[A-Za-z0-9-_.+\/=]+)?)
+    - (^|[^.0-9A-Za-z_/+-])(?P<value>eyJ[=A-Za-z0-9_-]{13,}[.=A-Za-z0-9_-]*)([^.0-9A-Za-z_/+-]|$)
   filter_type: GeneralPattern
   use_ml: true
   required_substrings:
@@ -622,7 +622,7 @@
   severity: high
   type: pattern
   values:
-    - (^|[^.0-9A-Za-z_/+-])(?P<value>eyJ[A-Za-z0-9_=-]{50,500}\.eyJ[A-Za-z0-9_=-]+\.[A-Za-z0-9_=-]+)
+    - (^|[^.0-9A-Za-z_/+-])(?P<value>eyJ[A-Za-z0-9_=-]{50,500}\.eyJ[A-Za-z0-9_=-]+\.[A-Za-z0-9_=-]+)([^.0-9A-Za-z_/+-]|$)
   filter_type:
     - ValueJsonWebTokenCheck
   required_substrings:

credsweeper/utils/util.py

@@ -1,4 +1,5 @@
 import ast
+import base64
 import json
 import logging
 import math
@@ -613,3 +614,17 @@ def parse_python(source: str) -> List[Any]:
         src = ast.parse(source)
         result = Util.ast_to_dict(src)
         return result
+
+    @staticmethod
+    def decode_base64(text: str, padding_safe: bool = False, urlsafe_detect=False) -> bytes:
+        """decode text to bytes with / without padding detect and urlsafe symbols"""
+        value = text
+        if padding_safe:
+            pad_num = 0x3 & len(value)
+            if pad_num:
+                value += '=' * (4 - pad_num)
+        if urlsafe_detect and '-' in value or '_' in value:
+            decoded = base64.urlsafe_b64decode(value)
+        else:
+            decoded = base64.standard_b64decode(value)
+        return decoded

tests/filters/test_value_json_web_token_check.py

@@ -11,17 +11,20 @@ class TestValueJsonWebTokenCheck:
 
     @pytest.mark.parametrize("line", ["12345:asbdsa:28yd"])
     def test_value_jwt_check_p(self, file_path: pytest.fixture, line: str) -> None:
+        jwt_like_line = base64.urlsafe_b64encode('{"typ":"JWT", "dummy": false}'.encode('ascii')).decode('ascii')
+        jwt_line_data = get_line_data(file_path, line=f"{jwt_like_line}", pattern=LINE_VALUE_PATTERN)
+        assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is False
         encoded_line = base64.b64encode(line.encode('ascii')).decode('ascii')
-        jwt_like_line = base64.b64encode('{"typ":"JWT", "dummy": false}'.encode('ascii')).decode('ascii')
         jwt_line_data = get_line_data(file_path, line=f"{jwt_like_line}.{encoded_line}", pattern=LINE_VALUE_PATTERN)
         assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is False
-        # partially line
+        assert '=' in jwt_like_line  # just demonstrate that encoded header contains padding symbol
+        jwt_like_line = jwt_like_line.replace('=', '')
         jwt_line_data = get_line_data(file_path, line=f"{jwt_like_line}.AnyTailOfString", pattern=LINE_VALUE_PATTERN)
         assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is False
 
     @pytest.mark.parametrize("line", ["1234f:asbdsa:28yd"])
     def test_value_jwt_check_n(self, file_path: pytest.fixture, line: str) -> None:
-        encoded_line = base64.b64encode(line.encode('ascii')).decode('ascii')
+        encoded_line = base64.urlsafe_b64encode(line.encode('ascii')).decode('ascii')
         jwt_line_data = get_line_data(file_path, line=f"eyJungle.{encoded_line}", pattern=LINE_VALUE_PATTERN)
         assert ValueJsonWebTokenCheck().run(jwt_line_data, DUMMY_ANALYSIS_TARGET) is True
         jwt_line_data = get_line_data(file_path, line="eyJungle", pattern=LINE_VALUE_PATTERN)

tests/samples/azure_access_token

@@ -1 +1,2 @@
-eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSJ9.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd
+eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEpsWSIsImtpZCI6Imk2bEdrM0ZaenhSY1ViMkMzbkVRN3N5SEoifQo.eyJhdWQiOiJlZjFkYTlkNC1mZjc3LTRjM2UtYTAwNS04NDBjM2Y4MzA3NDUiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC9mYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTUyMjIyOS8iLCJpYXQiOjE1MzcyMzMxMDYsIm5iZiI6MTUzNzIzMzEwNiwiZXhwIjoxNTM3MjM3MDA2LCJhY3IiOiIxIiwiYWlvIjoiQVhRQWkvOElBQUFBRm0rRS9RVEcrZ0ZuVnhMaldkdzhLKzYxQUdyU091TU1GNmViYU1qN1hPM0libUQzZkdtck95RCtOdlp5R24yVmFUL2tES1h3NE1JaHJnR1ZxNkJuOHdMWG9UMUxrSVorRnpRVmtKUFBMUU9WNEtjWHFTbENWUERTL0RpQ0RnRTIyMlRJbU12V05hRU1hVU9Uc0lHdlRRPT0iLCJhbXIiOlsid2lhIl0sImFwcGlkIjoiNzVkYmU3N2YtMTBhMy00ZTU5LTg1ZmQtOGMxMjc1NDRmMTdjIiwiYXBwaWRhY3IiOiIwIiwiZW1haWwiOiJBYmVMaUBtaWNyb3NvZnQuY29tIiwiZmFtaWx5X25hbWUiOiJMaW5jb2xuIiwiZ2l2ZW5fbmFtZSI6IkFiZSAoTVNGVCkiLCJpZHAiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMjIyNDcvIiwiaXBhZGRyIjoiMjIyLjIyMi4yMjIuMjIiLCJuYW1lIjoiYWJlbGkiLCJvaWQiOiIwMjIyM2I2Yi1hYTFkLTQyZDQtOWVjMC0xYjJiYjkxOTQ0MzgiLCJyaCI6IkkiLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJsM19yb0lTUVUyMjJiVUxTOXlpMmswWHBxcE9pTXo1SDNaQUNvMUdlWEEiLCJ0aWQiOiJmYTE1ZDY5Mi1lOWM3LTQ0NjAtYTc0My0yOWYyOTU2ZmQ0MjkiLCJ1bmlxdWVfbmFtZSI6ImFiZWxpQG1pY3Jvc29mdC5jb20iLCJ1dGkiOiJGVnNHeFlYSTMwLVR1aWt1dVVvRkFBIiwidmVyIjoiMS4wIn0.D3H6pMUtQnoJAGq6AHd
+