From c81daefa927f207a54fde7ebf405f18483319190 Mon Sep 17 00:00:00 2001 From: dr-rompecabezas Date: Thu, 13 Mar 2025 15:08:27 -0400 Subject: [PATCH 1/8] Working version of sanitization helper Needs refactoring! --- debug_toolbar/utils.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/debug_toolbar/utils.py b/debug_toolbar/utils.py index dc3cc1adc..daf84d045 100644 --- a/debug_toolbar/utils.py +++ b/debug_toolbar/utils.py @@ -14,10 +14,12 @@ from django.template import Node from django.utils.html import format_html from django.utils.safestring import SafeString, mark_safe +from django.views.debug import get_default_exception_reporter_filter from debug_toolbar import _stubs as stubs, settings as dt_settings _local_data = Local() +safe_filter = get_default_exception_reporter_filter() def _is_excluded_frame(frame: Any, excluded_modules: Sequence[str] | None) -> bool: @@ -215,18 +217,34 @@ def getframeinfo(frame: Any, context: int = 1) -> inspect.Traceback: return inspect.Traceback(filename, lineno, frame.f_code.co_name, lines, index) -def get_sorted_request_variable( +def sanitize_and_sort_request_vars( variable: dict[str, Any] | QueryDict, ) -> dict[str, list[tuple[str, Any]] | Any]: """ Get a data structure for showing a sorted list of variables from the - request data. + request data with sensitive values redacted. """ + if not isinstance(variable, (dict, QueryDict)): + return {"raw": variable} + try: - if isinstance(variable, dict): - return {"list": [(k, variable.get(k)) for k in sorted(variable)]} + try: + keys = sorted(variable) + except TypeError: + keys = list(variable) + + if isinstance(variable, QueryDict): + result = [] + for k in keys: + values = variable.getlist(k) + # Return single value if there's only one, otherwise keep as list + value = values[0] if len(values) == 1 else values + result.append((k, safe_filter.cleanse_setting(k, value))) else: - return {"list": [(k, variable.getlist(k)) for k in sorted(variable)]} + result = [ + (k, safe_filter.cleanse_setting(k, variable.get(k))) for k in keys + ] + return {"list": result} except TypeError: return {"raw": variable} From d155fc4117f02fb3aef72202a5641b5998c5b033 Mon Sep 17 00:00:00 2001 From: dr-rompecabezas Date: Thu, 13 Mar 2025 15:10:23 -0400 Subject: [PATCH 2/8] Sanitize data in RequestPanel - Refactor sesssion data handling --- debug_toolbar/panels/request.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/debug_toolbar/panels/request.py b/debug_toolbar/panels/request.py index b77788637..5a24d6179 100644 --- a/debug_toolbar/panels/request.py +++ b/debug_toolbar/panels/request.py @@ -3,7 +3,7 @@ from django.utils.translation import gettext_lazy as _ from debug_toolbar.panels import Panel -from debug_toolbar.utils import get_name_from_obj, get_sorted_request_variable +from debug_toolbar.utils import get_name_from_obj, sanitize_and_sort_request_vars class RequestPanel(Panel): @@ -26,9 +26,9 @@ def nav_subtitle(self): def generate_stats(self, request, response): self.record_stats( { - "get": get_sorted_request_variable(request.GET), - "post": get_sorted_request_variable(request.POST), - "cookies": get_sorted_request_variable(request.COOKIES), + "get": sanitize_and_sort_request_vars(request.GET), + "post": sanitize_and_sort_request_vars(request.POST), + "cookies": sanitize_and_sort_request_vars(request.COOKIES), } ) @@ -59,13 +59,5 @@ def generate_stats(self, request, response): self.record_stats(view_info) if hasattr(request, "session"): - try: - session_list = [ - (k, request.session.get(k)) for k in sorted(request.session.keys()) - ] - except TypeError: - session_list = [ - (k, request.session.get(k)) - for k in request.session.keys() # (it's not a dict) - ] - self.record_stats({"session": {"list": session_list}}) + session_data = dict(request.session) + self.record_stats({"session": sanitize_and_sort_request_vars(session_data)}) From 12ef38f8b4f067d922bc8ab440d0e78fc6c97b36 Mon Sep 17 00:00:00 2001 From: dr-rompecabezas Date: Thu, 13 Mar 2025 15:12:30 -0400 Subject: [PATCH 3/8] Add tests for sanitizing sensitive data - Test handling in dicts and QueryDicts - Test string substitution --- tests/panels/test_request.py | 73 ++++++++++++++++++++++++++++++++++++ tests/test_utils.py | 62 ++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) diff --git a/tests/panels/test_request.py b/tests/panels/test_request.py index 707b50bb4..2eb7ba610 100644 --- a/tests/panels/test_request.py +++ b/tests/panels/test_request.py @@ -136,3 +136,76 @@ def test_session_list_sorted_or_not(self): self.panel.generate_stats(self.request, response) panel_stats = self.panel.get_stats() self.assertEqual(panel_stats["session"], data) + + def test_sensitive_post_data_sanitized(self): + """Test that sensitive POST data is redacted.""" + self.request.POST = {"username": "testuser", "password": "secret123"} + response = self.panel.process_request(self.request) + self.panel.generate_stats(self.request, response) + + # Check that password is redacted in panel content + content = self.panel.content + self.assertIn("username", content) + self.assertIn("testuser", content) + self.assertIn("password", content) + self.assertNotIn("secret123", content) + self.assertIn("********************", content) + + def test_sensitive_get_data_sanitized(self): + """Test that sensitive GET data is redacted.""" + self.request.GET = {"api_key": "abc123", "q": "search term"} + response = self.panel.process_request(self.request) + self.panel.generate_stats(self.request, response) + + # Check that api_key is redacted in panel content + content = self.panel.content + self.assertIn("api_key", content) + self.assertNotIn("abc123", content) + self.assertIn("********************", content) + self.assertIn("q", content) + self.assertIn("search term", content) + + def test_sensitive_cookie_data_sanitized(self): + """Test that sensitive cookie data is redacted.""" + self.request.COOKIES = {"session_id": "abc123", "auth_token": "xyz789"} + response = self.panel.process_request(self.request) + self.panel.generate_stats(self.request, response) + + # Check that auth_token is redacted in panel content + content = self.panel.content + self.assertIn("session_id", content) + self.assertIn("abc123", content) + self.assertIn("auth_token", content) + self.assertNotIn("xyz789", content) + self.assertIn("********************", content) + + def test_sensitive_session_data_sanitized(self): + """Test that sensitive session data is redacted.""" + self.request.session = {"user_id": 123, "auth_token": "xyz789"} + response = self.panel.process_request(self.request) + self.panel.generate_stats(self.request, response) + + # Check that auth_token is redacted in panel content + content = self.panel.content + self.assertIn("user_id", content) + self.assertIn("123", content) + self.assertIn("auth_token", content) + self.assertNotIn("xyz789", content) + self.assertIn("********************", content) + + def test_querydict_sanitized(self): + """Test that sensitive data in QueryDict objects is properly redacted.""" + query_dict = QueryDict("username=testuser&password=secret123&token=abc456") + self.request.GET = query_dict + response = self.panel.process_request(self.request) + self.panel.generate_stats(self.request, response) + + # Check that sensitive data is redacted in panel content + content = self.panel.content + self.assertIn("username", content) + self.assertIn("testuser", content) + self.assertIn("password", content) + self.assertNotIn("secret123", content) + self.assertIn("token", content) + self.assertNotIn("abc456", content) + self.assertIn("********************", content) diff --git a/tests/test_utils.py b/tests/test_utils.py index 26bfce005..646b6a5ad 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,6 @@ import unittest +from django.http import QueryDict from django.test import override_settings import debug_toolbar.utils @@ -8,6 +9,7 @@ get_stack, get_stack_trace, render_stacktrace, + sanitize_and_sort_request_vars, tidy_stacktrace, ) @@ -109,3 +111,63 @@ def __init__(self, value): rendered_stack_2 = render_stacktrace(stack_2_wrapper.value) self.assertNotIn("test_locals_value_1", rendered_stack_2) self.assertIn("test_locals_value_2", rendered_stack_2) + + +class SanitizeAndSortRequestVarsTestCase(unittest.TestCase): + """Tests for the sanitize_and_sort_request_vars function.""" + + def test_dict_sanitization(self): + """Test sanitization of a regular dictionary.""" + test_dict = { + "username": "testuser", + "password": "secret123", + "api_key": "abc123", + } + result = sanitize_and_sort_request_vars(test_dict) + + # Convert to dict for easier testing + result_dict = dict(result["list"]) + + self.assertEqual(result_dict["username"], "testuser") + self.assertEqual(result_dict["password"], "********************") + self.assertEqual(result_dict["api_key"], "********************") + + def test_querydict_sanitization(self): + """Test sanitization of a QueryDict.""" + query_dict = QueryDict("username=testuser&password=secret123&api_key=abc123") + result = sanitize_and_sort_request_vars(query_dict) + + # Convert to dict for easier testing + result_dict = dict(result["list"]) + + self.assertEqual(result_dict["username"], "testuser") + self.assertEqual(result_dict["password"], "********************") + self.assertEqual(result_dict["api_key"], "********************") + + def test_non_sortable_dict_keys(self): + """Test dictionary with keys that can't be sorted.""" + test_dict = { + 1: "one", + "2": "two", + None: "none", + } + result = sanitize_and_sort_request_vars(test_dict) + self.assertEqual(len(result["list"]), 3) + result_dict = dict(result["list"]) + self.assertEqual(result_dict[1], "one") + self.assertEqual(result_dict["2"], "two") + self.assertEqual(result_dict[None], "none") + + def test_querydict_multiple_values(self): + """Test QueryDict with multiple values for the same key.""" + query_dict = QueryDict("name=bar1&name=bar2&title=value") + result = sanitize_and_sort_request_vars(query_dict) + result_dict = dict(result["list"]) + self.assertEqual(result_dict["name"], ["bar1", "bar2"]) + self.assertEqual(result_dict["title"], "value") + + def test_non_dict_input(self): + """Test handling of non-dict input.""" + test_input = ["not", "a", "dict"] + result = sanitize_and_sort_request_vars(test_input) + self.assertEqual(result["raw"], test_input) From d82fe4b96eb40818f60b00b291673ef515f65548 Mon Sep 17 00:00:00 2001 From: dr-rompecabezas Date: Thu, 13 Mar 2025 15:13:19 -0400 Subject: [PATCH 4/8] Add RequestPanel sanitization to changes --- docs/changes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changes.rst b/docs/changes.rst index f11d4889e..5995c7b22 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -16,6 +16,7 @@ Pending of each target. * Avoided reinitializing the staticfiles storage during instrumentation. * Fix for exception-unhandled "forked" Promise chain in rebound window.fetch +* Added feature to cleanse sensitive data in the Request Panel. 5.0.1 (2025-01-13) ------------------ From 0100833bc4bf2501ac0d2ecc64024e904c90d853 Mon Sep 17 00:00:00 2001 From: dr-rompecabezas Date: Thu, 13 Mar 2025 15:18:57 -0400 Subject: [PATCH 5/8] Refactor sanitize_and_sort_request_vars - Use helper functions to refactor ugly-looking code --- debug_toolbar/utils.py | 48 ++++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/debug_toolbar/utils.py b/debug_toolbar/utils.py index daf84d045..64c2ae005 100644 --- a/debug_toolbar/utils.py +++ b/debug_toolbar/utils.py @@ -228,27 +228,47 @@ def sanitize_and_sort_request_vars( return {"raw": variable} try: - try: - keys = sorted(variable) - except TypeError: - keys = list(variable) - + # Get sorted keys if possible, otherwise just list them + keys = _get_sorted_keys(variable) + + # Process the variable based on its type if isinstance(variable, QueryDict): - result = [] - for k in keys: - values = variable.getlist(k) - # Return single value if there's only one, otherwise keep as list - value = values[0] if len(values) == 1 else values - result.append((k, safe_filter.cleanse_setting(k, value))) + result = _process_query_dict(variable, keys) else: - result = [ - (k, safe_filter.cleanse_setting(k, variable.get(k))) for k in keys - ] + result = _process_dict(variable, keys) + return {"list": result} except TypeError: + # If any processing fails, return raw variable return {"raw": variable} +def _get_sorted_keys(variable): + """Helper function to get sorted keys if possible.""" + try: + return sorted(variable) + except TypeError: + return list(variable) + + +def _process_query_dict(query_dict, keys): + """Process a QueryDict into a list of (key, sanitized_value) tuples.""" + result = [] + for k in keys: + values = query_dict.getlist(k) + # Return single value if there's only one, otherwise keep as list + value = values[0] if len(values) == 1 else values + result.append((k, safe_filter.cleanse_setting(k, value))) + return result + + +def _process_dict(dictionary, keys): + """Process a dictionary into a list of (key, sanitized_value) tuples.""" + return [ + (k, safe_filter.cleanse_setting(k, dictionary.get(k))) for k in keys + ] + + def get_stack(context=1) -> list[stubs.InspectStack]: """ Get a list of records for a frame and all higher (calling) frames. From 5afe3b3a8659688905c9661128188544b6736604 Mon Sep 17 00:00:00 2001 From: dr-rompecabezas Date: Thu, 13 Mar 2025 15:28:15 -0400 Subject: [PATCH 6/8] Replace 'cleanse' with 'sanitize' in changes - For consistency with the code --- docs/changes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changes.rst b/docs/changes.rst index 5995c7b22..72627338e 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -16,7 +16,7 @@ Pending of each target. * Avoided reinitializing the staticfiles storage during instrumentation. * Fix for exception-unhandled "forked" Promise chain in rebound window.fetch -* Added feature to cleanse sensitive data in the Request Panel. +* Added feature to sanitize sensitive data in the Request Panel. 5.0.1 (2025-01-13) ------------------ From b79487d05c344b0a6c1d73ab14aff4572b3a9b00 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 13 Mar 2025 19:34:57 +0000 Subject: [PATCH 7/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- debug_toolbar/utils.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/debug_toolbar/utils.py b/debug_toolbar/utils.py index 64c2ae005..0858e5047 100644 --- a/debug_toolbar/utils.py +++ b/debug_toolbar/utils.py @@ -230,13 +230,13 @@ def sanitize_and_sort_request_vars( try: # Get sorted keys if possible, otherwise just list them keys = _get_sorted_keys(variable) - + # Process the variable based on its type if isinstance(variable, QueryDict): result = _process_query_dict(variable, keys) else: result = _process_dict(variable, keys) - + return {"list": result} except TypeError: # If any processing fails, return raw variable @@ -264,9 +264,7 @@ def _process_query_dict(query_dict, keys): def _process_dict(dictionary, keys): """Process a dictionary into a list of (key, sanitized_value) tuples.""" - return [ - (k, safe_filter.cleanse_setting(k, dictionary.get(k))) for k in keys - ] + return [(k, safe_filter.cleanse_setting(k, dictionary.get(k))) for k in keys] def get_stack(context=1) -> list[stubs.InspectStack]: From ca19dc225b81f0ae5d223d4c769ad8d7b28d4f98 Mon Sep 17 00:00:00 2001 From: dr-rompecabezas Date: Sun, 16 Mar 2025 21:12:56 -0400 Subject: [PATCH 8/8] Refactor sanitize_and_sort_request_vars function The try-except block in sanitize_and_sort_request_vars was removed as it was not necessary. The function now directly processes the variable based on its type and returns the result. --- debug_toolbar/utils.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/debug_toolbar/utils.py b/debug_toolbar/utils.py index 0858e5047..f4b3eac38 100644 --- a/debug_toolbar/utils.py +++ b/debug_toolbar/utils.py @@ -227,20 +227,16 @@ def sanitize_and_sort_request_vars( if not isinstance(variable, (dict, QueryDict)): return {"raw": variable} - try: - # Get sorted keys if possible, otherwise just list them - keys = _get_sorted_keys(variable) + # Get sorted keys if possible, otherwise just list them + keys = _get_sorted_keys(variable) - # Process the variable based on its type - if isinstance(variable, QueryDict): - result = _process_query_dict(variable, keys) - else: - result = _process_dict(variable, keys) + # Process the variable based on its type + if isinstance(variable, QueryDict): + result = _process_query_dict(variable, keys) + else: + result = _process_dict(variable, keys) - return {"list": result} - except TypeError: - # If any processing fails, return raw variable - return {"raw": variable} + return {"list": result} def _get_sorted_keys(variable):