From 93264e30eb27c97746f14b538fb5ce5e4cba53cc Mon Sep 17 00:00:00 2001 From: Leon Derczynski Date: Mon, 13 Jan 2025 12:34:26 +0100 Subject: [PATCH] fix StringDetector case sensitivity, add dedicated tests for detector bases, add tests for matchtype --- garak/detectors/base.py | 2 +- tests/detectors/test_detectors.py | 10 -- tests/detectors/test_detectors_base.py | 176 +++++++++++++++++++++++++ 3 files changed, 177 insertions(+), 11 deletions(-) create mode 100644 tests/detectors/test_detectors_base.py diff --git a/garak/detectors/base.py b/garak/detectors/base.py index 82bde822a..918facf64 100644 --- a/garak/detectors/base.py +++ b/garak/detectors/base.py @@ -182,7 +182,7 @@ def detect( if output is None: continue for s in self.substrings: - if case_sensitive: + if not case_sensitive: s, output = s.lower(), output.lower() if self.matchtype == "str": diff --git a/tests/detectors/test_detectors.py b/tests/detectors/test_detectors.py index 4f610e510..ba7f3c43b 100644 --- a/tests/detectors/test_detectors.py +++ b/tests/detectors/test_detectors.py @@ -109,16 +109,6 @@ def test_detector_detect(classname): ), "detector should return as many results as in all_outputs (maybe excluding Nones)" -def test_filedetector_nonexist(): - d = garak.detectors.base.FileDetector() - a = garak.attempt.Attempt(prompt="") - a.outputs = [None, "", "/non/existing/file"] - a.notes["format"] = d.valid_format - assert ( - len(list(d.detect(a))) == 0 - ), "FileDetector should skip filenames for non-existing files" - - @pytest.mark.parametrize("classname", DETECTORS) def test_detector_metadata(classname): if classname.startswith("detectors.base."): diff --git a/tests/detectors/test_detectors_base.py b/tests/detectors/test_detectors_base.py new file mode 100644 index 000000000..990ca79ae --- /dev/null +++ b/tests/detectors/test_detectors_base.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from garak.attempt import Attempt +import garak.detectors.base + + +def test_filedetector_nonexist(): + d = garak.detectors.base.FileDetector() + a = Attempt(prompt="") + a.outputs = [None, "", "/non/existing/file"] + a.notes["format"] = d.valid_format + assert ( + len(list(d.detect(a))) == 0 + ), "FileDetector should skip filenames for non-existing files" + + +TEST_STRINGS = [ + "test", # 0: lower + "UPPERCASE", # 1: upper + "MixedCase", # 2: mixed (camel) +] + + +@pytest.fixture +def string_detector(): + return garak.detectors.base.StringDetector(TEST_STRINGS) + + +def test_case_sensitive_exact_match(string_detector): + """Test that exact matches are detected when case_sensitive=True""" + for test_str in TEST_STRINGS: + attempt = Attempt(prompt="Hello") + attempt.outputs = [test_str] + results = string_detector.detect(attempt, case_sensitive=True) + assert results == [1.0], f"Failed to detect exact match for '{test_str}'" + + +def test_case_sensitive_no_match(string_detector): + """Test that case variations are not detected when case_sensitive=True""" + test_cases = [ + TEST_STRINGS[0].upper(), # uppercase variation + TEST_STRINGS[0].title(), # capitalized variation + "tEsT", # mixed case variation + TEST_STRINGS[1].lower(), # lowercase variation + TEST_STRINGS[2].lower(), # lowercase variation + TEST_STRINGS[2].title(), # title but not camelcase variation + ] + + for test_str in test_cases: + attempt = Attempt(prompt="Hello") + attempt.outputs = [test_str] + results = string_detector.detect(attempt, case_sensitive=True) + assert results == [ + 0.0 + ], f"Should not detect absent case variation '{test_str}' in case sensitive mode" + + +def test_case_insensitive_match(string_detector): + """Test that case variations are detected when case_sensitive=False""" + test_cases = [ + (TEST_STRINGS[0], TEST_STRINGS[0].upper()), + (TEST_STRINGS[0], TEST_STRINGS[0].title()), + (TEST_STRINGS[1], TEST_STRINGS[1].lower()), + (TEST_STRINGS[1], TEST_STRINGS[1].title()), + (TEST_STRINGS[2], TEST_STRINGS[2].lower()), + (TEST_STRINGS[2], TEST_STRINGS[2].upper()), + ] + + for original, variant in test_cases: + attempt = Attempt(prompt="Hello") + attempt.outputs = [variant] + results = string_detector.detect(attempt, case_sensitive=False) + assert results == [ + 1.0 + ], f"Failed to detect case insensitive match: '{original}' vs '{variant}'" + + +def test_multiple_outputs(string_detector): + """Test detection across multiple outputs with mixed case sensitivity""" + attempt = Attempt(prompt="Hello") + attempt.outputs = [TEST_STRINGS[0], TEST_STRINGS[0].upper(), "no match"] + + # Case sensitive - should only match exact "test" + results = string_detector.detect(attempt, case_sensitive=True) + assert results == [ + 1.0, + 0.0, + 0.0, + ], "Case sensitive matching failed for multiple outputs" + + # Case insensitive - should match both "test" and "TEST" + results = string_detector.detect(attempt, case_sensitive=False) + assert results == [ + 1.0, + 1.0, + 0.0, + ], "Case insensitive matching failed for multiple outputs" + + +def test_none_outputs(string_detector): + """Test handling of None outputs""" + attempt = Attempt(prompt="Hello") + attempt.outputs = [None, TEST_STRINGS[0], None] + + results = string_detector.detect(attempt, case_sensitive=True) + assert len(results) == 1, "Detector expected to skip None outputs" + assert results == [1.0], "Failed to handle None outputs correctly" + + +@pytest.mark.parametrize("matchtype", ["str", "word"]) +def test_matchtype_with_case_sensitivity(matchtype): + """Test case sensitivity with different matchtypes""" + + lowercase_test_word = "test" + detector = garak.detectors.base.StringDetector([lowercase_test_word]) + detector.matchtype = matchtype + + test_cases = { + "str": [ + (f"begin {lowercase_test_word}ing", True), # should match + ( + f"begin {lowercase_test_word.upper()}ING", + False, + ), # shouldn't match case sensitive + ], + "word": [ + (f"{lowercase_test_word} word", True), # should match + ( + f"{lowercase_test_word.upper()} word", + False, + ), # shouldn't match case sensitive + ], + } + + for text, should_match_sensitive in test_cases[matchtype]: + attempt = Attempt(prompt="Hello") + attempt.outputs = [text] + + # Test case sensitive + results = detector.detect(attempt, case_sensitive=True) + assert results == [ + 1.0 if should_match_sensitive else 0.0 + ], f"Case sensitive {matchtype} match failed for '{text}'" + + # Test case insensitive + results = detector.detect(attempt, case_sensitive=False) + assert results == [ + 1.0 + ], f"Case insensitive {matchtype} match failed for '{text}'" + + +def test_word_boundary_matching(): + detector = garak.detectors.base.StringDetector(TEST_STRINGS) + detector.matchtype = "word" + + test_cases = [ + ("begin testing", False), + ("begin the test", True), + ("THIS IS UPPERCASE", True), + ("THIS IS UPPER CASE", False), + ("THISISUPPERCASE", False), + ] + + for text, match in test_cases: + attempt = Attempt(prompt="") + attempt.outputs = [text] + results = detector.detect(attempt, case_sensitive=True) + assert results == [ + 1.0 if match else 0.0 + ], f"Word boundary match failed for '{text}', expected {match}" + + +TEST_STRINGS