Feature/reporting categories (#389)

leondz · web-flow · commit 7aa344f5e770 · 2023-12-20T14:06:16.000+01:00
* add reporting top-level config object, and taxonomy variable for it

* move report_dir config into reporting config

* move report_prefix config item to reporting config

* update report html to support flexible grouping by top-level tag

* presentation tweaks, add second-order sorts
diff --git a/garak/_config.py b/garak/_config.py
@@ -18,11 +18,10 @@
 
 version = -1  # eh why this is here? hm. who references it
 
-system_params = (
-    "verbose report_prefix narrow_output parallel_requests parallel_attempts".split()
-)
+system_params = "verbose narrow_output parallel_requests parallel_attempts".split()
 run_params = "seed deprefix eval_threshold generations probe_tags".split()
 plugins_params = "model_type model_name extended_detectors".split()
+reporting_params = "taxonomy report_prefix".split()
 
 
 loaded = False
@@ -45,19 +44,22 @@ class TransientConfig(GarakSubConfig):
     basedir = pathlib.Path(__file__).parents[0]
     starttime = None
     starttime_iso = None
-    report_dir = "runs"
 
 
 transient = TransientConfig()
 
 system = GarakSubConfig()
 run = GarakSubConfig()
 plugins = GarakSubConfig()
+reporting = GarakSubConfig()
 plugins.probes = {}
 plugins.generators = {}
 plugins.detectors = {}
 plugins.buffs = {}
 plugins.harnesses = {}
+reporting.report_dir = "runs"
+reporting.taxonomy = None  # set here to enable report_digest to be called directly
+
 
 config_files = []
 
@@ -95,11 +97,12 @@ def _load_yaml_config(settings_filenames) -> dict:
 
 
 def _store_config(settings_files) -> None:
-    global system, run, plugins
+    global system, run, plugins, reporting
     settings = _load_yaml_config(settings_files)
     system = _set_settings(system, settings["system"])
     run = _set_settings(run, settings["run"])
     plugins = _set_settings(plugins, settings["plugins"])
+    reporting = _set_settings(reporting, settings["reporting"])
 
 
 def load_base_config() -> None:
diff --git a/garak/analyze/report_digest.py b/garak/analyze/report_digest.py
@@ -11,6 +11,8 @@
 
 import jinja2
 
+from garak import _config, _plugins
+
 templateLoader = jinja2.FileSystemLoader(searchpath=".")
 templateEnv = jinja2.Environment(loader=templateLoader)
 
@@ -20,9 +22,7 @@
 footer_template = templateEnv.get_template(
     "garak/analyze/templates/digest_footer.jinja"
 )
-module_template = templateEnv.get_template(
-    "garak/analyze/templates/digest_module.jinja"
-)
+group_template = templateEnv.get_template("garak/analyze/templates/digest_group.jinja")
 probe_template = templateEnv.get_template("garak/analyze/templates/digest_probe.jinja")
 detector_template = templateEnv.get_template(
     "garak/analyze/templates/digest_detector.jinja"
@@ -43,7 +43,7 @@ def map_score(score):
     return 4
 
 
-def compile_digest(report_path):
+def compile_digest(report_path, taxonomy=_config.reporting.taxonomy):
     evals = []
     setup = defaultdict(str)
     with open(report_path, "r", encoding="utf-8") as reportfile:
@@ -78,6 +78,7 @@ def compile_digest(report_path):
 
     create_table = """create table results(
         probe_module VARCHAR(255) not null,
+        probe_group VARCHAR(255) not null,
         probe_class VARCHAR(255) not null,
         detector VARCHAR(255) not null, 
         score FLOAT not null,
@@ -92,22 +93,38 @@ def compile_digest(report_path):
         detector = eval["detector"].replace("detector.", "")
         score = eval["passed"] / eval["total"]
         instances = eval["total"]
-        cursor.execute(
-            f"insert into results values ('{pm}', '{pc}', '{detector}', '{score}', '{instances}')"
-        )
+        groups = []
+        if taxonomy is not None:
+            # get the probe tags
+            m = importlib.import_module(f"garak.probes.{pm}")
+            tags = getattr(m, pc).tags
+            for tag in tags:
+                if tag.split(":")[0] == taxonomy:
+                    groups.append(":".join(tag.split(":")[1:]))
+            if groups == []:
+                groups = ["other"]
+        else:
+            groups = [pm]
+        # add a row for each group
+        for group in groups:
+            cursor.execute(
+                f"insert into results values ('{pm}', '{group}', '{pc}', '{detector}', '{score}', '{instances}')"
+            )
 
     # calculate per-probe scores
 
-    res = cursor.execute("select distinct probe_module from results")
-    module_names = [i[0] for i in res.fetchall()]
+    res = cursor.execute(
+        "select distinct probe_group from results order by probe_group"
+    )
+    group_names = [i[0] for i in res.fetchall()]
 
     # top score: % of passed probes
     # probe score: mean of detector scores
 
     # let's build a dict of per-probe score
 
-    for probe_module in module_names:
-        sql = f"select avg(score)*100 as s from results where probe_module = '{probe_module}' order by s asc;"
+    for probe_group in group_names:
+        sql = f"select avg(score)*100 as s from results where probe_group = '{probe_group}' order by s asc, probe_class asc;"
         # sql = f"select probe_module || '.' || probe_class, avg(score) as s from results where probe_module = '{probe_module}' group by probe_module, probe_class order by desc(s)"
         res = cursor.execute(sql)
         # probe_scores = res.fetchall()
@@ -116,24 +133,38 @@ def compile_digest(report_path):
         # top_score = passing_probe_count / probe_count
         top_score = res.fetchone()[0]
 
-        probe_module = re.sub("[^0-9A-Za-z_]", "", probe_module)
-        m = importlib.import_module(f"garak.probes.{probe_module}")
-        module_doc = markdown.markdown(m.__doc__)
+        group_doc = f"Probes tagged {probe_group}"
+        group_link = ""
+
+        probe_group_name = probe_group
+        if taxonomy is None:
+            probe_module = re.sub("[^0-9A-Za-z_]", "", probe_group)
+            m = importlib.import_module(f"garak.probes.{probe_module}")
+            group_doc = markdown.markdown(m.__doc__)
+            group_link = (
+                f"https://reference.garak.ai/en/latest/garak.probes.{probe_group}.html"
+            )
+        elif probe_group != "other":
+            probe_group_name = f"{taxonomy}:{probe_group}"
+        else:
+            probe_group_name = "Uncategorized"
 
-        digest_content += module_template.render(
+        digest_content += group_template.render(
             {
-                "module": probe_module,
+                "module": probe_group_name,
                 "module_score": f"{top_score:.1f}%",
                 "severity": map_score(top_score),
-                "module_doc": module_doc,
+                "module_doc": group_doc,
+                "group_link": group_link,
             }
         )
 
         if top_score < 100.0:
             res = cursor.execute(
-                f"select probe_class, avg(score)*100 as s from results where probe_module='{probe_module}' group by probe_class order by s asc;"
+                f"select probe_module, probe_class, avg(score)*100 as s from results where probe_group='{probe_group}' group by probe_class order by s asc, probe_class asc;"
             )
-            for probe_class, score in res.fetchall():
+            for probe_module, probe_class, score in res.fetchall():
+                m = importlib.import_module(f"garak.probes.{probe_module}")
                 digest_content += probe_template.render(
                     {
                         "plugin_name": probe_class,
@@ -145,7 +176,7 @@ def compile_digest(report_path):
                 # print(f"\tplugin: {probe_module}.{probe_class} - {score:.1f}%")
                 if score < 100.0:
                     res = cursor.execute(
-                        f"select detector, score*100 from results where probe_module='{probe_module}' and probe_class='{probe_class}' order by score asc;"
+                        f"select detector, score*100 from results where probe_group='{probe_group}' and probe_class='{probe_class}' order by score asc, detector asc;"
                     )
                     for detector, score in res.fetchall():
                         detector = re.sub(r"[^0-9A-Za-z_.]", "", detector)
@@ -176,5 +207,8 @@ def compile_digest(report_path):
 
 if __name__ == "__main__":
     report_path = sys.argv[1]
-    digest_content = compile_digest(report_path)
+    taxonomy = None
+    if len(sys.argv) == 3:
+        taxonomy = sys.argv[2]
+    digest_content = compile_digest(report_path, taxonomy=taxonomy)
     print(digest_content)
diff --git a/garak/analyze/templates/digest_group.jinja b/garak/analyze/templates/digest_group.jinja
@@ -0,0 +1,13 @@
+
+<button class="defcon{{severity}} accordion">{{ module }} - {{ module_score }}</button>
+<div class="panel">
+<p>{{module_doc}}</p>
+{%if module_score != "100.0%"%}
+<ul><li>Probes under
+{%if group_link%}
+<a href="{{group_link}}" target="_new">{{module}}</a>
+{%else%}
+"{{module}}"
+{%endif%}
+ scored the system a {{ module_score }} pass rate.</li></ul>
+{%endif%}
diff --git a/garak/analyze/templates/digest_module.jinja b/garak/analyze/templates/digest_module.jinja
diff --git a/garak/analyze/templates/digest_probe.jinja b/garak/analyze/templates/digest_probe.jinja
@@ -1,5 +1,5 @@
 <h3 class="defcon{{severity}}">probe: {{ plugin_name }} {{ plugin_score }}</h3>
 {%if plugin_score != "100.0%"%}
-<p class="probe">Tests for: {{plugin_descr}}</p>
+<p class="probe">Tests description: {{plugin_descr}}</p>
 <p class="probe">{{plugin_name}}: <b>{{ plugin_score }}</b> resilient.</p>
 {%endif%}
diff --git a/garak/cli.py b/garak/cli.py
@@ -32,6 +32,7 @@ def main(arguments=[]) -> None:
     import argparse
 
     parser = argparse.ArgumentParser(
+        prog="python -m garak",
         description="LLM safety & security scanning tool",
         epilog="See https://github.com/leondz/garak",
     )
@@ -47,7 +48,7 @@ def main(arguments=[]) -> None:
     parser.add_argument(
         "--report_prefix",
         type=str,
-        default=_config.system.report_prefix,
+        default=_config.reporting.report_prefix,
         help="Specify an optional prefix for the report and hit logs",
     )
     parser.add_argument(
@@ -173,6 +174,14 @@ def main(arguments=[]) -> None:
         help="buff to use",
     )
 
+    ## REPORTING
+    parser.add_argument(
+        "--taxonomy",
+        type=str,
+        default=_config.reporting.taxonomy,
+        help="specify a MISP top-level taxonomy to be used for grouping probes in reporting. e.g. 'avid-effect', 'owasp' ",
+    )
+
     ## COMMANDS
     # items placed here also need to be listed in command_options below
     parser.add_argument(
@@ -292,10 +301,13 @@ def main(arguments=[]) -> None:
             setattr(_config.run, param, value)
         elif param in _config.plugins_params:
             setattr(_config.plugins, param, value)
+        elif param in _config.reporting_params:
+            setattr(_config.reporting, param, value)
         else:
             ignored_params.append((param, value))
+    logging.debug("non-config params: %s", ignored_params)
 
-    # put plguin spec into the _spec config value, if set at cli
+    # put plugin spec into the _spec config value, if set at cli
     if "probes" in args:
         _config.plugins.probe_spec = args.probes
     if "detectors" in args:
diff --git a/garak/command.py b/garak/command.py
@@ -40,19 +40,19 @@ def start_run():
     logging.info("started at %s", _config.transient.starttime_iso)
     # print("ASSIGN UUID", args)
     _config.transient.run_id = str(uuid.uuid4())  # uuid1 is safe but leaks host info
-    if not _config.system.report_prefix:
-        if not os.path.isdir(_config.transient.report_dir):
+    if not _config.reporting.report_prefix:
+        if not os.path.isdir(_config.reporting.report_dir):
             try:
-                os.mkdir(_config.transient.report_dir)
+                os.mkdir(_config.reporting.report_dir)
             except PermissionError as e:
                 raise PermissionError(
                     "Can't create logging directory %s, quitting",
-                    _config.transient.report_dir,
+                    _config.reporting.report_dir,
                 ) from e
-        _config.transient.report_filename = f"{_config.transient.report_dir}/garak.{_config.transient.run_id}.report.jsonl"
+        _config.transient.report_filename = f"{_config.reporting.report_dir}/garak.{_config.transient.run_id}.report.jsonl"
     else:
         _config.transient.report_filename = (
-            _config.system.report_prefix + ".report.jsonl"
+            _config.reporting.report_prefix + ".report.jsonl"
         )
     _config.transient.reportfile = open(
         _config.transient.report_filename, "w", buffering=1, encoding="utf-8"
@@ -70,7 +70,7 @@ def start_run():
             type(None),
         ):
             setup_dict[f"_config.{k}"] = v
-    for subset in "system transient run plugins".split():
+    for subset in "system transient run plugins reporting".split():
         for k, v in getattr(_config, subset).__dict__.items():
             if k[:2] != "__" and type(v) in (
                 str,
diff --git a/garak/evaluators/base.py b/garak/evaluators/base.py
@@ -64,11 +64,11 @@ def evaluate(self, attempts: List[garak.attempt.Attempt]) -> None:
                 for idx, score in enumerate(attempt.detector_results[detector]):
                     if not self.test(score):  # if we don't pass
                         if not _config.transient.hitlogfile:
-                            if not _config.system.report_prefix:
-                                hitlog_filename = f"{_config.transient.report_dir}/garak.{_config.transient.run_id}.hitlog.jsonl"
+                            if not _config.reporting.report_prefix:
+                                hitlog_filename = f"{_config.reporting.report_dir}/garak.{_config.transient.run_id}.hitlog.jsonl"
                             else:
                                 hitlog_filename = (
-                                    _config.system.report_prefix + ".hitlog.jsonl"
+                                    _config.reporting.report_prefix + ".hitlog.jsonl"
                                 )
                             logging.info("hit log in %s", hitlog_filename)
                             _config.transient.hitlogfile = open(
diff --git a/garak/resources/garak.core.yaml b/garak/resources/garak.core.yaml
@@ -1,7 +1,6 @@
 ---
 system:
   verbose: 0
-  report_prefix:
   narrow_output: false
   parallel_requests: false
   parallel_attempts: false
@@ -28,3 +27,7 @@ plugins:
   buff_spec:
   buffs: {}
   harnesses: {}
+
+reporting:
+  report_prefix:
+  taxonomy:
diff --git a/tests/test_config.py b/tests/test_config.py
@@ -51,6 +51,8 @@
     param_locs[p] = "run"
 for p in garak._config.plugins_params:
     param_locs[p] = "plugins"
+for p in garak._config.reporting_params:
+    param_locs[p] = "reporting"
 
 
 # test CLI assertions of each var
