add external configurability (#331)

* start integrating settings by config file

* tests pass (coverage isn't high enough yet)

* add basic docs re: config loading

* have code access new locations withing _config; stop accessing garak._config indirectly

* clear up references to _config.transient.reportfile

* cli_args now stores params directly specified on CLI only (plus command requests, for convenience) and then overrides args; better logging of cli/arg processing

* search for plugin spec in config, not in args

* amend reporting write call

* plugin planning now uses spec & spec parsing

* uuid only generated once

* configure logging before it's used in _config

* re-enable CLI short form options & CLI param typing

* reinstate unit on completion timer

* add list_config command; avoid configs getting all merged into one subconfig object; correct config var locations in cli

* moved model_name, model_type to _config.plugins

* track config param name locations in _config.py

* add config testing, include CLI option tests

* implement config file priority; remove dynaconf (doesn't support hierarchical merging)

* runs can be launched from config file alone

* support loading generator, probe options from yaml

* add stub failing tests for config

* reconfig and test probe, generator CLI cfg file loading & structure

* clean up cli param setting

* set up plugin config-on-load; start removing generator/plugin_options params; do config merging of CLI options

* add probespec parsing tests

* test generator parameter cli loading

* test generator config loading

* check that cli probe options override yaml

* check that cli probe options override yaml

* check that cli probe options override yaml

* check CLI generations param overrides run param

* test that site cfg overrides core cfg

* config tests passing

Author: leondz

docs/source/basic.rst

@@ -4,7 +4,22 @@ garak
 garak._config
 -------------
 
-This module holds runtime config values.
+This module holds config values.
+
+These are broken into the following major categories:
+
+* system: options that don't affect the security assessment
+* run: options that describe how a garak run will be conducted
+* plugins: config for plugins (generators, probes, detectors, buffs)
+* transient: internal values local to a single garak execution
+
+Config values are loaded in the following priority (lowest-first):
+
+* Plugin defaults in the code
+* Core config: from ``garak/resources/garak.core.yaml``; not to be overridden
+* Site config: from ``garak/garak.site.yaml``
+* Runtime config: from an optional config file specified manually, via e.g. CLI parameter
+* Command-line options
 
 Code
 ^^^^

garak/_config.py

@@ -1,16 +1,152 @@
 #!/usr/bin/env python3
 """garak global config"""
 
+# SPDX-FileCopyrightText: Portions Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# plugin code < base config < site config < run config < cli params
+
+# logging should be set up before config is loaded
+
+from dataclasses import dataclass
+import logging
+import os
 import pathlib
+from typing import List
+import yaml
+
+version = -1  # eh why this is here? hm. who references it
+
+system_params = (
+    "verbose report_prefix narrow_output parallel_requests parallel_attempts".split()
+)
+run_params = "seed deprefix eval_threshold generations".split()
+plugins_params = "model_type model_name extended_detectors".split()
+
+
+@dataclass
+class GarakSubConfig:
+    pass
+
+
+@dataclass
+class TransientConfig(GarakSubConfig):
+    """Object to hold transient global config items not set externally"""
+
+    report_filename = None
+    reportfile = None
+    hitlogfile = None
+    args = None  # only access this when determining what was passed on CLI
+    run_id = None
+    basedir = pathlib.Path(__file__).parents[0]
+    starttime = None
+    starttime_iso = None
+
+
+transient = TransientConfig()
+
+system = GarakSubConfig()
+run = GarakSubConfig()
+plugins = GarakSubConfig()
+plugins.probes = {}
+plugins.generators = {}
+plugins.detectors = {}
+plugins.buffs = {}
+
+# this is so popular, let's set a default. what other defaults are worth setting? what's the policy?
+run.seed = None
+
+# placeholder
+# generator, probe, detector, buff = {}, {}, {}, {}
+
+
+def _set_settings(config_obj, settings_obj: dict):
+    for k, v in settings_obj.items():
+        setattr(config_obj, k, v)
+    return config_obj
+
+
+def _combine_into(d: dict, combined: dict) -> None:
+    for k, v in d.items():
+        if isinstance(v, dict):
+            _combine_into(v, combined.setdefault(k, {}))
+        else:
+            combined[k] = v
+    return combined
+
+
+def _load_yaml_config(settings_filenames) -> dict:
+    config = {}
+    for settings_filename in settings_filenames:
+        settings = yaml.safe_load(open(settings_filename, encoding="utf-8"))
+        if settings is not None:
+            config = _combine_into(settings, config)
+    return config
+
+
+def _store_config(settings_files) -> None:
+    global system, run, plugins
+    settings = _load_yaml_config(settings_files)
+    system = _set_settings(system, settings["system"])
+    run = _set_settings(run, settings["run"])
+    plugins = _set_settings(plugins, settings["plugins"])
+
+
+def load_base_config() -> None:
+    global system
+    settings_files = [str(transient.basedir / "resources/garak.core.yaml")]
+    logging.debug("Loading configs from: %s", ",".join(settings_files))
+    _store_config(settings_files=settings_files)
+
+
+def load_config(
+    site_config_filename="garak.site.yaml", run_config_filename=None
+) -> None:
+    # would be good to bubble up things from run_config, e.g. generator, probe(s), detector(s)
+    # and then not have cli be upset when these are not given as cli params
+    global system, run, plugins
+
+    settings_files = [str(transient.basedir / "resources/garak.core.yaml")]
+
+    fq_site_config_filename = str(transient.basedir / site_config_filename)
+    if os.path.isfile(fq_site_config_filename):
+        settings_files.append(fq_site_config_filename)
+    else:
+        # warning, not error, because this one has a default value
+        logging.warning("site config not found: %s", fq_site_config_filename)
+
+    if run_config_filename is not None:
+        fq_run_config_filename = str(transient.basedir / run_config_filename)
+        if os.path.isfile(fq_run_config_filename):
+            settings_files.append(fq_run_config_filename)
+        else:
+            logging.error("run config not found: %s", fq_run_config_filename)
+
+    logging.debug("Loading configs from: %s", ",".join(settings_files))
+    _store_config(settings_files=settings_files)
+
+
+def parse_plugin_spec(spec: str, category: str) -> List[str]:
+    from garak._plugins import enumerate_plugins
+
+    if spec is None or spec.lower() in ("", "auto", "none"):
+        return []
+    if spec.lower() == "all":
+        plugin_names = [
+            name
+            for name, active in enumerate_plugins(category=category)
+            if active is True
+        ]
+    else:
+        plugin_names = []
+        for clause in spec.split(","):
+            if clause.count(".") < 1:
+                plugin_names += [
+                    p
+                    for p, a in enumerate_plugins(category=category)
+                    if p.startswith(f"{category}.{clause}.") and a is True
+                ]
+            else:
+                plugin_names += [f"{category}.{clause}"]  # spec parsing
 
-basedir = pathlib.Path(__file__).parents[0]
-
-args = None
-reportfile = None
-hitlogfile = None
-seed = None
-run_id = None
-generator_model = None
-generator_name = None
-probe_options = {}
-generator_options = {}
+    return plugin_names

garak/_plugins.py

@@ -11,10 +11,12 @@
 import os
 from typing import List
 
-import garak._config
+from garak import _config
 
 
-def enumerate_plugins(category: str = "probes", skip_base_classes=True) -> List[str]:
+def enumerate_plugins(
+    category: str = "probes", skip_base_classes=True
+) -> List[tuple[str, bool]]:
     """A function for listing all modules & plugins of the specified kind.
 
     garak's plugins are organised into four packages - probes, detectors, generators
@@ -54,7 +56,7 @@ def enumerate_plugins(category: str = "probes", skip_base_classes=True) -> List[
 
     plugin_class_names = []
 
-    for module_filename in sorted(os.listdir(garak._config.basedir / category)):
+    for module_filename in sorted(os.listdir(_config.transient.basedir / category)):
         if not module_filename.endswith(".py"):
             continue
         if module_filename.startswith("__"):
@@ -82,7 +84,17 @@ def enumerate_plugins(category: str = "probes", skip_base_classes=True) -> List[
     return plugin_class_names
 
 
-def load_plugin(path, break_on_fail=True):
+def configure_plugin(plugin_path: str, plugin: object) -> object:
+    category, module_name, plugin_class_name = plugin_path.split(".")
+    plugin_name = f"{module_name}.{plugin_class_name}"
+    plugin_type_config = getattr(_config.plugins, category)
+    if plugin_name in plugin_type_config:
+        for k, v in plugin_type_config[plugin_name].items():
+            setattr(plugin, k, v)
+    return plugin
+
+
+def load_plugin(path, break_on_fail=True) -> object:
     """load_plugin takes a path to a plugin class, and attempts to load that class.
     If successful, it returns an instance of that class.
 
@@ -94,41 +106,45 @@ def load_plugin(path, break_on_fail=True):
     """
     try:
         category, module_name, plugin_class_name = path.split(".")
-    except ValueError:
+    except ValueError as ve:
         if break_on_fail:
             raise ValueError(
                 f'Expected plugin name in format category.module_name.class_name, got "{path}"'
-            )
+            ) from ve
         else:
             return False
     module_path = f"garak.{category}.{module_name}"
     try:
         mod = importlib.import_module(module_path)
-    except:
-        logging.warning(f"Exception failed import of {module_path}")
+    except Exception as e:
+        logging.warning("Exception failed import of %s", module_path)
         if break_on_fail:
-            raise ValueError("Didn't successfully import " + module_name)
+            raise ValueError("Didn't successfully import " + module_name) from e
         else:
             return False
 
     try:
         plugin_instance = getattr(mod, plugin_class_name)()
-    except AttributeError:
+    except AttributeError as ae:
         logging.warning(
-            f"Exception failed instantiation of {module_path}.{plugin_class_name}"
+            "Exception failed instantiation of %s.%s", module_path, plugin_class_name
         )
         if break_on_fail:
             raise ValueError(
                 f"Plugin {plugin_class_name} not found in {category}.{module_name}"
-            )
+            ) from ae
         else:
             return False
     except Exception as e:
         # print("error in: module", mod.__name__, "class", plugin_class_name)
-        # logging.warning(f"error in: module {mod} class {plugin_class_name}")
+        logging.warning(
+            "error instantiating module %s class %s", str(mod), plugin_class_name
+        )
         if break_on_fail:
-            raise Exception(e)
+            raise Exception(e) from e
         else:
             return False
 
+    plugin_instance = configure_plugin(path, plugin_instance)
+
     return plugin_instance