docs: require core config top- and second-level params to be documented

tests/test_docs.py

@@ -1,8 +1,11 @@
 import importlib
 import os
 from pathlib import Path
+import yaml
+
 import pytest
 
+
 TOP_PATHS = ["probes", "detectors", "harnesses", "generators", "evaluators", "buffs"]
 DOC_SOURCE = os.path.join("docs", "source")
 
@@ -165,3 +168,34 @@ def test_root_modules_docs(doc_index_source_text, root_module: str):
         assert (
             f" {root_module.stem}\n" in doc_index_source_text
         ), f"root module doc page for {root_module.name} should be linked from doc root index"
+
+
+def test_core_config_options_explained():
+    import garak._config
+
+    core_config_file_name = (
+        garak._config.transient.package_dir / "resources" / "garak.core.yaml"
+    )
+    l1_nodes_to_check = []
+    l2_nodes_to_check = []
+
+    with open(core_config_file_name, encoding="utf-8") as settings_file:
+        settings = yaml.safe_load(settings_file)
+        for top_level_setting in settings:
+            l1_nodes_to_check.append(top_level_setting)
+            for second_level_setting in settings[top_level_setting]:
+                l2_nodes_to_check.append(second_level_setting)
+
+    configurable_rst = open(
+        Path("docs") / "source" / "configurable.rst", "r", encoding="utf-8"
+    ).read()
+
+    for l1_node in l1_nodes_to_check:
+        assert (
+            f"\n``{l1_node}`` config items\n" in configurable_rst
+        ), f"core config value '{l1_node}' must be documented in configurable.rst"
+
+    for l2_node in l2_nodes_to_check:
+        assert (
+            f"\n* ``{l2_node}`` - " in configurable_rst
+        ), f"core config value '{l2_node}' must be documented in configurable.rst"