From 76bac931fb3fde3f6eb95968eb9f07dc42a2fbb1 Mon Sep 17 00:00:00 2001 From: Leon Derczynski Date: Wed, 30 Oct 2024 08:53:51 +0100 Subject: [PATCH 1/2] require core config top- and second-level params to be documented --- tests/test_docs.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_docs.py b/tests/test_docs.py index 7f7c0a9af..b352aabc6 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,8 +1,11 @@ import importlib import os from pathlib import Path +import yaml + import pytest + TOP_PATHS = ["probes", "detectors", "harnesses", "generators", "evaluators", "buffs"] DOC_SOURCE = os.path.join("docs", "source") @@ -165,3 +168,26 @@ def test_root_modules_docs(doc_index_source_text, root_module: str): assert ( f" {root_module.stem}\n" in doc_index_source_text ), f"root module doc page for {root_module.name} should be linked from doc root index" + + +def test_core_config_options_explained(): + import garak._config + + core_config_file_name = ( + garak._config.transient.package_dir / "resources" / "garak.core.yaml" + ) + doc_nodes_to_check = [] + with open(core_config_file_name, encoding="utf-8") as settings_file: + settings = yaml.safe_load(settings_file) + for top_level_setting in settings: + doc_nodes_to_check.append(top_level_setting) + for second_level_setting in settings[top_level_setting]: + doc_nodes_to_check.append(second_level_setting) + + configurable_rst = open( + Path("docs") / "source" / "configurable.rst", "r", encoding="utf-8" + ).read() + for doc_node_to_check in doc_nodes_to_check: + assert ( + f"``{doc_node_to_check}``" in configurable_rst + ), f"core config value '{doc_node_to_check}' must be documented in configurable.rst" From abb292a7ffd050f356be484c0d1852c8d002b729 Mon Sep 17 00:00:00 2001 From: Leon Derczynski Date: Wed, 30 Oct 2024 08:58:01 +0100 Subject: [PATCH 2/2] tighter formatting check to reduce false positives --- tests/test_docs.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/test_docs.py b/tests/test_docs.py index b352aabc6..73109ee38 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -176,18 +176,26 @@ def test_core_config_options_explained(): core_config_file_name = ( garak._config.transient.package_dir / "resources" / "garak.core.yaml" ) - doc_nodes_to_check = [] + l1_nodes_to_check = [] + l2_nodes_to_check = [] + with open(core_config_file_name, encoding="utf-8") as settings_file: settings = yaml.safe_load(settings_file) for top_level_setting in settings: - doc_nodes_to_check.append(top_level_setting) + l1_nodes_to_check.append(top_level_setting) for second_level_setting in settings[top_level_setting]: - doc_nodes_to_check.append(second_level_setting) + l2_nodes_to_check.append(second_level_setting) configurable_rst = open( Path("docs") / "source" / "configurable.rst", "r", encoding="utf-8" ).read() - for doc_node_to_check in doc_nodes_to_check: + + for l1_node in l1_nodes_to_check: + assert ( + f"\n``{l1_node}`` config items\n" in configurable_rst + ), f"core config value '{l1_node}' must be documented in configurable.rst" + + for l2_node in l2_nodes_to_check: assert ( - f"``{doc_node_to_check}``" in configurable_rst - ), f"core config value '{doc_node_to_check}' must be documented in configurable.rst" + f"\n* ``{l2_node}`` - " in configurable_rst + ), f"core config value '{l2_node}' must be documented in configurable.rst"