Create TOML V2

Given that the new things we're adding to the user file aren't
compatible with older versions of mantid, we're bumping up to TOML V2 so
that we can keep being backwards compatible. This is mainly to stop new
user files containing this information from being loaded by older
versions, making it clear that you need to upgrade to use these new
features and toml files.

RE mantidproject#38524

Author: cailafinn

scripts/SANS/sans/gui_logic/presenter/run_tab_presenter.py

@@ -18,7 +18,7 @@
 from typing import Optional
 
 from mantidqt.utils.observer_pattern import GenericObserver
-from sans.user_file.toml_parsers.toml_v1_schema import TomlValidationError
+from sans.user_file.toml_parsers.toml_base_schema import TomlValidationError
 from ui.sans_isis import SANSSaveOtherWindow
 from ui.sans_isis.sans_data_processor_gui import SANSDataProcessorGui
 from ui.sans_isis.sans_gui_observable import SansGuiObservable

scripts/SANS/sans/state/StateObjects/StatePolarization.py

@@ -58,6 +58,7 @@ def __init__(self):
         # Relevant to He3 filters (Polarisers and Analysers)
         self.cell_length = None
         self.gas_pressure = None
+        self.empty_cell = None
         # Relevant to all polarisers and analysers.
         self.initial_polarization = None
 

scripts/SANS/sans/test_helper/toml_parser_test_helpers.py

@@ -0,0 +1,43 @@
+# Mantid Repository : https://github.com/mantidproject/mantid
+#
+# Copyright © 2025 ISIS Rutherford Appleton Laboratory UKRI,
+#     NScD Oak Ridge National Laboratory, European Spallation Source
+#     & Institut Laue - Langevin
+# SPDX - License - Identifier: GPL - 3.0 +
+
+from typing import List, Dict
+
+from sans.common.enums import SANSInstrument, SANSFacility
+from sans.state.StateObjects.StateData import get_data_builder
+from sans.state.StateObjects.StateData import StateData
+from sans.test_helper.file_information_mock import SANSFileInformationMock
+
+
+def get_mock_data_info():
+    # TODO I really really dislike having to do this in a test, but
+    # TODO de-coupling StateData is required to avoid it
+    file_information = SANSFileInformationMock(instrument=SANSInstrument.SANS2D, run_number=22024)
+    data_builder = get_data_builder(SANSFacility.ISIS, file_information)
+    data_builder.set_sample_scatter("SANS2D00022024")
+    data_builder.set_sample_scatter_period(3)
+    return data_builder.build()
+
+
+def setup_parser_dict(dict_vals) -> tuple[dict, StateData]:
+    def _add_missing_mandatory_key(dict_to_check: Dict, key_path: List[str], replacement_val):
+        _dict = dict_to_check
+        for key in key_path[0:-1]:
+            if key not in _dict:
+                _dict[key] = {}
+            _dict = _dict[key]
+
+        if key_path[-1] not in _dict:
+            _dict[key_path[-1]] = replacement_val  # Add in child value
+        return dict_to_check
+
+    mocked_data_info = get_mock_data_info()
+    # instrument key needs to generally be present
+    dict_vals = _add_missing_mandatory_key(dict_vals, ["instrument", "name"], "LOQ")
+    dict_vals = _add_missing_mandatory_key(dict_vals, ["detector", "configuration", "selected_detector"], "rear")
+
+    return dict_vals, mocked_data_info

scripts/SANS/sans/user_file/toml_parsers/toml_base_parser.py

@@ -0,0 +1,27 @@
+# Mantid Repository : https://github.com/mantidproject/mantid
+#
+# Copyright © 2025 ISIS Rutherford Appleton Laboratory UKRI,
+#     NScD Oak Ridge National Laboratory, European Spallation Source
+#     & Institut Laue - Langevin
+# SPDX - License - Identifier: GPL - 3.0 +
+
+from abc import ABCMeta, abstractmethod
+
+from sans.state.IStateParser import IStateParser
+from sans.user_file.toml_parsers.toml_base_schema import TomlSchemaValidator
+
+
+class TomlParserBase(IStateParser, metaclass=ABCMeta):
+    def __init__(self, dict_to_parse, file_information, schema_validator: TomlSchemaValidator):
+        self._validator = schema_validator
+        self._validator.validate()
+
+        self._implementation = None
+        data_info = self.get_state_data(file_information)
+        self._implementation = self._get_impl(dict_to_parse, data_info)
+        self._implementation.parse_all()
+
+    @staticmethod
+    @abstractmethod
+    def _get_impl(*args):
+        pass

scripts/SANS/sans/user_file/toml_parsers/toml_base_schema.py

@@ -0,0 +1,73 @@
+# Mantid Repository : https://github.com/mantidproject/mantid
+#
+# Copyright © 2025 ISIS Rutherford Appleton Laboratory UKRI,
+#     NScD Oak Ridge National Laboratory, European Spallation Source
+#     & Institut Laue - Langevin
+# SPDX - License - Identifier: GPL - 3.0 +
+
+import re
+from abc import abstractmethod, ABCMeta
+
+
+class TomlValidationError(Exception):
+    # A key error would be more appropriate, but there is special
+    # handling on KeyError which escapes any new lines making it un-readable
+    pass
+
+
+class TomlSchemaValidator(object, metaclass=ABCMeta):
+    # As of the current TOML release there is no way to validate a schema so
+    # we must provide an implementation
+
+    def __init__(self, dict_to_validate):
+        self._expected_list = self._build_nested_keys(self.reference_schema())
+        self._to_validate_list = self._build_nested_keys(dict_to_validate)
+
+    def validate(self):
+        self._to_validate_list = filter(lambda s: not s.startswith("metadata"), self._to_validate_list)
+        unrecognised = set(self._to_validate_list).difference(self._expected_list)
+
+        if not unrecognised:
+            return
+
+        # Build any with wildcards
+        wildcard_matchers = [re.compile(s) for s in self._expected_list if "*" in s]
+        # Remove anything which matches any the regex wildcards
+        unrecognised = [s for s in unrecognised if not any(wild_matcher.match(s) for wild_matcher in wildcard_matchers)]
+
+        if len(unrecognised) > 0:
+            err = "The following keys were not recognised:\n"
+            err += "".join("{0} \n".format(k) for k in unrecognised)
+            raise TomlValidationError(err)
+
+    @staticmethod
+    @abstractmethod
+    def reference_schema():
+        """
+        Returns a dictionary layout of all supported keys
+        :return: Dictionary containing all keys, and values set to None
+        """
+        pass
+
+    @staticmethod
+    def _build_nested_keys(d, path="", current_out=None):
+        if not current_out:
+            current_out = []
+
+        def make_path(current_path, new_key):
+            return current_path + "." + new_key if current_path else new_key
+
+        for key, v in d.items():
+            new_path = make_path(path, key)
+            if isinstance(v, dict):
+                # Recurse into dict
+                current_out = TomlSchemaValidator._build_nested_keys(v, new_path, current_out)
+            elif isinstance(v, set):
+                # Pack all in from the set of names
+                for name in v:
+                    current_out.append(make_path(new_path, name))
+            else:
+                # This means its a value type with nothing special, so keep name
+                current_out.append(new_path)
+
+        return current_out

scripts/SANS/sans/user_file/toml_parsers/toml_parser.py

@@ -8,6 +8,7 @@
 from sans.state.IStateParser import IStateParser
 from sans.user_file.toml_parsers.toml_reader import TomlReader
 from sans.user_file.toml_parsers.toml_v1_parser import TomlV1Parser
+from sans.user_file.toml_parsers.toml_v2_parser import TomlV2Parser
 
 
 class TomlParser(object):
@@ -33,5 +34,7 @@ def get_versioned_parser(toml_dict, file_information) -> IStateParser:
         version = toml_dict["toml_file_version"]
         if version == 1:
             return TomlV1Parser(toml_dict, file_information=file_information)
+        if version == 2:
+            return TomlV2Parser(toml_dict, file_information=file_information)
         else:
             raise NotImplementedError("Version {0} of the SANS Toml Format is not supported".format(version))

scripts/SANS/sans/user_file/toml_parsers/toml_v1_parser.py

@@ -5,11 +5,8 @@
 #     & Institut Laue - Langevin
 # SPDX - License - Identifier: GPL - 3.0 +
 
-from typing import Optional
-
 from sans.common.enums import SANSInstrument, ReductionMode, DetectorType, RangeStepType, FitModeForMerge, DataType, FitType, RebinType
 from sans.common.general_functions import get_bank_for_spectrum_number, get_detector_types_from_instrument
-from sans.state.IStateParser import IStateParser
 from sans.state.StateObjects.StateAdjustment import StateAdjustment
 from sans.state.StateObjects.StateCalculateTransmission import get_calculate_transmission
 from sans.state.StateObjects.StateCompatibility import StateCompatibility
@@ -18,7 +15,7 @@
 from sans.state.StateObjects.StateMaskDetectors import get_mask_builder, StateMaskDetectors
 from sans.state.StateObjects.StateMoveDetectors import get_move_builder
 from sans.state.StateObjects.StateNormalizeToMonitor import get_normalize_to_monitor_builder
-from sans.state.StateObjects.StatePolarization import StatePolarization, StateComponent, StateFilter, StateField
+from sans.state.StateObjects.StatePolarization import StatePolarization
 from sans.state.StateObjects.StateReductionMode import StateReductionMode
 from sans.state.StateObjects.StateSave import StateSave
 from sans.state.StateObjects.StateScale import StateScale
@@ -28,22 +25,18 @@
 from sans.user_file.parser_helpers.toml_parser_impl_base import TomlParserImplBase
 from sans.user_file.parser_helpers.wavelength_parser import DuplicateWavelengthStates, WavelengthTomlParser
 from sans.user_file.toml_parsers.toml_v1_schema import TomlSchemaV1Validator
+from sans.user_file.toml_parsers.toml_base_parser import TomlParserBase
 
 
-class TomlV1Parser(IStateParser):
+class TomlV1Parser(TomlParserBase):
     def __init__(self, dict_to_parse, file_information, schema_validator=None):
-        self._validator = schema_validator if schema_validator else TomlSchemaV1Validator(dict_to_parse)
-        self._validator.validate()
-
-        self._implementation = None
-        data_info = self.get_state_data(file_information)
-        self._implementation = self._get_impl(dict_to_parse, data_info)
-        self._implementation.parse_all()
+        validator = schema_validator if schema_validator else TomlSchemaV1Validator(dict_to_parse)
+        super(TomlV1Parser, self).__init__(dict_to_parse, file_information, validator)
 
     @staticmethod
     def _get_impl(*args):
         # Wrapper which can replaced with a mock
-        return _TomlV1ParserImpl(*args)
+        return TomlV1ParserImpl(*args)
 
     def get_state_data(self, file_information):
         state_data = super().get_state_data(file_information)
@@ -76,8 +69,9 @@ def get_state_normalize_to_monitor(self, _):
     def get_state_reduction_mode(self):
         return self._implementation.reduction_mode
 
-    def get_state_polarization(self) -> Optional[StatePolarization]:
-        return self._implementation.polarization
+    def get_state_polarization(self) -> StatePolarization:
+        # Not supported by TOML V1, but we return a blank one to keep the parsing results consistent.
+        return StatePolarization()
 
     def get_state_save(self):
         return StateSave()
@@ -98,9 +92,9 @@ def get_state_wavelength_and_pixel_adjustment(self):
         return self._implementation.wavelength_and_pixel
 
 
-class _TomlV1ParserImpl(TomlParserImplBase):
+class TomlV1ParserImpl(TomlParserImplBase):
     def __init__(self, input_dict, data_info: StateData):
-        super(_TomlV1ParserImpl, self).__init__(toml_dict=input_dict)
+        super(TomlV1ParserImpl, self).__init__(toml_dict=input_dict)
         # Always take the instrument from the TOML file rather than guessing in the new parser
         data_info.instrument = self.instrument
         self._create_state_objs(data_info=data_info)
@@ -118,7 +112,6 @@ def parse_all(self):
         self._parse_transmission()
         self._parse_transmission_roi()
         self._parse_transmission_fitting()
-        self._parse_polarization()
 
     @property
     def instrument(self):
@@ -141,7 +134,6 @@ def _create_state_objs(self, data_info):
         self.scale = StateScale()
         self.wavelength = StateWavelength()
         self.wavelength_and_pixel = get_wavelength_and_pixel_adjustment_builder(data_info=data_info).build()
-        self.polarization = StatePolarization()
 
         # Ensure they are linked up correctly
         self.adjustment.calculate_transmission = self.calculate_transmission
@@ -519,60 +511,6 @@ def _parse_mask(self):
             if "stop" in phi_mask:
                 self.mask.phi_max = phi_mask["stop"]
 
-    def _parse_polarization(self):
-        polarization_dict = self.get_val("polarization")
-        if polarization_dict is None:
-            return
-        self.polarization.flipper_configuration = self.get_val("flipper_configuration", polarization_dict)
-        self.polarization.spin_configuration = self.get_val("spin_configuration", polarization_dict)
-        flipper_dicts = self.get_val("flipper", polarization_dict)
-        if flipper_dicts:
-            for flipper_dict in flipper_dicts.values():
-                self.polarization.flippers.append(self._parse_component(flipper_dict))
-        self.polarization.polarizer = self._parse_filter(self.get_val("polarizer", polarization_dict))
-        self.polarization.analyzer = self._parse_filter(self.get_val("analyzer", polarization_dict))
-        self.polarization.magnetic_field = self._parse_field(self.get_val("magnetic_field", polarization_dict))
-        self.polarization.electric_field = self._parse_field(self.get_val("electric_field", polarization_dict))
-        self.polarization.validate()
-
-    def _parse_component(self, component_dict: dict) -> StateComponent:
-        component_state = StateComponent()
-        if component_dict is None:
-            return component_state
-        component_state.idf_component_name = self.get_val("idf_component_name", component_dict)
-        component_state.device_name = self.get_val("device_name", component_dict)
-        component_state.device_type = self.get_val("device_type", component_dict)
-        location_dict = self.get_val("location", component_dict)
-        if location_dict:
-            component_state.location_x = self.get_val("x", location_dict)
-            component_state.location_y = self.get_val("y", location_dict)
-            component_state.location_z = self.get_val("z", location_dict)
-        component_state.transmission = self.get_val("transmission", component_dict)
-        component_state.efficiency = self.get_val("efficiency", component_dict)
-        return component_state
-
-    def _parse_filter(self, filter_dict: dict) -> StateFilter:
-        if filter_dict is None:
-            return StateFilter()
-        filter_state = self._parse_component(filter_dict)
-        filter_state.__class__ = StateFilter
-        filter_state.cell_length = self.get_val("cell_length", filter_dict)
-        filter_state.gas_pressure = self.get_val("gas_pressure", filter_dict)
-        return filter_state
-
-    def _parse_field(self, field_dict: dict) -> StateField:
-        field_state = StateField()
-        if field_dict is None:
-            return field_state
-        field_state.sample_strength_log = self.get_val("sample_strength_log", field_dict)
-        direction_dict = self.get_val("sample_direction", field_dict)
-        if direction_dict:
-            field_state.sample_direction_a = self.get_val("a", direction_dict)
-            field_state.sample_direction_p = self.get_val("p", direction_dict)
-            field_state.sample_direction_d = self.get_val("d", direction_dict)
-        field_state.sample_direction_log = self.get_val("sample_direction_log", field_dict)
-        return field_state
-
     @staticmethod
     def _get_1d_min_max(one_d_binning: str):
         # TODO: We have to do some special parsing for this type on behalf of the sans codebase