From 71e7ac173456036e91bac8dd50fae75d4dbaaef2 Mon Sep 17 00:00:00 2001
From: Jonathan Karlsen <JONAK@equinor.com>
Date: Wed, 12 Mar 2025 12:24:14 +0100
Subject: [PATCH] Have DESIGN_MATRIX not require design_sheet and default_sheet
This commit makes the DESIGN_MATRIX keyword not require arguments for design and default
sheet. Now DESIGN_SHEET defaults to `DesignSheet` and `DEFAULT_SHEET` defaults to "DefaultSheet".
This is the same defaults that are used in design2params.
---
src/ert/config/design_matrix.py | 12 +----
src/ert/config/parsing/config_schema.py | 4 +-
.../cli/analysis/test_design_matrix.py | 12 ++---
.../unit_tests/config/test_analysis_config.py | 34 -------------
.../ert/unit_tests/config/test_ert_config.py | 34 ++++++++++---
.../gui/simulation/test_run_dialog.py | 6 +--
.../test_design_matrix.py | 48 +++++++++----------
tests/ert/unit_tests/test_libres_facade.py | 6 +--
8 files changed, 67 insertions(+), 89 deletions(-)
diff --git a/src/ert/config/design_matrix.py b/src/ert/config/design_matrix.py
index 859324bc4a4..72a6126065f 100644
--- a/src/ert/config/design_matrix.py
+++ b/src/ert/config/design_matrix.py
@@ -43,8 +43,8 @@ def __post_init__(self) -> None:
def from_config_list(cls, config_list: list[str | dict[str, str]]) -> DesignMatrix:
filename = Path(cast(str, config_list[0]))
options = cast(dict[str, str], config_list[1])
- design_sheet = options.get("DESIGN_SHEET")
- default_sheet = options.get("DEFAULT_SHEET")
+ design_sheet = options.get("DESIGN_SHEET", "DesignSheet")
+ default_sheet = options.get("DEFAULT_SHEET", "DefaultSheet")
errors = []
if filename.suffix not in {
".xlsx",
@@ -55,14 +55,6 @@ def from_config_list(cls, config_list: list[str | dict[str, str]]) -> DesignMatr
f"DESIGN_MATRIX must be of format .xls or .xlsx; is '{filename}'"
).set_context(config_list)
)
- if design_sheet is None:
- errors.append(
- ErrorInfo("Missing required DESIGN_SHEET").set_context(config_list)
- )
- if default_sheet is None:
- errors.append(
- ErrorInfo("Missing required DEFAULT_SHEET").set_context(config_list)
- )
if design_sheet is not None and design_sheet == default_sheet:
errors.append(
ErrorInfo(
diff --git a/src/ert/config/parsing/config_schema.py b/src/ert/config/parsing/config_schema.py
index 9a9d026982f..329841bf097 100644
--- a/src/ert/config/parsing/config_schema.py
+++ b/src/ert/config/parsing/config_schema.py
@@ -261,8 +261,8 @@ def install_job_directory_keyword() -> SchemaItem:
def design_matrix_keyword() -> SchemaItem:
return SchemaItem(
kw=ConfigKeys.DESIGN_MATRIX,
- argc_min=2,
- argc_max=2,
+ argc_min=1,
+ argc_max=3,
type_map=[
SchemaItemType.EXISTING_PATH,
SchemaItemType.STRING,
diff --git a/tests/ert/ui_tests/cli/analysis/test_design_matrix.py b/tests/ert/ui_tests/cli/analysis/test_design_matrix.py
index 7e52ef403d5..6eb284c4f3d 100644
--- a/tests/ert/ui_tests/cli/analysis/test_design_matrix.py
+++ b/tests/ert/ui_tests/cli/analysis/test_design_matrix.py
@@ -22,7 +22,7 @@
def _create_design_matrix(filename, design_sheet_df, default_sheet_df=None):
with pd.ExcelWriter(filename) as xl_write:
- design_sheet_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
+ design_sheet_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
if default_sheet_df is not None:
default_sheet_df.to_excel(
xl_write, index=False, sheet_name="DefaultSheet", header=False
@@ -54,7 +54,7 @@ def test_run_poly_example_with_design_matrix():
NUM_REALIZATIONS 10
MIN_REALIZATIONS 1
GEN_DATA POLY_RES RESULT_FILE:poly.out
- DESIGN_MATRIX poly_design.xlsx DESIGN_SHEET:DesignSheet01 DEFAULT_SHEET:DefaultSheet
+ DESIGN_MATRIX poly_design.xlsx
INSTALL_JOB poly_eval POLY_EVAL
FORWARD_MODEL poly_eval
"""
@@ -156,7 +156,7 @@ def test_run_poly_example_with_design_matrix_and_genkw_merge(default_values, err
MIN_REALIZATIONS 1
GEN_DATA POLY_RES RESULT_FILE:poly.out
GEN_KW COEFFS my_template my_output coeff_priors
- DESIGN_MATRIX poly_design.xlsx DESIGN_SHEET:DesignSheet01 DEFAULT_SHEET:DefaultSheet
+ DESIGN_MATRIX poly_design.xlsx DESIGN_SHEET:DesignSheet DEFAULT_SHEET:DefaultSheet
INSTALL_JOB poly_eval POLY_EVAL
FORWARD_MODEL poly_eval
"""
@@ -268,8 +268,8 @@ def test_run_poly_example_with_multiple_design_matrix_instances():
NUM_REALIZATIONS 10
MIN_REALIZATIONS 1
GEN_DATA POLY_RES RESULT_FILE:poly.out
- DESIGN_MATRIX poly_design_1.xlsx DESIGN_SHEET:DesignSheet01 DEFAULT_SHEET:DefaultSheet
- DESIGN_MATRIX poly_design_2.xlsx DESIGN_SHEET:DesignSheet01 DEFAULT_SHEET:DefaultSheet
+ DESIGN_MATRIX poly_design_1.xlsx
+ DESIGN_MATRIX poly_design_2.xlsx
INSTALL_JOB poly_eval POLY_EVAL
FORWARD_MODEL poly_eval
"""
@@ -358,7 +358,7 @@ def test_design_matrix_on_esmda(experiment_mode, ensemble_name, iterations):
GEN_KW COEFFS_B coeff_priors_b
GEN_KW COEFFS_C coeff_priors_c
GEN_DATA POLY_RES RESULT_FILE:poly.out
- DESIGN_MATRIX design_matrix.xlsx DESIGN_SHEET:DesignSheet01 DEFAULT_SHEET:DefaultSheet
+ DESIGN_MATRIX design_matrix.xlsx
INSTALL_JOB poly_eval POLY_EVAL
FORWARD_MODEL poly_eval
"""
diff --git a/tests/ert/unit_tests/config/test_analysis_config.py b/tests/ert/unit_tests/config/test_analysis_config.py
index 7abaf145a78..bc416ba4439 100644
--- a/tests/ert/unit_tests/config/test_analysis_config.py
+++ b/tests/ert/unit_tests/config/test_analysis_config.py
@@ -126,40 +126,6 @@ def test_invalid_design_matrix_format_raises_validation_error():
)
-def test_design_matrix_without_design_sheet_raises_validation_error():
- with pytest.raises(ConfigValidationError, match="Missing required DESIGN_SHEET"):
- AnalysisConfig.from_dict(
- {
- ConfigKeys.DESIGN_MATRIX: [
- [
- "my_matrix.xlsx",
- {
- "DESIGN_": "design",
- "DEFAULT_SHEET": "default",
- },
- ]
- ],
- }
- )
-
-
-def test_design_matrix_without_default_sheet_raises_validation_error():
- with pytest.raises(ConfigValidationError, match="Missing required DEFAULT_SHEET"):
- AnalysisConfig.from_dict(
- {
- ConfigKeys.DESIGN_MATRIX: [
- [
- "my_matrix.xlsx",
- {
- "DESIGN_SHEET": "design",
- "DEFAULT_": "default",
- },
- ]
- ],
- }
- )
-
-
def test_invalid_min_realization_percentage_raises_config_validation_error():
with pytest.raises(
ConfigValidationError,
diff --git a/tests/ert/unit_tests/config/test_ert_config.py b/tests/ert/unit_tests/config/test_ert_config.py
index ef27bae01a3..cfa75ee4eb8 100644
--- a/tests/ert/unit_tests/config/test_ert_config.py
+++ b/tests/ert/unit_tests/config/test_ert_config.py
@@ -2020,7 +2020,7 @@ def test_design2params_also_validates_design_matrix(tmp_path, caplog, monkeypatc
{"DESIGN2PARAMS": mock_design2params},
)
ErtConfig.from_file_contents(
- f"NUM_REALIZATIONS 1\nFORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file}, <designsheet>=DesignSheet01,<defaultssheet>=DefaultSheet)"
+ f"NUM_REALIZATIONS 1\nFORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file}, <designsheet>=DesignSheet,<defaultssheet>=DefaultSheet)"
)
assert "DESIGN_MATRIX validation of DESIGN2PARAMS" in caplog.text
@@ -2058,8 +2058,8 @@ def test_two_design2params_validates_design_matrix_merging(
ErtConfig.from_file_contents(
f"""\
NUM_REALIZATIONS 1
- FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file}, <designsheet>=DesignSheet01,<defaultssheet>=DefaultSheet)
- FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file2}, <designsheet>=DesignSheet01,<defaultssheet>=DefaultSheet)
+ FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file}, <designsheet>=DesignSheet,<defaultssheet>=DefaultSheet)
+ FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file2}, <designsheet>=DesignSheet,<defaultssheet>=DefaultSheet)
"""
)
assert (
@@ -2112,12 +2112,34 @@ def test_three_design2params_validates_design_matrix_merging(
ErtConfig.from_file_contents(
f"""\
NUM_REALIZATIONS 1
- FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file}, <designsheet>=DesignSheet01,<defaultssheet>=DefaultSheet)
- FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file2}, <designsheet>=DesignSheet01,<defaultssheet>=DefaultSheet)
- FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file3}, <designsheet>=DesignSheet01,<defaultssheet>=DefaultSheet)
+ FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file}, <designsheet>=DesignSheet,<defaultssheet>=DefaultSheet)
+ FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file2}, <designsheet>=DesignSheet,<defaultssheet>=DefaultSheet)
+ FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file3}, <designsheet>=DesignSheet,<defaultssheet>=DefaultSheet)
"""
)
assert (
"Design matrix merging would have failed due to: Design Matrices don't have the same active realizations"
in caplog.text
)
+
+
+def test_design_matrix_default_argument(tmp_path):
+ design_matrix_file = tmp_path / "my_design_matrix.xlsx"
+ _create_design_matrix(
+ design_matrix_file,
+ pd.DataFrame(
+ {
+ "REAL": [1],
+ "a": [1],
+ "category": ["cat1"],
+ }
+ ),
+ pd.DataFrame([["b", 1], ["c", 2]]),
+ )
+
+ config = ErtConfig.from_file_contents(
+ f"NUM_REALIZATIONS 1\nDESIGN_MATRIX {design_matrix_file}"
+ )
+ assert config.analysis_config.design_matrix
+ assert config.analysis_config.design_matrix.design_sheet == "DesignSheet"
+ assert config.analysis_config.design_matrix.default_sheet == "DefaultSheet"
diff --git a/tests/ert/unit_tests/gui/simulation/test_run_dialog.py b/tests/ert/unit_tests/gui/simulation/test_run_dialog.py
index 0301bf2ad87..3599de7f146 100644
--- a/tests/ert/unit_tests/gui/simulation/test_run_dialog.py
+++ b/tests/ert/unit_tests/gui/simulation/test_run_dialog.py
@@ -684,7 +684,7 @@ def test_that_design_matrix_show_parameters_button_is_visible(
)
default_sheet_df = pd.DataFrame([["b", 1], ["c", 2]])
with pd.ExcelWriter(xls_filename) as xl_write:
- design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
+ design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
xl_write, index=False, sheet_name="DefaultSheet", header=False
)
@@ -693,9 +693,7 @@ def test_that_design_matrix_show_parameters_button_is_visible(
with open(config_file, "w", encoding="utf-8") as f:
f.write("NUM_REALIZATIONS 1")
if design_matrix_entry:
- f.write(
- f"\nDESIGN_MATRIX {xls_filename} DESIGN_SHEET:DesignSheet01 DEFAULT_SHEET:DefaultSheet"
- )
+ f.write(f"\nDESIGN_MATRIX {xls_filename}")
args_mock = Mock()
args_mock.config = config_file
diff --git a/tests/ert/unit_tests/sensitivity_analysis/test_design_matrix.py b/tests/ert/unit_tests/sensitivity_analysis/test_design_matrix.py
index 68605b52833..37db2bc3885 100644
--- a/tests/ert/unit_tests/sensitivity_analysis/test_design_matrix.py
+++ b/tests/ert/unit_tests/sensitivity_analysis/test_design_matrix.py
@@ -12,11 +12,11 @@
def _create_design_matrix(xls_path, design_matrix_df, default_sheet_df) -> DesignMatrix:
with pd.ExcelWriter(xls_path) as xl_write:
- design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
+ design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
- xl_write, index=False, sheet_name="DefaultValues", header=False
+ xl_write, index=False, sheet_name="DefaultSheet", header=False
)
- return DesignMatrix(xls_path, "DesignSheet01", "DefaultValues")
+ return DesignMatrix(xls_path, "DesignSheet", "DefaultSheet")
@pytest.mark.parametrize(
@@ -145,11 +145,11 @@ def test_read_and_merge_with_existing_parameters(tmp_path, parameters, error_msg
)
default_sheet_df = pd.DataFrame([["a", 1], ["b", 4]])
with pd.ExcelWriter(design_path) as xl_write:
- design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
+ design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
- xl_write, index=False, sheet_name="DefaultValues", header=False
+ xl_write, index=False, sheet_name="DefaultSheet", header=False
)
- design_matrix = DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
+ design_matrix = DesignMatrix(design_path, "DesignSheet", "DefaultSheet")
if error_msg:
with pytest.raises(ValueError, match=error_msg):
design_matrix.merge_with_existing_parameters(extra_genkw_config)
@@ -180,11 +180,11 @@ def test_reading_design_matrix(tmp_path):
)
default_sheet_df = pd.DataFrame([["one", 1], ["b", 4], ["d", "case_name"]])
with pd.ExcelWriter(design_path) as xl_write:
- design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
+ design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
- xl_write, index=False, sheet_name="DefaultValues", header=False
+ xl_write, index=False, sheet_name="DefaultSheet", header=False
)
- design_matrix = DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
+ design_matrix = DesignMatrix(design_path, "DesignSheet", "DefaultSheet")
design_params = design_matrix.parameter_configuration
assert all(param in design_params for param in ("a", "b", "c", "one", "d"))
assert design_matrix.active_realizations == [True, True, False, False, True]
@@ -218,13 +218,13 @@ def test_reading_design_matrix_validate_reals(tmp_path, real_column, error_msg):
)
default_sheet_df = pd.DataFrame()
with pd.ExcelWriter(design_path) as xl_write:
- design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
+ design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
- xl_write, index=False, sheet_name="DefaultValues", header=False
+ xl_write, index=False, sheet_name="DefaultSheet", header=False
)
with pytest.raises(ValueError, match=error_msg):
- DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
+ DesignMatrix(design_path, "DesignSheet", "DefaultSheet")
@pytest.mark.parametrize(
@@ -270,13 +270,13 @@ def test_reading_design_matrix_validate_headers(tmp_path, column_names, error_ms
)
default_sheet_df = pd.DataFrame([["one", 1], ["b", 4], ["d", 6]])
with pd.ExcelWriter(design_path) as xl_write:
- design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
+ design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
- xl_write, index=False, sheet_name="DefaultValues", header=False
+ xl_write, index=False, sheet_name="DefaultSheet", header=False
)
with pytest.raises(ValueError, match=error_msg):
- DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
+ DesignMatrix(design_path, "DesignSheet", "DefaultSheet")
@pytest.mark.parametrize(
@@ -306,13 +306,13 @@ def test_reading_design_matrix_validate_cells(tmp_path, values, error_msg):
)
default_sheet_df = pd.DataFrame()
with pd.ExcelWriter(design_path) as xl_write:
- design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
+ design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
- xl_write, index=False, sheet_name="DefaultValues", header=False
+ xl_write, index=False, sheet_name="DefaultSheet", header=False
)
with pytest.raises(ValueError, match=error_msg):
- DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
+ DesignMatrix(design_path, "DesignSheet", "DefaultSheet")
@pytest.mark.parametrize(
@@ -352,13 +352,13 @@ def test_reading_default_sheet_validation(tmp_path, data, error_msg):
)
default_sheet_df = pd.DataFrame(data)
with pd.ExcelWriter(design_path) as xl_write:
- design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
+ design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
- xl_write, index=False, sheet_name="DefaultValues", header=False
+ xl_write, index=False, sheet_name="DefaultSheet", header=False
)
with pytest.raises(ValueError, match=error_msg):
- DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
+ DesignMatrix(design_path, "DesignSheet", "DefaultSheet")
def test_default_values_used(tmp_path):
@@ -373,11 +373,11 @@ def test_default_values_used(tmp_path):
)
default_sheet_df = pd.DataFrame([["one", 1], ["b", 4], ["d", "case_name"]])
with pd.ExcelWriter(design_path) as xl_write:
- design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
+ design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
- xl_write, index=False, sheet_name="DefaultValues", header=False
+ xl_write, index=False, sheet_name="DefaultSheet", header=False
)
- design_matrix = DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
+ design_matrix = DesignMatrix(design_path, "DesignSheet", "DefaultSheet")
df = design_matrix.design_matrix_df
np.testing.assert_equal(df["one"], np.array([1, 1, 1, 1]))
np.testing.assert_equal(df["b"], np.array([0, 2, 0, 1]))
diff --git a/tests/ert/unit_tests/test_libres_facade.py b/tests/ert/unit_tests/test_libres_facade.py
index 1f8646a7ca4..4a8c45a021e 100644
--- a/tests/ert/unit_tests/test_libres_facade.py
+++ b/tests/ert/unit_tests/test_libres_facade.py
@@ -276,11 +276,11 @@ def test_save_parameters_to_storage_from_design_dataframe(
c_values = np.random.default_rng().uniform(-5, 5, 10)
design_matrix_df = DataFrame({"a": a_values, "b": b_values, "c": c_values})
with ExcelWriter(design_path) as xl_write:
- design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
+ design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
DataFrame().to_excel(
- xl_write, index=False, sheet_name="DefaultValues", header=False
+ xl_write, index=False, sheet_name="DefaultSheet", header=False
)
- design_matrix = DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
+ design_matrix = DesignMatrix(design_path, "DesignSheet", "DefaultSheet")
with open_storage(tmp_path / "storage", mode="w") as storage:
experiment_id = storage.create_experiment(
parameters=[design_matrix.parameter_configuration]