Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Have DESIGN_MATRIX not require design_sheet and default_sheet #10272

Merged
merged 1 commit into from
Mar 20, 2025
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 2 additions & 10 deletions src/ert/config/design_matrix.py
Original file line number Diff line number Diff line change
@@ -43,8 +43,8 @@ def __post_init__(self) -> None:
def from_config_list(cls, config_list: list[str | dict[str, str]]) -> DesignMatrix:
filename = Path(cast(str, config_list[0]))
options = cast(dict[str, str], config_list[1])
design_sheet = options.get("DESIGN_SHEET")
default_sheet = options.get("DEFAULT_SHEET")
design_sheet = options.get("DESIGN_SHEET", "DesignSheet")
default_sheet = options.get("DEFAULT_SHEET", "DefaultSheet")
errors = []
if filename.suffix not in {
".xlsx",
@@ -55,14 +55,6 @@ def from_config_list(cls, config_list: list[str | dict[str, str]]) -> DesignMatr
f"DESIGN_MATRIX must be of format .xls or .xlsx; is '{filename}'"
).set_context(config_list)
)
if design_sheet is None:
errors.append(
ErrorInfo("Missing required DESIGN_SHEET").set_context(config_list)
)
if default_sheet is None:
errors.append(
ErrorInfo("Missing required DEFAULT_SHEET").set_context(config_list)
)
if design_sheet is not None and design_sheet == default_sheet:
errors.append(
ErrorInfo(
4 changes: 2 additions & 2 deletions src/ert/config/parsing/config_schema.py
Original file line number Diff line number Diff line change
@@ -261,8 +261,8 @@ def install_job_directory_keyword() -> SchemaItem:
def design_matrix_keyword() -> SchemaItem:
return SchemaItem(
kw=ConfigKeys.DESIGN_MATRIX,
argc_min=2,
argc_max=2,
argc_min=1,
argc_max=3,
type_map=[
SchemaItemType.EXISTING_PATH,
SchemaItemType.STRING,
12 changes: 6 additions & 6 deletions tests/ert/ui_tests/cli/analysis/test_design_matrix.py
Original file line number Diff line number Diff line change
@@ -22,7 +22,7 @@

def _create_design_matrix(filename, design_sheet_df, default_sheet_df=None):
with pd.ExcelWriter(filename) as xl_write:
design_sheet_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
design_sheet_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
if default_sheet_df is not None:
default_sheet_df.to_excel(
xl_write, index=False, sheet_name="DefaultSheet", header=False
@@ -54,7 +54,7 @@ def test_run_poly_example_with_design_matrix():
NUM_REALIZATIONS 10
MIN_REALIZATIONS 1
GEN_DATA POLY_RES RESULT_FILE:poly.out
DESIGN_MATRIX poly_design.xlsx DESIGN_SHEET:DesignSheet01 DEFAULT_SHEET:DefaultSheet
DESIGN_MATRIX poly_design.xlsx
INSTALL_JOB poly_eval POLY_EVAL
FORWARD_MODEL poly_eval
"""
@@ -156,7 +156,7 @@ def test_run_poly_example_with_design_matrix_and_genkw_merge(default_values, err
MIN_REALIZATIONS 1
GEN_DATA POLY_RES RESULT_FILE:poly.out
GEN_KW COEFFS my_template my_output coeff_priors
DESIGN_MATRIX poly_design.xlsx DESIGN_SHEET:DesignSheet01 DEFAULT_SHEET:DefaultSheet
DESIGN_MATRIX poly_design.xlsx DESIGN_SHEET:DesignSheet DEFAULT_SHEET:DefaultSheet
INSTALL_JOB poly_eval POLY_EVAL
FORWARD_MODEL poly_eval
"""
@@ -268,8 +268,8 @@ def test_run_poly_example_with_multiple_design_matrix_instances():
NUM_REALIZATIONS 10
MIN_REALIZATIONS 1
GEN_DATA POLY_RES RESULT_FILE:poly.out
DESIGN_MATRIX poly_design_1.xlsx DESIGN_SHEET:DesignSheet01 DEFAULT_SHEET:DefaultSheet
DESIGN_MATRIX poly_design_2.xlsx DESIGN_SHEET:DesignSheet01 DEFAULT_SHEET:DefaultSheet
DESIGN_MATRIX poly_design_1.xlsx
DESIGN_MATRIX poly_design_2.xlsx
INSTALL_JOB poly_eval POLY_EVAL
FORWARD_MODEL poly_eval
"""
@@ -358,7 +358,7 @@ def test_design_matrix_on_esmda(experiment_mode, ensemble_name, iterations):
GEN_KW COEFFS_B coeff_priors_b
GEN_KW COEFFS_C coeff_priors_c
GEN_DATA POLY_RES RESULT_FILE:poly.out
DESIGN_MATRIX design_matrix.xlsx DESIGN_SHEET:DesignSheet01 DEFAULT_SHEET:DefaultSheet
DESIGN_MATRIX design_matrix.xlsx
INSTALL_JOB poly_eval POLY_EVAL
FORWARD_MODEL poly_eval
"""
34 changes: 0 additions & 34 deletions tests/ert/unit_tests/config/test_analysis_config.py
Original file line number Diff line number Diff line change
@@ -126,40 +126,6 @@ def test_invalid_design_matrix_format_raises_validation_error():
)


def test_design_matrix_without_design_sheet_raises_validation_error():
with pytest.raises(ConfigValidationError, match="Missing required DESIGN_SHEET"):
AnalysisConfig.from_dict(
{
ConfigKeys.DESIGN_MATRIX: [
[
"my_matrix.xlsx",
{
"DESIGN_": "design",
"DEFAULT_SHEET": "default",
},
]
],
}
)


def test_design_matrix_without_default_sheet_raises_validation_error():
with pytest.raises(ConfigValidationError, match="Missing required DEFAULT_SHEET"):
AnalysisConfig.from_dict(
{
ConfigKeys.DESIGN_MATRIX: [
[
"my_matrix.xlsx",
{
"DESIGN_SHEET": "design",
"DEFAULT_": "default",
},
]
],
}
)


def test_invalid_min_realization_percentage_raises_config_validation_error():
with pytest.raises(
ConfigValidationError,
34 changes: 28 additions & 6 deletions tests/ert/unit_tests/config/test_ert_config.py
Original file line number Diff line number Diff line change
@@ -2020,7 +2020,7 @@ def test_design2params_also_validates_design_matrix(tmp_path, caplog, monkeypatc
{"DESIGN2PARAMS": mock_design2params},
)
ErtConfig.from_file_contents(
f"NUM_REALIZATIONS 1\nFORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file}, <designsheet>=DesignSheet01,<defaultssheet>=DefaultSheet)"
f"NUM_REALIZATIONS 1\nFORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file}, <designsheet>=DesignSheet,<defaultssheet>=DefaultSheet)"
)
assert "DESIGN_MATRIX validation of DESIGN2PARAMS" in caplog.text

@@ -2058,8 +2058,8 @@ def test_two_design2params_validates_design_matrix_merging(
ErtConfig.from_file_contents(
f"""\
NUM_REALIZATIONS 1
FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file}, <designsheet>=DesignSheet01,<defaultssheet>=DefaultSheet)
FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file2}, <designsheet>=DesignSheet01,<defaultssheet>=DefaultSheet)
FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file}, <designsheet>=DesignSheet,<defaultssheet>=DefaultSheet)
FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file2}, <designsheet>=DesignSheet,<defaultssheet>=DefaultSheet)
"""
)
assert (
@@ -2112,12 +2112,34 @@ def test_three_design2params_validates_design_matrix_merging(
ErtConfig.from_file_contents(
f"""\
NUM_REALIZATIONS 1
FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file}, <designsheet>=DesignSheet01,<defaultssheet>=DefaultSheet)
FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file2}, <designsheet>=DesignSheet01,<defaultssheet>=DefaultSheet)
FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file3}, <designsheet>=DesignSheet01,<defaultssheet>=DefaultSheet)
FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file}, <designsheet>=DesignSheet,<defaultssheet>=DefaultSheet)
FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file2}, <designsheet>=DesignSheet,<defaultssheet>=DefaultSheet)
FORWARD_MODEL DESIGN2PARAMS(<xls_filename>={design_matrix_file3}, <designsheet>=DesignSheet,<defaultssheet>=DefaultSheet)
"""
)
assert (
"Design matrix merging would have failed due to: Design Matrices don't have the same active realizations"
in caplog.text
)


def test_design_matrix_default_argument(tmp_path):
design_matrix_file = tmp_path / "my_design_matrix.xlsx"
_create_design_matrix(
design_matrix_file,
pd.DataFrame(
{
"REAL": [1],
"a": [1],
"category": ["cat1"],
}
),
pd.DataFrame([["b", 1], ["c", 2]]),
)

config = ErtConfig.from_file_contents(
f"NUM_REALIZATIONS 1\nDESIGN_MATRIX {design_matrix_file}"
)
assert config.analysis_config.design_matrix
assert config.analysis_config.design_matrix.design_sheet == "DesignSheet"
assert config.analysis_config.design_matrix.default_sheet == "DefaultSheet"
6 changes: 2 additions & 4 deletions tests/ert/unit_tests/gui/simulation/test_run_dialog.py
Original file line number Diff line number Diff line change
@@ -684,7 +684,7 @@ def test_that_design_matrix_show_parameters_button_is_visible(
)
default_sheet_df = pd.DataFrame([["b", 1], ["c", 2]])
with pd.ExcelWriter(xls_filename) as xl_write:
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
xl_write, index=False, sheet_name="DefaultSheet", header=False
)
@@ -693,9 +693,7 @@ def test_that_design_matrix_show_parameters_button_is_visible(
with open(config_file, "w", encoding="utf-8") as f:
f.write("NUM_REALIZATIONS 1")
if design_matrix_entry:
f.write(
f"\nDESIGN_MATRIX {xls_filename} DESIGN_SHEET:DesignSheet01 DEFAULT_SHEET:DefaultSheet"
)
f.write(f"\nDESIGN_MATRIX {xls_filename}")

args_mock = Mock()
args_mock.config = config_file
48 changes: 24 additions & 24 deletions tests/ert/unit_tests/sensitivity_analysis/test_design_matrix.py
Original file line number Diff line number Diff line change
@@ -12,11 +12,11 @@

def _create_design_matrix(xls_path, design_matrix_df, default_sheet_df) -> DesignMatrix:
with pd.ExcelWriter(xls_path) as xl_write:
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
xl_write, index=False, sheet_name="DefaultValues", header=False
xl_write, index=False, sheet_name="DefaultSheet", header=False
)
return DesignMatrix(xls_path, "DesignSheet01", "DefaultValues")
return DesignMatrix(xls_path, "DesignSheet", "DefaultSheet")


@pytest.mark.parametrize(
@@ -145,11 +145,11 @@ def test_read_and_merge_with_existing_parameters(tmp_path, parameters, error_msg
)
default_sheet_df = pd.DataFrame([["a", 1], ["b", 4]])
with pd.ExcelWriter(design_path) as xl_write:
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
xl_write, index=False, sheet_name="DefaultValues", header=False
xl_write, index=False, sheet_name="DefaultSheet", header=False
)
design_matrix = DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
design_matrix = DesignMatrix(design_path, "DesignSheet", "DefaultSheet")
if error_msg:
with pytest.raises(ValueError, match=error_msg):
design_matrix.merge_with_existing_parameters(extra_genkw_config)
@@ -180,11 +180,11 @@ def test_reading_design_matrix(tmp_path):
)
default_sheet_df = pd.DataFrame([["one", 1], ["b", 4], ["d", "case_name"]])
with pd.ExcelWriter(design_path) as xl_write:
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
xl_write, index=False, sheet_name="DefaultValues", header=False
xl_write, index=False, sheet_name="DefaultSheet", header=False
)
design_matrix = DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
design_matrix = DesignMatrix(design_path, "DesignSheet", "DefaultSheet")
design_params = design_matrix.parameter_configuration
assert all(param in design_params for param in ("a", "b", "c", "one", "d"))
assert design_matrix.active_realizations == [True, True, False, False, True]
@@ -218,13 +218,13 @@ def test_reading_design_matrix_validate_reals(tmp_path, real_column, error_msg):
)
default_sheet_df = pd.DataFrame()
with pd.ExcelWriter(design_path) as xl_write:
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
xl_write, index=False, sheet_name="DefaultValues", header=False
xl_write, index=False, sheet_name="DefaultSheet", header=False
)

with pytest.raises(ValueError, match=error_msg):
DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
DesignMatrix(design_path, "DesignSheet", "DefaultSheet")


@pytest.mark.parametrize(
@@ -270,13 +270,13 @@ def test_reading_design_matrix_validate_headers(tmp_path, column_names, error_ms
)
default_sheet_df = pd.DataFrame([["one", 1], ["b", 4], ["d", 6]])
with pd.ExcelWriter(design_path) as xl_write:
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
xl_write, index=False, sheet_name="DefaultValues", header=False
xl_write, index=False, sheet_name="DefaultSheet", header=False
)

with pytest.raises(ValueError, match=error_msg):
DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
DesignMatrix(design_path, "DesignSheet", "DefaultSheet")


@pytest.mark.parametrize(
@@ -306,13 +306,13 @@ def test_reading_design_matrix_validate_cells(tmp_path, values, error_msg):
)
default_sheet_df = pd.DataFrame()
with pd.ExcelWriter(design_path) as xl_write:
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
xl_write, index=False, sheet_name="DefaultValues", header=False
xl_write, index=False, sheet_name="DefaultSheet", header=False
)

with pytest.raises(ValueError, match=error_msg):
DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
DesignMatrix(design_path, "DesignSheet", "DefaultSheet")


@pytest.mark.parametrize(
@@ -352,13 +352,13 @@ def test_reading_default_sheet_validation(tmp_path, data, error_msg):
)
default_sheet_df = pd.DataFrame(data)
with pd.ExcelWriter(design_path) as xl_write:
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
xl_write, index=False, sheet_name="DefaultValues", header=False
xl_write, index=False, sheet_name="DefaultSheet", header=False
)

with pytest.raises(ValueError, match=error_msg):
DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
DesignMatrix(design_path, "DesignSheet", "DefaultSheet")


def test_default_values_used(tmp_path):
@@ -373,11 +373,11 @@ def test_default_values_used(tmp_path):
)
default_sheet_df = pd.DataFrame([["one", 1], ["b", 4], ["d", "case_name"]])
with pd.ExcelWriter(design_path) as xl_write:
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
default_sheet_df.to_excel(
xl_write, index=False, sheet_name="DefaultValues", header=False
xl_write, index=False, sheet_name="DefaultSheet", header=False
)
design_matrix = DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
design_matrix = DesignMatrix(design_path, "DesignSheet", "DefaultSheet")
df = design_matrix.design_matrix_df
np.testing.assert_equal(df["one"], np.array([1, 1, 1, 1]))
np.testing.assert_equal(df["b"], np.array([0, 2, 0, 1]))
6 changes: 3 additions & 3 deletions tests/ert/unit_tests/test_libres_facade.py
Original file line number Diff line number Diff line change
@@ -276,11 +276,11 @@ def test_save_parameters_to_storage_from_design_dataframe(
c_values = np.random.default_rng().uniform(-5, 5, 10)
design_matrix_df = DataFrame({"a": a_values, "b": b_values, "c": c_values})
with ExcelWriter(design_path) as xl_write:
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet")
DataFrame().to_excel(
xl_write, index=False, sheet_name="DefaultValues", header=False
xl_write, index=False, sheet_name="DefaultSheet", header=False
)
design_matrix = DesignMatrix(design_path, "DesignSheet01", "DefaultValues")
design_matrix = DesignMatrix(design_path, "DesignSheet", "DefaultSheet")
with open_storage(tmp_path / "storage", mode="w") as storage:
experiment_id = storage.create_experiment(
parameters=[design_matrix.parameter_configuration]