diff --git a/src/model_config_tests/config_tests/conftest.py b/src/model_config_tests/config_tests/conftest.py index 74b98e1..0c8e3b9 100644 --- a/src/model_config_tests/config_tests/conftest.py +++ b/src/model_config_tests/config_tests/conftest.py @@ -131,6 +131,18 @@ def pytest_configure(config): "markers", "repro_determinism_restart: mark tests that check determinism restart", ) + config.addinivalue_line( + "markers", + "repro_payu_setup: mark tests that check payu setup reproducibility", + ) + config.addinivalue_line( + "markers", + "manifests: mark tests that check payu setup does not change manifests files or md5", + ) + config.addinivalue_line( + "markers", + "manifests_unchanged: mark tests that check payu setup does not change manifests files", + ) config.addinivalue_line("markers", "slow: mark tests that are slow to run") config.addinivalue_line( "markers", diff --git a/src/model_config_tests/config_tests/test_bit_reproducibility.py b/src/model_config_tests/config_tests/test_bit_reproducibility.py index c4852c4..ad79fd5 100644 --- a/src/model_config_tests/config_tests/test_bit_reproducibility.py +++ b/src/model_config_tests/config_tests/test_bit_reproducibility.py @@ -9,7 +9,7 @@ import pytest -from model_config_tests.exp_test_helper import Experiments, ExpTestHelper +from model_config_tests.exp_test_helper import Experiments, ExpTestHelper, setup_exp from model_config_tests.util import DAY_IN_SECONDS, HOUR_IN_SECONDS # Names of shared experiments @@ -328,3 +328,32 @@ def test_repro_determinism_restart( ) assert produced == expected + + +@pytest.mark.repro +@pytest.mark.manifests +@pytest.mark.repro_payu_setup +def test_payu_setup_repro_flag(control_path, output_path): + """ + Test payu setup with `--repro` flag which errors if md5 of any files in payu manifests are changed. + """ + experiment = setup_exp(control_path, output_path, exp_name="repro_payu_setup") + try: + experiment.setup_reproduce() + except Exception as error: + pytest.fail(f"{error}") + + +@pytest.mark.manifests +@pytest.mark.manifests_unchanged +def test_manifests_unchanged(control_path, output_path): + """ + Test payu setup with `git diff` which errors if any files in payu manifests are changed. + """ + experiment = setup_exp( + control_path, output_path, exp_name="setup_unchanged_manifests" + ) + try: + experiment.setup_manifests_unchanged() + except Exception as error: + pytest.fail(f"{error}") diff --git a/src/model_config_tests/exp_test_helper.py b/src/model_config_tests/exp_test_helper.py index eaabbf5..51ddc28 100644 --- a/src/model_config_tests/exp_test_helper.py +++ b/src/model_config_tests/exp_test_helper.py @@ -82,6 +82,86 @@ def has_run(self): """ return self.model.output_exists() + def setup_reproduce(self): + """ + Run payu setup with `--repro` flag to check if md5 hashes have changed in the manifests. + """ + owd = Path.cwd() + # Change to experiment directory and run. + os.chdir(self.control_path) + + try: + setup_command = [ + "payu", + "setup", + "--lab", + str(self.lab_path), + "--reproduce", + ] + print(f"Running payu setup command: {setup_command}") + result = sp.run(setup_command, capture_output=True, text=True) + finally: + # Change back to original working directory + os.chdir(owd) + + if result.returncode != 0: + raise RuntimeError( + f"Failed to run payu setup with --reproduce. Error: {result.stderr}\n" + f"Full output: {result.stdout}" + ) + + def setup_manifests_unchanged(self): + """ + Run payu setup command and check if manifests files have been changed with `git diff`. + """ + owd = Path.cwd() + # Change to experiment directory and run. + os.chdir(self.control_path) + + try: + setup_command = ["payu", "setup", "--lab", str(self.lab_path)] + print(f"Running payu setup command: {setup_command}") + setup_result = sp.run(setup_command, capture_output=True, text=True) + if setup_result.returncode != 0: + raise RuntimeError( + f"Error during payu setup. \n" + f"{'='*10}STDOUT{'='*10}\n {setup_result.stdout}\n" + f"{'='*10}STDERR{'='*10}\n {setup_result.stderr}\n" + ) + + result = sp.run( + ["git", "diff", "--name-only", "manifests/"], + capture_output=True, + text=True, + ) + finally: + # Change back to original working directory + os.chdir(owd) + + if result.stdout != "": + # Collect and display the top 10 lines of the diff for each modified file + files = result.stdout.strip().split("\n") + error_message = "Modifications are detected in file:\n" + error_message += "\n".join(" - " + file for file in files) + "\n" + error_message += "\nIf md5 hashes have changed, this indicates file contents being different." + error_message += """ +If binhashes/paths have changed but md5's are the same, +this will mean the configuration can reproduce the manifests +but `payu setup` will take longer to run as it needs to re-calculate all the md5 hashes. + """ + for file in files: + diff_details = sp.run( + ["git", "-C", str(self.control_path), "diff", f"{file}"], + capture_output=True, + text=True, + ) + diff_lines = diff_details.stdout.splitlines() + top_lines = "\n".join(diff_lines[2:12]) + if len(diff_lines) > 12: + top_lines += "\n... (truncated)" + error_message += f"\n{'='*10} Diff for {file} {'='*10}\n{top_lines}\n" + raise RuntimeError(f"{error_message}") + def setup_for_test_run(self): """ Various config.yaml settings need to be modified in order to run in the diff --git a/tests/test_exp_test_helper.py b/tests/test_exp_test_helper.py index 7ffc633..0a7417e 100644 --- a/tests/test_exp_test_helper.py +++ b/tests/test_exp_test_helper.py @@ -1,7 +1,7 @@ import shutil import subprocess from pathlib import Path -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, Mock, patch import pytest import yaml @@ -556,3 +556,74 @@ def test_experiments_check_experiment_error(tmp_path): ) with pytest.raises(RuntimeError, match=error_msg): exps.check_experiment("error_exp") + + +@patch("subprocess.run") +def test_setup_reproduce_error(mock_run, exp): + """Test that payu setup --repro fails raises an error and return to original work directory""" + # Mock the payu setup --repro to fail + mock_result = MagicMock() + mock_result.returncode = 1 + mock_result.stderr = "MD5 mismatch" + mock_result.stdout = "Check manifest" + mock_run.return_value = mock_result + + with pytest.raises(RuntimeError) as excinfo: + exp.setup_reproduce() + + assert ( + f"Failed to run payu setup with --reproduce. Error: {mock_result.stderr}" + in str(excinfo.value) + ) + assert f"Full output: {mock_result.stdout}" in str(excinfo.value) + + # assert returning to the original work directory + assert Path.cwd() == exp.control_path + + +@patch("subprocess.run") +def test_setup_manifests_unchanged_fail_setup(mock_run, exp): + """Test that an error is raised when payu setup fails in setup_manifests_unchanged()""" + # Mock the payu setup --repro to fail with unchanged manifests + mock_result = MagicMock() + mock_result.returncode = 1 + mock_result.stderr = "Setup failed" + mock_result.stdout = "Payu setup output" + mock_run.return_value = mock_result + + with pytest.raises(RuntimeError) as excinfo: + exp.setup_manifests_unchanged() + + assert "Error during payu setup." in str(excinfo.value) + assert f"{'='*10}STDOUT{'='*10}\n {mock_result.stdout}\n" in str(excinfo.value) + + # assert returning to the original work directory + assert Path.cwd() == exp.control_path + + +@patch("subprocess.run") +def test_setup_manifests_unchanged_show_changes(mock_run, exp): + """Test that when manifests are changed, the `git diff` results are printed to stdout""" + # Mock the `payu setup` succeed first + setup_success = MagicMock(returncode=0, stdout="Payu setup succeeded") + + # Then mock the `git diff --name-only` to show which files are changed + git_diff_name_only = MagicMock(returncode=1, stdout="manifests/input.yaml") + + # Mock the `git diff` to show the detailed changes in the file + git_diff_output = ( + "--- a/manifests/input.yaml\n+++ b/manifests/input.yaml\n+new line\n-old line" + ) + git_diff_run = MagicMock(returncode=0, stdout=git_diff_output) + + # Run these mocks in sequence + mock_run.side_effect = [setup_success, git_diff_name_only, git_diff_run] + + with pytest.raises(RuntimeError) as excinfo: + exp.setup_manifests_unchanged() + + assert "Modifications are detected in file:\n" in str(excinfo.value) + assert git_diff_output in str(excinfo.value) + + # assert returning to the original work directory + assert Path.cwd() == exp.control_path