From 5993c7e3a2bd73f04bc10cc4a989586a7834e622 Mon Sep 17 00:00:00 2001 From: Eivind Jahren Date: Wed, 22 Jan 2025 08:07:54 +0100 Subject: [PATCH] Skip slow tests in mac ci --- .github/workflows/build_and_test.yml | 1 + .github/workflows/test_ert.yml | 11 +++++++---- .github/workflows/test_everest.yml | 4 ++-- pyproject.toml | 2 +- tests/ert/ui_tests/cli/test_parameter_passing.py | 3 ++- tests/ert/ui_tests/gui/test_plotting_of_snake_oil.py | 1 + .../dark_storage/test_dark_storage_state.py | 4 +++- tests/ert/unit_tests/gui/model/test_snapshot.py | 1 + tests/everest/entry_points/test_everexport.py | 10 +++++----- tests/everest/functional/test_main_everest_entry.py | 2 +- tests/everest/test_detached.py | 2 +- tests/everest/test_everlint.py | 2 +- tests/everest/test_res_initialization.py | 2 +- tests/everest/test_yaml_parser.py | 4 ++-- 14 files changed, 29 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 16736f5146c..cc6f2b743b1 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -109,6 +109,7 @@ jobs: test-type: [ 'performance-tests', 'unit-tests', 'gui-tests', 'cli-tests' ] python-version: [ '3.12' ] os: [ 'macos-latest'] + select-string: '"not skip_mac_ci"' uses: ./.github/workflows/test_ert.yml with: diff --git a/.github/workflows/test_ert.yml b/.github/workflows/test_ert.yml index 9e0da2f02d7..548c7fa6f11 100644 --- a/.github/workflows/test_ert.yml +++ b/.github/workflows/test_ert.yml @@ -7,6 +7,9 @@ on: type: string test-type: type: string + select-string: + type: string + default: "" env: ERT_SHOW_BACKTRACE: 1 @@ -40,7 +43,7 @@ jobs: - name: GUI Test if: inputs.test-type == 'gui-tests' run: | - pytest --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -v --mpl --benchmark-disable tests/ert/ui_tests/gui --durations=25 + pytest -m ${{ inputs.select-string }} --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -v --mpl --benchmark-disable tests/ert/ui_tests/gui --durations=25 - name: Upload artifact images uses: actions/upload-artifact@v4 @@ -53,18 +56,18 @@ jobs: - name: CLI Test if: inputs.test-type == 'cli-tests' run: | - pytest --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -v --benchmark-disable --dist loadgroup tests/ert/ui_tests/cli --durations=25 + pytest -m ${{ inputs.select-string }} --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -v --benchmark-disable --dist loadgroup tests/ert/ui_tests/cli --durations=25 - name: Unit Test if: inputs.test-type == 'unit-tests' run: | - pytest --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -n logical --show-capture=stderr -v --benchmark-disable --mpl --dist loadgroup tests/ert/unit_tests --durations=25 + pytest -m ${{ inputs.select-string }} --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -n logical --show-capture=stderr -v --benchmark-disable --mpl --dist loadgroup tests/ert/unit_tests --durations=25 pytest --doctest-modules --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov2.xml src/ --ignore src/ert/dark_storage - name: Performance Test if: inputs.test-type == 'performance-tests' run: | - pytest --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -n logical --show-capture=stderr -v --benchmark-disable --dist loadgroup tests/ert/performance_tests --durations=25 + pytest -m ${{ inputs.select-string }} --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -n logical --show-capture=stderr -v --benchmark-disable --dist loadgroup tests/ert/performance_tests --durations=25 - name: Upload coverage to Codecov id: codecov1 diff --git a/.github/workflows/test_everest.yml b/.github/workflows/test_everest.yml index f3dfb60f845..f3e400d39cd 100644 --- a/.github/workflows/test_everest.yml +++ b/.github/workflows/test_everest.yml @@ -46,7 +46,7 @@ jobs: - name: Run Tests macOS if: ${{ inputs.test-type == 'test' && runner.os == 'macOS'}} run: | - python -m pytest tests/everest -n 4 --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -m "not integration_test and not fails_on_macos_github_workflow" --dist loadgroup -sv + python -m pytest tests/everest -n 4 --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -m "not integration_test and not skip_mac_ci" --dist loadgroup -sv - name: Run Integration Tests Linux if: ${{inputs.test-type == 'integration-test' && runner.os != 'macOS'}} @@ -56,7 +56,7 @@ jobs: - name: Run Integration Tests macOS if: ${{inputs.test-type == 'integration-test' && runner.os == 'macOS'}} run: | - python -m pytest tests/everest -n 4 --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -m "integration_test and not fails_on_macos_github_workflow" --dist loadgroup + python -m pytest tests/everest -n 4 --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -m "integration_test and not skip_mac_ci" --dist loadgroup - name: Build Documentation if: inputs.test-type == 'doc' diff --git a/pyproject.toml b/pyproject.toml index 010607a8298..c08c27b0a6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -171,7 +171,7 @@ markers = [ "slow", "everest_models_test", "integration_test", - "fails_on_macos_github_workflow", # Tests marked fail due to gui-related issues + "skip_mac_ci", ] log_cli = "false" asyncio_mode = "auto" diff --git a/tests/ert/ui_tests/cli/test_parameter_passing.py b/tests/ert/ui_tests/cli/test_parameter_passing.py index b808a467c65..e1090ed7f6f 100644 --- a/tests/ert/ui_tests/cli/test_parameter_passing.py +++ b/tests/ert/ui_tests/cli/test_parameter_passing.py @@ -19,7 +19,7 @@ import xtgeo from hypothesis import given, note, settings from hypothesis.extra.numpy import arrays -from pytest import MonkeyPatch, TempPathFactory +from pytest import MonkeyPatch, TempPathFactory, mark from resdata import ResDataType from resdata.grid import GridGenerator from resdata.resfile import ResdataKW @@ -409,6 +409,7 @@ def check(self, io_source: IoProvider, mask, num_realizations: int): max_size=3, ), ) +@mark.skip_mac_ci # test is slow def test_that_parameters_are_placed_in_the_runpath_as_expected( io_source: IoProvider, grid_format: Literal["grid", "egrid"], diff --git a/tests/ert/ui_tests/gui/test_plotting_of_snake_oil.py b/tests/ert/ui_tests/gui/test_plotting_of_snake_oil.py index 617acceb206..1e5691bc290 100644 --- a/tests/ert/ui_tests/gui/test_plotting_of_snake_oil.py +++ b/tests/ert/ui_tests/gui/test_plotting_of_snake_oil.py @@ -121,6 +121,7 @@ def plot_figure(qtbot, heat_equation_storage, snake_oil_case_storage, request): # The tolerance is chosen by guess, in one bug we observed a # mismatch of 58 which would fail the test by being above 10.0 @pytest.mark.mpl_image_compare(tolerance=10.0) +@pytest.mark.skip_mac_ci # test is slow def test_that_all_snake_oil_visualisations_matches_snapshot(plot_figure): return plot_figure diff --git a/tests/ert/unit_tests/dark_storage/test_dark_storage_state.py b/tests/ert/unit_tests/dark_storage/test_dark_storage_state.py index d93320e4982..6a218bd8bf9 100644 --- a/tests/ert/unit_tests/dark_storage/test_dark_storage_state.py +++ b/tests/ert/unit_tests/dark_storage/test_dark_storage_state.py @@ -111,4 +111,6 @@ def teardown(self): del os.environ["ERT_STORAGE_ENS_PATH"] -TestDarkStorage = pytest.mark.integration_test(DarkStorageStateTest.TestCase) +TestDarkStorage = pytest.mark.skip_mac_ci( + pytest.mark.integration_test(DarkStorageStateTest.TestCase) +) diff --git a/tests/ert/unit_tests/gui/model/test_snapshot.py b/tests/ert/unit_tests/gui/model/test_snapshot.py index b8a369bbd68..13a29eb1019 100644 --- a/tests/ert/unit_tests/gui/model/test_snapshot.py +++ b/tests/ert/unit_tests/gui/model/test_snapshot.py @@ -10,6 +10,7 @@ @pytest.mark.integration_test +@pytest.mark.skip_mac_ci # slow def test_using_qt_model_tester(qtmodeltester, full_snapshot): model = SnapshotModel() diff --git a/tests/everest/entry_points/test_everexport.py b/tests/everest/entry_points/test_everexport.py index 88dc26ff4c8..e16ca6f4a57 100644 --- a/tests/everest/entry_points/test_everexport.py +++ b/tests/everest/entry_points/test_everexport.py @@ -101,7 +101,7 @@ def test_everexport_entry_empty(mocked_func, cached_example): side_effect=validate_export_mock, ) @patch("everest.bin.utils.export_data") -@pytest.mark.fails_on_macos_github_workflow +@pytest.mark.skip_mac_ci def test_everexport_entry_batches(mocked_func, validate_export_mock, cached_example): """Test running everexport with the --batches flag""" _, config_file, _ = cached_example("math_func/config_minimal.yml") @@ -156,7 +156,7 @@ def test_everexport_entry_empty_export(mocked_func, cached_example): @patch("everest.bin.utils.export_data") -@pytest.mark.fails_on_macos_github_workflow +@pytest.mark.skip_mac_ci def test_everexport_entry_no_usr_def_ecl_keys(mocked_func, cached_example): """Test running everexport with config file containing only the keywords label without any list of keys""" @@ -190,7 +190,7 @@ def condition(config): @patch("everest.bin.utils.export_data") -@pytest.mark.fails_on_macos_github_workflow +@pytest.mark.skip_mac_ci def test_everexport_entry_internalized_usr_def_ecl_keys(mocked_func, cached_example): """Test running everexport with config file containing a key in the list of user defined ecl keywords, that has been internalized on @@ -226,7 +226,7 @@ def condition(config): @patch("everest.bin.utils.export_data") -@pytest.mark.fails_on_macos_github_workflow +@pytest.mark.skip_mac_ci def test_everexport_entry_non_int_usr_def_ecl_keys(mocked_func, caplog, cached_example): """Test running everexport when config file contains non internalized ecl keys in the user defined keywords list""" @@ -269,7 +269,7 @@ def condition(config): @patch("everest.bin.utils.export_data") -@pytest.mark.fails_on_macos_github_workflow +@pytest.mark.skip_mac_ci def test_everexport_entry_not_available_batches(mocked_func, caplog, cached_example): """Test running everexport when config file contains non existing batch numbers in the list of user defined batches""" diff --git a/tests/everest/functional/test_main_everest_entry.py b/tests/everest/functional/test_main_everest_entry.py index 028248e0d5a..1ec7695a22a 100644 --- a/tests/everest/functional/test_main_everest_entry.py +++ b/tests/everest/functional/test_main_everest_entry.py @@ -41,7 +41,7 @@ def test_everest_main_entry_bad_command(): @pytest.mark.flaky(reruns=5) -@pytest.mark.fails_on_macos_github_workflow +@pytest.mark.skip_mac_ci @pytest.mark.integration_test def test_everest_entry_run(cached_example): _, config_file, _ = cached_example("math_func/config_minimal.yml") diff --git a/tests/everest/test_detached.py b/tests/everest/test_detached.py index e3cba39d7fc..994de4c3fd8 100644 --- a/tests/everest/test_detached.py +++ b/tests/everest/test_detached.py @@ -42,7 +42,7 @@ @pytest.mark.flaky(reruns=5) @pytest.mark.integration_test -@pytest.mark.fails_on_macos_github_workflow +@pytest.mark.skip_mac_ci @pytest.mark.xdist_group(name="starts_everest") async def test_https_requests(copy_math_func_test_data_to_tmp): everest_config = EverestConfig.load_file("config_minimal.yml") diff --git a/tests/everest/test_everlint.py b/tests/everest/test_everlint.py index d76c74f89e7..12f5886a410 100644 --- a/tests/everest/test_everlint.py +++ b/tests/everest/test_everlint.py @@ -333,7 +333,7 @@ def test_date_type(date, valid, min_config): EverestConfig(**min_config) -@pytest.mark.fails_on_macos_github_workflow +@pytest.mark.skip_mac_ci def test_lint_everest_models_jobs(): pytest.importorskip("everest_models") config_file = relpath("../../test-data/everest/egg/everest/model/config.yml") diff --git a/tests/everest/test_res_initialization.py b/tests/everest/test_res_initialization.py index 27fc2e5db81..56f1dacf614 100644 --- a/tests/everest/test_res_initialization.py +++ b/tests/everest/test_res_initialization.py @@ -241,7 +241,7 @@ def test_summary_default(copy_egg_test_data_to_tmp): @hide_opm @skipif_no_everest_models @pytest.mark.everest_models_test -@pytest.mark.fails_on_macos_github_workflow +@pytest.mark.skip_mac_ci def test_summary_default_no_opm(copy_egg_test_data_to_tmp): config_dir = "everest/model" config_file = os.path.join(config_dir, "config.yml") diff --git a/tests/everest/test_yaml_parser.py b/tests/everest/test_yaml_parser.py index 8d5376ce418..d173c53c188 100644 --- a/tests/everest/test_yaml_parser.py +++ b/tests/everest/test_yaml_parser.py @@ -106,7 +106,7 @@ def test_valid_config_file(copy_test_data_to_tmp, monkeypatch): assert "could not find expected ':'" in parser.get_error() -@pytest.mark.fails_on_macos_github_workflow +@pytest.mark.skip_mac_ci @skipif_no_everest_models @pytest.mark.everest_models_test def test_valid_forward_model_config_files(copy_test_data_to_tmp, monkeypatch): @@ -121,7 +121,7 @@ def test_valid_forward_model_config_files(copy_test_data_to_tmp, monkeypatch): @skipif_no_everest_models @pytest.mark.everest_models_test -@pytest.mark.fails_on_macos_github_workflow +@pytest.mark.skip_mac_ci def test_invalid_forward_model_config_files(copy_test_data_to_tmp, monkeypatch): monkeypatch.chdir("valid_config_file/forward_models") parser = MockParser()