Skip slow tests in mac ci

Author: eivindjahren

.github/workflows/build_and_test.yml

@@ -109,6 +109,7 @@ jobs:
         test-type: [ 'performance-tests', 'unit-tests', 'gui-tests', 'cli-tests' ]
         python-version: [ '3.12' ]
         os: [ 'macos-latest']
+        select-string: ['"not skip_mac_ci"']
 
     uses: ./.github/workflows/test_ert.yml
     with:

.github/workflows/test_ert.yml

@@ -7,6 +7,9 @@ on:
         type: string
       test-type:
         type: string
+      select-string:
+        type: string
+        default: "''"
 
 env:
   ERT_SHOW_BACKTRACE: 1
@@ -40,7 +43,7 @@ jobs:
     - name: GUI Test
       if: inputs.test-type == 'gui-tests'
       run: |
-        pytest --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -v --mpl --benchmark-disable tests/ert/ui_tests/gui --durations=25
+        pytest -m ${{ inputs.select-string }} --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -v --mpl --benchmark-disable tests/ert/ui_tests/gui --durations=25
 
     - name: Upload artifact images
       uses: actions/upload-artifact@v4
@@ -53,18 +56,18 @@ jobs:
     - name: CLI Test
       if: inputs.test-type == 'cli-tests'
       run: |
-        pytest --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -v --benchmark-disable  --dist loadgroup tests/ert/ui_tests/cli --durations=25
+        pytest -m ${{ inputs.select-string }} --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -v --benchmark-disable  --dist loadgroup tests/ert/ui_tests/cli --durations=25
 
     - name: Unit Test
       if: inputs.test-type == 'unit-tests'
       run: |
-        pytest --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -n logical --show-capture=stderr -v --benchmark-disable --mpl --dist loadgroup tests/ert/unit_tests --durations=25
+        pytest -m ${{ inputs.select-string }} --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -n logical --show-capture=stderr -v --benchmark-disable --mpl --dist loadgroup tests/ert/unit_tests --durations=25
         pytest --doctest-modules --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov2.xml src/ --ignore src/ert/dark_storage
 
     - name: Performance Test
       if: inputs.test-type == 'performance-tests'
       run: |
-        pytest --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -n logical --show-capture=stderr -v --benchmark-disable  --dist loadgroup tests/ert/performance_tests --durations=25
+        pytest -m ${{ inputs.select-string }} --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -n logical --show-capture=stderr -v --benchmark-disable  --dist loadgroup tests/ert/performance_tests --durations=25
 
     - name: Upload coverage to Codecov
       id: codecov1

.github/workflows/test_everest.yml

@@ -46,7 +46,7 @@ jobs:
     - name: Run Tests macOS
       if: ${{ inputs.test-type == 'test' && runner.os == 'macOS'}}
       run: |
-        python -m pytest tests/everest -n 4 --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -m "not integration_test and not fails_on_macos_github_workflow" --dist loadgroup -sv
+        python -m pytest tests/everest -n 4 --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -m "not integration_test and not skip_mac_ci" --dist loadgroup -sv
 
     - name: Run Integration Tests Linux
       if: ${{inputs.test-type == 'integration-test' && runner.os != 'macOS'}}
@@ -56,7 +56,7 @@ jobs:
     - name: Run Integration Tests macOS
       if: ${{inputs.test-type == 'integration-test' && runner.os == 'macOS'}}
       run: |
-        python -m pytest tests/everest -n 4 --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -m "integration_test and not fails_on_macos_github_workflow" --dist loadgroup
+        python -m pytest tests/everest -n 4 --cov=ert --cov=everest --cov=_ert --cov-report=xml:cov1.xml --junit-xml=junit.xml -o junit_family=legacy -m "integration_test and not skip_mac_ci" --dist loadgroup
 
     - name: Build Documentation
       if: inputs.test-type == 'doc'

pyproject.toml

@@ -171,7 +171,7 @@ markers = [
     "slow",
     "everest_models_test",
     "integration_test",
-    "fails_on_macos_github_workflow", # Tests marked fail due to gui-related issues
+    "skip_mac_ci",
 ]
 log_cli = "false"
 asyncio_mode = "auto"

tests/ert/ui_tests/cli/test_parameter_passing.py

@@ -19,7 +19,7 @@
 import xtgeo
 from hypothesis import given, note, settings
 from hypothesis.extra.numpy import arrays
-from pytest import MonkeyPatch, TempPathFactory
+from pytest import MonkeyPatch, TempPathFactory, mark
 from resdata import ResDataType
 from resdata.grid import GridGenerator
 from resdata.resfile import ResdataKW
@@ -409,6 +409,7 @@ def check(self, io_source: IoProvider, mask, num_realizations: int):
         max_size=3,
     ),
 )
+@mark.skip_mac_ci  # test is slow
 def test_that_parameters_are_placed_in_the_runpath_as_expected(
     io_source: IoProvider,
     grid_format: Literal["grid", "egrid"],

tests/ert/ui_tests/gui/test_plotting_of_snake_oil.py

@@ -121,6 +121,7 @@ def plot_figure(qtbot, heat_equation_storage, snake_oil_case_storage, request):
 # The tolerance is chosen by guess, in one bug we observed a
 # mismatch of 58 which would fail the test by being above 10.0
 @pytest.mark.mpl_image_compare(tolerance=10.0)
+@pytest.mark.skip_mac_ci  # test is slow
 def test_that_all_snake_oil_visualisations_matches_snapshot(plot_figure):
     return plot_figure
 

tests/ert/unit_tests/dark_storage/test_dark_storage_state.py

@@ -111,4 +111,6 @@ def teardown(self):
             del os.environ["ERT_STORAGE_ENS_PATH"]
 
 
-TestDarkStorage = pytest.mark.integration_test(DarkStorageStateTest.TestCase)
+TestDarkStorage = pytest.mark.skip_mac_ci(
+    pytest.mark.integration_test(DarkStorageStateTest.TestCase)
+)

tests/ert/unit_tests/gui/model/test_snapshot.py

@@ -10,6 +10,7 @@
 
 
 @pytest.mark.integration_test
+@pytest.mark.skip_mac_ci  # slow
 def test_using_qt_model_tester(qtmodeltester, full_snapshot):
     model = SnapshotModel()
 

tests/everest/entry_points/test_everexport.py

@@ -101,7 +101,7 @@ def test_everexport_entry_empty(mocked_func, cached_example):
     side_effect=validate_export_mock,
 )
 @patch("everest.bin.utils.export_data")
-@pytest.mark.fails_on_macos_github_workflow
+@pytest.mark.skip_mac_ci
 def test_everexport_entry_batches(mocked_func, validate_export_mock, cached_example):
     """Test running everexport with the --batches flag"""
     _, config_file, _ = cached_example("math_func/config_minimal.yml")
@@ -156,7 +156,7 @@ def test_everexport_entry_empty_export(mocked_func, cached_example):
 
 
 @patch("everest.bin.utils.export_data")
-@pytest.mark.fails_on_macos_github_workflow
+@pytest.mark.skip_mac_ci
 def test_everexport_entry_no_usr_def_ecl_keys(mocked_func, cached_example):
     """Test running everexport with config file containing only the
     keywords label without any list of keys"""
@@ -190,7 +190,7 @@ def condition(config):
 
 
 @patch("everest.bin.utils.export_data")
-@pytest.mark.fails_on_macos_github_workflow
+@pytest.mark.skip_mac_ci
 def test_everexport_entry_internalized_usr_def_ecl_keys(mocked_func, cached_example):
     """Test running everexport with config file containing a key in the
     list of user defined ecl keywords, that has been internalized on
@@ -226,7 +226,7 @@ def condition(config):
 
 
 @patch("everest.bin.utils.export_data")
-@pytest.mark.fails_on_macos_github_workflow
+@pytest.mark.skip_mac_ci
 def test_everexport_entry_non_int_usr_def_ecl_keys(mocked_func, caplog, cached_example):
     """Test running everexport  when config file contains non internalized
     ecl keys in the user defined keywords list"""
@@ -269,7 +269,7 @@ def condition(config):
 
 
 @patch("everest.bin.utils.export_data")
-@pytest.mark.fails_on_macos_github_workflow
+@pytest.mark.skip_mac_ci
 def test_everexport_entry_not_available_batches(mocked_func, caplog, cached_example):
     """Test running everexport  when config file contains non existing
     batch numbers in the list of user defined batches"""

tests/everest/functional/test_main_everest_entry.py

@@ -41,7 +41,7 @@ def test_everest_main_entry_bad_command():
 
 
 @pytest.mark.flaky(reruns=5)
-@pytest.mark.fails_on_macos_github_workflow
+@pytest.mark.skip_mac_ci
 @pytest.mark.integration_test
 def test_everest_entry_run(cached_example):
     _, config_file, _ = cached_example("math_func/config_minimal.yml")

tests/everest/test_detached.py

@@ -42,7 +42,7 @@
 
 @pytest.mark.flaky(reruns=5)
 @pytest.mark.integration_test
-@pytest.mark.fails_on_macos_github_workflow
+@pytest.mark.skip_mac_ci
 @pytest.mark.xdist_group(name="starts_everest")
 async def test_https_requests(copy_math_func_test_data_to_tmp):
     everest_config = EverestConfig.load_file("config_minimal.yml")

tests/everest/test_everlint.py

@@ -333,7 +333,7 @@ def test_date_type(date, valid, min_config):
         EverestConfig(**min_config)
 
 
-@pytest.mark.fails_on_macos_github_workflow
+@pytest.mark.skip_mac_ci
 def test_lint_everest_models_jobs():
     pytest.importorskip("everest_models")
     config_file = relpath("../../test-data/everest/egg/everest/model/config.yml")

tests/everest/test_res_initialization.py

@@ -241,7 +241,7 @@ def test_summary_default(copy_egg_test_data_to_tmp):
 @hide_opm
 @skipif_no_everest_models
 @pytest.mark.everest_models_test
-@pytest.mark.fails_on_macos_github_workflow
+@pytest.mark.skip_mac_ci
 def test_summary_default_no_opm(copy_egg_test_data_to_tmp):
     config_dir = "everest/model"
     config_file = os.path.join(config_dir, "config.yml")

tests/everest/test_yaml_parser.py

@@ -106,7 +106,7 @@ def test_valid_config_file(copy_test_data_to_tmp, monkeypatch):
     assert "could not find expected ':'" in parser.get_error()
 
 
-@pytest.mark.fails_on_macos_github_workflow
+@pytest.mark.skip_mac_ci
 @skipif_no_everest_models
 @pytest.mark.everest_models_test
 def test_valid_forward_model_config_files(copy_test_data_to_tmp, monkeypatch):
@@ -121,7 +121,7 @@ def test_valid_forward_model_config_files(copy_test_data_to_tmp, monkeypatch):
 
 @skipif_no_everest_models
 @pytest.mark.everest_models_test
-@pytest.mark.fails_on_macos_github_workflow
+@pytest.mark.skip_mac_ci
 def test_invalid_forward_model_config_files(copy_test_data_to_tmp, monkeypatch):
     monkeypatch.chdir("valid_config_file/forward_models")
     parser = MockParser()