Add option --order-marker-prefix

- allows to use custom markers for ordering

Author: mrbean-bremen

CHANGELOG.md

@@ -1,7 +1,10 @@
 # pytest-order Release Notes
 
+### New features
+* added option `--order-marker-prefix` to allow using custom markers for ordering
+
 ### Infrastructure
-* add pre-commit hook for linters
+* added pre-commit hook for linters
 
 ## [Version 1.1.0](https://pypi.org/project/pytest-order/1.1.0/) (2023-03-10)
 Adds support for executing tests more than once using order marks.

README.md

@@ -48,6 +48,8 @@ Features
   [here](https://pytest-order.readthedocs.io/en/stable/other_plugins.html#relationship-with-pytest-dependency)
 - sparse ordering of tests via the
   [sparse-ordering](https://pytest-order.readthedocs.io/en/stable/configuration.html#sparse-ordering) option
+- usage of custom markers for ordering using the
+  [sparse-ordering](https://pytest-order.readthedocs.io/en/stable/configuration.html#order-marker-prefix) option
 
 Overview
 --------

docs/source/configuration.rst

@@ -420,6 +420,84 @@ adds ordering to the existing functionality if needed.
   detected at collection time and therefore are not included in ordering.
   The same is true for the `dynamic compilation of marked parameters`_.
 
+.. _order-marker-prefix:
+
+``--order-marker-prefix``
+-------------------------
+Consider the following: You have several groups of tests where you want to decide
+which test groups to execute in a certain test run. This is usually done using custom markers,
+so that you can filter the tests by the markers using the "-m" option:
+
+.. code:: python
+
+  import pytest
+
+
+  @pytest.mark.m3
+  def test_a():
+      assert True
+
+
+  @pytest.mark.m1
+  def test_b():
+      assert True
+
+
+  @pytest.mark.m2
+  def test_c():
+      assert True
+
+Running these you get::
+
+    $ pytest tests -vv -m "m2 or m3"
+    ============================= test session starts ==============================
+    ...
+    test_module.py:5: test_a PASSED
+    test_module.py:15: test_c PASSED
+
+Now consider that the test groups shall always be executed in a certain order, e.g.
+the group with the marker "m1" shall always be executed before the tests with "m2" etc.
+This can be achieved by adding an additional order marker to each test:
+
+.. code:: python
+
+  import pytest
+
+
+  @pytest.mark.order(3)
+  @pytest.mark.m3
+  def test_a():
+      assert True
+
+
+  @pytest.mark.order(1)
+  @pytest.mark.m1
+  def test_b():
+      assert True
+
+etc. Running these you get the desired order::
+
+    $ pytest tests -vv -m "m2 or m3"
+    ============================= test session starts ==============================
+    ...
+    test_module.py:18: test_c PASSED
+    test_module.py:6: test_a PASSED
+
+This looks redundant and is also error-prone. If you want to order them instead
+just using your own marker (which has the order index already in the name), you can use
+the option ``--order-marker-prefix``. Running the original tests without any order marker
+gives you now::
+
+    $ pytest tests -vv -m "m2 or m3" --order-merker-prefix=m
+    ============================= test session starts ==============================
+    ...
+    test_module.py:18: test_c PASSED
+    test_module.py:6: test_a PASSED
+
+.. note::
+  As usually, you are responsible for registering your own markers, either in the
+  code or in the ``pytest.ini`` file. If you forget this, pytest will give you warnings about unknown markers.
+
 .. _indulgent-ordering:
 
 ``--indulgent-ordering``

pytest_order/plugin.py

@@ -101,6 +101,15 @@ def pytest_addoption(parser: Parser) -> None:
             "if needed."
         ),
     )
+    group.addoption(
+        "--order-marker-prefix",
+        action="store",
+        dest="order_marker_prefix",
+        help=(
+            "If set, markers starting with the given prefix followed by a number "
+            "are handled like order markers with an index."
+        ),
+    )
 
 
 def _get_mark_description(mark: Mark):

pytest_order/settings.py

@@ -22,6 +22,7 @@ class Settings:
     def __init__(self, config: Config) -> None:
         self.sparse_ordering: bool = config.getoption("sparse_ordering")
         self.order_dependencies: bool = config.getoption("order_dependencies")
+        self.marker_prefix: str = config.getoption("order_marker_prefix")
         scope: str = config.getoption("order_scope")
         if scope in self.valid_scopes:
             self.scope: Scope = self.valid_scopes[scope]

pytest_order/sorter.py

@@ -120,6 +120,14 @@ def mark_binning(
         keys = item.item.keywords.keys()
         has_dependency = "dependency" in keys
         has_order = "order" in keys
+        if not has_order and self.settings.marker_prefix:
+            for key in keys:
+                if key.startswith(self.settings.marker_prefix):
+                    try:
+                        index = int(key[len(self.settings.marker_prefix)])
+                        item.order = index
+                    except ValueError:
+                        pass
         if has_dependency or self.settings.auto_mark_dep:
             self.handle_dependency_mark(item, has_order, dep_marks, aliases)
         if has_order:

tests/conftest.py

@@ -37,6 +37,7 @@ def ignore_settings(mocker):
     settings.return_value.scope = Scope.SESSION
     settings.return_value.group_scope = Scope.SESSION
     settings.return_value.scope_level = 0
+    settings.return_value.marker_prefix = None
     yield settings
 
 

tests/test_marker_prefix.py

@@ -0,0 +1,141 @@
+import pytest
+
+
+@pytest.fixture(scope="module")
+def marker_test_file():
+    yield (
+        """
+        import pytest
+
+        @pytest.mark.my3
+        def test_a():
+            pass
+
+        @pytest.mark.my1
+        def test_b():
+            pass
+
+        @pytest.mark.my2
+        def test_c():
+            pass
+        """
+    )
+
+
+@pytest.fixture
+def marker_test(test_path, marker_test_file):
+    test_path.makepyfile(test_marker=marker_test_file)
+    yield test_path
+
+
+def test_no_ordering(marker_test_file, item_names_for):
+    assert item_names_for(marker_test_file) == ["test_a", "test_b", "test_c"]
+
+
+def test_order_with_marker_prefix(marker_test):
+    result = marker_test.runpytest("-v", "--order-marker-prefix=my")
+    result.assert_outcomes(passed=3, skipped=0)
+    result.stdout.fnmatch_lines(
+        [
+            "test_marker.py::test_b PASSED",
+            "test_marker.py::test_c PASSED",
+            "test_marker.py::test_a PASSED",
+        ]
+    )
+
+
+def test_order_with_marker_prefix_filtered(marker_test):
+    result = marker_test.runpytest("-v", "--order-marker-prefix=my", "-m", "my2 or my3")
+    result.assert_outcomes(passed=2, skipped=0)
+    result.stdout.fnmatch_lines(
+        [
+            "test_marker.py::test_c PASSED",
+            "test_marker.py::test_a PASSED",
+        ]
+    )
+
+
+def test_no_ordering_with_incorrect_marker_prefix(marker_test):
+    result = marker_test.runpytest("-v", "--order-marker-prefix=mi")
+    result.assert_outcomes(passed=3, skipped=0)
+    result.stdout.fnmatch_lines(
+        [
+            "test_marker.py::test_a PASSED",
+            "test_marker.py::test_b PASSED",
+            "test_marker.py::test_c PASSED",
+        ]
+    )
+
+
+def test_no_ordering_with_shorter_marker_prefix(marker_test):
+    result = marker_test.runpytest("-v", "--order-marker-prefix=m")
+    result.assert_outcomes(passed=3, skipped=0)
+    result.stdout.fnmatch_lines(
+        [
+            "test_marker.py::test_a PASSED",
+            "test_marker.py::test_b PASSED",
+            "test_marker.py::test_c PASSED",
+        ]
+    )
+
+
+def test_marker_prefix_does_not_interfere_with_order_marks(test_path):
+    test_path.makepyfile(
+        test_marker=(
+            """
+            import pytest
+
+            @pytest.mark.order(3)
+            def test_a():
+                pass
+
+            @pytest.mark.order(1)
+            def test_b():
+                pass
+
+            @pytest.mark.order(2)
+            def test_c():
+                pass
+            """
+        )
+    )
+    result = test_path.runpytest("-v", "--order-marker-prefix=m")
+    result.assert_outcomes(passed=3, skipped=0)
+    result.stdout.fnmatch_lines(
+        [
+            "test_marker.py::test_b PASSED",
+            "test_marker.py::test_c PASSED",
+            "test_marker.py::test_a PASSED",
+        ]
+    )
+
+
+def test_mix_marker_prefix_with_order_marks(test_path):
+    test_path.makepyfile(
+        test_marker=(
+            """
+            import pytest
+
+            @pytest.mark.order(3)
+            def test_a():
+                pass
+
+            @pytest.mark.my1
+            def test_b():
+                pass
+
+            @pytest.mark.my2
+            def test_c():
+                pass
+            """
+        )
+    )
+    result = test_path.runpytest("-v", "--order-marker-prefix=my")
+    result.assert_outcomes(passed=3, skipped=0)
+    result.stdout.fnmatch_lines(
+        [
+            "test_marker.py::test_b PASSED",
+            "test_marker.py::test_c PASSED",
+            "test_marker.py::test_a PASSED",
+        ]
+    )