diff --git a/pytest.ini b/pytest.ini index 157b0fb55b..38a6c5dad3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -31,7 +31,7 @@ markers = spark: test for Spark trino: test for Trino (Hive connector) trino_iceberg: test for Trino (Iceberg connector) -addopts = -n 0 --dist=worksteal +addopts = -n 0 --dist=loadgroup # Set this to True to enable logging during tests log_cli = False diff --git a/setup.py b/setup.py index 274dbd7542..39d5ca10ef 100644 --- a/setup.py +++ b/setup.py @@ -90,10 +90,8 @@ # https://github.com/dpgaspar/Flask-AppBuilder/blob/master/requirements.txt#L25 "pydantic<2.6.0", "PyGithub", - # 8.0.0 broke compatability with lazy-fixture - "pytest<8.0.0", + "pytest", "pytest-asyncio<0.23.0", - "pytest-lazy-fixture", "pytest-mock", "pytest-xdist", "pyspark==3.4.0", diff --git a/tests/core/engine_adapter/test_integration.py b/tests/core/engine_adapter/test_integration.py index 11597f1f69..ce47416868 100644 --- a/tests/core/engine_adapter/test_integration.py +++ b/tests/core/engine_adapter/test_integration.py @@ -402,6 +402,7 @@ def config() -> Config: pytest.mark.duckdb, pytest.mark.engine, pytest.mark.slow, + pytest.mark.xdist_group("engine_integration_duckdb"), ], ), pytest.param( @@ -410,6 +411,7 @@ def config() -> Config: pytest.mark.docker, pytest.mark.engine, pytest.mark.postgres, + pytest.mark.xdist_group("engine_integration_postgres"), ], ), pytest.param( @@ -418,6 +420,7 @@ def config() -> Config: pytest.mark.docker, pytest.mark.engine, pytest.mark.mysql, + pytest.mark.xdist_group("engine_integration_mysql"), ], ), pytest.param( @@ -426,6 +429,7 @@ def config() -> Config: pytest.mark.docker, pytest.mark.engine, pytest.mark.mssql, + pytest.mark.xdist_group("engine_integration_mssql"), ], ), pytest.param( @@ -434,6 +438,7 @@ def config() -> Config: pytest.mark.docker, pytest.mark.engine, pytest.mark.trino, + pytest.mark.xdist_group("engine_integration_trino"), ], ), pytest.param( @@ -442,6 +447,7 @@ def config() -> Config: pytest.mark.docker, pytest.mark.engine, pytest.mark.trino_iceberg, + pytest.mark.xdist_group("engine_integration_trino_iceberg"), ], ), pytest.param( @@ -450,6 +456,7 @@ def config() -> Config: pytest.mark.docker, pytest.mark.engine, pytest.mark.spark, + pytest.mark.xdist_group("engine_integration_spark"), ], ), pytest.param( @@ -458,6 +465,7 @@ def config() -> Config: pytest.mark.bigquery, pytest.mark.engine, pytest.mark.remote, + pytest.mark.xdist_group("engine_integration_bigquery"), ], ), pytest.param( @@ -466,6 +474,7 @@ def config() -> Config: pytest.mark.databricks, pytest.mark.engine, pytest.mark.remote, + pytest.mark.xdist_group("engine_integration_databricks"), ], ), pytest.param( @@ -474,6 +483,7 @@ def config() -> Config: pytest.mark.motherduck, pytest.mark.engine, pytest.mark.remote, + pytest.mark.xdist_group("engine_integration_motherduck"), ], ), pytest.param( @@ -482,6 +492,7 @@ def config() -> Config: pytest.mark.engine, pytest.mark.remote, pytest.mark.redshift, + pytest.mark.xdist_group("engine_integration_redshift"), ], ), pytest.param( @@ -490,6 +501,7 @@ def config() -> Config: pytest.mark.engine, pytest.mark.remote, pytest.mark.snowflake, + pytest.mark.xdist_group("engine_integration_snowflake"), ], ), ] diff --git a/tests/core/test_connection_config.py b/tests/core/test_connection_config.py index 7e90112830..6b320a6715 100644 --- a/tests/core/test_connection_config.py +++ b/tests/core/test_connection_config.py @@ -2,7 +2,7 @@ import typing as t import pytest -from pytest_lazyfixture import lazy_fixture +from _pytest.fixtures import FixtureRequest from sqlmesh.core.config.connection import ( BigQueryConnectionConfig, @@ -181,27 +181,37 @@ def test_snowflake(make_config, snowflake_key_passphrase_bytes, snowflake_oauth_ @pytest.mark.parametrize( - "key_path, key_pem, key_b64, key_bytes, key_passphrase", + "key_path_fixture, key_pem_fixture, key_b64_fixture, key_bytes_fixture, key_passphrase", [ ( - lazy_fixture("snowflake_key_no_passphrase_path"), - lazy_fixture("snowflake_key_no_passphrase_pem"), - lazy_fixture("snowflake_key_no_passphrase_b64"), - lazy_fixture("snowflake_key_no_passphrase_bytes"), + "snowflake_key_no_passphrase_path", + "snowflake_key_no_passphrase_pem", + "snowflake_key_no_passphrase_b64", + "snowflake_key_no_passphrase_bytes", None, ), ( - lazy_fixture("snowflake_key_passphrase_path"), - lazy_fixture("snowflake_key_passphrase_pem"), - lazy_fixture("snowflake_key_passphrase_b64"), - lazy_fixture("snowflake_key_passphrase_bytes"), + "snowflake_key_passphrase_path", + "snowflake_key_passphrase_pem", + "snowflake_key_passphrase_b64", + "snowflake_key_passphrase_bytes", "insecure", ), ], ) def test_snowflake_private_key_pass( - make_config, key_path, key_pem, key_b64, key_bytes, key_passphrase + make_config, + key_path_fixture, + key_pem_fixture, + key_b64_fixture, + key_bytes_fixture, + key_passphrase, + request: FixtureRequest, ): + key_path = request.getfixturevalue(key_path_fixture) + key_pem = request.getfixturevalue(key_pem_fixture) + key_b64 = request.getfixturevalue(key_b64_fixture) + key_bytes = request.getfixturevalue(key_bytes_fixture) common_kwargs = dict( type="snowflake", account="test", diff --git a/tests/dbt/test_manifest.py b/tests/dbt/test_manifest.py index 9fc16148d7..8d5849d009 100644 --- a/tests/dbt/test_manifest.py +++ b/tests/dbt/test_manifest.py @@ -100,6 +100,7 @@ def test_tests_referencing_disabled_models(): assert "not_null_disabled_model_one" not in helper.tests() +@pytest.mark.xdist_group("dbt_manifest") def test_variable_override(): project_path = Path("tests/fixtures/dbt/sushi_test") profile = Profile.load(DbtContext(project_path)) diff --git a/tests/dbt/test_transformation.py b/tests/dbt/test_transformation.py index 76cd908dff..037a2f2b00 100644 --- a/tests/dbt/test_transformation.py +++ b/tests/dbt/test_transformation.py @@ -285,6 +285,7 @@ def test_seed_columns(): assert sqlmesh_seed.column_descriptions == expected_column_descriptions +@pytest.mark.xdist_group("dbt_manifest") @pytest.mark.parametrize( "model_fqn", ['"memory"."sushi"."waiters"', '"memory"."sushi"."waiter_names"'] ) @@ -310,6 +311,7 @@ def test_hooks(sushi_test_dbt_context: Context, model_fqn: str): assert "post-hook" in mock_logger.call_args[0][0] +@pytest.mark.xdist_group("dbt_manifest") def test_target_jinja(sushi_test_project: Project): context = sushi_test_project.context @@ -321,11 +323,13 @@ def test_target_jinja(sushi_test_project: Project): assert context.render("{{ target.profile_name }}") == "None" +@pytest.mark.xdist_group("dbt_manifest") def test_project_name_jinja(sushi_test_project: Project): context = sushi_test_project.context assert context.render("{{ project_name }}") == "sushi" +@pytest.mark.xdist_group("dbt_manifest") def test_schema_jinja(sushi_test_project: Project, assert_exp_eq): model_config = ModelConfig( name="model", @@ -341,6 +345,7 @@ def test_schema_jinja(sushi_test_project: Project, assert_exp_eq): ) +@pytest.mark.xdist_group("dbt_manifest") def test_config_jinja(sushi_test_project: Project): hook = "{{ config(alias='bar') }} {{ config.alias }}" model_config = ModelConfig( @@ -357,6 +362,7 @@ def test_config_jinja(sushi_test_project: Project): assert model.render_pre_statements()[0].sql() == '"bar"' +@pytest.mark.xdist_group("dbt_manifest") def test_model_this(assert_exp_eq, sushi_test_project: Project): model_config = ModelConfig( name="model", @@ -372,6 +378,7 @@ def test_model_this(assert_exp_eq, sushi_test_project: Project): ) +@pytest.mark.xdist_group("dbt_manifest") def test_test_this(assert_exp_eq, sushi_test_project: Project): test_config = TestConfig( name="test", @@ -389,6 +396,7 @@ def test_test_this(assert_exp_eq, sushi_test_project: Project): ) +@pytest.mark.xdist_group("dbt_manifest") def test_statement(sushi_test_project: Project, runtime_renderer: t.Callable): context = sushi_test_project.context assert context.target @@ -405,6 +413,7 @@ def test_statement(sushi_test_project: Project, runtime_renderer: t.Callable): ) +@pytest.mark.xdist_group("dbt_manifest") def test_run_query(sushi_test_project: Project, runtime_renderer: t.Callable): context = sushi_test_project.context assert context.target @@ -418,6 +427,7 @@ def test_run_query(sushi_test_project: Project, runtime_renderer: t.Callable): ) +@pytest.mark.xdist_group("dbt_manifest") def test_logging(sushi_test_project: Project, runtime_renderer: t.Callable): context = sushi_test_project.context assert context.target @@ -434,6 +444,7 @@ def test_logging(sushi_test_project: Project, runtime_renderer: t.Callable): assert "bar" in mock_logger.call_args[0][0] +@pytest.mark.xdist_group("dbt_manifest") def test_exceptions(sushi_test_project: Project): context = sushi_test_project.context @@ -446,6 +457,7 @@ def test_exceptions(sushi_test_project: Project): context.render('{{ exceptions.raise_compiler_error("Error") }}') +@pytest.mark.xdist_group("dbt_manifest") def test_modules(sushi_test_project: Project): context = sushi_test_project.context @@ -468,6 +480,7 @@ def test_modules(sushi_test_project: Project): assert context.render(itertools_jinja) == "5" +@pytest.mark.xdist_group("dbt_manifest") def test_flags(sushi_test_project: Project): context = sushi_test_project.context @@ -476,6 +489,7 @@ def test_flags(sushi_test_project: Project): assert context.render("{{ flags.WHICH }}") == "parse" +@pytest.mark.xdist_group("dbt_manifest") def test_relation(sushi_test_project: Project): context = sushi_test_project.context @@ -492,6 +506,7 @@ def test_relation(sushi_test_project: Project): assert context.render(jinja) == "sushi waiters" +@pytest.mark.xdist_group("dbt_manifest") def test_column(sushi_test_project: Project): context = sushi_test_project.context @@ -504,6 +519,7 @@ def test_column(sushi_test_project: Project): assert context.render(jinja) == "True False" +@pytest.mark.xdist_group("dbt_manifest") def test_quote(sushi_test_project: Project): context = sushi_test_project.context @@ -511,6 +527,7 @@ def test_quote(sushi_test_project: Project): assert context.render(jinja) == '"foo" "bar"' +@pytest.mark.xdist_group("dbt_manifest") def test_as_filters(sushi_test_project: Project): context = sushi_test_project.context @@ -527,6 +544,7 @@ def test_as_filters(sushi_test_project: Project): assert context.render("{{ None | as_native }}") == "None" +@pytest.mark.xdist_group("dbt_manifest") def test_set(sushi_test_project: Project): context = sushi_test_project.context @@ -538,6 +556,7 @@ def test_set(sushi_test_project: Project): assert context.render("{{ set_strict(1) }}") +@pytest.mark.xdist_group("dbt_manifest") def test_json(sushi_test_project: Project): context = sushi_test_project.context @@ -548,6 +567,7 @@ def test_json(sushi_test_project: Project): assert context.render("""{{ fromjson('invalid') }}""") == "None" +@pytest.mark.xdist_group("dbt_manifest") def test_yaml(sushi_test_project: Project): context = sushi_test_project.context @@ -557,6 +577,7 @@ def test_yaml(sushi_test_project: Project): assert context.render("""{{ fromyaml('key: value') }}""") == "{'key': 'value'}" +@pytest.mark.xdist_group("dbt_manifest") def test_zip(sushi_test_project: Project): context = sushi_test_project.context @@ -568,12 +589,14 @@ def test_zip(sushi_test_project: Project): context.render("{{ zip_strict(12, ['a', 'b']) }}") +@pytest.mark.xdist_group("dbt_manifest") def test_dbt_version(sushi_test_project: Project): context = sushi_test_project.context assert context.render("{{ dbt_version }}").startswith("1.") +@pytest.mark.xdist_group("dbt_manifest") def test_parsetime_adapter_call( assert_exp_eq, sushi_test_project: Project, sushi_test_dbt_context: Context ): @@ -603,6 +626,7 @@ def test_parsetime_adapter_call( ) +@pytest.mark.xdist_group("dbt_manifest") def test_partition_by(sushi_test_project: Project): context = sushi_test_project.context context.target = BigQueryConfig(name="production", database="main", schema="sushi") @@ -644,6 +668,7 @@ def test_partition_by(sushi_test_project: Project): assert model_config.to_sqlmesh(context).partitioned_by == [exp.to_column("ds")] +@pytest.mark.xdist_group("dbt_manifest") def test_relation_info_to_relation(): assert _relation_info_to_relation( {"quote_policy": {}}, @@ -676,6 +701,7 @@ def test_relation_info_to_relation(): ).quote_policy == Policy(database=False, schema=False, identifier=False) +@pytest.mark.xdist_group("dbt_manifest") def test_is_incremental(sushi_test_project: Project, assert_exp_eq, mocker): model_config = ModelConfig( name="model", @@ -705,6 +731,7 @@ def test_is_incremental(sushi_test_project: Project, assert_exp_eq, mocker): ) +@pytest.mark.xdist_group("dbt_manifest") def test_dbt_max_partition(sushi_test_project: Project, assert_exp_eq, mocker: MockerFixture): model_config = ModelConfig( name="model", @@ -742,6 +769,7 @@ def test_dbt_max_partition(sushi_test_project: Project, assert_exp_eq, mocker: M assert d.parse_one(pre_statement.sql()) == pre_statement +@pytest.mark.xdist_group("dbt_manifest") def test_bigquery_table_properties(sushi_test_project: Project, mocker: MockerFixture): context = sushi_test_project.context context.target = BigQueryConfig( @@ -780,6 +808,7 @@ def test_bigquery_table_properties(sushi_test_project: Project, mocker: MockerFi } +@pytest.mark.xdist_group("dbt_manifest") def test_snapshot_json_payload(): sushi_context = Context(paths=["tests/fixtures/dbt/sushi_test"]) snapshot_json = json.loads( @@ -794,6 +823,7 @@ def test_snapshot_json_payload(): } +@pytest.mark.xdist_group("dbt_manifest") @freeze_time("2023-01-08 00:00:00") def test_dbt_package_macros(sushi_test_project: Project): context = sushi_test_project.context @@ -804,6 +834,7 @@ def test_dbt_package_macros(sushi_test_project: Project): assert context.render("{{ dbt.run_started_at }}") == "2023-01-08 00:00:00+00:00" +@pytest.mark.xdist_group("dbt_manifest") def test_dbt_vars(sushi_test_project: Project): context = sushi_test_project.context @@ -816,6 +847,7 @@ def test_dbt_vars(sushi_test_project: Project): assert context.render("{{ var.has_var('missing') }}") == "False" +@pytest.mark.xdist_group("dbt_manifest") def test_snowflake_session_properties(sushi_test_project: Project, mocker: MockerFixture): context = sushi_test_project.context context.target = SnowflakeConfig( diff --git a/tests/schedulers/airflow/test_plan.py b/tests/schedulers/airflow/test_plan.py index f0075a3b85..e348ad8be4 100644 --- a/tests/schedulers/airflow/test_plan.py +++ b/tests/schedulers/airflow/test_plan.py @@ -3,8 +3,8 @@ from unittest import mock import pytest +from _pytest.fixtures import FixtureRequest from _pytest.monkeypatch import MonkeyPatch -from pytest_lazyfixture import lazy_fixture from pytest_mock.plugin import MockerFixture from sqlglot import parse_one @@ -62,12 +62,12 @@ def depends_on_past_snapshot(make_snapshot, random_name) -> Snapshot: @pytest.mark.parametrize( - "the_snapshot, expected_intervals, paused_forward_only", + "snapshot_fixture, expected_intervals, paused_forward_only", [ - (lazy_fixture("snapshot"), [(to_datetime("2022-01-01"), to_datetime("2022-01-05"))], False), - (lazy_fixture("snapshot"), [(to_datetime("2022-01-01"), to_datetime("2022-01-05"))], True), + ("snapshot", [(to_datetime("2022-01-01"), to_datetime("2022-01-05"))], False), + ("snapshot", [(to_datetime("2022-01-01"), to_datetime("2022-01-05"))], True), ( - lazy_fixture("depends_on_past_snapshot"), + "depends_on_past_snapshot", [ (to_datetime("2022-01-01"), to_datetime("2022-01-02")), (to_datetime("2022-01-02"), to_datetime("2022-01-03")), @@ -80,11 +80,13 @@ def depends_on_past_snapshot(make_snapshot, random_name) -> Snapshot: ) def test_create_plan_dag_spec( mocker: MockerFixture, - the_snapshot: Snapshot, + snapshot_fixture: str, expected_intervals: t.List[t.Tuple[datetime, datetime]], paused_forward_only: bool, random_name, + request: FixtureRequest, ): + the_snapshot = request.getfixturevalue(snapshot_fixture) the_snapshot.categorize_as( SnapshotChangeCategory.FORWARD_ONLY if paused_forward_only @@ -188,14 +190,14 @@ def test_create_plan_dag_spec( @pytest.mark.parametrize( - "the_snapshot, expected_intervals", + "snapshot_fixture, expected_intervals", [ ( - lazy_fixture("snapshot"), + "snapshot", [(to_datetime("2022-01-02"), to_datetime("2022-01-04"))], ), ( - lazy_fixture("depends_on_past_snapshot"), + "depends_on_past_snapshot", [ (to_datetime("2022-01-02"), to_datetime("2022-01-03")), (to_datetime("2022-01-03"), to_datetime("2022-01-04")), @@ -206,10 +208,12 @@ def test_create_plan_dag_spec( def test_restatement( mocker: MockerFixture, monkeypatch: MonkeyPatch, - the_snapshot: Snapshot, + snapshot_fixture: str, expected_intervals: t.List[t.Tuple[datetime, datetime]], random_name, + request: FixtureRequest, ): + the_snapshot = request.getfixturevalue(snapshot_fixture) environment_name = random_name() new_environment = Environment( name=environment_name, diff --git a/tests/utils/test_transactional_file.py b/tests/utils/test_transactional_file.py index 0fbf2cf496..7729e40076 100644 --- a/tests/utils/test_transactional_file.py +++ b/tests/utils/test_transactional_file.py @@ -6,7 +6,7 @@ from sqlmesh.utils.transactional_file import TransactionalFile -@pytest.fixture(scope="session") +@pytest.fixture() def fs() -> fsspec.AbstractFileSystem: return fsspec.filesystem("file")