diff --git a/dbt-sail/.gitignore b/dbt-sail/.gitignore new file mode 100644 index 0000000000..6d1ed8221f --- /dev/null +++ b/dbt-sail/.gitignore @@ -0,0 +1,8 @@ +logs/ +.venv/ +__pycache__/ +*.pyc +*.egg-info/ +.pytest_cache/ +dist/ +build/ diff --git a/dbt-sail/README.md b/dbt-sail/README.md new file mode 100644 index 0000000000..f9ada01969 --- /dev/null +++ b/dbt-sail/README.md @@ -0,0 +1,5 @@ +# dbt-sail + +The [Sail](https://github.com/lakehq/sail) adapter plugin for [dbt](https://www.getdbt.com/). + +Sail is a drop-in replacement for Apache Spark. This adapter is a thin wrapper around `dbt-spark` that connects to Sail via the Spark Connect protocol. diff --git a/dbt-sail/hatch.toml b/dbt-sail/hatch.toml new file mode 100644 index 0000000000..342f728cfb --- /dev/null +++ b/dbt-sail/hatch.toml @@ -0,0 +1,30 @@ +[version] +path = "src/dbt/adapters/sail/__version__.py" +pattern = "version = \"(?P[^\"]+)\"" + +[build.targets.wheel] +packages = ["src/dbt"] + +[envs.default] +python = "3.13.2" +pre-install-commands = [ + "pip install -e ../dbt-adapters", + "pip install -e ../dbt-tests-adapter", + "pip install -e ../dbt-spark", +] +dependencies = [ + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", + "pyspark-client==4.1.1", + "pre-commit==3.7.0", + "pytest>=7.0,<8.0", + "pytest-xdist", +] + +[envs.default.scripts] +setup = [ + "pre-commit install", +] +code-quality = "pre-commit run --all-files" +unit-tests = "python -m pytest {args:tests/unit}" +integration-tests = "python -m pytest {args:tests/functional}" diff --git a/dbt-sail/pyproject.toml b/dbt-sail/pyproject.toml new file mode 100644 index 0000000000..c9d55212d0 --- /dev/null +++ b/dbt-sail/pyproject.toml @@ -0,0 +1,49 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +dynamic = ["version"] +name = "dbt-sail" +description = "The Sail adapter plugin for dbt" +readme = "README.md" +keywords = [ + "dbt", + "adapter", + "adapters", + "database", + "elt", + "dbt-core", + "sail", + "spark", +] +requires-python = ">=3.10.0" +authors = [{ name = "LakeSail", email = "kyle@lakesail.com" }] +classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +dependencies = ["dbt-spark[session]>=1.8.0", "pysail"] + +[project.optional-dependencies] +dev = ["dbt-tests-adapter", "pytest", "pytest-xdist"] + +# Need to update these when published +[project.urls] +Homepage = "https://github.com/lakehq/sail" +Repository = "https://github.com/lakehq/sail.git" + +[tool.pytest.ini_options] +testpaths = ["tests/unit", "tests/functional"] +addopts = "-v --color=yes" +filterwarnings = [ + "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", + "ignore:unclosed file .*:ResourceWarning", +] diff --git a/dbt-sail/src/dbt/adapters/sail/__init__.py b/dbt-sail/src/dbt/adapters/sail/__init__.py new file mode 100644 index 0000000000..4004a82040 --- /dev/null +++ b/dbt-sail/src/dbt/adapters/sail/__init__.py @@ -0,0 +1,12 @@ +from dbt.adapters.base import AdapterPlugin + +from dbt.adapters.sail.connections import SailConnectionManager, SailCredentials # noqa: F401 +from dbt.adapters.sail.impl import SailAdapter +from dbt.include import sail + +Plugin = AdapterPlugin( + adapter=SailAdapter, # type: ignore + credentials=SailCredentials, + include_path=sail.PACKAGE_PATH, + dependencies=["spark"], +) diff --git a/dbt-sail/src/dbt/adapters/sail/__version__.py b/dbt-sail/src/dbt/adapters/sail/__version__.py new file mode 100644 index 0000000000..9f7a875c56 --- /dev/null +++ b/dbt-sail/src/dbt/adapters/sail/__version__.py @@ -0,0 +1 @@ +version = "0.1.0" diff --git a/dbt-sail/src/dbt/adapters/sail/connections.py b/dbt-sail/src/dbt/adapters/sail/connections.py new file mode 100644 index 0000000000..e941297f13 --- /dev/null +++ b/dbt-sail/src/dbt/adapters/sail/connections.py @@ -0,0 +1,129 @@ +from dataclasses import dataclass, field +from typing import Any, Dict + +from dbt.adapters.contracts.connection import Connection, ConnectionState +from dbt.adapters.events.logging import AdapterLogger +from dbt.adapters.exceptions import FailedToConnectError +from dbt.adapters.spark.connections import SparkConnectionManager, SparkCredentials +from dbt_common.dataclass_schema import StrEnum +from dbt_common.exceptions import DbtConfigError + +logger = AdapterLogger("Sail") + + +class SailConnectionMethod(StrEnum): + EMBEDDED = "embedded" + REMOTE = "remote" + + +@dataclass +class SailCredentials(SparkCredentials): + mode: SailConnectionMethod = SailConnectionMethod.EMBEDDED # type: ignore + server_side_parameters: Dict[str, str] = field(default_factory=dict) + + @property + def type(self) -> str: + return "sail" + + @property + def unique_field(self) -> str: + return self.host or "embedded" + + def __post_init__(self) -> None: + # For embedded mode, host is not required — set defaults + if self.mode == SailConnectionMethod.EMBEDDED: + if self.host is None: + self.host = "127.0.0.1" + if self.method is None: + # Set method to session so SparkCredentials validation doesn't fail + from dbt.adapters.spark.connections import SparkConnectionMethod + + self.method = SparkConnectionMethod.SESSION + elif self.mode == SailConnectionMethod.REMOTE: + if self.host is None: + raise DbtConfigError("Must specify `host` for remote mode") + if self.method is None: + from dbt.adapters.spark.connections import SparkConnectionMethod + + self.method = SparkConnectionMethod.SESSION + + if self.schema is None: + raise DbtConfigError("Must specify `schema` in profile") + + # Spark treats database and schema as the same thing + if self.database is not None and self.database != self.schema: + raise DbtConfigError( + f" schema: {self.schema} \n" + f" database: {self.database} \n" + f"On Sail, database must be omitted or have the same value as schema." + ) + self.database = None + + +class SailConnectionManager(SparkConnectionManager): + TYPE = "sail" + + # Track the embedded server so it can be reused/stopped + _server = None + + @classmethod + def open(cls, connection: Connection) -> Connection: + if connection.state == ConnectionState.OPEN: + logger.debug("Connection is already open, skipping open.") + return connection + + creds: SailCredentials = connection.credentials # type: ignore + + try: + if creds.mode == SailConnectionMethod.EMBEDDED: + handle = cls._open_embedded(creds) + elif creds.mode == SailConnectionMethod.REMOTE: + handle = cls._open_remote(creds) + else: + raise DbtConfigError(f"invalid Sail connection mode: {creds.mode}") + except Exception as e: + logger.debug(f"Error opening connection: {e}") + connection.handle = None + connection.state = ConnectionState.FAIL + raise FailedToConnectError(f"failed to connect to Sail: {e}") from e + + connection.handle = handle + connection.state = ConnectionState.OPEN + return connection + + @classmethod + def _open_embedded(cls, creds: SailCredentials) -> Any: + """Start a SparkConnectServer in-process via pysail, then connect PySpark to it.""" + from pysail.spark import SparkConnectServer + from dbt.adapters.spark.session import ( + SessionConnectionWrapper, + Connection as SessionConnection, + ) + + if cls._server is None or not cls._server.running: + port = creds.port if creds.port != 443 else 0 + cls._server = SparkConnectServer(ip=creds.host or "127.0.0.1", port=port) + cls._server.start(background=True) + logger.debug(f"Started embedded Sail server at {cls._server.listening_address}") + + ip, port = cls._server.listening_address + remote_url = f"sc://{ip}:{port}" + + params = dict(creds.server_side_parameters) + params["spark.remote"] = remote_url + return SessionConnectionWrapper(SessionConnection(server_side_parameters=params)) + + @classmethod + def _open_remote(cls, creds: SailCredentials) -> Any: + """Connect PySpark to an already-running Sail server via Spark Connect.""" + from dbt.adapters.spark.session import ( + SessionConnectionWrapper, + Connection as SessionConnection, + ) + + port = creds.port if creds.port != 443 else 50051 + remote_url = f"sc://{creds.host}:{port}" + + params = dict(creds.server_side_parameters) + params["spark.remote"] = remote_url + return SessionConnectionWrapper(SessionConnection(server_side_parameters=params)) diff --git a/dbt-sail/src/dbt/adapters/sail/impl.py b/dbt-sail/src/dbt/adapters/sail/impl.py new file mode 100644 index 0000000000..59d4f55f06 --- /dev/null +++ b/dbt-sail/src/dbt/adapters/sail/impl.py @@ -0,0 +1,10 @@ +from dbt.adapters.spark.impl import SparkAdapter +from dbt.adapters.sail.connections import SailConnectionManager + + +class SailAdapter(SparkAdapter): + ConnectionManager = SailConnectionManager + + @classmethod + def type(cls) -> str: + return "sail" diff --git a/dbt-sail/src/dbt/include/sail/__init__.py b/dbt-sail/src/dbt/include/sail/__init__.py new file mode 100644 index 0000000000..b177e5d493 --- /dev/null +++ b/dbt-sail/src/dbt/include/sail/__init__.py @@ -0,0 +1,3 @@ +import os + +PACKAGE_PATH = os.path.dirname(__file__) diff --git a/dbt-sail/src/dbt/include/sail/dbt_project.yml b/dbt-sail/src/dbt/include/sail/dbt_project.yml new file mode 100644 index 0000000000..0b7c8889de --- /dev/null +++ b/dbt-sail/src/dbt/include/sail/dbt_project.yml @@ -0,0 +1,5 @@ +config-version: 2 +name: dbt_sail +version: 1.0 + +macro-paths: ["macros"] diff --git a/dbt-sail/src/dbt/include/sail/macros/adapters.sql b/dbt-sail/src/dbt/include/sail/macros/adapters.sql new file mode 100644 index 0000000000..db08b4df1c --- /dev/null +++ b/dbt-sail/src/dbt/include/sail/macros/adapters.sql @@ -0,0 +1,11 @@ +{% macro sail__list_relations_without_caching(relation) %} + {{ return(spark__list_relations_without_caching(relation)) }} +{% endmacro %} + +{% macro sail__get_columns_in_relation_raw(relation) -%} + {{ return(spark__get_columns_in_relation_raw(relation)) }} +{% endmacro %} + +{% macro sail__get_columns_in_relation(relation) -%} + {{ return(spark__get_columns_in_relation(relation)) }} +{% endmacro %} diff --git a/dbt-sail/tests/__init__.py b/dbt-sail/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbt-sail/tests/functional/__init__.py b/dbt-sail/tests/functional/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbt-sail/tests/functional/adapter/SPARK_SESSION_TEST_DEBUG_REPORT.md b/dbt-sail/tests/functional/adapter/SPARK_SESSION_TEST_DEBUG_REPORT.md new file mode 100644 index 0000000000..cd59683343 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/SPARK_SESSION_TEST_DEBUG_REPORT.md @@ -0,0 +1,121 @@ +# Spark-Session Pruned Test Debug Report (dbt-sail) + +## Summary +- Total tests: **52** +- Passed: **21** +- Failed/Error: **30** +- Skipped: **1** + +## Failure Categories +- `exit-state mismatch`: 16 +- `sql/parser incompatibility`: 6 +- `type mapping mismatch`: 5 +- `error-message mismatch`: 1 +- `state/cleanup issue`: 1 +- `time precision/tolerance mismatch`: 1 + +## Error Signature Legend +| ID | Count | Category | What's Happening | Example | +|---|---:|---|---|---| +| `E01` | 16 | exit-state mismatch | dbt command result differs from test expectation. | AssertionError: dbt exit state did not match expected self = project = args = [], expected = 5 @pytest.mark.parametrize( "args,expected", [ ([], 5), # default limit (["--limit", 3 | +| `E02` | 1 | state/cleanup issue | test expects clean relation state but object already exists. | AssertionError: assert 'Compilation Error' in 'Runtime Error in model incremental_fail (models/incremental_fail.sql)\n Runtime Error\n table already exists: incremental_fail' + where 'Runtime Error in model incremental_fail (models/incremental_fail.sql)\n Runtime Error\n table already exists: increm | +| `E03` | 1 | error-message mismatch | same logical failure class, but assertion expects different text. | assert "Invalid column name: 'invalid_column_name' in unit test fixture for 'my_upstream_model'." in '\x1b[0m03:44:22 Running with dbt=1.11.7\n\x1b[0m03:44:22 Registered adapter: sail=0.1.0\n\x1b[0m03:44:22 Found 2 m...odel (models/unit_tests.yml)\n\x1b[0m03:44:22 \n\x1b[0m03:44:22 Done. PASS=0 WARN | +| `E04` | 6 | sql/parser incompatibility | SQL generated by shared test macro is rejected by Sail parser. | dbt_common.exceptions.base.DbtRuntimeError: Runtime Error invalid argument: found . at 278:279 expected ',', 'FROM', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', or ')' self = project = def test_check_types_assert_match(self, project): run_dbt(["build"]) # check contents | +| `E06` | 1 | time precision/tolerance mismatch | timestamp precision/timezone offset differs from test expectation. | AssertionError: SQL timestamp 2026-04-03T20:44:25.633143 is not close enough to Python UTC 2026-04-04T03:44:25.634810 assert (datetime.datetime(2026, 4, 3, 20, 44, 25, 633143) > (datetime.datetime(2026, 4, 4, 3, 44, 25, 634810) - datetime.timedelta(seconds=300))) self = 0 + +{% endtest %} +""".strip() + + +_SEED_CSV = """ +id,orderid,paymentmethod,status,amount,amount_usd,created +1,1,credit_card,success,1000,10.00,2018-01-01 +2,2,credit_card,success,2000,20.00,2018-01-02 +3,3,coupon,success,100,1.00,2018-01-04 +4,4,coupon,success,2500,25.00,2018-01-05 +5,5,bank_transfer,fail,1700,17.00,2018-01-05 +6,5,bank_transfer,success,1700,17.00,2018-01-05 +7,6,credit_card,success,600,6.00,2018-01-07 +8,7,credit_card,success,1600,16.00,2018-01-09 +9,8,credit_card,success,2300,23.00,2018-01-11 +10,9,gift_card,success,2300,23.00,2018-01-12 +""".strip() + + +_SEED_YML = """ +version: 2 + +seeds: + - name: payments + config: + column_types: + id: string + orderid: string + paymentmethod: string + status: string + amount: integer + amount_usd: decimal(20,2) + created: timestamp + tests: + - is_type: + column_map: + id: ["string", "not number"] + orderid: ["string", "not number"] + paymentmethod: ["string", "not number"] + status: ["string", "not number"] + amount: ["integer", "number"] + amount_usd: ["decimal", "number"] + created: ["timestamp", "string"] +""".strip() diff --git a/dbt-sail/tests/functional/adapter/test_basic.py b/dbt-sail/tests/functional/adapter/test_basic.py new file mode 100644 index 0000000000..2b04e535c3 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/test_basic.py @@ -0,0 +1,73 @@ +from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod +from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations +from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests +from dbt.tests.adapter.basic.test_singular_tests_ephemeral import ( + BaseSingularTestsEphemeral, +) +from dbt.tests.adapter.basic.test_empty import BaseEmpty +from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral +from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests +from dbt.tests.adapter.basic.test_incremental import BaseIncremental +from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols +from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp +import pytest + + +class TestSimpleMaterializationsSpark(BaseSimpleMaterializations): + pass + + +class TestSingularTestsSpark(BaseSingularTests): + pass + + +# The local cluster currently tests on spark 2.x, which does not support this +# if we upgrade it to 3.x, we can enable this test +class TestSingularTestsEphemeralSpark(BaseSingularTestsEphemeral): + pass + + +class TestEmptySpark(BaseEmpty): + pass + + +class TestEphemeralSpark(BaseEphemeral): + pass + + +class TestIncrementalSpark(BaseIncremental): + pass + + +class TestGenericTestsSpark(BaseGenericTests): + pass + + +class TestSnapshotCheckColsSpark(BaseSnapshotCheckCols): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "+file_format": "delta", + }, + "snapshots": { + "+file_format": "delta", + }, + } + + +class TestSnapshotTimestampSpark(BaseSnapshotTimestamp): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "+file_format": "delta", + }, + "snapshots": { + "+file_format": "delta", + }, + } + + +class TestBaseAdapterMethod(BaseAdapterMethod): + pass diff --git a/dbt-sail/tests/functional/adapter/test_simple_seed.py b/dbt-sail/tests/functional/adapter/test_simple_seed.py new file mode 100644 index 0000000000..c610967c69 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/test_simple_seed.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.simple_seed.test_seed import BaseTestEmptySeed + + +class TestBigQueryEmptySeed(BaseTestEmptySeed): + pass diff --git a/dbt-sail/tests/functional/adapter/unit_testing/test_unit_testing.py b/dbt-sail/tests/functional/adapter/unit_testing/test_unit_testing.py new file mode 100644 index 0000000000..b70c581d11 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -0,0 +1,34 @@ +import pytest + +from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes +from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity +from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput + + +class TestSparkUnitTestingTypes(BaseUnitTestingTypes): + @pytest.fixture + def data_types(self): + # sql_value, yaml_value + return [ + ["1", "1"], + ["2.0", "2.0"], + ["'12345'", "12345"], + ["'string'", "string"], + ["true", "true"], + ["date '2011-11-11'", "2011-11-11"], + ["timestamp '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + ["array(1, 2, 3)", "'array(1, 2, 3)'"], + [ + "map('10', 't', '15', 'f', '20', NULL)", + """'map("10", "t", "15", "f", "20", NULL)'""", + ], + ['named_struct("a", 1, "b", 2, "c", 3)', """'named_struct("a", 1, "b", 2, "c", 3)'"""], + ] + + +class TestSparkUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): + pass + + +class TestSparkUnitTestInvalidInput(BaseUnitTestInvalidInput): + pass diff --git a/dbt-sail/tests/functional/adapter/utils/fixture_listagg.py b/dbt-sail/tests/functional/adapter/utils/fixture_listagg.py new file mode 100644 index 0000000000..0262ca2340 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/utils/fixture_listagg.py @@ -0,0 +1,61 @@ +# SparkSQL does not support 'order by' for its 'listagg' equivalent +# the argument is ignored, so let's ignore those fields when checking equivalency + +models__test_listagg_no_order_by_sql = """ +with data as ( + select * from {{ ref('data_listagg') }} +), +data_output as ( + select * from {{ ref('data_listagg_output') }} +), +calculate as ( +/* + + select + group_col, + {{ listagg('string_text', "'_|_'", "order by order_col") }} as actual, + 'bottom_ordered' as version + from data + group by group_col + union all + select + group_col, + {{ listagg('string_text', "'_|_'", "order by order_col", 2) }} as actual, + 'bottom_ordered_limited' as version + from data + group by group_col + union all + +*/ + select + group_col, + {{ listagg('string_text', "', '") }} as actual, + 'comma_whitespace_unordered' as version + from data + where group_col = 3 + group by group_col + union all + select + group_col, + {{ listagg('DISTINCT string_text', "','") }} as actual, + 'distinct_comma' as version + from data + where group_col = 3 + group by group_col + union all + select + group_col, + {{ listagg('string_text') }} as actual, + 'no_params' as version + from data + where group_col = 3 + group by group_col +) +select + calculate.actual, + data_output.expected +from calculate +left join data_output +on calculate.group_col = data_output.group_col +and calculate.version = data_output.version +""" diff --git a/dbt-sail/tests/functional/adapter/utils/test_data_types.py b/dbt-sail/tests/functional/adapter/utils/test_data_types.py new file mode 100644 index 0000000000..8ca38ab1e9 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/utils/test_data_types.py @@ -0,0 +1,76 @@ +import pytest +from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt +from dbt.tests.adapter.utils.data_types.test_type_float import ( + BaseTypeFloat, + seeds__expected_csv as seeds__float_expected_csv, +) +from dbt.tests.adapter.utils.data_types.test_type_int import ( + BaseTypeInt, + seeds__expected_csv as seeds__int_expected_csv, +) +from dbt.tests.adapter.utils.data_types.test_type_numeric import BaseTypeNumeric +from dbt.tests.adapter.utils.data_types.test_type_string import BaseTypeString +from dbt.tests.adapter.utils.data_types.test_type_timestamp import BaseTypeTimestamp +from dbt.tests.adapter.utils.data_types.test_type_boolean import BaseTypeBoolean + + +class TestTypeBigInt(BaseTypeBigInt): + pass + + +# need to explicitly cast this to avoid it being inferred/loaded as a DOUBLE on Spark +# in SparkSQL, the two are equivalent for `=` comparison, but distinct for EXCEPT comparison +seeds__float_expected_yml = """ +version: 2 +seeds: + - name: expected + config: + column_types: + float_col: float +""" + + +class TestTypeFloat(BaseTypeFloat): + @pytest.fixture(scope="class") + def seeds(self): + return { + "expected.csv": seeds__float_expected_csv, + "expected.yml": seeds__float_expected_yml, + } + + +# need to explicitly cast this to avoid it being inferred/loaded as a BIGINT on Spark +seeds__int_expected_yml = """ +version: 2 +seeds: + - name: expected + config: + column_types: + int_col: int +""" + + +class TestTypeInt(BaseTypeInt): + @pytest.fixture(scope="class") + def seeds(self): + return { + "expected.csv": seeds__int_expected_csv, + "expected.yml": seeds__int_expected_yml, + } + + +class TestTypeNumeric(BaseTypeNumeric): + def numeric_fixture_type(self): + return "decimal(28,6)" + + +class TestTypeString(BaseTypeString): + pass + + +class TestTypeTimestamp(BaseTypeTimestamp): + pass + + +class TestTypeBoolean(BaseTypeBoolean): + pass diff --git a/dbt-sail/tests/functional/adapter/utils/test_timestamps.py b/dbt-sail/tests/functional/adapter/utils/test_timestamps.py new file mode 100644 index 0000000000..d05d23997d --- /dev/null +++ b/dbt-sail/tests/functional/adapter/utils/test_timestamps.py @@ -0,0 +1,18 @@ +import pytest +from dbt.tests.adapter.utils.test_timestamps import BaseCurrentTimestamps + + +class TestCurrentTimestampSpark(BaseCurrentTimestamps): + @pytest.fixture(scope="class") + def models(self): + return { + "get_current_timestamp.sql": "select {{ current_timestamp() }} as current_timestamp" + } + + @pytest.fixture(scope="class") + def expected_schema(self): + return {"current_timestamp": "timestamp"} + + @pytest.fixture(scope="class") + def expected_sql(self): + return """select current_timestamp() as current_timestamp""" diff --git a/dbt-sail/tests/functional/adapter/utils/test_utils.py b/dbt-sail/tests/functional/adapter/utils/test_utils.py new file mode 100644 index 0000000000..88d8a0fdde --- /dev/null +++ b/dbt-sail/tests/functional/adapter/utils/test_utils.py @@ -0,0 +1,133 @@ +import pytest + +from dbt.tests.adapter.utils.test_array_append import BaseArrayAppend +from dbt.tests.adapter.utils.test_array_concat import BaseArrayConcat +from dbt.tests.adapter.utils.test_array_construct import BaseArrayConstruct +from dbt.tests.adapter.utils.test_any_value import BaseAnyValue +from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr +from dbt.tests.adapter.utils.test_cast import BaseCast +from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText +from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampNaive +from dbt.tests.adapter.utils.test_date import BaseDate +from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd +from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc +from dbt.tests.adapter.utils.test_equals import BaseEquals +from dbt.tests.adapter.utils.test_escape_single_quotes import BaseEscapeSingleQuotesBackslash +from dbt.tests.adapter.utils.test_except import BaseExcept +from dbt.tests.adapter.utils.test_intersect import BaseIntersect +from dbt.tests.adapter.utils.test_length import BaseLength +from dbt.tests.adapter.utils.test_position import BasePosition +from dbt.tests.adapter.utils.test_safe_cast import BaseSafeCast + +from dbt.tests.adapter.utils.test_split_part import BaseSplitPart +from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral + +# requires modification +from dbt.tests.adapter.utils.test_listagg import BaseListagg +from dbt.tests.adapter.utils.fixture_listagg import models__test_listagg_yml +from tests.functional.adapter.utils.fixture_listagg import models__test_listagg_no_order_by_sql + +seeds__data_split_part_csv = """parts,split_on,result_1,result_2,result_3,result_4 +a|b|c,|,a,b,c,c +1|2|3,|,1,2,3,3 +EMPTY|EMPTY|EMPTY,|,EMPTY,EMPTY,EMPTY,EMPTY +""" + + +# skipped: ,month, + + +class TestAnyValue(BaseAnyValue): + pass + + +class TestArrayAppend(BaseArrayAppend): + pass + + +class TestArrayConcat(BaseArrayConcat): + pass + + +class TestArrayConstruct(BaseArrayConstruct): + pass + + +class TestBoolOr(BaseBoolOr): + pass + + +class TestCast(BaseCast): + pass + + +class TestCastBoolToText(BaseCastBoolToText): + pass + + +# Use either BaseCurrentTimestampAware or BaseCurrentTimestampNaive but not both +class TestCurrentTimestamp(BaseCurrentTimestampNaive): + pass + + +class TestDate(BaseDate): + pass + + +class TestDateAdd(BaseDateAdd): + pass + + +class TestDateTrunc(BaseDateTrunc): + pass + + +class TestEquals(BaseEquals): + pass + + +class TestEscapeSingleQuotes(BaseEscapeSingleQuotesBackslash): + pass + + +class TestExcept(BaseExcept): + pass + + +class TestIntersect(BaseIntersect): + pass + + +class TestLength(BaseLength): + pass + + +# SparkSQL does not support 'order by' for its 'listagg' equivalent +# the argument is ignored, so let's ignore those fields when checking equivalency +class TestListagg(BaseListagg): + @pytest.fixture(scope="class") + def models(self): + return { + "test_listagg.yml": models__test_listagg_yml, + "test_listagg.sql": self.interpolate_macro_namespace( + models__test_listagg_no_order_by_sql, "listagg" + ), + } + + +class TestPosition(BasePosition): + pass + + +class TestSafeCast(BaseSafeCast): + pass + + +class TestSplitPart(BaseSplitPart): + @pytest.fixture(scope="class") + def seeds(self): + return {"data_split_part.csv": seeds__data_split_part_csv} + + +class TestStringLiteral(BaseStringLiteral): + pass diff --git a/dbt-sail/tests/functional/conftest.py b/dbt-sail/tests/functional/conftest.py new file mode 100644 index 0000000000..3e9dc97a47 --- /dev/null +++ b/dbt-sail/tests/functional/conftest.py @@ -0,0 +1,15 @@ +import pytest + + +pytest_plugins = ["dbt.tests.fixtures.project"] + + +@pytest.fixture(scope="class") +def dbt_profile_target(): + return { + "type": "sail", + "mode": "embedded", + "host": "127.0.0.1", + "schema": "test_sail", + "threads": 1, + } diff --git a/dbt-sail/tests/unit/__init__.py b/dbt-sail/tests/unit/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbt-sail/tests/unit/conftest.py b/dbt-sail/tests/unit/conftest.py new file mode 100644 index 0000000000..7fc366ecba --- /dev/null +++ b/dbt-sail/tests/unit/conftest.py @@ -0,0 +1 @@ +from .fixtures.profiles import * # noqa: F401, F403 diff --git a/dbt-sail/tests/unit/fixtures/__init__.py b/dbt-sail/tests/unit/fixtures/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbt-sail/tests/unit/fixtures/profiles.py b/dbt-sail/tests/unit/fixtures/profiles.py new file mode 100644 index 0000000000..459577ba11 --- /dev/null +++ b/dbt-sail/tests/unit/fixtures/profiles.py @@ -0,0 +1,55 @@ +import pytest + +from tests.unit.utils import config_from_parts_or_dicts + + +@pytest.fixture(scope="session", autouse=True) +def base_project_cfg(): + return { + "name": "X", + "version": "0.1", + "profile": "test", + "project-root": "/tmp/dbt/does-not-exist", + "quoting": { + "identifier": False, + "schema": False, + }, + "config-version": 2, + } + + +@pytest.fixture(scope="session", autouse=True) +def target_sail_embedded(base_project_cfg): + return config_from_parts_or_dicts( + base_project_cfg, + { + "outputs": { + "test": { + "type": "sail", + "mode": "embedded", + "schema": "analytics", + "host": "127.0.0.1", + } + }, + "target": "test", + }, + ) + + +@pytest.fixture(scope="session", autouse=True) +def target_sail_remote(base_project_cfg): + return config_from_parts_or_dicts( + base_project_cfg, + { + "outputs": { + "test": { + "type": "sail", + "mode": "remote", + "schema": "analytics", + "host": "myorg.sailhost.com", + "port": 50051, + } + }, + "target": "test", + }, + ) diff --git a/dbt-sail/tests/unit/test_adapter.py b/dbt-sail/tests/unit/test_adapter.py new file mode 100644 index 0000000000..694ccee29e --- /dev/null +++ b/dbt-sail/tests/unit/test_adapter.py @@ -0,0 +1,143 @@ +import unittest +import pytest +from multiprocessing import get_context +from unittest import mock + +from dbt_common.exceptions import DbtRuntimeError +from agate import Row +from dbt.adapters.sail import SailAdapter +from dbt.adapters.spark import SparkRelation +from dbt.adapters.spark.impl import ( + SCHEMA_NOT_FOUND_MESSAGES, + TABLE_OR_VIEW_NOT_FOUND_MESSAGES, +) + + +class TestSailAdapter(unittest.TestCase): + @pytest.fixture(autouse=True) + def set_up_fixtures(self, target_sail_embedded, base_project_cfg): + self.base_project_cfg = base_project_cfg + self.target_sail_embedded = target_sail_embedded + + def test_adapter_type(self): + assert SailAdapter.type() == "sail" + + def test_parse_relation(self): + self.maxDiff = None + rel_type = SparkRelation.get_relation_type.Table + + relation = SparkRelation.create( + schema="default_schema", identifier="mytable", type=rel_type + ) + assert relation.database is None + + plain_rows = [ + ("col1", "decimal(22,0)"), + ("col2", "string"), + ("dt", "date"), + ("struct_col", "struct"), + ("# Partition Information", "data_type"), + ("# col_name", "data_type"), + ("dt", "date"), + (None, None), + ("# Detailed Table Information", None), + ("Database", None), + ("Owner", "root"), + ("Created Time", "Wed Feb 04 18:15:00 UTC 1815"), + ("Last Access", "Wed May 20 19:25:00 UTC 1925"), + ("Type", "MANAGED"), + ("Provider", "delta"), + ("Location", "/mnt/vo"), + ("Serde Library", "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"), + ("InputFormat", "org.apache.hadoop.mapred.SequenceFileInputFormat"), + ("OutputFormat", "org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat"), + ("Partition Provider", "Catalog"), + ] + + input_cols = [Row(keys=["col_name", "data_type"], values=r) for r in plain_rows] + + rows = SailAdapter( + self.target_sail_embedded, get_context("spawn") + ).parse_describe_extended(relation, input_cols) + self.assertEqual(len(rows), 4) + self.assertEqual(rows[0].to_column_dict(omit_none=False)["column"], "col1") + self.assertEqual(rows[0].to_column_dict(omit_none=False)["dtype"], "decimal(22,0)") + self.assertEqual(rows[1].to_column_dict(omit_none=False)["column"], "col2") + self.assertEqual(rows[2].to_column_dict(omit_none=False)["column"], "dt") + self.assertEqual(rows[3].to_column_dict(omit_none=False)["column"], "struct_col") + + def test_relation_with_database(self): + adapter = SailAdapter(self.target_sail_embedded, get_context("spawn")) + # fine + adapter.Relation.create(schema="different", identifier="table") + with self.assertRaises(DbtRuntimeError): + # not fine - database set + adapter.Relation.create(database="something", schema="different", identifier="table") + + +class TestListRelationsWithoutCaching(unittest.TestCase): + @pytest.fixture(autouse=True) + def set_up_fixtures(self, target_sail_embedded): + self.target_sail_embedded = target_sail_embedded + + def _make_adapter(self): + return SailAdapter(self.target_sail_embedded, get_context("spawn")) + + def _make_schema_relation(self, adapter, schema="analytics"): + return adapter.Relation.create(schema=schema, identifier="").without_identifier() + + def test_unknown_error_is_raised(self): + adapter = self._make_adapter() + schema_relation = self._make_schema_relation(adapter) + + with mock.patch.object( + adapter, + "execute_macro", + side_effect=DbtRuntimeError("Connection failed"), + ): + with self.assertRaises(DbtRuntimeError): + adapter.list_relations_without_caching(schema_relation) + + def test_schema_not_found_returns_empty(self): + adapter = self._make_adapter() + schema_relation = self._make_schema_relation(adapter, schema="nonexistent") + + with mock.patch.object( + adapter, + "execute_macro", + side_effect=DbtRuntimeError("Database not found"), + ): + result = adapter.list_relations_without_caching(schema_relation) + self.assertEqual(result, []) + + +@pytest.mark.parametrize("not_found_msg", SCHEMA_NOT_FOUND_MESSAGES) +def test_all_schema_not_found_messages_return_empty(not_found_msg, target_sail_embedded): + adapter = SailAdapter(target_sail_embedded, get_context("spawn")) + schema_relation = adapter.Relation.create( + schema="nonexistent", identifier="" + ).without_identifier() + + with mock.patch.object( + adapter, + "execute_macro", + side_effect=DbtRuntimeError(not_found_msg), + ): + result = adapter.list_relations_without_caching(schema_relation) + assert result == [] + + +@pytest.mark.parametrize("not_found_msg", TABLE_OR_VIEW_NOT_FOUND_MESSAGES) +def test_all_table_or_view_not_found_messages_return_empty(not_found_msg, target_sail_embedded): + adapter = SailAdapter(target_sail_embedded, get_context("spawn")) + schema_relation = adapter.Relation.create( + schema="nonexistent", identifier="" + ).without_identifier() + + with mock.patch.object( + adapter, + "execute_macro", + side_effect=DbtRuntimeError(not_found_msg), + ): + result = adapter.list_relations_without_caching(schema_relation) + assert result == [] diff --git a/dbt-sail/tests/unit/test_credentials.py b/dbt-sail/tests/unit/test_credentials.py new file mode 100644 index 0000000000..d0dcbc0985 --- /dev/null +++ b/dbt-sail/tests/unit/test_credentials.py @@ -0,0 +1,77 @@ +import pytest + +from dbt.adapters.sail.connections import SailConnectionMethod, SailCredentials +from dbt_common.exceptions import DbtConfigError + + +def test_embedded_credentials_defaults() -> None: + credentials = SailCredentials( + mode=SailConnectionMethod.EMBEDDED, # type: ignore + schema="analytics", + ) + assert credentials.type == "sail" + assert credentials.host == "127.0.0.1" + assert credentials.schema == "analytics" + assert credentials.database is None + + +def test_remote_credentials() -> None: + credentials = SailCredentials( + mode=SailConnectionMethod.REMOTE, # type: ignore + host="myorg.sailhost.com", + port=50051, + schema="analytics", + ) + assert credentials.type == "sail" + assert credentials.host == "myorg.sailhost.com" + assert credentials.port == 50051 + + +def test_remote_credentials_requires_host() -> None: + with pytest.raises(DbtConfigError): + SailCredentials( + mode=SailConnectionMethod.REMOTE, # type: ignore + schema="analytics", + ) + + +def test_credentials_schema_required() -> None: + with pytest.raises(DbtConfigError): + SailCredentials( + mode=SailConnectionMethod.EMBEDDED, # type: ignore + ) + + +def test_credentials_database_must_match_schema() -> None: + with pytest.raises(DbtConfigError): + SailCredentials( + mode=SailConnectionMethod.EMBEDDED, # type: ignore + schema="analytics", + database="different", + ) + + +def test_credentials_server_side_parameters() -> None: + credentials = SailCredentials( + mode=SailConnectionMethod.EMBEDDED, # type: ignore + schema="analytics", + server_side_parameters={"spark.driver.memory": "4g"}, + ) + assert credentials.server_side_parameters["spark.driver.memory"] == "4g" + + +def test_credentials_unique_field_embedded() -> None: + credentials = SailCredentials( + mode=SailConnectionMethod.EMBEDDED, # type: ignore + schema="analytics", + ) + assert credentials.unique_field == "127.0.0.1" + + +def test_credentials_unique_field_remote() -> None: + credentials = SailCredentials( + mode=SailConnectionMethod.REMOTE, # type: ignore + host="myorg.sailhost.com", + schema="analytics", + ) + assert credentials.unique_field == "myorg.sailhost.com" diff --git a/dbt-sail/tests/unit/utils.py b/dbt-sail/tests/unit/utils.py new file mode 100644 index 0000000000..5419f50d2e --- /dev/null +++ b/dbt-sail/tests/unit/utils.py @@ -0,0 +1,13 @@ +"""Re-export utility functions from dbt-spark's test utilities.""" + +import sys +import os + +# Add dbt-spark's test directory to the path so we can import its utilities +_dbt_adapters_root = os.path.join( + os.path.dirname(__file__), "..", "..", "..", "..", "dbt-adapters", "dbt-spark" +) +if os.path.isdir(_dbt_adapters_root): + sys.path.insert(0, os.path.join(_dbt_adapters_root, "tests")) + +from unit.utils import config_from_parts_or_dicts # noqa: E402, F401 diff --git a/dbt-spark/dagger/audit_spark_session_test_speeds.py b/dbt-spark/dagger/audit_spark_session_test_speeds.py new file mode 100644 index 0000000000..e95fa94ea2 --- /dev/null +++ b/dbt-spark/dagger/audit_spark_session_test_speeds.py @@ -0,0 +1,212 @@ +from __future__ import annotations + +import argparse +import concurrent.futures +import contextlib +import io +import json +import subprocess +import sys +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import List + +import pytest + + +DBT_SPARK_ROOT = Path(__file__).resolve().parents[1] +RUNNER_PATH = DBT_SPARK_ROOT / "dagger" / "run_dbt_spark_tests.py" +DEFAULT_SCOPE = "tests/functional/adapter" +DEFAULT_OUTPUT_DIR = DBT_SPARK_ROOT / "target" / "spark-session-audit" + + +@dataclass +class TestAuditResult: + nodeid: str + return_code: int + timed_out: bool + duration_seconds: float + log_file: str + + +def collect_runnable_nodeids(scope: str, profile: str) -> List[str]: + class Collector: + def __init__(self) -> None: + self.items = [] + + def pytest_collection_modifyitems(self, session, config, items): + self.items = list(items) + + collector = Collector() + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + pytest.main([scope, "--collect-only", "-q", "--profile", profile], plugins=[collector]) + + runnable: List[str] = [] + for item in collector.items: + if not item.nodeid.startswith("tests/"): + continue + marker = item.get_closest_marker("skip_profile") + marked_profiles = set(marker.args) if marker else set() + if profile in marked_profiles: + continue + runnable.append(item.nodeid) + return sorted(set(runnable)) + + +def run_one_test( + nodeid: str, profile: str, timeout_seconds: int, output_dir: Path +) -> TestAuditResult: + safe_name = nodeid.replace("/", "__").replace("::", "__") + log_file = output_dir / f"{safe_name}.log" + cmd = [ + sys.executable, + str(RUNNER_PATH), + "--profile", + profile, + "--test-path", + nodeid, + ] + + started = time.monotonic() + try: + completed = subprocess.run( + cmd, + cwd=DBT_SPARK_ROOT, + text=True, + capture_output=True, + check=False, + timeout=timeout_seconds, + ) + timed_out = False + return_code = completed.returncode + output = completed.stdout + if completed.stderr: + output += "\n\n=== STDERR ===\n" + completed.stderr + except subprocess.TimeoutExpired as exc: + timed_out = True + return_code = 124 + out = exc.stdout or "" + err = exc.stderr or "" + if isinstance(out, bytes): + out = out.decode(errors="replace") + if isinstance(err, bytes): + err = err.decode(errors="replace") + output = f"Timed out after {timeout_seconds}s.\n\n{out}" + if err: + output += "\n\n=== STDERR ===\n" + err + + duration_seconds = round(time.monotonic() - started, 2) + log_file.write_text(output) + return TestAuditResult( + nodeid=nodeid, + return_code=return_code, + timed_out=timed_out, + duration_seconds=duration_seconds, + log_file=str(log_file.relative_to(DBT_SPARK_ROOT)), + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Run each spark_session-runnable test individually with timeout. " + "Tests that do not finish before timeout are killed and classified as slow." + ) + ) + parser.add_argument("--profile", default="spark_session") + parser.add_argument("--scope", default=DEFAULT_SCOPE) + parser.add_argument("--jobs", type=int, default=3) + parser.add_argument("--slow-timeout-seconds", type=int, default=180) + parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR)) + parser.add_argument( + "--max-tests", type=int, default=0, help="Run only first N tests (0 = all)" + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + nodeids = collect_runnable_nodeids(args.scope, args.profile) + if args.max_tests > 0: + nodeids = nodeids[: args.max_tests] + + if not nodeids: + print("No runnable tests found.") + return 1 + + print(f"Profile: {args.profile}") + print(f"Scope: {args.scope}") + print(f"Runnable tests to audit: {len(nodeids)}") + print( + f"Running individually with jobs={args.jobs}; " + f"kill threshold={args.slow_timeout_seconds}s" + ) + + results: List[TestAuditResult] = [] + max_workers = min(max(args.jobs, 1), len(nodeids)) + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit( + run_one_test, + nodeid, + args.profile, + args.slow_timeout_seconds, + output_dir, + ): nodeid + for nodeid in nodeids + } + for future in concurrent.futures.as_completed(futures): + result = future.result() + results.append(result) + status = "FAST" if result.return_code == 0 else "SLOW" + timeout_note = " timeout" if result.timed_out else "" + print(f"[{status}] {result.duration_seconds:6.2f}s{timeout_note} {result.nodeid}") + + results = sorted(results, key=lambda item: item.nodeid) + fast = sorted( + [r for r in results if r.return_code == 0], key=lambda item: item.duration_seconds + ) + slow = sorted([r for r in results if r.return_code != 0], key=lambda item: item.nodeid) + + fast_list = output_dir / "fast_tests.txt" + slow_list = output_dir / "slow_tests.txt" + json_report = output_dir / "audit_report.json" + + fast_list.write_text("\n".join([r.nodeid for r in fast]) + ("\n" if fast else "")) + slow_list.write_text("\n".join([r.nodeid for r in slow]) + ("\n" if slow else "")) + json_report.write_text( + json.dumps( + { + "profile": args.profile, + "scope": args.scope, + "jobs": args.jobs, + "slow_timeout_seconds": args.slow_timeout_seconds, + "total_audited": len(results), + "fast_count": len(fast), + "slow_count": len(slow), + "fast_tests_file": str(fast_list.relative_to(DBT_SPARK_ROOT)), + "slow_tests_file": str(slow_list.relative_to(DBT_SPARK_ROOT)), + "results": [asdict(r) for r in results], + }, + indent=2, + ) + + "\n" + ) + + print("\nAudit summary:") + print(f" Total audited: {len(results)}") + print(f" Fast tests: {len(fast)}") + print(f" Slow tests: {len(slow)}") + print(f" Fast list: {fast_list.relative_to(DBT_SPARK_ROOT)}") + print(f" Slow list: {slow_list.relative_to(DBT_SPARK_ROOT)}") + print(f" JSON report: {json_report.relative_to(DBT_SPARK_ROOT)}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dbt-spark/dagger/run_dbt_spark_tests.py b/dbt-spark/dagger/run_dbt_spark_tests.py index 3d5432e0c9..80fbe17e35 100644 --- a/dbt-spark/dagger/run_dbt_spark_tests.py +++ b/dbt-spark/dagger/run_dbt_spark_tests.py @@ -92,7 +92,6 @@ def get_spark_container(client: dagger.Client) -> Tuple[dagger.Service, str]: async def test_spark(test_args): async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client: - # create cache volumes, these are persisted between runs saving time when developing locally tst_container = ( client.container(platform=dagger.Platform("linux/amd64")) @@ -154,13 +153,28 @@ async def test_spark(test_args): tst_container = tst_container.with_exec(["./scripts/install_jdk.sh"]) # run the tests - result = ( - await tst_container.with_workdir("/src") - .with_exec( - ["hatch", "run", "pytest", "--profile", test_args.profile, test_args.test_path] - ) - .stdout() - ) + pytest_cmd = [ + "hatch", + "run", + "pytest", + "--profile", + test_args.profile, + *test_args.test_path, + ] + if test_args.profile == "spark_session": + # Spark session mode is unstable under xdist parallelism in Dagger; run serially. + pytest_cmd.extend(["-n", "0"]) + + result = await tst_container.with_workdir("/src").with_exec(pytest_cmd).stdout() + + print(result) + + # Print summary at the end + for line in result.strip().splitlines(): + if "passed" in line or "failed" in line or "error" in line: + print(f"\n{'=' * 60}") + print(f"SUMMARY: {line.strip()}") + print(f"{'=' * 60}") return result @@ -168,7 +182,13 @@ async def test_spark(test_args): # TODO: update this to align more closely with the pytest api, e.g. --test-path should be an arg instead of a kwarg parser = argparse.ArgumentParser() parser.add_argument("--profile", required=True, type=str) -parser.add_argument("--test-path", required=False, type=str, default="tests/functional/adapter") +parser.add_argument( + "--test-path", + required=False, + nargs="+", + default=["tests/functional/adapter"], + help="One or more pytest paths to run", +) args = parser.parse_args() anyio.run(test_spark, args) diff --git a/dbt-spark/dagger/run_dbt_spark_tests_sharded.py b/dbt-spark/dagger/run_dbt_spark_tests_sharded.py new file mode 100644 index 0000000000..dedd6ec0d6 --- /dev/null +++ b/dbt-spark/dagger/run_dbt_spark_tests_sharded.py @@ -0,0 +1,530 @@ +from __future__ import annotations + +import argparse +import concurrent.futures +import contextlib +import io +import os +import re +import signal +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List + +import pytest + +DBT_SPARK_ROOT = Path(__file__).resolve().parents[1] +RUNNER_PATH = DBT_SPARK_ROOT / "dagger" / "run_dbt_spark_tests.py" +DEFAULT_GLOB = "tests/functional/adapter/**/test_*.py" +LOG_DIR = DBT_SPARK_ROOT / "target" / "dagger-shards" +ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") +STATUS_RE_1 = re.compile(r"(PASSED|SKIPPED|FAILED|ERROR)\s+(tests/functional/adapter/\S+)") +STATUS_RE_2 = re.compile(r"(tests/functional/adapter/\S+)\s+(PASSED|SKIPPED|FAILED|ERROR)") +PYTEST_SUMMARY_RE = re.compile(r"=+\s+.*(passed|skipped|failed|error).* in .*=+") + + +@dataclass +class TaskSpec: + task_index: int + test_paths: List[str] + + +@dataclass +class TaskResult: + task_index: int + test_paths: List[str] + return_code: int + timed_out: bool + attempts: int + log_file: Path + summary_line: str + status_by_test: Dict[str, str] + + +def _collect_ancestor_pids() -> set[int]: + ancestors: set[int] = set() + pid = os.getpid() + for _ in range(10): + if pid <= 1: + break + ancestors.add(pid) + try: + parent = ( + os.getppid() + if pid == os.getpid() + else int( + subprocess.check_output( + ["ps", "-o", "ppid=", "-p", str(pid)], + text=True, + ).strip() + ) + ) + except Exception: + break + if parent in ancestors: + break + pid = parent + return ancestors + + +def find_runner_processes_for_profile(profile: str) -> List[int]: + try: + ps_output = subprocess.check_output(["ps", "-eo", "pid,args"], text=True) + except subprocess.CalledProcessError: + return [] + + current_and_ancestors = _collect_ancestor_pids() + matching: List[int] = [] + for line in ps_output.splitlines(): + line = line.strip() + if not line: + continue + parts = line.split(None, 1) + if len(parts) != 2: + continue + pid_str, args = parts + if not pid_str.isdigit(): + continue + pid = int(pid_str) + if pid in current_and_ancestors: + continue + if "run_dbt_spark_tests.py" not in args: + continue + if f"--profile {profile}" not in args: + continue + matching.append(pid) + return sorted(set(matching)) + + +def terminate_pids(pids: List[int], grace_seconds: float = 1.5) -> List[int]: + if not pids: + return [] + + for pid in pids: + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGTERM) + time.sleep(grace_seconds) + + survivors: List[int] = [] + for pid in pids: + try: + os.kill(pid, 0) + survivors.append(pid) + except ProcessLookupError: + continue + + for pid in survivors: + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGKILL) + return survivors + + +def cleanup_stale_runner_processes(profile: str) -> None: + stale_pids = find_runner_processes_for_profile(profile) + if not stale_pids: + return + print( + f"Found {len(stale_pids)} stale runner process(es) for profile '{profile}': " + + ", ".join(str(pid) for pid in stale_pids) + ) + killed_after_term = terminate_pids(stale_pids) + if killed_after_term: + print( + "Escalated to SIGKILL for pid(s): " + ", ".join(str(pid) for pid in killed_after_term) + ) + print("Stale runner cleanup complete.") + + +def discover_test_files(test_glob: str) -> List[str]: + files = [ + str(path.relative_to(DBT_SPARK_ROOT)) + for path in DBT_SPARK_ROOT.glob(test_glob) + if path.is_file() + ] + return sorted(files) + + +def build_shards(paths: List[str], shard_count: int) -> List[List[str]]: + if shard_count < 1: + raise ValueError("shard_count must be >= 1") + shards: List[List[str]] = [[] for _ in range(shard_count)] + for idx, path in enumerate(paths): + shards[idx % shard_count].append(path) + return [shard for shard in shards if shard] + + +def clean_line(line: str) -> str: + return ANSI_RE.sub("", line) + + +def parse_statuses(output: str) -> Dict[str, str]: + statuses: Dict[str, str] = {} + for raw_line in output.splitlines(): + line = clean_line(raw_line) + match = STATUS_RE_1.search(line) + if match: + statuses[match.group(2)] = match.group(1) + continue + match = STATUS_RE_2.search(line) + if match: + statuses[match.group(1)] = match.group(2) + return statuses + + +def extract_pytest_summary(output: str) -> str: + summary = "" + for raw_line in output.splitlines(): + line = clean_line(raw_line) + if PYTEST_SUMMARY_RE.search(line): + summary = line.strip() + return summary + + +def run_task(task: TaskSpec, profile: str, timeout_seconds: int, retries: int) -> TaskResult: + cmd = [ + sys.executable, + str(RUNNER_PATH), + "--profile", + profile, + "--test-path", + *task.test_paths, + ] + + LOG_DIR.mkdir(parents=True, exist_ok=True) + log_file = LOG_DIR / f"task_{task.task_index:03d}.log" + + merged_output = "" + return_code = 1 + timed_out = False + attempts_used = 0 + + for attempt in range(1, retries + 2): + attempts_used = attempt + try: + completed = subprocess.run( + cmd, + cwd=DBT_SPARK_ROOT, + text=True, + capture_output=True, + check=False, + timeout=timeout_seconds if timeout_seconds > 0 else None, + ) + attempt_output = completed.stdout + if completed.stderr: + attempt_output += "\n\n=== STDERR ===\n" + completed.stderr + merged_output += f"\n\n===== ATTEMPT {attempt} =====\n" + attempt_output + return_code = completed.returncode + timed_out = False + if return_code == 0: + break + except subprocess.TimeoutExpired as exc: + timed_out = True + timed_stdout = exc.stdout or "" + timed_stderr = exc.stderr or "" + if isinstance(timed_stdout, bytes): + timed_stdout = timed_stdout.decode(errors="replace") + if isinstance(timed_stderr, bytes): + timed_stderr = timed_stderr.decode(errors="replace") + merged_output += ( + f"\n\n===== ATTEMPT {attempt} =====\n" + f"Task timed out after {timeout_seconds}s.\n" + timed_stdout + ) + if timed_stderr: + merged_output += "\n\n=== STDERR ===\n" + timed_stderr + return_code = 124 + + log_file.write_text(merged_output) + status_by_test = parse_statuses(merged_output) + summary_line = extract_pytest_summary(merged_output) + return TaskResult( + task_index=task.task_index, + test_paths=task.test_paths, + return_code=return_code, + timed_out=timed_out, + attempts=attempts_used, + log_file=log_file, + summary_line=summary_line, + status_by_test=status_by_test, + ) + + +def run_all_tasks( + tasks: List[TaskSpec], + profile: str, + jobs: int, + timeout_seconds: int, + retries: int, + cleanup_stale: bool = True, +) -> List[TaskResult]: + results: List[TaskResult] = [] + if cleanup_stale and profile == "spark_session": + cleanup_stale_runner_processes(profile) + max_workers = min(max(jobs, 1), len(tasks)) + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + future_map = { + executor.submit(run_task, task, profile, timeout_seconds, retries): task.task_index + for task in tasks + } + for future in concurrent.futures.as_completed(future_map): + result = future.result() + results.append(result) + status = "PASS" if result.return_code == 0 else "FAIL" + timeout_note = " (timeout)" if result.timed_out else "" + print(f"[{status}] task {result.task_index}{timeout_note} -> {result.log_file}") + if result.summary_line: + print(f" {result.summary_line}") + return sorted(results, key=lambda r: r.task_index) + + +def collect_nodeids_for_path(path: str, profile: str) -> List[str]: + class Collector: + def __init__(self) -> None: + self.items = [] + + def pytest_collection_modifyitems(self, session, config, items): + self.items = list(items) + + collector = Collector() + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + pytest.main([path, "--collect-only", "-q", "--profile", profile], plugins=[collector]) + nodeids: List[str] = [] + for item in collector.items: + nodeid = item.nodeid + if nodeid.startswith("tests/"): + nodeids.append(nodeid) + return sorted(set(nodeids)) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Run dbt-spark Dagger integration tests with isolated parallel tasks. " + "Use file mode for maximum stability in spark_session." + ) + ) + parser.add_argument("--profile", default="spark_session", help="dbt-spark test profile") + parser.add_argument( + "--mode", + choices=["files", "shards"], + default="files", + help="files: one test file per task, shards: grouped files per task", + ) + parser.add_argument("--shards", type=int, default=6, help="Number of shards when mode=shards") + parser.add_argument("--jobs", type=int, default=3, help="Maximum tasks to run in parallel") + parser.add_argument( + "--timeout-seconds", type=int, default=1500, help="Per-task timeout (0 disables timeout)" + ) + parser.add_argument("--retries", type=int, default=1, help="Retries per failed/timed-out task") + parser.add_argument( + "--fallback-serial", + action="store_true", + help="On failure/timeout, rerun failed files serially (jobs=1)", + ) + parser.add_argument( + "--fallback-timeout-seconds", + type=int, + default=0, + help="Per-file timeout for fallback reruns (0 = 2x --timeout-seconds)", + ) + parser.add_argument( + "--fallback-retries", + type=int, + default=0, + help="Retries for fallback per-file reruns", + ) + parser.add_argument( + "--fallback-split-tests", + action="store_true", + help="If fallback file reruns still fail, split failed files into per-test reruns", + ) + parser.add_argument( + "--fallback-split-jobs", + type=int, + default=2, + help="Parallelism for fallback split-test reruns", + ) + parser.add_argument( + "--test-glob", + default=DEFAULT_GLOB, + help=f"Glob used to discover test files (default: {DEFAULT_GLOB})", + ) + parser.add_argument( + "--test-path", + nargs="*", + default=[], + help="Optional explicit test file paths (used instead of discovery)", + ) + parser.add_argument( + "--always-split-path", + nargs="*", + default=[], + help="File paths that should always be split into per-test tasks up front", + ) + parser.add_argument( + "--max-tasks", + type=int, + default=0, + help="If > 0, run only first N tasks (smoke testing)", + ) + parser.add_argument("--dry-run", action="store_true", help="Print task plan and exit") + parser.add_argument( + "--no-cleanup-stale-runners", + action="store_true", + help="Skip preflight cleanup of stale run_dbt_spark_tests.py processes", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + if args.test_path: + test_files = sorted(args.test_path) + else: + test_files = discover_test_files(args.test_glob) + + if not test_files: + print("No test files found.") + return 1 + + split_paths = set(args.always_split_path) + if args.mode == "files": + task_paths: List[List[str]] = [] + for path in test_files: + if path in split_paths: + nodeids = collect_nodeids_for_path(path, args.profile) + if nodeids: + task_paths.extend([[nodeid] for nodeid in nodeids]) + else: + task_paths.append([path]) + else: + task_paths.append([path]) + else: + task_paths = build_shards(test_files, min(args.shards, len(test_files))) + + tasks = [ + TaskSpec(task_index=idx, test_paths=paths) for idx, paths in enumerate(task_paths, start=1) + ] + if args.max_tasks > 0: + tasks = tasks[: args.max_tasks] + + print(f"Discovered {len(test_files)} test files") + print( + f"Running {len(tasks)} task(s) in {args.mode} mode with up to {args.jobs} parallel job(s); " + f"timeout={args.timeout_seconds}s retries={args.retries}" + ) + for task in tasks: + print(f" task {task.task_index}: {len(task.test_paths)} file(s)") + + if args.dry_run: + return 0 + + results = run_all_tasks( + tasks, + args.profile, + args.jobs, + args.timeout_seconds, + args.retries, + cleanup_stale=not args.no_cleanup_stale_runners, + ) + + fallback_results: List[TaskResult] = [] + if args.fallback_serial: + failed_paths = sorted( + {path for result in results if result.return_code != 0 for path in result.test_paths} + ) + if failed_paths: + print("\nRunning fallback serial reruns for failed paths...") + fallback_tasks = [ + TaskSpec(task_index=1000 + idx, test_paths=[path]) + for idx, path in enumerate(failed_paths, start=1) + ] + fallback_timeout = ( + args.fallback_timeout_seconds + if args.fallback_timeout_seconds > 0 + else (args.timeout_seconds * 2 if args.timeout_seconds > 0 else 0) + ) + fallback_results = run_all_tasks( + fallback_tasks, + args.profile, + jobs=1, + timeout_seconds=fallback_timeout, + retries=args.fallback_retries, + cleanup_stale=not args.no_cleanup_stale_runners, + ) + results.extend(fallback_results) + + failed = [result for result in results if result.return_code != 0] + unresolved_paths = {path for result in failed for path in result.test_paths} + for result in fallback_results: + if result.return_code == 0: + unresolved_paths.discard(result.test_paths[0]) + + if unresolved_paths and args.fallback_split_tests: + print("\nRunning fallback per-test reruns for unresolved paths...") + split_tasks: List[TaskSpec] = [] + split_index = 2000 + for path in sorted(unresolved_paths): + nodeids = collect_nodeids_for_path(path, args.profile) + if not nodeids: + split_tasks.append(TaskSpec(task_index=split_index, test_paths=[path])) + split_index += 1 + continue + for nodeid in nodeids: + split_tasks.append(TaskSpec(task_index=split_index, test_paths=[nodeid])) + split_index += 1 + + split_timeout = ( + args.fallback_timeout_seconds + if args.fallback_timeout_seconds > 0 + else (args.timeout_seconds * 2 if args.timeout_seconds > 0 else 0) + ) + split_results = run_all_tasks( + split_tasks, + args.profile, + jobs=args.fallback_split_jobs, + timeout_seconds=split_timeout, + retries=args.fallback_retries, + cleanup_stale=not args.no_cleanup_stale_runners, + ) + results.extend(split_results) + unresolved_paths = set() + for result in split_results: + if result.return_code != 0: + unresolved_paths.update(result.test_paths) + + combined_status: Dict[str, str] = {} + for result in results: + combined_status.update(result.status_by_test) + counts = {"PASSED": 0, "SKIPPED": 0, "FAILED": 0, "ERROR": 0} + for status in combined_status.values(): + if status in counts: + counts[status] += 1 + print("\nAggregated status counts:") + print(f" PASSED: {counts['PASSED']}") + print(f" SKIPPED: {counts['SKIPPED']}") + print(f" FAILED: {counts['FAILED']}") + print(f" ERROR: {counts['ERROR']}") + print(f" WITH STATUS: {len(combined_status)}") + + if unresolved_paths: + print("\nFailed tasks:") + for result in failed: + files = ", ".join(result.test_paths) + timeout_note = " (timeout)" if result.timed_out else "" + print(f" task {result.task_index}{timeout_note}: {files}") + print(f" log: {result.log_file}") + print("\nUnresolved failed paths:") + for path in sorted(unresolved_paths): + print(f" {path}") + return 1 + + print("\nAll tasks passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dbt-spark/dagger/run_dbt_spark_tests_tiered.py b/dbt-spark/dagger/run_dbt_spark_tests_tiered.py new file mode 100644 index 0000000000..d001177b4b --- /dev/null +++ b/dbt-spark/dagger/run_dbt_spark_tests_tiered.py @@ -0,0 +1,439 @@ +from __future__ import annotations + +import argparse +import concurrent.futures +import contextlib +import io +from pathlib import Path +from typing import List, Set, Tuple + +import pytest + +from run_dbt_spark_tests_sharded import ( + TaskSpec, + build_shards, + cleanup_stale_runner_processes, + run_all_tasks, +) + +DEFAULT_TEST_SCOPE = "tests/functional/adapter" +DBT_SPARK_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_FAST_LIST_FILE = DBT_SPARK_ROOT / "dagger" / "test_lists" / "spark_session_fast.txt" +DEFAULT_FAST_TAIL_LIST_FILE = ( + DBT_SPARK_ROOT / "dagger" / "test_lists" / "spark_session_fast_tail_grouped.txt" +) +DEFAULT_SLOW_WORKING_LIST_FILE = ( + DBT_SPARK_ROOT / "dagger" / "test_lists" / "spark_session_slow_but_working.txt" +) +DEFAULT_EXCLUDED_LIST_FILES = [ + DBT_SPARK_ROOT / "dagger" / "test_lists" / "spark_session_timed_out.txt", + DBT_SPARK_ROOT / "dagger" / "test_lists" / "spark_session_errored.txt", +] + +# Observed long/hang-prone tests from recent runs. +SLOW_NODEIDS = { + "tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChangeSpecialChars::test_incremental_append_new_columns_with_special_characters", + "tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_ignore", + "tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_fail_on_schema_change", + "tests/functional/adapter/utils/test_utils.py::TestAnyValue::test_build_assert_equal", + "tests/functional/adapter/utils/test_utils.py::TestBoolOr::test_build_assert_equal", + "tests/functional/adapter/utils/test_utils.py::TestDateAdd::test_build_assert_equal", + "tests/functional/adapter/utils/test_utils.py::TestEquals::test_build_assert_equal", + "tests/functional/adapter/utils/test_utils.py::TestListagg::test_build_assert_equal", +} + + +def collect_runnable_nodeids(path: str, profile: str) -> List[str]: + class Collector: + def __init__(self) -> None: + self.items = [] + + def pytest_collection_modifyitems(self, session, config, items): + self.items = list(items) + + collector = Collector() + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + pytest.main([path, "--collect-only", "-q", "--profile", profile], plugins=[collector]) + + runnable: List[str] = [] + for item in collector.items: + nodeid = item.nodeid + if not nodeid.startswith("tests/"): + continue + marker = item.get_closest_marker("skip_profile") + marked_profiles = set(marker.args) if marker else set() + if profile in marked_profiles: + continue + runnable.append(nodeid) + return sorted(set(runnable)) + + +def build_groups(all_runnable: List[str]) -> Tuple[List[str], List[str]]: + collected = set(all_runnable) + slow = sorted(collected & SLOW_NODEIDS) + fast = sorted(collected - set(slow)) + return fast, slow + + +def run_lane( + nodeids: List[str], + profile: str, + jobs: int, + timeout_seconds: int, + retries: int, + task_offset: int, + shards: int = 0, + cleanup_stale: bool = True, +): + if shards > 0: + chunked = build_shards(nodeids, min(shards, len(nodeids))) + tasks = [ + TaskSpec(task_index=task_offset + idx, test_paths=batch) + for idx, batch in enumerate(chunked, start=1) + ] + else: + tasks = [ + TaskSpec(task_index=task_offset + idx, test_paths=[nodeid]) + for idx, nodeid in enumerate(nodeids, start=1) + ] + return run_all_tasks( + tasks, + profile, + jobs=jobs, + timeout_seconds=timeout_seconds, + retries=retries, + cleanup_stale=cleanup_stale, + ) + + +def summarize_status_counts(results) -> dict: + combined = {} + for result in results: + combined.update(result.status_by_test) + counts = {"PASSED": 0, "SKIPPED": 0, "FAILED": 0, "ERROR": 0} + for status in combined.values(): + if status in counts: + counts[status] += 1 + counts["WITH_STATUS"] = len(combined) + return counts + + +def run_grouped_tail_lane( + nodeids: List[str], + profile: str, + timeout_seconds: int, + retries: int, + task_offset: int, + cleanup_stale: bool = True, +): + if not nodeids: + return [] + tasks = [TaskSpec(task_index=task_offset, test_paths=nodeids)] + return run_all_tasks( + tasks, + profile, + jobs=1, + timeout_seconds=timeout_seconds, + retries=retries, + cleanup_stale=cleanup_stale, + ) + + +def load_nodeids_from_file(path: Path) -> Set[str]: + if not path.exists(): + return set() + return { + line.strip() + for line in path.read_text().splitlines() + if line.strip() and not line.strip().startswith("#") + } + + +def run_fast_subpath(args: argparse.Namespace, fast: List[str], fast_tail: List[str]): + print(f"\nStep 1/2: Running fast subpath ({len(fast)} tests)...") + fast_main_results = run_lane( + nodeids=fast, + profile=args.profile, + jobs=args.fast_jobs, + timeout_seconds=args.fast_timeout_seconds, + retries=args.retries, + task_offset=0, + shards=args.fast_shards, + cleanup_stale=not args.no_cleanup_stale_runners, + ) + fast_tail_results = [] + if fast_tail: + print(f"\nStep 1b/2: Running grouped fast tail ({len(fast_tail)} tests)...") + fast_tail_results = run_grouped_tail_lane( + nodeids=fast_tail, + profile=args.profile, + timeout_seconds=args.fast_tail_timeout_seconds, + retries=args.retries, + task_offset=4500, + cleanup_stale=not args.no_cleanup_stale_runners, + ) + return fast_main_results + fast_tail_results + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Effective spark_session test runner: execute fast subpath first, " + "then run remaining tests for full coverage." + ) + ) + parser.add_argument("--profile", default="spark_session", help="dbt-spark profile") + parser.add_argument( + "--test-scope", + default=DEFAULT_TEST_SCOPE, + help="Pytest path to collect tests from", + ) + parser.add_argument( + "--group", + choices=["all", "effective", "fast", "slow"], + default="effective", + help="Run effective (full), fast subpath only, or slow lane only", + ) + parser.add_argument("--fast-jobs", type=int, default=6, help="Parallel jobs for fast lane") + parser.add_argument( + "--fast-shards", + type=int, + default=6, + help="Number of batched tasks for fast lane (0 = one test per task)", + ) + parser.add_argument("--slow-jobs", type=int, default=1, help="Parallel jobs for slow lane") + parser.add_argument( + "--fast-timeout-seconds", type=int, default=600, help="Per-test timeout in fast lane" + ) + parser.add_argument( + "--slow-timeout-seconds", type=int, default=1500, help="Per-test timeout in slow lane" + ) + parser.add_argument("--retries", type=int, default=1, help="Retries per failed/timed-out test") + parser.add_argument( + "--fast-list-file", + default=str(DEFAULT_FAST_LIST_FILE), + help="Optional newline-delimited fast test list; used if file exists", + ) + parser.add_argument( + "--exclude-list-files", + nargs="*", + default=[str(path) for path in DEFAULT_EXCLUDED_LIST_FILES], + help="Optional newline-delimited lists of problematic tests to exclude from fast lane", + ) + parser.add_argument( + "--fast-tail-list-file", + default=str(DEFAULT_FAST_TAIL_LIST_FILE), + help="Optional newline-delimited tests to run as one grouped tail task after fast lane", + ) + parser.add_argument( + "--slow-working-list-file", + default=str(DEFAULT_SLOW_WORKING_LIST_FILE), + help="Optional newline-delimited slow-but-working tests to force into effective runner's slow lane", + ) + parser.add_argument( + "--fast-tail-timeout-seconds", + type=int, + default=600, + help="Timeout for grouped fast-tail task", + ) + parser.add_argument("--dry-run", action="store_true", help="Print grouped tests and exit") + parser.add_argument( + "--effective-lanes-parallel", + action="store_true", + help="Run effective mode with fast subpath and slow lane in parallel", + ) + parser.add_argument( + "--no-cleanup-stale-runners", + action="store_true", + help="Skip preflight cleanup of stale run_dbt_spark_tests.py processes", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if args.group == "effective": + args.group = "all" + + all_runnable = collect_runnable_nodeids(args.test_scope, args.profile) + computed_fast, slow_seed = build_groups(all_runnable) + runnable_set = set(all_runnable) + + excluded_from_files: Set[str] = set() + for raw_path in args.exclude_list_files: + excluded_from_files.update(load_nodeids_from_file(Path(raw_path))) + excluded_from_files = excluded_from_files & runnable_set + if excluded_from_files: + slow_seed = sorted(set(slow_seed) | excluded_from_files) + computed_fast = sorted(runnable_set - set(slow_seed)) + + fast_tail_path = Path(args.fast_tail_list_file) + fast_tail = sorted(load_nodeids_from_file(fast_tail_path) & runnable_set) + if fast_tail: + slow_seed = sorted(set(slow_seed) - set(fast_tail)) + computed_fast = sorted(set(computed_fast) - set(fast_tail)) + + slow_working_path = Path(args.slow_working_list_file) + slow_working = sorted(load_nodeids_from_file(slow_working_path) & runnable_set) + if slow_working: + slow_seed = sorted(set(slow_seed) | set(slow_working)) + computed_fast = sorted(runnable_set - set(slow_seed) - set(fast_tail)) + + fast = computed_fast + fast_list_path = Path(args.fast_list_file) + if fast_list_path.exists(): + listed_fast = load_nodeids_from_file(fast_list_path) & runnable_set + loaded_fast = sorted(listed_fast) + if loaded_fast: + fast = loaded_fast + # Explicit fast-list entries take precedence over seeded slow buckets. + slow_seed = sorted(set(slow_seed) - set(fast)) + print(f"Using fast list from {fast_list_path}") + else: + print( + "Fast list file exists but had no usable runnable entries; using computed fast group." + ) + if excluded_from_files: + print(f"Loaded {len(excluded_from_files)} excluded test(s) from --exclude-list-files.") + if fast_tail: + print(f"Loaded {len(fast_tail)} grouped fast-tail test(s) from {fast_tail_path}.") + if slow_working: + print(f"Loaded {len(slow_working)} slow-working test(s) from {slow_working_path}.") + + print(f"Profile: {args.profile}") + print(f"Scope: {args.test_scope}") + print(f"Total runnable tests: {len(all_runnable)}") + print(f"Fast tests: {len(fast)}") + print(f"Fast tail grouped tests: {len(fast_tail)}") + print(f"Slow working tests: {len(slow_working)}") + print(f"Slow seed tests: {len(slow_seed)}") + + if args.dry_run: + print("\nFast group:") + for nodeid in fast: + print(f" {nodeid}") + print("\nFast tail grouped task:") + for nodeid in fast_tail: + print(f" {nodeid}") + print("\nSlow seed group:") + for nodeid in slow_seed: + print(f" {nodeid}") + return 0 + + if args.group == "fast": + fast_results = run_fast_subpath(args, fast, fast_tail) + failed_fast_tasks = [r for r in fast_results if r.return_code != 0] + fast_counts = summarize_status_counts(fast_results) + print("\nFast-only result:") + print(f" Tasks failed: {len(failed_fast_tasks)}") + print(f" PASSED: {fast_counts['PASSED']}") + print(f" SKIPPED: {fast_counts['SKIPPED']}") + print(f" FAILED: {fast_counts['FAILED']}") + print(f" ERROR: {fast_counts['ERROR']}") + print(f" WITH STATUS: {fast_counts['WITH_STATUS']}") + return 0 if not failed_fast_tasks else 1 + + if args.group == "slow": + print(f"\nRunning slow lane only ({len(slow_seed)} tests)...") + slow_results = run_lane( + nodeids=slow_seed, + profile=args.profile, + jobs=args.slow_jobs, + timeout_seconds=args.slow_timeout_seconds, + retries=args.retries, + task_offset=5000, + cleanup_stale=not args.no_cleanup_stale_runners, + ) + failed_slow = [r for r in slow_results if r.return_code != 0] + print( + f"\nSlow-only result: passed={len(slow_results) - len(failed_slow)} failed={len(failed_slow)}" + ) + return 0 if not failed_slow else 1 + + # Full coverage effective run: + # 1) Run fast subpath. + # 2) Run remaining slow lane. + # Optionally run steps 1 and 2 in parallel. + if args.effective_lanes_parallel: + print("\nRunning effective mode with parallel lanes...") + if not args.no_cleanup_stale_runners: + cleanup_stale_runner_processes(args.profile) + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + fast_future = executor.submit(run_fast_subpath, args, fast, fast_tail) + slow_future = executor.submit( + run_lane, + slow_seed, + args.profile, + args.slow_jobs, + args.slow_timeout_seconds, + args.retries, + 5000, + 0, + False, + ) + fast_results = fast_future.result() + slow_results = slow_future.result() + else: + fast_results = run_fast_subpath(args, fast, fast_tail) + + moved_from_fast = sorted( + [ + path + for result in fast_results + if result.return_code != 0 + for path in result.test_paths + ] + ) + if moved_from_fast: + print(f"\nMoving {len(moved_from_fast)} test(s) from fast lane to slow lane.") + + final_slow = sorted(set(slow_seed) | set(moved_from_fast)) + print(f"\nStep 2/2: Running remaining slow lane ({len(final_slow)} tests)...") + slow_results = run_lane( + nodeids=final_slow, + profile=args.profile, + jobs=args.slow_jobs, + timeout_seconds=args.slow_timeout_seconds, + retries=args.retries, + task_offset=5000, + cleanup_stale=not args.no_cleanup_stale_runners, + ) + + moved_from_fast = sorted( + [path for result in fast_results if result.return_code != 0 for path in result.test_paths] + ) + if args.effective_lanes_parallel and moved_from_fast: + print(f"\nRunning follow-up slow rerun for {len(moved_from_fast)} fast-lane failure(s)...") + followup_results = run_lane( + nodeids=moved_from_fast, + profile=args.profile, + jobs=args.slow_jobs, + timeout_seconds=args.slow_timeout_seconds, + retries=args.retries, + task_offset=7000, + cleanup_stale=False, + ) + slow_results.extend(followup_results) + + final_slow = sorted(set(slow_seed) | set(moved_from_fast)) + + failed_fast_final = [result for result in fast_results if result.return_code != 0] + failed_slow_final = [result for result in slow_results if result.return_code != 0] + total_failed = len(failed_fast_final) + len(failed_slow_final) + + print("\nTiered summary:") + print(f" Total runnable: {len(all_runnable)}") + print(f" Fast lane initial: {len(fast)}") + print(f" Fast tail grouped: {len(fast_tail)}") + print(f" Slow lane seed: {len(slow_seed)}") + print(f" Moved fast->slow: {len(moved_from_fast)}") + print(f" Slow lane final: {len(final_slow)}") + print(f" Fast passed (first pass): {len(fast_results) - len(failed_fast_final)}") + print(f" Slow passed: {len(slow_results) - len(failed_slow_final)}") + print(f" Total failed: {total_failed}") + + return 0 if total_failed == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dbt-spark/dagger/test_lists/spark_session_errored.txt b/dbt-spark/dagger/test_lists/spark_session_errored.txt new file mode 100644 index 0000000000..009bc12734 --- /dev/null +++ b/dbt-spark/dagger/test_lists/spark_session_errored.txt @@ -0,0 +1 @@ +tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestDeltaStrategies::test_delta_strategies_overwrite diff --git a/dbt-spark/dagger/test_lists/spark_session_fast.txt b/dbt-spark/dagger/test_lists/spark_session_fast.txt new file mode 100644 index 0000000000..dc3c4bfbc6 --- /dev/null +++ b/dbt-spark/dagger/test_lists/spark_session_fast.txt @@ -0,0 +1,44 @@ +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowSqlHeader::test_sql_header +tests/functional/adapter/empty/test_empty.py::TestSparkEmptyInlineSourceRef::test_run_with_empty +tests/functional/adapter/empty/test_empty.py::TestSparkEmpty::test_run_with_empty +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_fail_on_schema_change +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChangeSpecialChars::test_incremental_append_new_columns_with_special_characters +tests/functional/adapter/test_basic.py::TestEmptySpark::test_empty +tests/functional/adapter/test_basic.py::TestSingularTestsEphemeralSpark::test_singular_tests_ephemeral +tests/functional/adapter/test_basic.py::TestSingularTestsSpark::test_singular_tests +tests/functional/adapter/test_simple_seed.py::TestBigQueryEmptySeed::test_empty_seeds +tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestCaseInsensitivity::test_case_insensitivity +tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestInvalidInput::test_invalid_input +tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestingTypes::test_unit_test_data_type +tests/functional/adapter/utils/test_data_types.py::TestTypeBigInt::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeBoolean::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeFloat::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeInt::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeNumeric::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeString::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeTimestamp::test_check_types_assert_match +tests/functional/adapter/utils/test_timestamps.py::TestCurrentTimestampSpark::test_current_timestamps +tests/functional/adapter/utils/test_utils.py::TestAnyValue::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestArrayAppend::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestArrayAppend::test_expected_actual +tests/functional/adapter/utils/test_utils.py::TestArrayConcat::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestArrayConcat::test_expected_actual +tests/functional/adapter/utils/test_utils.py::TestArrayConstruct::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestArrayConstruct::test_expected_actual +tests/functional/adapter/utils/test_utils.py::TestBoolOr::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestCast::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestCastBoolToText::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestCurrentTimestamp::test_current_timestamp_matches_utc +tests/functional/adapter/utils/test_utils.py::TestCurrentTimestamp::test_current_timestamp_type +tests/functional/adapter/utils/test_utils.py::TestDate::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestDateAdd::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestDateTrunc::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestEquals::test_equal_values +tests/functional/adapter/utils/test_utils.py::TestEscapeSingleQuotes::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestExcept::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestIntersect::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestLength::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestPosition::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestSafeCast::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestSplitPart::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestStringLiteral::test_build_assert_equal diff --git a/dbt-spark/dagger/test_lists/spark_session_fast_tail_grouped.txt b/dbt-spark/dagger/test_lists/spark_session_fast_tail_grouped.txt new file mode 100644 index 0000000000..d5f7fe5071 --- /dev/null +++ b/dbt-spark/dagger/test_lists/spark_session_fast_tail_grouped.txt @@ -0,0 +1,3 @@ +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args0-5] +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args1-3] +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args2-7] diff --git a/dbt-spark/dagger/test_lists/spark_session_slow_but_working.txt b/dbt-spark/dagger/test_lists/spark_session_slow_but_working.txt new file mode 100644 index 0000000000..c1a4c7a0dd --- /dev/null +++ b/dbt-spark/dagger/test_lists/spark_session_slow_but_working.txt @@ -0,0 +1,3 @@ +# Tests known to be valid but currently high-latency in spark_session. +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_ignore +tests/functional/adapter/test_basic.py::TestGenericTestsSpark::test_generic_tests diff --git a/dbt-spark/dagger/test_lists/spark_session_timed_out.txt b/dbt-spark/dagger/test_lists/spark_session_timed_out.txt new file mode 100644 index 0000000000..a671715004 --- /dev/null +++ b/dbt-spark/dagger/test_lists/spark_session_timed_out.txt @@ -0,0 +1,7 @@ +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args0-5] +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args1-3] +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args2-7] +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_ignore +tests/functional/adapter/test_basic.py::TestGenericTestsSpark::test_generic_tests +tests/functional/adapter/utils/test_utils.py::TestEquals::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestListagg::test_build_assert_equal diff --git a/dbt-spark/docker/spark.Dockerfile b/dbt-spark/docker/spark.Dockerfile index 3e12962050..9685f9d86d 100644 --- a/dbt-spark/docker/spark.Dockerfile +++ b/dbt-spark/docker/spark.Dockerfile @@ -1,4 +1,4 @@ -ARG OPENJDK_VERSION=8 +ARG OPENJDK_VERSION=17 FROM eclipse-temurin:${OPENJDK_VERSION}-jre ARG BUILD_DATE diff --git a/dbt-spark/hatch.toml b/dbt-spark/hatch.toml index f5edc69dab..b59148c7b3 100644 --- a/dbt-spark/hatch.toml +++ b/dbt-spark/hatch.toml @@ -6,6 +6,7 @@ packages = ["src/dbt"] sources = ["src"] [envs.default] +python = "3.12" pre-install-commands = [ "pip install -e ../dbt-adapters", "pip install -e ../dbt-tests-adapter", @@ -13,7 +14,6 @@ pre-install-commands = [ dependencies = [ "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", - "ddtrace==2.3.0", "ipdb~=0.13.13", "pre-commit==3.7.0", "freezegun", @@ -35,6 +35,16 @@ setup = [ code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" integration-tests = "python dagger/run_dbt_spark_tests.py {args:--profile apache_spark}" +integration-tests-sharded = "python dagger/run_dbt_spark_tests_sharded.py {args:--profile spark_session}" +integration-tests-safe = "python dagger/run_dbt_spark_tests_sharded.py {args:--profile spark_session --mode files --jobs 3 --retries 1 --timeout-seconds 1500 --fallback-serial --always-split-path tests/functional/adapter/incremental/test_incremental_on_schema_change.py tests/functional/adapter/utils/test_utils.py}" +# Effective spark_session runner (fast subpath + remaining coverage). +integration-tests-spark-session-effective = "python dagger/run_dbt_spark_tests_tiered.py {args:--profile spark_session --group effective}" +# Fast subpath only (used by effective runner step 1). +integration-tests-spark-session-fast-subpath = "python dagger/run_dbt_spark_tests_tiered.py {args:--profile spark_session --group fast}" +integration-tests-spark-session-fast = "python dagger/run_dbt_spark_tests_tiered.py {args:--profile spark_session --group fast}" +integration-tests-spark-session-slow = "python dagger/run_dbt_spark_tests_tiered.py {args:--profile spark_session --group slow}" +integration-tests-spark-session-tiered-all = "python dagger/run_dbt_spark_tests_tiered.py {args:--profile spark_session --group all}" +integration-tests-spark-session-audit-speeds = "python dagger/audit_spark_session_test_speeds.py {args:--profile spark_session --scope tests/functional/adapter --jobs 3 --slow-timeout-seconds 180}" docker-prod = "docker build -f docker/Dockerfile -t dbt-spark ." [envs.build] @@ -70,7 +80,7 @@ pre-install-commands = [ dependencies = [ "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", - "ddtrace==2.3.0", + "freezegun", "pytest>=7.0,<8.0", "pytest-csv~=3.0", @@ -91,7 +101,7 @@ integration-tests = [ [envs.cd] pre-install-commands = [] dependencies = [ - "ddtrace==2.3.0", + "freezegun", "pytest>=7.0,<8.0", "pytest-csv~=3.0", diff --git a/dbt-spark/pyproject.toml b/dbt-spark/pyproject.toml index b0012687cc..82dd631b14 100644 --- a/dbt-spark/pyproject.toml +++ b/dbt-spark/pyproject.toml @@ -7,7 +7,18 @@ dynamic = ["version"] name = "dbt-spark" description = "The Apache Spark adapter plugin for dbt" readme = "README.md" -keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs", "spark"] +keywords = [ + "dbt", + "adapter", + "adapters", + "database", + "elt", + "dbt-core", + "dbt Core", + "dbt Cloud", + "dbt Labs", + "spark", +] requires-python = ">=3.10.0" authors = [{ name = "dbt Labs", email = "info@dbtlabs.com" }] maintainers = [{ name = "dbt Labs", email = "info@dbtlabs.com" }] @@ -31,10 +42,7 @@ dependencies = [ ] [project.optional-dependencies] ODBC = ["pyodbc>=5.1,<5.3"] -PyHive = [ - "PyHive[hive_pure_sasl]~=0.7.0", - "thrift>=0.11.0,<0.23.0", -] +PyHive = ["PyHive[hive_pure_sasl]~=0.7.0", "thrift>=0.11.0,<0.23.0"] session = ["pyspark>=3.0.0,<5.0.0"] all = [ "pyodbc>=5.1,<5.3", diff --git a/dbt-spark/test_results.txt b/dbt-spark/test_results.txt new file mode 100644 index 0000000000..d999d85f08 --- /dev/null +++ b/dbt-spark/test_results.txt @@ -0,0 +1,539 @@ +Obtaining file:///dbt-adapters + Installing build dependencies: started + Installing build dependencies: finished with status 'done' + Checking if build backend supports build_editable: started + Checking if build backend supports build_editable: finished with status 'done' + Getting requirements to build editable: started + Getting requirements to build editable: finished with status 'done' + Installing backend dependencies: started + Installing backend dependencies: finished with status 'done' + Preparing editable metadata (pyproject.toml): started + Preparing editable metadata (pyproject.toml): finished with status 'done' +Collecting agate<2.0,>=1.0 (from dbt-adapters==1.22.9) + Using cached agate-1.14.2-py3-none-any.whl.metadata (3.1 kB) +Collecting dbt-common<2.0,>=1.36 (from dbt-adapters==1.22.9) + Using cached dbt_common-1.37.3-py3-none-any.whl.metadata (4.9 kB) +Collecting dbt-protos<2.0,>=1.0.291 (from dbt-adapters==1.22.9) + Using cached dbt_protos-1.0.443-py3-none-any.whl.metadata (859 bytes) +Collecting mashumaro<3.15,>=3.9 (from mashumaro[msgpack]<3.15,>=3.9->dbt-adapters==1.22.9) + Using cached mashumaro-3.14-py3-none-any.whl.metadata (114 kB) +Collecting protobuf<7.0,>=6.0 (from dbt-adapters==1.22.9) + Using cached protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl.metadata (593 bytes) +Collecting pytz>=2015.7 (from dbt-adapters==1.22.9) + Using cached pytz-2026.1.post1-py2.py3-none-any.whl.metadata (22 kB) +Collecting typing-extensions<5.0,>=4.0 (from dbt-adapters==1.22.9) + Using cached typing_extensions-4.15.0-py3-none-any.whl.metadata (3.3 kB) +Collecting Babel>=2.0 (from agate<2.0,>=1.0->dbt-adapters==1.22.9) + Using cached babel-2.18.0-py3-none-any.whl.metadata (2.2 kB) +Collecting isodate>=0.5.4 (from agate<2.0,>=1.0->dbt-adapters==1.22.9) + Using cached isodate-0.7.2-py3-none-any.whl.metadata (11 kB) +Collecting leather>=0.3.2 (from agate<2.0,>=1.0->dbt-adapters==1.22.9) + Using cached leather-0.4.1-py3-none-any.whl.metadata (3.0 kB) +Collecting parsedatetime!=2.5,>=2.1 (from agate<2.0,>=1.0->dbt-adapters==1.22.9) + Using cached parsedatetime-2.6-py3-none-any.whl.metadata (4.7 kB) +Collecting python-slugify>=1.2.1 (from agate<2.0,>=1.0->dbt-adapters==1.22.9) + Using cached python_slugify-8.0.4-py2.py3-none-any.whl.metadata (8.5 kB) +Collecting pytimeparse>=1.1.5 (from agate<2.0,>=1.0->dbt-adapters==1.22.9) + Using cached pytimeparse-1.1.8-py2.py3-none-any.whl.metadata (3.4 kB) +Collecting agate<2.0,>=1.0 (from dbt-adapters==1.22.9) + Using cached agate-1.9.1-py2.py3-none-any.whl.metadata (3.2 kB) +Collecting colorama<0.5,>=0.3.9 (from dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached colorama-0.4.6-py2.py3-none-any.whl.metadata (17 kB) +Collecting deepdiff<9.0,>=7.0 (from dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached deepdiff-8.6.2-py3-none-any.whl.metadata (8.8 kB) +Collecting jinja2<4,>=3.1.3 (from dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached jinja2-3.1.6-py3-none-any.whl.metadata (2.9 kB) +Collecting jsonschema<5.0,>=4.0 (from dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached jsonschema-4.26.0-py3-none-any.whl.metadata (7.6 kB) +Collecting pathspec<0.13,>=0.9 (from dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached pathspec-0.12.1-py3-none-any.whl.metadata (21 kB) +Collecting python-dateutil<3.0,>=2.0 (from dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl.metadata (8.4 kB) +Collecting requests<3.0.0 (from dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached requests-2.33.1-py3-none-any.whl.metadata (4.8 kB) +Collecting orderly-set<6,>=5.4.1 (from deepdiff<9.0,>=7.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached orderly_set-5.5.0-py3-none-any.whl.metadata (6.6 kB) +Collecting MarkupSafe>=2.0 (from jinja2<4,>=3.1.3->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (2.7 kB) +Collecting attrs>=22.2.0 (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached attrs-26.1.0-py3-none-any.whl.metadata (8.8 kB) +Collecting jsonschema-specifications>=2023.03.6 (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached jsonschema_specifications-2025.9.1-py3-none-any.whl.metadata (2.9 kB) +Collecting referencing>=0.28.4 (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached referencing-0.37.0-py3-none-any.whl.metadata (2.8 kB) +Collecting rpds-py>=0.25.0 (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.1 kB) +Collecting msgpack>=0.5.6 (from mashumaro[msgpack]<3.15,>=3.9->dbt-adapters==1.22.9) + Using cached msgpack-1.1.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (8.1 kB) +Collecting six>=1.5 (from python-dateutil<3.0,>=2.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached six-1.17.0-py2.py3-none-any.whl.metadata (1.7 kB) +Collecting charset_normalizer<4,>=2 (from requests<3.0.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (40 kB) +Collecting idna<4,>=2.5 (from requests<3.0.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached idna-3.11-py3-none-any.whl.metadata (8.4 kB) +Collecting urllib3<3,>=1.26 (from requests<3.0.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached urllib3-2.6.3-py3-none-any.whl.metadata (6.9 kB) +Collecting certifi>=2023.5.7 (from requests<3.0.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached certifi-2026.2.25-py3-none-any.whl.metadata (2.5 kB) +Collecting text-unidecode>=1.3 (from python-slugify>=1.2.1->agate<2.0,>=1.0->dbt-adapters==1.22.9) + Using cached text_unidecode-1.3-py2.py3-none-any.whl.metadata (2.4 kB) +Using cached dbt_common-1.37.3-py3-none-any.whl (87 kB) +Using cached agate-1.9.1-py2.py3-none-any.whl (95 kB) +Using cached colorama-0.4.6-py2.py3-none-any.whl (25 kB) +Using cached dbt_protos-1.0.443-py3-none-any.whl (186 kB) +Using cached deepdiff-8.6.2-py3-none-any.whl (91 kB) +Using cached isodate-0.7.2-py3-none-any.whl (22 kB) +Using cached jinja2-3.1.6-py3-none-any.whl (134 kB) +Using cached jsonschema-4.26.0-py3-none-any.whl (90 kB) +Using cached mashumaro-3.14-py3-none-any.whl (92 kB) +Using cached orderly_set-5.5.0-py3-none-any.whl (13 kB) +Using cached pathspec-0.12.1-py3-none-any.whl (31 kB) +Using cached protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl (323 kB) +Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) +Using cached requests-2.33.1-py3-none-any.whl (64 kB) +Using cached charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (207 kB) +Using cached idna-3.11-py3-none-any.whl (71 kB) +Using cached typing_extensions-4.15.0-py3-none-any.whl (44 kB) +Using cached urllib3-2.6.3-py3-none-any.whl (131 kB) +Using cached attrs-26.1.0-py3-none-any.whl (67 kB) +Using cached babel-2.18.0-py3-none-any.whl (10.2 MB) +Using cached certifi-2026.2.25-py3-none-any.whl (153 kB) +Using cached jsonschema_specifications-2025.9.1-py3-none-any.whl (18 kB) +Using cached leather-0.4.1-py3-none-any.whl (30 kB) +Using cached markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (22 kB) +Using cached msgpack-1.1.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (427 kB) +Using cached parsedatetime-2.6-py3-none-any.whl (42 kB) +Using cached python_slugify-8.0.4-py2.py3-none-any.whl (10 kB) +Using cached pytimeparse-1.1.8-py2.py3-none-any.whl (10.0 kB) +Using cached pytz-2026.1.post1-py2.py3-none-any.whl (510 kB) +Using cached referencing-0.37.0-py3-none-any.whl (26 kB) +Using cached rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (394 kB) +Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) +Using cached text_unidecode-1.3-py2.py3-none-any.whl (78 kB) +Building wheels for collected packages: dbt-adapters + Building editable for dbt-adapters (pyproject.toml): started + Building editable for dbt-adapters (pyproject.toml): finished with status 'done' + Created wheel for dbt-adapters: filename=dbt_adapters-1.22.9-py3-none-any.whl size=6737 sha256=4ba22a9f5c88aaa0f14ab0dc54869c1235726737a0fdef0884593f8f8b0fcf8b + Stored in directory: /tmp/pip-ephem-wheel-cache-uz474tpw/wheels/9f/29/3a/fd477b8fd9e53b7ef60b1d83670c4eb11250b0cd70e60d2dcd +Successfully built dbt-adapters +Installing collected packages: text-unidecode, pytz, pytimeparse, parsedatetime, leather, urllib3, typing-extensions, six, rpds-py, python-slugify, protobuf, pathspec, orderly-set, msgpack, MarkupSafe, isodate, idna, colorama, charset_normalizer, certifi, Babel, attrs, requests, referencing, python-dateutil, mashumaro, jinja2, deepdiff, dbt-protos, agate, jsonschema-specifications, jsonschema, dbt-common, dbt-adapters + +Successfully installed Babel-2.18.0 MarkupSafe-3.0.3 agate-1.9.1 attrs-26.1.0 certifi-2026.2.25 charset_normalizer-3.4.6 colorama-0.4.6 dbt-adapters-1.22.9 dbt-common-1.37.3 dbt-protos-1.0.443 deepdiff-8.6.2 idna-3.11 isodate-0.7.2 jinja2-3.1.6 jsonschema-4.26.0 jsonschema-specifications-2025.9.1 leather-0.4.1 mashumaro-3.14 msgpack-1.1.2 orderly-set-5.5.0 parsedatetime-2.6 pathspec-0.12.1 protobuf-6.33.6 python-dateutil-2.9.0.post0 python-slugify-8.0.4 pytimeparse-1.1.8 pytz-2026.1.post1 referencing-0.37.0 requests-2.33.1 rpds-py-0.30.0 six-1.17.0 text-unidecode-1.3 typing-extensions-4.15.0 urllib3-2.6.3 +Obtaining file:///dbt-tests-adapter + Installing build dependencies: started + Installing build dependencies: finished with status 'done' + Checking if build backend supports build_editable: started + Checking if build backend supports build_editable: finished with status 'done' + Getting requirements to build editable: started + Getting requirements to build editable: finished with status 'done' + Installing backend dependencies: started + Installing backend dependencies: finished with status 'done' + Preparing editable metadata (pyproject.toml): started + Preparing editable metadata (pyproject.toml): finished with status 'done' +Requirement already satisfied: dbt-adapters<2.0,>=1.14.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-tests-adapter==1.19.7) (1.22.9) +Requirement already satisfied: dbt-common<2.0,>=1.34.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-tests-adapter==1.19.7) (1.37.3) +Collecting dbt-core>=1.8.0a1 (from dbt-tests-adapter==1.19.7) + Using cached dbt_core-1.11.7-py3-none-any.whl.metadata (4.4 kB) +Collecting freezegun (from dbt-tests-adapter==1.19.7) + Using cached freezegun-1.5.5-py3-none-any.whl.metadata (13 kB) +Collecting pyyaml (from dbt-tests-adapter==1.19.7) + Using cached pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (2.4 kB) +Requirement already satisfied: agate<2.0,>=1.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (1.9.1) +Requirement already satisfied: dbt-protos<2.0,>=1.0.291 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (1.0.443) +Requirement already satisfied: mashumaro<3.15,>=3.9 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from mashumaro[msgpack]<3.15,>=3.9->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (3.14) +Requirement already satisfied: protobuf<7.0,>=6.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (6.33.6) +Requirement already satisfied: pytz>=2015.7 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (2026.1.post1) +Requirement already satisfied: typing-extensions<5.0,>=4.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (4.15.0) +Requirement already satisfied: Babel>=2.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from agate<2.0,>=1.0->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (2.18.0) +Requirement already satisfied: isodate>=0.5.4 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from agate<2.0,>=1.0->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (0.7.2) +Requirement already satisfied: leather>=0.3.2 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from agate<2.0,>=1.0->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (0.4.1) +Requirement already satisfied: parsedatetime!=2.5,>=2.1 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from agate<2.0,>=1.0->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (2.6) +Requirement already satisfied: python-slugify>=1.2.1 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from agate<2.0,>=1.0->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (8.0.4) +Requirement already satisfied: pytimeparse>=1.1.5 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from agate<2.0,>=1.0->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (1.1.8) +Requirement already satisfied: colorama<0.5,>=0.3.9 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (0.4.6) +Requirement already satisfied: deepdiff<9.0,>=7.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (8.6.2) +Requirement already satisfied: jinja2<4,>=3.1.3 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (3.1.6) +Requirement already satisfied: jsonschema<5.0,>=4.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (4.26.0) +Requirement already satisfied: pathspec<0.13,>=0.9 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (0.12.1) +Requirement already satisfied: python-dateutil<3.0,>=2.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (2.9.0.post0) +Requirement already satisfied: requests<3.0.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (2.33.1) +Requirement already satisfied: orderly-set<6,>=5.4.1 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from deepdiff<9.0,>=7.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (5.5.0) +Requirement already satisfied: MarkupSafe>=2.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from jinja2<4,>=3.1.3->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (3.0.3) +Requirement already satisfied: attrs>=22.2.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (26.1.0) +Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (2025.9.1) +Requirement already satisfied: referencing>=0.28.4 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (0.37.0) +Requirement already satisfied: rpds-py>=0.25.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (0.30.0) +Requirement already satisfied: msgpack>=0.5.6 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from mashumaro[msgpack]<3.15,>=3.9->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (1.1.2) +Requirement already satisfied: six>=1.5 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from python-dateutil<3.0,>=2.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (1.17.0) +Requirement already satisfied: charset_normalizer<4,>=2 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from requests<3.0.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (3.4.6) +Requirement already satisfied: idna<4,>=2.5 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from requests<3.0.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (3.11) +Requirement already satisfied: urllib3<3,>=1.26 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from requests<3.0.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (2.6.3) +Requirement already satisfied: certifi>=2023.5.7 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from requests<3.0.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (2026.2.25) +Collecting click<9.0,>=8.3.0 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached click-8.3.1-py3-none-any.whl.metadata (2.6 kB) +Collecting daff>=1.3.46 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached daff-1.4.2-py3-none-any.whl.metadata (10 kB) +Collecting dbt-extractor<=0.6,>=0.5.0 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.6 kB) +Collecting dbt-semantic-interfaces<0.10,>=0.9.0 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached dbt_semantic_interfaces-0.9.0-py3-none-any.whl.metadata (2.6 kB) +Collecting networkx<4.0,>=2.3 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached networkx-3.6.1-py3-none-any.whl.metadata (6.8 kB) +Collecting packaging>20.9 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached packaging-26.0-py3-none-any.whl.metadata (3.3 kB) +Collecting pydantic<3 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached pydantic-2.12.5-py3-none-any.whl.metadata (90 kB) +Collecting snowplow-tracker<2.0,>=1.0.2 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached snowplow_tracker-1.1.0-py3-none-any.whl.metadata (5.7 kB) +Collecting sqlparse<0.5.5,>=0.5.0 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached sqlparse-0.5.4-py3-none-any.whl.metadata (4.7 kB) +Collecting importlib-metadata<9,>=6.0 (from dbt-semantic-interfaces<0.10,>=0.9.0->dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached importlib_metadata-8.9.0-py3-none-any.whl.metadata (4.5 kB) +Collecting more-itertools<11.0,>=8.0 (from dbt-semantic-interfaces<0.10,>=0.9.0->dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached more_itertools-10.8.0-py3-none-any.whl.metadata (39 kB) +Collecting zipp>=3.20 (from importlib-metadata<9,>=6.0->dbt-semantic-interfaces<0.10,>=0.9.0->dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached zipp-3.23.0-py3-none-any.whl.metadata (3.6 kB) +Collecting annotated-types>=0.6.0 (from pydantic<3->dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached annotated_types-0.7.0-py3-none-any.whl.metadata (15 kB) +Collecting pydantic-core==2.41.5 (from pydantic<3->dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.3 kB) +Collecting typing-inspection>=0.4.2 (from pydantic<3->dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached typing_inspection-0.4.2-py3-none-any.whl.metadata (2.6 kB) +Requirement already satisfied: text-unidecode>=1.3 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from python-slugify>=1.2.1->agate<2.0,>=1.0->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (1.3) +Using cached dbt_core-1.11.7-py3-none-any.whl (1.0 MB) +Using cached click-8.3.1-py3-none-any.whl (108 kB) +Using cached dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (442 kB) +Using cached dbt_semantic_interfaces-0.9.0-py3-none-any.whl (147 kB) +Using cached importlib_metadata-8.9.0-py3-none-any.whl (27 kB) +Using cached more_itertools-10.8.0-py3-none-any.whl (69 kB) +Using cached networkx-3.6.1-py3-none-any.whl (2.1 MB) +Using cached pydantic-2.12.5-py3-none-any.whl (463 kB) +Using cached pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB) +Using cached pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (807 kB) +Using cached snowplow_tracker-1.1.0-py3-none-any.whl (44 kB) +Using cached sqlparse-0.5.4-py3-none-any.whl (45 kB) +Using cached annotated_types-0.7.0-py3-none-any.whl (13 kB) +Using cached daff-1.4.2-py3-none-any.whl (144 kB) +Using cached packaging-26.0-py3-none-any.whl (74 kB) +Using cached typing_inspection-0.4.2-py3-none-any.whl (14 kB) +Using cached zipp-3.23.0-py3-none-any.whl (10 kB) +Using cached freezegun-1.5.5-py3-none-any.whl (19 kB) +Building wheels for collected packages: dbt-tests-adapter + Building editable for dbt-tests-adapter (pyproject.toml): started + Building editable for dbt-tests-adapter (pyproject.toml): finished with status 'done' + Created wheel for dbt-tests-adapter: filename=dbt_tests_adapter-1.19.7-py3-none-any.whl size=7285 sha256=5c68c5d67059edf6c3b535b7f98e0d2eac916db602b17a0b4d72bf4f8adc4223 + Stored in directory: /tmp/pip-ephem-wheel-cache-n61hgt2s/wheels/ba/2c/73/c6e0aedccba1b655bc8415507b8fc68862d843de1da8f54445 +Successfully built dbt-tests-adapter +Installing collected packages: daff, zipp, typing-inspection, sqlparse, pyyaml, pydantic-core, packaging, networkx, more-itertools, dbt-extractor, click, annotated-types, snowplow-tracker, pydantic, importlib-metadata, freezegun, dbt-semantic-interfaces, dbt-core, dbt-tests-adapter + +Successfully installed annotated-types-0.7.0 click-8.3.1 daff-1.4.2 dbt-core-1.11.7 dbt-extractor-0.6.0 dbt-semantic-interfaces-0.9.0 dbt-tests-adapter-1.19.7 freezegun-1.5.5 importlib-metadata-8.9.0 more-itertools-10.8.0 networkx-3.6.1 packaging-26.0 pydantic-2.12.5 pydantic-core-2.41.5 pyyaml-6.0.3 snowplow-tracker-1.1.0 sqlparse-0.5.4 typing-inspection-0.4.2 zipp-3.23.0 +============================= test session starts ============================== +platform linux -- Python 3.12.12, pytest-7.4.4, pluggy-1.6.0 -- /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/bin/python +cachedir: .pytest_cache +rootdir: /src +configfile: pyproject.toml +plugins: xdist-3.8.0, logbook-1.2.0, csv-3.0.0, dotenv-0.5.2, mock-3.15.1 +created: 18/18 workers +18 workers [142 items] + +scheduling tests via LoadScheduling + +tests/functional/adapter/test_basic.py::TestSingularTestsEphemeralSpark::test_singular_tests_ephemeral +tests/functional/adapter/test_basic.py::TestSnapshotTimestampSpark::test_snapshot_timestamp +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_order +tests/functional/adapter/test_basic.py::TestGenericTestsSpark::test_generic_tests +tests/functional/adapter/test_basic.py::TestSimpleMaterializationsSpark::test_base +tests/functional/adapter/test_basic.py::TestEphemeralSpark::test_ephemeral +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsDdlEnforcement::test__constraints_ddl +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_order +[gw13] [ 0%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_names +[gw13] [ 1%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_names +[gw10] [ 2%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_correct_column_data_types +[gw10] [ 2%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_correct_column_data_types +[gw12] [ 3%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_correct_column_data_types +[gw12] [ 4%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_correct_column_data_types +[gw8] [ 4%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_correct_column_data_types +[gw8] [ 5%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_correct_column_data_types +[gw14] [ 6%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_correct_column_data_types +[gw14] [ 7%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_correct_column_data_types +[gw1] [ 7%] SKIPPED tests/functional/adapter/test_basic.py::TestSingularTestsEphemeralSpark::test_singular_tests_ephemeral +[gw16] [ 8%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_correct_column_data_types +[gw16] [ 9%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_correct_column_data_types +[gw9] [ 9%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_names +[gw9] [ 10%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_names +[gw5] [ 11%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_names +[gw5] [ 11%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_names +[gw15] [ 12%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_names +[gw15] [ 13%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_names +[gw17] [ 14%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsDdlEnforcement::test__constraints_ddl +[gw11] [ 14%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_names +[gw11] [ 15%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_names +[gw6] [ 16%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_correct_column_data_types +[gw6] [ 16%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_correct_column_data_types +[gw7] [ 17%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_names +[gw7] [ 18%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_names +[gw4] [ 19%] SKIPPED tests/functional/adapter/test_basic.py::TestSnapshotTimestampSpark::test_snapshot_timestamp +tests/functional/adapter/test_grants.py::TestSeedGrantsSpark::test_seed_grants +tests/functional/adapter/test_basic.py::TestEmptySpark::test_empty +tests/functional/adapter/test_get_columns_in_relation.py::TestColumnsInRelation::test_get_columns_in_relation +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsRollback::test__constraints_enforcement_rollback +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsRollback::test__constraints_enforcement_rollback +tests/functional/adapter/test_constraints.py::TestSparkModelConstraintsRuntimeEnforcement::test__model_constraints_ddl +tests/functional/adapter/test_grants.py::TestModelGrantsSpark::test_view_table_grants +tests/functional/adapter/test_grants.py::TestIncrementalGrantsSpark::test_incremental_grants +tests/functional/adapter/test_grants.py::TestSnapshotGrantsSpark::test_snapshot_grants +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsDdlEnforcement::test__constraints_ddl +tests/functional/adapter/test_grants.py::TestInvalidGrantsSpark::test_invalid_grants +tests/functional/adapter/test_python_model.py::TestPythonModelSpark::test_singular_tests +tests/functional/adapter/test_python_model.py::TestPySpark::test_different_dataframes +[gw5] [ 19%] SKIPPED tests/functional/adapter/test_grants.py::TestSeedGrantsSpark::test_seed_grants +tests/functional/adapter/test_constraints.py::TestSparkConstraintQuotedColumn::test__constraints_ddl +[gw12] [ 20%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsRollback::test__constraints_enforcement_rollback +[gw10] [ 21%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsRollback::test__constraints_enforcement_rollback +[gw8] [ 21%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkModelConstraintsRuntimeEnforcement::test__model_constraints_ddl +[gw16] [ 22%] SKIPPED tests/functional/adapter/test_grants.py::TestModelGrantsSpark::test_view_table_grants +[gw9] [ 23%] SKIPPED tests/functional/adapter/test_grants.py::TestIncrementalGrantsSpark::test_incremental_grants +[gw15] [ 23%] SKIPPED tests/functional/adapter/test_grants.py::TestSnapshotGrantsSpark::test_snapshot_grants +[gw17] [ 24%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsDdlEnforcement::test__constraints_ddl +[gw11] [ 25%] SKIPPED tests/functional/adapter/test_grants.py::TestInvalidGrantsSpark::test_invalid_grants +[gw6] [ 26%] SKIPPED tests/functional/adapter/test_python_model.py::TestPythonModelSpark::test_singular_tests +[gw7] [ 26%] SKIPPED tests/functional/adapter/test_python_model.py::TestPySpark::test_different_dataframes +[gw13] [ 27%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkConstraintQuotedColumn::test__constraints_ddl +[gw1] [ 28%] PASSED tests/functional/adapter/test_basic.py::TestEmptySpark::test_empty +tests/functional/adapter/test_python_model.py::TestPythonIncrementalModelSpark::test_incremental +tests/functional/adapter/test_basic.py::TestBaseAdapterMethod::test_adapter_methods +tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsProjectLevelView::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsProjectLevelOff::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsInteractions::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/test_store_test_failures.py::TestSparkStoreTestFailuresWithDelta::test_store_and_assert_failure_with_delta +tests/functional/adapter/test_store_test_failures.py::TestSparkStoreTestFailures::test_store_and_assert +tests/functional/adapter/test_simple_seed.py::TestBigQueryEmptySeed::test_empty_seeds +tests/functional/adapter/dbt_clone/test_dbt_clone.py::TestSparkClonePossible::test_can_clone_true +[gw5] [ 28%] SKIPPED tests/functional/adapter/test_python_model.py::TestPythonIncrementalModelSpark::test_incremental +tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsProjectLevelEphemeral::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsExceptions::test_tests_run_unsuccessfully_and_raise_appropriate_exception +tests/functional/adapter/test_python_model.py::TestChangingSchemaSpark::test_changing_schema_with_log_validation +tests/functional/adapter/dbt_clone/test_dbt_clone.py::TestSparkClonePossible::test_clone_no_state +[gw8] [ 29%] SKIPPED tests/functional/adapter/test_store_test_failures.py::TestSparkStoreTestFailuresWithDelta::test_store_and_assert_failure_with_delta +[gw7] [ 30%] SKIPPED tests/functional/adapter/dbt_clone/test_dbt_clone.py::TestSparkClonePossible::test_can_clone_true +tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsGeneric::test_tests_run_successfully_and_are_stored_as_expected +[gw1] [ 30%] SKIPPED tests/functional/adapter/test_python_model.py::TestChangingSchemaSpark::test_changing_schema_with_log_validation +[gw13] [ 31%] SKIPPED tests/functional/adapter/dbt_clone/test_dbt_clone.py::TestSparkClonePossible::test_clone_no_state +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args0-5] +tests/functional/adapter/empty/test_empty.py::TestSparkEmpty::test_run_with_empty +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestDeltaOnSchemaChange::test_run_incremental_append_new_columns +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestInsertOverwriteOnSchemaChange::test_run_incremental_fail_on_schema_change +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_ignore +[gw13] [ 32%] SKIPPED tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestDeltaOnSchemaChange::test_run_incremental_append_new_columns +tests/functional/adapter/incremental/test_incremental_predicates.py::TestPredicatesMergeSpark::test__incremental_predicates +[gw13] [ 33%] SKIPPED tests/functional/adapter/incremental/test_incremental_predicates.py::TestPredicatesMergeSpark::test__incremental_predicates +[gw12] [ 33%] PASSED tests/functional/adapter/test_simple_seed.py::TestBigQueryEmptySeed::test_empty_seeds +tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py::TestMergeExcludeColumns::test__merge_exclude_columns +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__one_unique_key +[gw12] [ 34%] SKIPPED tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py::TestMergeExcludeColumns::test__merge_exclude_columns +[gw13] [ 35%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__one_unique_key +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__no_unique_keys +[gw13] [ 35%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__no_unique_keys +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__empty_str_unique_key +[gw13] [ 36%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__empty_str_unique_key +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__unary_unique_key_list +[gw13] [ 37%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__unary_unique_key_list +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__duplicated_unary_unique_key_list +[gw13] [ 38%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__duplicated_unary_unique_key_list +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__trinary_unique_key_list +[gw13] [ 38%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__trinary_unique_key_list +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__trinary_unique_key_list_no_update +[gw13] [ 39%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__trinary_unique_key_list_no_update +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__bad_unique_key_list +[gw12] [ 40%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__bad_unique_key_list +tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestInsertOverwrite::test_insert_overwrite +tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestDefaultAppend::test_default_append +[gw14] [ 40%] PASSED tests/functional/adapter/test_get_columns_in_relation.py::TestColumnsInRelation::test_get_columns_in_relation +tests/functional/adapter/test_query_timeout.py::TestQueryTimeout::test_query_timeout_exceeded +[gw5] [ 41%] PASSED tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args0-5] +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChangeSpecialChars::test_incremental_append_new_columns_with_special_characters +[gw6] [ 42%] PASSED tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsExceptions::test_tests_run_unsuccessfully_and_raise_appropriate_exception +[gw4] [ 42%] PASSED tests/functional/adapter/test_basic.py::TestBaseAdapterMethod::test_adapter_methods +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestInsertOverwriteOnSchemaChange::test_run_incremental_ignore +[gw2] [ 43%] PASSED tests/functional/adapter/test_basic.py::TestEphemeralSpark::test_ephemeral +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args1-3] +tests/functional/adapter/test_basic.py::TestIncrementalSpark::test_incremental +[gw3] [ 44%] PASSED tests/functional/adapter/test_basic.py::TestGenericTestsSpark::test_generic_tests +tests/functional/adapter/test_basic.py::TestSnapshotCheckColsSpark::test_snapshot_check_cols +[gw3] [ 45%] SKIPPED tests/functional/adapter/test_basic.py::TestSnapshotCheckColsSpark::test_snapshot_check_cols +[gw1] [ 45%] PASSED tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestInsertOverwriteOnSchemaChange::test_run_incremental_fail_on_schema_change +tests/functional/adapter/seed_column_types/test_seed_column_types.py::TestSeedColumnTypesCast::test_column_seed_type +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__bad_unique_key +[gw1] [ 46%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__bad_unique_key +[gw11] [ 47%] PASSED tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsProjectLevelEphemeral::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestCaseInsensitivity::test_case_insensitivity +[gw8] [ 47%] PASSED tests/functional/adapter/empty/test_empty.py::TestSparkEmpty::test_run_with_empty +[gw14] [ 48%] PASSED tests/functional/adapter/test_query_timeout.py::TestQueryTimeout::test_query_timeout_exceeded +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_fail_on_schema_change +tests/functional/adapter/incremental/test_incremental_predicates.py::TestIncrementalPredicatesMergeSpark::test__incremental_predicates +[gw17] [ 49%] PASSED tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsGeneric::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestBadStrategies::test_bad_strategies +[gw8] [ 50%] SKIPPED tests/functional/adapter/incremental/test_incremental_predicates.py::TestIncrementalPredicatesMergeSpark::test__incremental_predicates +[gw4] [ 50%] PASSED tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args1-3] +tests/functional/adapter/utils/test_data_types.py::TestTypeFloat::test_check_types_assert_match +[gw9] [ 51%] PASSED tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsProjectLevelOff::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestDeltaOnSchemaChange::test_run_incremental_sync_all_columns +tests/functional/adapter/persist_docs/test_persist_docs.py::TestPersistDocsDeltaView::test_delta_comments +[gw17] [ 52%] SKIPPED tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestDeltaOnSchemaChange::test_run_incremental_sync_all_columns +[gw14] [ 52%] PASSED tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestBadStrategies::test_bad_strategies +[gw4] [ 53%] SKIPPED tests/functional/adapter/persist_docs/test_persist_docs.py::TestPersistDocsDeltaView::test_delta_comments +tests/functional/adapter/utils/test_data_types.py::TestTypeString::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeInt::test_check_types_assert_match +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowSqlHeader::test_sql_header +tests/functional/adapter/utils/test_data_types.py::TestTypeTimestamp::test_check_types_assert_match +[gw3] [ 54%] PASSED tests/functional/adapter/seed_column_types/test_seed_column_types.py::TestSeedColumnTypesCast::test_column_seed_type +[gw15] [ 54%] PASSED tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsProjectLevelView::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestingTypes::test_unit_test_data_type +[gw9] [ 55%] PASSED tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowSqlHeader::test_sql_header +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args2-7] +tests/functional/adapter/utils/test_utils.py::TestAnyValue::test_build_assert_equal +[gw1] [ 56%] PASSED tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestCaseInsensitivity::test_case_insensitivity +tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestInvalidInput::test_invalid_input +[gw7] [ 57%] PASSED tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_ignore +[gw13] [ 57%] PASSED tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestDefaultAppend::test_default_append +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__empty_unique_key_list +tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestDeltaStrategies::test_delta_strategies_overwrite +[gw13] [ 58%] SKIPPED tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestDeltaStrategies::test_delta_strategies_overwrite +tests/functional/adapter/utils/test_utils.py::TestBoolOr::test_build_assert_equal +[gw7] [ 59%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__empty_unique_key_list +tests/functional/adapter/utils/test_utils.py::TestArrayConstruct::test_expected_actual +[gw8] [ 59%] PASSED tests/functional/adapter/utils/test_data_types.py::TestTypeFloat::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeNumeric::test_check_types_assert_match +[gw17] [ 60%] PASSED tests/functional/adapter/utils/test_data_types.py::TestTypeString::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeBoolean::test_check_types_assert_match +[gw4] [ 61%] PASSED tests/functional/adapter/utils/test_data_types.py::TestTypeTimestamp::test_check_types_assert_match +[gw14] [ 61%] PASSED tests/functional/adapter/utils/test_data_types.py::TestTypeInt::test_check_types_assert_match +tests/functional/adapter/utils/test_utils.py::TestArrayAppend::test_build_assert_equal +[gw15] [ 62%] PASSED tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args2-7] +tests/functional/adapter/utils/test_timestamps.py::TestCurrentTimestampSpark::test_current_timestamps +tests/functional/adapter/utils/test_utils.py::TestArrayConcat::test_build_assert_equal +[gw11] [ 63%] PASSED tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_fail_on_schema_change +[gw1] [ 64%] PASSED tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestInvalidInput::test_invalid_input +tests/functional/adapter/utils/test_data_types.py::TestTypeBigInt::test_check_types_assert_match +[gw9] [ 64%] PASSED tests/functional/adapter/utils/test_utils.py::TestAnyValue::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestArrayConstruct::test_build_assert_equal +[gw12] [ 65%] PASSED tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestInsertOverwrite::test_insert_overwrite +tests/functional/adapter/utils/test_utils.py::TestArrayConcat::test_expected_actual +[gw13] [ 66%] PASSED tests/functional/adapter/utils/test_utils.py::TestBoolOr::test_build_assert_equal +[gw7] [ 66%] PASSED tests/functional/adapter/utils/test_utils.py::TestArrayConstruct::test_expected_actual +tests/functional/adapter/utils/test_utils.py::TestCast::test_build_assert_equal +[gw4] [ 67%] PASSED tests/functional/adapter/utils/test_utils.py::TestArrayAppend::test_build_assert_equal +tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestDeltaStrategies::test_delta_strategies +tests/functional/adapter/utils/test_utils.py::TestCastBoolToText::test_build_assert_equal +[gw14] [ 68%] PASSED tests/functional/adapter/utils/test_timestamps.py::TestCurrentTimestampSpark::test_current_timestamps +[gw12] [ 69%] SKIPPED tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestDeltaStrategies::test_delta_strategies +tests/functional/adapter/utils/test_utils.py::TestCurrentTimestamp::test_current_timestamp_type +tests/functional/adapter/utils/test_utils.py::TestEscapeSingleQuotes::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestDate::test_build_assert_equal +[gw15] [ 69%] PASSED tests/functional/adapter/utils/test_utils.py::TestArrayConcat::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestDateAdd::test_build_assert_equal +[gw2] [ 70%] PASSED tests/functional/adapter/test_basic.py::TestIncrementalSpark::test_incremental +[gw1] [ 71%] PASSED tests/functional/adapter/utils/test_utils.py::TestArrayConstruct::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestDateTrunc::test_build_assert_equal +tests/functional/adapter/persist_docs/test_persist_docs.py::TestPersistDocsMissingColumn::test_missing_column +[gw17] [ 71%] PASSED tests/functional/adapter/utils/test_data_types.py::TestTypeBoolean::test_check_types_assert_match +[gw2] [ 72%] SKIPPED tests/functional/adapter/persist_docs/test_persist_docs.py::TestPersistDocsMissingColumn::test_missing_column +[gw7] [ 73%] PASSED tests/functional/adapter/utils/test_utils.py::TestCastBoolToText::test_build_assert_equal +[gw8] [ 73%] PASSED tests/functional/adapter/utils/test_data_types.py::TestTypeNumeric::test_check_types_assert_match +[gw4] [ 74%] PASSED tests/functional/adapter/utils/test_utils.py::TestCurrentTimestamp::test_current_timestamp_type +tests/functional/adapter/utils/test_utils.py::TestCurrentTimestamp::test_current_timestamp_matches_utc +tests/functional/adapter/utils/test_utils.py::TestExcept::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestPosition::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestConcat::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestHash::test_build_assert_equal +[gw14] [ 75%] PASSED tests/functional/adapter/utils/test_utils.py::TestDate::test_build_assert_equal +[gw11] [ 76%] PASSED tests/functional/adapter/utils/test_data_types.py::TestTypeBigInt::test_check_types_assert_match +tests/functional/adapter/utils/test_utils.py::TestLastDay::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestDateDiff::test_build_assert_equal +[gw13] [ 76%] PASSED tests/functional/adapter/utils/test_utils.py::TestCast::test_build_assert_equal +[gw5] [ 77%] PASSED tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChangeSpecialChars::test_incremental_append_new_columns_with_special_characters +[gw11] [ 78%] SKIPPED tests/functional/adapter/utils/test_utils.py::TestDateDiff::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestEquals::test_equal_values +[gw12] [ 78%] PASSED tests/functional/adapter/utils/test_utils.py::TestEscapeSingleQuotes::test_build_assert_equal +tests/functional/adapter/incremental_strategies/test_microbatch.py::TestMicrobatch::test_run_with_event_time +[gw17] [ 79%] PASSED tests/functional/adapter/utils/test_utils.py::TestCurrentTimestamp::test_current_timestamp_matches_utc +tests/functional/adapter/utils/test_utils.py::TestIntersect::test_build_assert_equal +[gw15] [ 80%] PASSED tests/functional/adapter/utils/test_utils.py::TestDateAdd::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestReplace::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestLength::test_build_assert_equal +[gw9] [ 80%] PASSED tests/functional/adapter/utils/test_utils.py::TestArrayConcat::test_expected_actual +[gw1] [ 81%] PASSED tests/functional/adapter/utils/test_utils.py::TestDateTrunc::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestEquals::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestListagg::test_build_assert_equal +[gw16] [ 82%] PASSED tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsInteractions::test_tests_run_successfully_and_are_stored_as_expected +[gw8] [ 83%] PASSED tests/functional/adapter/utils/test_utils.py::TestConcat::test_build_assert_equal +[gw4] [ 83%] PASSED tests/functional/adapter/utils/test_utils.py::TestHash::test_build_assert_equal +[gw2] [ 84%] PASSED tests/functional/adapter/utils/test_utils.py::TestPosition::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestStringLiteral::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestSplitPart::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestSafeCast::test_build_assert_equal +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowDoesNotHandleDoubleLimit::test_double_limit_throws_syntax_error +[gw16] [ 85%] SKIPPED tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowDoesNotHandleDoubleLimit::test_double_limit_throws_syntax_error +[gw17] [ 85%] PASSED tests/functional/adapter/utils/test_utils.py::TestReplace::test_build_assert_equal +[gw0] [ 86%] PASSED tests/functional/adapter/test_basic.py::TestSimpleMaterializationsSpark::test_base +[gw15] [ 87%] PASSED tests/functional/adapter/utils/test_utils.py::TestLength::test_build_assert_equal +[gw4] [ 88%] PASSED tests/functional/adapter/utils/test_utils.py::TestStringLiteral::test_build_assert_equal +tests/functional/adapter/test_basic.py::TestSingularTestsSpark::test_singular_tests +[gw14] [ 88%] PASSED tests/functional/adapter/utils/test_utils.py::TestLastDay::test_build_assert_equal +[gw9] [ 89%] PASSED tests/functional/adapter/utils/test_utils.py::TestEquals::test_build_assert_equal +[gw0] [ 90%] PASSED tests/functional/adapter/test_basic.py::TestSingularTestsSpark::test_singular_tests +[gw13] [ 90%] PASSED tests/functional/adapter/utils/test_utils.py::TestEquals::test_equal_values +[gw8] [ 91%] PASSED tests/functional/adapter/utils/test_utils.py::TestSplitPart::test_build_assert_equal +[gw2] [ 92%] PASSED tests/functional/adapter/utils/test_utils.py::TestSafeCast::test_build_assert_equal +[gw6] [ 92%] PASSED tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestInsertOverwriteOnSchemaChange::test_run_incremental_ignore +[gw1] [ 93%] PASSED tests/functional/adapter/utils/test_utils.py::TestListagg::test_build_assert_equal +tests/functional/adapter/persist_docs/test_persist_docs.py::TestPersistDocsDeltaTable::test_delta_comments +[gw6] [ 94%] SKIPPED tests/functional/adapter/persist_docs/test_persist_docs.py::TestPersistDocsDeltaTable::test_delta_comments +[gw12] [ 95%] PASSED tests/functional/adapter/utils/test_utils.py::TestIntersect::test_build_assert_equal +[gw7] [ 95%] PASSED tests/functional/adapter/utils/test_utils.py::TestExcept::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestRight::test_build_assert_equal +[gw3] [ 96%] PASSED tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestingTypes::test_unit_test_data_type +tests/functional/adapter/utils/test_utils.py::TestArrayAppend::test_expected_actual +[gw7] [ 97%] PASSED tests/functional/adapter/utils/test_utils.py::TestRight::test_build_assert_equal +[gw3] [ 97%] PASSED tests/functional/adapter/utils/test_utils.py::TestArrayAppend::test_expected_actual +[gw5] [ 98%] PASSED tests/functional/adapter/incremental_strategies/test_microbatch.py::TestMicrobatch::test_run_with_event_time +[gw10] [ 99%] PASSED tests/functional/adapter/test_store_test_failures.py::TestSparkStoreTestFailures::test_store_and_assert +tests/functional/adapter/empty/test_empty.py::TestSparkEmptyInlineSourceRef::test_run_with_empty +[gw10] [100%] PASSED tests/functional/adapter/empty/test_empty.py::TestSparkEmptyInlineSourceRef::test_run_with_empty + +================== 75 passed, 67 skipped in 85.47s (0:01:25) =================== + + +============================================================ +SUMMARY: tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowDoesNotHandleDoubleLimit::test_double_limit_throws_syntax_error +============================================================ + +============================================================ +SUMMARY: [gw16] [ 85%] SKIPPED tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowDoesNotHandleDoubleLimit::test_double_limit_throws_syntax_error +============================================================ + +============================================================ +SUMMARY: ================== 75 passed, 67 skipped in 85.47s (0:01:25) =================== +============================================================ diff --git a/dbt-spark/tests/functional/fixtures/profiles.py b/dbt-spark/tests/functional/fixtures/profiles.py index 900428e10f..c3b9b90882 100644 --- a/dbt-spark/tests/functional/fixtures/profiles.py +++ b/dbt-spark/tests/functional/fixtures/profiles.py @@ -96,6 +96,7 @@ def spark_session_target(): "type": "spark", "host": "localhost", "method": "session", + "server_side_parameters": {"spark.ui.enabled": "false"}, }