From 281dd144bc997685ce14de4f67a82192cc3c1b9b Mon Sep 17 00:00:00 2001 From: Kyle Pomykala Date: Sun, 29 Mar 2026 20:10:34 -0700 Subject: [PATCH 1/6] feat: add dbt-sail adapter for Sail (Spark-compatible compute engine) Thin wrapper around dbt-spark using the same pattern as dbt-redshift wrapping dbt-postgres. Connects to Sail via Spark Connect protocol with embedded (in-process) and remote connection modes. Co-Authored-By: Claude Opus 4.6 (1M context) --- dbt-sail/.gitignore | 8 + dbt-sail/README.md | 5 + dbt-sail/pyproject.toml | 53 +++++ dbt-sail/src/dbt/adapters/sail/__init__.py | 12 + dbt-sail/src/dbt/adapters/sail/__version__.py | 1 + dbt-sail/src/dbt/adapters/sail/connections.py | 123 ++++++++++ dbt-sail/src/dbt/adapters/sail/impl.py | 66 ++++++ dbt-sail/src/dbt/include/sail/__init__.py | 3 + dbt-sail/src/dbt/include/sail/dbt_project.yml | 5 + .../src/dbt/include/sail/macros/adapters.sql | 28 +++ dbt-sail/tests/__init__.py | 0 dbt-sail/tests/functional/__init__.py | 0 dbt-sail/tests/functional/adapter/__init__.py | 0 .../functional/adapter/dbt_clone/fixtures.py | 98 ++++++++ .../adapter/dbt_clone/test_dbt_clone.py | 77 ++++++ .../adapter/dbt_show/test_dbt_show.py | 19 ++ .../functional/adapter/empty/test_empty.py | 9 + .../test_incremental_merge_exclude_columns.py | 15 ++ .../test_incremental_on_schema_change.py | 45 ++++ .../test_incremental_predicates.py | 47 ++++ .../incremental/test_incremental_unique_id.py | 13 + .../incremental_strategies/fixtures.py | 222 ++++++++++++++++++ .../adapter/incremental_strategies/seeds.py | 27 +++ .../test_incremental_strategies.py | 108 +++++++++ .../incremental_strategies/test_microbatch.py | 17 ++ .../adapter/persist_docs/fixtures.py | 156 ++++++++++++ .../adapter/persist_docs/test_persist_docs.py | 156 ++++++++++++ .../adapter/seed_column_types/fixtures.py | 113 +++++++++ .../test_seed_column_types.py | 22 ++ .../tests/functional/adapter/test_basic.py | 72 ++++++ .../functional/adapter/test_constraints.py | 131 +++++++++++ .../adapter/test_get_columns_in_relation.py | 32 +++ .../tests/functional/adapter/test_grants.py | 60 +++++ .../functional/adapter/test_python_model.py | 18 ++ .../functional/adapter/test_simple_seed.py | 5 + .../adapter/test_store_test_failures.py | 21 ++ .../adapter/unit_testing/test_unit_testing.py | 34 +++ .../adapter/utils/fixture_listagg.py | 61 +++++ .../adapter/utils/test_data_types.py | 73 ++++++ .../adapter/utils/test_timestamps.py | 18 ++ .../functional/adapter/utils/test_utils.py | 162 +++++++++++++ dbt-sail/tests/functional/conftest.py | 15 ++ dbt-sail/tests/unit/__init__.py | 0 dbt-sail/tests/unit/conftest.py | 1 + dbt-sail/tests/unit/fixtures/__init__.py | 0 dbt-sail/tests/unit/fixtures/profiles.py | 55 +++++ dbt-sail/tests/unit/test_adapter.py | 145 ++++++++++++ dbt-sail/tests/unit/test_credentials.py | 77 ++++++ dbt-sail/tests/unit/utils.py | 13 + 49 files changed, 2441 insertions(+) create mode 100644 dbt-sail/.gitignore create mode 100644 dbt-sail/README.md create mode 100644 dbt-sail/pyproject.toml create mode 100644 dbt-sail/src/dbt/adapters/sail/__init__.py create mode 100644 dbt-sail/src/dbt/adapters/sail/__version__.py create mode 100644 dbt-sail/src/dbt/adapters/sail/connections.py create mode 100644 dbt-sail/src/dbt/adapters/sail/impl.py create mode 100644 dbt-sail/src/dbt/include/sail/__init__.py create mode 100644 dbt-sail/src/dbt/include/sail/dbt_project.yml create mode 100644 dbt-sail/src/dbt/include/sail/macros/adapters.sql create mode 100644 dbt-sail/tests/__init__.py create mode 100644 dbt-sail/tests/functional/__init__.py create mode 100644 dbt-sail/tests/functional/adapter/__init__.py create mode 100644 dbt-sail/tests/functional/adapter/dbt_clone/fixtures.py create mode 100644 dbt-sail/tests/functional/adapter/dbt_clone/test_dbt_clone.py create mode 100644 dbt-sail/tests/functional/adapter/dbt_show/test_dbt_show.py create mode 100644 dbt-sail/tests/functional/adapter/empty/test_empty.py create mode 100644 dbt-sail/tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py create mode 100644 dbt-sail/tests/functional/adapter/incremental/test_incremental_on_schema_change.py create mode 100644 dbt-sail/tests/functional/adapter/incremental/test_incremental_predicates.py create mode 100644 dbt-sail/tests/functional/adapter/incremental/test_incremental_unique_id.py create mode 100644 dbt-sail/tests/functional/adapter/incremental_strategies/fixtures.py create mode 100644 dbt-sail/tests/functional/adapter/incremental_strategies/seeds.py create mode 100644 dbt-sail/tests/functional/adapter/incremental_strategies/test_incremental_strategies.py create mode 100644 dbt-sail/tests/functional/adapter/incremental_strategies/test_microbatch.py create mode 100644 dbt-sail/tests/functional/adapter/persist_docs/fixtures.py create mode 100644 dbt-sail/tests/functional/adapter/persist_docs/test_persist_docs.py create mode 100644 dbt-sail/tests/functional/adapter/seed_column_types/fixtures.py create mode 100644 dbt-sail/tests/functional/adapter/seed_column_types/test_seed_column_types.py create mode 100644 dbt-sail/tests/functional/adapter/test_basic.py create mode 100644 dbt-sail/tests/functional/adapter/test_constraints.py create mode 100644 dbt-sail/tests/functional/adapter/test_get_columns_in_relation.py create mode 100644 dbt-sail/tests/functional/adapter/test_grants.py create mode 100644 dbt-sail/tests/functional/adapter/test_python_model.py create mode 100644 dbt-sail/tests/functional/adapter/test_simple_seed.py create mode 100644 dbt-sail/tests/functional/adapter/test_store_test_failures.py create mode 100644 dbt-sail/tests/functional/adapter/unit_testing/test_unit_testing.py create mode 100644 dbt-sail/tests/functional/adapter/utils/fixture_listagg.py create mode 100644 dbt-sail/tests/functional/adapter/utils/test_data_types.py create mode 100644 dbt-sail/tests/functional/adapter/utils/test_timestamps.py create mode 100644 dbt-sail/tests/functional/adapter/utils/test_utils.py create mode 100644 dbt-sail/tests/functional/conftest.py create mode 100644 dbt-sail/tests/unit/__init__.py create mode 100644 dbt-sail/tests/unit/conftest.py create mode 100644 dbt-sail/tests/unit/fixtures/__init__.py create mode 100644 dbt-sail/tests/unit/fixtures/profiles.py create mode 100644 dbt-sail/tests/unit/test_adapter.py create mode 100644 dbt-sail/tests/unit/test_credentials.py create mode 100644 dbt-sail/tests/unit/utils.py diff --git a/dbt-sail/.gitignore b/dbt-sail/.gitignore new file mode 100644 index 0000000000..6d1ed8221f --- /dev/null +++ b/dbt-sail/.gitignore @@ -0,0 +1,8 @@ +logs/ +.venv/ +__pycache__/ +*.pyc +*.egg-info/ +.pytest_cache/ +dist/ +build/ diff --git a/dbt-sail/README.md b/dbt-sail/README.md new file mode 100644 index 0000000000..f9ada01969 --- /dev/null +++ b/dbt-sail/README.md @@ -0,0 +1,5 @@ +# dbt-sail + +The [Sail](https://github.com/lakehq/sail) adapter plugin for [dbt](https://www.getdbt.com/). + +Sail is a drop-in replacement for Apache Spark. This adapter is a thin wrapper around `dbt-spark` that connects to Sail via the Spark Connect protocol. diff --git a/dbt-sail/pyproject.toml b/dbt-sail/pyproject.toml new file mode 100644 index 0000000000..cd08283e57 --- /dev/null +++ b/dbt-sail/pyproject.toml @@ -0,0 +1,53 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +dynamic = ["version"] +name = "dbt-sail" +description = "The Sail adapter plugin for dbt" +readme = "README.md" +keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "sail", "spark"] +requires-python = ">=3.10.0" +authors = [{ name = "LakeSail", email = "info@lakesail.com" }] +classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +dependencies = [ + "dbt-spark[session]>=1.8.0", + "pysail", +] + +[project.optional-dependencies] +dev = [ + "dbt-tests-adapter", + "pytest", + "pytest-xdist", +] + +[project.urls] +Homepage = "https://github.com/lakehq/sail" +Repository = "https://github.com/lakehq/sail.git" + +[tool.hatch.version] +path = "src/dbt/adapters/sail/__version__.py" +pattern = "version = \"(?P[^\"]+)\"" + +[tool.hatch.build.targets.wheel] +packages = ["src/dbt"] + +[tool.pytest.ini_options] +testpaths = ["tests/unit", "tests/functional"] +addopts = "-v --color=yes" +filterwarnings = [ + "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", + "ignore:unclosed file .*:ResourceWarning", +] diff --git a/dbt-sail/src/dbt/adapters/sail/__init__.py b/dbt-sail/src/dbt/adapters/sail/__init__.py new file mode 100644 index 0000000000..4004a82040 --- /dev/null +++ b/dbt-sail/src/dbt/adapters/sail/__init__.py @@ -0,0 +1,12 @@ +from dbt.adapters.base import AdapterPlugin + +from dbt.adapters.sail.connections import SailConnectionManager, SailCredentials # noqa: F401 +from dbt.adapters.sail.impl import SailAdapter +from dbt.include import sail + +Plugin = AdapterPlugin( + adapter=SailAdapter, # type: ignore + credentials=SailCredentials, + include_path=sail.PACKAGE_PATH, + dependencies=["spark"], +) diff --git a/dbt-sail/src/dbt/adapters/sail/__version__.py b/dbt-sail/src/dbt/adapters/sail/__version__.py new file mode 100644 index 0000000000..9f7a875c56 --- /dev/null +++ b/dbt-sail/src/dbt/adapters/sail/__version__.py @@ -0,0 +1 @@ +version = "0.1.0" diff --git a/dbt-sail/src/dbt/adapters/sail/connections.py b/dbt-sail/src/dbt/adapters/sail/connections.py new file mode 100644 index 0000000000..d740ec7714 --- /dev/null +++ b/dbt-sail/src/dbt/adapters/sail/connections.py @@ -0,0 +1,123 @@ +from dataclasses import dataclass, field +from typing import Any, Dict, Optional + +from dbt.adapters.contracts.connection import Connection, ConnectionState +from dbt.adapters.events.logging import AdapterLogger +from dbt.adapters.exceptions import FailedToConnectError +from dbt.adapters.spark.connections import SparkConnectionManager, SparkCredentials +from dbt_common.dataclass_schema import StrEnum +from dbt_common.exceptions import DbtConfigError + +logger = AdapterLogger("Sail") + + +class SailConnectionMethod(StrEnum): + EMBEDDED = "embedded" + REMOTE = "remote" + + +@dataclass +class SailCredentials(SparkCredentials): + mode: SailConnectionMethod = SailConnectionMethod.EMBEDDED # type: ignore + server_side_parameters: Dict[str, str] = field(default_factory=dict) + + @property + def type(self) -> str: + return "sail" + + @property + def unique_field(self) -> str: + return self.host or "embedded" + + def __post_init__(self) -> None: + # For embedded mode, host is not required — set defaults + if self.mode == SailConnectionMethod.EMBEDDED: + if self.host is None: + self.host = "127.0.0.1" + if self.method is None: + # Set method to session so SparkCredentials validation doesn't fail + from dbt.adapters.spark.connections import SparkConnectionMethod + + self.method = SparkConnectionMethod.SESSION + elif self.mode == SailConnectionMethod.REMOTE: + if self.host is None: + raise DbtConfigError("Must specify `host` for remote mode") + if self.method is None: + from dbt.adapters.spark.connections import SparkConnectionMethod + + self.method = SparkConnectionMethod.SESSION + + if self.schema is None: + raise DbtConfigError("Must specify `schema` in profile") + + # Spark treats database and schema as the same thing + if self.database is not None and self.database != self.schema: + raise DbtConfigError( + f" schema: {self.schema} \n" + f" database: {self.database} \n" + f"On Sail, database must be omitted or have the same value as schema." + ) + self.database = None + + +class SailConnectionManager(SparkConnectionManager): + TYPE = "sail" + + # Track the embedded server so it can be reused/stopped + _server = None + + @classmethod + def open(cls, connection: Connection) -> Connection: + if connection.state == ConnectionState.OPEN: + logger.debug("Connection is already open, skipping open.") + return connection + + creds: SailCredentials = connection.credentials # type: ignore + + try: + if creds.mode == SailConnectionMethod.EMBEDDED: + handle = cls._open_embedded(creds) + elif creds.mode == SailConnectionMethod.REMOTE: + handle = cls._open_remote(creds) + else: + raise DbtConfigError(f"invalid Sail connection mode: {creds.mode}") + except Exception as e: + logger.debug(f"Error opening connection: {e}") + connection.handle = None + connection.state = ConnectionState.FAIL + raise FailedToConnectError(f"failed to connect to Sail: {e}") from e + + connection.handle = handle + connection.state = ConnectionState.OPEN + return connection + + @classmethod + def _open_embedded(cls, creds: SailCredentials) -> Any: + """Start a SparkConnectServer in-process via pysail, then connect PySpark to it.""" + from pysail.spark import SparkConnectServer + from dbt.adapters.spark.session import SessionConnectionWrapper, Connection as SessionConnection + + if cls._server is None or not cls._server.running: + port = creds.port if creds.port != 443 else 0 + cls._server = SparkConnectServer(ip=creds.host or "127.0.0.1", port=port) + cls._server.start(background=True) + logger.debug(f"Started embedded Sail server at {cls._server.listening_address}") + + ip, port = cls._server.listening_address + remote_url = f"sc://{ip}:{port}" + + params = dict(creds.server_side_parameters) + params["spark.remote"] = remote_url + return SessionConnectionWrapper(SessionConnection(server_side_parameters=params)) + + @classmethod + def _open_remote(cls, creds: SailCredentials) -> Any: + """Connect PySpark to an already-running Sail server via Spark Connect.""" + from dbt.adapters.spark.session import SessionConnectionWrapper, Connection as SessionConnection + + port = creds.port if creds.port != 443 else 50051 + remote_url = f"sc://{creds.host}:{port}" + + params = dict(creds.server_side_parameters) + params["spark.remote"] = remote_url + return SessionConnectionWrapper(SessionConnection(server_side_parameters=params)) diff --git a/dbt-sail/src/dbt/adapters/sail/impl.py b/dbt-sail/src/dbt/adapters/sail/impl.py new file mode 100644 index 0000000000..8135242ebf --- /dev/null +++ b/dbt-sail/src/dbt/adapters/sail/impl.py @@ -0,0 +1,66 @@ +from typing import List + +from dbt.adapters.base import BaseRelation +from dbt.adapters.events.logging import AdapterLogger +from dbt.adapters.spark.impl import ( + SparkAdapter, + LIST_RELATIONS_MACRO_NAME, + DESCRIBE_TABLE_EXTENDED_MACRO_NAME, + SCHEMA_NOT_FOUND_MESSAGES, + TABLE_OR_VIEW_NOT_FOUND_MESSAGES, +) +from dbt.adapters.sail.connections import SailConnectionManager +from dbt_common.exceptions import DbtRuntimeError +from dbt_common.utils import AttrDict + +logger = AdapterLogger("Sail") + + +class SailAdapter(SparkAdapter): + ConnectionManager = SailConnectionManager + + @classmethod + def type(cls) -> str: + return "sail" + + def _get_relation_information(self, row) -> tuple: + """Parse a row from Sail's SHOW TABLES (6 columns) and fetch extended info.""" + # Sail returns: tableName, catalog, namespace, description, tableType, isTemporary + name = row[0] + namespace = row[2] + _schema = namespace[0] if namespace else "" + + table_name = f"{_schema}.{name}" + try: + table_results = self.execute_macro( + DESCRIBE_TABLE_EXTENDED_MACRO_NAME, kwargs={"table_name": table_name} + ) + except DbtRuntimeError as e: + logger.debug(f"Error while retrieving information about {table_name}: {e.msg}") + table_results = AttrDict() + + information = "" + for info_row in table_results: + info_type, info_value, _ = info_row + if not info_type.startswith("#"): + information += f"{info_type}: {info_value}\n" + + return _schema, name, information + + def list_relations_without_caching(self, schema_relation: BaseRelation) -> List[BaseRelation]: + kwargs = {"schema_relation": schema_relation} + + try: + show_table_rows = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) + return self._build_spark_relation_list( + row_list=show_table_rows, + relation_info_func=self._get_relation_information, + ) + except DbtRuntimeError as e: + errmsg = getattr(e, "msg", "") + if any(msg in errmsg for msg in SCHEMA_NOT_FOUND_MESSAGES) or any( + msg in errmsg for msg in TABLE_OR_VIEW_NOT_FOUND_MESSAGES + ): + return [] + else: + raise diff --git a/dbt-sail/src/dbt/include/sail/__init__.py b/dbt-sail/src/dbt/include/sail/__init__.py new file mode 100644 index 0000000000..b177e5d493 --- /dev/null +++ b/dbt-sail/src/dbt/include/sail/__init__.py @@ -0,0 +1,3 @@ +import os + +PACKAGE_PATH = os.path.dirname(__file__) diff --git a/dbt-sail/src/dbt/include/sail/dbt_project.yml b/dbt-sail/src/dbt/include/sail/dbt_project.yml new file mode 100644 index 0000000000..0b7c8889de --- /dev/null +++ b/dbt-sail/src/dbt/include/sail/dbt_project.yml @@ -0,0 +1,5 @@ +config-version: 2 +name: dbt_sail +version: 1.0 + +macro-paths: ["macros"] diff --git a/dbt-sail/src/dbt/include/sail/macros/adapters.sql b/dbt-sail/src/dbt/include/sail/macros/adapters.sql new file mode 100644 index 0000000000..fd94b20b77 --- /dev/null +++ b/dbt-sail/src/dbt/include/sail/macros/adapters.sql @@ -0,0 +1,28 @@ +{% macro sail__list_relations_without_caching(relation) %} + {% call statement('list_relations_without_caching', fetch_result=True) -%} + show tables in {{ relation.schema }} like '*' + {% endcall %} + + {% do return(load_result('list_relations_without_caching').table) %} +{% endmacro %} + +{% macro sail__get_columns_in_relation_raw(relation) -%} + {% call statement('get_columns_in_relation_raw', fetch_result=True) %} + describe table extended {{ relation }} + {% endcall %} + {% do return(load_result('get_columns_in_relation_raw').table) %} +{% endmacro %} + +{% macro sail__get_columns_in_relation(relation) -%} + {% call statement('get_columns_in_relation', fetch_result=True) %} + describe table extended {{ relation.include(schema=(schema is not none)) }} + {% endcall %} + {% do return(load_result('get_columns_in_relation').table) %} +{% endmacro %} + +{% macro describe_table_extended_without_caching(table_name) %} + {% call statement('describe_table_extended_without_caching', fetch_result=True) -%} + describe table extended {{ table_name }} + {% endcall %} + {% do return(load_result('describe_table_extended_without_caching').table) %} +{% endmacro %} diff --git a/dbt-sail/tests/__init__.py b/dbt-sail/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbt-sail/tests/functional/__init__.py b/dbt-sail/tests/functional/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbt-sail/tests/functional/adapter/__init__.py b/dbt-sail/tests/functional/adapter/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbt-sail/tests/functional/adapter/dbt_clone/fixtures.py b/dbt-sail/tests/functional/adapter/dbt_clone/fixtures.py new file mode 100644 index 0000000000..4939f36c91 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/dbt_clone/fixtures.py @@ -0,0 +1,98 @@ +# Re-export fixtures from dbt-spark's clone test fixtures. +# These are Spark SQL model constants that work identically with Sail. + +seed_csv = """id,name +1,Alice +2,Bob +""" + +table_model_sql = """ +{{ config(materialized='table') }} +select * from {{ ref('ephemeral_model') }} +-- establish a macro dependency to trigger state:modified.macros +-- depends on: {{ my_macro() }} +""" + +view_model_sql = """ +{{ config(materialized='view') }} +select * from {{ ref('seed') }} +-- establish a macro dependency that trips infinite recursion if not handled +-- depends on: {{ my_infinitely_recursive_macro() }} +""" + +ephemeral_model_sql = """ +{{ config(materialized='ephemeral') }} +select * from {{ ref('view_model') }} +""" + +exposures_yml = """ +version: 2 +exposures: + - name: my_exposure + type: application + depends_on: + - ref('view_model') + owner: + email: test@example.com +""" + +schema_yml = """ +version: 2 +models: + - name: view_model + columns: + - name: id + tests: + - unique: + severity: error + - not_null + - name: name +""" + +get_schema_name_sql = """ +{% macro generate_schema_name(custom_schema_name, node) -%} + {%- set default_schema = target.schema -%} + {%- if custom_schema_name is not none -%} + {{ return(default_schema ~ '_' ~ custom_schema_name|trim) }} + {%- elif target.name == 'default' and node.resource_type == 'seed' -%} + {{ return(default_schema ~ '_' ~ 'seeds') }} + {%- else -%} + {{ return(default_schema) }} + {%- endif -%} +{%- endmacro %} +""" + +snapshot_sql = """ +{% snapshot my_cool_snapshot %} + {{ + config( + target_database=database, + target_schema=schema, + unique_key='id', + strategy='check', + check_cols=['id'], + ) + }} + select * from {{ ref('view_model') }} +{% endsnapshot %} +""" + +macros_sql = """ +{% macro my_macro() %} + {% do log('in a macro' ) %} +{% endmacro %} +""" + +infinite_macros_sql = """ +{# trigger infinite recursion if not handled #} +{% macro my_infinitely_recursive_macro() %} + {{ return(adapter.dispatch('my_infinitely_recursive_macro')()) }} +{% endmacro %} +{% macro default__my_infinitely_recursive_macro() %} + {% if unmet_condition %} + {{ my_infinitely_recursive_macro() }} + {% else %} + {{ return('') }} + {% endif %} +{% endmacro %} +""" diff --git a/dbt-sail/tests/functional/adapter/dbt_clone/test_dbt_clone.py b/dbt-sail/tests/functional/adapter/dbt_clone/test_dbt_clone.py new file mode 100644 index 0000000000..529bafb2a9 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/dbt_clone/test_dbt_clone.py @@ -0,0 +1,77 @@ +import pytest +from dbt.tests.adapter.dbt_clone.test_dbt_clone import BaseClonePossible +from tests.functional.adapter.dbt_clone.fixtures import ( + seed_csv, + table_model_sql, + view_model_sql, + ephemeral_model_sql, + exposures_yml, + schema_yml, + snapshot_sql, + get_schema_name_sql, + macros_sql, + infinite_macros_sql, +) + + +class TestSailClonePossible(BaseClonePossible): + @pytest.fixture(scope="class") + def models(self): + return { + "table_model.sql": table_model_sql, + "view_model.sql": view_model_sql, + "ephemeral_model.sql": ephemeral_model_sql, + "schema.yml": schema_yml, + "exposures.yml": exposures_yml, + } + + @pytest.fixture(scope="class") + def macros(self): + return { + "macros.sql": macros_sql, + "infinite_macros.sql": infinite_macros_sql, + "get_schema_name.sql": get_schema_name_sql, + } + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": seed_csv, + } + + @pytest.fixture(scope="class") + def snapshots(self): + return { + "snapshot.sql": snapshot_sql, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+file_format": "delta", + }, + "seeds": { + "test": { + "quote_columns": False, + }, + "+file_format": "delta", + }, + "snapshots": { + "+file_format": "delta", + }, + } + + @pytest.fixture(autouse=True) + def clean_up(self, project): + yield + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, schema=f"{project.test_schema}_seeds" + ) + project.adapter.drop_schema(relation) + + relation = project.adapter.Relation.create( + database=project.database, schema=project.test_schema + ) + project.adapter.drop_schema(relation) diff --git a/dbt-sail/tests/functional/adapter/dbt_show/test_dbt_show.py b/dbt-sail/tests/functional/adapter/dbt_show/test_dbt_show.py new file mode 100644 index 0000000000..e15df5dcbc --- /dev/null +++ b/dbt-sail/tests/functional/adapter/dbt_show/test_dbt_show.py @@ -0,0 +1,19 @@ +import pytest + +from dbt.tests.adapter.dbt_show.test_dbt_show import ( + BaseShowLimit, + BaseShowSqlHeader, + BaseShowDoesNotHandleDoubleLimit, +) + + +class TestSailShowLimit(BaseShowLimit): + pass + + +class TestSailShowSqlHeader(BaseShowSqlHeader): + pass + + +class TestSailShowDoesNotHandleDoubleLimit(BaseShowDoesNotHandleDoubleLimit): + pass diff --git a/dbt-sail/tests/functional/adapter/empty/test_empty.py b/dbt-sail/tests/functional/adapter/empty/test_empty.py new file mode 100644 index 0000000000..85c38b473f --- /dev/null +++ b/dbt-sail/tests/functional/adapter/empty/test_empty.py @@ -0,0 +1,9 @@ +from dbt.tests.adapter.empty.test_empty import BaseTestEmpty, BaseTestEmptyInlineSourceRef + + +class TestEmptySail(BaseTestEmpty): + pass + + +class TestEmptyInlineSourceRefSail(BaseTestEmptyInlineSourceRef): + pass diff --git a/dbt-sail/tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py b/dbt-sail/tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py new file mode 100644 index 0000000000..9a34f641a5 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py @@ -0,0 +1,15 @@ +import pytest +from dbt.tests.adapter.incremental.test_incremental_merge_exclude_columns import ( + BaseMergeExcludeColumns, +) + + +class TestMergeExcludeColumnsSail(BaseMergeExcludeColumns): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+file_format": "delta", + "+incremental_strategy": "merge", + } + } diff --git a/dbt-sail/tests/functional/adapter/incremental/test_incremental_on_schema_change.py b/dbt-sail/tests/functional/adapter/incremental/test_incremental_on_schema_change.py new file mode 100644 index 0000000000..c97677fd9a --- /dev/null +++ b/dbt-sail/tests/functional/adapter/incremental/test_incremental_on_schema_change.py @@ -0,0 +1,45 @@ +import pytest + +from dbt.tests.util import run_dbt + +from dbt.tests.adapter.incremental.test_incremental_on_schema_change import ( + BaseIncrementalOnSchemaChangeSetup, +) + + +class IncrementalOnSchemaChangeIgnoreFail(BaseIncrementalOnSchemaChangeSetup): + def test_run_dbt(self, project): + run_dbt(["run"]) + run_dbt(["run"]) + + +class TestAppendOnSchemaChangeSail(IncrementalOnSchemaChangeIgnoreFail): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+incremental_strategy": "append", + } + } + + +class TestInsertOverwriteOnSchemaChangeSail(IncrementalOnSchemaChangeIgnoreFail): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+incremental_strategy": "insert_overwrite", + } + } + + +class TestDeltaOnSchemaChangeSail(BaseIncrementalOnSchemaChangeSetup): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+file_format": "delta", + "+incremental_strategy": "merge", + "+unique_key": "id", + } + } diff --git a/dbt-sail/tests/functional/adapter/incremental/test_incremental_predicates.py b/dbt-sail/tests/functional/adapter/incremental/test_incremental_predicates.py new file mode 100644 index 0000000000..ba659667e3 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/incremental/test_incremental_predicates.py @@ -0,0 +1,47 @@ +import pytest +from dbt.tests.adapter.incremental.test_incremental_predicates import BaseIncrementalPredicates + + +models__spark_incremental_predicates_sql = """ +{{ config( + materialized = 'incremental', + unique_key = 'id', + incremental_strategy = 'merge', + file_format = 'delta', +) }} + +{% if not is_incremental() %} + +select 1 as id, 'hello' as msg, 'blue' as color + +{% else %} + +-- incremental load +select 2 as id, 'goodbye' as msg, 'red' as color + +{% endif %} +""" + + +class TestIncrementalPredicatesMergeSail(BaseIncrementalPredicates): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+incremental_predicates": ["id != 2"], + "+file_format": "delta", + "+incremental_strategy": "merge", + } + } + + +class TestPredicatesMergeSail(BaseIncrementalPredicates): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+predicates": ["id != 2"], + "+file_format": "delta", + "+incremental_strategy": "merge", + } + } diff --git a/dbt-sail/tests/functional/adapter/incremental/test_incremental_unique_id.py b/dbt-sail/tests/functional/adapter/incremental/test_incremental_unique_id.py new file mode 100644 index 0000000000..b4172d43ca --- /dev/null +++ b/dbt-sail/tests/functional/adapter/incremental/test_incremental_unique_id.py @@ -0,0 +1,13 @@ +import pytest +from dbt.tests.adapter.incremental.test_incremental_unique_id import BaseIncrementalUniqueKey + + +class TestUniqueKeySail(BaseIncrementalUniqueKey): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+file_format": "delta", + "+incremental_strategy": "merge", + } + } diff --git a/dbt-sail/tests/functional/adapter/incremental_strategies/fixtures.py b/dbt-sail/tests/functional/adapter/incremental_strategies/fixtures.py new file mode 100644 index 0000000000..58acfc623b --- /dev/null +++ b/dbt-sail/tests/functional/adapter/incremental_strategies/fixtures.py @@ -0,0 +1,222 @@ +# Spark SQL model fixtures for incremental strategy tests. +# These are identical to dbt-spark's fixtures since Sail speaks Spark SQL. + +default_append_sql = """ +{{ config( + materialized = 'incremental', +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg +union all +select cast(2 as bigint) as id, 'goodbye' as msg + +{% else %} + +select cast(2 as bigint) as id, 'yo' as msg +union all +select cast(3 as bigint) as id, 'anyway' as msg + +{% endif %} +""".lstrip() + +bad_file_format_sql = """ +{{ config( + materialized = 'incremental', + file_format = 'something_else', +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg +union all +select cast(2 as bigint) as id, 'goodbye' as msg + +{% else %} + +select cast(2 as bigint) as id, 'yo' as msg +union all +select cast(3 as bigint) as id, 'anyway' as msg + +{% endif %} +""".lstrip() + +bad_merge_not_delta_sql = """ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg +union all +select cast(2 as bigint) as id, 'goodbye' as msg + +{% else %} + +select cast(2 as bigint) as id, 'yo' as msg +union all +select cast(3 as bigint) as id, 'anyway' as msg + +{% endif %} +""".lstrip() + +bad_strategy_sql = """ +{{ config( + materialized = 'incremental', + incremental_strategy = 'something_else', +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg +union all +select cast(2 as bigint) as id, 'goodbye' as msg + +{% else %} + +select cast(2 as bigint) as id, 'yo' as msg +union all +select cast(3 as bigint) as id, 'anyway' as msg + +{% endif %} +""".lstrip() + +append_delta_sql = """ +{{ config( + materialized = 'incremental', + incremental_strategy = 'append', + file_format = 'delta', +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg +union all +select cast(2 as bigint) as id, 'goodbye' as msg + +{% else %} + +select cast(2 as bigint) as id, 'yo' as msg +union all +select cast(3 as bigint) as id, 'anyway' as msg + +{% endif %} +""".lstrip() + +delta_merge_no_key_sql = """ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + file_format = 'delta', +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg +union all +select cast(2 as bigint) as id, 'goodbye' as msg + +{% else %} + +select cast(2 as bigint) as id, 'yo' as msg +union all +select cast(3 as bigint) as id, 'anyway' as msg + +{% endif %} +""".lstrip() + +delta_merge_unique_key_sql = """ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + file_format = 'delta', + unique_key = 'id', +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg +union all +select cast(2 as bigint) as id, 'goodbye' as msg + +{% else %} + +select cast(2 as bigint) as id, 'yo' as msg +union all +select cast(3 as bigint) as id, 'anyway' as msg + +{% endif %} +""".lstrip() + +delta_merge_update_columns_sql = """ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + file_format = 'delta', + unique_key = 'id', + merge_update_columns = ['msg'], +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg, 'blue' as color +union all +select cast(2 as bigint) as id, 'goodbye' as msg, 'red' as color + +{% else %} + +-- msg will be updated, color will be ignored +select cast(2 as bigint) as id, 'yo' as msg, 'green' as color +union all +select cast(3 as bigint) as id, 'anyway' as msg, 'purple' as color + +{% endif %} +""".lstrip() + +insert_overwrite_no_partitions_sql = """ +{{ config( + materialized = 'incremental', + incremental_strategy = 'insert_overwrite', + file_format = 'parquet', +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg +union all +select cast(2 as bigint) as id, 'goodbye' as msg + +{% else %} + +select cast(2 as bigint) as id, 'yo' as msg +union all +select cast(3 as bigint) as id, 'anyway' as msg + +{% endif %} +""".lstrip() + +insert_overwrite_partitions_sql = """ +{{ config( + materialized = 'incremental', + incremental_strategy = 'insert_overwrite', + partition_by = 'id', + file_format = 'parquet', +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg +union all +select cast(2 as bigint) as id, 'goodbye' as msg + +{% else %} + +select cast(2 as bigint) as id, 'yo' as msg +union all +select cast(3 as bigint) as id, 'anyway' as msg + +{% endif %} +""".lstrip() diff --git a/dbt-sail/tests/functional/adapter/incremental_strategies/seeds.py b/dbt-sail/tests/functional/adapter/incremental_strategies/seeds.py new file mode 100644 index 0000000000..c27561e04c --- /dev/null +++ b/dbt-sail/tests/functional/adapter/incremental_strategies/seeds.py @@ -0,0 +1,27 @@ +expected_append_csv = """ +id,msg +1,hello +2,goodbye +2,yo +3,anyway +""".lstrip() + +expected_overwrite_csv = """ +id,msg +2,yo +3,anyway +""".lstrip() + +expected_partial_upsert_csv = """ +id,msg,color +1,hello,blue +2,yo,red +3,anyway,purple +""".lstrip() + +expected_upsert_csv = """ +id,msg +1,hello +2,yo +3,anyway +""".lstrip() diff --git a/dbt-sail/tests/functional/adapter/incremental_strategies/test_incremental_strategies.py b/dbt-sail/tests/functional/adapter/incremental_strategies/test_incremental_strategies.py new file mode 100644 index 0000000000..d87b1b43c4 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/incremental_strategies/test_incremental_strategies.py @@ -0,0 +1,108 @@ +import pytest + +from dbt.tests.util import run_dbt, check_relations_equal +from dbt.tests.adapter.simple_seed.test_seed import SeedConfigBase +from tests.functional.adapter.incremental_strategies.seeds import ( + expected_append_csv, + expected_overwrite_csv, + expected_upsert_csv, + expected_partial_upsert_csv, +) +from tests.functional.adapter.incremental_strategies.fixtures import ( + bad_file_format_sql, + bad_merge_not_delta_sql, + bad_strategy_sql, + default_append_sql, + insert_overwrite_no_partitions_sql, + insert_overwrite_partitions_sql, + append_delta_sql, + delta_merge_no_key_sql, + delta_merge_unique_key_sql, + delta_merge_update_columns_sql, +) + + +class BaseIncrementalStrategies(SeedConfigBase): + @pytest.fixture(scope="class") + def seeds(self): + return { + "expected_append.csv": expected_append_csv, + "expected_overwrite.csv": expected_overwrite_csv, + "expected_upsert.csv": expected_upsert_csv, + "expected_partial_upsert.csv": expected_partial_upsert_csv, + } + + @staticmethod + def seed_and_run_once(): + run_dbt(["seed"]) + run_dbt(["run"]) + + @staticmethod + def seed_and_run_twice(): + run_dbt(["seed"]) + run_dbt(["run"]) + run_dbt(["run"]) + + +class TestDefaultAppendSail(BaseIncrementalStrategies): + @pytest.fixture(scope="class") + def models(self): + return {"default_append.sql": default_append_sql} + + def test_default_append(self, project): + self.seed_and_run_twice() + check_relations_equal(project.adapter, ["default_append", "expected_append"]) + + +class TestInsertOverwriteSail(BaseIncrementalStrategies): + @pytest.fixture(scope="class") + def models(self): + return { + "insert_overwrite_no_partitions.sql": insert_overwrite_no_partitions_sql, + "insert_overwrite_partitions.sql": insert_overwrite_partitions_sql, + } + + def test_insert_overwrite(self, project): + self.seed_and_run_twice() + check_relations_equal( + project.adapter, ["insert_overwrite_no_partitions", "expected_overwrite"] + ) + check_relations_equal(project.adapter, ["insert_overwrite_partitions", "expected_upsert"]) + + +class TestDeltaStrategiesSail(BaseIncrementalStrategies): + @pytest.fixture(scope="class") + def models(self): + return { + "append_delta.sql": append_delta_sql, + "merge_no_key.sql": delta_merge_no_key_sql, + "merge_unique_key.sql": delta_merge_unique_key_sql, + "merge_update_columns.sql": delta_merge_update_columns_sql, + } + + def test_delta_strategies(self, project): + self.seed_and_run_twice() + check_relations_equal(project.adapter, ["append_delta", "expected_append"]) + check_relations_equal(project.adapter, ["merge_no_key", "expected_append"]) + check_relations_equal(project.adapter, ["merge_unique_key", "expected_upsert"]) + check_relations_equal(project.adapter, ["merge_update_columns", "expected_partial_upsert"]) + + +class TestBadStrategiesSail(BaseIncrementalStrategies): + @pytest.fixture(scope="class") + def models(self): + return { + "bad_file_format.sql": bad_file_format_sql, + "bad_merge_not_delta.sql": bad_merge_not_delta_sql, + "bad_strategy.sql": bad_strategy_sql, + } + + @staticmethod + def run_and_test(): + run_results = run_dbt(["run"], expect_pass=False) + for result in run_results: + assert result.status == "error" + assert "Compilation Error in model" in result.message + + def test_bad_strategies(self, project): + self.run_and_test() diff --git a/dbt-sail/tests/functional/adapter/incremental_strategies/test_microbatch.py b/dbt-sail/tests/functional/adapter/incremental_strategies/test_microbatch.py new file mode 100644 index 0000000000..4e7845667b --- /dev/null +++ b/dbt-sail/tests/functional/adapter/incremental_strategies/test_microbatch.py @@ -0,0 +1,17 @@ +import pytest + +from dbt.tests.adapter.incremental.test_incremental_microbatch import ( + BaseMicrobatch, +) + +_microbatch_model_no_unique_id_sql = """ +{{ config(materialized='incremental', incremental_strategy='microbatch', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0), partition_by=['date_day'], file_format='parquet') }} +select *, cast(event_time as date) as date_day +from {{ ref('input_model') }} +""" + + +class TestMicrobatchSail(BaseMicrobatch): + @pytest.fixture(scope="class") + def microbatch_model_sql(self) -> str: + return _microbatch_model_no_unique_id_sql diff --git a/dbt-sail/tests/functional/adapter/persist_docs/fixtures.py b/dbt-sail/tests/functional/adapter/persist_docs/fixtures.py new file mode 100644 index 0000000000..6c57079e93 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/persist_docs/fixtures.py @@ -0,0 +1,156 @@ +_MODELS__MY_FUN_DOCS = """ +{% docs my_fun_doc %} +name Column description "with double quotes" +and with 'single quotes' as welll as other; +'''abc123''' +reserved -- characters +-- +/* comment */ +Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + +{% enddocs %} +""" + +_MODELS__INCREMENTAL_DELTA = """ +{{ config(materialized='incremental', file_format='delta') }} +select 1 as id, 'Joe' as name +""" + +_MODELS__TABLE_DELTA_MODEL = """ +{{ config(materialized='table', file_format='delta') }} +select 1 as id, 'Joe' as name +""" + +_MODELS__VIEW_DELTA_MODEL = """ +{{ config(materialized='view') }} +select id, count(*) as count from {{ ref('table_delta_model') }} group by id +""" + +_MODELS__TABLE_DELTA_MODEL_MISSING_COLUMN = """ +{{ config(materialized='table', file_format='delta') }} +select 1 as id, 'Joe' as different_name +""" + +_VIEW_PROPERTIES_MODELS = """ +version: 2 + +models: + - name: view_delta_model + description: | + View model description "with double quotes" + and with 'single quotes' as welll as other; + '''abc123''' + reserved -- characters + -- + /* comment */ + Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + columns: + - name: id + description: | + id Column description "with double quotes" + and with 'single quotes' as welll as other; + '''abc123''' + reserved -- characters + -- + /* comment */ + Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting +""" + +_PROPERTIES__MODELS = """ +version: 2 + +models: + - name: table_delta_model + description: | + Table model description "with double quotes" + and with 'single quotes' as welll as other; + '''abc123''' + reserved -- characters + -- + /* comment */ + Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + columns: + - name: id + description: | + id Column description "with double quotes" + and with 'single quotes' as welll as other; + '''abc123''' + reserved -- characters + -- + /* comment */ + Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + - name: name + description: | + Some stuff here and then a call to + {{ doc('my_fun_doc')}} + + - name: incremental_delta_model + description: | + Incremental model description "with double quotes" + and with 'single quotes' as welll as other; + '''abc123''' + reserved -- characters + -- + /* comment */ + Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + columns: + - name: id + description: | + id Column description "with double quotes" + and with 'single quotes' as welll as other; + '''abc123''' + reserved -- characters + -- + /* comment */ + Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + - name: name + description: | + Some stuff here and then a call to + {{ doc('my_fun_doc')}} +""" + +_PROPERTIES__SEEDS = """ +version: 2 + +seeds: + - name: seed + description: | + Seed model description "with double quotes" + and with 'single quotes' as welll as other; + '''abc123''' + reserved -- characters + -- + /* comment */ + Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + columns: + - name: id + description: | + id Column description "with double quotes" + and with 'single quotes' as welll as other; + '''abc123''' + reserved -- characters + -- + /* comment */ + Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + - name: name + description: | + Some stuff here and then a call to + {{ doc('my_fun_doc')}} +""" + +_PROPERTIES__MISSING_COLUMN = """ +version: 2 + +models: + - name: table_delta_model + columns: + - name: id + description: "id column" + - name: column_that_does_not_exist + description: "this column does not exist" +""" + +_SEEDS__BASIC = """id,name +1,Alice +2,Bob +""" diff --git a/dbt-sail/tests/functional/adapter/persist_docs/test_persist_docs.py b/dbt-sail/tests/functional/adapter/persist_docs/test_persist_docs.py new file mode 100644 index 0000000000..ab9f8c5f3a --- /dev/null +++ b/dbt-sail/tests/functional/adapter/persist_docs/test_persist_docs.py @@ -0,0 +1,156 @@ +import pytest + +from dbt.tests.util import run_dbt, run_dbt_and_capture + +from tests.functional.adapter.persist_docs.fixtures import ( + _MODELS__MY_FUN_DOCS, + _MODELS__INCREMENTAL_DELTA, + _MODELS__TABLE_DELTA_MODEL, + _MODELS__TABLE_DELTA_MODEL_MISSING_COLUMN, + _PROPERTIES__MODELS, + _PROPERTIES__MISSING_COLUMN, + _PROPERTIES__SEEDS, + _SEEDS__BASIC, + _MODELS__VIEW_DELTA_MODEL, + _VIEW_PROPERTIES_MODELS, +) + + +class TestPersistDocsDeltaTableSail: + @pytest.fixture(scope="class") + def models(self): + return { + "incremental_delta_model.sql": _MODELS__INCREMENTAL_DELTA, + "my_fun_docs.md": _MODELS__MY_FUN_DOCS, + "table_delta_model.sql": _MODELS__TABLE_DELTA_MODEL, + "schema.yml": _PROPERTIES__MODELS, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"seed.csv": _SEEDS__BASIC, "seed.yml": _PROPERTIES__SEEDS} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "+persist_docs": { + "relation": True, + "columns": True, + }, + } + }, + "seeds": { + "test": { + "+persist_docs": { + "relation": True, + "columns": True, + }, + "+file_format": "delta", + "+quote_columns": True, + } + }, + } + + def test_delta_comments(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + + for table, whatis in [ + ("table_delta_model", "Table"), + ("seed", "Seed"), + ("incremental_delta_model", "Incremental"), + ]: + results = project.run_sql( + "describe extended {schema}.{table}".format( + schema=project.test_schema, table=table + ), + fetch="all", + ) + + for result in results: + if result[0] == "Comment": + assert result[1].startswith(f"{whatis} model description") + if result[0] == "id": + assert result[2].startswith("id Column description") + if result[0] == "name": + assert result[2].startswith("Some stuff here and then a call to") + + +class TestPersistDocsDeltaViewSail: + @pytest.fixture(scope="class") + def models(self): + return { + "table_delta_model.sql": _MODELS__TABLE_DELTA_MODEL, + "view_delta_model.sql": _MODELS__VIEW_DELTA_MODEL, + "schema.yml": _VIEW_PROPERTIES_MODELS, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "+persist_docs": { + "relation": True, + "columns": True, + }, + } + }, + } + + def test_delta_comments(self, project): + run_dbt(["run"]) + + results = project.run_sql( + "describe extended {schema}.{table}".format( + schema=project.test_schema, table="view_delta_model" + ), + fetch="all", + ) + + for result in results: + if result[0] == "Comment": + assert result[1].startswith("View model description") + if result[0] == "id": + assert result[2].startswith("id Column description") + if result[0] == "count": + assert result[2] is None + + +class TestPersistDocsMissingColumnSail: + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "+persist_docs": { + "columns": True, + }, + } + } + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"seed.csv": _SEEDS__BASIC, "seed.yml": _PROPERTIES__SEEDS} + + @pytest.fixture(scope="class") + def models(self): + return { + "table_delta_model.sql": _MODELS__TABLE_DELTA_MODEL_MISSING_COLUMN, + "my_fun_docs.md": _MODELS__MY_FUN_DOCS, + } + + @pytest.fixture(scope="class") + def properties(self): + return {"schema.yml": _PROPERTIES__MISSING_COLUMN} + + def test_missing_column(self, project): + run_dbt(["seed"]) + _, logs = run_dbt_and_capture(["run"]) + assert ( + "The following columns are specified in the schema but are not present in the database: column_that_does_not_exist" + in logs + ) diff --git a/dbt-sail/tests/functional/adapter/seed_column_types/fixtures.py b/dbt-sail/tests/functional/adapter/seed_column_types/fixtures.py new file mode 100644 index 0000000000..ffc8fc35c6 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/seed_column_types/fixtures.py @@ -0,0 +1,113 @@ +_MACRO_TEST_IS_TYPE_SQL = """ +{% macro simple_type_check_column(column, check) %} + {% set checks = { + 'string': column.is_string, + 'float': column.is_float, + 'number': column.is_number, + 'numeric': column.is_numeric, + 'integer': column.is_integer, + } %} + + {{ return(checks[check]()) }} +{% endmacro %} + +{% macro type_check_column(column, type_checks) %} + {% set failures = [] %} + {% for type_check in type_checks %} + {% if type_check.startswith('not ') %} + {% if simple_type_check_column(column, type_check[4:]) %} + {% do failures.append(type_check) %} + {% endif %} + {% else %} + {% if not simple_type_check_column(column, type_check) %} + {% do failures.append(type_check) %} + {% endif %} + {% endif %} + {% endfor %} + + {% do return((failures | length) == 0) %} +{% endmacro %} + +{% macro is_bad_column(column, column_map) %} + {% set column_key = (column.name | lower) %} + {% if column_key not in column_map %} + {% do exceptions.raise_compiler_error('column key ' ~ column_key ~ ' not found in ' ~ (column_map | list | string)) %} + {% endif %} + + {% set type_checks = column_map[column_key] %} + {% if not type_checks %} + {% do exceptions.raise_compiler_error('no type checks?') %} + {% endif %} + + {{ return(not type_check_column(column, type_checks)) }} +{% endmacro %} + +{% test is_type(model, column_map) %} + {% if not execute %} + {{ return(None) }} + {% endif %} + + {% set columns = adapter.get_columns_in_relation(model) %} + {% if (column_map | length) != (columns | length) %} + {% set column_map_keys = (column_map | list | string) %} + {% set column_names = (columns | map(attribute='name') | list | string) %} + {% do exceptions.raise_compiler_error('did not get all the columns/all columns not specified:\\n' ~ column_map_keys ~ '\\nvs\\n' ~ column_names) %} + {% endif %} + + {% set bad_columns = [] %} + {% for column in columns %} + {% if is_bad_column(column, column_map) %} + {% do bad_columns.append(column.name) %} + {% endif %} + {% endfor %} + + {% set num_bad_columns = (bad_columns | length) %} + + select '{{ num_bad_columns }}' as bad_column + group by 1 + having bad_column > 0 + +{% endtest %} +""".strip() + + +_SEED_CSV = """ +id,orderid,paymentmethod,status,amount,amount_usd,created +1,1,credit_card,success,1000,10.00,2018-01-01 +2,2,credit_card,success,2000,20.00,2018-01-02 +3,3,coupon,success,100,1.00,2018-01-04 +4,4,coupon,success,2500,25.00,2018-01-05 +5,5,bank_transfer,fail,1700,17.00,2018-01-05 +6,5,bank_transfer,success,1700,17.00,2018-01-05 +7,6,credit_card,success,600,6.00,2018-01-07 +8,7,credit_card,success,1600,16.00,2018-01-09 +9,8,credit_card,success,2300,23.00,2018-01-11 +10,9,gift_card,success,2300,23.00,2018-01-12 +""".strip() + + +_SEED_YML = """ +version: 2 + +seeds: + - name: payments + config: + column_types: + id: string + orderid: string + paymentmethod: string + status: string + amount: integer + amount_usd: decimal(20,2) + created: timestamp + tests: + - is_type: + column_map: + id: ["string", "not number"] + orderid: ["string", "not number"] + paymentmethod: ["string", "not number"] + status: ["string", "not number"] + amount: ["integer", "number"] + amount_usd: ["decimal", "number"] + created: ["timestamp", "string"] +""".strip() diff --git a/dbt-sail/tests/functional/adapter/seed_column_types/test_seed_column_types.py b/dbt-sail/tests/functional/adapter/seed_column_types/test_seed_column_types.py new file mode 100644 index 0000000000..9395dfd6b9 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/seed_column_types/test_seed_column_types.py @@ -0,0 +1,22 @@ +import pytest +from dbt.tests.util import run_dbt +from tests.functional.adapter.seed_column_types.fixtures import ( + _MACRO_TEST_IS_TYPE_SQL, + _SEED_CSV, + _SEED_YML, +) + + +class TestSeedColumnTypesCastSail: + @pytest.fixture(scope="class") + def macros(self): + return {"test_is_type.sql": _MACRO_TEST_IS_TYPE_SQL} + + @pytest.fixture(scope="class") + def seeds(self): + return {"payments.csv": _SEED_CSV, "schema.yml": _SEED_YML} + + def test_column_seed_type(self, project): + results = run_dbt(["seed"]) + assert len(results) == 1 + run_dbt(["test"], expect_pass=False) diff --git a/dbt-sail/tests/functional/adapter/test_basic.py b/dbt-sail/tests/functional/adapter/test_basic.py new file mode 100644 index 0000000000..567bac3d8e --- /dev/null +++ b/dbt-sail/tests/functional/adapter/test_basic.py @@ -0,0 +1,72 @@ +import pytest + +from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations +from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests +from dbt.tests.adapter.basic.test_singular_tests_ephemeral import ( + BaseSingularTestsEphemeral, +) +from dbt.tests.adapter.basic.test_empty import BaseEmpty +from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral +from dbt.tests.adapter.basic.test_incremental import BaseIncremental +from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests +from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols +from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp +from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod + + +class TestSimpleMaterializationsSail(BaseSimpleMaterializations): + pass + + +class TestSingularTestsSail(BaseSingularTests): + pass + + +class TestSingularTestsEphemeralSail(BaseSingularTestsEphemeral): + pass + + +class TestEmptySail(BaseEmpty): + pass + + +class TestEphemeralSail(BaseEphemeral): + pass + + +class TestIncrementalSail(BaseIncremental): + pass + + +class TestGenericTestsSail(BaseGenericTests): + pass + + +class TestSnapshotCheckColsSail(BaseSnapshotCheckCols): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "+file_format": "delta", + }, + "snapshots": { + "+file_format": "delta", + }, + } + + +class TestSnapshotTimestampSail(BaseSnapshotTimestamp): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "+file_format": "delta", + }, + "snapshots": { + "+file_format": "delta", + }, + } + + +class TestBaseAdapterMethodSail(BaseAdapterMethod): + pass diff --git a/dbt-sail/tests/functional/adapter/test_constraints.py b/dbt-sail/tests/functional/adapter/test_constraints.py new file mode 100644 index 0000000000..7b4836e134 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/test_constraints.py @@ -0,0 +1,131 @@ +import pytest +from dbt.tests.adapter.constraints.test_constraints import ( + BaseTableConstraintsColumnsEqual, + BaseViewConstraintsColumnsEqual, + BaseIncrementalConstraintsColumnsEqual, + BaseConstraintQuotedColumn, + BaseConstraintsRuntimeDdlEnforcement, + BaseIncrementalConstraintsRuntimeDdlEnforcement, + BaseConstraintsRollback, + BaseIncrementalConstraintsRollback, + BaseModelConstraintsRuntimeEnforcement, +) + + +class SailConstraintsSetup: + @pytest.fixture(scope="class") + def string_type(self): + return "string" + + @pytest.fixture(scope="class") + def int_type(self): + return "int" + + @pytest.fixture(scope="class") + def schema_string_type(self): + return "string" + + @pytest.fixture(scope="class") + def schema_int_type(self): + return "int" + + @pytest.fixture(scope="class") + def data_types(self, schema_int_type, int_type, string_type): + return [ + (schema_int_type, int_type, "1"), + (string_type, string_type, "'1'"), + ("boolean", "boolean", "true"), + ("date", "date", "'2024-01-01'"), + ("timestamp", "timestamp", "'2024-01-01 00:00:00'"), + ("double", "double", "1.0"), + ] + + +class TestSailTableConstraintsColumnsEqual(SailConstraintsSetup, BaseTableConstraintsColumnsEqual): + pass + + +class TestSailViewConstraintsColumnsEqual(SailConstraintsSetup, BaseViewConstraintsColumnsEqual): + pass + + +class TestSailIncrementalConstraintsColumnsEqual( + SailConstraintsSetup, BaseIncrementalConstraintsColumnsEqual +): + pass + + +class BaseSailConstraintsDdlEnforcementSetup: + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+file_format": "delta", + } + } + + @pytest.fixture(scope="class") + def expected_color(self): + return "blue" + + +class TestSailTableConstraintsDdlEnforcement( + BaseSailConstraintsDdlEnforcementSetup, BaseConstraintsRuntimeDdlEnforcement +): + pass + + +class TestSailIncrementalConstraintsDdlEnforcement( + BaseSailConstraintsDdlEnforcementSetup, BaseIncrementalConstraintsRuntimeDdlEnforcement +): + pass + + +class TestSailConstraintQuotedColumn(BaseConstraintQuotedColumn): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+file_format": "delta", + } + } + + +class BaseSailConstraintsRollbackSetup: + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+file_format": "delta", + } + } + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["NOT NULL constraint violated"] + + +class TestSailTableConstraintsRollback( + BaseSailConstraintsRollbackSetup, BaseConstraintsRollback +): + pass + + +class TestSailIncrementalConstraintsRollback( + BaseSailConstraintsRollbackSetup, BaseIncrementalConstraintsRollback +): + pass + + +class TestSailModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+file_format": "delta", + } + } + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return ["NOT NULL constraint violated"] diff --git a/dbt-sail/tests/functional/adapter/test_get_columns_in_relation.py b/dbt-sail/tests/functional/adapter/test_get_columns_in_relation.py new file mode 100644 index 0000000000..d9e2269c7a --- /dev/null +++ b/dbt-sail/tests/functional/adapter/test_get_columns_in_relation.py @@ -0,0 +1,32 @@ +import pytest + +from dbt.tests.util import run_dbt, relation_from_name, check_relations_equal_with_relations + + +_MODEL_PARENT = """ +select 1 as id, 'hello' as msg +""" + +_MODEL_CHILD = """ +select * from {{ adapter.get_columns_in_relation(ref('parent')) }} +from {{ ref('parent') }} +""" + + +class TestColumnsInRelationSail: + @pytest.fixture(scope="class") + def models(self): + return { + "parent.sql": _MODEL_PARENT, + "child.sql": _MODEL_CHILD, + } + + def test_get_columns_in_relation(self, project): + run_dbt(["run"]) + check_relations_equal_with_relations( + project.adapter, + [ + relation_from_name(project.adapter, "parent"), + relation_from_name(project.adapter, "child"), + ], + ) diff --git a/dbt-sail/tests/functional/adapter/test_grants.py b/dbt-sail/tests/functional/adapter/test_grants.py new file mode 100644 index 0000000000..f9484eb463 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/test_grants.py @@ -0,0 +1,60 @@ +import pytest +from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants +from dbt.tests.adapter.grants.test_incremental_grants import BaseIncrementalGrants +from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants +from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants +from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants + + +class TestModelGrantsSail(BaseModelGrants): + def privilege_grantee_name_overrides(self): + return { + "select": "select", + } + + +class TestIncrementalGrantsSail(BaseIncrementalGrants): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+file_format": "delta", + "+incremental_strategy": "merge", + } + } + + def privilege_grantee_name_overrides(self): + return { + "select": "select", + } + + +class TestSeedGrantsSail(BaseSeedGrants): + def seeds_support_partial_refresh(self): + return False + + +class TestSnapshotGrantsSail(BaseSnapshotGrants): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "+file_format": "delta", + }, + "snapshots": { + "+file_format": "delta", + }, + } + + def privilege_grantee_name_overrides(self): + return { + "select": "select", + } + + +class TestInvalidGrantsSail(BaseInvalidGrants): + def grantee_does_not_exist_error(self): + return "Principal does not exist" + + def privilege_does_not_exist_error(self): + return "Action Unknown" diff --git a/dbt-sail/tests/functional/adapter/test_python_model.py b/dbt-sail/tests/functional/adapter/test_python_model.py new file mode 100644 index 0000000000..d94e5f36f3 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/test_python_model.py @@ -0,0 +1,18 @@ +import pytest +from dbt.tests.adapter.python_model.test_python_model import ( + BasePythonModelTests, + BasePythonIncrementalTests, +) +from dbt.tests.adapter.python_model.test_spark import BasePySparkTests + + +class TestPythonModelSail(BasePythonModelTests): + pass + + +class TestPySparkSail(BasePySparkTests): + pass + + +class TestPythonIncrementalModelSail(BasePythonIncrementalTests): + pass diff --git a/dbt-sail/tests/functional/adapter/test_simple_seed.py b/dbt-sail/tests/functional/adapter/test_simple_seed.py new file mode 100644 index 0000000000..a706ff73a8 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/test_simple_seed.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.simple_seed.test_seed import BaseTestEmptySeed + + +class TestEmptySeedSail(BaseTestEmptySeed): + pass diff --git a/dbt-sail/tests/functional/adapter/test_store_test_failures.py b/dbt-sail/tests/functional/adapter/test_store_test_failures.py new file mode 100644 index 0000000000..db1d4c758b --- /dev/null +++ b/dbt-sail/tests/functional/adapter/test_store_test_failures.py @@ -0,0 +1,21 @@ +import pytest + +from dbt.tests.adapter.store_test_failures_tests import basic +from dbt.tests.adapter.store_test_failures_tests.test_store_test_failures import ( + StoreTestFailuresBase, +) + + +class TestSailStoreTestFailures(StoreTestFailuresBase): + @pytest.fixture(scope="function", autouse=True) + def teardown_method(self, project): + yield + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, schema=project.test_schema + ) + project.adapter.drop_schema(relation) + + def test_store_and_assert(self, project): + self.run_tests_store_one_failure(project) + self.run_tests_store_failures_and_assert(project) diff --git a/dbt-sail/tests/functional/adapter/unit_testing/test_unit_testing.py b/dbt-sail/tests/functional/adapter/unit_testing/test_unit_testing.py new file mode 100644 index 0000000000..ce1a692904 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -0,0 +1,34 @@ +import pytest + +from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes +from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity +from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput + + +class TestSailUnitTestingTypes(BaseUnitTestingTypes): + @pytest.fixture + def data_types(self): + # sql_value, yaml_value + return [ + ["1", "1"], + ["2.0", "2.0"], + ["'12345'", "12345"], + ["'string'", "string"], + ["true", "true"], + ["date '2011-11-11'", "2011-11-11"], + ["timestamp '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + ["array(1, 2, 3)", "'array(1, 2, 3)'"], + [ + "map('10', 't', '15', 'f', '20', NULL)", + """'map("10", "t", "15", "f", "20", NULL)'""", + ], + ['named_struct("a", 1, "b", 2, "c", 3)', """'named_struct("a", 1, "b", 2, "c", 3)'"""], + ] + + +class TestSailUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): + pass + + +class TestSailUnitTestInvalidInput(BaseUnitTestInvalidInput): + pass diff --git a/dbt-sail/tests/functional/adapter/utils/fixture_listagg.py b/dbt-sail/tests/functional/adapter/utils/fixture_listagg.py new file mode 100644 index 0000000000..0262ca2340 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/utils/fixture_listagg.py @@ -0,0 +1,61 @@ +# SparkSQL does not support 'order by' for its 'listagg' equivalent +# the argument is ignored, so let's ignore those fields when checking equivalency + +models__test_listagg_no_order_by_sql = """ +with data as ( + select * from {{ ref('data_listagg') }} +), +data_output as ( + select * from {{ ref('data_listagg_output') }} +), +calculate as ( +/* + + select + group_col, + {{ listagg('string_text', "'_|_'", "order by order_col") }} as actual, + 'bottom_ordered' as version + from data + group by group_col + union all + select + group_col, + {{ listagg('string_text', "'_|_'", "order by order_col", 2) }} as actual, + 'bottom_ordered_limited' as version + from data + group by group_col + union all + +*/ + select + group_col, + {{ listagg('string_text', "', '") }} as actual, + 'comma_whitespace_unordered' as version + from data + where group_col = 3 + group by group_col + union all + select + group_col, + {{ listagg('DISTINCT string_text', "','") }} as actual, + 'distinct_comma' as version + from data + where group_col = 3 + group by group_col + union all + select + group_col, + {{ listagg('string_text') }} as actual, + 'no_params' as version + from data + where group_col = 3 + group by group_col +) +select + calculate.actual, + data_output.expected +from calculate +left join data_output +on calculate.group_col = data_output.group_col +and calculate.version = data_output.version +""" diff --git a/dbt-sail/tests/functional/adapter/utils/test_data_types.py b/dbt-sail/tests/functional/adapter/utils/test_data_types.py new file mode 100644 index 0000000000..ad0bd45a74 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/utils/test_data_types.py @@ -0,0 +1,73 @@ +import pytest +from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt +from dbt.tests.adapter.utils.data_types.test_type_float import ( + BaseTypeFloat, + seeds__expected_csv as seeds__float_expected_csv, +) +from dbt.tests.adapter.utils.data_types.test_type_int import ( + BaseTypeInt, + seeds__expected_csv as seeds__int_expected_csv, +) +from dbt.tests.adapter.utils.data_types.test_type_numeric import BaseTypeNumeric +from dbt.tests.adapter.utils.data_types.test_type_string import BaseTypeString +from dbt.tests.adapter.utils.data_types.test_type_timestamp import BaseTypeTimestamp +from dbt.tests.adapter.utils.data_types.test_type_boolean import BaseTypeBoolean + + +class TestTypeBigIntSail(BaseTypeBigInt): + pass + + +seeds__float_expected_yml = """ +version: 2 +seeds: + - name: expected + config: + column_types: + float_col: float +""" + + +class TestTypeFloatSail(BaseTypeFloat): + @pytest.fixture(scope="class") + def seeds(self): + return { + "expected.csv": seeds__float_expected_csv, + "expected.yml": seeds__float_expected_yml, + } + + +seeds__int_expected_yml = """ +version: 2 +seeds: + - name: expected + config: + column_types: + int_col: int +""" + + +class TestTypeIntSail(BaseTypeInt): + @pytest.fixture(scope="class") + def seeds(self): + return { + "expected.csv": seeds__int_expected_csv, + "expected.yml": seeds__int_expected_yml, + } + + +class TestTypeNumericSail(BaseTypeNumeric): + def numeric_fixture_type(self): + return "decimal(28,6)" + + +class TestTypeStringSail(BaseTypeString): + pass + + +class TestTypeTimestampSail(BaseTypeTimestamp): + pass + + +class TestTypeBooleanSail(BaseTypeBoolean): + pass diff --git a/dbt-sail/tests/functional/adapter/utils/test_timestamps.py b/dbt-sail/tests/functional/adapter/utils/test_timestamps.py new file mode 100644 index 0000000000..4656552977 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/utils/test_timestamps.py @@ -0,0 +1,18 @@ +import pytest +from dbt.tests.adapter.utils.test_timestamps import BaseCurrentTimestamps + + +class TestCurrentTimestampSail(BaseCurrentTimestamps): + @pytest.fixture(scope="class") + def models(self): + return { + "get_current_timestamp.sql": "select {{ current_timestamp() }} as current_timestamp" + } + + @pytest.fixture(scope="class") + def expected_schema(self): + return {"current_timestamp": "timestamp"} + + @pytest.fixture(scope="class") + def expected_sql(self): + return """select current_timestamp() as current_timestamp""" diff --git a/dbt-sail/tests/functional/adapter/utils/test_utils.py b/dbt-sail/tests/functional/adapter/utils/test_utils.py new file mode 100644 index 0000000000..1e68522491 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/utils/test_utils.py @@ -0,0 +1,162 @@ +import pytest + +from dbt.tests.adapter.utils.test_array_append import BaseArrayAppend +from dbt.tests.adapter.utils.test_array_concat import BaseArrayConcat +from dbt.tests.adapter.utils.test_array_construct import BaseArrayConstruct +from dbt.tests.adapter.utils.test_any_value import BaseAnyValue +from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr +from dbt.tests.adapter.utils.test_cast import BaseCast +from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText +from dbt.tests.adapter.utils.test_concat import BaseConcat +from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampNaive +from dbt.tests.adapter.utils.test_date import BaseDate +from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd +from dbt.tests.adapter.utils.test_datediff import BaseDateDiff +from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc +from dbt.tests.adapter.utils.test_equals import BaseEquals +from dbt.tests.adapter.utils.test_escape_single_quotes import BaseEscapeSingleQuotesBackslash +from dbt.tests.adapter.utils.test_except import BaseExcept +from dbt.tests.adapter.utils.test_hash import BaseHash +from dbt.tests.adapter.utils.test_intersect import BaseIntersect +from dbt.tests.adapter.utils.test_last_day import BaseLastDay +from dbt.tests.adapter.utils.test_length import BaseLength +from dbt.tests.adapter.utils.test_position import BasePosition +from dbt.tests.adapter.utils.test_replace import BaseReplace +from dbt.tests.adapter.utils.test_right import BaseRight +from dbt.tests.adapter.utils.test_safe_cast import BaseSafeCast +from dbt.tests.adapter.utils.test_split_part import BaseSplitPart +from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral +from dbt.tests.adapter.utils.test_listagg import BaseListagg +from dbt.tests.adapter.utils.fixture_listagg import models__test_listagg_yml +from tests.functional.adapter.utils.fixture_listagg import models__test_listagg_no_order_by_sql + +seeds__data_split_part_csv = """parts,split_on,result_1,result_2,result_3,result_4 +a|b|c,|,a,b,c,c +1|2|3,|,1,2,3,3 +EMPTY|EMPTY|EMPTY,|,EMPTY,EMPTY,EMPTY,EMPTY +""" + +seeds__data_last_day_csv = """date_day,date_part,result +2018-01-02,month,2018-01-31 +2018-01-02,quarter,2018-03-31 +2018-01-02,year,2018-12-31 +""" + + +class TestAnyValueSail(BaseAnyValue): + pass + + +class TestArrayAppendSail(BaseArrayAppend): + pass + + +class TestArrayConcatSail(BaseArrayConcat): + pass + + +class TestArrayConstructSail(BaseArrayConstruct): + pass + + +class TestBoolOrSail(BaseBoolOr): + pass + + +class TestCastSail(BaseCast): + pass + + +class TestCastBoolToTextSail(BaseCastBoolToText): + pass + + +class TestConcatSail(BaseConcat): + pass + + +class TestCurrentTimestampSail(BaseCurrentTimestampNaive): + pass + + +class TestDateSail(BaseDate): + pass + + +class TestDateAddSail(BaseDateAdd): + pass + + +class TestDateDiffSail(BaseDateDiff): + pass + + +class TestDateTruncSail(BaseDateTrunc): + pass + + +class TestEqualsSail(BaseEquals): + pass + + +class TestEscapeSingleQuotesSail(BaseEscapeSingleQuotesBackslash): + pass + + +class TestExceptSail(BaseExcept): + pass + + +class TestHashSail(BaseHash): + pass + + +class TestIntersectSail(BaseIntersect): + pass + + +class TestLastDaySail(BaseLastDay): + @pytest.fixture(scope="class") + def seeds(self): + return {"data_last_day.csv": seeds__data_last_day_csv} + + +class TestLengthSail(BaseLength): + pass + + +class TestListaggSail(BaseListagg): + @pytest.fixture(scope="class") + def models(self): + return { + "test_listagg.yml": models__test_listagg_yml, + "test_listagg.sql": self.interpolate_macro_namespace( + models__test_listagg_no_order_by_sql, "listagg" + ), + } + + +class TestPositionSail(BasePosition): + pass + + +class TestReplaceSail(BaseReplace): + pass + + +class TestRightSail(BaseRight): + pass + + +class TestSafeCastSail(BaseSafeCast): + pass + + +class TestSplitPartSail(BaseSplitPart): + @pytest.fixture(scope="class") + def seeds(self): + return {"data_split_part.csv": seeds__data_split_part_csv} + + +class TestStringLiteralSail(BaseStringLiteral): + pass diff --git a/dbt-sail/tests/functional/conftest.py b/dbt-sail/tests/functional/conftest.py new file mode 100644 index 0000000000..3e9dc97a47 --- /dev/null +++ b/dbt-sail/tests/functional/conftest.py @@ -0,0 +1,15 @@ +import pytest + + +pytest_plugins = ["dbt.tests.fixtures.project"] + + +@pytest.fixture(scope="class") +def dbt_profile_target(): + return { + "type": "sail", + "mode": "embedded", + "host": "127.0.0.1", + "schema": "test_sail", + "threads": 1, + } diff --git a/dbt-sail/tests/unit/__init__.py b/dbt-sail/tests/unit/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbt-sail/tests/unit/conftest.py b/dbt-sail/tests/unit/conftest.py new file mode 100644 index 0000000000..7fc366ecba --- /dev/null +++ b/dbt-sail/tests/unit/conftest.py @@ -0,0 +1 @@ +from .fixtures.profiles import * # noqa: F401, F403 diff --git a/dbt-sail/tests/unit/fixtures/__init__.py b/dbt-sail/tests/unit/fixtures/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbt-sail/tests/unit/fixtures/profiles.py b/dbt-sail/tests/unit/fixtures/profiles.py new file mode 100644 index 0000000000..459577ba11 --- /dev/null +++ b/dbt-sail/tests/unit/fixtures/profiles.py @@ -0,0 +1,55 @@ +import pytest + +from tests.unit.utils import config_from_parts_or_dicts + + +@pytest.fixture(scope="session", autouse=True) +def base_project_cfg(): + return { + "name": "X", + "version": "0.1", + "profile": "test", + "project-root": "/tmp/dbt/does-not-exist", + "quoting": { + "identifier": False, + "schema": False, + }, + "config-version": 2, + } + + +@pytest.fixture(scope="session", autouse=True) +def target_sail_embedded(base_project_cfg): + return config_from_parts_or_dicts( + base_project_cfg, + { + "outputs": { + "test": { + "type": "sail", + "mode": "embedded", + "schema": "analytics", + "host": "127.0.0.1", + } + }, + "target": "test", + }, + ) + + +@pytest.fixture(scope="session", autouse=True) +def target_sail_remote(base_project_cfg): + return config_from_parts_or_dicts( + base_project_cfg, + { + "outputs": { + "test": { + "type": "sail", + "mode": "remote", + "schema": "analytics", + "host": "myorg.sailhost.com", + "port": 50051, + } + }, + "target": "test", + }, + ) diff --git a/dbt-sail/tests/unit/test_adapter.py b/dbt-sail/tests/unit/test_adapter.py new file mode 100644 index 0000000000..fff47a7ad8 --- /dev/null +++ b/dbt-sail/tests/unit/test_adapter.py @@ -0,0 +1,145 @@ +import unittest +import pytest +from multiprocessing import get_context +from unittest import mock + +from dbt_common.exceptions import DbtRuntimeError +from agate import Row +from dbt.adapters.sail import SailAdapter +from dbt.adapters.spark import SparkRelation +from dbt.adapters.spark.impl import ( + SCHEMA_NOT_FOUND_MESSAGES, + TABLE_OR_VIEW_NOT_FOUND_MESSAGES, +) + + +class TestSailAdapter(unittest.TestCase): + @pytest.fixture(autouse=True) + def set_up_fixtures(self, target_sail_embedded, base_project_cfg): + self.base_project_cfg = base_project_cfg + self.target_sail_embedded = target_sail_embedded + + def test_adapter_type(self): + assert SailAdapter.type() == "sail" + + def test_parse_relation(self): + self.maxDiff = None + rel_type = SparkRelation.get_relation_type.Table + + relation = SparkRelation.create( + schema="default_schema", identifier="mytable", type=rel_type + ) + assert relation.database is None + + plain_rows = [ + ("col1", "decimal(22,0)"), + ("col2", "string"), + ("dt", "date"), + ("struct_col", "struct"), + ("# Partition Information", "data_type"), + ("# col_name", "data_type"), + ("dt", "date"), + (None, None), + ("# Detailed Table Information", None), + ("Database", None), + ("Owner", "root"), + ("Created Time", "Wed Feb 04 18:15:00 UTC 1815"), + ("Last Access", "Wed May 20 19:25:00 UTC 1925"), + ("Type", "MANAGED"), + ("Provider", "delta"), + ("Location", "/mnt/vo"), + ("Serde Library", "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"), + ("InputFormat", "org.apache.hadoop.mapred.SequenceFileInputFormat"), + ("OutputFormat", "org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat"), + ("Partition Provider", "Catalog"), + ] + + input_cols = [Row(keys=["col_name", "data_type"], values=r) for r in plain_rows] + + rows = SailAdapter( + self.target_sail_embedded, get_context("spawn") + ).parse_describe_extended(relation, input_cols) + self.assertEqual(len(rows), 4) + self.assertEqual(rows[0].to_column_dict(omit_none=False)["column"], "col1") + self.assertEqual(rows[0].to_column_dict(omit_none=False)["dtype"], "decimal(22,0)") + self.assertEqual(rows[1].to_column_dict(omit_none=False)["column"], "col2") + self.assertEqual(rows[2].to_column_dict(omit_none=False)["column"], "dt") + self.assertEqual(rows[3].to_column_dict(omit_none=False)["column"], "struct_col") + + def test_relation_with_database(self): + adapter = SailAdapter(self.target_sail_embedded, get_context("spawn")) + # fine + adapter.Relation.create(schema="different", identifier="table") + with self.assertRaises(DbtRuntimeError): + # not fine - database set + adapter.Relation.create( + database="something", schema="different", identifier="table" + ) + + +class TestListRelationsWithoutCaching(unittest.TestCase): + @pytest.fixture(autouse=True) + def set_up_fixtures(self, target_sail_embedded): + self.target_sail_embedded = target_sail_embedded + + def _make_adapter(self): + return SailAdapter(self.target_sail_embedded, get_context("spawn")) + + def _make_schema_relation(self, adapter, schema="analytics"): + return adapter.Relation.create(schema=schema, identifier="").without_identifier() + + def test_unknown_error_is_raised(self): + adapter = self._make_adapter() + schema_relation = self._make_schema_relation(adapter) + + with mock.patch.object( + adapter, + "execute_macro", + side_effect=DbtRuntimeError("Connection failed"), + ): + with self.assertRaises(DbtRuntimeError): + adapter.list_relations_without_caching(schema_relation) + + def test_schema_not_found_returns_empty(self): + adapter = self._make_adapter() + schema_relation = self._make_schema_relation(adapter, schema="nonexistent") + + with mock.patch.object( + adapter, + "execute_macro", + side_effect=DbtRuntimeError("Database not found"), + ): + result = adapter.list_relations_without_caching(schema_relation) + self.assertEqual(result, []) + + +@pytest.mark.parametrize("not_found_msg", SCHEMA_NOT_FOUND_MESSAGES) +def test_all_schema_not_found_messages_return_empty(not_found_msg, target_sail_embedded): + adapter = SailAdapter(target_sail_embedded, get_context("spawn")) + schema_relation = adapter.Relation.create( + schema="nonexistent", identifier="" + ).without_identifier() + + with mock.patch.object( + adapter, + "execute_macro", + side_effect=DbtRuntimeError(not_found_msg), + ): + result = adapter.list_relations_without_caching(schema_relation) + assert result == [] + + +@pytest.mark.parametrize("not_found_msg", TABLE_OR_VIEW_NOT_FOUND_MESSAGES) +def test_all_table_or_view_not_found_messages_return_empty(not_found_msg, target_sail_embedded): + adapter = SailAdapter(target_sail_embedded, get_context("spawn")) + schema_relation = adapter.Relation.create( + schema="nonexistent", identifier="" + ).without_identifier() + + with mock.patch.object( + adapter, + "execute_macro", + side_effect=DbtRuntimeError(not_found_msg), + ): + result = adapter.list_relations_without_caching(schema_relation) + assert result == [] diff --git a/dbt-sail/tests/unit/test_credentials.py b/dbt-sail/tests/unit/test_credentials.py new file mode 100644 index 0000000000..d0dcbc0985 --- /dev/null +++ b/dbt-sail/tests/unit/test_credentials.py @@ -0,0 +1,77 @@ +import pytest + +from dbt.adapters.sail.connections import SailConnectionMethod, SailCredentials +from dbt_common.exceptions import DbtConfigError + + +def test_embedded_credentials_defaults() -> None: + credentials = SailCredentials( + mode=SailConnectionMethod.EMBEDDED, # type: ignore + schema="analytics", + ) + assert credentials.type == "sail" + assert credentials.host == "127.0.0.1" + assert credentials.schema == "analytics" + assert credentials.database is None + + +def test_remote_credentials() -> None: + credentials = SailCredentials( + mode=SailConnectionMethod.REMOTE, # type: ignore + host="myorg.sailhost.com", + port=50051, + schema="analytics", + ) + assert credentials.type == "sail" + assert credentials.host == "myorg.sailhost.com" + assert credentials.port == 50051 + + +def test_remote_credentials_requires_host() -> None: + with pytest.raises(DbtConfigError): + SailCredentials( + mode=SailConnectionMethod.REMOTE, # type: ignore + schema="analytics", + ) + + +def test_credentials_schema_required() -> None: + with pytest.raises(DbtConfigError): + SailCredentials( + mode=SailConnectionMethod.EMBEDDED, # type: ignore + ) + + +def test_credentials_database_must_match_schema() -> None: + with pytest.raises(DbtConfigError): + SailCredentials( + mode=SailConnectionMethod.EMBEDDED, # type: ignore + schema="analytics", + database="different", + ) + + +def test_credentials_server_side_parameters() -> None: + credentials = SailCredentials( + mode=SailConnectionMethod.EMBEDDED, # type: ignore + schema="analytics", + server_side_parameters={"spark.driver.memory": "4g"}, + ) + assert credentials.server_side_parameters["spark.driver.memory"] == "4g" + + +def test_credentials_unique_field_embedded() -> None: + credentials = SailCredentials( + mode=SailConnectionMethod.EMBEDDED, # type: ignore + schema="analytics", + ) + assert credentials.unique_field == "127.0.0.1" + + +def test_credentials_unique_field_remote() -> None: + credentials = SailCredentials( + mode=SailConnectionMethod.REMOTE, # type: ignore + host="myorg.sailhost.com", + schema="analytics", + ) + assert credentials.unique_field == "myorg.sailhost.com" diff --git a/dbt-sail/tests/unit/utils.py b/dbt-sail/tests/unit/utils.py new file mode 100644 index 0000000000..5419f50d2e --- /dev/null +++ b/dbt-sail/tests/unit/utils.py @@ -0,0 +1,13 @@ +"""Re-export utility functions from dbt-spark's test utilities.""" + +import sys +import os + +# Add dbt-spark's test directory to the path so we can import its utilities +_dbt_adapters_root = os.path.join( + os.path.dirname(__file__), "..", "..", "..", "..", "dbt-adapters", "dbt-spark" +) +if os.path.isdir(_dbt_adapters_root): + sys.path.insert(0, os.path.join(_dbt_adapters_root, "tests")) + +from unit.utils import config_from_parts_or_dicts # noqa: E402, F401 From 417a2c75201353834aed0d1931c90b9bb40434f1 Mon Sep 17 00:00:00 2001 From: pomykalakyle Date: Wed, 1 Apr 2026 11:14:07 -0700 Subject: [PATCH 2/6] Refactor tests and classes for Spark compatibility - Updated test classes to reflect Spark-specific naming conventions, replacing "Sail" with "Spark" in class names. - Added pytest markers to skip tests for specific profiles, ensuring compatibility with Apache Spark and Databricks environments. - Refactored model definitions and configurations to align with Spark's requirements, including adjustments for data types and incremental strategies. - Introduced new tests for query timeout functionality and enhanced existing tests for incremental models, ensuring robust coverage for Spark integration. --- dbt-sail/src/dbt/adapters/sail/__init__.py | 1 + .../functional/adapter/dbt_clone/fixtures.py | 11 +- .../adapter/dbt_clone/test_dbt_clone.py | 5 +- .../adapter/dbt_show/test_dbt_show.py | 13 +- .../functional/adapter/empty/test_empty.py | 4 +- .../test_incremental_merge_exclude_columns.py | 11 +- .../test_incremental_on_schema_change.py | 176 ++++++++- .../test_incremental_predicates.py | 42 +- .../incremental/test_incremental_unique_id.py | 3 +- .../incremental_strategies/fixtures.py | 182 ++++++++- .../test_incremental_strategies.py | 50 ++- .../incremental_strategies/test_microbatch.py | 6 +- .../adapter/persist_docs/fixtures.py | 2 - .../adapter/persist_docs/test_persist_docs.py | 15 +- .../adapter/seed_column_types/fixtures.py | 2 +- .../test_seed_column_types.py | 5 +- .../tests/functional/adapter/test_basic.py | 33 +- .../functional/adapter/test_constraints.py | 368 +++++++++++++++--- .../adapter/test_get_columns_in_relation.py | 29 +- .../tests/functional/adapter/test_grants.py | 40 +- .../functional/adapter/test_python_model.py | 114 +++++- .../functional/adapter/test_query_timeout.py | 76 ++++ .../functional/adapter/test_simple_seed.py | 2 +- .../adapter/test_store_test_failures.py | 84 +++- .../adapter/unit_testing/test_unit_testing.py | 6 +- .../adapter/utils/test_data_types.py | 17 +- .../adapter/utils/test_timestamps.py | 2 +- .../functional/adapter/utils/test_utils.py | 70 ++-- dbt-spark/docker/spark.Dockerfile | 2 +- 29 files changed, 1173 insertions(+), 198 deletions(-) create mode 100644 dbt-sail/tests/functional/adapter/test_query_timeout.py diff --git a/dbt-sail/src/dbt/adapters/sail/__init__.py b/dbt-sail/src/dbt/adapters/sail/__init__.py index 4004a82040..0518dece62 100644 --- a/dbt-sail/src/dbt/adapters/sail/__init__.py +++ b/dbt-sail/src/dbt/adapters/sail/__init__.py @@ -10,3 +10,4 @@ include_path=sail.PACKAGE_PATH, dependencies=["spark"], ) + diff --git a/dbt-sail/tests/functional/adapter/dbt_clone/fixtures.py b/dbt-sail/tests/functional/adapter/dbt_clone/fixtures.py index 4939f36c91..a4bb12a463 100644 --- a/dbt-sail/tests/functional/adapter/dbt_clone/fixtures.py +++ b/dbt-sail/tests/functional/adapter/dbt_clone/fixtures.py @@ -1,6 +1,3 @@ -# Re-export fixtures from dbt-spark's clone test fixtures. -# These are Spark SQL model constants that work identically with Sail. - seed_csv = """id,name 1,Alice 2,Bob @@ -54,6 +51,7 @@ {%- set default_schema = target.schema -%} {%- if custom_schema_name is not none -%} {{ return(default_schema ~ '_' ~ custom_schema_name|trim) }} + -- put seeds into a separate schema in "prod", to verify that cloning in "dev" still works {%- elif target.name == 'default' and node.resource_type == 'seed' -%} {{ return(default_schema ~ '_' ~ 'seeds') }} {%- else -%} @@ -76,7 +74,6 @@ select * from {{ ref('view_model') }} {% endsnapshot %} """ - macros_sql = """ {% macro my_macro() %} {% do log('in a macro' ) %} @@ -96,3 +93,9 @@ {% endif %} {% endmacro %} """ + +custom_can_clone_tables_false_macros_sql = """ +{% macro can_clone_table() %} + {{ return(False) }} +{% endmacro %} +""" diff --git a/dbt-sail/tests/functional/adapter/dbt_clone/test_dbt_clone.py b/dbt-sail/tests/functional/adapter/dbt_clone/test_dbt_clone.py index 529bafb2a9..80e919a24a 100644 --- a/dbt-sail/tests/functional/adapter/dbt_clone/test_dbt_clone.py +++ b/dbt-sail/tests/functional/adapter/dbt_clone/test_dbt_clone.py @@ -14,7 +14,8 @@ ) -class TestSailClonePossible(BaseClonePossible): +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestSparkClonePossible(BaseClonePossible): @pytest.fixture(scope="class") def models(self): return { @@ -75,3 +76,5 @@ def clean_up(self, project): database=project.database, schema=project.test_schema ) project.adapter.drop_schema(relation) + + pass diff --git a/dbt-sail/tests/functional/adapter/dbt_show/test_dbt_show.py b/dbt-sail/tests/functional/adapter/dbt_show/test_dbt_show.py index e15df5dcbc..bc56fd908f 100644 --- a/dbt-sail/tests/functional/adapter/dbt_show/test_dbt_show.py +++ b/dbt-sail/tests/functional/adapter/dbt_show/test_dbt_show.py @@ -1,19 +1,22 @@ import pytest from dbt.tests.adapter.dbt_show.test_dbt_show import ( - BaseShowLimit, BaseShowSqlHeader, + BaseShowLimit, BaseShowDoesNotHandleDoubleLimit, ) -class TestSailShowLimit(BaseShowLimit): +class TestSparkShowLimit(BaseShowLimit): pass -class TestSailShowSqlHeader(BaseShowSqlHeader): +class TestSparkShowSqlHeader(BaseShowSqlHeader): pass -class TestSailShowDoesNotHandleDoubleLimit(BaseShowDoesNotHandleDoubleLimit): - pass +@pytest.mark.skip_profile("apache_spark", "spark_session", "databricks_http_cluster") +class TestSparkShowDoesNotHandleDoubleLimit(BaseShowDoesNotHandleDoubleLimit): + """The syntax message is quite variable across clusters, but this hits two at once.""" + + DATABASE_ERROR_MESSAGE = "limit" diff --git a/dbt-sail/tests/functional/adapter/empty/test_empty.py b/dbt-sail/tests/functional/adapter/empty/test_empty.py index 85c38b473f..90f32953a7 100644 --- a/dbt-sail/tests/functional/adapter/empty/test_empty.py +++ b/dbt-sail/tests/functional/adapter/empty/test_empty.py @@ -1,9 +1,9 @@ from dbt.tests.adapter.empty.test_empty import BaseTestEmpty, BaseTestEmptyInlineSourceRef -class TestEmptySail(BaseTestEmpty): +class TestSparkEmpty(BaseTestEmpty): pass -class TestEmptyInlineSourceRefSail(BaseTestEmptyInlineSourceRef): +class TestSparkEmptyInlineSourceRef(BaseTestEmptyInlineSourceRef): pass diff --git a/dbt-sail/tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py b/dbt-sail/tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py index 9a34f641a5..7560b25ce4 100644 --- a/dbt-sail/tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py +++ b/dbt-sail/tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py @@ -1,15 +1,12 @@ import pytest + from dbt.tests.adapter.incremental.test_incremental_merge_exclude_columns import ( BaseMergeExcludeColumns, ) -class TestMergeExcludeColumnsSail(BaseMergeExcludeColumns): +@pytest.mark.skip_profile("spark_session", "apache_spark") +class TestMergeExcludeColumns(BaseMergeExcludeColumns): @pytest.fixture(scope="class") def project_config_update(self): - return { - "models": { - "+file_format": "delta", - "+incremental_strategy": "merge", - } - } + return {"models": {"+file_format": "delta"}} diff --git a/dbt-sail/tests/functional/adapter/incremental/test_incremental_on_schema_change.py b/dbt-sail/tests/functional/adapter/incremental/test_incremental_on_schema_change.py index c97677fd9a..8d002cec06 100644 --- a/dbt-sail/tests/functional/adapter/incremental/test_incremental_on_schema_change.py +++ b/dbt-sail/tests/functional/adapter/incremental/test_incremental_on_schema_change.py @@ -8,12 +8,21 @@ class IncrementalOnSchemaChangeIgnoreFail(BaseIncrementalOnSchemaChangeSetup): - def test_run_dbt(self, project): - run_dbt(["run"]) - run_dbt(["run"]) + def test_run_incremental_ignore(self, project): + select = "model_a incremental_ignore incremental_ignore_target" + compare_source = "incremental_ignore" + compare_target = "incremental_ignore_target" + self.run_twice_and_assert(select, compare_source, compare_target, project) + def test_run_incremental_fail_on_schema_change(self, project): + select = "model_a incremental_fail" + run_dbt(["run", "--models", select, "--full-refresh"]) + results_two = run_dbt(["run", "--models", select], expect_pass=False) + assert "Compilation Error" in results_two[1].message -class TestAppendOnSchemaChangeSail(IncrementalOnSchemaChangeIgnoreFail): + +@pytest.mark.skip_profile("databricks_sql_endpoint", "spark_http_odbc") +class TestAppendOnSchemaChange(IncrementalOnSchemaChangeIgnoreFail): @pytest.fixture(scope="class") def project_config_update(self): return { @@ -23,17 +32,21 @@ def project_config_update(self): } -class TestInsertOverwriteOnSchemaChangeSail(IncrementalOnSchemaChangeIgnoreFail): +@pytest.mark.skip_profile("databricks_sql_endpoint", "spark_session", "spark_http_odbc") +class TestInsertOverwriteOnSchemaChange(IncrementalOnSchemaChangeIgnoreFail): @pytest.fixture(scope="class") def project_config_update(self): return { "models": { + "+file_format": "parquet", + "+partition_by": "id", "+incremental_strategy": "insert_overwrite", } } -class TestDeltaOnSchemaChangeSail(BaseIncrementalOnSchemaChangeSetup): +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestDeltaOnSchemaChange(BaseIncrementalOnSchemaChangeSetup): @pytest.fixture(scope="class") def project_config_update(self): return { @@ -43,3 +56,154 @@ def project_config_update(self): "+unique_key": "id", } } + + def run_incremental_sync_all_columns(self, project): + select = "model_a incremental_sync_all_columns incremental_sync_all_columns_target" + run_dbt(["run", "--models", select, "--full-refresh"]) + # Delta Lake doesn"t support removing columns -- show a nice compilation error + results = run_dbt(["run", "--models", select], expect_pass=False) + assert "Compilation Error" in results[1].message + + def run_incremental_sync_remove_only(self, project): + select = "model_a incremental_sync_remove_only incremental_sync_remove_only_target" + run_dbt(["run", "--models", select, "--full-refresh"]) + # Delta Lake doesn"t support removing columns -- show a nice compilation error + results = run_dbt(["run", "--models", select], expect_pass=False) + assert "Compilation Error" in results[1].message + + def test_run_incremental_append_new_columns(self, project): + # only adding new columns in supported + self.run_incremental_append_new_columns(project) + # handling columns that have been removed doesn"t work on Delta Lake today + # self.run_incremental_append_new_columns_remove_one(project) + + def test_run_incremental_sync_all_columns(self, project): + self.run_incremental_sync_all_columns(project) + self.run_incremental_sync_remove_only(project) + + +# Test fixtures for special character column names +_MODEL_A_SPECIAL_CHARS = """ +{{ + config(materialized='table') +}} + +with source_data as ( + + select 1 as id, 'bbb' as `select`, 111 as `field-with-dash` + union all select 2 as id, 'ddd' as `select`, 222 as `field-with-dash` + union all select 3 as id, 'fff' as `select`, 333 as `field-with-dash` + union all select 4 as id, 'hhh' as `select`, 444 as `field-with-dash` + union all select 5 as id, 'jjj' as `select`, 555 as `field-with-dash` + union all select 6 as id, 'lll' as `select`, 666 as `field-with-dash` + +) + +select id + ,`select` + ,`field-with-dash` + +from source_data +""" + +_MODEL_INCREMENTAL_APPEND_NEW_SPECIAL_CHARS = """ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='append_new_columns' + ) +}} + +WITH source_data AS (SELECT * FROM {{ ref('model_a_special_chars') }} ) + +{% if is_incremental() %} + +SELECT id, + `select`, + `field-with-dash` +FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) + +{% else %} + +SELECT id, + `select` +FROM source_data where id <= 3 + +{% endif %} +""" + +_MODEL_INCREMENTAL_APPEND_NEW_SPECIAL_CHARS_TARGET = """ +{{ + config(materialized='table') +}} + +with source_data as ( + + select * from {{ ref('model_a_special_chars') }} + +) + +select id + ,`select` + ,CASE WHEN id <= 3 THEN NULL ELSE `field-with-dash` END AS `field-with-dash` + +from source_data +""" + + +@pytest.mark.skip_profile("databricks_sql_endpoint", "spark_http_odbc") +class TestAppendOnSchemaChangeSpecialChars(BaseIncrementalOnSchemaChangeSetup): + """Test incremental models with special character column names using append strategy + + Tests column quoting with SQL keywords and dashes in column names. + Note: Spaces and dots in column names are not supported by some databases. + """ + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+incremental_strategy": "append", + } + } + + @pytest.fixture(scope="class") + def models(self): + return { + "model_a_special_chars.sql": _MODEL_A_SPECIAL_CHARS, + "incremental_append_new_special_chars.sql": _MODEL_INCREMENTAL_APPEND_NEW_SPECIAL_CHARS, + "incremental_append_new_special_chars_target.sql": _MODEL_INCREMENTAL_APPEND_NEW_SPECIAL_CHARS_TARGET, + } + + def test_incremental_append_new_columns_with_special_characters(self, project): + """Test that incremental models work correctly with special character column names when adding new columns""" + + # First run - creates initial table + run_dbt( + [ + "run", + "--models", + "model_a_special_chars incremental_append_new_special_chars incremental_append_new_special_chars_target", + ] + ) + + # Second run - should append new columns with special characters + run_dbt( + [ + "run", + "--models", + "model_a_special_chars incremental_append_new_special_chars incremental_append_new_special_chars_target", + ] + ) + + # Verify results match expected output + from dbt.tests.util import check_relations_equal + + check_relations_equal( + project.adapter, + [ + "incremental_append_new_special_chars", + "incremental_append_new_special_chars_target", + ], + ) diff --git a/dbt-sail/tests/functional/adapter/incremental/test_incremental_predicates.py b/dbt-sail/tests/functional/adapter/incremental/test_incremental_predicates.py index ba659667e3..52c01a7477 100644 --- a/dbt-sail/tests/functional/adapter/incremental/test_incremental_predicates.py +++ b/dbt-sail/tests/functional/adapter/incremental/test_incremental_predicates.py @@ -5,43 +5,61 @@ models__spark_incremental_predicates_sql = """ {{ config( materialized = 'incremental', - unique_key = 'id', - incremental_strategy = 'merge', - file_format = 'delta', + unique_key = 'id' ) }} {% if not is_incremental() %} -select 1 as id, 'hello' as msg, 'blue' as color +select cast(1 as bigint) as id, 'hello' as msg, 'blue' as color +union all +select cast(2 as bigint) as id, 'goodbye' as msg, 'red' as color {% else %} --- incremental load -select 2 as id, 'goodbye' as msg, 'red' as color +-- merge will not happen on the above record where id = 2, so new record will fall to insert +select cast(1 as bigint) as id, 'hey' as msg, 'blue' as color +union all +select cast(2 as bigint) as id, 'yo' as msg, 'green' as color +union all +select cast(3 as bigint) as id, 'anyway' as msg, 'purple' as color {% endif %} """ -class TestIncrementalPredicatesMergeSail(BaseIncrementalPredicates): +@pytest.mark.skip_profile("spark_session", "apache_spark") +class TestIncrementalPredicatesMergeSpark(BaseIncrementalPredicates): @pytest.fixture(scope="class") def project_config_update(self): return { "models": { - "+incremental_predicates": ["id != 2"], - "+file_format": "delta", + "+incremental_predicates": ["dbt_internal_dest.id != 2"], "+incremental_strategy": "merge", + "+file_format": "delta", } } + @pytest.fixture(scope="class") + def models(self): + return { + "delete_insert_incremental_predicates.sql": models__spark_incremental_predicates_sql + } + -class TestPredicatesMergeSail(BaseIncrementalPredicates): +@pytest.mark.skip_profile("spark_session", "apache_spark") +class TestPredicatesMergeSpark(BaseIncrementalPredicates): @pytest.fixture(scope="class") def project_config_update(self): return { "models": { - "+predicates": ["id != 2"], - "+file_format": "delta", + "+predicates": ["dbt_internal_dest.id != 2"], "+incremental_strategy": "merge", + "+file_format": "delta", } } + + @pytest.fixture(scope="class") + def models(self): + return { + "delete_insert_incremental_predicates.sql": models__spark_incremental_predicates_sql + } diff --git a/dbt-sail/tests/functional/adapter/incremental/test_incremental_unique_id.py b/dbt-sail/tests/functional/adapter/incremental/test_incremental_unique_id.py index b4172d43ca..de8cb652c1 100644 --- a/dbt-sail/tests/functional/adapter/incremental/test_incremental_unique_id.py +++ b/dbt-sail/tests/functional/adapter/incremental/test_incremental_unique_id.py @@ -2,7 +2,8 @@ from dbt.tests.adapter.incremental.test_incremental_unique_id import BaseIncrementalUniqueKey -class TestUniqueKeySail(BaseIncrementalUniqueKey): +@pytest.mark.skip_profile("spark_session", "apache_spark") +class TestUniqueKeySpark(BaseIncrementalUniqueKey): @pytest.fixture(scope="class") def project_config_update(self): return { diff --git a/dbt-sail/tests/functional/adapter/incremental_strategies/fixtures.py b/dbt-sail/tests/functional/adapter/incremental_strategies/fixtures.py index 58acfc623b..9cee477df8 100644 --- a/dbt-sail/tests/functional/adapter/incremental_strategies/fixtures.py +++ b/dbt-sail/tests/functional/adapter/incremental_strategies/fixtures.py @@ -1,5 +1,6 @@ -# Spark SQL model fixtures for incremental strategy tests. -# These are identical to dbt-spark's fixtures since Sail speaks Spark SQL. +# +# Models +# default_append_sql = """ {{ config( @@ -21,6 +22,10 @@ {% endif %} """.lstrip() +# +# Bad Models +# + bad_file_format_sql = """ {{ config( materialized = 'incremental', @@ -84,6 +89,10 @@ {% endif %} """.lstrip() +# +# Delta Models +# + append_delta_sql = """ {{ config( materialized = 'incremental', @@ -106,6 +115,30 @@ {% endif %} """.lstrip() +insert_overwrite_partitions_delta_sql = """ +{{ config( + materialized='incremental', + incremental_strategy='insert_overwrite', + partition_by='id', + file_format='delta' +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg +union all +select cast(2 as bigint) as id, 'goodbye' as msg + +{% else %} + +select cast(2 as bigint) as id, 'yo' as msg +union all +select cast(3 as bigint) as id, 'anyway' as msg + +{% endif %} +""" + + delta_merge_no_key_sql = """ {{ config( materialized = 'incremental', @@ -176,6 +209,10 @@ {% endif %} """.lstrip() +# +# Insert Overwrite +# + insert_overwrite_no_partitions_sql = """ {{ config( materialized = 'incremental', @@ -220,3 +257,144 @@ {% endif %} """.lstrip() + +# +# Hudi Models +# + +append_hudi_sql = """ +{{ config( + materialized = 'incremental', + incremental_strategy = 'append', + file_format = 'hudi', +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg +union all +select cast(2 as bigint) as id, 'goodbye' as msg + +{% else %} + +select cast(2 as bigint) as id, 'yo' as msg +union all +select cast(3 as bigint) as id, 'anyway' as msg + +{% endif %} +""".lstrip() + +hudi_insert_overwrite_no_partitions_sql = """ +{{ config( + materialized = 'incremental', + incremental_strategy = 'insert_overwrite', + file_format = 'hudi', +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg +union all +select cast(2 as bigint) as id, 'goodbye' as msg + +{% else %} + +select cast(2 as bigint) as id, 'yo' as msg +union all +select cast(3 as bigint) as id, 'anyway' as msg + +{% endif %} +""".lstrip() + +hudi_insert_overwrite_partitions_sql = """ +{{ config( + materialized = 'incremental', + incremental_strategy = 'insert_overwrite', + partition_by = 'id', + file_format = 'hudi', +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg +union all +select cast(2 as bigint) as id, 'goodbye' as msg + +{% else %} + +select cast(2 as bigint) as id, 'yo' as msg +union all +select cast(3 as bigint) as id, 'anyway' as msg + +{% endif %} +""".lstrip() + +hudi_merge_no_key_sql = """ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + file_format = 'hudi', +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg +union all +select cast(2 as bigint) as id, 'goodbye' as msg + +{% else %} + +select cast(2 as bigint) as id, 'yo' as msg +union all +select cast(3 as bigint) as id, 'anyway' as msg + +{% endif %} +""".lstrip() + +hudi_merge_unique_key_sql = """ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + file_format = 'hudi', + unique_key = 'id', +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg +union all +select cast(2 as bigint) as id, 'goodbye' as msg + +{% else %} + +select cast(2 as bigint) as id, 'yo' as msg +union all +select cast(3 as bigint) as id, 'anyway' as msg + +{% endif %} +""".lstrip() + +hudi_update_columns_sql = """ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + file_format = 'hudi', + unique_key = 'id', + merge_update_columns = ['msg'], +) }} + +{% if not is_incremental() %} + +select cast(1 as bigint) as id, 'hello' as msg, 'blue' as color +union all +select cast(2 as bigint) as id, 'goodbye' as msg, 'red' as color + +{% else %} + +-- msg will be updated, color will be ignored +select cast(2 as bigint) as id, 'yo' as msg, 'green' as color +union all +select cast(3 as bigint) as id, 'anyway' as msg, 'purple' as color + +{% endif %} +""".lstrip() diff --git a/dbt-sail/tests/functional/adapter/incremental_strategies/test_incremental_strategies.py b/dbt-sail/tests/functional/adapter/incremental_strategies/test_incremental_strategies.py index d87b1b43c4..a44a1d23ec 100644 --- a/dbt-sail/tests/functional/adapter/incremental_strategies/test_incremental_strategies.py +++ b/dbt-sail/tests/functional/adapter/incremental_strategies/test_incremental_strategies.py @@ -19,6 +19,7 @@ delta_merge_no_key_sql, delta_merge_unique_key_sql, delta_merge_update_columns_sql, + # Skip: CT-1873 insert_overwrite_partitions_delta_sql, ) @@ -44,17 +45,23 @@ def seed_and_run_twice(): run_dbt(["run"]) -class TestDefaultAppendSail(BaseIncrementalStrategies): +class TestDefaultAppend(BaseIncrementalStrategies): @pytest.fixture(scope="class") def models(self): return {"default_append.sql": default_append_sql} - def test_default_append(self, project): + def run_and_test(self, project): self.seed_and_run_twice() check_relations_equal(project.adapter, ["default_append", "expected_append"]) + @pytest.mark.skip_profile( + "databricks_http_cluster", "databricks_sql_endpoint", "spark_session", "spark_http_odbc" + ) + def test_default_append(self, project): + self.run_and_test(project) + -class TestInsertOverwriteSail(BaseIncrementalStrategies): +class TestInsertOverwrite(BaseIncrementalStrategies): @pytest.fixture(scope="class") def models(self): return { @@ -62,15 +69,21 @@ def models(self): "insert_overwrite_partitions.sql": insert_overwrite_partitions_sql, } - def test_insert_overwrite(self, project): + def run_and_test(self, project): self.seed_and_run_twice() check_relations_equal( project.adapter, ["insert_overwrite_no_partitions", "expected_overwrite"] ) check_relations_equal(project.adapter, ["insert_overwrite_partitions", "expected_upsert"]) + @pytest.mark.skip_profile( + "databricks_http_cluster", "databricks_sql_endpoint", "spark_session", "spark_http_odbc" + ) + def test_insert_overwrite(self, project): + self.run_and_test(project) + -class TestDeltaStrategiesSail(BaseIncrementalStrategies): +class TestDeltaStrategies(BaseIncrementalStrategies): @pytest.fixture(scope="class") def models(self): return { @@ -78,17 +91,38 @@ def models(self): "merge_no_key.sql": delta_merge_no_key_sql, "merge_unique_key.sql": delta_merge_unique_key_sql, "merge_update_columns.sql": delta_merge_update_columns_sql, + # Skip: cannot be acnive on any endpoint with grants + # "insert_overwrite_partitions_delta.sql": insert_overwrite_partitions_delta_sql, } - def test_delta_strategies(self, project): + def run_and_test(self, project): self.seed_and_run_twice() check_relations_equal(project.adapter, ["append_delta", "expected_append"]) check_relations_equal(project.adapter, ["merge_no_key", "expected_append"]) check_relations_equal(project.adapter, ["merge_unique_key", "expected_upsert"]) check_relations_equal(project.adapter, ["merge_update_columns", "expected_partial_upsert"]) + @pytest.mark.skip_profile( + "apache_spark", + "databricks_http_cluster", + "databricks_sql_endpoint", + "spark_session", + "spark_http_odbc", + ) + def test_delta_strategies(self, project): + self.run_and_test(project) + + @pytest.mark.skip( + reason="this feature is incompatible with databricks settings required for grants" + ) + def test_delta_strategies_overwrite(self, project): + self.seed_and_run_twice() + check_relations_equal( + project.adapter, ["insert_overwrite_partitions_delta", "expected_upsert"] + ) + -class TestBadStrategiesSail(BaseIncrementalStrategies): +class TestBadStrategies(BaseIncrementalStrategies): @pytest.fixture(scope="class") def models(self): return { @@ -100,9 +134,11 @@ def models(self): @staticmethod def run_and_test(): run_results = run_dbt(["run"], expect_pass=False) + # assert all models fail with compilation errors for result in run_results: assert result.status == "error" assert "Compilation Error in model" in result.message + @pytest.mark.skip_profile("databricks_http_cluster", "spark_session") def test_bad_strategies(self, project): self.run_and_test() diff --git a/dbt-sail/tests/functional/adapter/incremental_strategies/test_microbatch.py b/dbt-sail/tests/functional/adapter/incremental_strategies/test_microbatch.py index 4e7845667b..088b35bafc 100644 --- a/dbt-sail/tests/functional/adapter/incremental_strategies/test_microbatch.py +++ b/dbt-sail/tests/functional/adapter/incremental_strategies/test_microbatch.py @@ -4,6 +4,7 @@ BaseMicrobatch, ) +# No requirement for a unique_id for spark microbatch! _microbatch_model_no_unique_id_sql = """ {{ config(materialized='incremental', incremental_strategy='microbatch', event_time='event_time', batch_size='day', begin=modules.datetime.datetime(2020, 1, 1, 0, 0, 0), partition_by=['date_day'], file_format='parquet') }} select *, cast(event_time as date) as date_day @@ -11,7 +12,10 @@ """ -class TestMicrobatchSail(BaseMicrobatch): +@pytest.mark.skip_profile( + "databricks_http_cluster", "databricks_sql_endpoint", "spark_session", "spark_http_odbc" +) +class TestMicrobatch(BaseMicrobatch): @pytest.fixture(scope="class") def microbatch_model_sql(self) -> str: return _microbatch_model_no_unique_id_sql diff --git a/dbt-sail/tests/functional/adapter/persist_docs/fixtures.py b/dbt-sail/tests/functional/adapter/persist_docs/fixtures.py index 6c57079e93..83137bc22a 100644 --- a/dbt-sail/tests/functional/adapter/persist_docs/fixtures.py +++ b/dbt-sail/tests/functional/adapter/persist_docs/fixtures.py @@ -30,7 +30,6 @@ {{ config(materialized='table', file_format='delta') }} select 1 as id, 'Joe' as different_name """ - _VIEW_PROPERTIES_MODELS = """ version: 2 @@ -55,7 +54,6 @@ /* comment */ Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting """ - _PROPERTIES__MODELS = """ version: 2 diff --git a/dbt-sail/tests/functional/adapter/persist_docs/test_persist_docs.py b/dbt-sail/tests/functional/adapter/persist_docs/test_persist_docs.py index ab9f8c5f3a..f6bf34b5c9 100644 --- a/dbt-sail/tests/functional/adapter/persist_docs/test_persist_docs.py +++ b/dbt-sail/tests/functional/adapter/persist_docs/test_persist_docs.py @@ -2,7 +2,7 @@ from dbt.tests.util import run_dbt, run_dbt_and_capture -from tests.functional.adapter.persist_docs.fixtures import ( +from fixtures import ( _MODELS__MY_FUN_DOCS, _MODELS__INCREMENTAL_DELTA, _MODELS__TABLE_DELTA_MODEL, @@ -16,7 +16,8 @@ ) -class TestPersistDocsDeltaTableSail: +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestPersistDocsDeltaTable: @pytest.fixture(scope="class") def models(self): return { @@ -78,7 +79,8 @@ def test_delta_comments(self, project): assert result[2].startswith("Some stuff here and then a call to") -class TestPersistDocsDeltaViewSail: +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestPersistDocsDeltaView: @pytest.fixture(scope="class") def models(self): return { @@ -119,7 +121,8 @@ def test_delta_comments(self, project): assert result[2] is None -class TestPersistDocsMissingColumnSail: +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestPersistDocsMissingColumn: @pytest.fixture(scope="class") def project_config_update(self): return { @@ -148,6 +151,10 @@ def properties(self): return {"schema.yml": _PROPERTIES__MISSING_COLUMN} def test_missing_column(self, project): + """ + With column filtering in alter_column_comment, non-existent columns + are now skipped instead of causing DB errors, and a warning is emitted. + """ run_dbt(["seed"]) _, logs = run_dbt_and_capture(["run"]) assert ( diff --git a/dbt-sail/tests/functional/adapter/seed_column_types/fixtures.py b/dbt-sail/tests/functional/adapter/seed_column_types/fixtures.py index ffc8fc35c6..e002d57bbe 100644 --- a/dbt-sail/tests/functional/adapter/seed_column_types/fixtures.py +++ b/dbt-sail/tests/functional/adapter/seed_column_types/fixtures.py @@ -51,7 +51,7 @@ {% if (column_map | length) != (columns | length) %} {% set column_map_keys = (column_map | list | string) %} {% set column_names = (columns | map(attribute='name') | list | string) %} - {% do exceptions.raise_compiler_error('did not get all the columns/all columns not specified:\\n' ~ column_map_keys ~ '\\nvs\\n' ~ column_names) %} + {% do exceptions.raise_compiler_error('did not get all the columns/all columns not specified:\n' ~ column_map_keys ~ '\nvs\n' ~ column_names) %} {% endif %} {% set bad_columns = [] %} diff --git a/dbt-sail/tests/functional/adapter/seed_column_types/test_seed_column_types.py b/dbt-sail/tests/functional/adapter/seed_column_types/test_seed_column_types.py index 9395dfd6b9..3326490f97 100644 --- a/dbt-sail/tests/functional/adapter/seed_column_types/test_seed_column_types.py +++ b/dbt-sail/tests/functional/adapter/seed_column_types/test_seed_column_types.py @@ -7,7 +7,8 @@ ) -class TestSeedColumnTypesCastSail: +@pytest.mark.skip_profile("spark_session") +class TestSeedColumnTypesCast: @pytest.fixture(scope="class") def macros(self): return {"test_is_type.sql": _MACRO_TEST_IS_TYPE_SQL} @@ -16,6 +17,8 @@ def macros(self): def seeds(self): return {"payments.csv": _SEED_CSV, "schema.yml": _SEED_YML} + # We want to test seed types because hive would cause all fields to be strings. + # setting column_types in project.yml should change them and pass. def test_column_seed_type(self, project): results = run_dbt(["seed"]) assert len(results) == 1 diff --git a/dbt-sail/tests/functional/adapter/test_basic.py b/dbt-sail/tests/functional/adapter/test_basic.py index 567bac3d8e..072d211d63 100644 --- a/dbt-sail/tests/functional/adapter/test_basic.py +++ b/dbt-sail/tests/functional/adapter/test_basic.py @@ -14,35 +14,44 @@ from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod -class TestSimpleMaterializationsSail(BaseSimpleMaterializations): +@pytest.mark.skip_profile("spark_session") +class TestSimpleMaterializationsSpark(BaseSimpleMaterializations): pass -class TestSingularTestsSail(BaseSingularTests): +class TestSingularTestsSpark(BaseSingularTests): pass -class TestSingularTestsEphemeralSail(BaseSingularTestsEphemeral): +# The local cluster currently tests on spark 2.x, which does not support this +# if we upgrade it to 3.x, we can enable this test +@pytest.mark.skip_profile("apache_spark") +class TestSingularTestsEphemeralSpark(BaseSingularTestsEphemeral): pass -class TestEmptySail(BaseEmpty): +class TestEmptySpark(BaseEmpty): pass -class TestEphemeralSail(BaseEphemeral): +@pytest.mark.skip_profile("spark_session") +class TestEphemeralSpark(BaseEphemeral): pass -class TestIncrementalSail(BaseIncremental): +@pytest.mark.skip_profile("spark_session") +class TestIncrementalSpark(BaseIncremental): pass -class TestGenericTestsSail(BaseGenericTests): +class TestGenericTestsSpark(BaseGenericTests): pass -class TestSnapshotCheckColsSail(BaseSnapshotCheckCols): +# These tests were not enabled in the dbtspec files, so skipping here. +# Error encountered was: Error running query: java.lang.ClassNotFoundException: delta.DefaultSource +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestSnapshotCheckColsSpark(BaseSnapshotCheckCols): @pytest.fixture(scope="class") def project_config_update(self): return { @@ -55,7 +64,10 @@ def project_config_update(self): } -class TestSnapshotTimestampSail(BaseSnapshotTimestamp): +# These tests were not enabled in the dbtspec files, so skipping here. +# Error encountered was: Error running query: java.lang.ClassNotFoundException: delta.DefaultSource +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestSnapshotTimestampSpark(BaseSnapshotTimestamp): @pytest.fixture(scope="class") def project_config_update(self): return { @@ -68,5 +80,6 @@ def project_config_update(self): } -class TestBaseAdapterMethodSail(BaseAdapterMethod): +@pytest.mark.skip_profile("spark_session") +class TestBaseAdapterMethod(BaseAdapterMethod): pass diff --git a/dbt-sail/tests/functional/adapter/test_constraints.py b/dbt-sail/tests/functional/adapter/test_constraints.py index 7b4836e134..f33359262c 100644 --- a/dbt-sail/tests/functional/adapter/test_constraints.py +++ b/dbt-sail/tests/functional/adapter/test_constraints.py @@ -1,61 +1,245 @@ import pytest from dbt.tests.adapter.constraints.test_constraints import ( + BaseModelConstraintsRuntimeEnforcement, BaseTableConstraintsColumnsEqual, BaseViewConstraintsColumnsEqual, BaseIncrementalConstraintsColumnsEqual, - BaseConstraintQuotedColumn, BaseConstraintsRuntimeDdlEnforcement, - BaseIncrementalConstraintsRuntimeDdlEnforcement, BaseConstraintsRollback, + BaseIncrementalConstraintsRuntimeDdlEnforcement, BaseIncrementalConstraintsRollback, - BaseModelConstraintsRuntimeEnforcement, + BaseConstraintQuotedColumn, +) +from dbt.tests.adapter.constraints.fixtures import ( + constrained_model_schema_yml, + my_model_sql, + my_model_wrong_order_sql, + my_model_wrong_name_sql, + model_schema_yml, + my_model_view_wrong_order_sql, + my_model_view_wrong_name_sql, + my_model_incremental_wrong_order_sql, + my_model_incremental_wrong_name_sql, + my_incremental_model_sql, + model_fk_constraint_schema_yml, + my_model_wrong_order_depends_on_fk_sql, + foreign_key_model_sql, + my_model_incremental_wrong_order_depends_on_fk_sql, + my_model_with_quoted_column_name_sql, + model_quoted_column_schema_yml, +) + +# constraints are enforced via 'alter' statements that run after table creation +_expected_sql_spark = """ +create or replace table + using delta + as +select + id, + color, + date_day +from + +( + -- depends_on: + select + 'blue' as color, + 1 as id, + '2019-01-01' as date_day ) as model_subq +""" + +_expected_sql_spark_model_constraints = """ +create or replace table + using delta + as +select + id, + color, + date_day +from + +( + -- depends_on: + select + 'blue' as color, + 1 as id, + '2019-01-01' as date_day ) as model_subq +""" + +# Different on Spark: +# - does not support a data type named 'text' (TODO handle this in the base test classes using string_type +constraints_yml = model_schema_yml.replace("text", "string").replace("primary key", "") +model_fk_constraint_schema_yml = model_fk_constraint_schema_yml.replace("text", "string").replace( + "primary key", "" ) +model_constraints_yml = constrained_model_schema_yml.replace("text", "string") -class SailConstraintsSetup: +class PyodbcSetup: @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "+file_format": "delta", + } + } + + @pytest.fixture def string_type(self): - return "string" + return "STR" - @pytest.fixture(scope="class") + @pytest.fixture def int_type(self): - return "int" + return "INT" - @pytest.fixture(scope="class") + @pytest.fixture def schema_string_type(self): - return "string" + return "STRING" - @pytest.fixture(scope="class") + @pytest.fixture def schema_int_type(self): - return "int" + return "INT" - @pytest.fixture(scope="class") - def data_types(self, schema_int_type, int_type, string_type): + @pytest.fixture + def data_types(self, int_type, schema_int_type, string_type, schema_string_type): + # sql_column_value, schema_data_type, error_data_type return [ - (schema_int_type, int_type, "1"), - (string_type, string_type, "'1'"), - ("boolean", "boolean", "true"), - ("date", "date", "'2024-01-01'"), - ("timestamp", "timestamp", "'2024-01-01 00:00:00'"), - ("double", "double", "1.0"), + ["1", schema_int_type, int_type], + ['"1"', schema_string_type, string_type], + ["true", "boolean", "BOOL"], + ['array("1","2","3")', "string", string_type], + ["array(1,2,3)", "string", string_type], + ["6.45", "decimal", "DECIMAL"], + ["cast('2019-01-01' as date)", "date", "DATE"], + ["cast('2019-01-01' as timestamp)", "timestamp", "DATETIME"], ] -class TestSailTableConstraintsColumnsEqual(SailConstraintsSetup, BaseTableConstraintsColumnsEqual): - pass +class DatabricksHTTPSetup: + @pytest.fixture + def string_type(self): + return "STRING_TYPE" + + @pytest.fixture + def int_type(self): + return "INT_TYPE" + + @pytest.fixture + def schema_string_type(self): + return "STRING" + + @pytest.fixture + def schema_int_type(self): + return "INT" + + @pytest.fixture + def data_types(self, int_type, schema_int_type, string_type, schema_string_type): + # sql_column_value, schema_data_type, error_data_type + return [ + ["1", schema_int_type, int_type], + ['"1"', schema_string_type, string_type], + ["true", "boolean", "BOOLEAN_TYPE"], + ['array("1","2","3")', "array", "ARRAY_TYPE"], + ["array(1,2,3)", "array", "ARRAY_TYPE"], + ["cast('2019-01-01' as date)", "date", "DATE_TYPE"], + ["cast('2019-01-01' as timestamp)", "timestamp", "TIMESTAMP_TYPE"], + ["cast(1.0 AS DECIMAL(4, 2))", "decimal", "DECIMAL_TYPE"], + ] + + +@pytest.mark.skip_profile("spark_session", "apache_spark", "databricks_http_cluster") +class TestSparkTableConstraintsColumnsEqualPyodbc(PyodbcSetup, BaseTableConstraintsColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_wrong_order_sql, + "my_model_wrong_name.sql": my_model_wrong_name_sql, + "constraints_schema.yml": constraints_yml, + } -class TestSailViewConstraintsColumnsEqual(SailConstraintsSetup, BaseViewConstraintsColumnsEqual): - pass +@pytest.mark.skip_profile("spark_session", "apache_spark", "databricks_http_cluster") +class TestSparkViewConstraintsColumnsEqualPyodbc(PyodbcSetup, BaseViewConstraintsColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_view_wrong_order_sql, + "my_model_wrong_name.sql": my_model_view_wrong_name_sql, + "constraints_schema.yml": constraints_yml, + } -class TestSailIncrementalConstraintsColumnsEqual( - SailConstraintsSetup, BaseIncrementalConstraintsColumnsEqual +@pytest.mark.skip_profile("spark_session", "apache_spark", "databricks_http_cluster") +class TestSparkIncrementalConstraintsColumnsEqualPyodbc( + PyodbcSetup, BaseIncrementalConstraintsColumnsEqual ): - pass + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_incremental_wrong_order_sql, + "my_model_wrong_name.sql": my_model_incremental_wrong_name_sql, + "constraints_schema.yml": constraints_yml, + } -class BaseSailConstraintsDdlEnforcementSetup: +@pytest.mark.skip_profile( + "spark_session", + "apache_spark", + "databricks_sql_endpoint", + "databricks_cluster", + "spark_http_odbc", +) +class TestSparkTableConstraintsColumnsEqualDatabricksHTTP( + DatabricksHTTPSetup, BaseTableConstraintsColumnsEqual +): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_wrong_order_sql, + "my_model_wrong_name.sql": my_model_wrong_name_sql, + "constraints_schema.yml": constraints_yml, + } + + +@pytest.mark.skip_profile( + "spark_session", + "apache_spark", + "databricks_sql_endpoint", + "databricks_cluster", + "spark_http_odbc", +) +class TestSparkViewConstraintsColumnsEqualDatabricksHTTP( + DatabricksHTTPSetup, BaseViewConstraintsColumnsEqual +): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_view_wrong_order_sql, + "my_model_wrong_name.sql": my_model_view_wrong_name_sql, + "constraints_schema.yml": constraints_yml, + } + + +@pytest.mark.skip_profile( + "spark_session", + "apache_spark", + "databricks_sql_endpoint", + "databricks_cluster", + "spark_http_odbc", +) +class TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP( + DatabricksHTTPSetup, BaseIncrementalConstraintsColumnsEqual +): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_incremental_wrong_order_sql, + "my_model_wrong_name.sql": my_model_incremental_wrong_name_sql, + "constraints_schema.yml": constraints_yml, + } + + +class BaseSparkConstraintsDdlEnforcementSetup: @pytest.fixture(scope="class") def project_config_update(self): return { @@ -65,33 +249,68 @@ def project_config_update(self): } @pytest.fixture(scope="class") - def expected_color(self): - return "blue" + def expected_sql(self): + return _expected_sql_spark -class TestSailTableConstraintsDdlEnforcement( - BaseSailConstraintsDdlEnforcementSetup, BaseConstraintsRuntimeDdlEnforcement +@pytest.mark.skip_profile("spark_session", "apache_spark") +class TestSparkTableConstraintsDdlEnforcement( + BaseSparkConstraintsDdlEnforcementSetup, BaseConstraintsRuntimeDdlEnforcement ): - pass + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": foreign_key_model_sql, + "constraints_schema.yml": model_fk_constraint_schema_yml, + } -class TestSailIncrementalConstraintsDdlEnforcement( - BaseSailConstraintsDdlEnforcementSetup, BaseIncrementalConstraintsRuntimeDdlEnforcement +@pytest.mark.skip_profile("spark_session", "apache_spark") +class TestSparkIncrementalConstraintsDdlEnforcement( + BaseSparkConstraintsDdlEnforcementSetup, BaseIncrementalConstraintsRuntimeDdlEnforcement ): - pass + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_incremental_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": foreign_key_model_sql, + "constraints_schema.yml": model_fk_constraint_schema_yml, + } -class TestSailConstraintQuotedColumn(BaseConstraintQuotedColumn): +@pytest.mark.skip_profile("spark_session", "apache_spark", "databricks_http_cluster") +class TestSparkConstraintQuotedColumn(PyodbcSetup, BaseConstraintQuotedColumn): @pytest.fixture(scope="class") - def project_config_update(self): + def models(self): return { - "models": { - "+file_format": "delta", - } + "my_model.sql": my_model_with_quoted_column_name_sql, + "constraints_schema.yml": model_quoted_column_schema_yml.replace( + "text", "string" + ).replace('"from"', "`from`"), } + @pytest.fixture(scope="class") + def expected_sql(self): + return """ +create or replace table + using delta + as +select + id, + `from`, + date_day +from + +( + select + 'blue' as `from`, + 1 as id, + '2019-01-01' as date_day ) as model_subq +""" -class BaseSailConstraintsRollbackSetup: + +class BaseSparkConstraintsRollbackSetup: @pytest.fixture(scope="class") def project_config_update(self): return { @@ -102,22 +321,61 @@ def project_config_update(self): @pytest.fixture(scope="class") def expected_error_messages(self): - return ["NOT NULL constraint violated"] + return [ + "violate the new CHECK constraint", + "DELTA_NEW_CHECK_CONSTRAINT_VIOLATION", + "DELTA_NEW_NOT_NULL_VIOLATION", + "violate the new NOT NULL constraint", + "(id > 0) violated by row with values:", # incremental mats + "DELTA_VIOLATE_CONSTRAINT_WITH_VALUES", # incremental mats + "NOT NULL constraint violated for col", + ] + def assert_expected_error_messages(self, error_message, expected_error_messages): + # This needs to be ANY instead of ALL + # The CHECK constraint is added before the NOT NULL constraint + # and different connection types display/truncate the error message in different ways... + assert any(msg in error_message for msg in expected_error_messages) -class TestSailTableConstraintsRollback( - BaseSailConstraintsRollbackSetup, BaseConstraintsRollback + +@pytest.mark.skip_profile("spark_session", "apache_spark") +class TestSparkTableConstraintsRollback( + BaseSparkConstraintsRollbackSetup, BaseConstraintsRollback ): - pass + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": constraints_yml, + } + + # On Spark/Databricks, constraints are applied *after* the table is replaced. + # We don't have any way to "rollback" the table to its previous happy state. + # So the 'color' column will be updated to 'red', instead of 'blue'. + @pytest.fixture(scope="class") + def expected_color(self): + return "red" -class TestSailIncrementalConstraintsRollback( - BaseSailConstraintsRollbackSetup, BaseIncrementalConstraintsRollback +@pytest.mark.skip_profile("spark_session", "apache_spark") +class TestSparkIncrementalConstraintsRollback( + BaseSparkConstraintsRollbackSetup, BaseIncrementalConstraintsRollback ): - pass + # color stays blue for incremental models since it's a new row that just + # doesn't get inserted + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_incremental_model_sql, + "constraints_schema.yml": constraints_yml, + } -class TestSailModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): +# TODO: Like the tests above, this does test that model-level constraints don't +# result in errors, but it does not verify that they are actually present in +# Spark and that the ALTER TABLE statement actually ran. +@pytest.mark.skip_profile("spark_session", "apache_spark") +class TestSparkModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): @pytest.fixture(scope="class") def project_config_update(self): return { @@ -127,5 +385,13 @@ def project_config_update(self): } @pytest.fixture(scope="class") - def expected_error_messages(self): - return ["NOT NULL constraint violated"] + def models(self): + return { + "my_model.sql": my_model_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": foreign_key_model_sql, + "constraints_schema.yml": model_fk_constraint_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return _expected_sql_spark_model_constraints diff --git a/dbt-sail/tests/functional/adapter/test_get_columns_in_relation.py b/dbt-sail/tests/functional/adapter/test_get_columns_in_relation.py index d9e2269c7a..a037bb1cae 100644 --- a/dbt-sail/tests/functional/adapter/test_get_columns_in_relation.py +++ b/dbt-sail/tests/functional/adapter/test_get_columns_in_relation.py @@ -3,30 +3,31 @@ from dbt.tests.util import run_dbt, relation_from_name, check_relations_equal_with_relations +_MODEL_CHILD = "select 1" + + _MODEL_PARENT = """ -select 1 as id, 'hello' as msg -""" +{% set cols = adapter.get_columns_in_relation(ref('child')) %} -_MODEL_CHILD = """ -select * from {{ adapter.get_columns_in_relation(ref('parent')) }} -from {{ ref('parent') }} +select + {% for col in cols %} + {{ adapter.quote(col.column) }}{%- if not loop.last %},{{ '\n ' }}{% endif %} + {% endfor %} +from {{ ref('child') }} """ -class TestColumnsInRelationSail: +class TestColumnsInRelation: @pytest.fixture(scope="class") def models(self): return { - "parent.sql": _MODEL_PARENT, "child.sql": _MODEL_CHILD, + "parent.sql": _MODEL_PARENT, } + @pytest.mark.skip_profile("databricks_http_cluster", "spark_session") def test_get_columns_in_relation(self, project): run_dbt(["run"]) - check_relations_equal_with_relations( - project.adapter, - [ - relation_from_name(project.adapter, "parent"), - relation_from_name(project.adapter, "child"), - ], - ) + child = relation_from_name(project.adapter, "child") + parent = relation_from_name(project.adapter, "parent") + check_relations_equal_with_relations(project.adapter, [child, parent]) diff --git a/dbt-sail/tests/functional/adapter/test_grants.py b/dbt-sail/tests/functional/adapter/test_grants.py index f9484eb463..1b1a005ad1 100644 --- a/dbt-sail/tests/functional/adapter/test_grants.py +++ b/dbt-sail/tests/functional/adapter/test_grants.py @@ -6,14 +6,20 @@ from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants -class TestModelGrantsSail(BaseModelGrants): +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestModelGrantsSpark(BaseModelGrants): def privilege_grantee_name_overrides(self): + # insert --> modify return { "select": "select", + "insert": "modify", + "fake_privilege": "fake_privilege", + "invalid_user": "invalid_user", } -class TestIncrementalGrantsSail(BaseIncrementalGrants): +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestIncrementalGrantsSpark(BaseIncrementalGrants): @pytest.fixture(scope="class") def project_config_update(self): return { @@ -23,38 +29,32 @@ def project_config_update(self): } } - def privilege_grantee_name_overrides(self): - return { - "select": "select", - } - -class TestSeedGrantsSail(BaseSeedGrants): +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestSeedGrantsSpark(BaseSeedGrants): + # seeds in dbt-spark are currently "full refreshed," in such a way that + # the grants are not carried over + # see https://github.com/dbt-labs/dbt-spark/issues/388 def seeds_support_partial_refresh(self): return False -class TestSnapshotGrantsSail(BaseSnapshotGrants): +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestSnapshotGrantsSpark(BaseSnapshotGrants): @pytest.fixture(scope="class") def project_config_update(self): return { - "seeds": { - "+file_format": "delta", - }, "snapshots": { "+file_format": "delta", - }, - } - - def privilege_grantee_name_overrides(self): - return { - "select": "select", + "+incremental_strategy": "merge", + } } -class TestInvalidGrantsSail(BaseInvalidGrants): +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestInvalidGrantsSpark(BaseInvalidGrants): def grantee_does_not_exist_error(self): - return "Principal does not exist" + return "RESOURCE_DOES_NOT_EXIST" def privilege_does_not_exist_error(self): return "Action Unknown" diff --git a/dbt-sail/tests/functional/adapter/test_python_model.py b/dbt-sail/tests/functional/adapter/test_python_model.py index d94e5f36f3..004e2f63c4 100644 --- a/dbt-sail/tests/functional/adapter/test_python_model.py +++ b/dbt-sail/tests/functional/adapter/test_python_model.py @@ -1,4 +1,6 @@ +import os import pytest +from dbt.tests.util import run_dbt, write_file from dbt.tests.adapter.python_model.test_python_model import ( BasePythonModelTests, BasePythonIncrementalTests, @@ -6,13 +8,115 @@ from dbt.tests.adapter.python_model.test_spark import BasePySparkTests -class TestPythonModelSail(BasePythonModelTests): +@pytest.mark.skip_profile( + "apache_spark", "spark_session", "databricks_sql_endpoint", "spark_http_odbc" +) +class TestPythonModelSpark(BasePythonModelTests): pass -class TestPySparkSail(BasePySparkTests): - pass +@pytest.mark.skip_profile( + "apache_spark", "spark_session", "databricks_sql_endpoint", "spark_http_odbc" +) +class TestPySpark(BasePySparkTests): + def test_different_dataframes(self, project): + """ + Test that python models are supported using dataframes from: + - pandas + - pyspark + - pyspark.pandas (formerly dataspark.koalas) + Note: + The CI environment is on Apache Spark >3.1, which includes koalas as pyspark.pandas. + The only Databricks runtime that supports Apache Spark <=3.1 is 9.1 LTS, which is EOL 2024-09-23. + For more information, see: + - https://github.com/databricks/koalas + - https://docs.databricks.com/en/release-notes/runtime/index.html + """ + results = run_dbt(["run", "--exclude", "koalas_df"]) + assert len(results) == 3 -class TestPythonIncrementalModelSail(BasePythonIncrementalTests): - pass + +@pytest.mark.skip_profile( + "apache_spark", "spark_session", "databricks_sql_endpoint", "spark_http_odbc" +) +class TestPythonIncrementalModelSpark(BasePythonIncrementalTests): + @pytest.fixture(scope="class") + def project_config_update(self): + return {} + + +models__simple_python_model = """ +def model(dbt, spark): + dbt.config( + materialized='table', + submission_method='job_cluster', + job_cluster_config={ + "spark_version": "12.2.x-scala2.12", + "node_type_id": "i3.xlarge", + "num_workers": 0, + "spark_conf": { + "spark.databricks.cluster.profile": "singleNode", + "spark.master": "local[*, 4]" + }, + "custom_tags": { + "ResourceClass": "SingleNode" + } + }, + packages=['pydantic'] + ) + data = [[1,2]] * 10 + return spark.createDataFrame(data, schema=['test', 'test2']) +""" +models__simple_python_model_v2 = """ +import pandas + +def model(dbt, spark): + dbt.config( + materialized='table', + ) + data = [[1,2]] * 10 + return spark.createDataFrame(data, schema=['test1', 'test3']) +""" + + +@pytest.mark.skip_profile( + "apache_spark", + "spark_session", + "databricks_sql_endpoint", + "spark_http_odbc", + "databricks_http_cluster", +) +class TestChangingSchemaSpark: + """ + Confirm that we can setup a spot instance and parse required packages into the Databricks job. + + Notes: + - This test generates a spot instance on demand using the settings from `job_cluster_config` + in `models__simple_python_model` above. It takes several minutes to run due to creating the cluster. + The job can be monitored via "Data Engineering > Job Runs" or "Workflows > Job Runs" + in the Databricks UI (instead of via the normal cluster). + - The `spark_version` argument will need to periodically be updated. It will eventually become + unsupported and start experiencing issues. + - See https://github.com/explosion/spaCy/issues/12659 for why we're pinning pydantic + """ + + @pytest.fixture(scope="class") + def models(self): + return {"simple_python_model.py": models__simple_python_model} + + def test_changing_schema_with_log_validation(self, project, logs_dir): + run_dbt(["run"]) + write_file( + models__simple_python_model_v2, + project.project_root + "/models", + "simple_python_model.py", + ) + run_dbt(["run"]) + log_file = os.path.join(logs_dir, "dbt.log") + with open(log_file, "r") as f: + log = f.read() + # validate #5510 log_code_execution works + assert "On model.test.simple_python_model:" in log + assert "spark.createDataFrame(data, schema=['test1', 'test3'])" in log + assert "Execution status: OK in" in log diff --git a/dbt-sail/tests/functional/adapter/test_query_timeout.py b/dbt-sail/tests/functional/adapter/test_query_timeout.py new file mode 100644 index 0000000000..5311140424 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/test_query_timeout.py @@ -0,0 +1,76 @@ +"""Functional tests for query timeout and retry behavior. + +NOTE: These tests only work with PyHive-based connections (http/thrift methods). +ODBC connections use PyodbcConnectionWrapper which doesn't have timeout/retry support yet. +""" + +import pytest +from dbt.tests.util import run_dbt, run_dbt_and_capture +from dbt_common.exceptions import DbtRuntimeError + + +# Model that creates a delay to simulate a long-running query +# Uses a large cross join to create computational delay +long_running_model = """ +{{ config(materialized='table') }} +-- Generate a large dataset to create a delay +-- This creates roughly 1 million rows which should take several seconds +with numbers as ( + select explode(sequence(1, 100000)) as n +), +cross_product as ( + select a.n as n1, b.n as n2 + from numbers a + cross join numbers b +) +select count(*) as total_count +from cross_product +""" + +simple_model = """ +{{ config(materialized='table') }} +select 1 as id +""" + + +@pytest.mark.skip_profile( + "spark_http_odbc", + "databricks_cluster", + "databricks_sql_endpoint", + "databricks_http_cluster", + "spark_session", +) +class TestQueryTimeout: + """Test query timeout functionality. + + Skipped on ODBC profiles because PyodbcConnectionWrapper doesn't use + the async polling mechanism with timeout support. + """ + + @pytest.fixture(scope="class") + def models(self): + return { + "long_running.sql": long_running_model, + "simple.sql": simple_model, + } + + @pytest.fixture(scope="class") + def dbt_profile_target(self, dbt_profile_target): + """Override profile to add timeout configuration.""" + dbt_profile_target["query_timeout"] = 1 # 1 second timeout + dbt_profile_target["poll_interval"] = 1 # Poll every second for faster test + dbt_profile_target["query_retries"] = 0 # Disable retries for clearer errors + return dbt_profile_target + + def test_query_timeout_exceeded(self, project): + """Test that queries exceeding timeout raise appropriate error.""" + + # Simple model should succeed (runs quickly) + results = run_dbt(["run", "--select", "simple"]) + assert results[0].status == "success" + + # Long-running model should timeout + # The cross-join query should take longer than 2 seconds on most systems + _, output = run_dbt_and_capture(["run", "--select", "long_running"], expect_pass=False) + + assert "exceeded timeout" in output.lower() diff --git a/dbt-sail/tests/functional/adapter/test_simple_seed.py b/dbt-sail/tests/functional/adapter/test_simple_seed.py index a706ff73a8..c610967c69 100644 --- a/dbt-sail/tests/functional/adapter/test_simple_seed.py +++ b/dbt-sail/tests/functional/adapter/test_simple_seed.py @@ -1,5 +1,5 @@ from dbt.tests.adapter.simple_seed.test_seed import BaseTestEmptySeed -class TestEmptySeedSail(BaseTestEmptySeed): +class TestBigQueryEmptySeed(BaseTestEmptySeed): pass diff --git a/dbt-sail/tests/functional/adapter/test_store_test_failures.py b/dbt-sail/tests/functional/adapter/test_store_test_failures.py index db1d4c758b..4723364052 100644 --- a/dbt-sail/tests/functional/adapter/test_store_test_failures.py +++ b/dbt-sail/tests/functional/adapter/test_store_test_failures.py @@ -3,19 +3,99 @@ from dbt.tests.adapter.store_test_failures_tests import basic from dbt.tests.adapter.store_test_failures_tests.test_store_test_failures import ( StoreTestFailuresBase, + TEST_AUDIT_SCHEMA_SUFFIX, ) -class TestSailStoreTestFailures(StoreTestFailuresBase): +@pytest.mark.skip_profile( + "spark_session", "databricks_cluster", "databricks_sql_endpoint", "spark_http_odbc" +) +class TestSparkStoreTestFailures(StoreTestFailuresBase): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "quote_columns": True, + }, + "tests": {"+schema": TEST_AUDIT_SCHEMA_SUFFIX, "+store_failures": True}, + } + @pytest.fixture(scope="function", autouse=True) def teardown_method(self, project): yield with project.adapter.connection_named("__test"): relation = project.adapter.Relation.create( - database=project.database, schema=project.test_schema + database=project.database, + schema=f"{project.test_schema}_{TEST_AUDIT_SCHEMA_SUFFIX}", ) + project.adapter.drop_schema(relation) def test_store_and_assert(self, project): self.run_tests_store_one_failure(project) self.run_tests_store_failures_and_assert(project) + + +@pytest.mark.skip_profile( + "apache_spark", "spark_session", "databricks_sql_endpoint", "spark_http_odbc" +) +class TestSparkStoreTestFailuresWithDelta(StoreTestFailuresBase): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "quote_columns": False, + "test": self.column_type_overrides(), + "+file_format": "delta", + }, + "tests": { + "+schema": TEST_AUDIT_SCHEMA_SUFFIX, + "+store_failures": True, + "+file_format": "delta", + }, + } + + @pytest.fixture(scope="function", autouse=True) + def teardown_method(self, project): + yield + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, + schema=f"{project.test_schema}_{TEST_AUDIT_SCHEMA_SUFFIX}", + ) + + project.adapter.drop_schema(relation) + + def test_store_and_assert_failure_with_delta(self, project): + self.run_tests_store_one_failure(project) + self.run_tests_store_failures_and_assert(project) + + +@pytest.mark.skip_profile("spark_session") +class TestStoreTestFailuresAsInteractions(basic.StoreTestFailuresAsInteractions): + pass + + +@pytest.mark.skip_profile("spark_session") +class TestStoreTestFailuresAsProjectLevelOff(basic.StoreTestFailuresAsProjectLevelOff): + pass + + +@pytest.mark.skip_profile("spark_session") +class TestStoreTestFailuresAsProjectLevelView(basic.StoreTestFailuresAsProjectLevelView): + pass + + +@pytest.mark.skip_profile("spark_session") +class TestStoreTestFailuresAsGeneric(basic.StoreTestFailuresAsGeneric): + pass + + +@pytest.mark.skip_profile("spark_session") +class TestStoreTestFailuresAsProjectLevelEphemeral(basic.StoreTestFailuresAsProjectLevelEphemeral): + pass + + +@pytest.mark.skip_profile("spark_session") +class TestStoreTestFailuresAsExceptions(basic.StoreTestFailuresAsExceptions): + pass diff --git a/dbt-sail/tests/functional/adapter/unit_testing/test_unit_testing.py b/dbt-sail/tests/functional/adapter/unit_testing/test_unit_testing.py index ce1a692904..b70c581d11 100644 --- a/dbt-sail/tests/functional/adapter/unit_testing/test_unit_testing.py +++ b/dbt-sail/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -5,7 +5,7 @@ from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput -class TestSailUnitTestingTypes(BaseUnitTestingTypes): +class TestSparkUnitTestingTypes(BaseUnitTestingTypes): @pytest.fixture def data_types(self): # sql_value, yaml_value @@ -26,9 +26,9 @@ def data_types(self): ] -class TestSailUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): +class TestSparkUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): pass -class TestSailUnitTestInvalidInput(BaseUnitTestInvalidInput): +class TestSparkUnitTestInvalidInput(BaseUnitTestInvalidInput): pass diff --git a/dbt-sail/tests/functional/adapter/utils/test_data_types.py b/dbt-sail/tests/functional/adapter/utils/test_data_types.py index ad0bd45a74..8ca38ab1e9 100644 --- a/dbt-sail/tests/functional/adapter/utils/test_data_types.py +++ b/dbt-sail/tests/functional/adapter/utils/test_data_types.py @@ -14,10 +14,12 @@ from dbt.tests.adapter.utils.data_types.test_type_boolean import BaseTypeBoolean -class TestTypeBigIntSail(BaseTypeBigInt): +class TestTypeBigInt(BaseTypeBigInt): pass +# need to explicitly cast this to avoid it being inferred/loaded as a DOUBLE on Spark +# in SparkSQL, the two are equivalent for `=` comparison, but distinct for EXCEPT comparison seeds__float_expected_yml = """ version: 2 seeds: @@ -28,7 +30,7 @@ class TestTypeBigIntSail(BaseTypeBigInt): """ -class TestTypeFloatSail(BaseTypeFloat): +class TestTypeFloat(BaseTypeFloat): @pytest.fixture(scope="class") def seeds(self): return { @@ -37,6 +39,7 @@ def seeds(self): } +# need to explicitly cast this to avoid it being inferred/loaded as a BIGINT on Spark seeds__int_expected_yml = """ version: 2 seeds: @@ -47,7 +50,7 @@ def seeds(self): """ -class TestTypeIntSail(BaseTypeInt): +class TestTypeInt(BaseTypeInt): @pytest.fixture(scope="class") def seeds(self): return { @@ -56,18 +59,18 @@ def seeds(self): } -class TestTypeNumericSail(BaseTypeNumeric): +class TestTypeNumeric(BaseTypeNumeric): def numeric_fixture_type(self): return "decimal(28,6)" -class TestTypeStringSail(BaseTypeString): +class TestTypeString(BaseTypeString): pass -class TestTypeTimestampSail(BaseTypeTimestamp): +class TestTypeTimestamp(BaseTypeTimestamp): pass -class TestTypeBooleanSail(BaseTypeBoolean): +class TestTypeBoolean(BaseTypeBoolean): pass diff --git a/dbt-sail/tests/functional/adapter/utils/test_timestamps.py b/dbt-sail/tests/functional/adapter/utils/test_timestamps.py index 4656552977..d05d23997d 100644 --- a/dbt-sail/tests/functional/adapter/utils/test_timestamps.py +++ b/dbt-sail/tests/functional/adapter/utils/test_timestamps.py @@ -2,7 +2,7 @@ from dbt.tests.adapter.utils.test_timestamps import BaseCurrentTimestamps -class TestCurrentTimestampSail(BaseCurrentTimestamps): +class TestCurrentTimestampSpark(BaseCurrentTimestamps): @pytest.fixture(scope="class") def models(self): return { diff --git a/dbt-sail/tests/functional/adapter/utils/test_utils.py b/dbt-sail/tests/functional/adapter/utils/test_utils.py index 1e68522491..db7a33d575 100644 --- a/dbt-sail/tests/functional/adapter/utils/test_utils.py +++ b/dbt-sail/tests/functional/adapter/utils/test_utils.py @@ -24,8 +24,11 @@ from dbt.tests.adapter.utils.test_replace import BaseReplace from dbt.tests.adapter.utils.test_right import BaseRight from dbt.tests.adapter.utils.test_safe_cast import BaseSafeCast + from dbt.tests.adapter.utils.test_split_part import BaseSplitPart from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral + +# requires modification from dbt.tests.adapter.utils.test_listagg import BaseListagg from dbt.tests.adapter.utils.fixture_listagg import models__test_listagg_yml from tests.functional.adapter.utils.fixture_listagg import models__test_listagg_no_order_by_sql @@ -43,89 +46,100 @@ """ -class TestAnyValueSail(BaseAnyValue): +# skipped: ,month, + + +class TestAnyValue(BaseAnyValue): pass -class TestArrayAppendSail(BaseArrayAppend): +class TestArrayAppend(BaseArrayAppend): pass -class TestArrayConcatSail(BaseArrayConcat): +class TestArrayConcat(BaseArrayConcat): pass -class TestArrayConstructSail(BaseArrayConstruct): +class TestArrayConstruct(BaseArrayConstruct): pass -class TestBoolOrSail(BaseBoolOr): +class TestBoolOr(BaseBoolOr): pass -class TestCastSail(BaseCast): +class TestCast(BaseCast): pass -class TestCastBoolToTextSail(BaseCastBoolToText): +class TestCastBoolToText(BaseCastBoolToText): pass -class TestConcatSail(BaseConcat): +@pytest.mark.skip_profile("spark_session") +class TestConcat(BaseConcat): pass -class TestCurrentTimestampSail(BaseCurrentTimestampNaive): +# Use either BaseCurrentTimestampAware or BaseCurrentTimestampNaive but not both +class TestCurrentTimestamp(BaseCurrentTimestampNaive): pass -class TestDateSail(BaseDate): +class TestDate(BaseDate): pass -class TestDateAddSail(BaseDateAdd): +class TestDateAdd(BaseDateAdd): pass -class TestDateDiffSail(BaseDateDiff): +# this generates too much SQL to run successfully in our testing environments :( +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestDateDiff(BaseDateDiff): pass -class TestDateTruncSail(BaseDateTrunc): +class TestDateTrunc(BaseDateTrunc): pass -class TestEqualsSail(BaseEquals): +class TestEquals(BaseEquals): pass -class TestEscapeSingleQuotesSail(BaseEscapeSingleQuotesBackslash): +class TestEscapeSingleQuotes(BaseEscapeSingleQuotesBackslash): pass -class TestExceptSail(BaseExcept): +class TestExcept(BaseExcept): pass -class TestHashSail(BaseHash): +@pytest.mark.skip_profile("spark_session") +class TestHash(BaseHash): pass -class TestIntersectSail(BaseIntersect): +class TestIntersect(BaseIntersect): pass -class TestLastDaySail(BaseLastDay): +@pytest.mark.skip_profile("spark_session") # spark session crashes in CI +class TestLastDay(BaseLastDay): @pytest.fixture(scope="class") def seeds(self): return {"data_last_day.csv": seeds__data_last_day_csv} -class TestLengthSail(BaseLength): +class TestLength(BaseLength): pass -class TestListaggSail(BaseListagg): +# SparkSQL does not support 'order by' for its 'listagg' equivalent +# the argument is ignored, so let's ignore those fields when checking equivalency +class TestListagg(BaseListagg): @pytest.fixture(scope="class") def models(self): return { @@ -136,27 +150,29 @@ def models(self): } -class TestPositionSail(BasePosition): +class TestPosition(BasePosition): pass -class TestReplaceSail(BaseReplace): +@pytest.mark.skip_profile("spark_session") +class TestReplace(BaseReplace): pass -class TestRightSail(BaseRight): +@pytest.mark.skip_profile("spark_session") +class TestRight(BaseRight): pass -class TestSafeCastSail(BaseSafeCast): +class TestSafeCast(BaseSafeCast): pass -class TestSplitPartSail(BaseSplitPart): +class TestSplitPart(BaseSplitPart): @pytest.fixture(scope="class") def seeds(self): return {"data_split_part.csv": seeds__data_split_part_csv} -class TestStringLiteralSail(BaseStringLiteral): +class TestStringLiteral(BaseStringLiteral): pass diff --git a/dbt-spark/docker/spark.Dockerfile b/dbt-spark/docker/spark.Dockerfile index 3e12962050..9685f9d86d 100644 --- a/dbt-spark/docker/spark.Dockerfile +++ b/dbt-spark/docker/spark.Dockerfile @@ -1,4 +1,4 @@ -ARG OPENJDK_VERSION=8 +ARG OPENJDK_VERSION=17 FROM eclipse-temurin:${OPENJDK_VERSION}-jre ARG BUILD_DATE From 696cb188dc0cc5afab0456df634c1d07ced703ac Mon Sep 17 00:00:00 2001 From: pomykalakyle Date: Mon, 6 Apr 2026 09:56:00 -0700 Subject: [PATCH 3/6] Refine dbt-sail test scope and spark session tooling Prune unsupported Sail adapter tests, capture the spark session debug report, and add shard-aware spark session runners so the branch reflects the intended session-focused coverage. --- dbt-sail/src/dbt/adapters/sail/__init__.py | 1 - dbt-sail/src/dbt/adapters/sail/connections.py | 12 +- .../SPARK_SESSION_TEST_DEBUG_REPORT.md | 121 ++++ .../adapter/dbt_clone/test_dbt_clone.py | 80 --- .../adapter/dbt_show/test_dbt_show.py | 10 - .../test_incremental_merge_exclude_columns.py | 12 - .../test_incremental_on_schema_change.py | 52 -- .../test_incremental_predicates.py | 65 --- .../incremental/test_incremental_unique_id.py | 14 - .../test_incremental_strategies.py | 105 ---- .../incremental_strategies/test_microbatch.py | 21 - .../adapter/persist_docs/test_persist_docs.py | 163 ------ .../test_seed_column_types.py | 25 - .../tests/functional/adapter/test_basic.py | 47 +- .../functional/adapter/test_constraints.py | 397 ------------- .../adapter/test_get_columns_in_relation.py | 33 -- .../tests/functional/adapter/test_grants.py | 60 -- .../functional/adapter/test_python_model.py | 122 ---- .../functional/adapter/test_query_timeout.py | 76 --- .../adapter/test_store_test_failures.py | 101 ---- .../functional/adapter/utils/test_utils.py | 45 -- dbt-sail/tests/unit/test_adapter.py | 4 +- .../dagger/audit_spark_session_test_speeds.py | 212 +++++++ dbt-spark/dagger/run_dbt_spark_tests.py | 38 +- .../dagger/run_dbt_spark_tests_sharded.py | 530 +++++++++++++++++ .../dagger/run_dbt_spark_tests_tiered.py | 439 ++++++++++++++ .../test_lists/spark_session_errored.txt | 1 + .../dagger/test_lists/spark_session_fast.txt | 44 ++ .../spark_session_fast_tail_grouped.txt | 3 + .../spark_session_slow_but_working.txt | 3 + .../test_lists/spark_session_timed_out.txt | 7 + dbt-spark/hatch.toml | 16 +- dbt-spark/test_results.txt | 539 ++++++++++++++++++ .../tests/functional/fixtures/profiles.py | 1 + 34 files changed, 1960 insertions(+), 1439 deletions(-) create mode 100644 dbt-sail/tests/functional/adapter/SPARK_SESSION_TEST_DEBUG_REPORT.md delete mode 100644 dbt-sail/tests/functional/adapter/dbt_clone/test_dbt_clone.py delete mode 100644 dbt-sail/tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py delete mode 100644 dbt-sail/tests/functional/adapter/incremental/test_incremental_predicates.py delete mode 100644 dbt-sail/tests/functional/adapter/incremental/test_incremental_unique_id.py delete mode 100644 dbt-sail/tests/functional/adapter/incremental_strategies/test_microbatch.py delete mode 100644 dbt-sail/tests/functional/adapter/persist_docs/test_persist_docs.py delete mode 100644 dbt-sail/tests/functional/adapter/seed_column_types/test_seed_column_types.py delete mode 100644 dbt-sail/tests/functional/adapter/test_constraints.py delete mode 100644 dbt-sail/tests/functional/adapter/test_get_columns_in_relation.py delete mode 100644 dbt-sail/tests/functional/adapter/test_grants.py delete mode 100644 dbt-sail/tests/functional/adapter/test_python_model.py delete mode 100644 dbt-sail/tests/functional/adapter/test_query_timeout.py delete mode 100644 dbt-sail/tests/functional/adapter/test_store_test_failures.py create mode 100644 dbt-spark/dagger/audit_spark_session_test_speeds.py create mode 100644 dbt-spark/dagger/run_dbt_spark_tests_sharded.py create mode 100644 dbt-spark/dagger/run_dbt_spark_tests_tiered.py create mode 100644 dbt-spark/dagger/test_lists/spark_session_errored.txt create mode 100644 dbt-spark/dagger/test_lists/spark_session_fast.txt create mode 100644 dbt-spark/dagger/test_lists/spark_session_fast_tail_grouped.txt create mode 100644 dbt-spark/dagger/test_lists/spark_session_slow_but_working.txt create mode 100644 dbt-spark/dagger/test_lists/spark_session_timed_out.txt create mode 100644 dbt-spark/test_results.txt diff --git a/dbt-sail/src/dbt/adapters/sail/__init__.py b/dbt-sail/src/dbt/adapters/sail/__init__.py index 0518dece62..4004a82040 100644 --- a/dbt-sail/src/dbt/adapters/sail/__init__.py +++ b/dbt-sail/src/dbt/adapters/sail/__init__.py @@ -10,4 +10,3 @@ include_path=sail.PACKAGE_PATH, dependencies=["spark"], ) - diff --git a/dbt-sail/src/dbt/adapters/sail/connections.py b/dbt-sail/src/dbt/adapters/sail/connections.py index d740ec7714..e941297f13 100644 --- a/dbt-sail/src/dbt/adapters/sail/connections.py +++ b/dbt-sail/src/dbt/adapters/sail/connections.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from typing import Any, Dict, Optional +from typing import Any, Dict from dbt.adapters.contracts.connection import Connection, ConnectionState from dbt.adapters.events.logging import AdapterLogger @@ -95,7 +95,10 @@ def open(cls, connection: Connection) -> Connection: def _open_embedded(cls, creds: SailCredentials) -> Any: """Start a SparkConnectServer in-process via pysail, then connect PySpark to it.""" from pysail.spark import SparkConnectServer - from dbt.adapters.spark.session import SessionConnectionWrapper, Connection as SessionConnection + from dbt.adapters.spark.session import ( + SessionConnectionWrapper, + Connection as SessionConnection, + ) if cls._server is None or not cls._server.running: port = creds.port if creds.port != 443 else 0 @@ -113,7 +116,10 @@ def _open_embedded(cls, creds: SailCredentials) -> Any: @classmethod def _open_remote(cls, creds: SailCredentials) -> Any: """Connect PySpark to an already-running Sail server via Spark Connect.""" - from dbt.adapters.spark.session import SessionConnectionWrapper, Connection as SessionConnection + from dbt.adapters.spark.session import ( + SessionConnectionWrapper, + Connection as SessionConnection, + ) port = creds.port if creds.port != 443 else 50051 remote_url = f"sc://{creds.host}:{port}" diff --git a/dbt-sail/tests/functional/adapter/SPARK_SESSION_TEST_DEBUG_REPORT.md b/dbt-sail/tests/functional/adapter/SPARK_SESSION_TEST_DEBUG_REPORT.md new file mode 100644 index 0000000000..cd59683343 --- /dev/null +++ b/dbt-sail/tests/functional/adapter/SPARK_SESSION_TEST_DEBUG_REPORT.md @@ -0,0 +1,121 @@ +# Spark-Session Pruned Test Debug Report (dbt-sail) + +## Summary +- Total tests: **52** +- Passed: **21** +- Failed/Error: **30** +- Skipped: **1** + +## Failure Categories +- `exit-state mismatch`: 16 +- `sql/parser incompatibility`: 6 +- `type mapping mismatch`: 5 +- `error-message mismatch`: 1 +- `state/cleanup issue`: 1 +- `time precision/tolerance mismatch`: 1 + +## Error Signature Legend +| ID | Count | Category | What's Happening | Example | +|---|---:|---|---|---| +| `E01` | 16 | exit-state mismatch | dbt command result differs from test expectation. | AssertionError: dbt exit state did not match expected self = project = args = [], expected = 5 @pytest.mark.parametrize( "args,expected", [ ([], 5), # default limit (["--limit", 3 | +| `E02` | 1 | state/cleanup issue | test expects clean relation state but object already exists. | AssertionError: assert 'Compilation Error' in 'Runtime Error in model incremental_fail (models/incremental_fail.sql)\n Runtime Error\n table already exists: incremental_fail' + where 'Runtime Error in model incremental_fail (models/incremental_fail.sql)\n Runtime Error\n table already exists: increm | +| `E03` | 1 | error-message mismatch | same logical failure class, but assertion expects different text. | assert "Invalid column name: 'invalid_column_name' in unit test fixture for 'my_upstream_model'." in '\x1b[0m03:44:22 Running with dbt=1.11.7\n\x1b[0m03:44:22 Registered adapter: sail=0.1.0\n\x1b[0m03:44:22 Found 2 m...odel (models/unit_tests.yml)\n\x1b[0m03:44:22 \n\x1b[0m03:44:22 Done. PASS=0 WARN | +| `E04` | 6 | sql/parser incompatibility | SQL generated by shared test macro is rejected by Sail parser. | dbt_common.exceptions.base.DbtRuntimeError: Runtime Error invalid argument: found . at 278:279 expected ',', 'FROM', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', or ')' self = project = def test_check_types_assert_match(self, project): run_dbt(["build"]) # check contents | +| `E06` | 1 | time precision/tolerance mismatch | timestamp precision/timezone offset differs from test expectation. | AssertionError: SQL timestamp 2026-04-03T20:44:25.633143 is not close enough to Python UTC 2026-04-04T03:44:25.634810 assert (datetime.datetime(2026, 4, 3, 20, 44, 25, 633143) > (datetime.datetime(2026, 4, 4, 3, 44, 25, 634810) - datetime.timedelta(seconds=300))) self = str: - return _microbatch_model_no_unique_id_sql diff --git a/dbt-sail/tests/functional/adapter/persist_docs/test_persist_docs.py b/dbt-sail/tests/functional/adapter/persist_docs/test_persist_docs.py deleted file mode 100644 index f6bf34b5c9..0000000000 --- a/dbt-sail/tests/functional/adapter/persist_docs/test_persist_docs.py +++ /dev/null @@ -1,163 +0,0 @@ -import pytest - -from dbt.tests.util import run_dbt, run_dbt_and_capture - -from fixtures import ( - _MODELS__MY_FUN_DOCS, - _MODELS__INCREMENTAL_DELTA, - _MODELS__TABLE_DELTA_MODEL, - _MODELS__TABLE_DELTA_MODEL_MISSING_COLUMN, - _PROPERTIES__MODELS, - _PROPERTIES__MISSING_COLUMN, - _PROPERTIES__SEEDS, - _SEEDS__BASIC, - _MODELS__VIEW_DELTA_MODEL, - _VIEW_PROPERTIES_MODELS, -) - - -@pytest.mark.skip_profile("apache_spark", "spark_session") -class TestPersistDocsDeltaTable: - @pytest.fixture(scope="class") - def models(self): - return { - "incremental_delta_model.sql": _MODELS__INCREMENTAL_DELTA, - "my_fun_docs.md": _MODELS__MY_FUN_DOCS, - "table_delta_model.sql": _MODELS__TABLE_DELTA_MODEL, - "schema.yml": _PROPERTIES__MODELS, - } - - @pytest.fixture(scope="class") - def seeds(self): - return {"seed.csv": _SEEDS__BASIC, "seed.yml": _PROPERTIES__SEEDS} - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "test": { - "+persist_docs": { - "relation": True, - "columns": True, - }, - } - }, - "seeds": { - "test": { - "+persist_docs": { - "relation": True, - "columns": True, - }, - "+file_format": "delta", - "+quote_columns": True, - } - }, - } - - def test_delta_comments(self, project): - run_dbt(["seed"]) - run_dbt(["run"]) - - for table, whatis in [ - ("table_delta_model", "Table"), - ("seed", "Seed"), - ("incremental_delta_model", "Incremental"), - ]: - results = project.run_sql( - "describe extended {schema}.{table}".format( - schema=project.test_schema, table=table - ), - fetch="all", - ) - - for result in results: - if result[0] == "Comment": - assert result[1].startswith(f"{whatis} model description") - if result[0] == "id": - assert result[2].startswith("id Column description") - if result[0] == "name": - assert result[2].startswith("Some stuff here and then a call to") - - -@pytest.mark.skip_profile("apache_spark", "spark_session") -class TestPersistDocsDeltaView: - @pytest.fixture(scope="class") - def models(self): - return { - "table_delta_model.sql": _MODELS__TABLE_DELTA_MODEL, - "view_delta_model.sql": _MODELS__VIEW_DELTA_MODEL, - "schema.yml": _VIEW_PROPERTIES_MODELS, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "test": { - "+persist_docs": { - "relation": True, - "columns": True, - }, - } - }, - } - - def test_delta_comments(self, project): - run_dbt(["run"]) - - results = project.run_sql( - "describe extended {schema}.{table}".format( - schema=project.test_schema, table="view_delta_model" - ), - fetch="all", - ) - - for result in results: - if result[0] == "Comment": - assert result[1].startswith("View model description") - if result[0] == "id": - assert result[2].startswith("id Column description") - if result[0] == "count": - assert result[2] is None - - -@pytest.mark.skip_profile("apache_spark", "spark_session") -class TestPersistDocsMissingColumn: - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "test": { - "+persist_docs": { - "columns": True, - }, - } - } - } - - @pytest.fixture(scope="class") - def seeds(self): - return {"seed.csv": _SEEDS__BASIC, "seed.yml": _PROPERTIES__SEEDS} - - @pytest.fixture(scope="class") - def models(self): - return { - "table_delta_model.sql": _MODELS__TABLE_DELTA_MODEL_MISSING_COLUMN, - "my_fun_docs.md": _MODELS__MY_FUN_DOCS, - } - - @pytest.fixture(scope="class") - def properties(self): - return {"schema.yml": _PROPERTIES__MISSING_COLUMN} - - def test_missing_column(self, project): - """ - With column filtering in alter_column_comment, non-existent columns - are now skipped instead of causing DB errors, and a warning is emitted. - """ - run_dbt(["seed"]) - _, logs = run_dbt_and_capture(["run"]) - assert ( - "The following columns are specified in the schema but are not present in the database: column_that_does_not_exist" - in logs - ) diff --git a/dbt-sail/tests/functional/adapter/seed_column_types/test_seed_column_types.py b/dbt-sail/tests/functional/adapter/seed_column_types/test_seed_column_types.py deleted file mode 100644 index 3326490f97..0000000000 --- a/dbt-sail/tests/functional/adapter/seed_column_types/test_seed_column_types.py +++ /dev/null @@ -1,25 +0,0 @@ -import pytest -from dbt.tests.util import run_dbt -from tests.functional.adapter.seed_column_types.fixtures import ( - _MACRO_TEST_IS_TYPE_SQL, - _SEED_CSV, - _SEED_YML, -) - - -@pytest.mark.skip_profile("spark_session") -class TestSeedColumnTypesCast: - @pytest.fixture(scope="class") - def macros(self): - return {"test_is_type.sql": _MACRO_TEST_IS_TYPE_SQL} - - @pytest.fixture(scope="class") - def seeds(self): - return {"payments.csv": _SEED_CSV, "schema.yml": _SEED_YML} - - # We want to test seed types because hive would cause all fields to be strings. - # setting column_types in project.yml should change them and pass. - def test_column_seed_type(self, project): - results = run_dbt(["seed"]) - assert len(results) == 1 - run_dbt(["test"], expect_pass=False) diff --git a/dbt-sail/tests/functional/adapter/test_basic.py b/dbt-sail/tests/functional/adapter/test_basic.py index 072d211d63..3c3fcde6c1 100644 --- a/dbt-sail/tests/functional/adapter/test_basic.py +++ b/dbt-sail/tests/functional/adapter/test_basic.py @@ -1,5 +1,4 @@ -import pytest - +from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests from dbt.tests.adapter.basic.test_singular_tests_ephemeral import ( @@ -7,14 +6,12 @@ ) from dbt.tests.adapter.basic.test_empty import BaseEmpty from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral -from dbt.tests.adapter.basic.test_incremental import BaseIncremental from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests +from dbt.tests.adapter.basic.test_incremental import BaseIncremental from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp -from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod -@pytest.mark.skip_profile("spark_session") class TestSimpleMaterializationsSpark(BaseSimpleMaterializations): pass @@ -25,7 +22,6 @@ class TestSingularTestsSpark(BaseSingularTests): # The local cluster currently tests on spark 2.x, which does not support this # if we upgrade it to 3.x, we can enable this test -@pytest.mark.skip_profile("apache_spark") class TestSingularTestsEphemeralSpark(BaseSingularTestsEphemeral): pass @@ -34,12 +30,10 @@ class TestEmptySpark(BaseEmpty): pass -@pytest.mark.skip_profile("spark_session") class TestEphemeralSpark(BaseEphemeral): pass -@pytest.mark.skip_profile("spark_session") class TestIncrementalSpark(BaseIncremental): pass @@ -48,38 +42,13 @@ class TestGenericTestsSpark(BaseGenericTests): pass -# These tests were not enabled in the dbtspec files, so skipping here. -# Error encountered was: Error running query: java.lang.ClassNotFoundException: delta.DefaultSource -@pytest.mark.skip_profile("apache_spark", "spark_session") class TestSnapshotCheckColsSpark(BaseSnapshotCheckCols): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "seeds": { - "+file_format": "delta", - }, - "snapshots": { - "+file_format": "delta", - }, - } - - -# These tests were not enabled in the dbtspec files, so skipping here. -# Error encountered was: Error running query: java.lang.ClassNotFoundException: delta.DefaultSource -@pytest.mark.skip_profile("apache_spark", "spark_session") + pass + + class TestSnapshotTimestampSpark(BaseSnapshotTimestamp): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "seeds": { - "+file_format": "delta", - }, - "snapshots": { - "+file_format": "delta", - }, - } - - -@pytest.mark.skip_profile("spark_session") + pass + + class TestBaseAdapterMethod(BaseAdapterMethod): pass diff --git a/dbt-sail/tests/functional/adapter/test_constraints.py b/dbt-sail/tests/functional/adapter/test_constraints.py deleted file mode 100644 index f33359262c..0000000000 --- a/dbt-sail/tests/functional/adapter/test_constraints.py +++ /dev/null @@ -1,397 +0,0 @@ -import pytest -from dbt.tests.adapter.constraints.test_constraints import ( - BaseModelConstraintsRuntimeEnforcement, - BaseTableConstraintsColumnsEqual, - BaseViewConstraintsColumnsEqual, - BaseIncrementalConstraintsColumnsEqual, - BaseConstraintsRuntimeDdlEnforcement, - BaseConstraintsRollback, - BaseIncrementalConstraintsRuntimeDdlEnforcement, - BaseIncrementalConstraintsRollback, - BaseConstraintQuotedColumn, -) -from dbt.tests.adapter.constraints.fixtures import ( - constrained_model_schema_yml, - my_model_sql, - my_model_wrong_order_sql, - my_model_wrong_name_sql, - model_schema_yml, - my_model_view_wrong_order_sql, - my_model_view_wrong_name_sql, - my_model_incremental_wrong_order_sql, - my_model_incremental_wrong_name_sql, - my_incremental_model_sql, - model_fk_constraint_schema_yml, - my_model_wrong_order_depends_on_fk_sql, - foreign_key_model_sql, - my_model_incremental_wrong_order_depends_on_fk_sql, - my_model_with_quoted_column_name_sql, - model_quoted_column_schema_yml, -) - -# constraints are enforced via 'alter' statements that run after table creation -_expected_sql_spark = """ -create or replace table - using delta - as -select - id, - color, - date_day -from - -( - -- depends_on: - select - 'blue' as color, - 1 as id, - '2019-01-01' as date_day ) as model_subq -""" - -_expected_sql_spark_model_constraints = """ -create or replace table - using delta - as -select - id, - color, - date_day -from - -( - -- depends_on: - select - 'blue' as color, - 1 as id, - '2019-01-01' as date_day ) as model_subq -""" - -# Different on Spark: -# - does not support a data type named 'text' (TODO handle this in the base test classes using string_type -constraints_yml = model_schema_yml.replace("text", "string").replace("primary key", "") -model_fk_constraint_schema_yml = model_fk_constraint_schema_yml.replace("text", "string").replace( - "primary key", "" -) -model_constraints_yml = constrained_model_schema_yml.replace("text", "string") - - -class PyodbcSetup: - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "+file_format": "delta", - } - } - - @pytest.fixture - def string_type(self): - return "STR" - - @pytest.fixture - def int_type(self): - return "INT" - - @pytest.fixture - def schema_string_type(self): - return "STRING" - - @pytest.fixture - def schema_int_type(self): - return "INT" - - @pytest.fixture - def data_types(self, int_type, schema_int_type, string_type, schema_string_type): - # sql_column_value, schema_data_type, error_data_type - return [ - ["1", schema_int_type, int_type], - ['"1"', schema_string_type, string_type], - ["true", "boolean", "BOOL"], - ['array("1","2","3")', "string", string_type], - ["array(1,2,3)", "string", string_type], - ["6.45", "decimal", "DECIMAL"], - ["cast('2019-01-01' as date)", "date", "DATE"], - ["cast('2019-01-01' as timestamp)", "timestamp", "DATETIME"], - ] - - -class DatabricksHTTPSetup: - @pytest.fixture - def string_type(self): - return "STRING_TYPE" - - @pytest.fixture - def int_type(self): - return "INT_TYPE" - - @pytest.fixture - def schema_string_type(self): - return "STRING" - - @pytest.fixture - def schema_int_type(self): - return "INT" - - @pytest.fixture - def data_types(self, int_type, schema_int_type, string_type, schema_string_type): - # sql_column_value, schema_data_type, error_data_type - return [ - ["1", schema_int_type, int_type], - ['"1"', schema_string_type, string_type], - ["true", "boolean", "BOOLEAN_TYPE"], - ['array("1","2","3")', "array", "ARRAY_TYPE"], - ["array(1,2,3)", "array", "ARRAY_TYPE"], - ["cast('2019-01-01' as date)", "date", "DATE_TYPE"], - ["cast('2019-01-01' as timestamp)", "timestamp", "TIMESTAMP_TYPE"], - ["cast(1.0 AS DECIMAL(4, 2))", "decimal", "DECIMAL_TYPE"], - ] - - -@pytest.mark.skip_profile("spark_session", "apache_spark", "databricks_http_cluster") -class TestSparkTableConstraintsColumnsEqualPyodbc(PyodbcSetup, BaseTableConstraintsColumnsEqual): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_wrong_order.sql": my_model_wrong_order_sql, - "my_model_wrong_name.sql": my_model_wrong_name_sql, - "constraints_schema.yml": constraints_yml, - } - - -@pytest.mark.skip_profile("spark_session", "apache_spark", "databricks_http_cluster") -class TestSparkViewConstraintsColumnsEqualPyodbc(PyodbcSetup, BaseViewConstraintsColumnsEqual): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_wrong_order.sql": my_model_view_wrong_order_sql, - "my_model_wrong_name.sql": my_model_view_wrong_name_sql, - "constraints_schema.yml": constraints_yml, - } - - -@pytest.mark.skip_profile("spark_session", "apache_spark", "databricks_http_cluster") -class TestSparkIncrementalConstraintsColumnsEqualPyodbc( - PyodbcSetup, BaseIncrementalConstraintsColumnsEqual -): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_wrong_order.sql": my_model_incremental_wrong_order_sql, - "my_model_wrong_name.sql": my_model_incremental_wrong_name_sql, - "constraints_schema.yml": constraints_yml, - } - - -@pytest.mark.skip_profile( - "spark_session", - "apache_spark", - "databricks_sql_endpoint", - "databricks_cluster", - "spark_http_odbc", -) -class TestSparkTableConstraintsColumnsEqualDatabricksHTTP( - DatabricksHTTPSetup, BaseTableConstraintsColumnsEqual -): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_wrong_order.sql": my_model_wrong_order_sql, - "my_model_wrong_name.sql": my_model_wrong_name_sql, - "constraints_schema.yml": constraints_yml, - } - - -@pytest.mark.skip_profile( - "spark_session", - "apache_spark", - "databricks_sql_endpoint", - "databricks_cluster", - "spark_http_odbc", -) -class TestSparkViewConstraintsColumnsEqualDatabricksHTTP( - DatabricksHTTPSetup, BaseViewConstraintsColumnsEqual -): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_wrong_order.sql": my_model_view_wrong_order_sql, - "my_model_wrong_name.sql": my_model_view_wrong_name_sql, - "constraints_schema.yml": constraints_yml, - } - - -@pytest.mark.skip_profile( - "spark_session", - "apache_spark", - "databricks_sql_endpoint", - "databricks_cluster", - "spark_http_odbc", -) -class TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP( - DatabricksHTTPSetup, BaseIncrementalConstraintsColumnsEqual -): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model_wrong_order.sql": my_model_incremental_wrong_order_sql, - "my_model_wrong_name.sql": my_model_incremental_wrong_name_sql, - "constraints_schema.yml": constraints_yml, - } - - -class BaseSparkConstraintsDdlEnforcementSetup: - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "+file_format": "delta", - } - } - - @pytest.fixture(scope="class") - def expected_sql(self): - return _expected_sql_spark - - -@pytest.mark.skip_profile("spark_session", "apache_spark") -class TestSparkTableConstraintsDdlEnforcement( - BaseSparkConstraintsDdlEnforcementSetup, BaseConstraintsRuntimeDdlEnforcement -): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_wrong_order_depends_on_fk_sql, - "foreign_key_model.sql": foreign_key_model_sql, - "constraints_schema.yml": model_fk_constraint_schema_yml, - } - - -@pytest.mark.skip_profile("spark_session", "apache_spark") -class TestSparkIncrementalConstraintsDdlEnforcement( - BaseSparkConstraintsDdlEnforcementSetup, BaseIncrementalConstraintsRuntimeDdlEnforcement -): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_incremental_wrong_order_depends_on_fk_sql, - "foreign_key_model.sql": foreign_key_model_sql, - "constraints_schema.yml": model_fk_constraint_schema_yml, - } - - -@pytest.mark.skip_profile("spark_session", "apache_spark", "databricks_http_cluster") -class TestSparkConstraintQuotedColumn(PyodbcSetup, BaseConstraintQuotedColumn): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_with_quoted_column_name_sql, - "constraints_schema.yml": model_quoted_column_schema_yml.replace( - "text", "string" - ).replace('"from"', "`from`"), - } - - @pytest.fixture(scope="class") - def expected_sql(self): - return """ -create or replace table - using delta - as -select - id, - `from`, - date_day -from - -( - select - 'blue' as `from`, - 1 as id, - '2019-01-01' as date_day ) as model_subq -""" - - -class BaseSparkConstraintsRollbackSetup: - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "+file_format": "delta", - } - } - - @pytest.fixture(scope="class") - def expected_error_messages(self): - return [ - "violate the new CHECK constraint", - "DELTA_NEW_CHECK_CONSTRAINT_VIOLATION", - "DELTA_NEW_NOT_NULL_VIOLATION", - "violate the new NOT NULL constraint", - "(id > 0) violated by row with values:", # incremental mats - "DELTA_VIOLATE_CONSTRAINT_WITH_VALUES", # incremental mats - "NOT NULL constraint violated for col", - ] - - def assert_expected_error_messages(self, error_message, expected_error_messages): - # This needs to be ANY instead of ALL - # The CHECK constraint is added before the NOT NULL constraint - # and different connection types display/truncate the error message in different ways... - assert any(msg in error_message for msg in expected_error_messages) - - -@pytest.mark.skip_profile("spark_session", "apache_spark") -class TestSparkTableConstraintsRollback( - BaseSparkConstraintsRollbackSetup, BaseConstraintsRollback -): - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_sql, - "constraints_schema.yml": constraints_yml, - } - - # On Spark/Databricks, constraints are applied *after* the table is replaced. - # We don't have any way to "rollback" the table to its previous happy state. - # So the 'color' column will be updated to 'red', instead of 'blue'. - @pytest.fixture(scope="class") - def expected_color(self): - return "red" - - -@pytest.mark.skip_profile("spark_session", "apache_spark") -class TestSparkIncrementalConstraintsRollback( - BaseSparkConstraintsRollbackSetup, BaseIncrementalConstraintsRollback -): - # color stays blue for incremental models since it's a new row that just - # doesn't get inserted - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_incremental_model_sql, - "constraints_schema.yml": constraints_yml, - } - - -# TODO: Like the tests above, this does test that model-level constraints don't -# result in errors, but it does not verify that they are actually present in -# Spark and that the ALTER TABLE statement actually ran. -@pytest.mark.skip_profile("spark_session", "apache_spark") -class TestSparkModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "+file_format": "delta", - } - } - - @pytest.fixture(scope="class") - def models(self): - return { - "my_model.sql": my_model_wrong_order_depends_on_fk_sql, - "foreign_key_model.sql": foreign_key_model_sql, - "constraints_schema.yml": model_fk_constraint_schema_yml, - } - - @pytest.fixture(scope="class") - def expected_sql(self): - return _expected_sql_spark_model_constraints diff --git a/dbt-sail/tests/functional/adapter/test_get_columns_in_relation.py b/dbt-sail/tests/functional/adapter/test_get_columns_in_relation.py deleted file mode 100644 index a037bb1cae..0000000000 --- a/dbt-sail/tests/functional/adapter/test_get_columns_in_relation.py +++ /dev/null @@ -1,33 +0,0 @@ -import pytest - -from dbt.tests.util import run_dbt, relation_from_name, check_relations_equal_with_relations - - -_MODEL_CHILD = "select 1" - - -_MODEL_PARENT = """ -{% set cols = adapter.get_columns_in_relation(ref('child')) %} - -select - {% for col in cols %} - {{ adapter.quote(col.column) }}{%- if not loop.last %},{{ '\n ' }}{% endif %} - {% endfor %} -from {{ ref('child') }} -""" - - -class TestColumnsInRelation: - @pytest.fixture(scope="class") - def models(self): - return { - "child.sql": _MODEL_CHILD, - "parent.sql": _MODEL_PARENT, - } - - @pytest.mark.skip_profile("databricks_http_cluster", "spark_session") - def test_get_columns_in_relation(self, project): - run_dbt(["run"]) - child = relation_from_name(project.adapter, "child") - parent = relation_from_name(project.adapter, "parent") - check_relations_equal_with_relations(project.adapter, [child, parent]) diff --git a/dbt-sail/tests/functional/adapter/test_grants.py b/dbt-sail/tests/functional/adapter/test_grants.py deleted file mode 100644 index 1b1a005ad1..0000000000 --- a/dbt-sail/tests/functional/adapter/test_grants.py +++ /dev/null @@ -1,60 +0,0 @@ -import pytest -from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants -from dbt.tests.adapter.grants.test_incremental_grants import BaseIncrementalGrants -from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants -from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants -from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants - - -@pytest.mark.skip_profile("apache_spark", "spark_session") -class TestModelGrantsSpark(BaseModelGrants): - def privilege_grantee_name_overrides(self): - # insert --> modify - return { - "select": "select", - "insert": "modify", - "fake_privilege": "fake_privilege", - "invalid_user": "invalid_user", - } - - -@pytest.mark.skip_profile("apache_spark", "spark_session") -class TestIncrementalGrantsSpark(BaseIncrementalGrants): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "+file_format": "delta", - "+incremental_strategy": "merge", - } - } - - -@pytest.mark.skip_profile("apache_spark", "spark_session") -class TestSeedGrantsSpark(BaseSeedGrants): - # seeds in dbt-spark are currently "full refreshed," in such a way that - # the grants are not carried over - # see https://github.com/dbt-labs/dbt-spark/issues/388 - def seeds_support_partial_refresh(self): - return False - - -@pytest.mark.skip_profile("apache_spark", "spark_session") -class TestSnapshotGrantsSpark(BaseSnapshotGrants): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "snapshots": { - "+file_format": "delta", - "+incremental_strategy": "merge", - } - } - - -@pytest.mark.skip_profile("apache_spark", "spark_session") -class TestInvalidGrantsSpark(BaseInvalidGrants): - def grantee_does_not_exist_error(self): - return "RESOURCE_DOES_NOT_EXIST" - - def privilege_does_not_exist_error(self): - return "Action Unknown" diff --git a/dbt-sail/tests/functional/adapter/test_python_model.py b/dbt-sail/tests/functional/adapter/test_python_model.py deleted file mode 100644 index 004e2f63c4..0000000000 --- a/dbt-sail/tests/functional/adapter/test_python_model.py +++ /dev/null @@ -1,122 +0,0 @@ -import os -import pytest -from dbt.tests.util import run_dbt, write_file -from dbt.tests.adapter.python_model.test_python_model import ( - BasePythonModelTests, - BasePythonIncrementalTests, -) -from dbt.tests.adapter.python_model.test_spark import BasePySparkTests - - -@pytest.mark.skip_profile( - "apache_spark", "spark_session", "databricks_sql_endpoint", "spark_http_odbc" -) -class TestPythonModelSpark(BasePythonModelTests): - pass - - -@pytest.mark.skip_profile( - "apache_spark", "spark_session", "databricks_sql_endpoint", "spark_http_odbc" -) -class TestPySpark(BasePySparkTests): - def test_different_dataframes(self, project): - """ - Test that python models are supported using dataframes from: - - pandas - - pyspark - - pyspark.pandas (formerly dataspark.koalas) - - Note: - The CI environment is on Apache Spark >3.1, which includes koalas as pyspark.pandas. - The only Databricks runtime that supports Apache Spark <=3.1 is 9.1 LTS, which is EOL 2024-09-23. - For more information, see: - - https://github.com/databricks/koalas - - https://docs.databricks.com/en/release-notes/runtime/index.html - """ - results = run_dbt(["run", "--exclude", "koalas_df"]) - assert len(results) == 3 - - -@pytest.mark.skip_profile( - "apache_spark", "spark_session", "databricks_sql_endpoint", "spark_http_odbc" -) -class TestPythonIncrementalModelSpark(BasePythonIncrementalTests): - @pytest.fixture(scope="class") - def project_config_update(self): - return {} - - -models__simple_python_model = """ -def model(dbt, spark): - dbt.config( - materialized='table', - submission_method='job_cluster', - job_cluster_config={ - "spark_version": "12.2.x-scala2.12", - "node_type_id": "i3.xlarge", - "num_workers": 0, - "spark_conf": { - "spark.databricks.cluster.profile": "singleNode", - "spark.master": "local[*, 4]" - }, - "custom_tags": { - "ResourceClass": "SingleNode" - } - }, - packages=['pydantic'] - ) - data = [[1,2]] * 10 - return spark.createDataFrame(data, schema=['test', 'test2']) -""" -models__simple_python_model_v2 = """ -import pandas - -def model(dbt, spark): - dbt.config( - materialized='table', - ) - data = [[1,2]] * 10 - return spark.createDataFrame(data, schema=['test1', 'test3']) -""" - - -@pytest.mark.skip_profile( - "apache_spark", - "spark_session", - "databricks_sql_endpoint", - "spark_http_odbc", - "databricks_http_cluster", -) -class TestChangingSchemaSpark: - """ - Confirm that we can setup a spot instance and parse required packages into the Databricks job. - - Notes: - - This test generates a spot instance on demand using the settings from `job_cluster_config` - in `models__simple_python_model` above. It takes several minutes to run due to creating the cluster. - The job can be monitored via "Data Engineering > Job Runs" or "Workflows > Job Runs" - in the Databricks UI (instead of via the normal cluster). - - The `spark_version` argument will need to periodically be updated. It will eventually become - unsupported and start experiencing issues. - - See https://github.com/explosion/spaCy/issues/12659 for why we're pinning pydantic - """ - - @pytest.fixture(scope="class") - def models(self): - return {"simple_python_model.py": models__simple_python_model} - - def test_changing_schema_with_log_validation(self, project, logs_dir): - run_dbt(["run"]) - write_file( - models__simple_python_model_v2, - project.project_root + "/models", - "simple_python_model.py", - ) - run_dbt(["run"]) - log_file = os.path.join(logs_dir, "dbt.log") - with open(log_file, "r") as f: - log = f.read() - # validate #5510 log_code_execution works - assert "On model.test.simple_python_model:" in log - assert "spark.createDataFrame(data, schema=['test1', 'test3'])" in log - assert "Execution status: OK in" in log diff --git a/dbt-sail/tests/functional/adapter/test_query_timeout.py b/dbt-sail/tests/functional/adapter/test_query_timeout.py deleted file mode 100644 index 5311140424..0000000000 --- a/dbt-sail/tests/functional/adapter/test_query_timeout.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Functional tests for query timeout and retry behavior. - -NOTE: These tests only work with PyHive-based connections (http/thrift methods). -ODBC connections use PyodbcConnectionWrapper which doesn't have timeout/retry support yet. -""" - -import pytest -from dbt.tests.util import run_dbt, run_dbt_and_capture -from dbt_common.exceptions import DbtRuntimeError - - -# Model that creates a delay to simulate a long-running query -# Uses a large cross join to create computational delay -long_running_model = """ -{{ config(materialized='table') }} --- Generate a large dataset to create a delay --- This creates roughly 1 million rows which should take several seconds -with numbers as ( - select explode(sequence(1, 100000)) as n -), -cross_product as ( - select a.n as n1, b.n as n2 - from numbers a - cross join numbers b -) -select count(*) as total_count -from cross_product -""" - -simple_model = """ -{{ config(materialized='table') }} -select 1 as id -""" - - -@pytest.mark.skip_profile( - "spark_http_odbc", - "databricks_cluster", - "databricks_sql_endpoint", - "databricks_http_cluster", - "spark_session", -) -class TestQueryTimeout: - """Test query timeout functionality. - - Skipped on ODBC profiles because PyodbcConnectionWrapper doesn't use - the async polling mechanism with timeout support. - """ - - @pytest.fixture(scope="class") - def models(self): - return { - "long_running.sql": long_running_model, - "simple.sql": simple_model, - } - - @pytest.fixture(scope="class") - def dbt_profile_target(self, dbt_profile_target): - """Override profile to add timeout configuration.""" - dbt_profile_target["query_timeout"] = 1 # 1 second timeout - dbt_profile_target["poll_interval"] = 1 # Poll every second for faster test - dbt_profile_target["query_retries"] = 0 # Disable retries for clearer errors - return dbt_profile_target - - def test_query_timeout_exceeded(self, project): - """Test that queries exceeding timeout raise appropriate error.""" - - # Simple model should succeed (runs quickly) - results = run_dbt(["run", "--select", "simple"]) - assert results[0].status == "success" - - # Long-running model should timeout - # The cross-join query should take longer than 2 seconds on most systems - _, output = run_dbt_and_capture(["run", "--select", "long_running"], expect_pass=False) - - assert "exceeded timeout" in output.lower() diff --git a/dbt-sail/tests/functional/adapter/test_store_test_failures.py b/dbt-sail/tests/functional/adapter/test_store_test_failures.py deleted file mode 100644 index 4723364052..0000000000 --- a/dbt-sail/tests/functional/adapter/test_store_test_failures.py +++ /dev/null @@ -1,101 +0,0 @@ -import pytest - -from dbt.tests.adapter.store_test_failures_tests import basic -from dbt.tests.adapter.store_test_failures_tests.test_store_test_failures import ( - StoreTestFailuresBase, - TEST_AUDIT_SCHEMA_SUFFIX, -) - - -@pytest.mark.skip_profile( - "spark_session", "databricks_cluster", "databricks_sql_endpoint", "spark_http_odbc" -) -class TestSparkStoreTestFailures(StoreTestFailuresBase): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "seeds": { - "quote_columns": True, - }, - "tests": {"+schema": TEST_AUDIT_SCHEMA_SUFFIX, "+store_failures": True}, - } - - @pytest.fixture(scope="function", autouse=True) - def teardown_method(self, project): - yield - with project.adapter.connection_named("__test"): - relation = project.adapter.Relation.create( - database=project.database, - schema=f"{project.test_schema}_{TEST_AUDIT_SCHEMA_SUFFIX}", - ) - - project.adapter.drop_schema(relation) - - def test_store_and_assert(self, project): - self.run_tests_store_one_failure(project) - self.run_tests_store_failures_and_assert(project) - - -@pytest.mark.skip_profile( - "apache_spark", "spark_session", "databricks_sql_endpoint", "spark_http_odbc" -) -class TestSparkStoreTestFailuresWithDelta(StoreTestFailuresBase): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "seeds": { - "quote_columns": False, - "test": self.column_type_overrides(), - "+file_format": "delta", - }, - "tests": { - "+schema": TEST_AUDIT_SCHEMA_SUFFIX, - "+store_failures": True, - "+file_format": "delta", - }, - } - - @pytest.fixture(scope="function", autouse=True) - def teardown_method(self, project): - yield - with project.adapter.connection_named("__test"): - relation = project.adapter.Relation.create( - database=project.database, - schema=f"{project.test_schema}_{TEST_AUDIT_SCHEMA_SUFFIX}", - ) - - project.adapter.drop_schema(relation) - - def test_store_and_assert_failure_with_delta(self, project): - self.run_tests_store_one_failure(project) - self.run_tests_store_failures_and_assert(project) - - -@pytest.mark.skip_profile("spark_session") -class TestStoreTestFailuresAsInteractions(basic.StoreTestFailuresAsInteractions): - pass - - -@pytest.mark.skip_profile("spark_session") -class TestStoreTestFailuresAsProjectLevelOff(basic.StoreTestFailuresAsProjectLevelOff): - pass - - -@pytest.mark.skip_profile("spark_session") -class TestStoreTestFailuresAsProjectLevelView(basic.StoreTestFailuresAsProjectLevelView): - pass - - -@pytest.mark.skip_profile("spark_session") -class TestStoreTestFailuresAsGeneric(basic.StoreTestFailuresAsGeneric): - pass - - -@pytest.mark.skip_profile("spark_session") -class TestStoreTestFailuresAsProjectLevelEphemeral(basic.StoreTestFailuresAsProjectLevelEphemeral): - pass - - -@pytest.mark.skip_profile("spark_session") -class TestStoreTestFailuresAsExceptions(basic.StoreTestFailuresAsExceptions): - pass diff --git a/dbt-sail/tests/functional/adapter/utils/test_utils.py b/dbt-sail/tests/functional/adapter/utils/test_utils.py index db7a33d575..88d8a0fdde 100644 --- a/dbt-sail/tests/functional/adapter/utils/test_utils.py +++ b/dbt-sail/tests/functional/adapter/utils/test_utils.py @@ -7,22 +7,16 @@ from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr from dbt.tests.adapter.utils.test_cast import BaseCast from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText -from dbt.tests.adapter.utils.test_concat import BaseConcat from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampNaive from dbt.tests.adapter.utils.test_date import BaseDate from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd -from dbt.tests.adapter.utils.test_datediff import BaseDateDiff from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc from dbt.tests.adapter.utils.test_equals import BaseEquals from dbt.tests.adapter.utils.test_escape_single_quotes import BaseEscapeSingleQuotesBackslash from dbt.tests.adapter.utils.test_except import BaseExcept -from dbt.tests.adapter.utils.test_hash import BaseHash from dbt.tests.adapter.utils.test_intersect import BaseIntersect -from dbt.tests.adapter.utils.test_last_day import BaseLastDay from dbt.tests.adapter.utils.test_length import BaseLength from dbt.tests.adapter.utils.test_position import BasePosition -from dbt.tests.adapter.utils.test_replace import BaseReplace -from dbt.tests.adapter.utils.test_right import BaseRight from dbt.tests.adapter.utils.test_safe_cast import BaseSafeCast from dbt.tests.adapter.utils.test_split_part import BaseSplitPart @@ -39,12 +33,6 @@ EMPTY|EMPTY|EMPTY,|,EMPTY,EMPTY,EMPTY,EMPTY """ -seeds__data_last_day_csv = """date_day,date_part,result -2018-01-02,month,2018-01-31 -2018-01-02,quarter,2018-03-31 -2018-01-02,year,2018-12-31 -""" - # skipped: ,month, @@ -77,11 +65,6 @@ class TestCastBoolToText(BaseCastBoolToText): pass -@pytest.mark.skip_profile("spark_session") -class TestConcat(BaseConcat): - pass - - # Use either BaseCurrentTimestampAware or BaseCurrentTimestampNaive but not both class TestCurrentTimestamp(BaseCurrentTimestampNaive): pass @@ -95,12 +78,6 @@ class TestDateAdd(BaseDateAdd): pass -# this generates too much SQL to run successfully in our testing environments :( -@pytest.mark.skip_profile("apache_spark", "spark_session") -class TestDateDiff(BaseDateDiff): - pass - - class TestDateTrunc(BaseDateTrunc): pass @@ -117,22 +94,10 @@ class TestExcept(BaseExcept): pass -@pytest.mark.skip_profile("spark_session") -class TestHash(BaseHash): - pass - - class TestIntersect(BaseIntersect): pass -@pytest.mark.skip_profile("spark_session") # spark session crashes in CI -class TestLastDay(BaseLastDay): - @pytest.fixture(scope="class") - def seeds(self): - return {"data_last_day.csv": seeds__data_last_day_csv} - - class TestLength(BaseLength): pass @@ -154,16 +119,6 @@ class TestPosition(BasePosition): pass -@pytest.mark.skip_profile("spark_session") -class TestReplace(BaseReplace): - pass - - -@pytest.mark.skip_profile("spark_session") -class TestRight(BaseRight): - pass - - class TestSafeCast(BaseSafeCast): pass diff --git a/dbt-sail/tests/unit/test_adapter.py b/dbt-sail/tests/unit/test_adapter.py index fff47a7ad8..694ccee29e 100644 --- a/dbt-sail/tests/unit/test_adapter.py +++ b/dbt-sail/tests/unit/test_adapter.py @@ -72,9 +72,7 @@ def test_relation_with_database(self): adapter.Relation.create(schema="different", identifier="table") with self.assertRaises(DbtRuntimeError): # not fine - database set - adapter.Relation.create( - database="something", schema="different", identifier="table" - ) + adapter.Relation.create(database="something", schema="different", identifier="table") class TestListRelationsWithoutCaching(unittest.TestCase): diff --git a/dbt-spark/dagger/audit_spark_session_test_speeds.py b/dbt-spark/dagger/audit_spark_session_test_speeds.py new file mode 100644 index 0000000000..e95fa94ea2 --- /dev/null +++ b/dbt-spark/dagger/audit_spark_session_test_speeds.py @@ -0,0 +1,212 @@ +from __future__ import annotations + +import argparse +import concurrent.futures +import contextlib +import io +import json +import subprocess +import sys +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import List + +import pytest + + +DBT_SPARK_ROOT = Path(__file__).resolve().parents[1] +RUNNER_PATH = DBT_SPARK_ROOT / "dagger" / "run_dbt_spark_tests.py" +DEFAULT_SCOPE = "tests/functional/adapter" +DEFAULT_OUTPUT_DIR = DBT_SPARK_ROOT / "target" / "spark-session-audit" + + +@dataclass +class TestAuditResult: + nodeid: str + return_code: int + timed_out: bool + duration_seconds: float + log_file: str + + +def collect_runnable_nodeids(scope: str, profile: str) -> List[str]: + class Collector: + def __init__(self) -> None: + self.items = [] + + def pytest_collection_modifyitems(self, session, config, items): + self.items = list(items) + + collector = Collector() + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + pytest.main([scope, "--collect-only", "-q", "--profile", profile], plugins=[collector]) + + runnable: List[str] = [] + for item in collector.items: + if not item.nodeid.startswith("tests/"): + continue + marker = item.get_closest_marker("skip_profile") + marked_profiles = set(marker.args) if marker else set() + if profile in marked_profiles: + continue + runnable.append(item.nodeid) + return sorted(set(runnable)) + + +def run_one_test( + nodeid: str, profile: str, timeout_seconds: int, output_dir: Path +) -> TestAuditResult: + safe_name = nodeid.replace("/", "__").replace("::", "__") + log_file = output_dir / f"{safe_name}.log" + cmd = [ + sys.executable, + str(RUNNER_PATH), + "--profile", + profile, + "--test-path", + nodeid, + ] + + started = time.monotonic() + try: + completed = subprocess.run( + cmd, + cwd=DBT_SPARK_ROOT, + text=True, + capture_output=True, + check=False, + timeout=timeout_seconds, + ) + timed_out = False + return_code = completed.returncode + output = completed.stdout + if completed.stderr: + output += "\n\n=== STDERR ===\n" + completed.stderr + except subprocess.TimeoutExpired as exc: + timed_out = True + return_code = 124 + out = exc.stdout or "" + err = exc.stderr or "" + if isinstance(out, bytes): + out = out.decode(errors="replace") + if isinstance(err, bytes): + err = err.decode(errors="replace") + output = f"Timed out after {timeout_seconds}s.\n\n{out}" + if err: + output += "\n\n=== STDERR ===\n" + err + + duration_seconds = round(time.monotonic() - started, 2) + log_file.write_text(output) + return TestAuditResult( + nodeid=nodeid, + return_code=return_code, + timed_out=timed_out, + duration_seconds=duration_seconds, + log_file=str(log_file.relative_to(DBT_SPARK_ROOT)), + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Run each spark_session-runnable test individually with timeout. " + "Tests that do not finish before timeout are killed and classified as slow." + ) + ) + parser.add_argument("--profile", default="spark_session") + parser.add_argument("--scope", default=DEFAULT_SCOPE) + parser.add_argument("--jobs", type=int, default=3) + parser.add_argument("--slow-timeout-seconds", type=int, default=180) + parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR)) + parser.add_argument( + "--max-tests", type=int, default=0, help="Run only first N tests (0 = all)" + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + nodeids = collect_runnable_nodeids(args.scope, args.profile) + if args.max_tests > 0: + nodeids = nodeids[: args.max_tests] + + if not nodeids: + print("No runnable tests found.") + return 1 + + print(f"Profile: {args.profile}") + print(f"Scope: {args.scope}") + print(f"Runnable tests to audit: {len(nodeids)}") + print( + f"Running individually with jobs={args.jobs}; " + f"kill threshold={args.slow_timeout_seconds}s" + ) + + results: List[TestAuditResult] = [] + max_workers = min(max(args.jobs, 1), len(nodeids)) + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit( + run_one_test, + nodeid, + args.profile, + args.slow_timeout_seconds, + output_dir, + ): nodeid + for nodeid in nodeids + } + for future in concurrent.futures.as_completed(futures): + result = future.result() + results.append(result) + status = "FAST" if result.return_code == 0 else "SLOW" + timeout_note = " timeout" if result.timed_out else "" + print(f"[{status}] {result.duration_seconds:6.2f}s{timeout_note} {result.nodeid}") + + results = sorted(results, key=lambda item: item.nodeid) + fast = sorted( + [r for r in results if r.return_code == 0], key=lambda item: item.duration_seconds + ) + slow = sorted([r for r in results if r.return_code != 0], key=lambda item: item.nodeid) + + fast_list = output_dir / "fast_tests.txt" + slow_list = output_dir / "slow_tests.txt" + json_report = output_dir / "audit_report.json" + + fast_list.write_text("\n".join([r.nodeid for r in fast]) + ("\n" if fast else "")) + slow_list.write_text("\n".join([r.nodeid for r in slow]) + ("\n" if slow else "")) + json_report.write_text( + json.dumps( + { + "profile": args.profile, + "scope": args.scope, + "jobs": args.jobs, + "slow_timeout_seconds": args.slow_timeout_seconds, + "total_audited": len(results), + "fast_count": len(fast), + "slow_count": len(slow), + "fast_tests_file": str(fast_list.relative_to(DBT_SPARK_ROOT)), + "slow_tests_file": str(slow_list.relative_to(DBT_SPARK_ROOT)), + "results": [asdict(r) for r in results], + }, + indent=2, + ) + + "\n" + ) + + print("\nAudit summary:") + print(f" Total audited: {len(results)}") + print(f" Fast tests: {len(fast)}") + print(f" Slow tests: {len(slow)}") + print(f" Fast list: {fast_list.relative_to(DBT_SPARK_ROOT)}") + print(f" Slow list: {slow_list.relative_to(DBT_SPARK_ROOT)}") + print(f" JSON report: {json_report.relative_to(DBT_SPARK_ROOT)}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dbt-spark/dagger/run_dbt_spark_tests.py b/dbt-spark/dagger/run_dbt_spark_tests.py index 3d5432e0c9..80fbe17e35 100644 --- a/dbt-spark/dagger/run_dbt_spark_tests.py +++ b/dbt-spark/dagger/run_dbt_spark_tests.py @@ -92,7 +92,6 @@ def get_spark_container(client: dagger.Client) -> Tuple[dagger.Service, str]: async def test_spark(test_args): async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client: - # create cache volumes, these are persisted between runs saving time when developing locally tst_container = ( client.container(platform=dagger.Platform("linux/amd64")) @@ -154,13 +153,28 @@ async def test_spark(test_args): tst_container = tst_container.with_exec(["./scripts/install_jdk.sh"]) # run the tests - result = ( - await tst_container.with_workdir("/src") - .with_exec( - ["hatch", "run", "pytest", "--profile", test_args.profile, test_args.test_path] - ) - .stdout() - ) + pytest_cmd = [ + "hatch", + "run", + "pytest", + "--profile", + test_args.profile, + *test_args.test_path, + ] + if test_args.profile == "spark_session": + # Spark session mode is unstable under xdist parallelism in Dagger; run serially. + pytest_cmd.extend(["-n", "0"]) + + result = await tst_container.with_workdir("/src").with_exec(pytest_cmd).stdout() + + print(result) + + # Print summary at the end + for line in result.strip().splitlines(): + if "passed" in line or "failed" in line or "error" in line: + print(f"\n{'=' * 60}") + print(f"SUMMARY: {line.strip()}") + print(f"{'=' * 60}") return result @@ -168,7 +182,13 @@ async def test_spark(test_args): # TODO: update this to align more closely with the pytest api, e.g. --test-path should be an arg instead of a kwarg parser = argparse.ArgumentParser() parser.add_argument("--profile", required=True, type=str) -parser.add_argument("--test-path", required=False, type=str, default="tests/functional/adapter") +parser.add_argument( + "--test-path", + required=False, + nargs="+", + default=["tests/functional/adapter"], + help="One or more pytest paths to run", +) args = parser.parse_args() anyio.run(test_spark, args) diff --git a/dbt-spark/dagger/run_dbt_spark_tests_sharded.py b/dbt-spark/dagger/run_dbt_spark_tests_sharded.py new file mode 100644 index 0000000000..dedd6ec0d6 --- /dev/null +++ b/dbt-spark/dagger/run_dbt_spark_tests_sharded.py @@ -0,0 +1,530 @@ +from __future__ import annotations + +import argparse +import concurrent.futures +import contextlib +import io +import os +import re +import signal +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List + +import pytest + +DBT_SPARK_ROOT = Path(__file__).resolve().parents[1] +RUNNER_PATH = DBT_SPARK_ROOT / "dagger" / "run_dbt_spark_tests.py" +DEFAULT_GLOB = "tests/functional/adapter/**/test_*.py" +LOG_DIR = DBT_SPARK_ROOT / "target" / "dagger-shards" +ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") +STATUS_RE_1 = re.compile(r"(PASSED|SKIPPED|FAILED|ERROR)\s+(tests/functional/adapter/\S+)") +STATUS_RE_2 = re.compile(r"(tests/functional/adapter/\S+)\s+(PASSED|SKIPPED|FAILED|ERROR)") +PYTEST_SUMMARY_RE = re.compile(r"=+\s+.*(passed|skipped|failed|error).* in .*=+") + + +@dataclass +class TaskSpec: + task_index: int + test_paths: List[str] + + +@dataclass +class TaskResult: + task_index: int + test_paths: List[str] + return_code: int + timed_out: bool + attempts: int + log_file: Path + summary_line: str + status_by_test: Dict[str, str] + + +def _collect_ancestor_pids() -> set[int]: + ancestors: set[int] = set() + pid = os.getpid() + for _ in range(10): + if pid <= 1: + break + ancestors.add(pid) + try: + parent = ( + os.getppid() + if pid == os.getpid() + else int( + subprocess.check_output( + ["ps", "-o", "ppid=", "-p", str(pid)], + text=True, + ).strip() + ) + ) + except Exception: + break + if parent in ancestors: + break + pid = parent + return ancestors + + +def find_runner_processes_for_profile(profile: str) -> List[int]: + try: + ps_output = subprocess.check_output(["ps", "-eo", "pid,args"], text=True) + except subprocess.CalledProcessError: + return [] + + current_and_ancestors = _collect_ancestor_pids() + matching: List[int] = [] + for line in ps_output.splitlines(): + line = line.strip() + if not line: + continue + parts = line.split(None, 1) + if len(parts) != 2: + continue + pid_str, args = parts + if not pid_str.isdigit(): + continue + pid = int(pid_str) + if pid in current_and_ancestors: + continue + if "run_dbt_spark_tests.py" not in args: + continue + if f"--profile {profile}" not in args: + continue + matching.append(pid) + return sorted(set(matching)) + + +def terminate_pids(pids: List[int], grace_seconds: float = 1.5) -> List[int]: + if not pids: + return [] + + for pid in pids: + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGTERM) + time.sleep(grace_seconds) + + survivors: List[int] = [] + for pid in pids: + try: + os.kill(pid, 0) + survivors.append(pid) + except ProcessLookupError: + continue + + for pid in survivors: + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGKILL) + return survivors + + +def cleanup_stale_runner_processes(profile: str) -> None: + stale_pids = find_runner_processes_for_profile(profile) + if not stale_pids: + return + print( + f"Found {len(stale_pids)} stale runner process(es) for profile '{profile}': " + + ", ".join(str(pid) for pid in stale_pids) + ) + killed_after_term = terminate_pids(stale_pids) + if killed_after_term: + print( + "Escalated to SIGKILL for pid(s): " + ", ".join(str(pid) for pid in killed_after_term) + ) + print("Stale runner cleanup complete.") + + +def discover_test_files(test_glob: str) -> List[str]: + files = [ + str(path.relative_to(DBT_SPARK_ROOT)) + for path in DBT_SPARK_ROOT.glob(test_glob) + if path.is_file() + ] + return sorted(files) + + +def build_shards(paths: List[str], shard_count: int) -> List[List[str]]: + if shard_count < 1: + raise ValueError("shard_count must be >= 1") + shards: List[List[str]] = [[] for _ in range(shard_count)] + for idx, path in enumerate(paths): + shards[idx % shard_count].append(path) + return [shard for shard in shards if shard] + + +def clean_line(line: str) -> str: + return ANSI_RE.sub("", line) + + +def parse_statuses(output: str) -> Dict[str, str]: + statuses: Dict[str, str] = {} + for raw_line in output.splitlines(): + line = clean_line(raw_line) + match = STATUS_RE_1.search(line) + if match: + statuses[match.group(2)] = match.group(1) + continue + match = STATUS_RE_2.search(line) + if match: + statuses[match.group(1)] = match.group(2) + return statuses + + +def extract_pytest_summary(output: str) -> str: + summary = "" + for raw_line in output.splitlines(): + line = clean_line(raw_line) + if PYTEST_SUMMARY_RE.search(line): + summary = line.strip() + return summary + + +def run_task(task: TaskSpec, profile: str, timeout_seconds: int, retries: int) -> TaskResult: + cmd = [ + sys.executable, + str(RUNNER_PATH), + "--profile", + profile, + "--test-path", + *task.test_paths, + ] + + LOG_DIR.mkdir(parents=True, exist_ok=True) + log_file = LOG_DIR / f"task_{task.task_index:03d}.log" + + merged_output = "" + return_code = 1 + timed_out = False + attempts_used = 0 + + for attempt in range(1, retries + 2): + attempts_used = attempt + try: + completed = subprocess.run( + cmd, + cwd=DBT_SPARK_ROOT, + text=True, + capture_output=True, + check=False, + timeout=timeout_seconds if timeout_seconds > 0 else None, + ) + attempt_output = completed.stdout + if completed.stderr: + attempt_output += "\n\n=== STDERR ===\n" + completed.stderr + merged_output += f"\n\n===== ATTEMPT {attempt} =====\n" + attempt_output + return_code = completed.returncode + timed_out = False + if return_code == 0: + break + except subprocess.TimeoutExpired as exc: + timed_out = True + timed_stdout = exc.stdout or "" + timed_stderr = exc.stderr or "" + if isinstance(timed_stdout, bytes): + timed_stdout = timed_stdout.decode(errors="replace") + if isinstance(timed_stderr, bytes): + timed_stderr = timed_stderr.decode(errors="replace") + merged_output += ( + f"\n\n===== ATTEMPT {attempt} =====\n" + f"Task timed out after {timeout_seconds}s.\n" + timed_stdout + ) + if timed_stderr: + merged_output += "\n\n=== STDERR ===\n" + timed_stderr + return_code = 124 + + log_file.write_text(merged_output) + status_by_test = parse_statuses(merged_output) + summary_line = extract_pytest_summary(merged_output) + return TaskResult( + task_index=task.task_index, + test_paths=task.test_paths, + return_code=return_code, + timed_out=timed_out, + attempts=attempts_used, + log_file=log_file, + summary_line=summary_line, + status_by_test=status_by_test, + ) + + +def run_all_tasks( + tasks: List[TaskSpec], + profile: str, + jobs: int, + timeout_seconds: int, + retries: int, + cleanup_stale: bool = True, +) -> List[TaskResult]: + results: List[TaskResult] = [] + if cleanup_stale and profile == "spark_session": + cleanup_stale_runner_processes(profile) + max_workers = min(max(jobs, 1), len(tasks)) + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + future_map = { + executor.submit(run_task, task, profile, timeout_seconds, retries): task.task_index + for task in tasks + } + for future in concurrent.futures.as_completed(future_map): + result = future.result() + results.append(result) + status = "PASS" if result.return_code == 0 else "FAIL" + timeout_note = " (timeout)" if result.timed_out else "" + print(f"[{status}] task {result.task_index}{timeout_note} -> {result.log_file}") + if result.summary_line: + print(f" {result.summary_line}") + return sorted(results, key=lambda r: r.task_index) + + +def collect_nodeids_for_path(path: str, profile: str) -> List[str]: + class Collector: + def __init__(self) -> None: + self.items = [] + + def pytest_collection_modifyitems(self, session, config, items): + self.items = list(items) + + collector = Collector() + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + pytest.main([path, "--collect-only", "-q", "--profile", profile], plugins=[collector]) + nodeids: List[str] = [] + for item in collector.items: + nodeid = item.nodeid + if nodeid.startswith("tests/"): + nodeids.append(nodeid) + return sorted(set(nodeids)) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Run dbt-spark Dagger integration tests with isolated parallel tasks. " + "Use file mode for maximum stability in spark_session." + ) + ) + parser.add_argument("--profile", default="spark_session", help="dbt-spark test profile") + parser.add_argument( + "--mode", + choices=["files", "shards"], + default="files", + help="files: one test file per task, shards: grouped files per task", + ) + parser.add_argument("--shards", type=int, default=6, help="Number of shards when mode=shards") + parser.add_argument("--jobs", type=int, default=3, help="Maximum tasks to run in parallel") + parser.add_argument( + "--timeout-seconds", type=int, default=1500, help="Per-task timeout (0 disables timeout)" + ) + parser.add_argument("--retries", type=int, default=1, help="Retries per failed/timed-out task") + parser.add_argument( + "--fallback-serial", + action="store_true", + help="On failure/timeout, rerun failed files serially (jobs=1)", + ) + parser.add_argument( + "--fallback-timeout-seconds", + type=int, + default=0, + help="Per-file timeout for fallback reruns (0 = 2x --timeout-seconds)", + ) + parser.add_argument( + "--fallback-retries", + type=int, + default=0, + help="Retries for fallback per-file reruns", + ) + parser.add_argument( + "--fallback-split-tests", + action="store_true", + help="If fallback file reruns still fail, split failed files into per-test reruns", + ) + parser.add_argument( + "--fallback-split-jobs", + type=int, + default=2, + help="Parallelism for fallback split-test reruns", + ) + parser.add_argument( + "--test-glob", + default=DEFAULT_GLOB, + help=f"Glob used to discover test files (default: {DEFAULT_GLOB})", + ) + parser.add_argument( + "--test-path", + nargs="*", + default=[], + help="Optional explicit test file paths (used instead of discovery)", + ) + parser.add_argument( + "--always-split-path", + nargs="*", + default=[], + help="File paths that should always be split into per-test tasks up front", + ) + parser.add_argument( + "--max-tasks", + type=int, + default=0, + help="If > 0, run only first N tasks (smoke testing)", + ) + parser.add_argument("--dry-run", action="store_true", help="Print task plan and exit") + parser.add_argument( + "--no-cleanup-stale-runners", + action="store_true", + help="Skip preflight cleanup of stale run_dbt_spark_tests.py processes", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + if args.test_path: + test_files = sorted(args.test_path) + else: + test_files = discover_test_files(args.test_glob) + + if not test_files: + print("No test files found.") + return 1 + + split_paths = set(args.always_split_path) + if args.mode == "files": + task_paths: List[List[str]] = [] + for path in test_files: + if path in split_paths: + nodeids = collect_nodeids_for_path(path, args.profile) + if nodeids: + task_paths.extend([[nodeid] for nodeid in nodeids]) + else: + task_paths.append([path]) + else: + task_paths.append([path]) + else: + task_paths = build_shards(test_files, min(args.shards, len(test_files))) + + tasks = [ + TaskSpec(task_index=idx, test_paths=paths) for idx, paths in enumerate(task_paths, start=1) + ] + if args.max_tasks > 0: + tasks = tasks[: args.max_tasks] + + print(f"Discovered {len(test_files)} test files") + print( + f"Running {len(tasks)} task(s) in {args.mode} mode with up to {args.jobs} parallel job(s); " + f"timeout={args.timeout_seconds}s retries={args.retries}" + ) + for task in tasks: + print(f" task {task.task_index}: {len(task.test_paths)} file(s)") + + if args.dry_run: + return 0 + + results = run_all_tasks( + tasks, + args.profile, + args.jobs, + args.timeout_seconds, + args.retries, + cleanup_stale=not args.no_cleanup_stale_runners, + ) + + fallback_results: List[TaskResult] = [] + if args.fallback_serial: + failed_paths = sorted( + {path for result in results if result.return_code != 0 for path in result.test_paths} + ) + if failed_paths: + print("\nRunning fallback serial reruns for failed paths...") + fallback_tasks = [ + TaskSpec(task_index=1000 + idx, test_paths=[path]) + for idx, path in enumerate(failed_paths, start=1) + ] + fallback_timeout = ( + args.fallback_timeout_seconds + if args.fallback_timeout_seconds > 0 + else (args.timeout_seconds * 2 if args.timeout_seconds > 0 else 0) + ) + fallback_results = run_all_tasks( + fallback_tasks, + args.profile, + jobs=1, + timeout_seconds=fallback_timeout, + retries=args.fallback_retries, + cleanup_stale=not args.no_cleanup_stale_runners, + ) + results.extend(fallback_results) + + failed = [result for result in results if result.return_code != 0] + unresolved_paths = {path for result in failed for path in result.test_paths} + for result in fallback_results: + if result.return_code == 0: + unresolved_paths.discard(result.test_paths[0]) + + if unresolved_paths and args.fallback_split_tests: + print("\nRunning fallback per-test reruns for unresolved paths...") + split_tasks: List[TaskSpec] = [] + split_index = 2000 + for path in sorted(unresolved_paths): + nodeids = collect_nodeids_for_path(path, args.profile) + if not nodeids: + split_tasks.append(TaskSpec(task_index=split_index, test_paths=[path])) + split_index += 1 + continue + for nodeid in nodeids: + split_tasks.append(TaskSpec(task_index=split_index, test_paths=[nodeid])) + split_index += 1 + + split_timeout = ( + args.fallback_timeout_seconds + if args.fallback_timeout_seconds > 0 + else (args.timeout_seconds * 2 if args.timeout_seconds > 0 else 0) + ) + split_results = run_all_tasks( + split_tasks, + args.profile, + jobs=args.fallback_split_jobs, + timeout_seconds=split_timeout, + retries=args.fallback_retries, + cleanup_stale=not args.no_cleanup_stale_runners, + ) + results.extend(split_results) + unresolved_paths = set() + for result in split_results: + if result.return_code != 0: + unresolved_paths.update(result.test_paths) + + combined_status: Dict[str, str] = {} + for result in results: + combined_status.update(result.status_by_test) + counts = {"PASSED": 0, "SKIPPED": 0, "FAILED": 0, "ERROR": 0} + for status in combined_status.values(): + if status in counts: + counts[status] += 1 + print("\nAggregated status counts:") + print(f" PASSED: {counts['PASSED']}") + print(f" SKIPPED: {counts['SKIPPED']}") + print(f" FAILED: {counts['FAILED']}") + print(f" ERROR: {counts['ERROR']}") + print(f" WITH STATUS: {len(combined_status)}") + + if unresolved_paths: + print("\nFailed tasks:") + for result in failed: + files = ", ".join(result.test_paths) + timeout_note = " (timeout)" if result.timed_out else "" + print(f" task {result.task_index}{timeout_note}: {files}") + print(f" log: {result.log_file}") + print("\nUnresolved failed paths:") + for path in sorted(unresolved_paths): + print(f" {path}") + return 1 + + print("\nAll tasks passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dbt-spark/dagger/run_dbt_spark_tests_tiered.py b/dbt-spark/dagger/run_dbt_spark_tests_tiered.py new file mode 100644 index 0000000000..d001177b4b --- /dev/null +++ b/dbt-spark/dagger/run_dbt_spark_tests_tiered.py @@ -0,0 +1,439 @@ +from __future__ import annotations + +import argparse +import concurrent.futures +import contextlib +import io +from pathlib import Path +from typing import List, Set, Tuple + +import pytest + +from run_dbt_spark_tests_sharded import ( + TaskSpec, + build_shards, + cleanup_stale_runner_processes, + run_all_tasks, +) + +DEFAULT_TEST_SCOPE = "tests/functional/adapter" +DBT_SPARK_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_FAST_LIST_FILE = DBT_SPARK_ROOT / "dagger" / "test_lists" / "spark_session_fast.txt" +DEFAULT_FAST_TAIL_LIST_FILE = ( + DBT_SPARK_ROOT / "dagger" / "test_lists" / "spark_session_fast_tail_grouped.txt" +) +DEFAULT_SLOW_WORKING_LIST_FILE = ( + DBT_SPARK_ROOT / "dagger" / "test_lists" / "spark_session_slow_but_working.txt" +) +DEFAULT_EXCLUDED_LIST_FILES = [ + DBT_SPARK_ROOT / "dagger" / "test_lists" / "spark_session_timed_out.txt", + DBT_SPARK_ROOT / "dagger" / "test_lists" / "spark_session_errored.txt", +] + +# Observed long/hang-prone tests from recent runs. +SLOW_NODEIDS = { + "tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChangeSpecialChars::test_incremental_append_new_columns_with_special_characters", + "tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_ignore", + "tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_fail_on_schema_change", + "tests/functional/adapter/utils/test_utils.py::TestAnyValue::test_build_assert_equal", + "tests/functional/adapter/utils/test_utils.py::TestBoolOr::test_build_assert_equal", + "tests/functional/adapter/utils/test_utils.py::TestDateAdd::test_build_assert_equal", + "tests/functional/adapter/utils/test_utils.py::TestEquals::test_build_assert_equal", + "tests/functional/adapter/utils/test_utils.py::TestListagg::test_build_assert_equal", +} + + +def collect_runnable_nodeids(path: str, profile: str) -> List[str]: + class Collector: + def __init__(self) -> None: + self.items = [] + + def pytest_collection_modifyitems(self, session, config, items): + self.items = list(items) + + collector = Collector() + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + pytest.main([path, "--collect-only", "-q", "--profile", profile], plugins=[collector]) + + runnable: List[str] = [] + for item in collector.items: + nodeid = item.nodeid + if not nodeid.startswith("tests/"): + continue + marker = item.get_closest_marker("skip_profile") + marked_profiles = set(marker.args) if marker else set() + if profile in marked_profiles: + continue + runnable.append(nodeid) + return sorted(set(runnable)) + + +def build_groups(all_runnable: List[str]) -> Tuple[List[str], List[str]]: + collected = set(all_runnable) + slow = sorted(collected & SLOW_NODEIDS) + fast = sorted(collected - set(slow)) + return fast, slow + + +def run_lane( + nodeids: List[str], + profile: str, + jobs: int, + timeout_seconds: int, + retries: int, + task_offset: int, + shards: int = 0, + cleanup_stale: bool = True, +): + if shards > 0: + chunked = build_shards(nodeids, min(shards, len(nodeids))) + tasks = [ + TaskSpec(task_index=task_offset + idx, test_paths=batch) + for idx, batch in enumerate(chunked, start=1) + ] + else: + tasks = [ + TaskSpec(task_index=task_offset + idx, test_paths=[nodeid]) + for idx, nodeid in enumerate(nodeids, start=1) + ] + return run_all_tasks( + tasks, + profile, + jobs=jobs, + timeout_seconds=timeout_seconds, + retries=retries, + cleanup_stale=cleanup_stale, + ) + + +def summarize_status_counts(results) -> dict: + combined = {} + for result in results: + combined.update(result.status_by_test) + counts = {"PASSED": 0, "SKIPPED": 0, "FAILED": 0, "ERROR": 0} + for status in combined.values(): + if status in counts: + counts[status] += 1 + counts["WITH_STATUS"] = len(combined) + return counts + + +def run_grouped_tail_lane( + nodeids: List[str], + profile: str, + timeout_seconds: int, + retries: int, + task_offset: int, + cleanup_stale: bool = True, +): + if not nodeids: + return [] + tasks = [TaskSpec(task_index=task_offset, test_paths=nodeids)] + return run_all_tasks( + tasks, + profile, + jobs=1, + timeout_seconds=timeout_seconds, + retries=retries, + cleanup_stale=cleanup_stale, + ) + + +def load_nodeids_from_file(path: Path) -> Set[str]: + if not path.exists(): + return set() + return { + line.strip() + for line in path.read_text().splitlines() + if line.strip() and not line.strip().startswith("#") + } + + +def run_fast_subpath(args: argparse.Namespace, fast: List[str], fast_tail: List[str]): + print(f"\nStep 1/2: Running fast subpath ({len(fast)} tests)...") + fast_main_results = run_lane( + nodeids=fast, + profile=args.profile, + jobs=args.fast_jobs, + timeout_seconds=args.fast_timeout_seconds, + retries=args.retries, + task_offset=0, + shards=args.fast_shards, + cleanup_stale=not args.no_cleanup_stale_runners, + ) + fast_tail_results = [] + if fast_tail: + print(f"\nStep 1b/2: Running grouped fast tail ({len(fast_tail)} tests)...") + fast_tail_results = run_grouped_tail_lane( + nodeids=fast_tail, + profile=args.profile, + timeout_seconds=args.fast_tail_timeout_seconds, + retries=args.retries, + task_offset=4500, + cleanup_stale=not args.no_cleanup_stale_runners, + ) + return fast_main_results + fast_tail_results + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Effective spark_session test runner: execute fast subpath first, " + "then run remaining tests for full coverage." + ) + ) + parser.add_argument("--profile", default="spark_session", help="dbt-spark profile") + parser.add_argument( + "--test-scope", + default=DEFAULT_TEST_SCOPE, + help="Pytest path to collect tests from", + ) + parser.add_argument( + "--group", + choices=["all", "effective", "fast", "slow"], + default="effective", + help="Run effective (full), fast subpath only, or slow lane only", + ) + parser.add_argument("--fast-jobs", type=int, default=6, help="Parallel jobs for fast lane") + parser.add_argument( + "--fast-shards", + type=int, + default=6, + help="Number of batched tasks for fast lane (0 = one test per task)", + ) + parser.add_argument("--slow-jobs", type=int, default=1, help="Parallel jobs for slow lane") + parser.add_argument( + "--fast-timeout-seconds", type=int, default=600, help="Per-test timeout in fast lane" + ) + parser.add_argument( + "--slow-timeout-seconds", type=int, default=1500, help="Per-test timeout in slow lane" + ) + parser.add_argument("--retries", type=int, default=1, help="Retries per failed/timed-out test") + parser.add_argument( + "--fast-list-file", + default=str(DEFAULT_FAST_LIST_FILE), + help="Optional newline-delimited fast test list; used if file exists", + ) + parser.add_argument( + "--exclude-list-files", + nargs="*", + default=[str(path) for path in DEFAULT_EXCLUDED_LIST_FILES], + help="Optional newline-delimited lists of problematic tests to exclude from fast lane", + ) + parser.add_argument( + "--fast-tail-list-file", + default=str(DEFAULT_FAST_TAIL_LIST_FILE), + help="Optional newline-delimited tests to run as one grouped tail task after fast lane", + ) + parser.add_argument( + "--slow-working-list-file", + default=str(DEFAULT_SLOW_WORKING_LIST_FILE), + help="Optional newline-delimited slow-but-working tests to force into effective runner's slow lane", + ) + parser.add_argument( + "--fast-tail-timeout-seconds", + type=int, + default=600, + help="Timeout for grouped fast-tail task", + ) + parser.add_argument("--dry-run", action="store_true", help="Print grouped tests and exit") + parser.add_argument( + "--effective-lanes-parallel", + action="store_true", + help="Run effective mode with fast subpath and slow lane in parallel", + ) + parser.add_argument( + "--no-cleanup-stale-runners", + action="store_true", + help="Skip preflight cleanup of stale run_dbt_spark_tests.py processes", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if args.group == "effective": + args.group = "all" + + all_runnable = collect_runnable_nodeids(args.test_scope, args.profile) + computed_fast, slow_seed = build_groups(all_runnable) + runnable_set = set(all_runnable) + + excluded_from_files: Set[str] = set() + for raw_path in args.exclude_list_files: + excluded_from_files.update(load_nodeids_from_file(Path(raw_path))) + excluded_from_files = excluded_from_files & runnable_set + if excluded_from_files: + slow_seed = sorted(set(slow_seed) | excluded_from_files) + computed_fast = sorted(runnable_set - set(slow_seed)) + + fast_tail_path = Path(args.fast_tail_list_file) + fast_tail = sorted(load_nodeids_from_file(fast_tail_path) & runnable_set) + if fast_tail: + slow_seed = sorted(set(slow_seed) - set(fast_tail)) + computed_fast = sorted(set(computed_fast) - set(fast_tail)) + + slow_working_path = Path(args.slow_working_list_file) + slow_working = sorted(load_nodeids_from_file(slow_working_path) & runnable_set) + if slow_working: + slow_seed = sorted(set(slow_seed) | set(slow_working)) + computed_fast = sorted(runnable_set - set(slow_seed) - set(fast_tail)) + + fast = computed_fast + fast_list_path = Path(args.fast_list_file) + if fast_list_path.exists(): + listed_fast = load_nodeids_from_file(fast_list_path) & runnable_set + loaded_fast = sorted(listed_fast) + if loaded_fast: + fast = loaded_fast + # Explicit fast-list entries take precedence over seeded slow buckets. + slow_seed = sorted(set(slow_seed) - set(fast)) + print(f"Using fast list from {fast_list_path}") + else: + print( + "Fast list file exists but had no usable runnable entries; using computed fast group." + ) + if excluded_from_files: + print(f"Loaded {len(excluded_from_files)} excluded test(s) from --exclude-list-files.") + if fast_tail: + print(f"Loaded {len(fast_tail)} grouped fast-tail test(s) from {fast_tail_path}.") + if slow_working: + print(f"Loaded {len(slow_working)} slow-working test(s) from {slow_working_path}.") + + print(f"Profile: {args.profile}") + print(f"Scope: {args.test_scope}") + print(f"Total runnable tests: {len(all_runnable)}") + print(f"Fast tests: {len(fast)}") + print(f"Fast tail grouped tests: {len(fast_tail)}") + print(f"Slow working tests: {len(slow_working)}") + print(f"Slow seed tests: {len(slow_seed)}") + + if args.dry_run: + print("\nFast group:") + for nodeid in fast: + print(f" {nodeid}") + print("\nFast tail grouped task:") + for nodeid in fast_tail: + print(f" {nodeid}") + print("\nSlow seed group:") + for nodeid in slow_seed: + print(f" {nodeid}") + return 0 + + if args.group == "fast": + fast_results = run_fast_subpath(args, fast, fast_tail) + failed_fast_tasks = [r for r in fast_results if r.return_code != 0] + fast_counts = summarize_status_counts(fast_results) + print("\nFast-only result:") + print(f" Tasks failed: {len(failed_fast_tasks)}") + print(f" PASSED: {fast_counts['PASSED']}") + print(f" SKIPPED: {fast_counts['SKIPPED']}") + print(f" FAILED: {fast_counts['FAILED']}") + print(f" ERROR: {fast_counts['ERROR']}") + print(f" WITH STATUS: {fast_counts['WITH_STATUS']}") + return 0 if not failed_fast_tasks else 1 + + if args.group == "slow": + print(f"\nRunning slow lane only ({len(slow_seed)} tests)...") + slow_results = run_lane( + nodeids=slow_seed, + profile=args.profile, + jobs=args.slow_jobs, + timeout_seconds=args.slow_timeout_seconds, + retries=args.retries, + task_offset=5000, + cleanup_stale=not args.no_cleanup_stale_runners, + ) + failed_slow = [r for r in slow_results if r.return_code != 0] + print( + f"\nSlow-only result: passed={len(slow_results) - len(failed_slow)} failed={len(failed_slow)}" + ) + return 0 if not failed_slow else 1 + + # Full coverage effective run: + # 1) Run fast subpath. + # 2) Run remaining slow lane. + # Optionally run steps 1 and 2 in parallel. + if args.effective_lanes_parallel: + print("\nRunning effective mode with parallel lanes...") + if not args.no_cleanup_stale_runners: + cleanup_stale_runner_processes(args.profile) + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + fast_future = executor.submit(run_fast_subpath, args, fast, fast_tail) + slow_future = executor.submit( + run_lane, + slow_seed, + args.profile, + args.slow_jobs, + args.slow_timeout_seconds, + args.retries, + 5000, + 0, + False, + ) + fast_results = fast_future.result() + slow_results = slow_future.result() + else: + fast_results = run_fast_subpath(args, fast, fast_tail) + + moved_from_fast = sorted( + [ + path + for result in fast_results + if result.return_code != 0 + for path in result.test_paths + ] + ) + if moved_from_fast: + print(f"\nMoving {len(moved_from_fast)} test(s) from fast lane to slow lane.") + + final_slow = sorted(set(slow_seed) | set(moved_from_fast)) + print(f"\nStep 2/2: Running remaining slow lane ({len(final_slow)} tests)...") + slow_results = run_lane( + nodeids=final_slow, + profile=args.profile, + jobs=args.slow_jobs, + timeout_seconds=args.slow_timeout_seconds, + retries=args.retries, + task_offset=5000, + cleanup_stale=not args.no_cleanup_stale_runners, + ) + + moved_from_fast = sorted( + [path for result in fast_results if result.return_code != 0 for path in result.test_paths] + ) + if args.effective_lanes_parallel and moved_from_fast: + print(f"\nRunning follow-up slow rerun for {len(moved_from_fast)} fast-lane failure(s)...") + followup_results = run_lane( + nodeids=moved_from_fast, + profile=args.profile, + jobs=args.slow_jobs, + timeout_seconds=args.slow_timeout_seconds, + retries=args.retries, + task_offset=7000, + cleanup_stale=False, + ) + slow_results.extend(followup_results) + + final_slow = sorted(set(slow_seed) | set(moved_from_fast)) + + failed_fast_final = [result for result in fast_results if result.return_code != 0] + failed_slow_final = [result for result in slow_results if result.return_code != 0] + total_failed = len(failed_fast_final) + len(failed_slow_final) + + print("\nTiered summary:") + print(f" Total runnable: {len(all_runnable)}") + print(f" Fast lane initial: {len(fast)}") + print(f" Fast tail grouped: {len(fast_tail)}") + print(f" Slow lane seed: {len(slow_seed)}") + print(f" Moved fast->slow: {len(moved_from_fast)}") + print(f" Slow lane final: {len(final_slow)}") + print(f" Fast passed (first pass): {len(fast_results) - len(failed_fast_final)}") + print(f" Slow passed: {len(slow_results) - len(failed_slow_final)}") + print(f" Total failed: {total_failed}") + + return 0 if total_failed == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dbt-spark/dagger/test_lists/spark_session_errored.txt b/dbt-spark/dagger/test_lists/spark_session_errored.txt new file mode 100644 index 0000000000..009bc12734 --- /dev/null +++ b/dbt-spark/dagger/test_lists/spark_session_errored.txt @@ -0,0 +1 @@ +tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestDeltaStrategies::test_delta_strategies_overwrite diff --git a/dbt-spark/dagger/test_lists/spark_session_fast.txt b/dbt-spark/dagger/test_lists/spark_session_fast.txt new file mode 100644 index 0000000000..dc3c4bfbc6 --- /dev/null +++ b/dbt-spark/dagger/test_lists/spark_session_fast.txt @@ -0,0 +1,44 @@ +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowSqlHeader::test_sql_header +tests/functional/adapter/empty/test_empty.py::TestSparkEmptyInlineSourceRef::test_run_with_empty +tests/functional/adapter/empty/test_empty.py::TestSparkEmpty::test_run_with_empty +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_fail_on_schema_change +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChangeSpecialChars::test_incremental_append_new_columns_with_special_characters +tests/functional/adapter/test_basic.py::TestEmptySpark::test_empty +tests/functional/adapter/test_basic.py::TestSingularTestsEphemeralSpark::test_singular_tests_ephemeral +tests/functional/adapter/test_basic.py::TestSingularTestsSpark::test_singular_tests +tests/functional/adapter/test_simple_seed.py::TestBigQueryEmptySeed::test_empty_seeds +tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestCaseInsensitivity::test_case_insensitivity +tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestInvalidInput::test_invalid_input +tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestingTypes::test_unit_test_data_type +tests/functional/adapter/utils/test_data_types.py::TestTypeBigInt::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeBoolean::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeFloat::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeInt::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeNumeric::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeString::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeTimestamp::test_check_types_assert_match +tests/functional/adapter/utils/test_timestamps.py::TestCurrentTimestampSpark::test_current_timestamps +tests/functional/adapter/utils/test_utils.py::TestAnyValue::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestArrayAppend::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestArrayAppend::test_expected_actual +tests/functional/adapter/utils/test_utils.py::TestArrayConcat::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestArrayConcat::test_expected_actual +tests/functional/adapter/utils/test_utils.py::TestArrayConstruct::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestArrayConstruct::test_expected_actual +tests/functional/adapter/utils/test_utils.py::TestBoolOr::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestCast::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestCastBoolToText::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestCurrentTimestamp::test_current_timestamp_matches_utc +tests/functional/adapter/utils/test_utils.py::TestCurrentTimestamp::test_current_timestamp_type +tests/functional/adapter/utils/test_utils.py::TestDate::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestDateAdd::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestDateTrunc::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestEquals::test_equal_values +tests/functional/adapter/utils/test_utils.py::TestEscapeSingleQuotes::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestExcept::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestIntersect::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestLength::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestPosition::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestSafeCast::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestSplitPart::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestStringLiteral::test_build_assert_equal diff --git a/dbt-spark/dagger/test_lists/spark_session_fast_tail_grouped.txt b/dbt-spark/dagger/test_lists/spark_session_fast_tail_grouped.txt new file mode 100644 index 0000000000..d5f7fe5071 --- /dev/null +++ b/dbt-spark/dagger/test_lists/spark_session_fast_tail_grouped.txt @@ -0,0 +1,3 @@ +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args0-5] +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args1-3] +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args2-7] diff --git a/dbt-spark/dagger/test_lists/spark_session_slow_but_working.txt b/dbt-spark/dagger/test_lists/spark_session_slow_but_working.txt new file mode 100644 index 0000000000..c1a4c7a0dd --- /dev/null +++ b/dbt-spark/dagger/test_lists/spark_session_slow_but_working.txt @@ -0,0 +1,3 @@ +# Tests known to be valid but currently high-latency in spark_session. +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_ignore +tests/functional/adapter/test_basic.py::TestGenericTestsSpark::test_generic_tests diff --git a/dbt-spark/dagger/test_lists/spark_session_timed_out.txt b/dbt-spark/dagger/test_lists/spark_session_timed_out.txt new file mode 100644 index 0000000000..a671715004 --- /dev/null +++ b/dbt-spark/dagger/test_lists/spark_session_timed_out.txt @@ -0,0 +1,7 @@ +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args0-5] +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args1-3] +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args2-7] +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_ignore +tests/functional/adapter/test_basic.py::TestGenericTestsSpark::test_generic_tests +tests/functional/adapter/utils/test_utils.py::TestEquals::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestListagg::test_build_assert_equal diff --git a/dbt-spark/hatch.toml b/dbt-spark/hatch.toml index f5edc69dab..b59148c7b3 100644 --- a/dbt-spark/hatch.toml +++ b/dbt-spark/hatch.toml @@ -6,6 +6,7 @@ packages = ["src/dbt"] sources = ["src"] [envs.default] +python = "3.12" pre-install-commands = [ "pip install -e ../dbt-adapters", "pip install -e ../dbt-tests-adapter", @@ -13,7 +14,6 @@ pre-install-commands = [ dependencies = [ "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", - "ddtrace==2.3.0", "ipdb~=0.13.13", "pre-commit==3.7.0", "freezegun", @@ -35,6 +35,16 @@ setup = [ code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" integration-tests = "python dagger/run_dbt_spark_tests.py {args:--profile apache_spark}" +integration-tests-sharded = "python dagger/run_dbt_spark_tests_sharded.py {args:--profile spark_session}" +integration-tests-safe = "python dagger/run_dbt_spark_tests_sharded.py {args:--profile spark_session --mode files --jobs 3 --retries 1 --timeout-seconds 1500 --fallback-serial --always-split-path tests/functional/adapter/incremental/test_incremental_on_schema_change.py tests/functional/adapter/utils/test_utils.py}" +# Effective spark_session runner (fast subpath + remaining coverage). +integration-tests-spark-session-effective = "python dagger/run_dbt_spark_tests_tiered.py {args:--profile spark_session --group effective}" +# Fast subpath only (used by effective runner step 1). +integration-tests-spark-session-fast-subpath = "python dagger/run_dbt_spark_tests_tiered.py {args:--profile spark_session --group fast}" +integration-tests-spark-session-fast = "python dagger/run_dbt_spark_tests_tiered.py {args:--profile spark_session --group fast}" +integration-tests-spark-session-slow = "python dagger/run_dbt_spark_tests_tiered.py {args:--profile spark_session --group slow}" +integration-tests-spark-session-tiered-all = "python dagger/run_dbt_spark_tests_tiered.py {args:--profile spark_session --group all}" +integration-tests-spark-session-audit-speeds = "python dagger/audit_spark_session_test_speeds.py {args:--profile spark_session --scope tests/functional/adapter --jobs 3 --slow-timeout-seconds 180}" docker-prod = "docker build -f docker/Dockerfile -t dbt-spark ." [envs.build] @@ -70,7 +80,7 @@ pre-install-commands = [ dependencies = [ "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", - "ddtrace==2.3.0", + "freezegun", "pytest>=7.0,<8.0", "pytest-csv~=3.0", @@ -91,7 +101,7 @@ integration-tests = [ [envs.cd] pre-install-commands = [] dependencies = [ - "ddtrace==2.3.0", + "freezegun", "pytest>=7.0,<8.0", "pytest-csv~=3.0", diff --git a/dbt-spark/test_results.txt b/dbt-spark/test_results.txt new file mode 100644 index 0000000000..d999d85f08 --- /dev/null +++ b/dbt-spark/test_results.txt @@ -0,0 +1,539 @@ +Obtaining file:///dbt-adapters + Installing build dependencies: started + Installing build dependencies: finished with status 'done' + Checking if build backend supports build_editable: started + Checking if build backend supports build_editable: finished with status 'done' + Getting requirements to build editable: started + Getting requirements to build editable: finished with status 'done' + Installing backend dependencies: started + Installing backend dependencies: finished with status 'done' + Preparing editable metadata (pyproject.toml): started + Preparing editable metadata (pyproject.toml): finished with status 'done' +Collecting agate<2.0,>=1.0 (from dbt-adapters==1.22.9) + Using cached agate-1.14.2-py3-none-any.whl.metadata (3.1 kB) +Collecting dbt-common<2.0,>=1.36 (from dbt-adapters==1.22.9) + Using cached dbt_common-1.37.3-py3-none-any.whl.metadata (4.9 kB) +Collecting dbt-protos<2.0,>=1.0.291 (from dbt-adapters==1.22.9) + Using cached dbt_protos-1.0.443-py3-none-any.whl.metadata (859 bytes) +Collecting mashumaro<3.15,>=3.9 (from mashumaro[msgpack]<3.15,>=3.9->dbt-adapters==1.22.9) + Using cached mashumaro-3.14-py3-none-any.whl.metadata (114 kB) +Collecting protobuf<7.0,>=6.0 (from dbt-adapters==1.22.9) + Using cached protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl.metadata (593 bytes) +Collecting pytz>=2015.7 (from dbt-adapters==1.22.9) + Using cached pytz-2026.1.post1-py2.py3-none-any.whl.metadata (22 kB) +Collecting typing-extensions<5.0,>=4.0 (from dbt-adapters==1.22.9) + Using cached typing_extensions-4.15.0-py3-none-any.whl.metadata (3.3 kB) +Collecting Babel>=2.0 (from agate<2.0,>=1.0->dbt-adapters==1.22.9) + Using cached babel-2.18.0-py3-none-any.whl.metadata (2.2 kB) +Collecting isodate>=0.5.4 (from agate<2.0,>=1.0->dbt-adapters==1.22.9) + Using cached isodate-0.7.2-py3-none-any.whl.metadata (11 kB) +Collecting leather>=0.3.2 (from agate<2.0,>=1.0->dbt-adapters==1.22.9) + Using cached leather-0.4.1-py3-none-any.whl.metadata (3.0 kB) +Collecting parsedatetime!=2.5,>=2.1 (from agate<2.0,>=1.0->dbt-adapters==1.22.9) + Using cached parsedatetime-2.6-py3-none-any.whl.metadata (4.7 kB) +Collecting python-slugify>=1.2.1 (from agate<2.0,>=1.0->dbt-adapters==1.22.9) + Using cached python_slugify-8.0.4-py2.py3-none-any.whl.metadata (8.5 kB) +Collecting pytimeparse>=1.1.5 (from agate<2.0,>=1.0->dbt-adapters==1.22.9) + Using cached pytimeparse-1.1.8-py2.py3-none-any.whl.metadata (3.4 kB) +Collecting agate<2.0,>=1.0 (from dbt-adapters==1.22.9) + Using cached agate-1.9.1-py2.py3-none-any.whl.metadata (3.2 kB) +Collecting colorama<0.5,>=0.3.9 (from dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached colorama-0.4.6-py2.py3-none-any.whl.metadata (17 kB) +Collecting deepdiff<9.0,>=7.0 (from dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached deepdiff-8.6.2-py3-none-any.whl.metadata (8.8 kB) +Collecting jinja2<4,>=3.1.3 (from dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached jinja2-3.1.6-py3-none-any.whl.metadata (2.9 kB) +Collecting jsonschema<5.0,>=4.0 (from dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached jsonschema-4.26.0-py3-none-any.whl.metadata (7.6 kB) +Collecting pathspec<0.13,>=0.9 (from dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached pathspec-0.12.1-py3-none-any.whl.metadata (21 kB) +Collecting python-dateutil<3.0,>=2.0 (from dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl.metadata (8.4 kB) +Collecting requests<3.0.0 (from dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached requests-2.33.1-py3-none-any.whl.metadata (4.8 kB) +Collecting orderly-set<6,>=5.4.1 (from deepdiff<9.0,>=7.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached orderly_set-5.5.0-py3-none-any.whl.metadata (6.6 kB) +Collecting MarkupSafe>=2.0 (from jinja2<4,>=3.1.3->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (2.7 kB) +Collecting attrs>=22.2.0 (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached attrs-26.1.0-py3-none-any.whl.metadata (8.8 kB) +Collecting jsonschema-specifications>=2023.03.6 (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached jsonschema_specifications-2025.9.1-py3-none-any.whl.metadata (2.9 kB) +Collecting referencing>=0.28.4 (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached referencing-0.37.0-py3-none-any.whl.metadata (2.8 kB) +Collecting rpds-py>=0.25.0 (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.1 kB) +Collecting msgpack>=0.5.6 (from mashumaro[msgpack]<3.15,>=3.9->dbt-adapters==1.22.9) + Using cached msgpack-1.1.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (8.1 kB) +Collecting six>=1.5 (from python-dateutil<3.0,>=2.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached six-1.17.0-py2.py3-none-any.whl.metadata (1.7 kB) +Collecting charset_normalizer<4,>=2 (from requests<3.0.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (40 kB) +Collecting idna<4,>=2.5 (from requests<3.0.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached idna-3.11-py3-none-any.whl.metadata (8.4 kB) +Collecting urllib3<3,>=1.26 (from requests<3.0.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached urllib3-2.6.3-py3-none-any.whl.metadata (6.9 kB) +Collecting certifi>=2023.5.7 (from requests<3.0.0->dbt-common<2.0,>=1.36->dbt-adapters==1.22.9) + Using cached certifi-2026.2.25-py3-none-any.whl.metadata (2.5 kB) +Collecting text-unidecode>=1.3 (from python-slugify>=1.2.1->agate<2.0,>=1.0->dbt-adapters==1.22.9) + Using cached text_unidecode-1.3-py2.py3-none-any.whl.metadata (2.4 kB) +Using cached dbt_common-1.37.3-py3-none-any.whl (87 kB) +Using cached agate-1.9.1-py2.py3-none-any.whl (95 kB) +Using cached colorama-0.4.6-py2.py3-none-any.whl (25 kB) +Using cached dbt_protos-1.0.443-py3-none-any.whl (186 kB) +Using cached deepdiff-8.6.2-py3-none-any.whl (91 kB) +Using cached isodate-0.7.2-py3-none-any.whl (22 kB) +Using cached jinja2-3.1.6-py3-none-any.whl (134 kB) +Using cached jsonschema-4.26.0-py3-none-any.whl (90 kB) +Using cached mashumaro-3.14-py3-none-any.whl (92 kB) +Using cached orderly_set-5.5.0-py3-none-any.whl (13 kB) +Using cached pathspec-0.12.1-py3-none-any.whl (31 kB) +Using cached protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl (323 kB) +Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) +Using cached requests-2.33.1-py3-none-any.whl (64 kB) +Using cached charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (207 kB) +Using cached idna-3.11-py3-none-any.whl (71 kB) +Using cached typing_extensions-4.15.0-py3-none-any.whl (44 kB) +Using cached urllib3-2.6.3-py3-none-any.whl (131 kB) +Using cached attrs-26.1.0-py3-none-any.whl (67 kB) +Using cached babel-2.18.0-py3-none-any.whl (10.2 MB) +Using cached certifi-2026.2.25-py3-none-any.whl (153 kB) +Using cached jsonschema_specifications-2025.9.1-py3-none-any.whl (18 kB) +Using cached leather-0.4.1-py3-none-any.whl (30 kB) +Using cached markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (22 kB) +Using cached msgpack-1.1.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (427 kB) +Using cached parsedatetime-2.6-py3-none-any.whl (42 kB) +Using cached python_slugify-8.0.4-py2.py3-none-any.whl (10 kB) +Using cached pytimeparse-1.1.8-py2.py3-none-any.whl (10.0 kB) +Using cached pytz-2026.1.post1-py2.py3-none-any.whl (510 kB) +Using cached referencing-0.37.0-py3-none-any.whl (26 kB) +Using cached rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (394 kB) +Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) +Using cached text_unidecode-1.3-py2.py3-none-any.whl (78 kB) +Building wheels for collected packages: dbt-adapters + Building editable for dbt-adapters (pyproject.toml): started + Building editable for dbt-adapters (pyproject.toml): finished with status 'done' + Created wheel for dbt-adapters: filename=dbt_adapters-1.22.9-py3-none-any.whl size=6737 sha256=4ba22a9f5c88aaa0f14ab0dc54869c1235726737a0fdef0884593f8f8b0fcf8b + Stored in directory: /tmp/pip-ephem-wheel-cache-uz474tpw/wheels/9f/29/3a/fd477b8fd9e53b7ef60b1d83670c4eb11250b0cd70e60d2dcd +Successfully built dbt-adapters +Installing collected packages: text-unidecode, pytz, pytimeparse, parsedatetime, leather, urllib3, typing-extensions, six, rpds-py, python-slugify, protobuf, pathspec, orderly-set, msgpack, MarkupSafe, isodate, idna, colorama, charset_normalizer, certifi, Babel, attrs, requests, referencing, python-dateutil, mashumaro, jinja2, deepdiff, dbt-protos, agate, jsonschema-specifications, jsonschema, dbt-common, dbt-adapters + +Successfully installed Babel-2.18.0 MarkupSafe-3.0.3 agate-1.9.1 attrs-26.1.0 certifi-2026.2.25 charset_normalizer-3.4.6 colorama-0.4.6 dbt-adapters-1.22.9 dbt-common-1.37.3 dbt-protos-1.0.443 deepdiff-8.6.2 idna-3.11 isodate-0.7.2 jinja2-3.1.6 jsonschema-4.26.0 jsonschema-specifications-2025.9.1 leather-0.4.1 mashumaro-3.14 msgpack-1.1.2 orderly-set-5.5.0 parsedatetime-2.6 pathspec-0.12.1 protobuf-6.33.6 python-dateutil-2.9.0.post0 python-slugify-8.0.4 pytimeparse-1.1.8 pytz-2026.1.post1 referencing-0.37.0 requests-2.33.1 rpds-py-0.30.0 six-1.17.0 text-unidecode-1.3 typing-extensions-4.15.0 urllib3-2.6.3 +Obtaining file:///dbt-tests-adapter + Installing build dependencies: started + Installing build dependencies: finished with status 'done' + Checking if build backend supports build_editable: started + Checking if build backend supports build_editable: finished with status 'done' + Getting requirements to build editable: started + Getting requirements to build editable: finished with status 'done' + Installing backend dependencies: started + Installing backend dependencies: finished with status 'done' + Preparing editable metadata (pyproject.toml): started + Preparing editable metadata (pyproject.toml): finished with status 'done' +Requirement already satisfied: dbt-adapters<2.0,>=1.14.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-tests-adapter==1.19.7) (1.22.9) +Requirement already satisfied: dbt-common<2.0,>=1.34.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-tests-adapter==1.19.7) (1.37.3) +Collecting dbt-core>=1.8.0a1 (from dbt-tests-adapter==1.19.7) + Using cached dbt_core-1.11.7-py3-none-any.whl.metadata (4.4 kB) +Collecting freezegun (from dbt-tests-adapter==1.19.7) + Using cached freezegun-1.5.5-py3-none-any.whl.metadata (13 kB) +Collecting pyyaml (from dbt-tests-adapter==1.19.7) + Using cached pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (2.4 kB) +Requirement already satisfied: agate<2.0,>=1.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (1.9.1) +Requirement already satisfied: dbt-protos<2.0,>=1.0.291 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (1.0.443) +Requirement already satisfied: mashumaro<3.15,>=3.9 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from mashumaro[msgpack]<3.15,>=3.9->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (3.14) +Requirement already satisfied: protobuf<7.0,>=6.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (6.33.6) +Requirement already satisfied: pytz>=2015.7 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (2026.1.post1) +Requirement already satisfied: typing-extensions<5.0,>=4.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (4.15.0) +Requirement already satisfied: Babel>=2.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from agate<2.0,>=1.0->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (2.18.0) +Requirement already satisfied: isodate>=0.5.4 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from agate<2.0,>=1.0->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (0.7.2) +Requirement already satisfied: leather>=0.3.2 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from agate<2.0,>=1.0->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (0.4.1) +Requirement already satisfied: parsedatetime!=2.5,>=2.1 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from agate<2.0,>=1.0->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (2.6) +Requirement already satisfied: python-slugify>=1.2.1 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from agate<2.0,>=1.0->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (8.0.4) +Requirement already satisfied: pytimeparse>=1.1.5 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from agate<2.0,>=1.0->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (1.1.8) +Requirement already satisfied: colorama<0.5,>=0.3.9 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (0.4.6) +Requirement already satisfied: deepdiff<9.0,>=7.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (8.6.2) +Requirement already satisfied: jinja2<4,>=3.1.3 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (3.1.6) +Requirement already satisfied: jsonschema<5.0,>=4.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (4.26.0) +Requirement already satisfied: pathspec<0.13,>=0.9 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (0.12.1) +Requirement already satisfied: python-dateutil<3.0,>=2.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (2.9.0.post0) +Requirement already satisfied: requests<3.0.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (2.33.1) +Requirement already satisfied: orderly-set<6,>=5.4.1 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from deepdiff<9.0,>=7.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (5.5.0) +Requirement already satisfied: MarkupSafe>=2.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from jinja2<4,>=3.1.3->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (3.0.3) +Requirement already satisfied: attrs>=22.2.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (26.1.0) +Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (2025.9.1) +Requirement already satisfied: referencing>=0.28.4 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (0.37.0) +Requirement already satisfied: rpds-py>=0.25.0 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from jsonschema<5.0,>=4.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (0.30.0) +Requirement already satisfied: msgpack>=0.5.6 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from mashumaro[msgpack]<3.15,>=3.9->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (1.1.2) +Requirement already satisfied: six>=1.5 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from python-dateutil<3.0,>=2.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (1.17.0) +Requirement already satisfied: charset_normalizer<4,>=2 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from requests<3.0.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (3.4.6) +Requirement already satisfied: idna<4,>=2.5 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from requests<3.0.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (3.11) +Requirement already satisfied: urllib3<3,>=1.26 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from requests<3.0.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (2.6.3) +Requirement already satisfied: certifi>=2023.5.7 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from requests<3.0.0->dbt-common<2.0,>=1.34.0->dbt-tests-adapter==1.19.7) (2026.2.25) +Collecting click<9.0,>=8.3.0 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached click-8.3.1-py3-none-any.whl.metadata (2.6 kB) +Collecting daff>=1.3.46 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached daff-1.4.2-py3-none-any.whl.metadata (10 kB) +Collecting dbt-extractor<=0.6,>=0.5.0 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.6 kB) +Collecting dbt-semantic-interfaces<0.10,>=0.9.0 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached dbt_semantic_interfaces-0.9.0-py3-none-any.whl.metadata (2.6 kB) +Collecting networkx<4.0,>=2.3 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached networkx-3.6.1-py3-none-any.whl.metadata (6.8 kB) +Collecting packaging>20.9 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached packaging-26.0-py3-none-any.whl.metadata (3.3 kB) +Collecting pydantic<3 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached pydantic-2.12.5-py3-none-any.whl.metadata (90 kB) +Collecting snowplow-tracker<2.0,>=1.0.2 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached snowplow_tracker-1.1.0-py3-none-any.whl.metadata (5.7 kB) +Collecting sqlparse<0.5.5,>=0.5.0 (from dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached sqlparse-0.5.4-py3-none-any.whl.metadata (4.7 kB) +Collecting importlib-metadata<9,>=6.0 (from dbt-semantic-interfaces<0.10,>=0.9.0->dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached importlib_metadata-8.9.0-py3-none-any.whl.metadata (4.5 kB) +Collecting more-itertools<11.0,>=8.0 (from dbt-semantic-interfaces<0.10,>=0.9.0->dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached more_itertools-10.8.0-py3-none-any.whl.metadata (39 kB) +Collecting zipp>=3.20 (from importlib-metadata<9,>=6.0->dbt-semantic-interfaces<0.10,>=0.9.0->dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached zipp-3.23.0-py3-none-any.whl.metadata (3.6 kB) +Collecting annotated-types>=0.6.0 (from pydantic<3->dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached annotated_types-0.7.0-py3-none-any.whl.metadata (15 kB) +Collecting pydantic-core==2.41.5 (from pydantic<3->dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.3 kB) +Collecting typing-inspection>=0.4.2 (from pydantic<3->dbt-core>=1.8.0a1->dbt-tests-adapter==1.19.7) + Using cached typing_inspection-0.4.2-py3-none-any.whl.metadata (2.6 kB) +Requirement already satisfied: text-unidecode>=1.3 in /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/lib/python3.12/site-packages (from python-slugify>=1.2.1->agate<2.0,>=1.0->dbt-adapters<2.0,>=1.14.0->dbt-tests-adapter==1.19.7) (1.3) +Using cached dbt_core-1.11.7-py3-none-any.whl (1.0 MB) +Using cached click-8.3.1-py3-none-any.whl (108 kB) +Using cached dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (442 kB) +Using cached dbt_semantic_interfaces-0.9.0-py3-none-any.whl (147 kB) +Using cached importlib_metadata-8.9.0-py3-none-any.whl (27 kB) +Using cached more_itertools-10.8.0-py3-none-any.whl (69 kB) +Using cached networkx-3.6.1-py3-none-any.whl (2.1 MB) +Using cached pydantic-2.12.5-py3-none-any.whl (463 kB) +Using cached pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB) +Using cached pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (807 kB) +Using cached snowplow_tracker-1.1.0-py3-none-any.whl (44 kB) +Using cached sqlparse-0.5.4-py3-none-any.whl (45 kB) +Using cached annotated_types-0.7.0-py3-none-any.whl (13 kB) +Using cached daff-1.4.2-py3-none-any.whl (144 kB) +Using cached packaging-26.0-py3-none-any.whl (74 kB) +Using cached typing_inspection-0.4.2-py3-none-any.whl (14 kB) +Using cached zipp-3.23.0-py3-none-any.whl (10 kB) +Using cached freezegun-1.5.5-py3-none-any.whl (19 kB) +Building wheels for collected packages: dbt-tests-adapter + Building editable for dbt-tests-adapter (pyproject.toml): started + Building editable for dbt-tests-adapter (pyproject.toml): finished with status 'done' + Created wheel for dbt-tests-adapter: filename=dbt_tests_adapter-1.19.7-py3-none-any.whl size=7285 sha256=5c68c5d67059edf6c3b535b7f98e0d2eac916db602b17a0b4d72bf4f8adc4223 + Stored in directory: /tmp/pip-ephem-wheel-cache-n61hgt2s/wheels/ba/2c/73/c6e0aedccba1b655bc8415507b8fc68862d843de1da8f54445 +Successfully built dbt-tests-adapter +Installing collected packages: daff, zipp, typing-inspection, sqlparse, pyyaml, pydantic-core, packaging, networkx, more-itertools, dbt-extractor, click, annotated-types, snowplow-tracker, pydantic, importlib-metadata, freezegun, dbt-semantic-interfaces, dbt-core, dbt-tests-adapter + +Successfully installed annotated-types-0.7.0 click-8.3.1 daff-1.4.2 dbt-core-1.11.7 dbt-extractor-0.6.0 dbt-semantic-interfaces-0.9.0 dbt-tests-adapter-1.19.7 freezegun-1.5.5 importlib-metadata-8.9.0 more-itertools-10.8.0 networkx-3.6.1 packaging-26.0 pydantic-2.12.5 pydantic-core-2.41.5 pyyaml-6.0.3 snowplow-tracker-1.1.0 sqlparse-0.5.4 typing-inspection-0.4.2 zipp-3.23.0 +============================= test session starts ============================== +platform linux -- Python 3.12.12, pytest-7.4.4, pluggy-1.6.0 -- /root/.local/share/hatch/env/virtual/dbt-spark/VsnhxLU2/dbt-spark/bin/python +cachedir: .pytest_cache +rootdir: /src +configfile: pyproject.toml +plugins: xdist-3.8.0, logbook-1.2.0, csv-3.0.0, dotenv-0.5.2, mock-3.15.1 +created: 18/18 workers +18 workers [142 items] + +scheduling tests via LoadScheduling + +tests/functional/adapter/test_basic.py::TestSingularTestsEphemeralSpark::test_singular_tests_ephemeral +tests/functional/adapter/test_basic.py::TestSnapshotTimestampSpark::test_snapshot_timestamp +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_order +tests/functional/adapter/test_basic.py::TestGenericTestsSpark::test_generic_tests +tests/functional/adapter/test_basic.py::TestSimpleMaterializationsSpark::test_base +tests/functional/adapter/test_basic.py::TestEphemeralSpark::test_ephemeral +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsDdlEnforcement::test__constraints_ddl +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_order +[gw13] [ 0%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_names +[gw13] [ 1%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_names +[gw10] [ 2%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_correct_column_data_types +[gw10] [ 2%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_correct_column_data_types +[gw12] [ 3%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_correct_column_data_types +[gw12] [ 4%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_correct_column_data_types +[gw8] [ 4%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_correct_column_data_types +[gw8] [ 5%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_correct_column_data_types +[gw14] [ 6%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_correct_column_data_types +[gw14] [ 7%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualDatabricksHTTP::test__constraints_correct_column_data_types +[gw1] [ 7%] SKIPPED tests/functional/adapter/test_basic.py::TestSingularTestsEphemeralSpark::test_singular_tests_ephemeral +[gw16] [ 8%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_correct_column_data_types +[gw16] [ 9%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_correct_column_data_types +[gw9] [ 9%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_names +[gw9] [ 10%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_names +[gw5] [ 11%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_names +[gw5] [ 11%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_names +[gw15] [ 12%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_names +[gw15] [ 13%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_names +[gw17] [ 14%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsDdlEnforcement::test__constraints_ddl +[gw11] [ 14%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_names +[gw11] [ 15%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualDatabricksHTTP::test__constraints_wrong_column_names +[gw6] [ 16%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_data_types +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_correct_column_data_types +[gw6] [ 16%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsColumnsEqualPyodbc::test__constraints_correct_column_data_types +[gw7] [ 17%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_order +tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_names +[gw7] [ 18%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkViewConstraintsColumnsEqualPyodbc::test__constraints_wrong_column_names +[gw4] [ 19%] SKIPPED tests/functional/adapter/test_basic.py::TestSnapshotTimestampSpark::test_snapshot_timestamp +tests/functional/adapter/test_grants.py::TestSeedGrantsSpark::test_seed_grants +tests/functional/adapter/test_basic.py::TestEmptySpark::test_empty +tests/functional/adapter/test_get_columns_in_relation.py::TestColumnsInRelation::test_get_columns_in_relation +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsRollback::test__constraints_enforcement_rollback +tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsRollback::test__constraints_enforcement_rollback +tests/functional/adapter/test_constraints.py::TestSparkModelConstraintsRuntimeEnforcement::test__model_constraints_ddl +tests/functional/adapter/test_grants.py::TestModelGrantsSpark::test_view_table_grants +tests/functional/adapter/test_grants.py::TestIncrementalGrantsSpark::test_incremental_grants +tests/functional/adapter/test_grants.py::TestSnapshotGrantsSpark::test_snapshot_grants +tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsDdlEnforcement::test__constraints_ddl +tests/functional/adapter/test_grants.py::TestInvalidGrantsSpark::test_invalid_grants +tests/functional/adapter/test_python_model.py::TestPythonModelSpark::test_singular_tests +tests/functional/adapter/test_python_model.py::TestPySpark::test_different_dataframes +[gw5] [ 19%] SKIPPED tests/functional/adapter/test_grants.py::TestSeedGrantsSpark::test_seed_grants +tests/functional/adapter/test_constraints.py::TestSparkConstraintQuotedColumn::test__constraints_ddl +[gw12] [ 20%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsRollback::test__constraints_enforcement_rollback +[gw10] [ 21%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkTableConstraintsRollback::test__constraints_enforcement_rollback +[gw8] [ 21%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkModelConstraintsRuntimeEnforcement::test__model_constraints_ddl +[gw16] [ 22%] SKIPPED tests/functional/adapter/test_grants.py::TestModelGrantsSpark::test_view_table_grants +[gw9] [ 23%] SKIPPED tests/functional/adapter/test_grants.py::TestIncrementalGrantsSpark::test_incremental_grants +[gw15] [ 23%] SKIPPED tests/functional/adapter/test_grants.py::TestSnapshotGrantsSpark::test_snapshot_grants +[gw17] [ 24%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkIncrementalConstraintsDdlEnforcement::test__constraints_ddl +[gw11] [ 25%] SKIPPED tests/functional/adapter/test_grants.py::TestInvalidGrantsSpark::test_invalid_grants +[gw6] [ 26%] SKIPPED tests/functional/adapter/test_python_model.py::TestPythonModelSpark::test_singular_tests +[gw7] [ 26%] SKIPPED tests/functional/adapter/test_python_model.py::TestPySpark::test_different_dataframes +[gw13] [ 27%] SKIPPED tests/functional/adapter/test_constraints.py::TestSparkConstraintQuotedColumn::test__constraints_ddl +[gw1] [ 28%] PASSED tests/functional/adapter/test_basic.py::TestEmptySpark::test_empty +tests/functional/adapter/test_python_model.py::TestPythonIncrementalModelSpark::test_incremental +tests/functional/adapter/test_basic.py::TestBaseAdapterMethod::test_adapter_methods +tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsProjectLevelView::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsProjectLevelOff::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsInteractions::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/test_store_test_failures.py::TestSparkStoreTestFailuresWithDelta::test_store_and_assert_failure_with_delta +tests/functional/adapter/test_store_test_failures.py::TestSparkStoreTestFailures::test_store_and_assert +tests/functional/adapter/test_simple_seed.py::TestBigQueryEmptySeed::test_empty_seeds +tests/functional/adapter/dbt_clone/test_dbt_clone.py::TestSparkClonePossible::test_can_clone_true +[gw5] [ 28%] SKIPPED tests/functional/adapter/test_python_model.py::TestPythonIncrementalModelSpark::test_incremental +tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsProjectLevelEphemeral::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsExceptions::test_tests_run_unsuccessfully_and_raise_appropriate_exception +tests/functional/adapter/test_python_model.py::TestChangingSchemaSpark::test_changing_schema_with_log_validation +tests/functional/adapter/dbt_clone/test_dbt_clone.py::TestSparkClonePossible::test_clone_no_state +[gw8] [ 29%] SKIPPED tests/functional/adapter/test_store_test_failures.py::TestSparkStoreTestFailuresWithDelta::test_store_and_assert_failure_with_delta +[gw7] [ 30%] SKIPPED tests/functional/adapter/dbt_clone/test_dbt_clone.py::TestSparkClonePossible::test_can_clone_true +tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsGeneric::test_tests_run_successfully_and_are_stored_as_expected +[gw1] [ 30%] SKIPPED tests/functional/adapter/test_python_model.py::TestChangingSchemaSpark::test_changing_schema_with_log_validation +[gw13] [ 31%] SKIPPED tests/functional/adapter/dbt_clone/test_dbt_clone.py::TestSparkClonePossible::test_clone_no_state +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args0-5] +tests/functional/adapter/empty/test_empty.py::TestSparkEmpty::test_run_with_empty +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestDeltaOnSchemaChange::test_run_incremental_append_new_columns +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestInsertOverwriteOnSchemaChange::test_run_incremental_fail_on_schema_change +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_ignore +[gw13] [ 32%] SKIPPED tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestDeltaOnSchemaChange::test_run_incremental_append_new_columns +tests/functional/adapter/incremental/test_incremental_predicates.py::TestPredicatesMergeSpark::test__incremental_predicates +[gw13] [ 33%] SKIPPED tests/functional/adapter/incremental/test_incremental_predicates.py::TestPredicatesMergeSpark::test__incremental_predicates +[gw12] [ 33%] PASSED tests/functional/adapter/test_simple_seed.py::TestBigQueryEmptySeed::test_empty_seeds +tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py::TestMergeExcludeColumns::test__merge_exclude_columns +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__one_unique_key +[gw12] [ 34%] SKIPPED tests/functional/adapter/incremental/test_incremental_merge_exclude_columns.py::TestMergeExcludeColumns::test__merge_exclude_columns +[gw13] [ 35%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__one_unique_key +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__no_unique_keys +[gw13] [ 35%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__no_unique_keys +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__empty_str_unique_key +[gw13] [ 36%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__empty_str_unique_key +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__unary_unique_key_list +[gw13] [ 37%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__unary_unique_key_list +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__duplicated_unary_unique_key_list +[gw13] [ 38%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__duplicated_unary_unique_key_list +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__trinary_unique_key_list +[gw13] [ 38%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__trinary_unique_key_list +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__trinary_unique_key_list_no_update +[gw13] [ 39%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__trinary_unique_key_list_no_update +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__bad_unique_key_list +[gw12] [ 40%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__bad_unique_key_list +tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestInsertOverwrite::test_insert_overwrite +tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestDefaultAppend::test_default_append +[gw14] [ 40%] PASSED tests/functional/adapter/test_get_columns_in_relation.py::TestColumnsInRelation::test_get_columns_in_relation +tests/functional/adapter/test_query_timeout.py::TestQueryTimeout::test_query_timeout_exceeded +[gw5] [ 41%] PASSED tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args0-5] +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChangeSpecialChars::test_incremental_append_new_columns_with_special_characters +[gw6] [ 42%] PASSED tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsExceptions::test_tests_run_unsuccessfully_and_raise_appropriate_exception +[gw4] [ 42%] PASSED tests/functional/adapter/test_basic.py::TestBaseAdapterMethod::test_adapter_methods +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestInsertOverwriteOnSchemaChange::test_run_incremental_ignore +[gw2] [ 43%] PASSED tests/functional/adapter/test_basic.py::TestEphemeralSpark::test_ephemeral +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args1-3] +tests/functional/adapter/test_basic.py::TestIncrementalSpark::test_incremental +[gw3] [ 44%] PASSED tests/functional/adapter/test_basic.py::TestGenericTestsSpark::test_generic_tests +tests/functional/adapter/test_basic.py::TestSnapshotCheckColsSpark::test_snapshot_check_cols +[gw3] [ 45%] SKIPPED tests/functional/adapter/test_basic.py::TestSnapshotCheckColsSpark::test_snapshot_check_cols +[gw1] [ 45%] PASSED tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestInsertOverwriteOnSchemaChange::test_run_incremental_fail_on_schema_change +tests/functional/adapter/seed_column_types/test_seed_column_types.py::TestSeedColumnTypesCast::test_column_seed_type +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__bad_unique_key +[gw1] [ 46%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__bad_unique_key +[gw11] [ 47%] PASSED tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsProjectLevelEphemeral::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestCaseInsensitivity::test_case_insensitivity +[gw8] [ 47%] PASSED tests/functional/adapter/empty/test_empty.py::TestSparkEmpty::test_run_with_empty +[gw14] [ 48%] PASSED tests/functional/adapter/test_query_timeout.py::TestQueryTimeout::test_query_timeout_exceeded +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_fail_on_schema_change +tests/functional/adapter/incremental/test_incremental_predicates.py::TestIncrementalPredicatesMergeSpark::test__incremental_predicates +[gw17] [ 49%] PASSED tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsGeneric::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestBadStrategies::test_bad_strategies +[gw8] [ 50%] SKIPPED tests/functional/adapter/incremental/test_incremental_predicates.py::TestIncrementalPredicatesMergeSpark::test__incremental_predicates +[gw4] [ 50%] PASSED tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args1-3] +tests/functional/adapter/utils/test_data_types.py::TestTypeFloat::test_check_types_assert_match +[gw9] [ 51%] PASSED tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsProjectLevelOff::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestDeltaOnSchemaChange::test_run_incremental_sync_all_columns +tests/functional/adapter/persist_docs/test_persist_docs.py::TestPersistDocsDeltaView::test_delta_comments +[gw17] [ 52%] SKIPPED tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestDeltaOnSchemaChange::test_run_incremental_sync_all_columns +[gw14] [ 52%] PASSED tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestBadStrategies::test_bad_strategies +[gw4] [ 53%] SKIPPED tests/functional/adapter/persist_docs/test_persist_docs.py::TestPersistDocsDeltaView::test_delta_comments +tests/functional/adapter/utils/test_data_types.py::TestTypeString::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeInt::test_check_types_assert_match +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowSqlHeader::test_sql_header +tests/functional/adapter/utils/test_data_types.py::TestTypeTimestamp::test_check_types_assert_match +[gw3] [ 54%] PASSED tests/functional/adapter/seed_column_types/test_seed_column_types.py::TestSeedColumnTypesCast::test_column_seed_type +[gw15] [ 54%] PASSED tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsProjectLevelView::test_tests_run_successfully_and_are_stored_as_expected +tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestingTypes::test_unit_test_data_type +[gw9] [ 55%] PASSED tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowSqlHeader::test_sql_header +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args2-7] +tests/functional/adapter/utils/test_utils.py::TestAnyValue::test_build_assert_equal +[gw1] [ 56%] PASSED tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestCaseInsensitivity::test_case_insensitivity +tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestInvalidInput::test_invalid_input +[gw7] [ 57%] PASSED tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_ignore +[gw13] [ 57%] PASSED tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestDefaultAppend::test_default_append +tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__empty_unique_key_list +tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestDeltaStrategies::test_delta_strategies_overwrite +[gw13] [ 58%] SKIPPED tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestDeltaStrategies::test_delta_strategies_overwrite +tests/functional/adapter/utils/test_utils.py::TestBoolOr::test_build_assert_equal +[gw7] [ 59%] SKIPPED tests/functional/adapter/incremental/test_incremental_unique_id.py::TestUniqueKeySpark::test__empty_unique_key_list +tests/functional/adapter/utils/test_utils.py::TestArrayConstruct::test_expected_actual +[gw8] [ 59%] PASSED tests/functional/adapter/utils/test_data_types.py::TestTypeFloat::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeNumeric::test_check_types_assert_match +[gw17] [ 60%] PASSED tests/functional/adapter/utils/test_data_types.py::TestTypeString::test_check_types_assert_match +tests/functional/adapter/utils/test_data_types.py::TestTypeBoolean::test_check_types_assert_match +[gw4] [ 61%] PASSED tests/functional/adapter/utils/test_data_types.py::TestTypeTimestamp::test_check_types_assert_match +[gw14] [ 61%] PASSED tests/functional/adapter/utils/test_data_types.py::TestTypeInt::test_check_types_assert_match +tests/functional/adapter/utils/test_utils.py::TestArrayAppend::test_build_assert_equal +[gw15] [ 62%] PASSED tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowLimit::test_limit[args2-7] +tests/functional/adapter/utils/test_timestamps.py::TestCurrentTimestampSpark::test_current_timestamps +tests/functional/adapter/utils/test_utils.py::TestArrayConcat::test_build_assert_equal +[gw11] [ 63%] PASSED tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChange::test_run_incremental_fail_on_schema_change +[gw1] [ 64%] PASSED tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestInvalidInput::test_invalid_input +tests/functional/adapter/utils/test_data_types.py::TestTypeBigInt::test_check_types_assert_match +[gw9] [ 64%] PASSED tests/functional/adapter/utils/test_utils.py::TestAnyValue::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestArrayConstruct::test_build_assert_equal +[gw12] [ 65%] PASSED tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestInsertOverwrite::test_insert_overwrite +tests/functional/adapter/utils/test_utils.py::TestArrayConcat::test_expected_actual +[gw13] [ 66%] PASSED tests/functional/adapter/utils/test_utils.py::TestBoolOr::test_build_assert_equal +[gw7] [ 66%] PASSED tests/functional/adapter/utils/test_utils.py::TestArrayConstruct::test_expected_actual +tests/functional/adapter/utils/test_utils.py::TestCast::test_build_assert_equal +[gw4] [ 67%] PASSED tests/functional/adapter/utils/test_utils.py::TestArrayAppend::test_build_assert_equal +tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestDeltaStrategies::test_delta_strategies +tests/functional/adapter/utils/test_utils.py::TestCastBoolToText::test_build_assert_equal +[gw14] [ 68%] PASSED tests/functional/adapter/utils/test_timestamps.py::TestCurrentTimestampSpark::test_current_timestamps +[gw12] [ 69%] SKIPPED tests/functional/adapter/incremental_strategies/test_incremental_strategies.py::TestDeltaStrategies::test_delta_strategies +tests/functional/adapter/utils/test_utils.py::TestCurrentTimestamp::test_current_timestamp_type +tests/functional/adapter/utils/test_utils.py::TestEscapeSingleQuotes::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestDate::test_build_assert_equal +[gw15] [ 69%] PASSED tests/functional/adapter/utils/test_utils.py::TestArrayConcat::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestDateAdd::test_build_assert_equal +[gw2] [ 70%] PASSED tests/functional/adapter/test_basic.py::TestIncrementalSpark::test_incremental +[gw1] [ 71%] PASSED tests/functional/adapter/utils/test_utils.py::TestArrayConstruct::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestDateTrunc::test_build_assert_equal +tests/functional/adapter/persist_docs/test_persist_docs.py::TestPersistDocsMissingColumn::test_missing_column +[gw17] [ 71%] PASSED tests/functional/adapter/utils/test_data_types.py::TestTypeBoolean::test_check_types_assert_match +[gw2] [ 72%] SKIPPED tests/functional/adapter/persist_docs/test_persist_docs.py::TestPersistDocsMissingColumn::test_missing_column +[gw7] [ 73%] PASSED tests/functional/adapter/utils/test_utils.py::TestCastBoolToText::test_build_assert_equal +[gw8] [ 73%] PASSED tests/functional/adapter/utils/test_data_types.py::TestTypeNumeric::test_check_types_assert_match +[gw4] [ 74%] PASSED tests/functional/adapter/utils/test_utils.py::TestCurrentTimestamp::test_current_timestamp_type +tests/functional/adapter/utils/test_utils.py::TestCurrentTimestamp::test_current_timestamp_matches_utc +tests/functional/adapter/utils/test_utils.py::TestExcept::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestPosition::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestConcat::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestHash::test_build_assert_equal +[gw14] [ 75%] PASSED tests/functional/adapter/utils/test_utils.py::TestDate::test_build_assert_equal +[gw11] [ 76%] PASSED tests/functional/adapter/utils/test_data_types.py::TestTypeBigInt::test_check_types_assert_match +tests/functional/adapter/utils/test_utils.py::TestLastDay::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestDateDiff::test_build_assert_equal +[gw13] [ 76%] PASSED tests/functional/adapter/utils/test_utils.py::TestCast::test_build_assert_equal +[gw5] [ 77%] PASSED tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestAppendOnSchemaChangeSpecialChars::test_incremental_append_new_columns_with_special_characters +[gw11] [ 78%] SKIPPED tests/functional/adapter/utils/test_utils.py::TestDateDiff::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestEquals::test_equal_values +[gw12] [ 78%] PASSED tests/functional/adapter/utils/test_utils.py::TestEscapeSingleQuotes::test_build_assert_equal +tests/functional/adapter/incremental_strategies/test_microbatch.py::TestMicrobatch::test_run_with_event_time +[gw17] [ 79%] PASSED tests/functional/adapter/utils/test_utils.py::TestCurrentTimestamp::test_current_timestamp_matches_utc +tests/functional/adapter/utils/test_utils.py::TestIntersect::test_build_assert_equal +[gw15] [ 80%] PASSED tests/functional/adapter/utils/test_utils.py::TestDateAdd::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestReplace::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestLength::test_build_assert_equal +[gw9] [ 80%] PASSED tests/functional/adapter/utils/test_utils.py::TestArrayConcat::test_expected_actual +[gw1] [ 81%] PASSED tests/functional/adapter/utils/test_utils.py::TestDateTrunc::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestEquals::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestListagg::test_build_assert_equal +[gw16] [ 82%] PASSED tests/functional/adapter/test_store_test_failures.py::TestStoreTestFailuresAsInteractions::test_tests_run_successfully_and_are_stored_as_expected +[gw8] [ 83%] PASSED tests/functional/adapter/utils/test_utils.py::TestConcat::test_build_assert_equal +[gw4] [ 83%] PASSED tests/functional/adapter/utils/test_utils.py::TestHash::test_build_assert_equal +[gw2] [ 84%] PASSED tests/functional/adapter/utils/test_utils.py::TestPosition::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestStringLiteral::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestSplitPart::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestSafeCast::test_build_assert_equal +tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowDoesNotHandleDoubleLimit::test_double_limit_throws_syntax_error +[gw16] [ 85%] SKIPPED tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowDoesNotHandleDoubleLimit::test_double_limit_throws_syntax_error +[gw17] [ 85%] PASSED tests/functional/adapter/utils/test_utils.py::TestReplace::test_build_assert_equal +[gw0] [ 86%] PASSED tests/functional/adapter/test_basic.py::TestSimpleMaterializationsSpark::test_base +[gw15] [ 87%] PASSED tests/functional/adapter/utils/test_utils.py::TestLength::test_build_assert_equal +[gw4] [ 88%] PASSED tests/functional/adapter/utils/test_utils.py::TestStringLiteral::test_build_assert_equal +tests/functional/adapter/test_basic.py::TestSingularTestsSpark::test_singular_tests +[gw14] [ 88%] PASSED tests/functional/adapter/utils/test_utils.py::TestLastDay::test_build_assert_equal +[gw9] [ 89%] PASSED tests/functional/adapter/utils/test_utils.py::TestEquals::test_build_assert_equal +[gw0] [ 90%] PASSED tests/functional/adapter/test_basic.py::TestSingularTestsSpark::test_singular_tests +[gw13] [ 90%] PASSED tests/functional/adapter/utils/test_utils.py::TestEquals::test_equal_values +[gw8] [ 91%] PASSED tests/functional/adapter/utils/test_utils.py::TestSplitPart::test_build_assert_equal +[gw2] [ 92%] PASSED tests/functional/adapter/utils/test_utils.py::TestSafeCast::test_build_assert_equal +[gw6] [ 92%] PASSED tests/functional/adapter/incremental/test_incremental_on_schema_change.py::TestInsertOverwriteOnSchemaChange::test_run_incremental_ignore +[gw1] [ 93%] PASSED tests/functional/adapter/utils/test_utils.py::TestListagg::test_build_assert_equal +tests/functional/adapter/persist_docs/test_persist_docs.py::TestPersistDocsDeltaTable::test_delta_comments +[gw6] [ 94%] SKIPPED tests/functional/adapter/persist_docs/test_persist_docs.py::TestPersistDocsDeltaTable::test_delta_comments +[gw12] [ 95%] PASSED tests/functional/adapter/utils/test_utils.py::TestIntersect::test_build_assert_equal +[gw7] [ 95%] PASSED tests/functional/adapter/utils/test_utils.py::TestExcept::test_build_assert_equal +tests/functional/adapter/utils/test_utils.py::TestRight::test_build_assert_equal +[gw3] [ 96%] PASSED tests/functional/adapter/unit_testing/test_unit_testing.py::TestSparkUnitTestingTypes::test_unit_test_data_type +tests/functional/adapter/utils/test_utils.py::TestArrayAppend::test_expected_actual +[gw7] [ 97%] PASSED tests/functional/adapter/utils/test_utils.py::TestRight::test_build_assert_equal +[gw3] [ 97%] PASSED tests/functional/adapter/utils/test_utils.py::TestArrayAppend::test_expected_actual +[gw5] [ 98%] PASSED tests/functional/adapter/incremental_strategies/test_microbatch.py::TestMicrobatch::test_run_with_event_time +[gw10] [ 99%] PASSED tests/functional/adapter/test_store_test_failures.py::TestSparkStoreTestFailures::test_store_and_assert +tests/functional/adapter/empty/test_empty.py::TestSparkEmptyInlineSourceRef::test_run_with_empty +[gw10] [100%] PASSED tests/functional/adapter/empty/test_empty.py::TestSparkEmptyInlineSourceRef::test_run_with_empty + +================== 75 passed, 67 skipped in 85.47s (0:01:25) =================== + + +============================================================ +SUMMARY: tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowDoesNotHandleDoubleLimit::test_double_limit_throws_syntax_error +============================================================ + +============================================================ +SUMMARY: [gw16] [ 85%] SKIPPED tests/functional/adapter/dbt_show/test_dbt_show.py::TestSparkShowDoesNotHandleDoubleLimit::test_double_limit_throws_syntax_error +============================================================ + +============================================================ +SUMMARY: ================== 75 passed, 67 skipped in 85.47s (0:01:25) =================== +============================================================ diff --git a/dbt-spark/tests/functional/fixtures/profiles.py b/dbt-spark/tests/functional/fixtures/profiles.py index 900428e10f..c3b9b90882 100644 --- a/dbt-spark/tests/functional/fixtures/profiles.py +++ b/dbt-spark/tests/functional/fixtures/profiles.py @@ -96,6 +96,7 @@ def spark_session_target(): "type": "spark", "host": "localhost", "method": "session", + "server_side_parameters": {"spark.ui.enabled": "false"}, } From 7375870dbf65dd242ffd97b3c3ffb5cd928bc358 Mon Sep 17 00:00:00 2001 From: pomykalakyle Date: Fri, 10 Apr 2026 16:19:48 -0700 Subject: [PATCH 4/6] Refine dbt-sail packaging and test setup Move dbt-sail Hatch config into its own file, reuse dbt-spark relation macros, and keep the snapshot test on Delta so the adapter package and repro flow exercise the intended Sail path. --- dbt-sail/hatch.toml | 28 +++++++++ dbt-sail/pyproject.toml | 32 +++++----- dbt-sail/src/dbt/adapters/sail/impl.py | 58 +------------------ .../src/dbt/include/sail/macros/adapters.sql | 23 +------- .../tests/functional/adapter/test_basic.py | 12 +++- dbt-spark/pyproject.toml | 18 ++++-- 6 files changed, 70 insertions(+), 101 deletions(-) create mode 100644 dbt-sail/hatch.toml diff --git a/dbt-sail/hatch.toml b/dbt-sail/hatch.toml new file mode 100644 index 0000000000..ede7350791 --- /dev/null +++ b/dbt-sail/hatch.toml @@ -0,0 +1,28 @@ +[version] +path = "src/dbt/adapters/sail/__version__.py" +pattern = "version = \"(?P[^\"]+)\"" + +[build.targets.wheel] +packages = ["src/dbt"] + +[envs.default] +pre-install-commands = [ + "pip install -e ../dbt-adapters", + "pip install -e ../dbt-tests-adapter", + "pip install -e ../dbt-spark", +] +dependencies = [ + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", + "pre-commit==3.7.0", + "pytest>=7.0,<8.0", + "pytest-xdist", +] + +[envs.default.scripts] +setup = [ + "pre-commit install", +] +code-quality = "pre-commit run --all-files" +unit-tests = "python -m pytest {args:tests/unit}" +integration-tests = "python -m pytest {args:tests/functional}" diff --git a/dbt-sail/pyproject.toml b/dbt-sail/pyproject.toml index cd08283e57..c9d55212d0 100644 --- a/dbt-sail/pyproject.toml +++ b/dbt-sail/pyproject.toml @@ -7,9 +7,18 @@ dynamic = ["version"] name = "dbt-sail" description = "The Sail adapter plugin for dbt" readme = "README.md" -keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "sail", "spark"] +keywords = [ + "dbt", + "adapter", + "adapters", + "database", + "elt", + "dbt-core", + "sail", + "spark", +] requires-python = ">=3.10.0" -authors = [{ name = "LakeSail", email = "info@lakesail.com" }] +authors = [{ name = "LakeSail", email = "kyle@lakesail.com" }] classifiers = [ "Development Status :: 3 - Alpha", "License :: OSI Approved :: Apache Software License", @@ -21,29 +30,16 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", ] -dependencies = [ - "dbt-spark[session]>=1.8.0", - "pysail", -] +dependencies = ["dbt-spark[session]>=1.8.0", "pysail"] [project.optional-dependencies] -dev = [ - "dbt-tests-adapter", - "pytest", - "pytest-xdist", -] +dev = ["dbt-tests-adapter", "pytest", "pytest-xdist"] +# Need to update these when published [project.urls] Homepage = "https://github.com/lakehq/sail" Repository = "https://github.com/lakehq/sail.git" -[tool.hatch.version] -path = "src/dbt/adapters/sail/__version__.py" -pattern = "version = \"(?P[^\"]+)\"" - -[tool.hatch.build.targets.wheel] -packages = ["src/dbt"] - [tool.pytest.ini_options] testpaths = ["tests/unit", "tests/functional"] addopts = "-v --color=yes" diff --git a/dbt-sail/src/dbt/adapters/sail/impl.py b/dbt-sail/src/dbt/adapters/sail/impl.py index 8135242ebf..59d4f55f06 100644 --- a/dbt-sail/src/dbt/adapters/sail/impl.py +++ b/dbt-sail/src/dbt/adapters/sail/impl.py @@ -1,19 +1,5 @@ -from typing import List - -from dbt.adapters.base import BaseRelation -from dbt.adapters.events.logging import AdapterLogger -from dbt.adapters.spark.impl import ( - SparkAdapter, - LIST_RELATIONS_MACRO_NAME, - DESCRIBE_TABLE_EXTENDED_MACRO_NAME, - SCHEMA_NOT_FOUND_MESSAGES, - TABLE_OR_VIEW_NOT_FOUND_MESSAGES, -) +from dbt.adapters.spark.impl import SparkAdapter from dbt.adapters.sail.connections import SailConnectionManager -from dbt_common.exceptions import DbtRuntimeError -from dbt_common.utils import AttrDict - -logger = AdapterLogger("Sail") class SailAdapter(SparkAdapter): @@ -22,45 +8,3 @@ class SailAdapter(SparkAdapter): @classmethod def type(cls) -> str: return "sail" - - def _get_relation_information(self, row) -> tuple: - """Parse a row from Sail's SHOW TABLES (6 columns) and fetch extended info.""" - # Sail returns: tableName, catalog, namespace, description, tableType, isTemporary - name = row[0] - namespace = row[2] - _schema = namespace[0] if namespace else "" - - table_name = f"{_schema}.{name}" - try: - table_results = self.execute_macro( - DESCRIBE_TABLE_EXTENDED_MACRO_NAME, kwargs={"table_name": table_name} - ) - except DbtRuntimeError as e: - logger.debug(f"Error while retrieving information about {table_name}: {e.msg}") - table_results = AttrDict() - - information = "" - for info_row in table_results: - info_type, info_value, _ = info_row - if not info_type.startswith("#"): - information += f"{info_type}: {info_value}\n" - - return _schema, name, information - - def list_relations_without_caching(self, schema_relation: BaseRelation) -> List[BaseRelation]: - kwargs = {"schema_relation": schema_relation} - - try: - show_table_rows = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) - return self._build_spark_relation_list( - row_list=show_table_rows, - relation_info_func=self._get_relation_information, - ) - except DbtRuntimeError as e: - errmsg = getattr(e, "msg", "") - if any(msg in errmsg for msg in SCHEMA_NOT_FOUND_MESSAGES) or any( - msg in errmsg for msg in TABLE_OR_VIEW_NOT_FOUND_MESSAGES - ): - return [] - else: - raise diff --git a/dbt-sail/src/dbt/include/sail/macros/adapters.sql b/dbt-sail/src/dbt/include/sail/macros/adapters.sql index fd94b20b77..db08b4df1c 100644 --- a/dbt-sail/src/dbt/include/sail/macros/adapters.sql +++ b/dbt-sail/src/dbt/include/sail/macros/adapters.sql @@ -1,28 +1,11 @@ {% macro sail__list_relations_without_caching(relation) %} - {% call statement('list_relations_without_caching', fetch_result=True) -%} - show tables in {{ relation.schema }} like '*' - {% endcall %} - - {% do return(load_result('list_relations_without_caching').table) %} + {{ return(spark__list_relations_without_caching(relation)) }} {% endmacro %} {% macro sail__get_columns_in_relation_raw(relation) -%} - {% call statement('get_columns_in_relation_raw', fetch_result=True) %} - describe table extended {{ relation }} - {% endcall %} - {% do return(load_result('get_columns_in_relation_raw').table) %} + {{ return(spark__get_columns_in_relation_raw(relation)) }} {% endmacro %} {% macro sail__get_columns_in_relation(relation) -%} - {% call statement('get_columns_in_relation', fetch_result=True) %} - describe table extended {{ relation.include(schema=(schema is not none)) }} - {% endcall %} - {% do return(load_result('get_columns_in_relation').table) %} -{% endmacro %} - -{% macro describe_table_extended_without_caching(table_name) %} - {% call statement('describe_table_extended_without_caching', fetch_result=True) -%} - describe table extended {{ table_name }} - {% endcall %} - {% do return(load_result('describe_table_extended_without_caching').table) %} + {{ return(spark__get_columns_in_relation(relation)) }} {% endmacro %} diff --git a/dbt-sail/tests/functional/adapter/test_basic.py b/dbt-sail/tests/functional/adapter/test_basic.py index 3c3fcde6c1..57dc41e48c 100644 --- a/dbt-sail/tests/functional/adapter/test_basic.py +++ b/dbt-sail/tests/functional/adapter/test_basic.py @@ -10,6 +10,7 @@ from dbt.tests.adapter.basic.test_incremental import BaseIncremental from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp +import pytest class TestSimpleMaterializationsSpark(BaseSimpleMaterializations): @@ -47,7 +48,16 @@ class TestSnapshotCheckColsSpark(BaseSnapshotCheckCols): class TestSnapshotTimestampSpark(BaseSnapshotTimestamp): - pass + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "+file_format": "delta", + }, + "snapshots": { + "+file_format": "delta", + }, + } class TestBaseAdapterMethod(BaseAdapterMethod): diff --git a/dbt-spark/pyproject.toml b/dbt-spark/pyproject.toml index b0012687cc..82dd631b14 100644 --- a/dbt-spark/pyproject.toml +++ b/dbt-spark/pyproject.toml @@ -7,7 +7,18 @@ dynamic = ["version"] name = "dbt-spark" description = "The Apache Spark adapter plugin for dbt" readme = "README.md" -keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs", "spark"] +keywords = [ + "dbt", + "adapter", + "adapters", + "database", + "elt", + "dbt-core", + "dbt Core", + "dbt Cloud", + "dbt Labs", + "spark", +] requires-python = ">=3.10.0" authors = [{ name = "dbt Labs", email = "info@dbtlabs.com" }] maintainers = [{ name = "dbt Labs", email = "info@dbtlabs.com" }] @@ -31,10 +42,7 @@ dependencies = [ ] [project.optional-dependencies] ODBC = ["pyodbc>=5.1,<5.3"] -PyHive = [ - "PyHive[hive_pure_sasl]~=0.7.0", - "thrift>=0.11.0,<0.23.0", -] +PyHive = ["PyHive[hive_pure_sasl]~=0.7.0", "thrift>=0.11.0,<0.23.0"] session = ["pyspark>=3.0.0,<5.0.0"] all = [ "pyodbc>=5.1,<5.3", From 03d6a946b1d058d44d75d7ba039bdd1450a59ce5 Mon Sep 17 00:00:00 2001 From: pomykalakyle Date: Mon, 13 Apr 2026 17:00:58 -0700 Subject: [PATCH 5/6] Refine dbt-sail snapshot test config Match the Spark adapter test setup by forcing Delta for the snapshot check-cols test so it no longer falls back to parquet and fail before exercising snapshot behavior. --- dbt-sail/tests/functional/adapter/test_basic.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dbt-sail/tests/functional/adapter/test_basic.py b/dbt-sail/tests/functional/adapter/test_basic.py index 57dc41e48c..2b04e535c3 100644 --- a/dbt-sail/tests/functional/adapter/test_basic.py +++ b/dbt-sail/tests/functional/adapter/test_basic.py @@ -44,7 +44,16 @@ class TestGenericTestsSpark(BaseGenericTests): class TestSnapshotCheckColsSpark(BaseSnapshotCheckCols): - pass + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "+file_format": "delta", + }, + "snapshots": { + "+file_format": "delta", + }, + } class TestSnapshotTimestampSpark(BaseSnapshotTimestamp): From 52446bf3e51cb479ffb3718841dd36b9fff1fc85 Mon Sep 17 00:00:00 2001 From: Shehab <11789402+shehabgamin@users.noreply.github.com> Date: Wed, 15 Apr 2026 13:41:01 -0700 Subject: [PATCH 6/6] save --- dbt-sail/hatch.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbt-sail/hatch.toml b/dbt-sail/hatch.toml index ede7350791..342f728cfb 100644 --- a/dbt-sail/hatch.toml +++ b/dbt-sail/hatch.toml @@ -6,6 +6,7 @@ pattern = "version = \"(?P[^\"]+)\"" packages = ["src/dbt"] [envs.default] +python = "3.13.2" pre-install-commands = [ "pip install -e ../dbt-adapters", "pip install -e ../dbt-tests-adapter", @@ -14,6 +15,7 @@ pre-install-commands = [ dependencies = [ "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", + "pyspark-client==4.1.1", "pre-commit==3.7.0", "pytest>=7.0,<8.0", "pytest-xdist",