feat: Sail

dbt-sail/.gitignore

@@ -0,0 +1,8 @@
+logs/
+.venv/
+__pycache__/
+*.pyc
+*.egg-info/
+.pytest_cache/
+dist/
+build/

dbt-sail/README.md

@@ -0,0 +1,5 @@
+# dbt-sail
+
+The [Sail](https://github.com/lakehq/sail) adapter plugin for [dbt](https://www.getdbt.com/).
+
+Sail is a drop-in replacement for Apache Spark. This adapter is a thin wrapper around `dbt-spark` that connects to Sail via the Spark Connect protocol.

dbt-sail/hatch.toml

@@ -0,0 +1,30 @@
+[version]
+path = "src/dbt/adapters/sail/__version__.py"
+pattern = "version = \"(?P<version>[^\"]+)\""
+
+[build.targets.wheel]
+packages = ["src/dbt"]
+
+[envs.default]
+python = "3.13.2"
+pre-install-commands = [
+    "pip install -e ../dbt-adapters",
+    "pip install -e ../dbt-tests-adapter",
+    "pip install -e ../dbt-spark",
+]
+dependencies = [
+    "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git",
+    "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core",
+    "pyspark-client==4.1.1",
+    "pre-commit==3.7.0",
+    "pytest>=7.0,<8.0",
+    "pytest-xdist",
+]
+
+[envs.default.scripts]
+setup = [
+    "pre-commit install",
+]
+code-quality = "pre-commit run --all-files"
+unit-tests = "python -m pytest {args:tests/unit}"
+integration-tests = "python -m pytest {args:tests/functional}"

dbt-sail/pyproject.toml

@@ -0,0 +1,49 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+dynamic = ["version"]
+name = "dbt-sail"
+description = "The Sail adapter plugin for dbt"
+readme = "README.md"
+keywords = [
+    "dbt",
+    "adapter",
+    "adapters",
+    "database",
+    "elt",
+    "dbt-core",
+    "sail",
+    "spark",
+]
+requires-python = ">=3.10.0"
+authors = [{ name = "LakeSail", email = "kyle@lakesail.com" }]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: MacOS :: MacOS X",
+    "Operating System :: Microsoft :: Windows",
+    "Operating System :: POSIX :: Linux",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+dependencies = ["dbt-spark[session]>=1.8.0", "pysail"]
+
+[project.optional-dependencies]
+dev = ["dbt-tests-adapter", "pytest", "pytest-xdist"]
+
+# Need to update these when published
+[project.urls]
+Homepage = "https://github.com/lakehq/sail"
+Repository = "https://github.com/lakehq/sail.git"
+
+[tool.pytest.ini_options]
+testpaths = ["tests/unit", "tests/functional"]
+addopts = "-v --color=yes"
+filterwarnings = [
+    "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning",
+    "ignore:unclosed file .*:ResourceWarning",
+]

dbt-sail/src/dbt/adapters/sail/__init__.py

@@ -0,0 +1,12 @@
+from dbt.adapters.base import AdapterPlugin
+
+from dbt.adapters.sail.connections import SailConnectionManager, SailCredentials  # noqa: F401
+from dbt.adapters.sail.impl import SailAdapter
+from dbt.include import sail
+
+Plugin = AdapterPlugin(
+    adapter=SailAdapter,  # type: ignore
+    credentials=SailCredentials,
+    include_path=sail.PACKAGE_PATH,
+    dependencies=["spark"],
+)

dbt-sail/src/dbt/adapters/sail/__version__.py

@@ -0,0 +1 @@
+version = "0.1.0"

dbt-sail/src/dbt/adapters/sail/connections.py

@@ -0,0 +1,129 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict
+
+from dbt.adapters.contracts.connection import Connection, ConnectionState
+from dbt.adapters.events.logging import AdapterLogger
+from dbt.adapters.exceptions import FailedToConnectError
+from dbt.adapters.spark.connections import SparkConnectionManager, SparkCredentials
+from dbt_common.dataclass_schema import StrEnum
+from dbt_common.exceptions import DbtConfigError
+
+logger = AdapterLogger("Sail")
+
+
+class SailConnectionMethod(StrEnum):
+    EMBEDDED = "embedded"
+    REMOTE = "remote"
+
+
+@dataclass
+class SailCredentials(SparkCredentials):
+    mode: SailConnectionMethod = SailConnectionMethod.EMBEDDED  # type: ignore
+    server_side_parameters: Dict[str, str] = field(default_factory=dict)
+
+    @property
+    def type(self) -> str:
+        return "sail"
+
+    @property
+    def unique_field(self) -> str:
+        return self.host or "embedded"
+
+    def __post_init__(self) -> None:
+        # For embedded mode, host is not required — set defaults
+        if self.mode == SailConnectionMethod.EMBEDDED:
+            if self.host is None:
+                self.host = "127.0.0.1"
+            if self.method is None:
+                # Set method to session so SparkCredentials validation doesn't fail
+                from dbt.adapters.spark.connections import SparkConnectionMethod
+
+                self.method = SparkConnectionMethod.SESSION
+        elif self.mode == SailConnectionMethod.REMOTE:
+            if self.host is None:
+                raise DbtConfigError("Must specify `host` for remote mode")
+            if self.method is None:
+                from dbt.adapters.spark.connections import SparkConnectionMethod
+
+                self.method = SparkConnectionMethod.SESSION
+
+        if self.schema is None:
+            raise DbtConfigError("Must specify `schema` in profile")
+
+        # Spark treats database and schema as the same thing
+        if self.database is not None and self.database != self.schema:
+            raise DbtConfigError(
+                f"    schema: {self.schema} \n"
+                f"    database: {self.database} \n"
+                f"On Sail, database must be omitted or have the same value as schema."
+            )
+        self.database = None
+
+
+class SailConnectionManager(SparkConnectionManager):
+    TYPE = "sail"
+
+    # Track the embedded server so it can be reused/stopped
+    _server = None
+
+    @classmethod
+    def open(cls, connection: Connection) -> Connection:
+        if connection.state == ConnectionState.OPEN:
+            logger.debug("Connection is already open, skipping open.")
+            return connection
+
+        creds: SailCredentials = connection.credentials  # type: ignore
+
+        try:
+            if creds.mode == SailConnectionMethod.EMBEDDED:
+                handle = cls._open_embedded(creds)
+            elif creds.mode == SailConnectionMethod.REMOTE:
+                handle = cls._open_remote(creds)
+            else:
+                raise DbtConfigError(f"invalid Sail connection mode: {creds.mode}")
+        except Exception as e:
+            logger.debug(f"Error opening connection: {e}")
+            connection.handle = None
+            connection.state = ConnectionState.FAIL
+            raise FailedToConnectError(f"failed to connect to Sail: {e}") from e
+
+        connection.handle = handle
+        connection.state = ConnectionState.OPEN
+        return connection
+
+    @classmethod
+    def _open_embedded(cls, creds: SailCredentials) -> Any:
+        """Start a SparkConnectServer in-process via pysail, then connect PySpark to it."""
+        from pysail.spark import SparkConnectServer
+        from dbt.adapters.spark.session import (
+            SessionConnectionWrapper,
+            Connection as SessionConnection,
+        )
+
+        if cls._server is None or not cls._server.running:
+            port = creds.port if creds.port != 443 else 0
+            cls._server = SparkConnectServer(ip=creds.host or "127.0.0.1", port=port)
+            cls._server.start(background=True)
+            logger.debug(f"Started embedded Sail server at {cls._server.listening_address}")
+
+        ip, port = cls._server.listening_address
+        remote_url = f"sc://{ip}:{port}"
+
+        params = dict(creds.server_side_parameters)
+        params["spark.remote"] = remote_url
+        return SessionConnectionWrapper(SessionConnection(server_side_parameters=params))
+
+    @classmethod
+    def _open_remote(cls, creds: SailCredentials) -> Any:
+        """Connect PySpark to an already-running Sail server via Spark Connect."""
+        from dbt.adapters.spark.session import (
+            SessionConnectionWrapper,
+            Connection as SessionConnection,
+        )
+
+        port = creds.port if creds.port != 443 else 50051
+        remote_url = f"sc://{creds.host}:{port}"
+
+        params = dict(creds.server_side_parameters)
+        params["spark.remote"] = remote_url
+        return SessionConnectionWrapper(SessionConnection(server_side_parameters=params))

dbt-sail/src/dbt/adapters/sail/impl.py

@@ -0,0 +1,10 @@
+from dbt.adapters.spark.impl import SparkAdapter
+from dbt.adapters.sail.connections import SailConnectionManager
+
+
+class SailAdapter(SparkAdapter):
+    ConnectionManager = SailConnectionManager
+
+    @classmethod
+    def type(cls) -> str:
+        return "sail"

dbt-sail/src/dbt/include/sail/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+PACKAGE_PATH = os.path.dirname(__file__)

dbt-sail/src/dbt/include/sail/dbt_project.yml

@@ -0,0 +1,5 @@
+config-version: 2
+name: dbt_sail
+version: 1.0
+
+macro-paths: ["macros"]

dbt-sail/src/dbt/include/sail/macros/adapters.sql

@@ -0,0 +1,11 @@
+{% macro sail__list_relations_without_caching(relation) %}
+  {{ return(spark__list_relations_without_caching(relation)) }}
+{% endmacro %}
+
+{% macro sail__get_columns_in_relation_raw(relation) -%}
+  {{ return(spark__get_columns_in_relation_raw(relation)) }}
+{% endmacro %}
+
+{% macro sail__get_columns_in_relation(relation) -%}
+  {{ return(spark__get_columns_in_relation(relation)) }}
+{% endmacro %}

dbt-sail/tests/functional/adapter/SPARK_SESSION_TEST_DEBUG_REPORT.md

@@ -0,0 +1,121 @@
+# Spark-Session Pruned Test Debug Report (dbt-sail)
+
+## Summary
+- Total tests: **52**
+- Passed: **21**
+- Failed/Error: **30**
+- Skipped: **1**
+
+## Failure Categories
+- `exit-state mismatch`: 16
+- `sql/parser incompatibility`: 6
+- `type mapping mismatch`: 5
+- `error-message mismatch`: 1
+- `state/cleanup issue`: 1
+- `time precision/tolerance mismatch`: 1
+
+## Error Signature Legend
+| ID | Count | Category | What's Happening | Example |
+|---|---:|---|---|---|
+| `E01` | 16 | exit-state mismatch | dbt command result differs from test expectation. | AssertionError: dbt exit state did not match expected self = <test_dbt_show.TestSparkShowLimit object at 0x10a6e6d50> project = <dbt.tests.fixtures.project.TestProjInfo object at 0x10ae31fd0> args = [], expected = 5 @pytest.mark.parametrize( "args,expected", [ ([], 5), # default limit (["--limit", 3 |
+| `E02` | 1 | state/cleanup issue | test expects clean relation state but object already exists. | AssertionError: assert 'Compilation Error' in 'Runtime Error in model incremental_fail (models/incremental_fail.sql)\n Runtime Error\n table already exists: incremental_fail' + where 'Runtime Error in model incremental_fail (models/incremental_fail.sql)\n Runtime Error\n table already exists: increm |
+| `E03` | 1 | error-message mismatch | same logical failure class, but assertion expects different text. | assert "Invalid column name: 'invalid_column_name' in unit test fixture for 'my_upstream_model'." in '\x1b[0m03:44:22 Running with dbt=1.11.7\n\x1b[0m03:44:22 Registered adapter: sail=0.1.0\n\x1b[0m03:44:22 Found 2 m...odel (models/unit_tests.yml)\n\x1b[0m03:44:22 \n\x1b[0m03:44:22 Done. PASS=0 WARN |
+| `E04` | 6 | sql/parser incompatibility | SQL generated by shared test macro is rejected by Sail parser. | dbt_common.exceptions.base.DbtRuntimeError: Runtime Error invalid argument: found . at 278:279 expected ',', 'FROM', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', or ')' self = <dbt.adapters.sail |
+| `E05` | 5 | type mapping mismatch | Spark-expected type strings differ from Sail-reported types. | AssertionError: Type difference detected: [('float_col', 'float', None)] vs. [] self = <test_data_types.TestTypeFloat object at 0x10a795f90> project = <dbt.tests.fixtures.project.TestProjInfo object at 0x128402990> def test_check_types_assert_match(self, project): run_dbt(["build"]) # check contents |
+| `E06` | 1 | time precision/tolerance mismatch | timestamp precision/timezone offset differs from test expectation. | AssertionError: SQL timestamp 2026-04-03T20:44:25.633143 is not close enough to Python UTC 2026-04-04T03:44:25.634810 assert (datetime.datetime(2026, 4, 3, 20, 44, 25, 633143) > (datetime.datetime(2026, 4, 4, 3, 44, 25, 634810) - datetime.timedelta(seconds=300))) self = <test_utils.TestCurrentTimest |
+| `PASS` | - | passed | test passed | - |
+| `SKIP` | - | skipped | test is intentionally skipped | see module table note |
+
+## Tests By Module
+
+### `tests.functional.adapter.dbt_show.test_dbt_show`
+| Class | Test | Status | Signature |
+|---|---|---|---|
+| `TestSparkShowLimit` | `test_limit[args0-5]` | failed | `E01` |
+| `TestSparkShowLimit` | `test_limit[args1-3]` | failed | `E01` |
+| `TestSparkShowLimit` | `test_limit[args2-7]` | failed | `E01` |
+| `TestSparkShowSqlHeader` | `test_sql_header` | passed | `PASS` |
+
+### `tests.functional.adapter.empty.test_empty`
+| Class | Test | Status | Signature |
+|---|---|---|---|
+| `TestSparkEmpty` | `test_run_with_empty` | passed | `PASS` |
+| `TestSparkEmptyInlineSourceRef` | `test_run_with_empty` | passed | `PASS` |
+
+### `tests.functional.adapter.incremental.test_incremental_on_schema_change`
+| Class | Test | Status | Signature |
+|---|---|---|---|
+| `TestAppendOnSchemaChange` | `test_run_incremental_fail_on_schema_change` | failed | `E02` |
+| `TestAppendOnSchemaChange` | `test_run_incremental_ignore` | failed | `E01` |
+| `TestAppendOnSchemaChangeSpecialChars` | `test_incremental_append_new_columns_with_special_characters` | failed | `E01` |
+
+### `tests.functional.adapter.incremental_strategies.test_incremental_strategies`
+| Class | Test | Status | Signature |
+|---|---|---|---|
+| `TestDeltaStrategies` | `test_delta_strategies_overwrite` | skipped | `SKIP` |
+
+### `tests.functional.adapter.test_basic`
+| Class | Test | Status | Signature |
+|---|---|---|---|
+| `TestEmptySpark` | `test_empty` | passed | `PASS` |
+| `TestGenericTestsSpark` | `test_generic_tests` | passed | `PASS` |
+| `TestSingularTestsEphemeralSpark` | `test_singular_tests_ephemeral` | passed | `PASS` |
+| `TestSingularTestsSpark` | `test_singular_tests` | passed | `PASS` |
+
+### `tests.functional.adapter.test_simple_seed`
+| Class | Test | Status | Signature |
+|---|---|---|---|
+| `TestBigQueryEmptySeed` | `test_empty_seeds` | passed | `PASS` |
+
+### `tests.functional.adapter.unit_testing.test_unit_testing`
+| Class | Test | Status | Signature |
+|---|---|---|---|
+| `TestSparkUnitTestCaseInsensitivity` | `test_case_insensitivity` | failed | `E01` |
+| `TestSparkUnitTestInvalidInput` | `test_invalid_input` | failed | `E03` |
+| `TestSparkUnitTestingTypes` | `test_unit_test_data_type` | failed | `E01` |
+
+### `tests.functional.adapter.utils.test_data_types`
+| Class | Test | Status | Signature |
+|---|---|---|---|
+| `TestTypeBigInt` | `test_check_types_assert_match` | failed | `E04` |
+| `TestTypeBoolean` | `test_check_types_assert_match` | failed | `E01` |
+| `TestTypeFloat` | `test_check_types_assert_match` | failed | `E05` |
+| `TestTypeInt` | `test_check_types_assert_match` | failed | `E05` |
+| `TestTypeNumeric` | `test_check_types_assert_match` | failed | `E05` |
+| `TestTypeString` | `test_check_types_assert_match` | failed | `E05` |
+| `TestTypeTimestamp` | `test_check_types_assert_match` | failed | `E05` |
+
+### `tests.functional.adapter.utils.test_timestamps`
+| Class | Test | Status | Signature |
+|---|---|---|---|
+| `TestCurrentTimestampSpark` | `test_current_timestamps` | failed | `E01` |
+
+### `tests.functional.adapter.utils.test_utils`
+| Class | Test | Status | Signature |
+|---|---|---|---|
+| `TestAnyValue` | `test_build_assert_equal` | passed | `PASS` |
+| `TestArrayAppend` | `test_build_assert_equal` | passed | `PASS` |
+| `TestArrayAppend` | `test_expected_actual` | failed | `E04` |
+| `TestArrayConcat` | `test_build_assert_equal` | passed | `PASS` |
+| `TestArrayConcat` | `test_expected_actual` | failed | `E04` |
+| `TestArrayConstruct` | `test_build_assert_equal` | passed | `PASS` |
+| `TestArrayConstruct` | `test_expected_actual` | failed | `E04` |
+| `TestBoolOr` | `test_build_assert_equal` | failed | `E01` |
+| `TestCast` | `test_build_assert_equal` | passed | `PASS` |
+| `TestCastBoolToText` | `test_build_assert_equal` | passed | `PASS` |
+| `TestCurrentTimestamp` | `test_current_timestamp_matches_utc` | failed | `E06` |
+| `TestCurrentTimestamp` | `test_current_timestamp_type` | passed | `PASS` |
+| `TestDate` | `test_build_assert_equal` | failed | `E01` |
+| `TestDateAdd` | `test_build_assert_equal` | failed | `E01` |
+| `TestDateTrunc` | `test_build_assert_equal` | failed | `E01` |
+| `TestEquals` | `test_build_assert_equal` | failed | `E01` |
+| `TestEquals` | `test_equal_values` | failed | `E01` |
+| `TestEscapeSingleQuotes` | `test_build_assert_equal` | passed | `PASS` |
+| `TestExcept` | `test_build_assert_equal` | failed | `E04` |
+| `TestIntersect` | `test_build_assert_equal` | failed | `E04` |
+| `TestLength` | `test_build_assert_equal` | passed | `PASS` |
+| `TestListagg` | `test_build_assert_equal` | passed | `PASS` |
+| `TestPosition` | `test_build_assert_equal` | passed | `PASS` |
+| `TestSafeCast` | `test_build_assert_equal` | passed | `PASS` |
+| `TestSplitPart` | `test_build_assert_equal` | failed | `E01` |
+| `TestStringLiteral` | `test_build_assert_equal` | passed | `PASS` |