diff --git a/airbyte_cdk/test/entrypoint_wrapper.py b/airbyte_cdk/test/entrypoint_wrapper.py index f8e85bfb0..43c84204a 100644 --- a/airbyte_cdk/test/entrypoint_wrapper.py +++ b/airbyte_cdk/test/entrypoint_wrapper.py @@ -82,6 +82,10 @@ def records(self) -> List[AirbyteMessage]: def state_messages(self) -> List[AirbyteMessage]: return self._get_message_by_types([Type.STATE]) + @property + def connection_status_messages(self) -> List[AirbyteMessage]: + return self._get_message_by_types([Type.CONNECTION_STATUS]) + @property def most_recent_state(self) -> Any: state_messages = self._get_message_by_types([Type.STATE]) diff --git a/airbyte_cdk/test/standard_tests/__init__.py b/airbyte_cdk/test/standard_tests/__init__.py new file mode 100644 index 000000000..c6aeaaf1c --- /dev/null +++ b/airbyte_cdk/test/standard_tests/__init__.py @@ -0,0 +1,46 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +'''FAST Airbyte Standard Tests + +This module provides a set of base classes for declarative connector test suites. +The goal of this module is to provide a robust and extensible framework for testing Airbyte +connectors. + +Example usage: + +```python +# `test_airbyte_standards.py` +from airbyte_cdk.test import standard_tests + +pytest_plugins = [ + "airbyte_cdk.test.standard_tests.pytest_hooks", +] + + +class TestSuiteSourcePokeAPI(standard_tests.DeclarativeSourceTestSuite): + """Test suite for the source.""" +``` + +Available test suites base classes: +- `DeclarativeSourceTestSuite`: A test suite for declarative sources. +- `SourceTestSuiteBase`: A test suite for sources. +- `DestinationTestSuiteBase`: A test suite for destinations. + +''' + +from airbyte_cdk.test.standard_tests.connector_base import ( + ConnectorTestScenario, + ConnectorTestSuiteBase, +) +from airbyte_cdk.test.standard_tests.declarative_sources import ( + DeclarativeSourceTestSuite, +) +from airbyte_cdk.test.standard_tests.destination_base import DestinationTestSuiteBase +from airbyte_cdk.test.standard_tests.source_base import SourceTestSuiteBase + +__all__ = [ + "ConnectorTestScenario", + "ConnectorTestSuiteBase", + "DeclarativeSourceTestSuite", + "DestinationTestSuiteBase", + "SourceTestSuiteBase", +] diff --git a/airbyte_cdk/test/standard_tests/_job_runner.py b/airbyte_cdk/test/standard_tests/_job_runner.py new file mode 100644 index 000000000..bab170361 --- /dev/null +++ b/airbyte_cdk/test/standard_tests/_job_runner.py @@ -0,0 +1,159 @@ +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. +"""Job runner for Airbyte Standard Tests.""" + +import logging +import tempfile +import uuid +from dataclasses import asdict +from pathlib import Path +from typing import Any, Callable, Literal + +import orjson +from typing_extensions import Protocol, runtime_checkable + +from airbyte_cdk.models import ( + ConfiguredAirbyteCatalog, + Status, +) +from airbyte_cdk.test import entrypoint_wrapper +from airbyte_cdk.test.standard_tests.models import ( + ConnectorTestScenario, +) + + +def _errors_to_str( + entrypoint_output: entrypoint_wrapper.EntrypointOutput, +) -> str: + """Convert errors from entrypoint output to a string.""" + if not entrypoint_output.errors: + # If there are no errors, return an empty string. + return "" + + return "\n" + "\n".join( + [ + str(error.trace.error).replace( + "\\n", + "\n", + ) + for error in entrypoint_output.errors + if error.trace + ], + ) + + +@runtime_checkable +class IConnector(Protocol): + """A connector that can be run in a test scenario. + + Note: We currently use 'spec' to determine if we have a connector object. + In the future, it would be preferred to leverage a 'launch' method instead, + directly on the connector (which doesn't yet exist). + """ + + def spec(self, logger: logging.Logger) -> Any: + """Connectors should have a `spec` method.""" + + +def run_test_job( + connector: IConnector | type[IConnector] | Callable[[], IConnector], + verb: Literal["read", "check", "discover"], + test_scenario: ConnectorTestScenario, + *, + catalog: ConfiguredAirbyteCatalog | dict[str, Any] | None = None, +) -> entrypoint_wrapper.EntrypointOutput: + """Run a test scenario from provided CLI args and return the result.""" + if not connector: + raise ValueError("Connector is required") + + if catalog and isinstance(catalog, ConfiguredAirbyteCatalog): + # Convert the catalog to a dict if it's already a ConfiguredAirbyteCatalog. + catalog = asdict(catalog) + + connector_obj: IConnector + if isinstance(connector, type) or callable(connector): + # If the connector is a class or a factory lambda, instantiate it. + connector_obj = connector() + elif isinstance(connector, IConnector): + connector_obj = connector + else: + raise ValueError( + f"Invalid connector input: {type(connector)}", + ) + + args: list[str] = [verb] + if test_scenario.config_path: + args += ["--config", str(test_scenario.config_path)] + elif test_scenario.config_dict: + config_path = ( + Path(tempfile.gettempdir()) / "airbyte-test" / f"temp_config_{uuid.uuid4().hex}.json" + ) + config_path.parent.mkdir(parents=True, exist_ok=True) + config_path.write_text(orjson.dumps(test_scenario.config_dict).decode()) + args += ["--config", str(config_path)] + + catalog_path: Path | None = None + if verb not in ["discover", "check"]: + # We need a catalog for read. + if catalog: + # Write the catalog to a temp json file and pass the path to the file as an argument. + catalog_path = ( + Path(tempfile.gettempdir()) + / "airbyte-test" + / f"temp_catalog_{uuid.uuid4().hex}.json" + ) + catalog_path.parent.mkdir(parents=True, exist_ok=True) + catalog_path.write_text(orjson.dumps(catalog).decode()) + elif test_scenario.configured_catalog_path: + catalog_path = Path(test_scenario.configured_catalog_path) + + if catalog_path: + args += ["--catalog", str(catalog_path)] + + # This is a bit of a hack because the source needs the catalog early. + # Because it *also* can fail, we have to redundantly wrap it in a try/except block. + + result: entrypoint_wrapper.EntrypointOutput = entrypoint_wrapper._run_command( # noqa: SLF001 # Non-public API + source=connector_obj, # type: ignore [arg-type] + args=args, + expecting_exception=test_scenario.expect_exception, + ) + if result.errors and not test_scenario.expect_exception: + raise AssertionError( + f"Expected no errors but got {len(result.errors)}: \n" + _errors_to_str(result) + ) + + if verb == "check": + # Check is expected to fail gracefully without an exception. + # Instead, we assert that we have a CONNECTION_STATUS message with + # a failure status. + assert len(result.connection_status_messages) == 1, ( + "Expected exactly one CONNECTION_STATUS message. Got " + f"{len(result.connection_status_messages)}:\n" + + "\n".join([str(msg) for msg in result.connection_status_messages]) + + _errors_to_str(result) + ) + if test_scenario.expect_exception: + conn_status = result.connection_status_messages[0].connectionStatus + assert conn_status, ( + "Expected CONNECTION_STATUS message to be present. Got: \n" + + "\n".join([str(msg) for msg in result.connection_status_messages]) + ) + assert conn_status.status == Status.FAILED, ( + "Expected CONNECTION_STATUS message to be FAILED. Got: \n" + + "\n".join([str(msg) for msg in result.connection_status_messages]) + ) + + return result + + # For all other verbs, we assert check that an exception is raised (or not). + if test_scenario.expect_exception: + if not result.errors: + raise AssertionError("Expected exception but got none.") + + return result + + assert not result.errors, ( + f"Expected no errors but got {len(result.errors)}: \n" + _errors_to_str(result) + ) + + return result diff --git a/airbyte_cdk/test/standard_tests/connector_base.py b/airbyte_cdk/test/standard_tests/connector_base.py new file mode 100644 index 000000000..964d0230d --- /dev/null +++ b/airbyte_cdk/test/standard_tests/connector_base.py @@ -0,0 +1,148 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +"""Base class for connector test suites.""" + +from __future__ import annotations + +import abc +import inspect +import sys +from collections.abc import Callable +from pathlib import Path +from typing import cast + +import yaml +from boltons.typeutils import classproperty + +from airbyte_cdk.models import ( + AirbyteMessage, + Type, +) +from airbyte_cdk.test import entrypoint_wrapper +from airbyte_cdk.test.standard_tests._job_runner import IConnector, run_test_job +from airbyte_cdk.test.standard_tests.models import ( + ConnectorTestScenario, +) + +ACCEPTANCE_TEST_CONFIG = "acceptance-test-config.yml" +MANIFEST_YAML = "manifest.yaml" + + +class ConnectorTestSuiteBase(abc.ABC): + """Base class for connector test suites.""" + + connector: type[IConnector] | Callable[[], IConnector] | None = None + """The connector class or a factory function that returns an scenario of IConnector.""" + + @classmethod + def get_test_class_dir(cls) -> Path: + """Get the file path that contains the class.""" + module = sys.modules[cls.__module__] + # Get the directory containing the test file + return Path(inspect.getfile(module)).parent + + @classmethod + def create_connector( + cls, + scenario: ConnectorTestScenario, + ) -> IConnector: + """Instantiate the connector class.""" + connector = cls.connector # type: ignore + if connector: + if callable(connector) or isinstance(connector, type): + # If the connector is a class or factory function, instantiate it: + return cast(IConnector, connector()) # type: ignore [redundant-cast] + + # Otherwise, we can't instantiate the connector. Fail with a clear error message. + raise NotImplementedError( + "No connector class or connector factory function provided. " + "Please provide a class or factory function in `cls.connector`, or " + "override `cls.create_connector()` to define a custom initialization process." + ) + + # Test Definitions + + def test_check( + self, + scenario: ConnectorTestScenario, + ) -> None: + """Run `connection` acceptance tests.""" + result: entrypoint_wrapper.EntrypointOutput = run_test_job( + self.create_connector(scenario), + "check", + test_scenario=scenario, + ) + conn_status_messages: list[AirbyteMessage] = [ + msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS + ] # noqa: SLF001 # Non-public API + assert len(conn_status_messages) == 1, ( + f"Expected exactly one CONNECTION_STATUS message. Got: {result._messages}" + ) + + @classmethod + def get_connector_root_dir(cls) -> Path: + """Get the root directory of the connector.""" + for parent in cls.get_test_class_dir().parents: + if (parent / MANIFEST_YAML).exists(): + return parent + if (parent / ACCEPTANCE_TEST_CONFIG).exists(): + return parent + if parent.name == "airbyte_cdk": + break + # If we reach here, we didn't find the manifest file in any parent directory + # Check if the manifest file exists in the current directory + for parent in Path.cwd().parents: + if (parent / MANIFEST_YAML).exists(): + return parent + if (parent / ACCEPTANCE_TEST_CONFIG).exists(): + return parent + if parent.name == "airbyte_cdk": + break + + raise FileNotFoundError( + "Could not find connector root directory relative to " + f"'{str(cls.get_test_class_dir())}' or '{str(Path.cwd())}'." + ) + + @classproperty + def acceptance_test_config_path(cls) -> Path: + """Get the path to the acceptance test config file.""" + result = cls.get_connector_root_dir() / ACCEPTANCE_TEST_CONFIG + if result.exists(): + return result + + raise FileNotFoundError(f"Acceptance test config file not found at: {str(result)}") + + @classmethod + def get_scenarios( + cls, + ) -> list[ConnectorTestScenario]: + """Get acceptance tests for a given category. + + This has to be a separate function because pytest does not allow + parametrization of fixtures with arguments from the test class itself. + """ + category = "connection" + all_tests_config = yaml.safe_load(cls.acceptance_test_config_path.read_text()) + if "acceptance_tests" not in all_tests_config: + raise ValueError( + f"Acceptance tests config not found in {cls.acceptance_test_config_path}." + f" Found only: {str(all_tests_config)}." + ) + if category not in all_tests_config["acceptance_tests"]: + return [] + if "tests" not in all_tests_config["acceptance_tests"][category]: + raise ValueError(f"No tests found for category {category}") + + tests_scenarios = [ + ConnectorTestScenario.model_validate(test) + for test in all_tests_config["acceptance_tests"][category]["tests"] + if "iam_role" not in test["config_path"] + ] + connector_root = cls.get_connector_root_dir().absolute() + for test in tests_scenarios: + if test.config_path: + test.config_path = connector_root / test.config_path + if test.configured_catalog_path: + test.configured_catalog_path = connector_root / test.configured_catalog_path + + return tests_scenarios diff --git a/airbyte_cdk/test/standard_tests/declarative_sources.py b/airbyte_cdk/test/standard_tests/declarative_sources.py new file mode 100644 index 000000000..18ac084fc --- /dev/null +++ b/airbyte_cdk/test/standard_tests/declarative_sources.py @@ -0,0 +1,92 @@ +import os +from hashlib import md5 +from pathlib import Path +from typing import Any, cast + +import yaml +from boltons.typeutils import classproperty + +from airbyte_cdk.sources.declarative.concurrent_declarative_source import ( + ConcurrentDeclarativeSource, +) +from airbyte_cdk.test.standard_tests._job_runner import IConnector +from airbyte_cdk.test.standard_tests.connector_base import MANIFEST_YAML +from airbyte_cdk.test.standard_tests.models import ConnectorTestScenario +from airbyte_cdk.test.standard_tests.source_base import SourceTestSuiteBase + + +def md5_checksum(file_path: Path) -> str: + """Helper function to calculate the MD5 checksum of a file. + + This is used to calculate the checksum of the `components.py` file, if it exists. + """ + with open(file_path, "rb") as file: + return md5(file.read()).hexdigest() + + +class DeclarativeSourceTestSuite(SourceTestSuiteBase): + """Declarative source test suite. + + This inherits from the Python-based source test suite and implements the + `create_connector` method to create a declarative source object instead of + requiring a custom Python source object. + + The class also automatically locates the `manifest.yaml` file and the + `components.py` file (if it exists) in the connector's directory. + """ + + @classproperty + def manifest_yaml_path(cls) -> Path: + """Get the path to the manifest.yaml file.""" + result = cls.get_connector_root_dir() / MANIFEST_YAML + if result.exists(): + return result + + raise FileNotFoundError( + f"Manifest YAML file not found at {result}. " + "Please ensure that the test suite is run in the correct directory.", + ) + + @classproperty + def components_py_path(cls) -> Path | None: + """Get the path to the `components.py` file, if one exists. + + If not `components.py` file exists, return None. + """ + result = cls.get_connector_root_dir() / "components.py" + if result.exists(): + return result + + return None + + @classmethod + def create_connector( + cls, + scenario: ConnectorTestScenario, + ) -> IConnector: + """Create a connector scenario for the test suite. + + This overrides `create_connector` from the create a declarative source object + instead of requiring a custom python source object. + + Subclasses should not need to override this method. + """ + config: dict[str, Any] = scenario.get_config_dict() + + manifest_dict = yaml.safe_load(cls.manifest_yaml_path.read_text()) + if cls.components_py_path and cls.components_py_path.exists(): + os.environ["AIRBYTE_ENABLE_UNSAFE_CODE"] = "true" + config["__injected_components_py"] = cls.components_py_path.read_text() + config["__injected_components_py_checksums"] = { + "md5": md5_checksum(cls.components_py_path), + } + + return cast( + IConnector, + ConcurrentDeclarativeSource( + config=config, + catalog=None, + state=None, + source_config=manifest_dict, + ), + ) diff --git a/airbyte_cdk/test/standard_tests/destination_base.py b/airbyte_cdk/test/standard_tests/destination_base.py new file mode 100644 index 000000000..985fc92a3 --- /dev/null +++ b/airbyte_cdk/test/standard_tests/destination_base.py @@ -0,0 +1,16 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +"""Base class for destination test suites.""" + +from airbyte_cdk.test.standard_tests.connector_base import ConnectorTestSuiteBase + + +class DestinationTestSuiteBase(ConnectorTestSuiteBase): + """Base class for destination test suites. + + This class provides a base set of functionality for testing destination connectors, and it + inherits all generic connector tests from the `ConnectorTestSuiteBase` class. + + TODO: As of now, this class does not add any additional functionality or tests specific to + destination connectors. However, it serves as a placeholder for future enhancements and + customizations that may be needed for destination connectors. + """ diff --git a/airbyte_cdk/test/standard_tests/models/__init__.py b/airbyte_cdk/test/standard_tests/models/__init__.py new file mode 100644 index 000000000..13d67e16a --- /dev/null +++ b/airbyte_cdk/test/standard_tests/models/__init__.py @@ -0,0 +1,7 @@ +from airbyte_cdk.test.standard_tests.models.scenario import ( + ConnectorTestScenario, +) + +__all__ = [ + "ConnectorTestScenario", +] diff --git a/airbyte_cdk/test/standard_tests/models/scenario.py b/airbyte_cdk/test/standard_tests/models/scenario.py new file mode 100644 index 000000000..944b60921 --- /dev/null +++ b/airbyte_cdk/test/standard_tests/models/scenario.py @@ -0,0 +1,74 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +"""Run acceptance tests in PyTest. + +These tests leverage the same `acceptance-test-config.yml` configuration files as the +acceptance tests in CAT, but they run in PyTest instead of CAT. This allows us to run +the acceptance tests in the same local environment as we are developing in, speeding +up iteration cycles. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Literal, cast + +import yaml +from pydantic import BaseModel + + +class ConnectorTestScenario(BaseModel): + """Acceptance test scenario, as a Pydantic model. + + This class represents an acceptance test scenario, which is a single test case + that can be run against a connector. It is used to deserialize and validate the + acceptance test configuration file. + """ + + class AcceptanceTestExpectRecords(BaseModel): + path: Path + exact_order: bool = False + + class AcceptanceTestFileTypes(BaseModel): + skip_test: bool + bypass_reason: str + + config_path: Path | None = None + config_dict: dict[str, Any] | None = None + + id: str | None = None + + configured_catalog_path: Path | None = None + timeout_seconds: int | None = None + expect_records: AcceptanceTestExpectRecords | None = None + file_types: AcceptanceTestFileTypes | None = None + status: Literal["succeed", "failed"] | None = None + + def get_config_dict(self) -> dict[str, Any]: + """Return the config dictionary. + + If a config dictionary has already been loaded, return it. Otherwise, load + the config file and return the dictionary. + """ + if self.config_dict: + return self.config_dict + + if self.config_path: + return cast(dict[str, Any], yaml.safe_load(self.config_path.read_text())) + + raise ValueError("No config dictionary or path provided.") + + @property + def expect_exception(self) -> bool: + return self.status and self.status == "failed" or False + + @property + def instance_name(self) -> str: + return self.config_path.stem if self.config_path else "Unnamed Scenario" + + def __str__(self) -> str: + if self.id: + return f"'{self.id}' Test Scenario" + if self.config_path: + return f"'{self.config_path.name}' Test Scenario" + + return f"'{hash(self)}' Test Scenario" diff --git a/airbyte_cdk/test/standard_tests/pytest_hooks.py b/airbyte_cdk/test/standard_tests/pytest_hooks.py new file mode 100644 index 000000000..b6197a0c3 --- /dev/null +++ b/airbyte_cdk/test/standard_tests/pytest_hooks.py @@ -0,0 +1,61 @@ +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. +"""Pytest hooks for Airbyte CDK tests. + +These hooks are used to customize the behavior of pytest during test discovery and execution. + +To use these hooks within a connector, add the following lines to the connector's `conftest.py` +file, or to another file that is imported during test discovery: + +```python +pytest_plugins = [ + "airbyte_cdk.test.standard_tests.pytest_hooks", +] +``` +""" + +import pytest + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + """ + A helper for pytest_generate_tests hook. + + If a test method (in a class subclassed from our base class) + declares an argument 'scenario', this function retrieves the + 'scenarios' attribute from the test class and parametrizes that + test with the values from 'scenarios'. + + ## Usage + + ```python + from airbyte_cdk.test.standard_tests.connector_base import ( + generate_tests, + ConnectorTestSuiteBase, + ) + + def pytest_generate_tests(metafunc): + generate_tests(metafunc) + + class TestMyConnector(ConnectorTestSuiteBase): + ... + + ``` + """ + # Check if the test function requires an 'scenario' argument + if "scenario" in metafunc.fixturenames: + # Retrieve the test class + test_class = metafunc.cls + if test_class is None: + return + + # Get the 'scenarios' attribute from the class + scenarios_attr = getattr(test_class, "get_scenarios", None) + if scenarios_attr is None: + raise ValueError( + f"Test class {test_class} does not have a 'scenarios' attribute. " + "Please define the 'scenarios' attribute in the test class." + ) + + scenarios = test_class.get_scenarios() + ids = [str(scenario) for scenario in scenarios] + metafunc.parametrize("scenario", scenarios, ids=ids) diff --git a/airbyte_cdk/test/standard_tests/source_base.py b/airbyte_cdk/test/standard_tests/source_base.py new file mode 100644 index 000000000..83cc7326f --- /dev/null +++ b/airbyte_cdk/test/standard_tests/source_base.py @@ -0,0 +1,140 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +"""Base class for source test suites.""" + +from dataclasses import asdict + +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + SyncMode, + Type, +) +from airbyte_cdk.test import entrypoint_wrapper +from airbyte_cdk.test.standard_tests._job_runner import run_test_job +from airbyte_cdk.test.standard_tests.connector_base import ( + ConnectorTestSuiteBase, +) +from airbyte_cdk.test.standard_tests.models import ( + ConnectorTestScenario, +) + + +class SourceTestSuiteBase(ConnectorTestSuiteBase): + """Base class for source test suites. + + This class provides a base set of functionality for testing source connectors, and it + inherits all generic connector tests from the `ConnectorTestSuiteBase` class. + """ + + def test_check( + self, + scenario: ConnectorTestScenario, + ) -> None: + """Run standard `check` tests on the connector. + + Assert that the connector returns a single CONNECTION_STATUS message. + This test is designed to validate the connector's ability to establish a connection + and return its status with the expected message type. + """ + result: entrypoint_wrapper.EntrypointOutput = run_test_job( + self.create_connector(scenario), + "check", + test_scenario=scenario, + ) + conn_status_messages: list[AirbyteMessage] = [ + msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS + ] # noqa: SLF001 # Non-public API + num_status_messages = len(conn_status_messages) + assert num_status_messages == 1, ( + f"Expected exactly one CONNECTION_STATUS message. Got {num_status_messages}: \n" + + "\n".join([str(m) for m in result._messages]) + ) + + def test_discover( + self, + scenario: ConnectorTestScenario, + ) -> None: + """Standard test for `discover`.""" + run_test_job( + self.create_connector(scenario), + "discover", + test_scenario=scenario, + ) + + def test_basic_read( + self, + scenario: ConnectorTestScenario, + ) -> None: + """Run standard `read` test on the connector. + + This test is designed to validate the connector's ability to read data + from the source and return records. It first runs a `discover` job to + obtain the catalog of streams, and then it runs a `read` job to fetch + records from those streams. + """ + discover_result = run_test_job( + self.create_connector(scenario), + "discover", + test_scenario=scenario, + ) + if scenario.expect_exception: + assert discover_result.errors, "Expected exception but got none." + return + + configured_catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=stream, + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.append_dedup, + ) + for stream in discover_result.catalog.catalog.streams # type: ignore [reportOptionalMemberAccess, union-attr] + ] + ) + result = run_test_job( + self.create_connector(scenario), + "read", + test_scenario=scenario, + catalog=configured_catalog, + ) + + if not result.records: + raise AssertionError("Expected records but got none.") # noqa: TRY003 + + def test_fail_read_with_bad_catalog( + self, + scenario: ConnectorTestScenario, + ) -> None: + """Standard test for `read` when passed a bad catalog file.""" + invalid_configured_catalog = ConfiguredAirbyteCatalog( + streams=[ + # Create ConfiguredAirbyteStream which is deliberately invalid + # with regard to the Airbyte Protocol. + # This should cause the connector to fail. + ConfiguredAirbyteStream( + stream=AirbyteStream( + name="__AIRBYTE__stream_that_does_not_exist", + json_schema={ + "type": "object", + "properties": {"f1": {"type": "string"}}, + }, + supported_sync_modes=[SyncMode.full_refresh], + ), + sync_mode="INVALID", # type: ignore [reportArgumentType] + destination_sync_mode="INVALID", # type: ignore [reportArgumentType] + ) + ] + ) + # Set expected status to "failed" to ensure the test fails if the connector. + scenario.status = "failed" + result: entrypoint_wrapper.EntrypointOutput = run_test_job( + self.create_connector(scenario), + "read", + test_scenario=scenario, + catalog=asdict(invalid_configured_catalog), + ) + assert result.errors, "Expected errors but got none." + assert result.trace_messages, "Expected trace messages but got none." diff --git a/poetry.lock b/poetry.lock index d415ac2ce..aec560f76 100644 --- a/poetry.lock +++ b/poetry.lock @@ -307,6 +307,19 @@ charset-normalizer = ["charset-normalizer"] html5lib = ["html5lib"] lxml = ["lxml"] +[[package]] +name = "boltons" +version = "25.0.0" +description = "When they're not builtins, they're boltons." +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "boltons-25.0.0-py3-none-any.whl", hash = "sha256:dc9fb38bf28985715497d1b54d00b62ea866eca3938938ea9043e254a3a6ca62"}, + {file = "boltons-25.0.0.tar.gz", hash = "sha256:e110fbdc30b7b9868cb604e3f71d4722dd8f4dcb4a5ddd06028ba8f1ab0b5ace"}, +] + [[package]] name = "bracex" version = "2.5.post1" @@ -5507,4 +5520,4 @@ vector-db-based = ["cohere", "langchain", "openai", "tiktoken"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "08cd7b4f8a9e5cc0fdf400cabafd6aaea7705e460422dbbccf212468d69eaea9" +content-hash = "9854ff162fb8407d116438ded5068bb03510e5692c62d81059c376c30c417948" diff --git a/pyproject.toml b/pyproject.toml index bbfed7639..90d018fed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ enable = true python = ">=3.10,<3.13" airbyte-protocol-models-dataclasses = "^0.14" backoff = "*" +boltons = "^25.0.0" cachetools = "*" dpath = "^2.1.6" dunamai = "^1.22.0" @@ -152,6 +153,9 @@ check-lockfile = {cmd = "poetry check", help = "Check the poetry lock file."} lint-fix = { cmd = "poetry run ruff check --fix .", help = "Auto-fix any lint issues that Ruff can automatically resolve (excluding 'unsafe' fixes)." } lint-fix-unsafe = { cmd = "poetry run ruff check --fix --unsafe-fixes .", help = "Lint-fix modified files, including 'unsafe' fixes. It is recommended to first commit any pending changes and then always manually review any unsafe changes applied." } +# ruff fix everything (ignoring non-Python fixes) +ruff-fix = { sequence = ["lint-fix", "_format-fix-ruff"] , help = "Lint-fix and format-fix all code." } + # Combined Check and Fix tasks check-all = {sequence = ["lint", "format-check", "type-check", "check-lockfile"], help = "Lint, format, and type-check modified files.", ignore_fail = "return_non_zero"} diff --git a/unit_tests/source_declarative_manifest/resources/__init__.py b/unit_tests/resources/__init__.py similarity index 100% rename from unit_tests/source_declarative_manifest/resources/__init__.py rename to unit_tests/resources/__init__.py diff --git a/unit_tests/source_declarative_manifest/resources/invalid_local_manifest.yaml b/unit_tests/resources/invalid_local_manifest.yaml similarity index 100% rename from unit_tests/source_declarative_manifest/resources/invalid_local_manifest.yaml rename to unit_tests/resources/invalid_local_manifest.yaml diff --git a/unit_tests/source_declarative_manifest/resources/invalid_local_pokeapi_config.json b/unit_tests/resources/invalid_local_pokeapi_config.json similarity index 100% rename from unit_tests/source_declarative_manifest/resources/invalid_local_pokeapi_config.json rename to unit_tests/resources/invalid_local_pokeapi_config.json diff --git a/unit_tests/source_declarative_manifest/resources/invalid_remote_config.json b/unit_tests/resources/invalid_remote_config.json similarity index 100% rename from unit_tests/source_declarative_manifest/resources/invalid_remote_config.json rename to unit_tests/resources/invalid_remote_config.json diff --git a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/README.md b/unit_tests/resources/source_pokeapi_w_components_py/README.md similarity index 100% rename from unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/README.md rename to unit_tests/resources/source_pokeapi_w_components_py/README.md diff --git a/unit_tests/resources/source_pokeapi_w_components_py/__init__.py b/unit_tests/resources/source_pokeapi_w_components_py/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/unit_tests/resources/source_pokeapi_w_components_py/acceptance-test-config.yml b/unit_tests/resources/source_pokeapi_w_components_py/acceptance-test-config.yml new file mode 100644 index 000000000..e707a9099 --- /dev/null +++ b/unit_tests/resources/source_pokeapi_w_components_py/acceptance-test-config.yml @@ -0,0 +1,29 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +# connector_image: airbyte/source-pokeapi:dev +acceptance_tests: + spec: + tests: + - spec_path: "manifest.yaml" + backward_compatibility_tests_config: + disable_for_version: "0.1.5" + connection: + tests: + - config_path: "valid_config.yaml" + status: "succeed" + discovery: + tests: + - config_path: "valid_config.yaml" + backward_compatibility_tests_config: + disable_for_version: "0.1.5" + basic_read: + tests: + - config_path: "valid_config.yaml" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + bypass_reason: "This connector does not implement incremental sync" + full_refresh: + tests: + - config_path: "valid_config.yaml" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components.py b/unit_tests/resources/source_pokeapi_w_components_py/components.py similarity index 51% rename from unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components.py rename to unit_tests/resources/source_pokeapi_w_components_py/components.py index 5e7e16f71..214a53641 100644 --- a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components.py +++ b/unit_tests/resources/source_pokeapi_w_components_py/components.py @@ -1,6 +1,7 @@ """A sample implementation of custom components that does nothing but will cause syncs to fail if missing.""" -from typing import Any, Mapping +from collections.abc import Iterable, MutableMapping +from typing import Any import requests @@ -18,3 +19,14 @@ class MyCustomExtractor(DpathExtractor): """ pass + + +class MyCustomFailingExtractor(DpathExtractor): + """Dummy class, intentionally raises an exception when extract_records is called.""" + + def extract_records( + self, + response: requests.Response, + ) -> Iterable[MutableMapping[Any, Any]]: + """Raise an exception when called.""" + raise IntentionalException("This is an intentional failure for testing purposes.") diff --git a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components_failing.py b/unit_tests/resources/source_pokeapi_w_components_py/components_failing.py similarity index 68% rename from unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components_failing.py rename to unit_tests/resources/source_pokeapi_w_components_py/components_failing.py index 5c05881e7..95a7c0662 100644 --- a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components_failing.py +++ b/unit_tests/resources/source_pokeapi_w_components_py/components_failing.py @@ -1,11 +1,7 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# """A sample implementation of custom components that does nothing but will cause syncs to fail if missing.""" from collections.abc import Iterable, MutableMapping -from dataclasses import InitVar, dataclass -from typing import Any, Mapping, Optional, Union +from typing import Any import requests @@ -17,8 +13,11 @@ class IntentionalException(Exception): class MyCustomExtractor(DpathExtractor): + """Dummy class, intentionally raises an exception when extract_records is called.""" + def extract_records( self, response: requests.Response, ) -> Iterable[MutableMapping[Any, Any]]: - raise IntentionalException + """Raise an exception when called.""" + raise IntentionalException("This is an intentional failure for testing purposes.") diff --git a/unit_tests/resources/source_pokeapi_w_components_py/integration_tests/__init__.py b/unit_tests/resources/source_pokeapi_w_components_py/integration_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/unit_tests/resources/source_pokeapi_w_components_py/integration_tests/test_airbyte_standards.py b/unit_tests/resources/source_pokeapi_w_components_py/integration_tests/test_airbyte_standards.py new file mode 100644 index 000000000..7229343ae --- /dev/null +++ b/unit_tests/resources/source_pokeapi_w_components_py/integration_tests/test_airbyte_standards.py @@ -0,0 +1,18 @@ +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. +"""FAST Airbyte Standard Tests for the source_pokeapi_w_components source.""" + +from airbyte_cdk.test.standard_tests import DeclarativeSourceTestSuite + +pytest_plugins = [ + "airbyte_cdk.test.standard_tests.pytest_hooks", +] + + +class TestSuiteSourcePokeAPI(DeclarativeSourceTestSuite): + """Test suite for the source_pokeapi_w_components source. + + This class inherits from SourceTestSuiteBase and implements all of the tests in the suite. + + As long as the class name starts with "Test", pytest will automatically discover and run the + tests in this class. + """ diff --git a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/manifest.yaml b/unit_tests/resources/source_pokeapi_w_components_py/manifest.yaml similarity index 100% rename from unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/manifest.yaml rename to unit_tests/resources/source_pokeapi_w_components_py/manifest.yaml diff --git a/unit_tests/resources/source_pokeapi_w_components_py/valid_config.yaml b/unit_tests/resources/source_pokeapi_w_components_py/valid_config.yaml new file mode 100644 index 000000000..209b0a787 --- /dev/null +++ b/unit_tests/resources/source_pokeapi_w_components_py/valid_config.yaml @@ -0,0 +1 @@ +{ "start_date": "2024-01-01", "pokemon_name": "pikachu" } diff --git a/unit_tests/source_declarative_manifest/resources/valid_local_manifest.yaml b/unit_tests/resources/valid_local_manifest.yaml similarity index 100% rename from unit_tests/source_declarative_manifest/resources/valid_local_manifest.yaml rename to unit_tests/resources/valid_local_manifest.yaml diff --git a/unit_tests/source_declarative_manifest/resources/valid_local_pokeapi_config.json b/unit_tests/resources/valid_local_pokeapi_config.json similarity index 100% rename from unit_tests/source_declarative_manifest/resources/valid_local_pokeapi_config.json rename to unit_tests/resources/valid_local_pokeapi_config.json diff --git a/unit_tests/source_declarative_manifest/resources/valid_remote_config.json b/unit_tests/resources/valid_remote_config.json similarity index 100% rename from unit_tests/source_declarative_manifest/resources/valid_remote_config.json rename to unit_tests/resources/valid_remote_config.json diff --git a/unit_tests/source_declarative_manifest/conftest.py b/unit_tests/source_declarative_manifest/conftest.py index 3d61e65e8..e1d135285 100644 --- a/unit_tests/source_declarative_manifest/conftest.py +++ b/unit_tests/source_declarative_manifest/conftest.py @@ -2,34 +2,34 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. # -import os +from pathlib import Path import pytest import yaml -def get_fixture_path(file_name): - return os.path.join(os.path.dirname(__file__), file_name) +def get_resource_path(file_name) -> str: + return Path(__file__).parent.parent / "resources" / file_name @pytest.fixture def valid_remote_config(): - return get_fixture_path("resources/valid_remote_config.json") + return get_resource_path("valid_remote_config.json") @pytest.fixture def invalid_remote_config(): - return get_fixture_path("resources/invalid_remote_config.json") + return get_resource_path("invalid_remote_config.json") @pytest.fixture def valid_local_manifest(): - return get_fixture_path("resources/valid_local_manifest.yaml") + return get_resource_path("valid_local_manifest.yaml") @pytest.fixture def invalid_local_manifest(): - return get_fixture_path("resources/invalid_local_manifest.yaml") + return get_resource_path("invalid_local_manifest.yaml") @pytest.fixture @@ -46,9 +46,9 @@ def invalid_local_manifest_yaml(invalid_local_manifest): @pytest.fixture def valid_local_config_file(): - return get_fixture_path("resources/valid_local_pokeapi_config.json") + return get_resource_path("valid_local_pokeapi_config.json") @pytest.fixture def invalid_local_config_file(): - return get_fixture_path("resources/invalid_local_pokeapi_config.json") + return get_resource_path("invalid_local_pokeapi_config.json") diff --git a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/valid_config.yaml b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/valid_config.yaml deleted file mode 100644 index 78af092bb..000000000 --- a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/valid_config.yaml +++ /dev/null @@ -1 +0,0 @@ -{ "start_date": "2024-01-01", "pokemon": "pikachu" } diff --git a/unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py b/unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py index 521572bec..12a236ad9 100644 --- a/unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py +++ b/unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py @@ -5,7 +5,6 @@ import datetime import json import logging -import os import sys import types from collections.abc import Callable, Mapping @@ -33,6 +32,7 @@ custom_code_execution_permitted, register_components_module_from_string, ) +from airbyte_cdk.test.standard_tests.connector_base import MANIFEST_YAML SAMPLE_COMPONENTS_PY_TEXT = """ def sample_function() -> str: @@ -44,8 +44,8 @@ def sample_method(self) -> str: """ -def get_fixture_path(file_name) -> str: - return os.path.join(os.path.dirname(__file__), file_name) +def get_resource_path(file_name) -> str: + return Path(__file__).parent.parent / "resources" / file_name def test_components_module_from_string() -> None: @@ -90,15 +90,14 @@ def get_py_components_config_dict( *, failing_components: bool = False, ) -> dict[str, Any]: - connector_dir = Path(get_fixture_path("resources/source_pokeapi_w_components_py")) - manifest_yml_path: Path = connector_dir / "manifest.yaml" + connector_dir = Path(get_resource_path("source_pokeapi_w_components_py")) + manifest_yaml_path: Path = connector_dir / MANIFEST_YAML custom_py_code_path: Path = connector_dir / ( "components.py" if not failing_components else "components_failing.py" ) config_yaml_path: Path = connector_dir / "valid_config.yaml" - secrets_yaml_path: Path = connector_dir / "secrets.yaml" - manifest_dict = yaml.safe_load(manifest_yml_path.read_text()) + manifest_dict = yaml.safe_load(manifest_yaml_path.read_text()) assert manifest_dict, "Failed to load the manifest file." assert isinstance(manifest_dict, Mapping), ( f"Manifest file is type {type(manifest_dict).__name__}, not a mapping: {manifest_dict}" @@ -266,8 +265,8 @@ def test_sync_with_injected_py_components( streams=[ ConfiguredAirbyteStream( stream=stream, - sync_mode="full_refresh", - destination_sync_mode="overwrite", + sync_mode="full_refresh", # type: ignore (intentional bad value) + destination_sync_mode="overwrite", # type: ignore (intentional bad value) ) for stream in catalog.streams ] diff --git a/unit_tests/test/test_standard_tests.py b/unit_tests/test/test_standard_tests.py new file mode 100644 index 000000000..aa2d38a2b --- /dev/null +++ b/unit_tests/test/test_standard_tests.py @@ -0,0 +1,31 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +"""Unit tests for FAST Airbyte Standard Tests.""" + +from typing import Any + +import pytest + +from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource +from airbyte_cdk.sources.source import Source +from airbyte_cdk.test.standard_tests._job_runner import IConnector + + +@pytest.mark.parametrize( + "input, expected", + [ + (DeclarativeSource, True), + (Source, True), + (None, False), + ("", False), + ([], False), + ({}, False), + (object(), False), + ], +) +def test_is_iconnector_check(input: Any, expected: bool) -> None: + """Assert whether inputs are valid as an IConnector object or class.""" + if isinstance(input, type): + assert issubclass(input, IConnector) == expected + return + + assert isinstance(input, IConnector) == expected