diff --git a/README.rst b/README.rst index 6f7477ca..d1466949 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,7 @@ List of currently supported connections: * Postgres * Oracle * MSSQL +* MySQL * HDFS * S3 diff --git a/docs/changelog/next_release/126.feature.rst b/docs/changelog/next_release/126.feature.rst new file mode 100644 index 00000000..6827d139 --- /dev/null +++ b/docs/changelog/next_release/126.feature.rst @@ -0,0 +1 @@ +Add MySQL API schema \ No newline at end of file diff --git a/syncmaster/backend/api/v1/connections.py b/syncmaster/backend/api/v1/connections.py index 678656a1..d0b2c02f 100644 --- a/syncmaster/backend/api/v1/connections.py +++ b/syncmaster/backend/api/v1/connections.py @@ -22,6 +22,7 @@ HDFS_TYPE, HIVE_TYPE, MSSQL_TYPE, + MYSQL_TYPE, ORACLE_TYPE, POSTGRES_TYPE, S3_TYPE, @@ -39,7 +40,7 @@ router = APIRouter(tags=["Connections"], responses=get_error_responses()) -CONNECTION_TYPES = ORACLE_TYPE, POSTGRES_TYPE, CLICKHOUSE_TYPE, HIVE_TYPE, MSSQL_TYPE, S3_TYPE, HDFS_TYPE +CONNECTION_TYPES = ORACLE_TYPE, POSTGRES_TYPE, CLICKHOUSE_TYPE, HIVE_TYPE, MSSQL_TYPE, MYSQL_TYPE, S3_TYPE, HDFS_TYPE @router.get("/connections") diff --git a/syncmaster/schemas/v1/connection_types.py b/syncmaster/schemas/v1/connection_types.py index 924493a3..24cf0a25 100644 --- a/syncmaster/schemas/v1/connection_types.py +++ b/syncmaster/schemas/v1/connection_types.py @@ -8,6 +8,7 @@ POSTGRES_TYPE = Literal["postgres"] CLICKHOUSE_TYPE = Literal["clickhouse"] MSSQL_TYPE = Literal["mssql"] +MYSQL_TYPE = Literal["mysql"] S3_TYPE = Literal["s3"] HDFS_TYPE = Literal["hdfs"] @@ -18,5 +19,6 @@ class ConnectionType(str, Enum): ORACLE = "oracle" CLICKHOUSE = "clickhouse" MSSQL = "mssql" + MYSQL = "mysql" S3 = "s3" HDFS = "hdfs" diff --git a/syncmaster/schemas/v1/connections/connection.py b/syncmaster/schemas/v1/connections/connection.py index ff597940..48fd27fd 100644 --- a/syncmaster/schemas/v1/connections/connection.py +++ b/syncmaster/schemas/v1/connections/connection.py @@ -35,6 +35,14 @@ UpdateMSSQLAuthSchema, UpdateMSSQLConnectionSchema, ) +from syncmaster.schemas.v1.connections.mysql import ( + CreateMySQLAuthSchema, + CreateMySQLConnectionSchema, + ReadMySQLAuthSchema, + ReadMySQLConnectionSchema, + UpdateMySQLAuthSchema, + UpdateMySQLConnectionSchema, +) from syncmaster.schemas.v1.connections.oracle import ( CreateOracleAuthSchema, CreateOracleConnectionSchema, @@ -69,6 +77,7 @@ | ReadPostgresConnectionSchema | ReadClickhouseConnectionSchema | ReadMSSQLConnectionSchema + | ReadMySQLConnectionSchema | S3ReadConnectionSchema ) CreateConnectionDataSchema = ( @@ -77,6 +86,7 @@ | CreatePostgresConnectionSchema | CreateClickhouseConnectionSchema | CreateMSSQLConnectionSchema + | CreateMySQLConnectionSchema | HDFSCreateConnectionSchema | S3CreateConnectionSchema ) @@ -88,6 +98,7 @@ | UpdatePostgresConnectionSchema | UpdateClickhouseConnectionSchema | UpdateMSSQLConnectionSchema + | UpdateMySQLConnectionSchema ) ReadConnectionAuthDataSchema = ( ReadHiveAuthSchema @@ -95,6 +106,7 @@ | ReadPostgresAuthSchema | ReadClickhouseAuthSchema | ReadMSSQLAuthSchema + | ReadMySQLAuthSchema | S3ReadAuthSchema | HDFSReadAuthSchema ) @@ -104,6 +116,7 @@ | CreatePostgresAuthSchema | CreateClickhouseAuthSchema | CreateMSSQLAuthSchema + | CreateMySQLAuthSchema | S3CreateAuthSchema | HDFSCreateAuthSchema ) @@ -113,6 +126,7 @@ | UpdatePostgresAuthSchema | UpdateClickhouseAuthSchema | UpdateMSSQLAuthSchema + | UpdateMySQLAuthSchema | S3UpdateAuthSchema | HDFSUpdateAuthSchema ) diff --git a/syncmaster/schemas/v1/connections/mysql.py b/syncmaster/schemas/v1/connections/mysql.py new file mode 100644 index 00000000..fe61d82d --- /dev/null +++ b/syncmaster/schemas/v1/connections/mysql.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from pydantic import BaseModel, Field, SecretStr + +from syncmaster.schemas.v1.connection_types import MYSQL_TYPE + + +class MySQLBaseSchema(BaseModel): + type: MYSQL_TYPE + + class Config: + from_attributes = True + + +class ReadMySQLConnectionSchema(MySQLBaseSchema): + host: str + port: int + database_name: str + additional_params: dict = Field(default_factory=dict) + + +class ReadMySQLAuthSchema(MySQLBaseSchema): + user: str + + +class UpdateMySQLConnectionSchema(MySQLBaseSchema): + host: str | None = None + port: int | None = None + database_name: str | None = None + additional_params: dict | None = Field(default_factory=dict) + + +class UpdateMySQLAuthSchema(MySQLBaseSchema): + user: str | None = None # noqa: F722 + password: SecretStr | None = None + + +class CreateMySQLConnectionSchema(MySQLBaseSchema): + host: str + port: int + database_name: str + additional_params: dict = Field(default_factory=dict) + + +class CreateMySQLAuthSchema(MySQLBaseSchema): + user: str + password: SecretStr diff --git a/syncmaster/schemas/v1/transfers/__init__.py b/syncmaster/schemas/v1/transfers/__init__.py index b998e3dd..d90732d3 100644 --- a/syncmaster/schemas/v1/transfers/__init__.py +++ b/syncmaster/schemas/v1/transfers/__init__.py @@ -10,6 +10,7 @@ ClickhouseReadTransferSourceAndTarget, HiveReadTransferSourceAndTarget, MSSQLReadTransferSourceAndTarget, + MySQLReadTransferSourceAndTarget, OracleReadTransferSourceAndTarget, PostgresReadTransferSourceAndTarget, ) @@ -35,6 +36,7 @@ | OracleReadTransferSourceAndTarget | ClickhouseReadTransferSourceAndTarget | MSSQLReadTransferSourceAndTarget + | MySQLReadTransferSourceAndTarget | S3ReadTransferSource ) @@ -45,6 +47,7 @@ | OracleReadTransferSourceAndTarget | ClickhouseReadTransferSourceAndTarget | MSSQLReadTransferSourceAndTarget + | MySQLReadTransferSourceAndTarget | S3ReadTransferTarget ) @@ -55,6 +58,7 @@ | OracleReadTransferSourceAndTarget | ClickhouseReadTransferSourceAndTarget | MSSQLReadTransferSourceAndTarget + | MySQLReadTransferSourceAndTarget | S3CreateTransferSource ) @@ -65,6 +69,7 @@ | OracleReadTransferSourceAndTarget | ClickhouseReadTransferSourceAndTarget | MSSQLReadTransferSourceAndTarget + | MySQLReadTransferSourceAndTarget | S3CreateTransferTarget ) @@ -75,6 +80,7 @@ | OracleReadTransferSourceAndTarget | ClickhouseReadTransferSourceAndTarget | MSSQLReadTransferSourceAndTarget + | MySQLReadTransferSourceAndTarget | S3CreateTransferSource | None ) @@ -86,6 +92,7 @@ | OracleReadTransferSourceAndTarget | ClickhouseReadTransferSourceAndTarget | MSSQLReadTransferSourceAndTarget + | MySQLReadTransferSourceAndTarget | S3CreateTransferTarget | None ) diff --git a/syncmaster/schemas/v1/transfers/db.py b/syncmaster/schemas/v1/transfers/db.py index 751bf57a..0305d807 100644 --- a/syncmaster/schemas/v1/transfers/db.py +++ b/syncmaster/schemas/v1/transfers/db.py @@ -8,6 +8,7 @@ CLICKHOUSE_TYPE, HIVE_TYPE, MSSQL_TYPE, + MYSQL_TYPE, ORACLE_TYPE, POSTGRES_TYPE, ) @@ -35,3 +36,7 @@ class ClickhouseReadTransferSourceAndTarget(ReadDBTransfer): class MSSQLReadTransferSourceAndTarget(ReadDBTransfer): type: MSSQL_TYPE + + +class MySQLReadTransferSourceAndTarget(ReadDBTransfer): + type: MYSQL_TYPE diff --git a/tests/test_unit/test_connections/connection_fixtures/group_connections_fixture.py b/tests/test_unit/test_connections/connection_fixtures/group_connections_fixture.py index b698ffa1..a679d89c 100644 --- a/tests/test_unit/test_connections/connection_fixtures/group_connections_fixture.py +++ b/tests/test_unit/test_connections/connection_fixtures/group_connections_fixture.py @@ -39,7 +39,7 @@ async def group_connections( "bucket": "bucket", }, ) - elif conn_type == ConnectionType.POSTGRES: + elif conn_type in [ConnectionType.POSTGRES, ConnectionType.MYSQL]: new_data.update( { "database_name": "database", diff --git a/tests/test_unit/test_connections/test_create_connection.py b/tests/test_unit/test_connections/test_create_connection.py index e0a43ec8..14cb2f73 100644 --- a/tests/test_unit/test_connections/test_create_connection.py +++ b/tests/test_unit/test_connections/test_create_connection.py @@ -282,7 +282,7 @@ async def test_check_fields_validation_on_create_connection( "context": { "discriminator": "'type'", "tag": "POSTGRESQL", - "expected_tags": "'hive', 'oracle', 'postgres', 'clickhouse', 'mssql', 'hdfs', 's3'", + "expected_tags": "'hive', 'oracle', 'postgres', 'clickhouse', 'mssql', 'mysql', 'hdfs', 's3'", }, "input": { "type": "POSTGRESQL", @@ -292,7 +292,7 @@ async def test_check_fields_validation_on_create_connection( "database_name": "postgres", }, "location": ["body", "connection_data"], - "message": "Input tag 'POSTGRESQL' found using 'type' does not match any of the expected tags: 'hive', 'oracle', 'postgres', 'clickhouse', 'mssql', 'hdfs', 's3'", + "message": "Input tag 'POSTGRESQL' found using 'type' does not match any of the expected tags: 'hive', 'oracle', 'postgres', 'clickhouse', 'mssql', 'mysql', 'hdfs', 's3'", "code": "union_tag_invalid", }, ], diff --git a/tests/test_unit/test_connections/test_db_connection/test_create_mysql_connection.py b/tests/test_unit/test_connections/test_db_connection/test_create_mysql_connection.py new file mode 100644 index 00000000..5ce60c8e --- /dev/null +++ b/tests/test_unit/test_connections/test_db_connection/test_create_mysql_connection.py @@ -0,0 +1,80 @@ +import pytest +from httpx import AsyncClient +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from syncmaster.db.models import AuthData, Connection +from syncmaster.db.repositories.utils import decrypt_auth_data +from syncmaster.settings import Settings +from tests.mocks import MockGroup, UserTestRoles + +pytestmark = [pytest.mark.asyncio, pytest.mark.backend, pytest.mark.mysql] + + +async def test_developer_plus_can_create_mysql_connection( + client: AsyncClient, + group: MockGroup, + session: AsyncSession, + settings: Settings, + role_developer_plus: UserTestRoles, +): + # Arrange + user = group.get_member_of_role(role_developer_plus) + + # Act + result = await client.post( + "v1/connections", + headers={"Authorization": f"Bearer {user.token}"}, + json={ + "group_id": group.id, + "name": "New connection", + "description": "", + "connection_data": { + "type": "mysql", + "host": "127.0.0.1", + "port": 3306, + "database_name": "database", + }, + "auth_data": { + "type": "mysql", + "user": "user", + "password": "secret", + }, + }, + ) + connection = ( + await session.scalars( + select(Connection).filter_by( + name="New connection", + ), + ) + ).first() + + creds = ( + await session.scalars( + select(AuthData).filter_by( + connection_id=connection.id, + ), + ) + ).one() + + # Assert + decrypted = decrypt_auth_data(creds.value, settings=settings) + assert result.status_code == 200 + assert result.json() == { + "id": connection.id, + "name": connection.name, + "description": connection.description, + "group_id": connection.group_id, + "connection_data": { + "type": connection.data["type"], + "host": connection.data["host"], + "port": connection.data["port"], + "database_name": connection.data["database_name"], + "additional_params": connection.data["additional_params"], + }, + "auth_data": { + "type": decrypted["type"], + "user": decrypted["user"], + }, + } diff --git a/tests/test_unit/test_connections/test_db_connection/test_update_mysql_connection.py b/tests/test_unit/test_connections/test_db_connection/test_update_mysql_connection.py new file mode 100644 index 00000000..33c8f6ce --- /dev/null +++ b/tests/test_unit/test_connections/test_db_connection/test_update_mysql_connection.py @@ -0,0 +1,72 @@ +import pytest +from httpx import AsyncClient + +from tests.mocks import MockConnection, UserTestRoles + +pytestmark = [pytest.mark.asyncio, pytest.mark.backend, pytest.mark.mysql] + + +@pytest.mark.parametrize( + "create_connection_data,create_connection_auth_data", + [ + ( + { + "type": "mysql", + "host": "127.0.0.1", + "port": 3306, + "database_name": "database", + }, + { + "type": "mysql", + "user": "user", + "password": "secret", + }, + ), + ], + indirect=True, +) +async def test_developer_plus_can_update_mysql_connection( + client: AsyncClient, + group_connection: MockConnection, + role_developer_plus: UserTestRoles, +): + # Arrange + user = group_connection.owner_group.get_member_of_role(role_developer_plus) + group_connection.connection.group.id + + # Act + result = await client.patch( + f"v1/connections/{group_connection.id}", + headers={"Authorization": f"Bearer {user.token}"}, + json={ + "connection_data": { + "type": "mysql", + "host": "127.0.1.1", + "database_name": "new_database", + }, + "auth_data": { + "type": "mysql", + "user": "new_user", + }, + }, + ) + + # Assert + assert result.status_code == 200 + assert result.json() == { + "id": group_connection.id, + "name": group_connection.name, + "description": group_connection.description, + "group_id": group_connection.group_id, + "connection_data": { + "type": group_connection.data["type"], + "host": "127.0.1.1", + "port": group_connection.data["port"], + "database_name": "new_database", + "additional_params": {}, + }, + "auth_data": { + "type": group_connection.credentials.value["type"], + "user": "new_user", + }, + } diff --git a/tests/test_unit/test_connections/test_read_connections.py b/tests/test_unit/test_connections/test_read_connections.py index 3cb6e124..54eb6826 100644 --- a/tests/test_unit/test_connections/test_read_connections.py +++ b/tests/test_unit/test_connections/test_read_connections.py @@ -307,10 +307,10 @@ async def test_search_connections_with_nonexistent_query( @pytest.mark.parametrize( "filter_params, expected_total", [ - ({}, 7), # No filters applied, expecting all connections + ({}, 8), # No filters applied, expecting all connections ({"type": ["oracle"]}, 1), ({"type": ["postgres", "hive"]}, 2), - ({"type": ["postgres", "hive", "oracle", "clickhouse", "mssql", "hdfs", "s3"]}, 7), + ({"type": ["postgres", "hive", "oracle", "clickhouse", "mssql", "mysql", "hdfs", "s3"]}, 8), ], ids=[ "no_filters", diff --git a/tests/test_unit/test_transfers/test_create_transfer.py b/tests/test_unit/test_transfers/test_create_transfer.py index 67a3bb5d..35b98ea4 100644 --- a/tests/test_unit/test_transfers/test_create_transfer.py +++ b/tests/test_unit/test_transfers/test_create_transfer.py @@ -370,12 +370,12 @@ async def test_superuser_can_create_transfer( "location": ["body", "source_params"], "message": ( "Input tag 'new some connection type' found using 'type' " - "does not match any of the expected tags: 'postgres', 'hdfs', 'hive', 'oracle', 'clickhouse', 'mssql', 's3'" + "does not match any of the expected tags: 'postgres', 'hdfs', 'hive', 'oracle', 'clickhouse', 'mssql', 'mysql', 's3'" ), "code": "union_tag_invalid", "context": { "discriminator": "'type'", - "expected_tags": "'postgres', 'hdfs', 'hive', 'oracle', 'clickhouse', 'mssql', 's3'", + "expected_tags": "'postgres', 'hdfs', 'hive', 'oracle', 'clickhouse', 'mssql', 'mysql', 's3'", "tag": "new some connection type", }, "input": { diff --git a/tests/test_unit/test_transfers/transfer_fixtures/transfers_fixture.py b/tests/test_unit/test_transfers/transfer_fixtures/transfers_fixture.py index f02d1998..9790372d 100644 --- a/tests/test_unit/test_transfers/transfer_fixtures/transfers_fixture.py +++ b/tests/test_unit/test_transfers/transfer_fixtures/transfers_fixture.py @@ -82,6 +82,7 @@ async def group_transfers( ConnectionType.ORACLE, ConnectionType.CLICKHOUSE, ConnectionType.MSSQL, + ConnectionType.MYSQL, ]: source_params["table_name"] = "source_table" target_params["table_name"] = "target_table"