diff --git a/docs/changelog/next_release/142.feature.rst b/docs/changelog/next_release/142.feature.rst new file mode 100644 index 00000000..10d534f5 --- /dev/null +++ b/docs/changelog/next_release/142.feature.rst @@ -0,0 +1 @@ +Add XML API schema \ No newline at end of file diff --git a/syncmaster/schemas/v1/__init__.py b/syncmaster/schemas/v1/__init__.py index 6fa935b0..bf3d0e21 100644 --- a/syncmaster/schemas/v1/__init__.py +++ b/syncmaster/schemas/v1/__init__.py @@ -36,7 +36,7 @@ PostgresReadTransferSourceAndTarget, ReadDBTransfer, ) -from syncmaster.schemas.v1.transfers.file_format import CSV, JSON, Excel, JSONLine +from syncmaster.schemas.v1.transfers.file_format import CSV, JSON, XML, Excel, JSONLine from syncmaster.schemas.v1.transfers.run import ( CreateRunSchema, ReadRunSchema, diff --git a/syncmaster/schemas/v1/file_formats.py b/syncmaster/schemas/v1/file_formats.py index ebf3e389..b0e038e6 100644 --- a/syncmaster/schemas/v1/file_formats.py +++ b/syncmaster/schemas/v1/file_formats.py @@ -6,3 +6,4 @@ JSONLINE_FORMAT = Literal["jsonline"] JSON_FORMAT = Literal["json"] EXCEL_FORMAT = Literal["excel"] +XML_FORMAT = Literal["xml"] diff --git a/syncmaster/schemas/v1/transfers/file/base.py b/syncmaster/schemas/v1/transfers/file/base.py index 3a131a4f..d4ddf4af 100644 --- a/syncmaster/schemas/v1/transfers/file/base.py +++ b/syncmaster/schemas/v1/transfers/file/base.py @@ -7,20 +7,24 @@ from pydantic import BaseModel, Field, field_validator -from syncmaster.schemas.v1.transfers.file_format import CSV, JSON, Excel, JSONLine +from syncmaster.schemas.v1.transfers.file_format import CSV, JSON, XML, Excel, JSONLine # At the moment the ReadTransferSourceParams and ReadTransferTargetParams # classes are identical but may change in the future class ReadFileTransferSource(BaseModel): directory_path: str - file_format: CSV | JSONLine | JSON | Excel = Field(..., discriminator="type") + file_format: CSV | JSONLine | JSON | Excel | XML = Field(..., discriminator="type") options: dict[str, Any] class ReadFileTransferTarget(BaseModel): directory_path: str - file_format: CSV | JSONLine | Excel = Field(..., discriminator="type") # JSON format is not supported for writing + # JSON format is not supported for writing + file_format: CSV | JSONLine | Excel | XML = Field( + ..., + discriminator="type", + ) options: dict[str, Any] @@ -28,7 +32,7 @@ class ReadFileTransferTarget(BaseModel): # classes are identical but may change in the future class CreateFileTransferSource(BaseModel): directory_path: str - file_format: CSV | JSONLine | JSON | Excel = Field(..., discriminator="type") + file_format: CSV | JSONLine | JSON | Excel | XML = Field(..., discriminator="type") options: dict[str, Any] = Field(default_factory=dict) class Config: @@ -44,7 +48,11 @@ def _directory_path_is_valid_path(cls, value): class CreateFileTransferTarget(BaseModel): directory_path: str - file_format: CSV | JSONLine | Excel = Field(..., discriminator="type") # JSON FORMAT IS NOT SUPPORTED AS A TARGET ! + # JSON format is not supported as a target + file_format: CSV | JSONLine | Excel | XML = Field( + ..., + discriminator="type", + ) options: dict[str, Any] = Field(default_factory=dict) class Config: diff --git a/syncmaster/schemas/v1/transfers/file_format.py b/syncmaster/schemas/v1/transfers/file_format.py index 2c91ef8d..da0f7e53 100644 --- a/syncmaster/schemas/v1/transfers/file_format.py +++ b/syncmaster/schemas/v1/transfers/file_format.py @@ -9,6 +9,7 @@ EXCEL_FORMAT, JSON_FORMAT, JSONLINE_FORMAT, + XML_FORMAT, ) @@ -38,3 +39,9 @@ class Excel(BaseModel): type: EXCEL_FORMAT include_header: bool = False start_cell: str | None = None + + +class XML(BaseModel): + type: XML_FORMAT + root_tag: str + row_tag: str diff --git a/tests/test_unit/test_transfers/test_create_transfer.py b/tests/test_unit/test_transfers/test_create_transfer.py index 3826adde..03245e54 100644 --- a/tests/test_unit/test_transfers/test_create_transfer.py +++ b/tests/test_unit/test_transfers/test_create_transfer.py @@ -636,11 +636,11 @@ async def test_developer_plus_can_not_create_transfer_with_target_format_json( "context": { "discriminator": "'type'", "tag": "json", - "expected_tags": "'csv', 'jsonline', 'excel'", + "expected_tags": "'csv', 'jsonline', 'excel', 'xml'", }, "input": {"type": "json", "lineSep": "\n", "encoding": "utf-8"}, "location": ["body", "target_params", "s3", "file_format"], - "message": "Input tag 'json' found using 'type' does not match any of the expected tags: 'csv', 'jsonline', 'excel'", + "message": "Input tag 'json' found using 'type' does not match any of the expected tags: 'csv', 'jsonline', 'excel', 'xml'", "code": "union_tag_invalid", }, ], diff --git a/tests/test_unit/test_transfers/test_file_transfers/test_create_transfer.py b/tests/test_unit/test_transfers/test_file_transfers/test_create_transfer.py index f60d8782..7f584075 100644 --- a/tests/test_unit/test_transfers/test_file_transfers/test_create_transfer.py +++ b/tests/test_unit/test_transfers/test_file_transfers/test_create_transfer.py @@ -51,6 +51,18 @@ "some": "option", }, }, + { + "type": "s3", + "directory_path": "/some/xml/path", + "file_format": { + "type": "xml", + "root_tag": "data", + "row_tag": "record", + }, + "options": { + "some": "option", + }, + }, ], ) async def test_developer_plus_can_create_s3_transfer( @@ -127,6 +139,11 @@ async def test_developer_plus_can_create_s3_transfer( "include_header": True, "start_cell": "A1", }, + "xml": { + "type": "xml", + "root_tag": "data", + "row_tag": "record", + }, } for params in (transfer.source_params, transfer.target_params): @@ -165,6 +182,15 @@ async def test_developer_plus_can_create_s3_transfer( "start_cell": "A1", }, }, + { + "type": "hdfs", + "directory_path": "/some/xml/path", + "file_format": { + "type": "xml", + "root_tag": "data", + "row_tag": "record", + }, + }, ], ) async def test_developer_plus_can_create_hdfs_transfer( @@ -242,6 +268,11 @@ async def test_developer_plus_can_create_hdfs_transfer( "include_header": True, "start_cell": "A1", }, + "xml": { + "type": "xml", + "root_tag": "data", + "row_tag": "record", + }, } for params in (transfer.source_params, transfer.target_params): @@ -280,6 +311,15 @@ async def test_developer_plus_can_create_hdfs_transfer( "include_header": True, }, }, + { + "type": "s3", + "directory_path": "some/path", + "file_format": { + "type": "xml", + "root_tag": "data", + "row_tag": "record", + }, + }, ], ) async def test_cannot_create_file_transfer_with_relative_path( diff --git a/tests/test_unit/test_transfers/test_file_transfers/test_read_transfer.py b/tests/test_unit/test_transfers/test_file_transfers/test_read_transfer.py index bb5e9e3a..5380f185 100644 --- a/tests/test_unit/test_transfers/test_file_transfers/test_read_transfer.py +++ b/tests/test_unit/test_transfers/test_file_transfers/test_read_transfer.py @@ -33,6 +33,16 @@ }, "options": {}, }, + { + "type": "s3", + "directory_path": "/some/xml/path", + "file_format": { + "type": "xml", + "root_tag": "data", + "row_tag": "record", + }, + "options": {}, + }, ], ) @pytest.mark.parametrize( diff --git a/tests/test_unit/test_transfers/test_file_transfers/test_update_transfer.py b/tests/test_unit/test_transfers/test_file_transfers/test_update_transfer.py index 182bab00..50398a11 100644 --- a/tests/test_unit/test_transfers/test_file_transfers/test_update_transfer.py +++ b/tests/test_unit/test_transfers/test_file_transfers/test_update_transfer.py @@ -33,6 +33,16 @@ }, "options": {}, }, + { + "type": "s3", + "directory_path": "/some/xml/path", + "file_format": { + "type": "xml", + "root_tag": "data", + "row_tag": "record", + }, + "options": {}, + }, ], ) @pytest.mark.parametrize(