diff --git a/src/runloop_api_client/sdk/async_.py b/src/runloop_api_client/sdk/async_.py index ef3191791..d91788857 100644 --- a/src/runloop_api_client/sdk/async_.py +++ b/src/runloop_api_client/sdk/async_.py @@ -2,8 +2,12 @@ from __future__ import annotations +import io +import asyncio +import tarfile from typing import Dict, Mapping, Optional from pathlib import Path +from datetime import timedelta from typing_extensions import Unpack import httpx @@ -350,7 +354,7 @@ async def upload_from_file( path = Path(file_path) try: - content = path.read_bytes() + content = await asyncio.to_thread(lambda: path.read_bytes()) except OSError as error: raise OSError(f"Failed to read file {path}: {error}") from error @@ -361,6 +365,50 @@ async def upload_from_file( await obj.complete() return obj + async def upload_from_dir( + self, + dir_path: str | Path, + *, + name: Optional[str] = None, + metadata: Optional[Dict[str, str]] = None, + ttl: Optional[timedelta] = None, + **options: Unpack[LongRequestOptions], + ) -> AsyncStorageObject: + """Create and upload an object from a local directory. + + The resulting object will be uploaded as a compressed tarball. + + :param dir_path: Local filesystem directory path to tar + :type dir_path: str | Path + :param name: Optional object name; defaults to the directory name + '.tar.gz' + :type name: Optional[str] + :param metadata: Optional key-value metadata + :type metadata: Optional[Dict[str, str]] + :param ttl: Optional Time-To-Live, after which the object is automatically deleted + :type ttl: Optional[timedelta] + :param options: See :typeddict:`~runloop_api_client.sdk._types.LongRequestOptions` for available options + :return: Wrapper for the uploaded object + :rtype: AsyncStorageObject + :raises OSError: If the local file cannot be read + """ + path = Path(dir_path) + name = name or f"{path.name}.tar.gz" + ttl_ms = int(ttl.total_seconds()) * 1000 if ttl else None + + def synchronous_io() -> bytes: + with io.BytesIO() as tar_buffer: + with tarfile.open(fileobj=tar_buffer, mode="w:gz") as tar: + tar.add(path, arcname=".", recursive=True) + tar_buffer.seek(0) + return tar_buffer.read() + + tar_bytes = await asyncio.to_thread(synchronous_io) + + obj = await self.create(name=name, content_type="tgz", metadata=metadata, ttl_ms=ttl_ms, **options) + await obj.upload_content(tar_bytes) + await obj.complete() + return obj + async def upload_from_text( self, text: str, diff --git a/src/runloop_api_client/sdk/async_storage_object.py b/src/runloop_api_client/sdk/async_storage_object.py index fd1f243b8..52e548ad0 100644 --- a/src/runloop_api_client/sdk/async_storage_object.py +++ b/src/runloop_api_client/sdk/async_storage_object.py @@ -2,6 +2,7 @@ from __future__ import annotations +from typing import Iterable from typing_extensions import Unpack, override from ._types import RequestOptions, LongRequestOptions, SDKObjectDownloadParams @@ -146,7 +147,7 @@ async def delete( **options, ) - async def upload_content(self, content: str | bytes) -> None: + async def upload_content(self, content: str | bytes | Iterable[bytes]) -> None: """Upload content to the object's pre-signed URL. :param content: Bytes or text payload to upload diff --git a/src/runloop_api_client/sdk/storage_object.py b/src/runloop_api_client/sdk/storage_object.py index 8b6ec341f..dbeed7f69 100644 --- a/src/runloop_api_client/sdk/storage_object.py +++ b/src/runloop_api_client/sdk/storage_object.py @@ -2,6 +2,7 @@ from __future__ import annotations +from typing import Iterable from typing_extensions import Unpack, override from ._types import RequestOptions, LongRequestOptions, SDKObjectDownloadParams @@ -146,7 +147,7 @@ def delete( **options, ) - def upload_content(self, content: str | bytes) -> None: + def upload_content(self, content: str | bytes | Iterable[bytes]) -> None: """Upload content to the object's pre-signed URL. :param content: Bytes or text payload to upload diff --git a/src/runloop_api_client/sdk/sync.py b/src/runloop_api_client/sdk/sync.py index 04d9b67cf..b2bc53cee 100644 --- a/src/runloop_api_client/sdk/sync.py +++ b/src/runloop_api_client/sdk/sync.py @@ -2,8 +2,11 @@ from __future__ import annotations +import io +import tarfile from typing import Dict, Mapping, Optional from pathlib import Path +from datetime import timedelta from typing_extensions import Unpack import httpx @@ -361,6 +364,46 @@ def upload_from_file( obj.complete() return obj + def upload_from_dir( + self, + dir_path: str | Path, + *, + name: Optional[str] = None, + metadata: Optional[Dict[str, str]] = None, + ttl: Optional[timedelta] = None, + **options: Unpack[LongRequestOptions], + ) -> StorageObject: + """Create and upload an object from a local directory. + + The resulting object will be uploaded as a compressed tarball. + + :param dir_path: Local filesystem directory path to tar + :type dir_path: str | Path + :param name: Optional object name; defaults to the directory name + '.tar.gz' + :type name: Optional[str] + :param metadata: Optional key-value metadata + :type metadata: Optional[Dict[str, str]] + :param ttl: Optional Time-To-Live, after which the object is automatically deleted + :type ttl: Optional[timedelta] + :param options: See :typeddict:`~runloop_api_client.sdk._types.LongRequestOptions` for available options + :return: Wrapper for the uploaded object + :rtype: StorageObject + :raises OSError: If the local file cannot be read + """ + path = Path(dir_path) + name = name or f"{path.name}.tar.gz" + ttl_ms = int(ttl.total_seconds()) * 1000 if ttl else None + + tar_buffer = io.BytesIO() + with tarfile.open(fileobj=tar_buffer, mode="w:gz") as tar: + tar.add(path, arcname=".", recursive=True) + tar_buffer.seek(0) + + obj = self.create(name=name, content_type="tgz", metadata=metadata, ttl_ms=ttl_ms, **options) + obj.upload_content(tar_buffer) + obj.complete() + return obj + def upload_from_text( self, text: str, diff --git a/tests/sdk/test_async_clients.py b/tests/sdk/test_async_clients.py index 6fa1dd9fb..92995eb1e 100644 --- a/tests/sdk/test_async_clients.py +++ b/tests/sdk/test_async_clients.py @@ -2,6 +2,8 @@ from __future__ import annotations +import io +import tarfile from types import SimpleNamespace from pathlib import Path from unittest.mock import AsyncMock @@ -327,6 +329,161 @@ async def test_upload_from_file_missing_path(self, mock_async_client: AsyncMock, with pytest.raises(OSError, match="Failed to read file"): await client.upload_from_file(missing_file) + @pytest.mark.asyncio + async def test_upload_from_dir( + self, mock_async_client: AsyncMock, object_view: MockObjectView, tmp_path: Path + ) -> None: + """Test upload_from_dir method.""" + mock_async_client.objects.create = AsyncMock(return_value=object_view) + mock_async_client.objects.complete = AsyncMock(return_value=object_view) + + # Create a temporary directory with some files + test_dir = tmp_path / "test_directory" + test_dir.mkdir() + (test_dir / "file1.txt").write_text("content1") + (test_dir / "file2.txt").write_text("content2") + subdir = test_dir / "subdir" + subdir.mkdir() + (subdir / "file3.txt").write_text("content3") + + http_client = AsyncMock() + mock_response = create_mock_httpx_response() + http_client.put = AsyncMock(return_value=mock_response) + mock_async_client._client = http_client + + client = AsyncStorageObjectOps(mock_async_client) + obj = await client.upload_from_dir(test_dir, name="archive.tar.gz", metadata={"key": "value"}) + + assert isinstance(obj, AsyncStorageObject) + assert obj.id == "obj_123" + mock_async_client.objects.create.assert_awaited_once_with( + name="archive.tar.gz", + content_type="tgz", + metadata={"key": "value"}, + ttl_ms=None, + ) + # Verify that put was called with tarball content + http_client.put.assert_awaited_once() + call_args = http_client.put.call_args + assert call_args[0][0] == object_view.upload_url + + # Verify it's a valid gzipped tarball + uploaded_content = call_args[1]["content"] + with tarfile.open(fileobj=io.BytesIO(uploaded_content), mode="r:gz") as tar: + members = tar.getmembers() + member_names = [m.name for m in members] + # Should contain our test files (may include directory entries) + assert any("file1.txt" in name for name in member_names) + assert any("file2.txt" in name for name in member_names) + assert any("file3.txt" in name for name in member_names) + + mock_async_client.objects.complete.assert_awaited_once() + + @pytest.mark.asyncio + async def test_upload_from_dir_default_name( + self, mock_async_client: AsyncMock, object_view: MockObjectView, tmp_path: Path + ) -> None: + """Test upload_from_dir uses directory name by default.""" + mock_async_client.objects.create = AsyncMock(return_value=object_view) + mock_async_client.objects.complete = AsyncMock(return_value=object_view) + + test_dir = tmp_path / "my_folder" + test_dir.mkdir() + (test_dir / "file.txt").write_text("content") + + http_client = AsyncMock() + mock_response = create_mock_httpx_response() + http_client.put = AsyncMock(return_value=mock_response) + mock_async_client._client = http_client + + client = AsyncStorageObjectOps(mock_async_client) + obj = await client.upload_from_dir(test_dir) + + assert isinstance(obj, AsyncStorageObject) + # Name should be directory name + .tar.gz + mock_async_client.objects.create.assert_awaited_once() + call_args = mock_async_client.objects.create.call_args + assert call_args[1]["name"] == "my_folder.tar.gz" + assert call_args[1]["content_type"] == "tgz" + + @pytest.mark.asyncio + async def test_upload_from_dir_with_ttl( + self, mock_async_client: AsyncMock, object_view: MockObjectView, tmp_path: Path + ) -> None: + """Test upload_from_dir with TTL.""" + from datetime import timedelta + + mock_async_client.objects.create = AsyncMock(return_value=object_view) + mock_async_client.objects.complete = AsyncMock(return_value=object_view) + + test_dir = tmp_path / "temp_dir" + test_dir.mkdir() + (test_dir / "file.txt").write_text("temporary content") + + http_client = AsyncMock() + mock_response = create_mock_httpx_response() + http_client.put = AsyncMock(return_value=mock_response) + mock_async_client._client = http_client + + client = AsyncStorageObjectOps(mock_async_client) + obj = await client.upload_from_dir(test_dir, ttl=timedelta(hours=2)) + + assert isinstance(obj, AsyncStorageObject) + mock_async_client.objects.create.assert_awaited_once() + call_args = mock_async_client.objects.create.call_args + # 2 hours = 7200 seconds = 7200000 milliseconds + assert call_args[1]["ttl_ms"] == 7200000 + + @pytest.mark.asyncio + async def test_upload_from_dir_empty_directory( + self, mock_async_client: AsyncMock, object_view: MockObjectView, tmp_path: Path + ) -> None: + """Test upload_from_dir with empty directory.""" + mock_async_client.objects.create = AsyncMock(return_value=object_view) + mock_async_client.objects.complete = AsyncMock(return_value=object_view) + + test_dir = tmp_path / "empty_dir" + test_dir.mkdir() + + http_client = AsyncMock() + mock_response = create_mock_httpx_response() + http_client.put = AsyncMock(return_value=mock_response) + mock_async_client._client = http_client + + client = AsyncStorageObjectOps(mock_async_client) + obj = await client.upload_from_dir(test_dir) + + assert isinstance(obj, AsyncStorageObject) + assert obj.id == "obj_123" + mock_async_client.objects.create.assert_awaited_once() + http_client.put.assert_awaited_once() + mock_async_client.objects.complete.assert_awaited_once() + + @pytest.mark.asyncio + async def test_upload_from_dir_with_string_path( + self, mock_async_client: AsyncMock, object_view: MockObjectView, tmp_path: Path + ) -> None: + """Test upload_from_dir with string path instead of Path object.""" + mock_async_client.objects.create = AsyncMock(return_value=object_view) + mock_async_client.objects.complete = AsyncMock(return_value=object_view) + + test_dir = tmp_path / "string_path_dir" + test_dir.mkdir() + (test_dir / "file.txt").write_text("content") + + http_client = AsyncMock() + mock_response = create_mock_httpx_response() + http_client.put = AsyncMock(return_value=mock_response) + mock_async_client._client = http_client + + client = AsyncStorageObjectOps(mock_async_client) + # Pass string path instead of Path object + obj = await client.upload_from_dir(str(test_dir)) + + assert isinstance(obj, AsyncStorageObject) + assert obj.id == "obj_123" + mock_async_client.objects.create.assert_awaited_once() + class TestAsyncRunloopSDK: """Tests for AsyncRunloopSDK class.""" diff --git a/tests/sdk/test_clients.py b/tests/sdk/test_clients.py index 246715008..f0e4b618d 100644 --- a/tests/sdk/test_clients.py +++ b/tests/sdk/test_clients.py @@ -306,6 +306,137 @@ def test_upload_from_file_missing_path(self, mock_client: Mock, tmp_path: Path) with pytest.raises(OSError, match="Failed to read file"): client.upload_from_file(missing_file) + def test_upload_from_dir(self, mock_client: Mock, object_view: MockObjectView, tmp_path: Path) -> None: + """Test upload_from_dir method.""" + mock_client.objects.create.return_value = object_view + + # Create a temporary directory with some files + test_dir = tmp_path / "test_directory" + test_dir.mkdir() + (test_dir / "file1.txt").write_text("content1") + (test_dir / "file2.txt").write_text("content2") + subdir = test_dir / "subdir" + subdir.mkdir() + (subdir / "file3.txt").write_text("content3") + + http_client = Mock() + mock_response = create_mock_httpx_response() + http_client.put.return_value = mock_response + mock_client._client = http_client + + client = StorageObjectOps(mock_client) + obj = client.upload_from_dir(test_dir, name="archive.tar.gz", metadata={"key": "value"}) + + assert isinstance(obj, StorageObject) + assert obj.id == "obj_123" + mock_client.objects.create.assert_called_once_with( + name="archive.tar.gz", + content_type="tgz", + metadata={"key": "value"}, + ttl_ms=None, + ) + # Verify that put was called with tarball content + http_client.put.assert_called_once() + call_args = http_client.put.call_args + assert call_args[0][0] == object_view.upload_url + # Verify it's a BytesIO object + uploaded_content = call_args[1]["content"] + assert hasattr(uploaded_content, "read") + mock_client.objects.complete.assert_called_once() + + def test_upload_from_dir_default_name(self, mock_client: Mock, object_view: MockObjectView, tmp_path: Path) -> None: + """Test upload_from_dir uses directory name by default.""" + mock_client.objects.create.return_value = object_view + + test_dir = tmp_path / "my_folder" + test_dir.mkdir() + (test_dir / "file.txt").write_text("content") + + http_client = Mock() + mock_response = create_mock_httpx_response() + http_client.put.return_value = mock_response + mock_client._client = http_client + + client = StorageObjectOps(mock_client) + obj = client.upload_from_dir(test_dir) + + assert isinstance(obj, StorageObject) + # Name should be directory name + .tar.gz + mock_client.objects.create.assert_called_once() + call_args = mock_client.objects.create.call_args + assert call_args[1]["name"] == "my_folder.tar.gz" + assert call_args[1]["content_type"] == "tgz" + + def test_upload_from_dir_with_ttl(self, mock_client: Mock, object_view: MockObjectView, tmp_path: Path) -> None: + """Test upload_from_dir with TTL.""" + from datetime import timedelta + + mock_client.objects.create.return_value = object_view + + test_dir = tmp_path / "temp_dir" + test_dir.mkdir() + (test_dir / "file.txt").write_text("temporary content") + + http_client = Mock() + mock_response = create_mock_httpx_response() + http_client.put.return_value = mock_response + mock_client._client = http_client + + client = StorageObjectOps(mock_client) + obj = client.upload_from_dir(test_dir, ttl=timedelta(hours=2)) + + assert isinstance(obj, StorageObject) + mock_client.objects.create.assert_called_once() + call_args = mock_client.objects.create.call_args + # 2 hours = 7200 seconds = 7200000 milliseconds + assert call_args[1]["ttl_ms"] == 7200000 + + def test_upload_from_dir_empty_directory( + self, mock_client: Mock, object_view: MockObjectView, tmp_path: Path + ) -> None: + """Test upload_from_dir with empty directory.""" + mock_client.objects.create.return_value = object_view + + test_dir = tmp_path / "empty_dir" + test_dir.mkdir() + + http_client = Mock() + mock_response = create_mock_httpx_response() + http_client.put.return_value = mock_response + mock_client._client = http_client + + client = StorageObjectOps(mock_client) + obj = client.upload_from_dir(test_dir) + + assert isinstance(obj, StorageObject) + assert obj.id == "obj_123" + mock_client.objects.create.assert_called_once() + http_client.put.assert_called_once() + mock_client.objects.complete.assert_called_once() + + def test_upload_from_dir_with_string_path( + self, mock_client: Mock, object_view: MockObjectView, tmp_path: Path + ) -> None: + """Test upload_from_dir with string path instead of Path object.""" + mock_client.objects.create.return_value = object_view + + test_dir = tmp_path / "string_path_dir" + test_dir.mkdir() + (test_dir / "file.txt").write_text("content") + + http_client = Mock() + mock_response = create_mock_httpx_response() + http_client.put.return_value = mock_response + mock_client._client = http_client + + client = StorageObjectOps(mock_client) + # Pass string path instead of Path object + obj = client.upload_from_dir(str(test_dir)) + + assert isinstance(obj, StorageObject) + assert obj.id == "obj_123" + mock_client.objects.create.assert_called_once() + class TestRunloopSDK: """Tests for RunloopSDK class.""" diff --git a/tests/smoketests/sdk/test_async_storage_object.py b/tests/smoketests/sdk/test_async_storage_object.py index b8d7b546b..cec495b79 100644 --- a/tests/smoketests/sdk/test_async_storage_object.py +++ b/tests/smoketests/sdk/test_async_storage_object.py @@ -157,6 +157,45 @@ async def test_upload_from_file(self, async_sdk_client: AsyncRunloopSDK) -> None finally: Path(tmp_path).unlink(missing_ok=True) + @pytest.mark.timeout(THIRTY_SECOND_TIMEOUT) + async def test_upload_from_dir(self, async_sdk_client: AsyncRunloopSDK) -> None: + """Test uploading from directory as tarball.""" + # Create temporary directory with files + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_path = Path(tmp_dir) + (tmp_path / "file1.txt").write_text("Async Content 1") + (tmp_path / "file2.txt").write_text("Async Content 2") + subdir = tmp_path / "subdir" + subdir.mkdir() + (subdir / "file3.txt").write_text("Async Content 3") + + obj = await async_sdk_client.storage_object.upload_from_dir( + tmp_path, + name=unique_name("sdk-async-dir-upload"), + metadata={"source": "upload_from_dir"}, + ) + + try: + assert obj.id is not None + + # Verify it's a tarball + info = await obj.refresh() + assert info.content_type == "tgz" + + # Download and verify tarball can be extracted + import io + import tarfile + + tarball_bytes = await obj.download_as_bytes(duration_seconds=120) + with tarfile.open(fileobj=io.BytesIO(tarball_bytes), mode="r:gz") as tar: + # Verify files exist in tarball + names = tar.getnames() + assert any("file1.txt" in name for name in names) + assert any("file2.txt" in name for name in names) + assert any("file3.txt" in name for name in names) + finally: + await obj.delete() + class TestAsyncStorageObjectDownloadMethods: """Test async storage object download methods.""" diff --git a/tests/smoketests/sdk/test_storage_object.py b/tests/smoketests/sdk/test_storage_object.py index cb0ce557e..da4740c06 100644 --- a/tests/smoketests/sdk/test_storage_object.py +++ b/tests/smoketests/sdk/test_storage_object.py @@ -157,6 +157,45 @@ def test_upload_from_file(self, sdk_client: RunloopSDK) -> None: finally: Path(tmp_path).unlink(missing_ok=True) + @pytest.mark.timeout(THIRTY_SECOND_TIMEOUT) + def test_upload_from_dir(self, sdk_client: RunloopSDK) -> None: + """Test uploading from directory as tarball.""" + # Create temporary directory with files + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_path = Path(tmp_dir) + (tmp_path / "file1.txt").write_text("Content 1") + (tmp_path / "file2.txt").write_text("Content 2") + subdir = tmp_path / "subdir" + subdir.mkdir() + (subdir / "file3.txt").write_text("Content 3") + + obj = sdk_client.storage_object.upload_from_dir( + tmp_path, + name=unique_name("sdk-dir-upload"), + metadata={"source": "upload_from_dir"}, + ) + + try: + assert obj.id is not None + + # Verify it's a tarball + info = obj.refresh() + assert info.content_type == "tgz" + + # Download and verify tarball can be extracted + import io + import tarfile + + tarball_bytes = obj.download_as_bytes(duration_seconds=120) + with tarfile.open(fileobj=io.BytesIO(tarball_bytes), mode="r:gz") as tar: + # Verify files exist in tarball + names = tar.getnames() + assert any("file1.txt" in name for name in names) + assert any("file2.txt" in name for name in names) + assert any("file3.txt" in name for name in names) + finally: + obj.delete() + class TestStorageObjectDownloadMethods: """Test storage object download methods."""