diff --git a/README-SDK.md b/README-SDK.md index b50d10b71..4c8d9db51 100644 --- a/README-SDK.md +++ b/README-SDK.md @@ -8,11 +8,23 @@ The `RunloopSDK` builds on top of the underlying REST client and provides a Pyth - [Quickstart (synchronous)](#quickstart-synchronous) - [Quickstart (asynchronous)](#quickstart-asynchronous) - [Core Concepts](#core-concepts) -- [Devbox](#devbox) -- [Blueprint](#blueprint) -- [Snapshot](#snapshot) -- [StorageObject](#storageobject) -- [Mounting Storage Objects to Devboxes](#mounting-storage-objects-to-devboxes) + - [RunloopSDK](#runloopsdk) + - [Available Resources](#available-resources) + - [Devbox](#devbox) + - [Command Execution](#command-execution) + - [Execution Management](#execution-management) + - [Execution Results](#execution-results) + - [Streaming Command Output](#streaming-command-output) + - [File Operations](#file-operations) + - [Network Operations](#network-operations) + - [Snapshot Operations](#snapshot-operations) + - [Devbox Lifecycle Management](#devbox-lifecycle-management) + - [Context Manager Support](#context-manager-support) + - [Blueprint](#blueprint) + - [Snapshot](#snapshot) + - [StorageObject](#storageobject) + - [Storage Object Upload Helpers](#storage-object-upload-helpers) + - [Mounting Storage Objects to Devboxes](#mounting-storage-objects-to-devboxes) - [Accessing the Underlying REST Client](#accessing-the-underlying-rest-client) - [Error Handling](#error-handling) - [Advanced Configuration](#advanced-configuration) @@ -409,6 +421,52 @@ blueprint = runloop.blueprint.create( system_setup_commands=["pip install numpy pandas"], ) +# Or create a blueprint with a Docker build context from a local directory +from pathlib import Path +from runloop_api_client.lib.context_loader import build_docker_context_tar + +context_root = Path("./my-app") +tar_bytes = build_docker_context_tar(context_root) + +build_ctx_obj = runloop.storage_object.upload_from_bytes( + data=tar_bytes, + name="my-app-context.tar.gz", + content_type="tgz", +) + +shared_root = Path("./shared-lib") +shared_tar = build_docker_context_tar(shared_root) + +shared_ctx_obj = runloop.storage_object.upload_from_bytes( + data=shared_tar, + name="shared-lib-context.tar.gz", + content_type="tgz", +) + +blueprint_with_context = runloop.blueprint.create( + name="my-blueprint-with-context", + dockerfile="""\ +FROM node:22 +WORKDIR /usr/src/app + +# copy using the build context from the object +COPY package.json package.json +COPY src src + +# copy from named context +COPY --from=shared / ./libs + +RUN npm install --only=production +CMD ["node", "src/app.js"] +""", + # Primary build context + build_context=build_ctx_obj.as_build_context(), + # Additional named build contexts (for Docker buildx-style usage) + named_build_contexts={ + "shared": shared_ctx_obj.as_build_context(), + }, +) + # Or get an existing one blueprint = runloop.blueprint.from_id(blueprint_id="bpt_123") diff --git a/src/runloop_api_client/lib/__init__.py b/src/runloop_api_client/lib/__init__.py new file mode 100644 index 000000000..a18bc710b --- /dev/null +++ b/src/runloop_api_client/lib/__init__.py @@ -0,0 +1,3 @@ +""" +Helpers for `runloop_api_client`. +""" diff --git a/src/runloop_api_client/lib/_ignore.py b/src/runloop_api_client/lib/_ignore.py new file mode 100644 index 000000000..d1b5d64b0 --- /dev/null +++ b/src/runloop_api_client/lib/_ignore.py @@ -0,0 +1,496 @@ +from __future__ import annotations + +import os +from abc import ABC, abstractmethod +from typing import Iterable, Optional, Sequence +from pathlib import Path, PurePosixPath +from dataclasses import dataclass +from typing_extensions import override + +__all__ = [ + "IgnorePattern", + "IgnoreMatcher", + "DockerIgnoreMatcher", + "FilePatternMatcher", + "read_ignorefile", + "compile_ignore", + "path_match", + "is_ignored", + "iter_included_files", +] + + +@dataclass(frozen=True) +class IgnorePattern: + """Single parsed ignore pattern. + + Follows Docker-style ``.dockerignore`` semantics and supports other ignore + use cases following the same approach. + """ + + pattern: str + """The normalized pattern text with leading and trailing ``/`` removed. + + Always uses POSIX ``'/'`` separators. + """ + + negated: bool + """Whether this is a negation pattern starting with ``!``.""" + + directory_only: bool + """Whether the original pattern ended with ``/`` and should apply only to + directories and their descendants. + """ + + anchored: bool + """Whether the pattern contains a path separator and should be matched + relative to the root path rather than at any depth. + """ + + +def _normalize_pattern_string(raw: str) -> str: + """Normalize a single ignore pattern string. + + Shared helper for patterns coming from both ignorefiles and inline pattern + lists. Handles: + + - Optional leading ``!`` negation marker (with surrounding whitespace + trimmed). + - ``os.path.normpath`` cleanup. + - Normalising path separators to POSIX ``'/'``. + - Stripping a single leading ``/`` so absolute-style patterns behave like + relative ones. + + Comment / blank-line handling is deliberately *not* included here; callers + are responsible for that. + """ + + if not raw: + return raw + + invert = raw[0] == "!" + pattern = raw[1:].strip() if invert else raw.strip() + + if pattern: + # filepath.Clean equivalent + pattern = os.path.normpath(pattern) + # filepath.ToSlash equivalent + pattern = pattern.replace(os.sep, "/") + # Leading forward-slashes are removed so "/some/path" and "some/path" + # are considered equivalent. + if len(pattern) > 1 and pattern[0] == "/": + pattern = pattern[1:] + + if invert: + pattern = "!" + pattern + + return pattern + + +def _normalize_pattern_line(raw: bytes, *, is_first_line: bool) -> Optional[str]: + """Normalize a single ignorefile line, mirroring moby's ignorefile.ReadAll. + + Behavior is based on: + https://github.com/moby/patternmatcher/blob/main/ignorefile/ignorefile.go + + :param raw: Raw line bytes from the ignore file, including any newline + characters. + :type raw: bytes + :param is_first_line: Whether this is the first line in the file (used to + detect and strip a UTF-8 BOM). + :type is_first_line: bool + :return: Normalized pattern string, or ``None`` if the line should be + ignored (empty or comment). + :rtype: Optional[str] + """ + + # Strip UTF-8 BOM from the first line if present + if is_first_line and raw.startswith(b"\xef\xbb\xbf"): + raw = raw[len(b"\xef\xbb\xbf") :] + + # Decode as UTF-8; we are strict here to surface bad encodings + text = raw.decode("utf-8", errors="strict") + text = text.rstrip("\r\n") + + # Lines starting with '#' are comments and are ignored before processing, + # i.e. we do *not* treat leading spaces as part of the comment detection. + if text.startswith("#"): + return None + + # Trim leading and trailing whitespace + pattern = text.strip() + if not pattern: + return None + + normalized = _normalize_pattern_string(pattern) + return normalized or None + + +def read_ignorefile(path: Path) -> list[str]: + """Read an ignore file and return a list of normalized pattern strings. + + This mirrors the behavior of moby's ``ignorefile.ReadAll``: + + - UTF-8 BOM on the first line is stripped. + - Lines starting with ``#`` are treated as comments and skipped. + - Remaining lines are trimmed, optionally negated with ``!``, cleaned, + have path separators normalized to ``/``, and leading and trailing ``/`` removed. + + :param path: Filesystem path to the ignore file to read. + :type path: Path + :return: List of normalized pattern strings in the order they appear in + the ignore file. + :rtype: list[str] + """ + + if not path.exists(): + return [] + + patterns: list[str] = [] + with path.open("rb") as f: + first = True + for raw in f: + normalized = _normalize_pattern_line(raw, is_first_line=first) + first = False + if normalized is None: + continue + patterns.append(normalized) + + return patterns + + +def compile_ignore(patterns: Sequence[str]) -> list[IgnorePattern]: + """Compile raw pattern strings into :class:`IgnorePattern` objects. + + :param patterns: Raw pattern strings following Docker-style semantics. + :type patterns: Sequence[str] + :return: Compiled ignore patterns. + :rtype: list[IgnorePattern] + """ + + compiled: list[IgnorePattern] = [] + + for raw in patterns: + if not raw: + continue + + negated = raw[0] == "!" + pattern_text = raw[1:] if negated else raw + + if not pattern_text: + # Bare "!" is ignored, matching Docker / moby behavior. + continue + + directory_only = pattern_text.endswith("/") + if directory_only: + pattern_text = pattern_text.rstrip("/") + + if not pattern_text: + continue + + # Treat patterns containing a path separator as anchored to the root + anchored = "/" in pattern_text + + compiled.append( + IgnorePattern( + pattern=PurePosixPath(pattern_text).as_posix(), + negated=negated, + directory_only=directory_only, + anchored=anchored, + ) + ) + + return compiled + + +def _segment_match(pattern_segment: str, path_segment: str) -> bool: + """Match a single path segment against a glob pattern segment. + + Supports: + + - ``*``: any sequence of characters except ``/``. + - ``?``: any single character except ``/``. + - ``[]``: character classes, excluding ``/``. + + :param pattern_segment: Glob-style pattern segment. + :type pattern_segment: str + :param path_segment: Path segment (no ``/``) to match against. + :type path_segment: str + :return: ``True`` if the path segment matches the pattern segment. + :rtype: bool + """ + + import re + + escaped = "" + i = 0 + while i < len(pattern_segment): + ch = pattern_segment[i] + if ch == "*": + escaped += "[^/]*" + elif ch == "?": + escaped += "[^/]" + elif ch == "[": + # Copy character class as-is until closing ']'. + j = i + 1 + while j < len(pattern_segment) and pattern_segment[j] != "]": + j += 1 + if j < len(pattern_segment): + escaped += pattern_segment[i : j + 1] + i = j + else: + # Unterminated '['; treat it literally. + escaped += re.escape(ch) + else: + escaped += re.escape(ch) + i += 1 + + regex = re.compile(rf"^{escaped}$") + return regex.match(path_segment) is not None + + +def _match_parts_recursive(pattern_parts: list[str], path_parts: list[str]) -> bool: + """Recursive helper implementing ``**`` segment semantics. + + :param pattern_parts: Pattern split into POSIX path segments. + :type pattern_parts: list[str] + :param path_parts: Path split into POSIX path segments. + :type path_parts: list[str] + :return: ``True`` if the pattern parts match the path parts. + :rtype: bool + """ + + if not pattern_parts: + return not path_parts + + if pattern_parts[0] == "**": + # '**' matches zero or more segments. + for i in range(len(path_parts) + 1): + if _match_parts_recursive(pattern_parts[1:], path_parts[i:]): + return True + return False + + if not path_parts: + return False + + if not _segment_match(pattern_parts[0], path_parts[0]): + return False + + return _match_parts_recursive(pattern_parts[1:], path_parts[1:]) + + +def path_match(pattern: IgnorePattern, relpath: str, *, is_dir: bool) -> bool: + """Return ``True`` if ``relpath`` matches a compiled ignore pattern. + + :param pattern: Compiled ignore pattern to test. + :type pattern: IgnorePattern + :param relpath: Path to test, relative to the ignore root. + :type relpath: str + :param is_dir: Whether ``relpath`` refers to a directory. + :type is_dir: bool + :return: ``True`` if the path is matched by the pattern. + :rtype: bool + """ + + relpath_posix = PurePosixPath(relpath).as_posix() + path_parts = PurePosixPath(relpath_posix).parts + pattern_parts = PurePosixPath(pattern.pattern).parts + + # Directory-only patterns never directly match files here; the effect on + # descendants is enforced by directory pruning in the traversal. + if pattern.directory_only and not is_dir: + return False + + if pattern.anchored: + return _match_parts_recursive(list(pattern_parts), list(path_parts)) + + for start in range(len(path_parts)): + if _match_parts_recursive(list(pattern_parts), list(path_parts[start:])): + return True + return False + + +def is_ignored(relpath: str, *, is_dir: bool, patterns: Sequence[IgnorePattern]) -> bool: + """Apply ignore patterns with 'last match wins' semantics. + + Examples:: + + *.log + !important.log + + excludes all ``.log`` files except ``important.log``. Patterns are applied + in order, and the last matching pattern determines inclusion. + + :param relpath: Path to evaluate, relative to the ignore root. + :type relpath: str + :param is_dir: Whether ``relpath`` refers to a directory. + :type is_dir: bool + :param patterns: Compiled ignore patterns to apply in order. + :type patterns: Sequence[IgnorePattern] + :return: ``True`` if the path should be treated as ignored. + :rtype: bool + """ + + included = True # include by default + for pat in patterns: + if path_match(pat, relpath, is_dir=is_dir): + included = pat.negated + return not included + + +def iter_included_files( + root: Path, + *, + patterns: Sequence[IgnorePattern], +) -> Iterable[Path]: + """Yield all files under ``root`` that are not ignored. + + This performs directory pruning so that ignored directories are never + traversed, mirroring Docker's behavior for ``.dockerignore``. + + :param root: Root directory to walk. + :type root: Path + :param patterns: Compiled ignore patterns controlling which files and + directories are included. + :type patterns: Sequence[IgnorePattern] + :return: Iterator over non-ignored file paths under ``root``. + :rtype: Iterable[Path] + """ + + if not root.is_dir(): + raise ValueError(f"root must be a directory, got: {root}") + + for dirpath, dirs, files in os.walk(root): + dir_path = Path(dirpath) + + # Prune ignored directories + for name in list(dirs): + subdir = dir_path / name + rel_dir = subdir.relative_to(root).as_posix() + if is_ignored(rel_dir, is_dir=True, patterns=patterns): + dirs.remove(name) + + # Yield non-ignored files + for name in files: + file_path = dir_path / name + rel_file = file_path.relative_to(root).as_posix() + if is_ignored(rel_file, is_dir=False, patterns=patterns): + continue + yield file_path + + +class IgnoreMatcher(ABC): + """Abstract interface for ignore matchers like .dockerignore and .gitignore. + + There is considerable variation for each ignore file format, so this interface + provides a minimal contract for supporting each format. Implementations are + responsible for interpreting any underlying ignore configuration (files, inline + patterns, etc.) and returning all files that should be included under a given + root directory. + """ + + @abstractmethod + def iter_paths(self, root: Path) -> Iterable[Path]: + """Yield filesystem paths to include under ``root``. + + :param root: Root directory to scan for files. + :type root: Path + :return: Iterator over filesystem paths that should be included. + :rtype: Iterable[Path] + """ + + +@dataclass(frozen=True) +class DockerIgnoreMatcher(IgnoreMatcher): + """Ignore matcher that mirrors Docker's .dockerignore semantics. + + This matcher: + + - Closely follows Docker's ``.dockerignore`` semantics. + - Always loads patterns from ``.dockerignore`` in the provided context + root, if present. + - Optionally loads additional patterns from an extra ignorefile. + - Optionally appends inline pattern strings. + + Note: Patterns follow Docker-style semantics (``!`` negation, ``**`` support). + """ + + extra_ignorefile: str | Path | None = None + """Optional path to an additional ignorefile whose patterns are appended + after the default ``.dockerignore``. + """ + + patterns: Sequence[str] | None = None + """Optional inline pattern strings appended after any ignorefiles.""" + + @override + def iter_paths(self, root: Path) -> Iterable[Path]: + """Yield non-ignored files under ``root`` honoring Docker-style patterns. + + :param root: Context directory whose contents should be filtered. + :type root: Path + :return: Iterator over non-ignored file paths under ``root``. + :rtype: Iterable[Path] + """ + root = root.resolve() + + all_patterns: list[str] = [] + + # 1) Always consider .dockerignore under the context root, if present. + default_ignorefile = root / ".dockerignore" + all_patterns.extend(read_ignorefile(default_ignorefile)) + + # 2) Optional additional ignorefile. + if self.extra_ignorefile is not None: + ignore_path = Path(self.extra_ignorefile) + if not ignore_path.exists(): + raise FileNotFoundError(f"Ignore file does not exist: {ignore_path}") + all_patterns.extend(read_ignorefile(ignore_path)) + + # 3) Optional inline patterns appended last using same rules as .dockerignore + # Some extra handling here for trailing slashes that is different from .gitignore. + if self.patterns: + for raw in self.patterns: + if not raw: + continue + normalized = _normalize_pattern_string(raw) + if normalized: + all_patterns.append(normalized) + + compiled: list[IgnorePattern] = compile_ignore(all_patterns) + return iter_included_files(root, patterns=compiled) + + +@dataclass(frozen=True) +class FilePatternMatcher(IgnoreMatcher): + """Ignore matcher that applies only inline patterns, without .dockerignore. + + Patterns follow the same semantics as :func:`compile_ignore` / Docker-style + ignore files and are treated as *ignore* rules (``!`` negation for + re-inclusion, ``**`` support, etc.). + + The constructor accepts either a single pattern string or a sequence of + pattern strings; a single string is automatically wrapped into a list. + """ + + patterns: Sequence[str] | str + """Pattern or patterns to apply as ignore rules when matching files.""" + + def __post_init__(self) -> None: + # Normalise a single pattern string into a list for downstream helpers. + if isinstance(self.patterns, str): + object.__setattr__(self, "patterns", [self.patterns]) + + @override + def iter_paths(self, root: Path) -> Iterable[Path]: + """Yield non-ignored files under ``root`` based only on ``patterns``. + + :param root: Root directory whose contents should be filtered. + :type root: Path + :return: Iterator over non-ignored file paths under ``root``. + :rtype: Iterable[Path] + """ + + root = root.resolve() + compiled: list[IgnorePattern] = compile_ignore(self.patterns) # type: ignore[arg-type] + return iter_included_files(root, patterns=compiled) diff --git a/src/runloop_api_client/lib/context_loader.py b/src/runloop_api_client/lib/context_loader.py new file mode 100644 index 000000000..bfa433a89 --- /dev/null +++ b/src/runloop_api_client/lib/context_loader.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import io +import tarfile +from typing import Callable, Optional, Sequence +from pathlib import Path + +from ._ignore import IgnoreMatcher, DockerIgnoreMatcher + +TarFilter = Callable[[tarfile.TarInfo], Optional[tarfile.TarInfo]] + + +def build_docker_context_tar( + context_root: Path, + *, + ignore: IgnoreMatcher | Sequence[str] | None = None, +) -> bytes: + """Create a .tar.gz of the build context, honoring Docker-style ignore patterns. + + - Treats ``context_root`` as the build context root. + - Always loads ``.dockerignore`` under ``context_root`` if present. + - An optional :class:`IgnoreMatcher` may be provided to customise how ignore + patterns are resolved; when omitted, :class:`DockerIgnoreMatcher` is used. + """ + + context_root = context_root.resolve() + + if ignore is None: + matcher: IgnoreMatcher = DockerIgnoreMatcher() + elif isinstance(ignore, IgnoreMatcher): + matcher = ignore + else: + # Treat sequences of pattern strings as additional inline patterns + # appended after ``.dockerignore`` (if present), mirroring + # :class:`DockerIgnoreMatcher` semantics. + matcher = DockerIgnoreMatcher(patterns=list(ignore)) + + buf = io.BytesIO() + + with tarfile.open(mode="w:gz", fileobj=buf) as tf: + for path in matcher.iter_paths(context_root): + rel = path.relative_to(context_root) + tf.add(path, arcname=rel.as_posix()) + + return buf.getvalue() + + +def build_directory_tar( + root: Path, + *, + tar_filter: TarFilter | None = None, +) -> bytes: + """Create a .tar.gz archive containing all files under ``root``. + + No ignore semantics are applied by default; callers may pass a tar filter + compatible with :meth:`tarfile.TarFile.add` to modify or exclude members. + """ + + root = root.resolve() + buf = io.BytesIO() + + def _wrapped_filter(ti: tarfile.TarInfo) -> Optional[tarfile.TarInfo]: + # Normalise member names so callers see paths relative to ``root`` + # without a leading ``./``, preserving existing TarFilter semantics and + # archive layout. This applies to both files and directories. + if ti.name.startswith("./"): + ti.name = ti.name[2:] + + if tar_filter is not None: + return tar_filter(ti) + return ti + + with tarfile.open(mode="w:gz", fileobj=buf) as tf: + # Add the root directory recursively in one call, delegating member + # handling to the wrapped filter above. + tf.add(root, arcname=".", filter=_wrapped_filter) + + return buf.getvalue() diff --git a/src/runloop_api_client/sdk/async_.py b/src/runloop_api_client/sdk/async_.py index 0825c6b46..16955642c 100644 --- a/src/runloop_api_client/sdk/async_.py +++ b/src/runloop_api_client/sdk/async_.py @@ -2,9 +2,7 @@ from __future__ import annotations -import io import asyncio -import tarfile from typing import Dict, Mapping, Optional from pathlib import Path from datetime import timedelta @@ -35,6 +33,7 @@ from .async_scorer import AsyncScorer from .async_snapshot import AsyncSnapshot from .async_blueprint import AsyncBlueprint +from ..lib.context_loader import TarFilter, build_directory_tar from .async_storage_object import AsyncStorageObject from ..types.object_create_params import ContentType @@ -399,6 +398,7 @@ async def upload_from_dir( name: Optional[str] = None, metadata: Optional[Dict[str, str]] = None, ttl: Optional[timedelta] = None, + ignore: TarFilter | None = None, **options: Unpack[LongRequestOptions], ) -> AsyncStorageObject: """Create and upload an object from a local directory. @@ -413,21 +413,26 @@ async def upload_from_dir( :type metadata: Optional[Dict[str, str]] :param ttl: Optional Time-To-Live, after which the object is automatically deleted :type ttl: Optional[timedelta] - :param options: See :typeddict:`~runloop_api_client.sdk._types.LongRequestOptions` for available options + :param ignore: Optional tar filter function compatible with + :meth:`tarfile.TarFile.add`. If provided, it will be called for each + member to allow modification or exclusion (by returning ``None``). + :type ignore: Optional[TarFilter] + :param options: See :typeddict:`~runloop_api_client.sdk._types.LongRequestOptions` + for available options :return: Wrapper for the uploaded object :rtype: AsyncStorageObject - :raises OSError: If the local file cannot be read + :raises OSError: If the local directory cannot be read + :raises ValueError: If ``dir_path`` does not point to a directory """ path = Path(dir_path) + if not path.is_dir(): + raise ValueError(f"dir_path must be a directory, got: {path}") + name = name or f"{path.name}.tar.gz" ttl_ms = int(ttl.total_seconds()) * 1000 if ttl else None def synchronous_io() -> bytes: - with io.BytesIO() as tar_buffer: - with tarfile.open(fileobj=tar_buffer, mode="w:gz") as tar: - tar.add(path, arcname=".", recursive=True) - tar_buffer.seek(0) - return tar_buffer.read() + return build_directory_tar(path, tar_filter=ignore) tar_bytes = await asyncio.to_thread(synchronous_io) diff --git a/src/runloop_api_client/sdk/async_storage_object.py b/src/runloop_api_client/sdk/async_storage_object.py index 5a3e9cb9e..377cc4ec0 100644 --- a/src/runloop_api_client/sdk/async_storage_object.py +++ b/src/runloop_api_client/sdk/async_storage_object.py @@ -8,6 +8,7 @@ from ._types import BaseRequestOptions, LongRequestOptions, SDKObjectDownloadParams from .._client import AsyncRunloop from ..types.object_view import ObjectView +from ..types.blueprint_create_params import BuildContext from ..types.object_download_url_view import ObjectDownloadURLView @@ -150,8 +151,10 @@ async def delete( async def upload_content(self, content: str | bytes | Iterable[bytes]) -> None: """Upload content to the object's pre-signed URL. - :param content: Bytes or text payload to upload - :type content: str | bytes + :param content: Bytes payload, text payload, or an iterable streaming bytes + :type content: str | bytes | Iterable[bytes] + :return: None + :rtype: None :raises RuntimeError: If no upload URL is available :raises httpx.HTTPStatusError: Propagated from the underlying ``httpx`` client when the upload fails """ @@ -159,6 +162,20 @@ async def upload_content(self, content: str | bytes | Iterable[bytes]) -> None: response = await self._client._client.put(url, content=content) response.raise_for_status() + def as_build_context(self) -> BuildContext: + """Return this object in the shape expected for a Blueprint build context. + + The returned mapping can be passed directly to ``build_context`` or + ``named_build_contexts`` when creating a blueprint. + + :return: Mapping suitable for use as a blueprint build context + :rtype: BuildContext + """ + return { + "object_id": self._id, + "type": "object", + } + def _ensure_upload_url(self) -> str: """Return the upload URL, ensuring it exists. diff --git a/src/runloop_api_client/sdk/storage_object.py b/src/runloop_api_client/sdk/storage_object.py index 8eb4898b8..28fad144d 100644 --- a/src/runloop_api_client/sdk/storage_object.py +++ b/src/runloop_api_client/sdk/storage_object.py @@ -8,6 +8,7 @@ from ._types import BaseRequestOptions, LongRequestOptions, SDKObjectDownloadParams from .._client import Runloop from ..types.object_view import ObjectView +from ..types.blueprint_create_params import BuildContext from ..types.object_download_url_view import ObjectDownloadURLView @@ -150,8 +151,10 @@ def delete( def upload_content(self, content: str | bytes | Iterable[bytes]) -> None: """Upload content to the object's pre-signed URL. - :param content: Bytes or text payload to upload - :type content: str | bytes + :param content: Bytes payload, text payload, or an iterable streaming bytes + :type content: str | bytes | Iterable[bytes] + :return: None + :rtype: None :raises RuntimeError: If no upload URL is available :raises httpx.HTTPStatusError: Propagated from the underlying ``httpx`` client when the upload fails """ @@ -159,6 +162,20 @@ def upload_content(self, content: str | bytes | Iterable[bytes]) -> None: response = self._client._client.put(url, content=content) response.raise_for_status() + def as_build_context(self) -> BuildContext: + """Return this object in the shape expected for a Blueprint build context. + + The returned mapping can be passed directly to ``build_context`` or + ``named_build_contexts`` when creating a blueprint. + + :return: Mapping suitable for use as a blueprint build context + :rtype: BuildContext + """ + return { + "object_id": self._id, + "type": "object", + } + def _ensure_upload_url(self) -> str: """Return the upload URL, ensuring it is present. diff --git a/src/runloop_api_client/sdk/sync.py b/src/runloop_api_client/sdk/sync.py index e4fad064c..c25c10cb5 100644 --- a/src/runloop_api_client/sdk/sync.py +++ b/src/runloop_api_client/sdk/sync.py @@ -2,8 +2,6 @@ from __future__ import annotations -import io -import tarfile from typing import Dict, Mapping, Optional from pathlib import Path from datetime import timedelta @@ -35,6 +33,7 @@ from .snapshot import Snapshot from .blueprint import Blueprint from .storage_object import StorageObject +from ..lib.context_loader import TarFilter, build_directory_tar from ..types.object_create_params import ContentType @@ -84,7 +83,6 @@ def create_from_blueprint_id( :param blueprint_id: Blueprint ID to create from :type blueprint_id: str :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKDevboxCreateFromImageParams` for available parameters - :type params: :return: Wrapper bound to the newly created devbox :rtype: Devbox """ @@ -227,11 +225,10 @@ class BlueprintOps: Example: >>> from datetime import timedelta >>> from runloop_api_client.types.blueprint_build_parameters import BuildContext - >>> >>> runloop = RunloopSDK() >>> obj = runloop.object_storage.upload_from_dir( ... "./", - ... ttl=timedelta(hours=1), + ... ttl=timedelta(hours=1), ... ) >>> blueprint = runloop.blueprint.create( ... name="my-blueprint", @@ -398,6 +395,7 @@ def upload_from_dir( name: Optional[str] = None, metadata: Optional[Dict[str, str]] = None, ttl: Optional[timedelta] = None, + ignore: TarFilter | None = None, **options: Unpack[LongRequestOptions], ) -> StorageObject: """Create and upload an object from a local directory. @@ -412,22 +410,28 @@ def upload_from_dir( :type metadata: Optional[Dict[str, str]] :param ttl: Optional Time-To-Live, after which the object is automatically deleted :type ttl: Optional[timedelta] - :param options: See :typeddict:`~runloop_api_client.sdk._types.LongRequestOptions` for available options + :param ignore: Optional tar filter function compatible with + :meth:`tarfile.TarFile.add`. If provided, it will be called for each + member to allow modification or exclusion (by returning ``None``). + :type ignore: Optional[TarFilter] + :param options: See :typeddict:`~runloop_api_client.sdk._types.LongRequestOptions` + for available options :return: Wrapper for the uploaded object :rtype: StorageObject :raises OSError: If the local file cannot be read + :raises ValueError: If ``dir_path`` does not point to a directory """ path = Path(dir_path) + if not path.is_dir(): + raise ValueError(f"dir_path must be a directory, got: {path}") + name = name or f"{path.name}.tar.gz" ttl_ms = int(ttl.total_seconds()) * 1000 if ttl else None - tar_buffer = io.BytesIO() - with tarfile.open(fileobj=tar_buffer, mode="w:gz") as tar: - tar.add(path, arcname=".", recursive=True) - tar_buffer.seek(0) + tar_bytes = build_directory_tar(path, tar_filter=ignore) obj = self.create(name=name, content_type="tgz", metadata=metadata, ttl_ms=ttl_ms, **options) - obj.upload_content(tar_buffer) + obj.upload_content(tar_bytes) obj.complete() return obj @@ -540,7 +544,7 @@ def list(self, **params: Unpack[SDKScorerListParams]) -> list[Scorer]: page = self._client.scenarios.scorers.list(**params) return [Scorer(self._client, item.id) for item in page] - + class AgentOps: """High-level manager for creating and managing agents. @@ -550,15 +554,10 @@ class AgentOps: Example: >>> runloop = RunloopSDK() >>> # Create agent from NPM package - >>> agent = runloop.agent.create_from_npm( - ... name="my-agent", - ... package_name="@runloop/example-agent" - ... ) + >>> agent = runloop.agent.create_from_npm(name="my-agent", package_name="@runloop/example-agent") >>> # Create agent from Git repository >>> agent = runloop.agent.create_from_git( - ... name="git-agent", - ... repository="https://github.com/user/agent-repo", - ... ref="main" + ... name="git-agent", repository="https://github.com/user/agent-repo", ref="main" ... ) >>> # List all agents >>> agents = runloop.agent.list(limit=10) @@ -600,9 +599,7 @@ def create_from_npm( Example: >>> agent = runloop.agent.create_from_npm( - ... name="my-npm-agent", - ... package_name="@runloop/example-agent", - ... npm_version="^1.0.0" + ... name="my-npm-agent", package_name="@runloop/example-agent", npm_version="^1.0.0" ... ) :param package_name: NPM package name @@ -619,7 +616,9 @@ def create_from_npm( :raises ValueError: If 'source' is provided in params """ if "source" in params: - raise ValueError("Cannot specify 'source' when using create_from_npm(); source is automatically set to npm configuration") + raise ValueError( + "Cannot specify 'source' when using create_from_npm(); source is automatically set to npm configuration" + ) npm_config: dict = {"package_name": package_name} if npm_version is not None: @@ -647,9 +646,7 @@ def create_from_pip( Example: >>> agent = runloop.agent.create_from_pip( - ... name="my-pip-agent", - ... package_name="runloop-example-agent", - ... pip_version=">=1.0.0" + ... name="my-pip-agent", package_name="runloop-example-agent", pip_version=">=1.0.0" ... ) :param package_name: Pip package name @@ -666,7 +663,9 @@ def create_from_pip( :raises ValueError: If 'source' is provided in params """ if "source" in params: - raise ValueError("Cannot specify 'source' when using create_from_pip(); source is automatically set to pip configuration") + raise ValueError( + "Cannot specify 'source' when using create_from_pip(); source is automatically set to pip configuration" + ) pip_config: dict = {"package_name": package_name} if pip_version is not None: @@ -696,7 +695,7 @@ def create_from_git( ... name="my-git-agent", ... repository="https://github.com/user/agent-repo", ... ref="main", - ... agent_setup=["npm install", "npm run build"] + ... agent_setup=["npm install", "npm run build"], ... ) :param repository: Git repository URL @@ -711,7 +710,9 @@ def create_from_git( :raises ValueError: If 'source' is provided in params """ if "source" in params: - raise ValueError("Cannot specify 'source' when using create_from_git(); source is automatically set to git configuration") + raise ValueError( + "Cannot specify 'source' when using create_from_git(); source is automatically set to git configuration" + ) git_config: dict = {"repository": repository} if ref is not None: @@ -738,9 +739,7 @@ def create_from_object( >>> obj = runloop.storage_object.upload_from_dir("./my-agent") >>> # Then create agent from the object >>> agent = runloop.agent.create_from_object( - ... name="my-object-agent", - ... object_id=obj.id, - ... agent_setup=["chmod +x setup.sh", "./setup.sh"] + ... name="my-object-agent", object_id=obj.id, agent_setup=["chmod +x setup.sh", "./setup.sh"] ... ) :param object_id: Storage object ID @@ -753,7 +752,9 @@ def create_from_object( :raises ValueError: If 'source' is provided in params """ if "source" in params: - raise ValueError("Cannot specify 'source' when using create_from_object(); source is automatically set to object configuration") + raise ValueError( + "Cannot specify 'source' when using create_from_object(); source is automatically set to object configuration" + ) object_config: dict = {"object_id": object_id} if agent_setup is not None: diff --git a/tests/sdk/test_async_ops.py b/tests/sdk/test_async_ops.py index 8a455cd68..8e0e6724b 100644 --- a/tests/sdk/test_async_ops.py +++ b/tests/sdk/test_async_ops.py @@ -474,6 +474,15 @@ async def test_upload_from_file_missing_path(self, mock_async_client: AsyncMock, with pytest.raises(OSError, match="Failed to read file"): await ops.upload_from_file(missing_file) + def test_as_build_context(self, mock_async_client: AsyncMock, object_view: MockObjectView) -> None: + """as_build_context should return the correct dict shape.""" + obj = AsyncStorageObject(mock_async_client, object_view.id, upload_url=None) + + assert obj.as_build_context() == { + "object_id": object_view.id, + "type": "object", + } + @pytest.mark.asyncio async def test_upload_from_dir( self, mock_async_client: AsyncMock, object_view: MockObjectView, tmp_path: Path @@ -524,6 +533,47 @@ async def test_upload_from_dir( mock_async_client.objects.complete.assert_awaited_once() + @pytest.mark.asyncio + async def test_upload_from_dir_with_inline_ignore_patterns( + self, mock_async_client: AsyncMock, object_view: MockObjectView, tmp_path: Path + ) -> None: + """upload_from_dir should respect inline ignore patterns.""" + mock_async_client.objects.create = AsyncMock(return_value=object_view) + mock_async_client.objects.complete = AsyncMock(return_value=object_view) + + test_dir = tmp_path / "ctx" + test_dir.mkdir() + (test_dir / "keep.txt").write_text("keep", encoding="utf-8") + (test_dir / "ignore.log").write_text("ignore", encoding="utf-8") + build_dir = test_dir / "build" + build_dir.mkdir() + (build_dir / "ignored.txt").write_text("ignored", encoding="utf-8") + + http_client = AsyncMock() + mock_response = create_mock_httpx_response() + http_client.put = AsyncMock(return_value=mock_response) + mock_async_client._client = http_client + + client = AsyncStorageObjectOps(mock_async_client) + + # Tar filter: drop logs and anything under build/ + def ignore_logs_and_build(ti: tarfile.TarInfo) -> tarfile.TarInfo | None: + if ti.name.endswith(".log") or ti.name.startswith("build/"): + return None + return ti + + obj = await client.upload_from_dir(test_dir, ignore=ignore_logs_and_build) + + assert isinstance(obj, AsyncStorageObject) + uploaded_content = http_client.put.call_args[1]["content"] + + with tarfile.open(fileobj=io.BytesIO(uploaded_content), mode="r:gz") as tar: + names = {m.name for m in tar.getmembers()} + + assert "keep.txt" in names + assert "ignore.log" not in names + assert not any(name.startswith("build/") for name in names) + @pytest.mark.asyncio async def test_upload_from_dir_default_name( self, mock_async_client: AsyncMock, object_view: MockObjectView, tmp_path: Path diff --git a/tests/sdk/test_ops.py b/tests/sdk/test_ops.py index cf3a7216d..bc6f1f445 100644 --- a/tests/sdk/test_ops.py +++ b/tests/sdk/test_ops.py @@ -2,6 +2,8 @@ from __future__ import annotations +import io +import tarfile from types import SimpleNamespace from pathlib import Path from unittest.mock import Mock @@ -440,6 +442,15 @@ def test_upload_from_file_missing_path(self, mock_client: Mock, tmp_path: Path) with pytest.raises(OSError, match="Failed to read file"): ops.upload_from_file(missing_file) + def test_as_build_context(self, mock_client: Mock, object_view: MockObjectView) -> None: + """as_build_context should return the correct dict shape.""" + obj = StorageObject(mock_client, object_view.id, upload_url=None) + + assert obj.as_build_context() == { + "object_id": object_view.id, + "type": "object", + } + def test_upload_from_dir(self, mock_client: Mock, object_view: MockObjectView, tmp_path: Path) -> None: """Test upload_from_dir method.""" mock_client.objects.create.return_value = object_view @@ -473,9 +484,9 @@ def test_upload_from_dir(self, mock_client: Mock, object_view: MockObjectView, t http_client.put.assert_called_once() call_args = http_client.put.call_args assert call_args[0][0] == object_view.upload_url - # Verify it's a BytesIO object uploaded_content = call_args[1]["content"] - assert hasattr(uploaded_content, "read") + # Verify it is bytes representing a gzipped tar archive + assert isinstance(uploaded_content, (bytes, bytearray)) mock_client.objects.complete.assert_called_once() def test_upload_from_dir_default_name(self, mock_client: Mock, object_view: MockObjectView, tmp_path: Path) -> None: @@ -587,6 +598,45 @@ def test_upload_from_dir_with_string_path( http_client.put.assert_called_once() mock_client.objects.complete.assert_called_once() + def test_upload_from_dir_respects_filter( + self, mock_client: Mock, object_view: MockObjectView, tmp_path: Path + ) -> None: + """upload_from_dir should respect a tar filter when provided.""" + mock_client.objects.create.return_value = object_view + + test_dir = tmp_path / "ctx" + test_dir.mkdir() + (test_dir / "keep.txt").write_text("keep", encoding="utf-8") + (test_dir / "ignore.log").write_text("ignore", encoding="utf-8") + build_dir = test_dir / "build" + build_dir.mkdir() + (build_dir / "ignored.txt").write_text("ignored", encoding="utf-8") + + http_client = Mock() + mock_response = create_mock_httpx_response() + http_client.put.return_value = mock_response + mock_client._client = http_client + + client = StorageObjectOps(mock_client) + + # Tar filter: drop logs and anything under build/ + def ignore_logs_and_build(ti: tarfile.TarInfo) -> tarfile.TarInfo | None: + if ti.name.endswith(".log") or ti.name.startswith("build/"): + return None + return ti + + obj = client.upload_from_dir(test_dir, ignore=ignore_logs_and_build) + + assert isinstance(obj, StorageObject) + uploaded_content = http_client.put.call_args[1]["content"] + + with tarfile.open(fileobj=io.BytesIO(uploaded_content), mode="r:gz") as tar: + names = {m.name for m in tar.getmembers()} + + assert "keep.txt" in names + assert "ignore.log" not in names + assert not any(name.startswith("build/") for name in names) + class TestScorerOps: """Tests for ScorerOps class.""" diff --git a/tests/test_utils/test_context_loader.py b/tests/test_utils/test_context_loader.py new file mode 100644 index 000000000..0a0bb7cc8 --- /dev/null +++ b/tests/test_utils/test_context_loader.py @@ -0,0 +1,207 @@ +import io +import tarfile +from pathlib import Path + +from runloop_api_client.lib._ignore import ( + IgnorePattern, + FilePatternMatcher, + is_ignored, + path_match, + compile_ignore, + read_ignorefile, + iter_included_files, +) +from runloop_api_client.lib.context_loader import build_docker_context_tar + + +def test_segment_match_basic_globs(): + patterns = compile_ignore(["*.log", "foo?", "[ab].txt"]) + pat_glob, pat_q, pat_class = patterns + + assert path_match(pat_glob, "app.log", is_dir=False) + assert not path_match(pat_glob, "app.txt", is_dir=False) + assert path_match(pat_q, "fooa", is_dir=False) + assert not path_match(pat_q, "fooba", is_dir=False) + assert path_match(pat_class, "a.txt", is_dir=False) + assert not path_match(pat_class, "c.txt", is_dir=False) + + +def test_path_match_anchored_and_unanchored(): + pat = IgnorePattern(pattern="foo/bar.txt", negated=False, directory_only=False, anchored=True) + assert path_match(pat, "foo/bar.txt", is_dir=False) + assert not path_match(pat, "a/foo/bar.txt", is_dir=False) + + pat_unanchored = IgnorePattern(pattern="foo/bar.txt", negated=False, directory_only=False, anchored=False) + assert path_match(pat_unanchored, "a/foo/bar.txt", is_dir=False) + assert path_match(pat_unanchored, "foo/bar.txt", is_dir=False) + + +def test_path_match_double_star(): + pat = IgnorePattern(pattern="**/*.log", negated=False, directory_only=False, anchored=False) + assert path_match(pat, "app.log", is_dir=False) + assert path_match(pat, "a/b/app.log", is_dir=False) + assert not path_match(pat, "a/b/app.txt", is_dir=False) + + +def test_is_ignored_last_match_wins(): + patterns = compile_ignore(["*.log", "!keep.log"]) + assert is_ignored("foo.log", is_dir=False, patterns=patterns) + assert not is_ignored("keep.log", is_dir=False, patterns=patterns) + + +def test_read_ignorefile_basic(tmp_path: Path): + dockerignore = tmp_path / ".dockerignore" + dockerignore.write_bytes(b"\xef\xbb\xbf# comment line\n*.log \n!keep.log\nbuild/\n") + + patterns = read_ignorefile(dockerignore) + assert patterns == ["*.log", "!keep.log", "build"] + + +def test_iter_build_context_files_respects_dockerignore(tmp_path: Path): + # Layout: + # foo.txt + # app.log + # build/ignored.txt + root = tmp_path + (root / "foo.txt").write_text("ok", encoding="utf-8") + (root / "app.log").write_text("ignored", encoding="utf-8") + build_dir = root / "build" + build_dir.mkdir() + (build_dir / "ignored.txt").write_text("ignored", encoding="utf-8") + + dockerignore = root / ".dockerignore" + dockerignore.write_text("*.log\nbuild/\n", encoding="utf-8") + + compiled = compile_ignore(read_ignorefile(dockerignore)) + files = {p.relative_to(root).as_posix() for p in iter_included_files(root, patterns=compiled)} + assert "foo.txt" in files + assert "app.log" not in files + assert "build/ignored.txt" not in files + + +def test_optional_patterns_trailing_slash_matches_file_in_context(tmp_path: Path) -> None: + """Inline ignore patterns should mirror .dockerignore trailing-slash behavior. + + Patterns like '/foo/bar/' provided via the optional ``ignore=`` parameter + should exclude a file at 'foo/bar' in the build context, matching Docker's + patternmatcher semantics where trailing slashes are not directory-only. + """ + + root = tmp_path + foo = root / "foo" + foo.mkdir() + (foo / "bar").write_text("ignored", encoding="utf-8") + (root / "keep.txt").write_text("keep", encoding="utf-8") + + # Use build_docker_context_tar with an inline pattern that includes a + # trailing slash; this should still exclude the file at 'foo/bar'. + tar_bytes = build_docker_context_tar(root, ignore=["/foo/bar/"]) + + with tarfile.open(fileobj=io.BytesIO(tar_bytes), mode="r:gz") as tf: + names = {m.name for m in tf.getmembers()} + + assert "keep.txt" in names + assert "foo/bar" not in names + + +def test_is_ignored_directory_pattern_affects_directory_entry_only() -> None: + """Directory patterns apply directly to directory entries, not to children.""" + + patterns = compile_ignore(["docs", "!docs/README.md"]) + + # The directory itself is ignored. + assert is_ignored("docs", is_dir=True, patterns=patterns) + # The child file is not ignored by the pattern set alone; directory pruning + # in ``iter_included_files`` is responsible for excluding its contents. + assert not is_ignored("docs/README.md", is_dir=False, patterns=patterns) + + +def test_compile_ignore_directory_only_and_files() -> None: + patterns = compile_ignore(["build/", "*.log"]) + + build_pat, log_pat = patterns + assert build_pat.directory_only + assert not log_pat.directory_only + + # Directory-only pattern does not directly match files at that path. + assert not path_match(build_pat, "build", is_dir=False) + assert path_match(build_pat, "build", is_dir=True) + + # Files under the directory are not ignored purely by the directory-only + # pattern; directory pruning in ``iter_included_files`` is responsible + # for skipping their traversal. + assert not is_ignored("build/output.bin", is_dir=False, patterns=patterns) + # Log files are ignored everywhere. + assert is_ignored("app.log", is_dir=False, patterns=patterns) + assert is_ignored("subdir/app.log", is_dir=False, patterns=patterns) + + +def test_double_star_matching_variants() -> None: + patterns = compile_ignore(["**", "dir/**", "**/file", "**/*.txt"]) + + any_pat, dir_pat, file_pat, txt_pat = patterns + + # '**' matches everything. + assert path_match(any_pat, "file", is_dir=False) + assert path_match(any_pat, "dir/file", is_dir=False) + + # 'dir/**' matches anything under dir. + assert path_match(dir_pat, "dir/file", is_dir=False) + assert path_match(dir_pat, "dir/sub/file", is_dir=False) + assert not path_match(dir_pat, "other/file", is_dir=False) + + # '**/file' matches at any depth. + assert path_match(file_pat, "file", is_dir=False) + assert path_match(file_pat, "dir/file", is_dir=False) + assert path_match(file_pat, "a/b/file", is_dir=False) + + # '**/*.txt' matches text files at any depth. + assert path_match(txt_pat, "file.txt", is_dir=False) + assert path_match(txt_pat, "dir/file.txt", is_dir=False) + assert not path_match(txt_pat, "dir/file.log", is_dir=False) + + +def test_iter_build_context_files_respects_directory_pruning(tmp_path: Path) -> None: + """Directories excluded by patterns are not traversed, even with negation.""" + + root = tmp_path + docs = root / "docs" + docs.mkdir() + (docs / "README.md").write_text("keep?", encoding="utf-8") + + ignorefile = root / ".dockerignore" + # Attempt to re-include a file under an ignored directory. + ignorefile.write_text("docs/\n!docs/README.md\n", encoding="utf-8") + + +def test_build_docker_context_tar_supports_pattern_list(tmp_path: Path) -> None: + """build_docker_context_tar should accept a sequence of ignore patterns.""" + + root = tmp_path + (root / "keep.txt").write_text("keep", encoding="utf-8") + (root / "env.venv").write_text("ignored", encoding="utf-8") + + tar_bytes = build_docker_context_tar(root, ignore=["*.venv"]) + + with tarfile.open(fileobj=io.BytesIO(tar_bytes), mode="r:gz") as tf: + names = {m.name for m in tf.getmembers()} + + assert "keep.txt" in names + assert "env.venv" not in names + + +def test_build_docker_context_tar_supports_file_pattern_matcher(tmp_path: Path) -> None: + """build_docker_context_tar should accept a FilePatternMatcher instance.""" + + root = tmp_path + (root / "keep.bin").write_text("keep", encoding="utf-8") + (root / "ignore.txt").write_text("ignored", encoding="utf-8") + + matcher = FilePatternMatcher("**/*.txt") + tar_bytes = build_docker_context_tar(root, ignore=matcher) + + with tarfile.open(fileobj=io.BytesIO(tar_bytes), mode="r:gz") as tf: + names = {m.name for m in tf.getmembers()} + + assert "keep.bin" in names + assert "ignore.txt" not in names