release: 0.24.0

.release-please-manifest.json

@@ -1,3 +1,3 @@
 {
-  ".": "0.23.0"
+  ".": "0.24.0"
 }

.stats.yml

@@ -1,2 +1,2 @@
 configured_endpoints: 77
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-a3d91c690527ff6a9040ade46943ba56916987f1f7d1fb45a9974546770ffe97.yml
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-cb8add05a7b418d6f8a5624be8477564853da49e8bf9671ae89b8ce49a04b6cd.yml

CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## 0.24.0 (2025-02-19)
+
+Full Changelog: [v0.23.0...v0.24.0](https://github.com/runloopai/api-client-python/compare/v0.23.0...v0.24.0)
+
+### Features
+
+* **api:** api update ([#549](https://github.com/runloopai/api-client-python/issues/549)) ([3e19f41](https://github.com/runloopai/api-client-python/commit/3e19f41cf8e2b9cd225439d9df90f80b34d89660))
+
+
+### Bug Fixes
+
+* asyncify on non-asyncio runtimes ([#547](https://github.com/runloopai/api-client-python/issues/547)) ([8ce7003](https://github.com/runloopai/api-client-python/commit/8ce700397fc5d6755f0a95b637efad74d6dc3fe4))
+
+
+### Chores
+
+* **internal:** codegen related update ([#543](https://github.com/runloopai/api-client-python/issues/543)) ([5c44e84](https://github.com/runloopai/api-client-python/commit/5c44e84a8a029b22461df339c280885dc972fa96))
+* **internal:** codegen related update ([#546](https://github.com/runloopai/api-client-python/issues/546)) ([d8b620d](https://github.com/runloopai/api-client-python/commit/d8b620d93c687d37a843262d5ac60fd4a9b815e8))
+* **internal:** update client tests ([#545](https://github.com/runloopai/api-client-python/issues/545)) ([30307f4](https://github.com/runloopai/api-client-python/commit/30307f4b78ba76240793b851dd735a0beb835b75))
+* **internal:** update client tests ([#548](https://github.com/runloopai/api-client-python/issues/548)) ([4e782f1](https://github.com/runloopai/api-client-python/commit/4e782f19bfa7e08ae318eca6f00fb4d5ffe5c9af))
+
 ## 0.23.0 (2025-02-11)
 
 Full Changelog: [v0.22.0...v0.23.0](https://github.com/runloopai/api-client-python/compare/v0.22.0...v0.23.0)

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "runloop_api_client"
-version = "0.23.0"
+version = "0.24.0"
 description = "The official Python library for the runloop API"
 dynamic = ["readme"]
 license = "MIT"

src/runloop_api_client/_utils/_sync.py

@@ -7,16 +7,20 @@
 from typing import Any, TypeVar, Callable, Awaitable
 from typing_extensions import ParamSpec
 
+import anyio
+import sniffio
+import anyio.to_thread
+
 T_Retval = TypeVar("T_Retval")
 T_ParamSpec = ParamSpec("T_ParamSpec")
 
 
 if sys.version_info >= (3, 9):
-    to_thread = asyncio.to_thread
+    _asyncio_to_thread = asyncio.to_thread
 else:
     # backport of https://docs.python.org/3/library/asyncio-task.html#asyncio.to_thread
     # for Python 3.8 support
-    async def to_thread(
+    async def _asyncio_to_thread(
         func: Callable[T_ParamSpec, T_Retval], /, *args: T_ParamSpec.args, **kwargs: T_ParamSpec.kwargs
     ) -> Any:
         """Asynchronously run function *func* in a separate thread.
@@ -34,6 +38,17 @@ async def to_thread(
         return await loop.run_in_executor(None, func_call)
 
 
+async def to_thread(
+    func: Callable[T_ParamSpec, T_Retval], /, *args: T_ParamSpec.args, **kwargs: T_ParamSpec.kwargs
+) -> T_Retval:
+    if sniffio.current_async_library() == "asyncio":
+        return await _asyncio_to_thread(func, *args, **kwargs)
+
+    return await anyio.to_thread.run_sync(
+        functools.partial(func, *args, **kwargs),
+    )
+
+
 # inspired by `asyncer`, https://github.com/tiangolo/asyncer
 def asyncify(function: Callable[T_ParamSpec, T_Retval]) -> Callable[T_ParamSpec, Awaitable[T_Retval]]:
     """

src/runloop_api_client/_version.py

@@ -1,4 +1,4 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 __title__ = "runloop_api_client"
-__version__ = "0.23.0"  # x-release-please-version
+__version__ = "0.24.0"  # x-release-please-version

src/runloop_api_client/resources/benchmarks/benchmarks.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 import httpx
 
@@ -69,6 +69,7 @@ def create(
         self,
         *,
         name: str,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
         scenario_ids: Optional[List[str]] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -84,6 +85,8 @@ def create(
         Args:
           name: The name of the Benchmark.
 
+          metadata: User defined metadata to attach to the benchmark for organization.
+
           scenario_ids: The Scenario IDs that make up the Benchmark.
 
           extra_headers: Send extra headers
@@ -101,6 +104,7 @@ def create(
             body=maybe_transform(
                 {
                     "name": name,
+                    "metadata": metadata,
                     "scenario_ids": scenario_ids,
                 },
                 benchmark_create_params.BenchmarkCreateParams,
@@ -246,6 +250,7 @@ def start_run(
         self,
         *,
         benchmark_id: str,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
         run_name: Optional[str] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -261,6 +266,8 @@ def start_run(
         Args:
           benchmark_id: ID of the Benchmark to run.
 
+          metadata: User defined metadata to attach to the benchmark run for organization.
+
           run_name: Display name of the run.
 
           extra_headers: Send extra headers
@@ -278,6 +285,7 @@ def start_run(
             body=maybe_transform(
                 {
                     "benchmark_id": benchmark_id,
+                    "metadata": metadata,
                     "run_name": run_name,
                 },
                 benchmark_start_run_params.BenchmarkStartRunParams,
@@ -321,6 +329,7 @@ async def create(
         self,
         *,
         name: str,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
         scenario_ids: Optional[List[str]] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -336,6 +345,8 @@ async def create(
         Args:
           name: The name of the Benchmark.
 
+          metadata: User defined metadata to attach to the benchmark for organization.
+
           scenario_ids: The Scenario IDs that make up the Benchmark.
 
           extra_headers: Send extra headers
@@ -353,6 +364,7 @@ async def create(
             body=await async_maybe_transform(
                 {
                     "name": name,
+                    "metadata": metadata,
                     "scenario_ids": scenario_ids,
                 },
                 benchmark_create_params.BenchmarkCreateParams,
@@ -498,6 +510,7 @@ async def start_run(
         self,
         *,
         benchmark_id: str,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
         run_name: Optional[str] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -513,6 +526,8 @@ async def start_run(
         Args:
           benchmark_id: ID of the Benchmark to run.
 
+          metadata: User defined metadata to attach to the benchmark run for organization.
+
           run_name: Display name of the run.
 
           extra_headers: Send extra headers
@@ -530,6 +545,7 @@ async def start_run(
             body=await async_maybe_transform(
                 {
                     "benchmark_id": benchmark_id,
+                    "metadata": metadata,
                     "run_name": run_name,
                 },
                 benchmark_start_run_params.BenchmarkStartRunParams,

src/runloop_api_client/resources/scenarios/scenarios.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from typing import Optional
+from typing import Dict, Optional
 
 import httpx
 
@@ -88,6 +88,8 @@ def create(
         name: str,
         scoring_contract: ScoringContractParam,
         environment_parameters: Optional[ScenarioEnvironmentParam] | NotGiven = NOT_GIVEN,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
+        reference_output: Optional[str] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
         extra_headers: Headers | None = None,
@@ -109,6 +111,12 @@ def create(
 
           environment_parameters: The Environment in which the Scenario will run.
 
+          metadata: User defined metadata to attach to the scenario for organization.
+
+          reference_output: A string representation of the reference output to solve the scenario. Commonly
+              can be the result of a git diff or a sequence of command actions to apply to the
+              environment.
+
           extra_headers: Send extra headers
 
           extra_query: Add additional query parameters to the request
@@ -127,6 +135,8 @@ def create(
                     "name": name,
                     "scoring_contract": scoring_contract,
                     "environment_parameters": environment_parameters,
+                    "metadata": metadata,
+                    "reference_output": reference_output,
                 },
                 scenario_create_params.ScenarioCreateParams,
             ),
@@ -281,6 +291,7 @@ def start_run(
         *,
         scenario_id: str,
         benchmark_run_id: Optional[str] | NotGiven = NOT_GIVEN,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
         run_name: Optional[str] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -298,6 +309,8 @@ def start_run(
 
           benchmark_run_id: Benchmark to associate the run.
 
+          metadata: User defined metadata to attach to the run for organization.
+
           run_name: Display name of the run.
 
           extra_headers: Send extra headers
@@ -316,6 +329,7 @@ def start_run(
                 {
                     "scenario_id": scenario_id,
                     "benchmark_run_id": benchmark_run_id,
+                    "metadata": metadata,
                     "run_name": run_name,
                 },
                 scenario_start_run_params.ScenarioStartRunParams,
@@ -423,6 +437,8 @@ async def create(
         name: str,
         scoring_contract: ScoringContractParam,
         environment_parameters: Optional[ScenarioEnvironmentParam] | NotGiven = NOT_GIVEN,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
+        reference_output: Optional[str] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
         extra_headers: Headers | None = None,
@@ -444,6 +460,12 @@ async def create(
 
           environment_parameters: The Environment in which the Scenario will run.
 
+          metadata: User defined metadata to attach to the scenario for organization.
+
+          reference_output: A string representation of the reference output to solve the scenario. Commonly
+              can be the result of a git diff or a sequence of command actions to apply to the
+              environment.
+
           extra_headers: Send extra headers
 
           extra_query: Add additional query parameters to the request
@@ -462,6 +484,8 @@ async def create(
                     "name": name,
                     "scoring_contract": scoring_contract,
                     "environment_parameters": environment_parameters,
+                    "metadata": metadata,
+                    "reference_output": reference_output,
                 },
                 scenario_create_params.ScenarioCreateParams,
             ),
@@ -616,6 +640,7 @@ async def start_run(
         *,
         scenario_id: str,
         benchmark_run_id: Optional[str] | NotGiven = NOT_GIVEN,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
         run_name: Optional[str] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -633,6 +658,8 @@ async def start_run(
 
           benchmark_run_id: Benchmark to associate the run.
 
+          metadata: User defined metadata to attach to the run for organization.
+
           run_name: Display name of the run.
 
           extra_headers: Send extra headers
@@ -651,6 +678,7 @@ async def start_run(
                 {
                     "scenario_id": scenario_id,
                     "benchmark_run_id": benchmark_run_id,
+                    "metadata": metadata,
                     "run_name": run_name,
                 },
                 scenario_start_run_params.ScenarioStartRunParams,

src/runloop_api_client/types/benchmark_create_params.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from typing import List, Optional
+from typing import Dict, List, Optional
 from typing_extensions import Required, TypedDict
 
 __all__ = ["BenchmarkCreateParams"]
@@ -12,5 +12,8 @@ class BenchmarkCreateParams(TypedDict, total=False):
     name: Required[str]
     """The name of the Benchmark."""
 
+    metadata: Optional[Dict[str, str]]
+    """User defined metadata to attach to the benchmark for organization."""
+
     scenario_ids: Optional[List[str]]
     """The Scenario IDs that make up the Benchmark."""

src/runloop_api_client/types/benchmark_run_view.py

@@ -1,6 +1,6 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-from typing import List, Optional
+from typing import Dict, List, Optional
 from typing_extensions import Literal
 
 from .._models import BaseModel
@@ -15,6 +15,12 @@ class BenchmarkRunView(BaseModel):
     benchmark_id: str
     """The ID of the Benchmark."""
 
+    metadata: Dict[str, str]
+    """User defined metadata to attach to the benchmark run for organization."""
+
+    pending_scenarios: List[str]
+    """List of Scenarios that need to be completed before benchmark can be completed."""
+
     start_time_ms: int
     """The time the benchmark run execution started (Unix timestamp milliseconds)."""
 
@@ -27,9 +33,6 @@ class BenchmarkRunView(BaseModel):
     name: Optional[str] = None
     """The name of the BenchmarkRun."""
 
-    pending_scenarios: Optional[List[str]] = None
-    """List of Scenarios that need to be completed before benchmark can be completed."""
-
     score: Optional[float] = None
     """The final score across the BenchmarkRun, present once completed.
 

src/runloop_api_client/types/benchmark_start_run_params.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from typing import Optional
+from typing import Dict, Optional
 from typing_extensions import Required, TypedDict
 
 __all__ = ["BenchmarkStartRunParams"]
@@ -12,5 +12,8 @@ class BenchmarkStartRunParams(TypedDict, total=False):
     benchmark_id: Required[str]
     """ID of the Benchmark to run."""
 
+    metadata: Optional[Dict[str, str]]
+    """User defined metadata to attach to the benchmark run for organization."""
+
     run_name: Optional[str]
     """Display name of the run."""