feat(scenarios): add scenario builder to sdk (#706)

* init commit

* formatting fix

* clean up imports in scenario ops unit tests

* use Blueprint and Snapshot objects directly in ScenarioBuilder

* consolidate from_blueprint and from_snapshot unit tests

* further consolidate scenario builder unit tests, make sure async coverage matches sync

* stricter type declaration for _build_params

* expose request options in push()

* add scenario creation smoketests, with push_or_update logic

* update sdk smoke tests with all ops

* avoid modifyng _scorers internal state when normalizing weights (create copy instead)

* formatting fixes

* update builder docstrings to use fluent pattern, replaced all references to RunloopSDK() to 'runloop' instead of 'sdk' or 'client'

* clarify from_blueprint and from_snapshot docstrings

* rename add_scorer methods to be more clear

* format fix

* address type check errors in scenario builder unit tests

* rename add_scorer methods in docstrings

* make sure it is clear that score is 0.0-1.0 inclusive

* update script scorer docstrings

* formatting

* clarify reference solution/gold patch terminology and validation strategy

* make name first argument passed to scenario builder

* add preview method

* clean up unit test imports, rename builder fixture to mock_builder

* added `preview()` method to scenario builder

* rename `_build_params` to `build` (now publicly exposed method)

* formatting

* update docstring examples to use builder.build() instead of builder.push()

* formatting

* update with_problem_statement and with_additional_context docstrings

Author: sid-rl

README.md

@@ -33,10 +33,10 @@ For a higher-level, Pythonic interface, check out the new [`RunloopSDK`](README-
 ```python
 from runloop_api_client import RunloopSDK
 
-sdk = RunloopSDK()  # Uses RUNLOOP_API_KEY environment variable by default
+runloop = RunloopSDK()  # Uses RUNLOOP_API_KEY environment variable by default
 
 # Create a devbox and execute commands with a clean, object-oriented interface
-with sdk.devbox.create(name="my-devbox") as devbox:
+with runloop.devbox.create(name="my-devbox") as devbox:
     result = devbox.cmd.exec("echo 'Hello from Runloop!'")
     print(result.stdout())
 ```

src/runloop_api_client/sdk/__init__.py

@@ -7,6 +7,7 @@
 
 from .sync import AgentOps, DevboxOps, ScorerOps, RunloopSDK, ScenarioOps, SnapshotOps, BlueprintOps, StorageObjectOps
 from .agent import Agent
+from ._types import ScenarioPreview
 from .async_ import (
     AsyncAgentOps,
     AsyncDevboxOps,
@@ -33,9 +34,11 @@
 from .async_blueprint import AsyncBlueprint
 from .async_execution import AsyncExecution
 from .execution_result import ExecutionResult
+from .scenario_builder import ScenarioBuilder
 from .async_scenario_run import AsyncScenarioRun
 from .async_storage_object import AsyncStorageObject
 from .async_execution_result import AsyncExecutionResult
+from .async_scenario_builder import AsyncScenarioBuilder
 
 __all__ = [
     # Main SDK entry points
@@ -71,6 +74,9 @@
     "AsyncScenario",
     "ScenarioRun",
     "AsyncScenarioRun",
+    "ScenarioBuilder",
+    "AsyncScenarioBuilder",
+    "ScenarioPreview",
     "Scorer",
     "AsyncScorer",
     "Snapshot",

src/runloop_api_client/sdk/_types.py

@@ -5,6 +5,8 @@
 from ..lib.polling import PollingConfig
 from ..types.devboxes import DiskSnapshotListParams, DiskSnapshotUpdateParams
 from ..types.scenarios import ScorerListParams, ScorerCreateParams, ScorerUpdateParams, ScorerValidateParams
+from ..types.input_context import InputContext
+from ..types.scenario_view import ScenarioView
 from ..types.agent_list_params import AgentListParams
 from ..types.devbox_list_params import DevboxListParams
 from ..types.object_list_params import ObjectListParams
@@ -186,3 +188,18 @@ class SDKScenarioRunAsyncParams(ScenarioStartRunBaseParams, LongRequestOptions):
 
 class SDKScenarioRunParams(ScenarioStartRunBaseParams, LongPollingRequestOptions):
     pass
+
+
+class InputContextPreview(InputContext):
+    problem_statement: Optional[str] = None  # type: ignore[assignment]
+    """The problem statement for the Scenario."""
+
+
+class ScenarioPreview(ScenarioView):
+    """Preview of scenario configuration with all fields optional."""
+
+    id: Optional[str] = None  # type: ignore[assignment]
+    """The ID of the Scenario."""
+
+    input_context: InputContextPreview  # type: ignore[assignment]
+    """The input context for the Scenario."""

src/runloop_api_client/sdk/async_.py

@@ -37,6 +37,7 @@
 from .async_blueprint import AsyncBlueprint
 from ..lib.context_loader import TarFilter, build_directory_tar
 from .async_storage_object import AsyncStorageObject
+from .async_scenario_builder import AsyncScenarioBuilder
 from ..types.object_create_params import ContentType
 from ..types.shared_params.agent_source import Git, Npm, Pip, Object
 
@@ -773,6 +774,16 @@ class AsyncScenarioOps:
         >>> scenario = runloop.scenario.from_id("scn-xxx")
         >>> run = await scenario.run()
         >>> scenarios = await runloop.scenario.list()
+
+    Example using builder:
+        >>> builder = (
+        ...     runloop.scenario.builder("my-scenario")
+        ...     .from_blueprint(blueprint)
+        ...     .with_problem_statement("Fix the bug")
+        ...     .add_test_command_scorer("tests", test_command="pytest")
+        ... )
+        >>> params = builder.build()
+        >>> scenario = await runloop.scenario.create(**params)  # equivalent to builder.push()
     """
 
     def __init__(self, client: AsyncRunloop) -> None:
@@ -783,6 +794,16 @@ def __init__(self, client: AsyncRunloop) -> None:
         """
         self._client = client
 
+    def builder(self, name: str) -> AsyncScenarioBuilder:
+        """Create a new scenario builder.
+
+        :param name: Name for the scenario
+        :type name: str
+        :return: A new AsyncScenarioBuilder instance
+        :rtype: AsyncScenarioBuilder
+        """
+        return AsyncScenarioBuilder(name, self._client)
+
     def from_id(self, scenario_id: str) -> AsyncScenario:
         """Get an AsyncScenario instance for an existing scenario ID.