diff --git a/src/sdk.ts b/src/sdk.ts index bc5f9e334..d96c9479f 100644 --- a/src/sdk.ts +++ b/src/sdk.ts @@ -11,6 +11,7 @@ import { NetworkPolicy } from './sdk/network-policy'; import { GatewayConfig } from './sdk/gateway-config'; import { McpConfig } from './sdk/mcp-config'; import { Scenario } from './sdk/scenario'; +import { ScenarioBuilder } from './sdk/scenario-builder'; import { Secret } from './sdk/secret'; // Import types used in this file @@ -1923,7 +1924,8 @@ export class McpConfigOps { * * ## Quickstart * - * Use `fromId()` to get a {@link Scenario} by ID, or `list()` to retrieve all scenarios. + * Use `fromId()` to get a {@link Scenario} by ID, `list()` to retrieve all scenarios, + * or `builder()` to construct a new scenario with a fluent API. * Once you have a scenario, call `scenario.run()` to start a {@link ScenarioRun} with * your agent mounted. * @@ -1967,6 +1969,26 @@ export class ScenarioOps { */ constructor(private client: RunloopAPI) {} + /** + * Create a new {@link ScenarioBuilder} for constructing a scenario with a fluent API. + * + * @example + * ```typescript + * const runloop = new RunloopSDK(); + * const scenario = await runloop.scenario + * .builder('my-scenario') + * .withProblemStatement('Fix the bug in main.py') + * .addTestCommandScorer('tests', { test_command: 'pytest' }) + * .push(); + * ``` + * + * @param {string} name - Name for the scenario + * @returns {ScenarioBuilder} A {@link ScenarioBuilder} instance + */ + builder(name: string): ScenarioBuilder { + return new ScenarioBuilder(name, this.client); + } + /** * Get a scenario object by its ID. * diff --git a/src/sdk/index.ts b/src/sdk/index.ts index 8950b855d..ae5bee7e6 100644 --- a/src/sdk/index.ts +++ b/src/sdk/index.ts @@ -12,3 +12,4 @@ export { McpConfig } from './mcp-config'; export { Secret } from './secret'; export { ScenarioRun } from './scenario-run'; export { Scenario, type ScenarioRunParams } from './scenario'; +export { ScenarioBuilder } from './scenario-builder'; diff --git a/src/sdk/scenario-builder.ts b/src/sdk/scenario-builder.ts new file mode 100644 index 000000000..ac12cd0cd --- /dev/null +++ b/src/sdk/scenario-builder.ts @@ -0,0 +1,444 @@ +import { Runloop } from '../index'; +import type * as Core from '../core'; +import type { + ScenarioCreateParams, + ScenarioEnvironment, + ScoringFunction, +} from '../resources/scenarios/scenarios'; +import { Scenario } from './scenario'; +import { Blueprint } from './blueprint'; +import { Snapshot } from './snapshot'; + +/** + * Fluent builder for constructing {@link ScenarioCreateParams}. + * + * @category Scenario + * + * @remarks + * ## Overview + * + * The `ScenarioBuilder` provides a step-by-step, chainable interface for + * configuring all aspects of a scenario before pushing it to the platform. + * + * ## Quickstart + * + * ```typescript + * import { RunloopSDK } from '@runloop/api-client'; + * + * const runloop = new RunloopSDK(); + * const scenario = await runloop.scenario + * .builder('my-scenario') + * .fromBlueprint(blueprint) + * .withWorkingDirectory('/app') + * .withProblemStatement('Fix the bug in main.py') + * .addTestCommandScorer('tests', { test_command: 'pytest' }) + * .push(); + * ``` + */ +export class ScenarioBuilder { + private client: Runloop; + private _name: string; + + // Environment + private _blueprint: Blueprint | null = null; + private _snapshot: Snapshot | null = null; + private _workingDirectory: string | null = null; + + // Input context + private _problemStatement: string | null = null; + private _additionalContext: unknown = null; + + // Scoring + private _scorers: ScoringFunction[] = []; + + // Metadata + private _metadata: Record = {}; + private _referenceOutput: string | null = null; + private _requiredEnvVars: string[] | null = null; + private _requiredSecrets: string[] | null = null; + private _validationType: 'UNSPECIFIED' | 'FORWARD' | 'REVERSE' | 'EVALUATION' | null = null; + + /** + * Create a new ScenarioBuilder. + * + * @param {string} name - Name for the scenario + * @param {Runloop} client - The Runloop client instance + */ + constructor(name: string, client: Runloop) { + this._name = name; + this.client = client; + } + + /** + * Get the scenario name. + * @returns {string} The scenario name + */ + get name(): string { + return this._name; + } + + /** + * Set a blueprint to define the baseline environment for the scenario. + * + * @param {Blueprint} blueprint - Blueprint to use + * @returns {this} Builder for chaining + */ + fromBlueprint(blueprint: Blueprint): this { + this._blueprint = blueprint; + this._snapshot = null; + return this; + } + + /** + * Set a snapshot to define the baseline environment for the scenario. + * + * @param {Snapshot} snapshot - Snapshot to use + * @returns {this} Builder for chaining + */ + fromSnapshot(snapshot: Snapshot): this { + this._snapshot = snapshot; + this._blueprint = null; + return this; + } + + /** + * Set the working directory for the scenario. + * + * @param {string} directory - Working directory path + * @returns {this} Builder for chaining + */ + withWorkingDirectory(directory: string): this { + this._workingDirectory = directory; + return this; + } + + /** + * Set the problem statement for the scenario. This will be provided as + * input context to the agent. + * + * @param {string} statement - Problem statement text + * @returns {this} Builder for chaining + */ + withProblemStatement(statement: string): this { + this._problemStatement = statement; + return this; + } + + /** + * Set additional structured context for the scenario. + * + * @param {unknown} context - Additional context (JSON-serializable) + * @returns {this} Builder for chaining + */ + withAdditionalContext(context: unknown): this { + this._additionalContext = context; + return this; + } + + /** + * Add a test-based scorer that runs a test command. + * + * @example + * ```typescript + * builder.addTestCommandScorer('tests', { + * test_command: 'pytest', + * test_files: [{ file_path: 'test_main.py', file_contents: 'def test_foo(): ...' }], + * }); + * ``` + * + * @param {string} name - Name of the scoring function + * @param {object} opts - Scorer options + * @param {string} opts.test_command - Command to run tests + * @param {number} [opts.weight] - Weight for this scorer (default: 1.0) + * @param {ScoringFunction.TestBasedScoringFunction.TestFile[]} [opts.test_files] - Test files to create before running + * @returns {this} Builder for chaining + */ + addTestCommandScorer( + name: string, + opts: { + test_command: string; + weight?: number; + test_files?: ScoringFunction.TestBasedScoringFunction.TestFile[]; + }, + ): this { + const scorer: ScoringFunction.TestBasedScoringFunction = { + type: 'test_based_scorer', + test_command: opts.test_command, + ...(opts.test_files !== undefined && { test_files: opts.test_files }), + }; + return this.addScorer(name, opts.weight ?? 1.0, scorer); + } + + /** + * Add a command scorer that runs a shell command. + * Scoring passes if the command returns exit code 0. + * + * @param {string} name - Name of the scoring function + * @param {object} opts - Scorer options + * @param {string} opts.command - Shell command to execute + * @param {number} [opts.weight] - Weight for this scorer (default: 1.0) + * @returns {this} Builder for chaining + */ + addShellCommandScorer(name: string, opts: { command: string; weight?: number }): this { + const scorer: ScoringFunction.CommandScoringFunction = { + type: 'command_scorer', + command: opts.command, + }; + return this.addScorer(name, opts.weight ?? 1.0, scorer); + } + + /** + * Add a standalone bash script scorer. + * The script should output "score=X.X" where X.X is a float between 0.0 and 1.0. + * + * @param {string} name - Name of the scoring function + * @param {object} opts - Scorer options + * @param {string} opts.bash_script - Bash script content + * @param {number} [opts.weight] - Weight for this scorer (default: 1.0) + * @returns {this} Builder for chaining + */ + addBashScriptScorer(name: string, opts: { bash_script: string; weight?: number }): this { + const scorer: ScoringFunction.BashScriptScoringFunction = { + type: 'bash_script_scorer', + bash_script: opts.bash_script, + }; + return this.addScorer(name, opts.weight ?? 1.0, scorer); + } + + /** + * Add a standalone Python script scorer. + * The script should print the score in the range [0.0, 1.0] to stdout. + * + * @param {string} name - Name of the scoring function + * @param {object} opts - Scorer options + * @param {string} opts.python_script - Python script content + * @param {number} [opts.weight] - Weight for this scorer (default: 1.0) + * @param {string} [opts.python_version_constraint] - Python version (default "==3.12.10") + * @param {string} [opts.requirements_contents] - pip requirements.txt content + * @returns {this} Builder for chaining + */ + addPythonScriptScorer( + name: string, + opts: { + python_script: string; + weight?: number; + python_version_constraint?: string; + requirements_contents?: string; + }, + ): this { + const scorer: ScoringFunction.PythonScriptScoringFunction = { + type: 'python_script_scorer', + python_script: opts.python_script, + ...(opts.python_version_constraint !== undefined && { + python_version_constraint: opts.python_version_constraint, + }), + ...(opts.requirements_contents !== undefined && { requirements_contents: opts.requirements_contents }), + }; + return this.addScorer(name, opts.weight ?? 1.0, scorer); + } + + /** + * Add an AST grep scorer that matches code patterns. + * + * @param {string} name - Name of the scoring function + * @param {object} opts - Scorer options + * @param {string} opts.pattern - AST pattern to match + * @param {number} [opts.weight] - Weight for this scorer (default: 1.0) + * @param {string} [opts.search_directory] - Directory to search (default: ".") + * @param {string} [opts.lang] - Language of the pattern + * @returns {this} Builder for chaining + */ + addAstGrepScorer( + name: string, + opts: { + pattern: string; + weight?: number; + search_directory?: string; + lang?: string; + }, + ): this { + const scorer: ScoringFunction.AstGrepScoringFunction = { + type: 'ast_grep_scorer', + pattern: opts.pattern, + search_directory: opts.search_directory ?? '.', + ...(opts.lang !== undefined && { lang: opts.lang }), + }; + return this.addScorer(name, opts.weight ?? 1.0, scorer); + } + + /** + * Add a custom scorer registered with Runloop. + * + * @param {string} name - Name of the scoring function + * @param {object} opts - Scorer options + * @param {string} opts.custom_scorer_type - Type identifier registered with Runloop + * @param {number} [opts.weight] - Weight for this scorer (default: 1.0) + * @param {unknown} [opts.scorer_params] - Additional JSON parameters for the scorer + * @returns {this} Builder for chaining + */ + addCustomScorer( + name: string, + opts: { + custom_scorer_type: string; + weight?: number; + scorer_params?: unknown; + }, + ): this { + const scorer: ScoringFunction.CustomScoringFunction = { + type: 'custom_scorer', + custom_scorer_type: opts.custom_scorer_type, + ...(opts.scorer_params !== undefined && { scorer_params: opts.scorer_params }), + }; + return this.addScorer(name, opts.weight ?? 1.0, scorer); + } + + /** + * Set metadata for the scenario. + * + * @param {Record} metadata - Key-value metadata + * @returns {this} Builder for chaining + */ + withMetadata(metadata: Record): this { + this._metadata = metadata; + return this; + } + + /** + * Set the reference solution or gold patch for validation. + * + * @param {string} output - Reference solution (e.g., git diff) + * @returns {this} Builder for chaining + */ + withReferenceOutput(output: string): this { + this._referenceOutput = output; + return this; + } + + /** + * Set required environment variables. + * + * @param {string[]} envVars - List of required environment variable names + * @returns {this} Builder for chaining + */ + withRequiredEnvVars(envVars: string[]): this { + this._requiredEnvVars = envVars; + return this; + } + + /** + * Set required secrets. + * + * @param {string[]} secrets - List of required secret names + * @returns {this} Builder for chaining + */ + withRequiredSecrets(secrets: string[]): this { + this._requiredSecrets = secrets; + return this; + } + + /** + * Set the validation strategy. + * + * @param {'UNSPECIFIED' | 'FORWARD' | 'REVERSE' | 'EVALUATION'} validationType - Validation type + * @returns {this} Builder for chaining + */ + withValidationType(validationType: 'UNSPECIFIED' | 'FORWARD' | 'REVERSE' | 'EVALUATION'): this { + this._validationType = validationType; + return this; + } + + /** + * Build the scenario creation parameters. + * + * Validates that required fields are set and normalizes scorer weights + * to sum to 1.0. + * + * @example + * ```typescript + * const params = builder.build(); + * // Use params with the raw API client + * const scenarioView = await client.scenarios.create(params); + * ``` + * + * @returns {ScenarioCreateParams} Parameters for scenario creation + * @throws {Error} If problem statement is missing or no scorers are configured + */ + build(): ScenarioCreateParams { + if (!this._problemStatement) { + throw new Error('Problem statement is required. Call withProblemStatement() first.'); + } + + if (this._scorers.length === 0) { + throw new Error( + 'At least one scorer is required. ' + + 'Call addTestCommandScorer(), addBashScriptScorer(), or another scorer method first.', + ); + } + + const totalWeight = this._scorers.reduce((sum, s) => sum + s.weight, 0); + const normalizedScorers: ScoringFunction[] = this._scorers.map((s) => ({ + ...s, + weight: s.weight / totalWeight, + })); + + const environmentParameters: ScenarioEnvironment | null = + this._blueprint || this._snapshot || this._workingDirectory ? + { + blueprint_id: this._blueprint?.id ?? null, + snapshot_id: this._snapshot?.id ?? null, + working_directory: this._workingDirectory ?? null, + } + : null; + + return { + name: this._name, + input_context: { + problem_statement: this._problemStatement, + ...(this._additionalContext !== null && { additional_context: this._additionalContext }), + }, + scoring_contract: { + scoring_function_parameters: normalizedScorers, + }, + ...(environmentParameters !== null && { environment_parameters: environmentParameters }), + ...(Object.keys(this._metadata).length > 0 && { metadata: this._metadata }), + ...(this._referenceOutput !== null && { reference_output: this._referenceOutput }), + ...(this._requiredEnvVars !== null && { required_environment_variables: this._requiredEnvVars }), + ...(this._requiredSecrets !== null && { required_secret_names: this._requiredSecrets }), + ...(this._validationType !== null && { validation_type: this._validationType }), + }; + } + + /** + * Create the scenario on the platform. + * + * Calls {@link build} to validate and assemble parameters, then creates + * the scenario via the API. + * + * @example + * ```typescript + * const scenario = await runloop.scenario + * .builder('my-scenario') + * .withProblemStatement('Fix the bug') + * .addTestCommandScorer('tests', { test_command: 'pytest' }) + * .push(); + * console.log(`Created scenario: ${scenario.id}`); + * ``` + * + * @param {Core.RequestOptions} [options] - Request options + * @returns {Promise} Created {@link Scenario} instance + * @throws {Error} If required fields are missing + */ + async push(options?: Core.RequestOptions): Promise { + const params = this.build(); + const view = await this.client.scenarios.create(params, options); + return Scenario.fromId(this.client, view.id); + } + + private addScorer(name: string, weight: number, scorer: ScoringFunction['scorer']): this { + if (!Number.isFinite(weight) || weight <= 0) { + throw new Error(`Scorer weight must be a finite positive number, got ${weight}`); + } + this._scorers.push({ name, weight, scorer }); + return this; + } +} diff --git a/tests/objects/scenario-builder.test.ts b/tests/objects/scenario-builder.test.ts new file mode 100644 index 000000000..2f61d0e4e --- /dev/null +++ b/tests/objects/scenario-builder.test.ts @@ -0,0 +1,267 @@ +import { ScenarioBuilder } from '../../src/sdk/scenario-builder'; +import { Scenario } from '../../src/sdk/scenario'; +import { Blueprint } from '../../src/sdk/blueprint'; +import { Snapshot } from '../../src/sdk/snapshot'; +import type { ScenarioView } from '../../src/resources/scenarios/scenarios'; + +// Mock the Runloop client +jest.mock('../../src/index'); + +describe('ScenarioBuilder', () => { + let mockClient: any; + + beforeEach(() => { + mockClient = { + scenarios: { + create: jest.fn().mockResolvedValue({ id: 'scn_created' } as ScenarioView), + }, + }; + }); + + describe('name getter', () => { + it('should return the name set in constructor', () => { + const builder = new ScenarioBuilder('test-scenario', mockClient); + expect(builder.name).toBe('test-scenario'); + }); + }); + + describe('build()', () => { + it('should produce correct params with all options and all scorer types', () => { + const mockBlueprint = Blueprint.fromId(mockClient, 'bp_123'); + + const params = new ScenarioBuilder('full-test', mockClient) + .fromBlueprint(mockBlueprint) + .withWorkingDirectory('/app') + .withProblemStatement('Fix the bug') + .withAdditionalContext({ hint: 'check main.py' }) + .addTestCommandScorer('test-scorer', { + test_command: 'pytest', + weight: 2, + test_files: [{ file_path: 'test.py', file_contents: 'def test(): pass' }], + }) + .addShellCommandScorer('cmd-scorer', { command: 'echo pass', weight: 3 }) + .addBashScriptScorer('bash-scorer', { bash_script: 'echo "score=1.0"', weight: 1 }) + .addPythonScriptScorer('py-scorer', { + python_script: 'print(1.0)', + weight: 2, + python_version_constraint: '>=3.11', + requirements_contents: 'numpy==1.26', + }) + .addAstGrepScorer('ast-scorer', { pattern: 'console.log($$$)', weight: 1, lang: 'javascript' }) + .addCustomScorer('custom-scorer', { + custom_scorer_type: 'my_scorer', + weight: 1, + scorer_params: { threshold: 0.5 }, + }) + .withMetadata({ env: 'test' }) + .withReferenceOutput('diff --git a/main.py') + .withRequiredEnvVars(['API_KEY']) + .withRequiredSecrets(['DB_PASS']) + .withValidationType('FORWARD') + .build(); + + // Required fields + expect(params.name).toBe('full-test'); + expect(params.input_context.problem_statement).toBe('Fix the bug'); + expect(params.input_context.additional_context).toEqual({ hint: 'check main.py' }); + + // Environment + expect(params.environment_parameters?.blueprint_id).toBe('bp_123'); + expect(params.environment_parameters?.snapshot_id).toBeNull(); + expect(params.environment_parameters?.working_directory).toBe('/app'); + + // Scorers - count and weight normalization + const scorers = params.scoring_contract.scoring_function_parameters; + expect(scorers).toHaveLength(6); + const totalWeight = scorers.reduce((sum, s) => sum + s.weight, 0); + expect(totalWeight).toBeCloseTo(1.0); + + // Verify each scorer type + expect(scorers[0]!.name).toBe('test-scorer'); + expect(scorers[0]!.scorer).toEqual({ + type: 'test_based_scorer', + test_command: 'pytest', + test_files: [{ file_path: 'test.py', file_contents: 'def test(): pass' }], + }); + + expect(scorers[1]!.scorer).toEqual({ type: 'command_scorer', command: 'echo pass' }); + expect(scorers[2]!.scorer).toEqual({ type: 'bash_script_scorer', bash_script: 'echo "score=1.0"' }); + expect(scorers[3]!.scorer).toEqual({ + type: 'python_script_scorer', + python_script: 'print(1.0)', + python_version_constraint: '>=3.11', + requirements_contents: 'numpy==1.26', + }); + expect(scorers[4]!.scorer).toEqual({ + type: 'ast_grep_scorer', + pattern: 'console.log($$$)', + search_directory: '.', + lang: 'javascript', + }); + expect(scorers[5]!.scorer).toEqual({ + type: 'custom_scorer', + custom_scorer_type: 'my_scorer', + scorer_params: { threshold: 0.5 }, + }); + + // Metadata fields + expect(params.metadata).toEqual({ env: 'test' }); + expect(params.reference_output).toBe('diff --git a/main.py'); + expect(params.required_environment_variables).toEqual(['API_KEY']); + expect(params.required_secret_names).toEqual(['DB_PASS']); + expect(params.validation_type).toBe('FORWARD'); + }); + + it('should default search_directory to "." for ast grep scorer', () => { + const params = new ScenarioBuilder('ast-test', mockClient) + .withProblemStatement('test') + .addAstGrepScorer('ast', { pattern: 'foo' }) + .build(); + + expect((params.scoring_contract.scoring_function_parameters[0]!.scorer as any).search_directory).toBe( + '.', + ); + }); + + it('should omit optional fields when not set', () => { + const params = new ScenarioBuilder('minimal', mockClient) + .withProblemStatement('test') + .addShellCommandScorer('s', { command: 'echo 1' }) + .build(); + + expect(params.environment_parameters).toBeUndefined(); + expect(params.metadata).toBeUndefined(); + expect(params.reference_output).toBeUndefined(); + expect(params.required_environment_variables).toBeUndefined(); + expect(params.required_secret_names).toBeUndefined(); + expect(params.validation_type).toBeUndefined(); + }); + }); + + describe('fromBlueprint / fromSnapshot mutual exclusion', () => { + it('should clear snapshot when setting blueprint', () => { + const snapshot = Snapshot.fromId(mockClient, 'snap_1'); + const blueprint = Blueprint.fromId(mockClient, 'bp_1'); + + const params = new ScenarioBuilder('test', mockClient) + .fromSnapshot(snapshot) + .fromBlueprint(blueprint) + .withProblemStatement('test') + .addShellCommandScorer('s', { command: 'echo 1' }) + .build(); + + expect(params.environment_parameters?.blueprint_id).toBe('bp_1'); + expect(params.environment_parameters?.snapshot_id).toBeNull(); + }); + + it('should clear blueprint when setting snapshot', () => { + const blueprint = Blueprint.fromId(mockClient, 'bp_1'); + const snapshot = Snapshot.fromId(mockClient, 'snap_1'); + + const params = new ScenarioBuilder('test', mockClient) + .fromBlueprint(blueprint) + .fromSnapshot(snapshot) + .withProblemStatement('test') + .addShellCommandScorer('s', { command: 'echo 1' }) + .build(); + + expect(params.environment_parameters?.snapshot_id).toBe('snap_1'); + expect(params.environment_parameters?.blueprint_id).toBeNull(); + }); + }); + + describe('validation errors', () => { + it('should throw when problem statement is missing', () => { + const builder = new ScenarioBuilder('test', mockClient).addShellCommandScorer('s', { + command: 'echo 1', + }); + + expect(() => builder.build()).toThrow('Problem statement is required'); + }); + + it('should throw when no scorers are configured', () => { + const builder = new ScenarioBuilder('test', mockClient).withProblemStatement('test'); + + expect(() => builder.build()).toThrow('At least one scorer is required'); + }); + + it('should throw when scorer weight is not a finite positive number', () => { + const builder = new ScenarioBuilder('test', mockClient); + + expect(() => builder.addShellCommandScorer('s', { command: 'echo 1', weight: 0 })).toThrow( + 'Scorer weight must be a finite positive number', + ); + + expect(() => builder.addShellCommandScorer('s', { command: 'echo 1', weight: -1 })).toThrow( + 'Scorer weight must be a finite positive number', + ); + + expect(() => builder.addShellCommandScorer('s', { command: 'echo 1', weight: NaN })).toThrow( + 'Scorer weight must be a finite positive number', + ); + + expect(() => builder.addShellCommandScorer('s', { command: 'echo 1', weight: Infinity })).toThrow( + 'Scorer weight must be a finite positive number', + ); + }); + }); + + describe('push()', () => { + it('should call client.scenarios.create with build output and return Scenario', async () => { + const scenario = await new ScenarioBuilder('push-test', mockClient) + .withProblemStatement('Fix it') + .addShellCommandScorer('s', { command: 'echo 1' }) + .push(); + + expect(mockClient.scenarios.create).toHaveBeenCalledTimes(1); + const callArgs = mockClient.scenarios.create.mock.calls[0]; + expect(callArgs[0].name).toBe('push-test'); + expect(callArgs[0].input_context.problem_statement).toBe('Fix it'); + + expect(scenario).toBeInstanceOf(Scenario); + expect(scenario.id).toBe('scn_created'); + }); + + it('should pass request options through', async () => { + const options = { timeout: 5000 }; + await new ScenarioBuilder('test', mockClient) + .withProblemStatement('test') + .addShellCommandScorer('s', { command: 'echo 1' }) + .push(options); + + expect(mockClient.scenarios.create).toHaveBeenCalledWith(expect.any(Object), options); + }); + + it('should throw build validation errors before calling API', async () => { + const builder = new ScenarioBuilder('test', mockClient); + + await expect(builder.push()).rejects.toThrow('Problem statement is required'); + expect(mockClient.scenarios.create).not.toHaveBeenCalled(); + }); + }); + + describe('fluent chaining', () => { + it('should return the same builder instance from all methods', () => { + const builder = new ScenarioBuilder('test', mockClient); + const blueprint = Blueprint.fromId(mockClient, 'bp_1'); + const snapshot = Snapshot.fromId(mockClient, 'snap_1'); + + expect(builder.fromBlueprint(blueprint)).toBe(builder); + expect(builder.fromSnapshot(snapshot)).toBe(builder); + expect(builder.withWorkingDirectory('/app')).toBe(builder); + expect(builder.withProblemStatement('test')).toBe(builder); + expect(builder.withAdditionalContext({})).toBe(builder); + expect(builder.addTestCommandScorer('t', { test_command: 'pytest' })).toBe(builder); + expect(builder.addShellCommandScorer('s', { command: 'echo 1' })).toBe(builder); + expect(builder.addBashScriptScorer('b', { bash_script: 'echo 1' })).toBe(builder); + expect(builder.addPythonScriptScorer('p', { python_script: 'print(1)' })).toBe(builder); + expect(builder.addAstGrepScorer('a', { pattern: 'foo' })).toBe(builder); + expect(builder.addCustomScorer('c', { custom_scorer_type: 'x' })).toBe(builder); + expect(builder.withMetadata({})).toBe(builder); + expect(builder.withReferenceOutput('ref')).toBe(builder); + expect(builder.withRequiredEnvVars([])).toBe(builder); + expect(builder.withRequiredSecrets([])).toBe(builder); + expect(builder.withValidationType('FORWARD')).toBe(builder); + }); + }); +}); diff --git a/tests/smoketests/object-oriented/scenario.test.ts b/tests/smoketests/object-oriented/scenario.test.ts index e159346ee..05a5bdd4e 100644 --- a/tests/smoketests/object-oriented/scenario.test.ts +++ b/tests/smoketests/object-oriented/scenario.test.ts @@ -1,4 +1,4 @@ -import { Scenario, ScenarioRun } from '@runloop/api-client/sdk'; +import { Scenario, ScenarioBuilder, ScenarioRun } from '@runloop/api-client/sdk'; import { makeClientSDK, SHORT_TIMEOUT, uniqueName } from '../utils'; const sdk = makeClientSDK(); @@ -89,6 +89,86 @@ describe('smoketest: object-oriented scenario', () => { ); }); + describe('ScenarioBuilder', () => { + test('builder() returns a ScenarioBuilder instance', () => { + const builder = sdk.scenario.builder('test-builder'); + expect(builder).toBeInstanceOf(ScenarioBuilder); + expect(builder.name).toBe('test-builder'); + }); + + test('build() produces valid params with all scorer types and normalizes weights', () => { + const blueprint = sdk.blueprint.fromId('bp_fake'); + + const params = sdk.scenario + .builder('build-test') + .fromBlueprint(blueprint) + .withWorkingDirectory('/app') + .withProblemStatement('Test problem') + .withAdditionalContext({ hint: 'test' }) + .addTestCommandScorer('test', { test_command: 'pytest', weight: 2 }) + .addShellCommandScorer('cmd', { command: 'echo 1.0', weight: 1 }) + .addBashScriptScorer('bash', { bash_script: 'echo "score=1.0"', weight: 1 }) + .addPythonScriptScorer('python', { python_script: 'print(1.0)', weight: 1 }) + .addAstGrepScorer('ast', { pattern: 'console.log($$$)', weight: 1 }) + .addCustomScorer('custom', { custom_scorer_type: 'test_type', weight: 1 }) + .withMetadata({ env: 'test' }) + .withReferenceOutput('reference') + .withRequiredEnvVars(['TEST_VAR']) + .withRequiredSecrets(['TEST_SECRET']) + .withValidationType('EVALUATION') + .build(); + + expect(params.name).toBe('build-test'); + expect(params.input_context.problem_statement).toBe('Test problem'); + expect(params.environment_parameters?.blueprint_id).toBe('bp_fake'); + expect(params.scoring_contract.scoring_function_parameters).toHaveLength(6); + + const totalWeight = params.scoring_contract.scoring_function_parameters.reduce( + (sum, s) => sum + s.weight, + 0, + ); + expect(totalWeight).toBeCloseTo(1.0); + }); + + test('build() with fromSnapshot sets snapshot_id', () => { + const snapshot = sdk.snapshot.fromId('snap_fake'); + const params = sdk.scenario + .builder('snap-test') + .fromSnapshot(snapshot) + .withProblemStatement('test') + .addShellCommandScorer('s', { command: 'echo 1' }) + .build(); + + expect(params.environment_parameters?.snapshot_id).toBe('snap_fake'); + expect(params.environment_parameters?.blueprint_id).toBeNull(); + }); + + test('build() throws without problem statement', () => { + const builder = sdk.scenario.builder('test').addShellCommandScorer('s', { command: 'echo 1' }); + expect(() => builder.build()).toThrow('Problem statement is required'); + }); + + test('build() throws without scorers', () => { + const builder = sdk.scenario.builder('test').withProblemStatement('test'); + expect(() => builder.build()).toThrow('At least one scorer is required'); + }); + + test( + 'push() creates scenario on platform', + async () => { + const scenario = await sdk.scenario + .builder(uniqueName('sdk-builder-push')) + .withProblemStatement('Builder push test') + .addShellCommandScorer('s', { command: 'echo 1.0' }) + .push(); + + expect(scenario).toBeInstanceOf(Scenario); + expect(scenario.id).toBeTruthy(); + }, + SHORT_TIMEOUT, + ); + }); + describe('Scenario run methods', () => { let scenario: Scenario;