Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/agent/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ export type {

export type {
ModelClient,
ModelErrorEvent,
ModelRequest,
ModelEvent,
ModelMessage,
Expand Down
2 changes: 1 addition & 1 deletion packages/agent/src/retrieval.ts
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,7 @@ export function createRetrievalStrategy(opts: RetrievalStrategyOpts = {}): Agent
} else if (ev.kind === 'usage') {
turnUsage = ev.usage;
} else if (ev.kind === 'error') {
yield { kind: 'error', message: ev.message };
yield ev;
return;
}
// `tool_call` events are impossible here (toolUseEnabled:false,
Expand Down
9 changes: 8 additions & 1 deletion packages/agent/src/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,14 @@ export function createAgentSession(config: AgentSessionConfig): AgentSession {
history = [...history, assistantMsg];
yield { kind: 'turn_completed', turnId, metrics, details: ev.details };
} else if (ev.kind === 'error') {
yield { kind: 'error', turnId, message: ev.message };
yield {
kind: 'error',
turnId,
message: ev.message,
...(ev.code ? { code: ev.code } : {}),
...(ev.retryable !== undefined ? { retryable: ev.retryable } : {}),
...(ev.details ? { details: ev.details } : {}),
};
return;
} else if (ev.kind === 'custom') {
yield { kind: 'strategy_event', name: ev.name, data: ev.data };
Expand Down
34 changes: 33 additions & 1 deletion packages/agent/src/strategy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@ import type { ToolHandler, ToolResult } from './types/tools.js';
interface ReactLoopOptions {
/** Cap on loop iterations to avoid runaway tool-call ping-pong. Default 24. */
maxTurns?: number;
/**
* Default strict behavior preserves existing ReAct semantics: any
* model error fails the run. `complete-with-warning` is for hosts
* where prior successful tool effects can be authoritative even when
* a later final model call fails with a known retryable no-output
* provider error.
*/
toolProgressErrorPolicy?: 'strict' | 'complete-with-warning';
/**
* Opt-in: when `true`, tool calls produced in a single turn are partitioned
* by the handler's `parallelSafe` tag. Parallel-safe calls run concurrently
Expand Down Expand Up @@ -59,6 +67,7 @@ const DEFAULT_CRITIQUE_SYSTEM_PROMPT =
export function createReactLoopStrategy(options: ReactLoopOptions = {}): AgentStrategy {
const maxTurns = options.maxTurns ?? 24;
const parallelDispatch = options.parallelDispatch === true;
const toolProgressErrorPolicy = options.toolProgressErrorPolicy ?? 'strict';
const reflexionEnabled = options.reflexion?.enabled === true;
const reflexionMaxRetries = Math.max(0, options.reflexion?.maxRetries ?? 1);
const critiqueSystemPrompt =
Expand All @@ -72,6 +81,7 @@ export function createReactLoopStrategy(options: ReactLoopOptions = {}): AgentSt
// critique branch is skipped entirely and the strategy returns
// on the first no-tool-calls turn exactly as before.
let reflexionRetriesRemaining = reflexionMaxRetries;
let successfulToolResultsThisRun = 0;

for (let turn = 0; turn < maxTurns; turn++) {
if (signal.aborted) {
Expand Down Expand Up @@ -160,7 +170,23 @@ export function createReactLoopStrategy(options: ReactLoopOptions = {}): AgentSt
} else if (ev.kind === 'usage') {
turnUsage = ev.usage;
} else if (ev.kind === 'error') {
yield { kind: 'error', message: ev.message };
const modelError = ev as Extract<StrategyEvent, { kind: 'error' }>;
if (
toolProgressErrorPolicy === 'complete-with-warning' &&
successfulToolResultsThisRun > 0 &&
isCompletableToolProgressError(modelError)
) {
yield {
kind: 'custom',
name: 'react_loop_model_warning',
data: {
error: modelError,
successfulToolResultCount: successfulToolResultsThisRun,
},
};
return;
}
yield modelError;
return;
}
}
Expand Down Expand Up @@ -451,6 +477,7 @@ export function createReactLoopStrategy(options: ReactLoopOptions = {}): AgentSt
resultJson: JSON.stringify(safeSerializable(result)),
text: '',
});
successfulToolResultsThisRun += 1;
}
} else {
// Default behavior: byte-for-byte identical to the pre-change
Expand All @@ -469,6 +496,7 @@ export function createReactLoopStrategy(options: ReactLoopOptions = {}): AgentSt
resultJson: JSON.stringify(safeSerializable(result)),
text: '',
});
successfulToolResultsThisRun += 1;
}
}
// Close the tool-dispatch segment for this iteration. Pair
Expand Down Expand Up @@ -500,6 +528,10 @@ export function createReactLoopStrategy(options: ReactLoopOptions = {}): AgentSt
};
}

function isCompletableToolProgressError(ev: Extract<StrategyEvent, { kind: 'error' }>): boolean {
return ev.code === 'gemini.thinking_only_stop' && ev.retryable === true;
}

function buildMessages(input: StrategyRunInput): ModelMessage[] {
const out: ModelMessage[] = [];
out.push({ role: 'system', text: input.systemPrompt });
Expand Down
8 changes: 8 additions & 0 deletions packages/agent/src/types/llm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,14 @@ export type {
ReasoningEffort,
} from '@inbrowser/model';

export interface ModelErrorEvent {
kind: 'error';
message: string;
code?: string;
retryable?: boolean;
details?: Record<string, unknown>;
}

export interface LlmConfig {
apiKey?: string;
model: string;
Expand Down
4 changes: 2 additions & 2 deletions packages/agent/src/types/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
*/

import type { ChatMessage, TurnDetails, TurnMetrics } from './chat.js';
import type { ModelClient } from './llm.js';
import type { ModelClient, ModelErrorEvent } from './llm.js';
import type { MetricsCollector } from './metrics.js';
import type { RuntimeState } from './runtime.js';
import type { AgentStrategy } from './strategy.js';
Expand Down Expand Up @@ -71,7 +71,7 @@ export type SessionEvent =
| { kind: 'workspace_changed'; workspace: Workspace }
| { kind: 'runtime_changed'; runtime: RuntimeState }
| { kind: 'turn_completed'; turnId: string; metrics: TurnMetrics; details: TurnDetails }
| { kind: 'error'; turnId?: string; message: string }
| (ModelErrorEvent & { turnId?: string })
| { kind: 'completed' }
/** Strategy-emitted milestones (planner phases, branch expansions, …)
* — generic envelope so new strategies can surface custom events
Expand Down
4 changes: 2 additions & 2 deletions packages/agent/src/types/strategy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
*/

import type { ChatMessage, TurnDetails } from './chat.js';
import type { ModelClient, ModelUsage } from './llm.js';
import type { ModelClient, ModelErrorEvent, ModelUsage } from './llm.js';
import type { RuntimeState } from './runtime.js';
import type { ToolContext, ToolDispatch, ToolHandler, ToolResult } from './tools.js';
import type { Tracer } from './trace.js';
Expand Down Expand Up @@ -61,7 +61,7 @@ export type StrategyEvent =
| { kind: 'tool_call'; id: string; name: string; args: unknown; signature?: string }
| { kind: 'tool_result'; id: string; result: ToolResult }
| { kind: 'turn_complete'; usage: ModelUsage; details: TurnDetails }
| { kind: 'error'; message: string }
| ModelErrorEvent
/** Custom milestone — name + arbitrary payload, surfaced as
* `SessionEvent.kind === 'strategy_event'` to the host. */
| { kind: 'custom'; name: string; data?: unknown };
Expand Down
36 changes: 36 additions & 0 deletions packages/agent/test/session.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,42 @@ describe('createAgentSession', () => {
expect(kinds[kinds.length - 1]).toBe('completed');
});

test('preserves structured strategy error metadata on session errors', async () => {
const strategy: AgentStrategy = {
id: 'structured-error',
async *run() {
yield {
kind: 'error',
message: 'Gemini produced no output',
code: 'gemini.thinking_only_stop',
retryable: true,
details: { finishReason: 'STOP' },
};
},
};
const session = createAgentSession({
strategy,
llm: fakeLlm([]),
tools: createDispatch(createToolRegistry()),
toolList: [],
toolContext: fakeCtx,
systemPromptBuilder: () => 'system',
metrics: createMetricsCollector(),
history: [],
});

const events = await collect(session.submit('hi', new AbortController().signal));
const error = events.find((e) => e.kind === 'error');

expect(error).toMatchObject({
kind: 'error',
message: 'Gemini produced no output',
code: 'gemini.thinking_only_stop',
retryable: true,
details: { finishReason: 'STOP' },
});
});

test('applies tool result patches to workspace + runtime + emits change events', async () => {
const writeRulesTool: ToolHandler<{ source: string }> = {
name: 'writeRules',
Expand Down
181 changes: 181 additions & 0 deletions packages/agent/test/strategy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,187 @@ describe('createReactLoopStrategy', () => {
}
});

test('default strict mode preserves structured model errors after tool progress', async () => {
const echoTool: ToolHandler<{ msg: string }, { msg: string }> = {
name: 'echo',
description: 'echo',
parameters: { type: 'object' },
async execute({ msg }) {
return { ok: true, summary: msg, data: { msg } };
},
};
const registry = createToolRegistry();
registry.register(echoTool);
const error = {
kind: 'error' as const,
message: 'Gemini produced no output',
code: 'gemini.thinking_only_stop',
retryable: true,
details: { finishReason: 'STOP' },
};
const llm = fakeLlm([
[
{ kind: 'tool_call', id: 'c1', name: 'echo', args: { msg: 'done' } },
{ kind: 'usage', usage: { promptTokens: 5, outputTokens: 1 } },
],
[error],
]);

const events = await collect(
createReactLoopStrategy().run(
{
prompt: 'use the tool',
history: [],
workspace: EMPTY_WORKSPACE,
runtime: EMPTY_RUNTIME,
llm,
tools: createDispatch(registry),
toolList: [echoTool],
toolContext: fakeCtx,
systemPrompt: 'You may call tools.',
},
new AbortController().signal,
),
);

const finalError = events.find((e) => e.kind === 'error');
expect(finalError).toMatchObject(error);
});

test('progress-aware mode completes with warning after tool progress and Gemini thinking-only STOP', async () => {
const echoTool: ToolHandler<{ msg: string }, { msg: string }> = {
name: 'echo',
description: 'echo',
parameters: { type: 'object' },
async execute({ msg }) {
return { ok: true, summary: msg, data: { msg } };
},
};
const registry = createToolRegistry();
registry.register(echoTool);
const providerError = {
kind: 'error' as const,
message: 'Gemini produced no output',
code: 'gemini.thinking_only_stop',
retryable: true,
details: { finishReason: 'STOP' },
};
const llm = fakeLlm([
[
{ kind: 'tool_call', id: 'c1', name: 'echo', args: { msg: 'done' } },
{ kind: 'usage', usage: { promptTokens: 5, outputTokens: 1 } },
],
[providerError],
]);

const events = await collect(
createReactLoopStrategy({ toolProgressErrorPolicy: 'complete-with-warning' }).run(
{
prompt: 'use the tool',
history: [],
workspace: EMPTY_WORKSPACE,
runtime: EMPTY_RUNTIME,
llm,
tools: createDispatch(registry),
toolList: [echoTool],
toolContext: fakeCtx,
systemPrompt: 'You may call tools.',
},
new AbortController().signal,
),
);

expect(events.some((e) => e.kind === 'error')).toBe(false);
expect(events.filter((e) => e.kind === 'turn_complete')).toHaveLength(1);
const warning = events.find(
(e) => e.kind === 'custom' && e.name === 'react_loop_model_warning',
);
expect(warning?.kind).toBe('custom');
if (warning?.kind === 'custom') {
expect(warning.data).toMatchObject({
error: providerError,
successfulToolResultCount: 1,
});
}
});

test('progress-aware mode does not swallow unknown provider errors', async () => {
const echoTool: ToolHandler<{ msg: string }, { msg: string }> = {
name: 'echo',
description: 'echo',
parameters: { type: 'object' },
async execute({ msg }) {
return { ok: true, summary: msg, data: { msg } };
},
};
const registry = createToolRegistry();
registry.register(echoTool);
const unknownError = {
kind: 'error' as const,
message: 'provider exploded',
code: 'provider.unknown',
retryable: true,
};
const llm = fakeLlm([
[
{ kind: 'tool_call', id: 'c1', name: 'echo', args: { msg: 'done' } },
{ kind: 'usage', usage: { promptTokens: 5, outputTokens: 1 } },
],
[unknownError],
]);

const events = await collect(
createReactLoopStrategy({ toolProgressErrorPolicy: 'complete-with-warning' }).run(
{
prompt: 'use the tool',
history: [],
workspace: EMPTY_WORKSPACE,
runtime: EMPTY_RUNTIME,
llm,
tools: createDispatch(registry),
toolList: [echoTool],
toolContext: fakeCtx,
systemPrompt: 'You may call tools.',
},
new AbortController().signal,
),
);

expect(events.find((e) => e.kind === 'error')).toMatchObject(unknownError);
});

test('progress-aware mode does not complete when no tool result was dispatched', async () => {
const providerError = {
kind: 'error' as const,
message: 'Gemini produced no output',
code: 'gemini.thinking_only_stop',
retryable: true,
};
const llm = fakeLlm([[providerError]]);

const events = await collect(
createReactLoopStrategy({ toolProgressErrorPolicy: 'complete-with-warning' }).run(
{
prompt: 'answer directly',
history: [],
workspace: EMPTY_WORKSPACE,
runtime: EMPTY_RUNTIME,
llm,
tools: createDispatch(createToolRegistry()),
toolList: [],
toolContext: fakeCtx,
systemPrompt: 'You are helpful.',
},
new AbortController().signal,
),
);

expect(events.find((e) => e.kind === 'error')).toMatchObject(providerError);
expect(events.some((e) => e.kind === 'custom' && e.name === 'react_loop_model_warning')).toBe(
false,
);
});

test('aborts when the signal fires before the turn starts', async () => {
const controller = new AbortController();
controller.abort();
Expand Down
Loading
Loading