diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b0eff4..3e1b5ccb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [`docs/workflow-syntax.md`](docs/workflow-syntax.md#mcp-steps) and [`examples/mcp-step.yaml`](examples/mcp-step.yaml). +### Fixed + +- **Pydantic AI structured-output agents explicitly require `final_result`** — + the generated output tool now tells models that they must call it before + finishing and that plain-text responses are not accepted. This improves + adherence for local models behind OpenAI- or Anthropic-compatible endpoints + without replacing tool-based output, weakening schema validation, or + changing authored system prompts. + ## [0.1.37](https://github.com/microsoft/conductor/compare/v0.1.36...v0.1.37) - 2026-09-09 ### Added diff --git a/src/conductor/providers/_pydantic_ai/agent_builder.py b/src/conductor/providers/_pydantic_ai/agent_builder.py index 4dc0f869..cd3c1ea1 100644 --- a/src/conductor/providers/_pydantic_ai/agent_builder.py +++ b/src/conductor/providers/_pydantic_ai/agent_builder.py @@ -9,7 +9,7 @@ import logging import os -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Final, Literal from anthropic import NOT_GIVEN as ANTHROPIC_NOT_GIVEN from anthropic import AsyncAnthropic @@ -53,6 +53,10 @@ # model deprecation risk. The "-latest" suffix lets Anthropic aliases keep the # identifier current without YAML changes. DEFAULT_ANTHROPIC_MODEL: str = "claude-3-5-sonnet-latest" +_FINAL_RESULT_TOOL_DESCRIPTION: Final[str] = ( + "Call this tool to return the final structured result and end the conversation. " + "You must call this tool before finishing; plain text responses are not accepted." +) # Default OpenAI model used when the agent and runtime fail to declare one. DEFAULT_OPENAI_MODEL: str = "gpt-5-mini" @@ -235,7 +239,7 @@ def _build_output_type( ) if dynamic_model is None: return None - return ToolOutput(dynamic_model) + return ToolOutput(dynamic_model, description=_FINAL_RESULT_TOOL_DESCRIPTION) def _resolve_anthropic_thinking( diff --git a/tests/test_providers/test_openai_http_stub.py b/tests/test_providers/test_openai_http_stub.py index 76d3c9f8..e86d5cb7 100644 --- a/tests/test_providers/test_openai_http_stub.py +++ b/tests/test_providers/test_openai_http_stub.py @@ -17,7 +17,7 @@ import pytest from pydantic_ai.exceptions import ModelHTTPError -from conductor.config.schema import AgentDef, ToolOutputConfig +from conductor.config.schema import AgentDef, OutputField, ToolOutputConfig from conductor.exceptions import ProviderError from conductor.providers._pydantic_ai.agent_builder import build_agent from conductor.providers._pydantic_ai.retry import RetryConfig @@ -199,6 +199,31 @@ async def test_openai_pipeline_400_reasoning_effort_is_fatal_one_request() -> No assert len(captured["urls"]) == 1 +# Requirement: the OpenAI request tells the model how structured output ends the run. +@pytest.mark.asyncio +async def test_openai_structured_output_sends_final_result_contract() -> None: + agent = AgentDef( + name="formatter", + model="gpt-5-mini", + prompt="format this", + output={"answer": OutputField(type="string")}, + ) + run, captured = _build_pipeline_runner( + agent, + responses=[(400, _make_openai_400_response())], + ) + + with pytest.raises(ProviderError): + await run() + + request_body = json.loads(captured["bodies"][0]) + output_tool = request_body["tools"][0]["function"] + assert output_tool["name"] == "final_result" + assert "must call" in output_tool["description"].lower() + assert "plain text" in output_tool["description"].lower() + assert request_body["tool_choice"] == "required" + + @pytest.mark.asyncio async def test_openai_pipeline_429_retries_then_succeeds_with_two_requests() -> None: """A 429 followed by a 200 retries once and results in exactly two Chat Completions requests.""" diff --git a/tests/test_providers/test_pydantic_ai_agent_builder.py b/tests/test_providers/test_pydantic_ai_agent_builder.py index 32206704..96b4a61d 100644 --- a/tests/test_providers/test_pydantic_ai_agent_builder.py +++ b/tests/test_providers/test_pydantic_ai_agent_builder.py @@ -360,6 +360,23 @@ def test_output_schema_becomes_tool_output(self) -> None: instance = output_model(answer="42") assert instance.answer == "42" + # Requirement: structured-output agents receive an explicit completion contract. + def test_output_tool_requires_final_result_call(self) -> None: + agent_def = AgentDef( + name="formatter", + output={"answer": OutputField(type="string")}, + ) + + pydantic_agent = build_agent(agent_def, system_prompt="", rendered_prompt="") + + toolset = pydantic_agent._output_schema.toolset + assert toolset is not None + output_tool = toolset._tool_defs[0] + assert output_tool.name == "final_result" + assert output_tool.description is not None + assert "must call" in output_tool.description.lower() + assert "plain text" in output_tool.description.lower() + def test_empty_output_schema_falls_back_to_text(self) -> None: """An empty or missing output schema must produce text output (str).""" agent_def = AgentDef(name="chatter")