From 13d528ca3614abe024f12b97113db2984e78f7ec Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Wed, 12 Mar 2025 10:40:35 +0200 Subject: [PATCH 01/13] WIP --- libs/agno/agno/models/openai/__init__.py | 1 + libs/agno/agno/models/openai/responses.py | 594 ++++++++++++++++++++++ libs/agno/agno/utils/openai_responses.py | 94 ++++ 3 files changed, 689 insertions(+) create mode 100644 libs/agno/agno/models/openai/responses.py create mode 100644 libs/agno/agno/utils/openai_responses.py diff --git a/libs/agno/agno/models/openai/__init__.py b/libs/agno/agno/models/openai/__init__.py index cbd773dafa..394d1a9518 100644 --- a/libs/agno/agno/models/openai/__init__.py +++ b/libs/agno/agno/models/openai/__init__.py @@ -1,2 +1,3 @@ from agno.models.openai.chat import OpenAIChat from agno.models.openai.like import OpenAILike +from agno.models.openai.responses import OpenAIResponses diff --git a/libs/agno/agno/models/openai/responses.py b/libs/agno/agno/models/openai/responses.py new file mode 100644 index 0000000000..e88ba51f14 --- /dev/null +++ b/libs/agno/agno/models/openai/responses.py @@ -0,0 +1,594 @@ +from dataclasses import dataclass, field +from typing import Any, AsyncGenerator, AsyncIterator, Dict, Iterator, List, Optional, Tuple, Union +import asyncio +from agno.models.response import ModelResponse +from agno.utils.openai_responses import images_to_message +import httpx + +from agno.exceptions import ModelProviderError +from agno.models.base import Model +from agno.models.message import Message +from agno.utils.log import logger + + +try: + import importlib.metadata as metadata + + from openai import AsyncOpenAI, OpenAI + from openai import APIConnectionError, APIStatusError, RateLimitError + from openai.resources.responses.responses import Response, ResponseStreamEvent + + from packaging import version + + # Get installed OpenAI version + openai_version = metadata.version("openai") + + # Check version compatibility + parsed_version = version.parse(openai_version) + if parsed_version.major == 0: + import warnings + warnings.warn("OpenAI v1.x is recommended for the Responses API", UserWarning) + +except ImportError as e: + # Handle different import error scenarios + if "openai" in str(e): + raise ImportError("OpenAI not installed. Install with `pip install openai`") from e + else: + raise ImportError("Missing dependencies. Install with `pip install packaging importlib-metadata`") from e + + +@dataclass +class OpenAIResponses(Model): + """ + Implementation for the OpenAI Responses API using direct chat completions. + + For more information, see: https://platform.openai.com/docs/api-reference/chat + """ + + id: str = "gpt-4o" + name: str = "OpenAIResponses" + provider: str = "OpenAI" + supports_structured_outputs: bool = True + + # API configuration + api_key: Optional[str] = None + organization: Optional[str] = None + base_url: Optional[Union[str, httpx.URL]] = None + timeout: Optional[float] = None + max_retries: Optional[int] = None + default_headers: Optional[Dict[str, str]] = None + default_query: Optional[Dict[str, str]] = None + http_client: Optional[httpx.Client] = None + client_params: Optional[Dict[str, Any]] = None + + # Response parameters + temperature: Optional[float] = None + top_p: Optional[float] = None + max_output_tokens: Optional[int] = None + response_format: Optional[Dict[str, str]] = None + metadata: Optional[Dict[str, Any]] = None + + # The role to map the message role to. + role_map = { + "system": "developer", + "user": "user", + "assistant": "assistant", + "tool": "tool", + } + + + # OpenAI clients + client: Optional[OpenAI] = None + async_client: Optional[AsyncOpenAI] = None + + # Internal parameters. Not used for API requests + # Whether to use the structured outputs with this Model. + structured_outputs: bool = False + + def _get_client_params(self) -> Dict[str, Any]: + """ + Get client parameters for API requests. + + Returns: + Dict[str, Any]: Client parameters + """ + import os + + # Fetch API key from env if not already set + if not self.api_key: + self.api_key = os.getenv("OPENAI_API_KEY") + if not self.api_key: + logger.error("OPENAI_API_KEY not set. Please set the OPENAI_API_KEY environment variable.") + + # Define base client params + base_params = { + "api_key": self.api_key, + "organization": self.organization, + "base_url": self.base_url, + "timeout": self.timeout, + "max_retries": self.max_retries, + "default_headers": self.default_headers, + "default_query": self.default_query, + } + + # Create client_params dict with non-None values + client_params = {k: v for k, v in base_params.items() if v is not None} + + # Add additional client params if provided + if self.client_params: + client_params.update(self.client_params) + + return client_params + + def get_client(self) -> OpenAI: + """ + Returns an OpenAI client. + + Returns: + OpenAI: An instance of the OpenAI client. + """ + if self.client: + return self.client + + client_params: Dict[str, Any] = self._get_client_params() + if self.http_client is not None: + client_params["http_client"] = self.http_client + + self.client = OpenAI(**client_params) + return self.client + + def get_async_client(self) -> AsyncOpenAI: + """ + Returns an asynchronous OpenAI client. + + Returns: + AsyncOpenAI: An instance of the asynchronous OpenAI client. + """ + if self.async_client: + return self.async_client + + client_params: Dict[str, Any] = self._get_client_params() + if self.http_client: + client_params["http_client"] = self.http_client + else: + # Create a new async HTTP client with custom limits + client_params["http_client"] = httpx.AsyncClient( + limits=httpx.Limits(max_connections=1000, max_keepalive_connections=100) + ) + + self.async_client = AsyncOpenAI(**client_params) + return self.async_client + + + @property + def request_kwargs(self) -> Dict[str, Any]: + """ + Returns keyword arguments for API requests. + + Returns: + Dict[str, Any]: A dictionary of keyword arguments for API requests. + """ + # Define base request parameters + base_params = { + "temperature": self.temperature, + "top_p": self.top_p, + "max_output_tokens": self.max_output_tokens, + # "response_format": self.response_format, # TODO: Add this back in + "metadata": self.metadata, + } + + if self.response_format is not None: + if self.structured_outputs: + base_params["text"] = { + "format": { + "type": "json_schema", + "name": self.response_model.__name__, + "schema": self.response_model.model_json_schema(), + "strict": True, + } + } + else: + # JSON mode + base_params["text"] = {"format": { "type": "json_object" }} + + # Filter out None values + request_params = {k: v for k, v in base_params.items() if v is not None} + + # Add tools + if self._tools is not None and len(self._tools) > 0: + request_params["tools"] = self._tools + + if self.tool_choice is not None: + request_params["tool_choice"] = self.tool_choice + + return request_params + + def _format_message(self, message: Message) -> Dict[str, Any]: + """ + Format a message into the format expected by OpenAI. + + Args: + message (Message): The message to format. + + Returns: + Dict[str, Any]: The formatted message. + """ + message_dict: Dict[str, Any] = { + "role": self.role_map[message.role], + "content": message.content, + } + message_dict = {k: v for k, v in message_dict.items() if v is not None} + + # Ignore non-string message content + # because we assume that the images/audio are already added to the message + if message.images is not None and len(message.images) > 0: + # Ignore non-string message content + # because we assume that the images/audio are already added to the message + if isinstance(message.content, str): + message_dict["content"] = [{"type": "input_text", "text": message.content}] + if message.images is not None: + message_dict["content"].extend(images_to_message(images=message.images)) + + # TODO: File support + + if message.audio is not None: + logger.warning("Audio input is currently unsupported.") + + if message.videos is not None: + logger.warning("Video input is currently unsupported.") + + # OpenAI expects the tool_calls to be None if empty, not an empty list + if message.tool_calls is not None and len(message.tool_calls) == 0: + message_dict["tool_calls"] = None + + # Manually add the content field even if it is None + if message.content is None: + message_dict["content"] = None + + return message_dict + + def invoke(self, messages: List[Message]) -> Response: + """ + Send a request to the OpenAI Responses API. + + Args: + messages (List[Message]): A list of messages to send to the model. + + Returns: + Response: The response from the API. + """ + try: + + return self.get_client().responses.create( + model=self.id, + input=[self._format_message(m) for m in messages], # type: ignore + **self.request_kwargs, + ) + except RateLimitError as e: + logger.error(f"Rate limit error from OpenAI API: {e}") + error_message = e.response.json().get("error", {}) + error_message = ( + error_message.get("message", "Unknown model error") + if isinstance(error_message, dict) + else error_message + ) + raise ModelProviderError( + message=error_message, + status_code=e.response.status_code, + model_name=self.name, + model_id=self.id, + ) from e + except APIConnectionError as e: + logger.error(f"API connection error from OpenAI API: {e}") + raise ModelProviderError(message=str(e), model_name=self.name, model_id=self.id) from e + except APIStatusError as e: + logger.error(f"API status error from OpenAI API: {e}") + error_message = e.response.json().get("error", {}) + error_message = ( + error_message.get("message", "Unknown model error") + if isinstance(error_message, dict) + else error_message + ) + raise ModelProviderError( + message=error_message, + status_code=e.response.status_code, + model_name=self.name, + model_id=self.id, + ) from e + except Exception as e: + logger.error(f"Error from OpenAI API: {e}") + raise ModelProviderError(message=str(e), model_name=self.name, model_id=self.id) from e + + async def ainvoke(self, messages: List[Message]) -> Response: + """ + Sends an asynchronous request to the OpenAI Responses API. + + Args: + messages (List[Message]): A list of messages to send to the model. + + Returns: + Response: The response from the API. + """ + try: + + return await self.get_async_client().responses.create( + model=self.id, + input=[self._format_message(m) for m in messages], # type: ignore + **self.request_kwargs, + ) + except RateLimitError as e: + logger.error(f"Rate limit error from OpenAI API: {e}") + error_message = e.response.json().get("error", {}) + error_message = ( + error_message.get("message", "Unknown model error") + if isinstance(error_message, dict) + else error_message + ) + raise ModelProviderError( + message=error_message, + status_code=e.response.status_code, + model_name=self.name, + model_id=self.id, + ) from e + except APIConnectionError as e: + logger.error(f"API connection error from OpenAI API: {e}") + raise ModelProviderError(message=str(e), model_name=self.name, model_id=self.id) from e + except APIStatusError as e: + logger.error(f"API status error from OpenAI API: {e}") + error_message = e.response.json().get("error", {}) + error_message = ( + error_message.get("message", "Unknown model error") + if isinstance(error_message, dict) + else error_message + ) + raise ModelProviderError( + message=error_message, + status_code=e.response.status_code, + model_name=self.name, + model_id=self.id, + ) from e + except Exception as e: + logger.error(f"Error from OpenAI API: {e}") + raise ModelProviderError(message=str(e), model_name=self.name, model_id=self.id) from e + + def invoke_stream(self, messages: List[Message]) -> Iterator[ResponseStreamEvent]: + """ + Send a streaming request to the OpenAI Responses API. + + Args: + messages (List[Message]): A list of messages to send to the model. + + Returns: + Iterator[ResponseStreamEvent]: An iterator of response stream events. + """ + try: + yield from self.get_client().responses.create( + model=self.id, + input=[self._format_message(m) for m in messages], # type: ignore + stream=True, + **self.request_kwargs, + ) # type: ignore + except RateLimitError as e: + logger.error(f"Rate limit error from OpenAI API: {e}") + error_message = e.response.json().get("error", {}) + error_message = ( + error_message.get("message", "Unknown model error") + if isinstance(error_message, dict) + else error_message + ) + raise ModelProviderError( + message=error_message, + status_code=e.response.status_code, + model_name=self.name, + model_id=self.id, + ) from e + except APIConnectionError as e: + logger.error(f"API connection error from OpenAI API: {e}") + raise ModelProviderError(message=str(e), model_name=self.name, model_id=self.id) from e + except APIStatusError as e: + logger.error(f"API status error from OpenAI API: {e}") + error_message = e.response.json().get("error", {}) + error_message = ( + error_message.get("message", "Unknown model error") + if isinstance(error_message, dict) + else error_message + ) + raise ModelProviderError( + message=error_message, + status_code=e.response.status_code, + model_name=self.name, + model_id=self.id, + ) from e + except Exception as e: + logger.error(f"Error from OpenAI API: {e}") + raise ModelProviderError(message=str(e), model_name=self.name, model_id=self.id) from e + + async def ainvoke_stream(self, messages: List[Message]) -> AsyncIterator[ResponseStreamEvent]: + """ + Sends an asynchronous streaming request to the OpenAI Responses API. + + Args: + messages (List[Message]): A list of messages to send to the model. + + Returns: + Any: An asynchronous iterator of chat completion chunks. + """ + try: + async_stream = await self.get_async_client().responses.create( + model=self.id, + input=[self._format_message(m) for m in messages], # type: ignore + stream=True, + **self.request_kwargs, + ) + async for chunk in async_stream: + yield chunk + except RateLimitError as e: + logger.error(f"Rate limit error from OpenAI API: {e}") + error_message = e.response.json().get("error", {}) + error_message = ( + error_message.get("message", "Unknown model error") + if isinstance(error_message, dict) + else error_message + ) + raise ModelProviderError( + message=error_message, + status_code=e.response.status_code, + model_name=self.name, + model_id=self.id, + ) from e + except APIConnectionError as e: + logger.error(f"API connection error from OpenAI API: {e}") + raise ModelProviderError(message=str(e), model_name=self.name, model_id=self.id) from e + except APIStatusError as e: + logger.error(f"API status error from OpenAI API: {e}") + error_message = e.response.json().get("error", {}) + error_message = ( + error_message.get("message", "Unknown model error") + if isinstance(error_message, dict) + else error_message + ) + raise ModelProviderError( + message=error_message, + status_code=e.response.status_code, + model_name=self.name, + model_id=self.id, + ) from e + except Exception as e: + logger.error(f"Error from OpenAI API: {e}") + raise ModelProviderError(message=str(e), model_name=self.name, model_id=self.id) from e + + def parse_provider_response(self, response: Response) -> ModelResponse: + """ + Parse the OpenAI response into a ModelResponse. + + Args: + response: Response from invoke() method + + Returns: + ModelResponse: Parsed response data + """ + model_response = ModelResponse() + + if hasattr(response, "error") and response.error: + raise ModelProviderError( + message=response.error.get("message", "Unknown model error"), + model_name=self.name, + model_id=self.id, + ) + + # Get response message + response_message = response.choices[0].message + + # Parse structured outputs if enabled + try: + if ( + self.response_format is not None + and self.structured_outputs + and issubclass(self.response_format, BaseModel) + ): + parsed_object = response_message.parsed # type: ignore + if parsed_object is not None: + model_response.parsed = parsed_object + except Exception as e: + logger.warning(f"Error retrieving structured outputs: {e}") + + # Add role + if response_message.role is not None: + model_response.role = response_message.role + + # Add content + if response_message.content is not None: + model_response.content = response_message.content + + # Add tool calls + if response_message.tool_calls is not None and len(response_message.tool_calls) > 0: + try: + model_response.tool_calls = [t.model_dump() for t in response_message.tool_calls] + except Exception as e: + logger.warning(f"Error processing tool calls: {e}") + + # Add audio transcript to content if available + response_audio: Optional[ChatCompletionAudio] = response_message.audio + if response_audio and response_audio.transcript and not model_response.content: + model_response.content = response_audio.transcript + + # Add audio if present + if hasattr(response_message, "audio") and response_message.audio is not None: + # If the audio output modality is requested, we can extract an audio response + try: + if isinstance(response_message.audio, dict): + model_response.audio = AudioResponse( + id=response_message.audio.get("id"), + content=response_message.audio.get("data"), + expires_at=response_message.audio.get("expires_at"), + transcript=response_message.audio.get("transcript"), + ) + else: + model_response.audio = AudioResponse( + id=response_message.audio.id, + content=response_message.audio.data, + expires_at=response_message.audio.expires_at, + transcript=response_message.audio.transcript, + ) + except Exception as e: + logger.warning(f"Error processing audio: {e}") + + if hasattr(response_message, "reasoning_content") and response_message.reasoning_content is not None: + model_response.reasoning_content = response_message.reasoning_content + + if response.usage is not None: + model_response.response_usage = response.usage + + return model_response + + def parse_provider_response_delta(self, response_delta: ChatCompletionChunk) -> ModelResponse: + """ + Parse the OpenAI streaming response into a ModelResponse. + + Args: + response_delta: Raw response chunk from OpenAI + + Returns: + ProviderResponse: Iterator of parsed response data + """ + model_response = ModelResponse() + if response_delta.choices and len(response_delta.choices) > 0: + delta: ChoiceDelta = response_delta.choices[0].delta + + # Add content + if delta.content is not None: + model_response.content = delta.content + + # Add tool calls + if delta.tool_calls is not None: + model_response.tool_calls = delta.tool_calls # type: ignore + + # Add audio if present + if hasattr(delta, "audio") and delta.audio is not None: + try: + if isinstance(delta.audio, dict): + model_response.audio = AudioResponse( + id=delta.audio.get("id"), + content=delta.audio.get("data"), + expires_at=delta.audio.get("expires_at"), + transcript=delta.audio.get("transcript"), + sample_rate=24000, + mime_type="pcm16", + ) + else: + model_response.audio = AudioResponse( + id=delta.audio.id, + content=delta.audio.data, + expires_at=delta.audio.expires_at, + transcript=delta.audio.transcript, + sample_rate=24000, + mime_type="pcm16", + ) + except Exception as e: + logger.warning(f"Error processing audio: {e}") + + # Add usage metrics if present + if response_delta.usage is not None: + model_response.response_usage = response_delta.usage + + return model_response diff --git a/libs/agno/agno/utils/openai_responses.py b/libs/agno/agno/utils/openai_responses.py new file mode 100644 index 0000000000..9e0da75d43 --- /dev/null +++ b/libs/agno/agno/utils/openai_responses.py @@ -0,0 +1,94 @@ +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Union + +from agno.media import Audio, Image +from agno.utils.log import logger + + + +def _process_bytes_image(image: bytes) -> Dict[str, Any]: + """Process bytes image data.""" + import base64 + + base64_image = base64.b64encode(image).decode("utf-8") + image_url = f"data:image/jpeg;base64,{base64_image}" + return {"type": "input_image", "image_url": image_url} + + +def _process_image_path(image_path: Union[Path, str]) -> Dict[str, Any]: + """Process image ( file path).""" + # Process local file image + import base64 + import mimetypes + + path = image_path if isinstance(image_path, Path) else Path(image_path) + if not path.exists(): + raise FileNotFoundError(f"Image file not found: {image_path}") + + mime_type = mimetypes.guess_type(image_path)[0] or "image/jpeg" + with open(path, "rb") as image_file: + base64_image = base64.b64encode(image_file.read()).decode("utf-8") + image_url = f"data:{mime_type};base64,{base64_image}" + return {"type": "input_image", "image_url": image_url} + + +def _process_image_url(image_url: str) -> Dict[str, Any]: + """Process image (base64 or URL).""" + + if image_url.startswith("data:image") or image_url.startswith(("http://", "https://")): + return {"type": "input_image", "image_url": image_url} + else: + raise ValueError("Image URL must start with 'data:image' or 'http(s)://'.") + + +def _process_image(image: Image) -> Optional[Dict[str, Any]]: + """Process an image based on the format.""" + + if image.url is not None: + image_payload = _process_image_url(image.url) + + elif image.filepath is not None: + image_payload = _process_image_path(image.filepath) + + elif image.content is not None: + image_payload = _process_bytes_image(image.content) + + else: + logger.warning(f"Unsupported image format: {image}") + return None + + if image.detail: + image_payload["image_url"]["detail"] = image.detail + + return image_payload + + +def images_to_message(images: Sequence[Image]) -> List[Dict[str, Any]]: + """ + Add images to a message for the model. By default, we use the OpenAI image format but other Models + can override this method to use a different image format. + + Args: + images: Sequence of images in various formats: + - str: base64 encoded image, URL, or file path + - Dict: pre-formatted image data + - bytes: raw image data + + Returns: + Message content with images added in the format expected by the model + """ + + # Create a default message content with text + image_messages: List[Dict[str, Any]] = [] + + # Add images to the message content + for image in images: + try: + image_data = _process_image(image) + if image_data: + image_messages.append(image_data) + except Exception as e: + logger.error(f"Failed to process image: {str(e)}") + continue + + return image_messages From 620254f6c3bd858bc9ee2492b7e4eb0db9c159fa Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Wed, 12 Mar 2025 11:43:29 +0200 Subject: [PATCH 02/13] Update --- cookbook/models/openai/{ => chat}/README.md | 0 cookbook/models/openai/{ => chat}/__init__.py | 0 .../models/openai/{ => chat}/async_basic.py | 0 .../openai/{ => chat}/async_basic_stream.py | 0 .../openai/{ => chat}/async_tool_use.py | 0 .../openai/{ => chat}/audio_input_agent.py | 0 .../audio_input_and_output_multi_turn.py | 0 .../audio_input_local_file_upload.py | 0 .../openai/{ => chat}/audio_output_agent.py | 0 .../openai/{ => chat}/audio_output_stream.py | 0 cookbook/models/openai/{ => chat}/basic.py | 0 .../models/openai/{ => chat}/basic_stream.py | 0 .../openai/{ => chat}/generate_images.py | 0 .../models/openai/{ => chat}/image_agent.py | 0 .../openai/{ => chat}/image_agent_bytes.py | 0 .../{ => chat}/image_agent_with_memory.py | 0 .../models/openai/{ => chat}/knowledge.py | 0 cookbook/models/openai/{ => chat}/memory.py | 0 cookbook/models/openai/{ => chat}/metrics.py | 0 .../openai/{ => chat}/reasoning/__init__.py | 0 .../models/openai/{ => chat}/reasoning/o1.py | 0 .../{ => chat}/reasoning/o3_mini_stream.py | 0 .../{ => chat}/reasoning/o3_mini_tool_use.py | 0 .../{ => chat}/reasoning/reasoning_effort.py | 0 cookbook/models/openai/{ => chat}/storage.py | 0 .../openai/{ => chat}/structured_output.py | 0 cookbook/models/openai/{ => chat}/tool_use.py | 0 .../openai/{ => chat}/tool_use_stream.py | 0 .../models/openai/responses/async_basic.py | 13 + .../openai/responses/async_basic_stream.py | 15 + cookbook/models/openai/responses/basic.py | 13 + .../models/openai/responses/basic_stream.py | 13 + libs/agno/agno/models/openai/chat.py | 2 +- libs/agno/agno/models/openai/responses.py | 283 +++++++++++------- 34 files changed, 228 insertions(+), 111 deletions(-) rename cookbook/models/openai/{ => chat}/README.md (100%) rename cookbook/models/openai/{ => chat}/__init__.py (100%) rename cookbook/models/openai/{ => chat}/async_basic.py (100%) rename cookbook/models/openai/{ => chat}/async_basic_stream.py (100%) rename cookbook/models/openai/{ => chat}/async_tool_use.py (100%) rename cookbook/models/openai/{ => chat}/audio_input_agent.py (100%) rename cookbook/models/openai/{ => chat}/audio_input_and_output_multi_turn.py (100%) rename cookbook/models/openai/{ => chat}/audio_input_local_file_upload.py (100%) rename cookbook/models/openai/{ => chat}/audio_output_agent.py (100%) rename cookbook/models/openai/{ => chat}/audio_output_stream.py (100%) rename cookbook/models/openai/{ => chat}/basic.py (100%) rename cookbook/models/openai/{ => chat}/basic_stream.py (100%) rename cookbook/models/openai/{ => chat}/generate_images.py (100%) rename cookbook/models/openai/{ => chat}/image_agent.py (100%) rename cookbook/models/openai/{ => chat}/image_agent_bytes.py (100%) rename cookbook/models/openai/{ => chat}/image_agent_with_memory.py (100%) rename cookbook/models/openai/{ => chat}/knowledge.py (100%) rename cookbook/models/openai/{ => chat}/memory.py (100%) rename cookbook/models/openai/{ => chat}/metrics.py (100%) rename cookbook/models/openai/{ => chat}/reasoning/__init__.py (100%) rename cookbook/models/openai/{ => chat}/reasoning/o1.py (100%) rename cookbook/models/openai/{ => chat}/reasoning/o3_mini_stream.py (100%) rename cookbook/models/openai/{ => chat}/reasoning/o3_mini_tool_use.py (100%) rename cookbook/models/openai/{ => chat}/reasoning/reasoning_effort.py (100%) rename cookbook/models/openai/{ => chat}/storage.py (100%) rename cookbook/models/openai/{ => chat}/structured_output.py (100%) rename cookbook/models/openai/{ => chat}/tool_use.py (100%) rename cookbook/models/openai/{ => chat}/tool_use_stream.py (100%) create mode 100644 cookbook/models/openai/responses/async_basic.py create mode 100644 cookbook/models/openai/responses/async_basic_stream.py create mode 100644 cookbook/models/openai/responses/basic.py create mode 100644 cookbook/models/openai/responses/basic_stream.py diff --git a/cookbook/models/openai/README.md b/cookbook/models/openai/chat/README.md similarity index 100% rename from cookbook/models/openai/README.md rename to cookbook/models/openai/chat/README.md diff --git a/cookbook/models/openai/__init__.py b/cookbook/models/openai/chat/__init__.py similarity index 100% rename from cookbook/models/openai/__init__.py rename to cookbook/models/openai/chat/__init__.py diff --git a/cookbook/models/openai/async_basic.py b/cookbook/models/openai/chat/async_basic.py similarity index 100% rename from cookbook/models/openai/async_basic.py rename to cookbook/models/openai/chat/async_basic.py diff --git a/cookbook/models/openai/async_basic_stream.py b/cookbook/models/openai/chat/async_basic_stream.py similarity index 100% rename from cookbook/models/openai/async_basic_stream.py rename to cookbook/models/openai/chat/async_basic_stream.py diff --git a/cookbook/models/openai/async_tool_use.py b/cookbook/models/openai/chat/async_tool_use.py similarity index 100% rename from cookbook/models/openai/async_tool_use.py rename to cookbook/models/openai/chat/async_tool_use.py diff --git a/cookbook/models/openai/audio_input_agent.py b/cookbook/models/openai/chat/audio_input_agent.py similarity index 100% rename from cookbook/models/openai/audio_input_agent.py rename to cookbook/models/openai/chat/audio_input_agent.py diff --git a/cookbook/models/openai/audio_input_and_output_multi_turn.py b/cookbook/models/openai/chat/audio_input_and_output_multi_turn.py similarity index 100% rename from cookbook/models/openai/audio_input_and_output_multi_turn.py rename to cookbook/models/openai/chat/audio_input_and_output_multi_turn.py diff --git a/cookbook/models/openai/audio_input_local_file_upload.py b/cookbook/models/openai/chat/audio_input_local_file_upload.py similarity index 100% rename from cookbook/models/openai/audio_input_local_file_upload.py rename to cookbook/models/openai/chat/audio_input_local_file_upload.py diff --git a/cookbook/models/openai/audio_output_agent.py b/cookbook/models/openai/chat/audio_output_agent.py similarity index 100% rename from cookbook/models/openai/audio_output_agent.py rename to cookbook/models/openai/chat/audio_output_agent.py diff --git a/cookbook/models/openai/audio_output_stream.py b/cookbook/models/openai/chat/audio_output_stream.py similarity index 100% rename from cookbook/models/openai/audio_output_stream.py rename to cookbook/models/openai/chat/audio_output_stream.py diff --git a/cookbook/models/openai/basic.py b/cookbook/models/openai/chat/basic.py similarity index 100% rename from cookbook/models/openai/basic.py rename to cookbook/models/openai/chat/basic.py diff --git a/cookbook/models/openai/basic_stream.py b/cookbook/models/openai/chat/basic_stream.py similarity index 100% rename from cookbook/models/openai/basic_stream.py rename to cookbook/models/openai/chat/basic_stream.py diff --git a/cookbook/models/openai/generate_images.py b/cookbook/models/openai/chat/generate_images.py similarity index 100% rename from cookbook/models/openai/generate_images.py rename to cookbook/models/openai/chat/generate_images.py diff --git a/cookbook/models/openai/image_agent.py b/cookbook/models/openai/chat/image_agent.py similarity index 100% rename from cookbook/models/openai/image_agent.py rename to cookbook/models/openai/chat/image_agent.py diff --git a/cookbook/models/openai/image_agent_bytes.py b/cookbook/models/openai/chat/image_agent_bytes.py similarity index 100% rename from cookbook/models/openai/image_agent_bytes.py rename to cookbook/models/openai/chat/image_agent_bytes.py diff --git a/cookbook/models/openai/image_agent_with_memory.py b/cookbook/models/openai/chat/image_agent_with_memory.py similarity index 100% rename from cookbook/models/openai/image_agent_with_memory.py rename to cookbook/models/openai/chat/image_agent_with_memory.py diff --git a/cookbook/models/openai/knowledge.py b/cookbook/models/openai/chat/knowledge.py similarity index 100% rename from cookbook/models/openai/knowledge.py rename to cookbook/models/openai/chat/knowledge.py diff --git a/cookbook/models/openai/memory.py b/cookbook/models/openai/chat/memory.py similarity index 100% rename from cookbook/models/openai/memory.py rename to cookbook/models/openai/chat/memory.py diff --git a/cookbook/models/openai/metrics.py b/cookbook/models/openai/chat/metrics.py similarity index 100% rename from cookbook/models/openai/metrics.py rename to cookbook/models/openai/chat/metrics.py diff --git a/cookbook/models/openai/reasoning/__init__.py b/cookbook/models/openai/chat/reasoning/__init__.py similarity index 100% rename from cookbook/models/openai/reasoning/__init__.py rename to cookbook/models/openai/chat/reasoning/__init__.py diff --git a/cookbook/models/openai/reasoning/o1.py b/cookbook/models/openai/chat/reasoning/o1.py similarity index 100% rename from cookbook/models/openai/reasoning/o1.py rename to cookbook/models/openai/chat/reasoning/o1.py diff --git a/cookbook/models/openai/reasoning/o3_mini_stream.py b/cookbook/models/openai/chat/reasoning/o3_mini_stream.py similarity index 100% rename from cookbook/models/openai/reasoning/o3_mini_stream.py rename to cookbook/models/openai/chat/reasoning/o3_mini_stream.py diff --git a/cookbook/models/openai/reasoning/o3_mini_tool_use.py b/cookbook/models/openai/chat/reasoning/o3_mini_tool_use.py similarity index 100% rename from cookbook/models/openai/reasoning/o3_mini_tool_use.py rename to cookbook/models/openai/chat/reasoning/o3_mini_tool_use.py diff --git a/cookbook/models/openai/reasoning/reasoning_effort.py b/cookbook/models/openai/chat/reasoning/reasoning_effort.py similarity index 100% rename from cookbook/models/openai/reasoning/reasoning_effort.py rename to cookbook/models/openai/chat/reasoning/reasoning_effort.py diff --git a/cookbook/models/openai/storage.py b/cookbook/models/openai/chat/storage.py similarity index 100% rename from cookbook/models/openai/storage.py rename to cookbook/models/openai/chat/storage.py diff --git a/cookbook/models/openai/structured_output.py b/cookbook/models/openai/chat/structured_output.py similarity index 100% rename from cookbook/models/openai/structured_output.py rename to cookbook/models/openai/chat/structured_output.py diff --git a/cookbook/models/openai/tool_use.py b/cookbook/models/openai/chat/tool_use.py similarity index 100% rename from cookbook/models/openai/tool_use.py rename to cookbook/models/openai/chat/tool_use.py diff --git a/cookbook/models/openai/tool_use_stream.py b/cookbook/models/openai/chat/tool_use_stream.py similarity index 100% rename from cookbook/models/openai/tool_use_stream.py rename to cookbook/models/openai/chat/tool_use_stream.py diff --git a/cookbook/models/openai/responses/async_basic.py b/cookbook/models/openai/responses/async_basic.py new file mode 100644 index 0000000000..dfdd69b982 --- /dev/null +++ b/cookbook/models/openai/responses/async_basic.py @@ -0,0 +1,13 @@ +import asyncio + +from agno.agent import Agent, RunResponse # noqa +from agno.models.openai import OpenAIResponses + +agent = Agent(model=OpenAIResponses(id="gpt-4o"), markdown=True) + +# Get the response in a variable +# run: RunResponse = agent.run("Share a 2 sentence horror story") +# print(run.content) + +# Print the response in the terminal +asyncio.run(agent.aprint_response("Share a 2 sentence horror story")) diff --git a/cookbook/models/openai/responses/async_basic_stream.py b/cookbook/models/openai/responses/async_basic_stream.py new file mode 100644 index 0000000000..0bb2509a69 --- /dev/null +++ b/cookbook/models/openai/responses/async_basic_stream.py @@ -0,0 +1,15 @@ +import asyncio +from typing import Iterator # noqa + +from agno.agent import Agent, RunResponse # noqa +from agno.models.openai import OpenAIResponses + +agent = Agent(model=OpenAIResponses(id="gpt-4o"), markdown=True) + +# Get the response in a variable +# run_response: Iterator[RunResponse] = agent.run("Share a 2 sentence horror story", stream=True) +# for chunk in run_response: +# print(chunk.content) + +# Print the response in the terminal +asyncio.run(agent.aprint_response("Share a 2 sentence horror story", stream=True)) diff --git a/cookbook/models/openai/responses/basic.py b/cookbook/models/openai/responses/basic.py new file mode 100644 index 0000000000..8ba45ee820 --- /dev/null +++ b/cookbook/models/openai/responses/basic.py @@ -0,0 +1,13 @@ +from agno.agent import Agent, RunResponse # noqa +from agno.models.openai import OpenAIResponses + +agent = Agent(model=OpenAIResponses(id="gpt-4o"), markdown=True) + +# Get the response in a variable +# run: RunResponse = agent.run("Share a 2 sentence horror story") +# print(run.content) + +# Print the response in the terminal +agent.print_response("Share a 2 sentence horror story") + +agent.run_response.metrics diff --git a/cookbook/models/openai/responses/basic_stream.py b/cookbook/models/openai/responses/basic_stream.py new file mode 100644 index 0000000000..ef81a62715 --- /dev/null +++ b/cookbook/models/openai/responses/basic_stream.py @@ -0,0 +1,13 @@ +from typing import Iterator # noqa +from agno.agent import Agent, RunResponse # noqa +from agno.models.openai import OpenAIResponses + +agent = Agent(model=OpenAIResponses(id="gpt-4o"), markdown=True) + +# Get the response in a variable +# run_response: Iterator[RunResponse] = agent.run("Share a 2 sentence horror story", stream=True) +# for chunk in run_response: +# print(chunk.content) + +# Print the response in the terminal +agent.print_response("Share a 2 sentence horror story", stream=True) diff --git a/libs/agno/agno/models/openai/chat.py b/libs/agno/agno/models/openai/chat.py index e0d94d5871..f8d68b672b 100644 --- a/libs/agno/agno/models/openai/chat.py +++ b/libs/agno/agno/models/openai/chat.py @@ -654,7 +654,7 @@ def parse_provider_response_delta(self, response_delta: ChatCompletionChunk) -> response_delta: Raw response chunk from OpenAI Returns: - ProviderResponse: Iterator of parsed response data + ModelResponse: Parsed response data """ model_response = ModelResponse() if response_delta.choices and len(response_delta.choices) > 0: diff --git a/libs/agno/agno/models/openai/responses.py b/libs/agno/agno/models/openai/responses.py index e88ba51f14..e64303fa70 100644 --- a/libs/agno/agno/models/openai/responses.py +++ b/libs/agno/agno/models/openai/responses.py @@ -1,12 +1,13 @@ from dataclasses import dataclass, field from typing import Any, AsyncGenerator, AsyncIterator, Dict, Iterator, List, Optional, Tuple, Union import asyncio +from agno.media import AudioResponse from agno.models.response import ModelResponse from agno.utils.openai_responses import images_to_message import httpx from agno.exceptions import ModelProviderError -from agno.models.base import Model +from agno.models.base import MessageData, Model from agno.models.message import Message from agno.utils.log import logger @@ -67,6 +68,13 @@ class OpenAIResponses(Model): max_output_tokens: Optional[int] = None response_format: Optional[Dict[str, str]] = None metadata: Optional[Dict[str, Any]] = None + store: Optional[bool] = None + reasoning_effort: Optional[str] = None + reasoning_summary: Optional[str] = None + + # Built-in tools + web_search: bool = False + # The role to map the message role to. role_map = { @@ -173,9 +181,14 @@ def request_kwargs(self) -> Dict[str, Any]: "temperature": self.temperature, "top_p": self.top_p, "max_output_tokens": self.max_output_tokens, - # "response_format": self.response_format, # TODO: Add this back in "metadata": self.metadata, + "store": self.store, } + if self.reasoning_effort is not None or self.reasoning_summary is not None: + base_params["reasoning"] = { + "effort": self.reasoning_effort, + "summary": self.reasoning_summary, + } if self.response_format is not None: if self.structured_outputs: @@ -194,12 +207,17 @@ def request_kwargs(self) -> Dict[str, Any]: # Filter out None values request_params = {k: v for k, v in base_params.items() if v is not None} + if self.web_search: + request_params.setdefault("tools", []) + request_params["tools"].append({"type": "web_search_preview"}) + # Add tools if self._tools is not None and len(self._tools) > 0: - request_params["tools"] = self._tools + request_params.setdefault("tools", []) + request_params["tools"].extend(self._tools) - if self.tool_choice is not None: - request_params["tool_choice"] = self.tool_choice + if self.tool_choice is not None: + request_params["tool_choice"] = self.tool_choice return request_params @@ -241,10 +259,6 @@ def _format_message(self, message: Message) -> Dict[str, Any]: if message.tool_calls is not None and len(message.tool_calls) == 0: message_dict["tool_calls"] = None - # Manually add the content field even if it is None - if message.content is None: - message_dict["content"] = None - return message_dict def invoke(self, messages: List[Message]) -> Response: @@ -469,126 +483,175 @@ def parse_provider_response(self, response: Response) -> ModelResponse: """ model_response = ModelResponse() - if hasattr(response, "error") and response.error: + if response.error: raise ModelProviderError( message=response.error.get("message", "Unknown model error"), model_name=self.name, model_id=self.id, ) - - # Get response message - response_message = response.choices[0].message - - # Parse structured outputs if enabled - try: - if ( - self.response_format is not None - and self.structured_outputs - and issubclass(self.response_format, BaseModel) - ): - parsed_object = response_message.parsed # type: ignore - if parsed_object is not None: - model_response.parsed = parsed_object - except Exception as e: - logger.warning(f"Error retrieving structured outputs: {e}") - + # Add role - if response_message.role is not None: - model_response.role = response_message.role - - # Add content - if response_message.content is not None: - model_response.content = response_message.content - - # Add tool calls - if response_message.tool_calls is not None and len(response_message.tool_calls) > 0: - try: - model_response.tool_calls = [t.model_dump() for t in response_message.tool_calls] - except Exception as e: - logger.warning(f"Error processing tool calls: {e}") - - # Add audio transcript to content if available - response_audio: Optional[ChatCompletionAudio] = response_message.audio - if response_audio and response_audio.transcript and not model_response.content: - model_response.content = response_audio.transcript - - # Add audio if present - if hasattr(response_message, "audio") and response_message.audio is not None: - # If the audio output modality is requested, we can extract an audio response - try: - if isinstance(response_message.audio, dict): - model_response.audio = AudioResponse( - id=response_message.audio.get("id"), - content=response_message.audio.get("data"), - expires_at=response_message.audio.get("expires_at"), - transcript=response_message.audio.get("transcript"), - ) - else: - model_response.audio = AudioResponse( - id=response_message.audio.id, - content=response_message.audio.data, - expires_at=response_message.audio.expires_at, - transcript=response_message.audio.transcript, - ) - except Exception as e: - logger.warning(f"Error processing audio: {e}") - - if hasattr(response_message, "reasoning_content") and response_message.reasoning_content is not None: - model_response.reasoning_content = response_message.reasoning_content + model_response.role = "assistant" + + for output in response.output: + if output.type == "message": + if model_response.content is None: + model_response.content = "" + # TODO: Support citations/annotations + for content in output.content: + if content.type == "output_text": + model_response.content += content.text + elif output.type == "function_call": + if model_response.tool_calls is None: + model_response.tool_calls = [] + model_response.tool_calls.append( + { + "id": output.id, + "name": output.name, + "arguments": output.arguments, + } + ) + + # i.e. we asked for reasoning, so we need to add the reasoning content + if self.reasoning_effort: + model_response.reasoning_content = response.output_text if response.usage is not None: model_response.response_usage = response.usage return model_response - def parse_provider_response_delta(self, response_delta: ChatCompletionChunk) -> ModelResponse: + + + # Override base method + @staticmethod + def parse_tool_calls(tool_calls_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """ - Parse the OpenAI streaming response into a ModelResponse. + Build tool calls from streamed tool call data. Args: - response_delta: Raw response chunk from OpenAI + tool_calls_data (List[Dict[str, Any]]): The tool call data to build from. Returns: - ProviderResponse: Iterator of parsed response data + List[Dict[str, Any]]: The built tool calls. + """ + tool_calls: List[Dict[str, Any]] = [] + for _tool_call in tool_calls_data: + _index = _tool_call.index or 0 + _tool_call_id = _tool_call.id + _tool_call_type = _tool_call.type + _function_name = _tool_call.function.name if _tool_call.function else None + _function_arguments = _tool_call.function.arguments if _tool_call.function else None + + if len(tool_calls) <= _index: + tool_calls.extend([{}] * (_index - len(tool_calls) + 1)) + tool_call_entry = tool_calls[_index] + if not tool_call_entry: + tool_call_entry["id"] = _tool_call_id + tool_call_entry["type"] = _tool_call_type + tool_call_entry["function"] = { + "name": _function_name or "", + "arguments": _function_arguments or "", + } + else: + if _function_name: + tool_call_entry["function"]["name"] += _function_name + if _function_arguments: + tool_call_entry["function"]["arguments"] += _function_arguments + if _tool_call_id: + tool_call_entry["id"] = _tool_call_id + if _tool_call_type: + tool_call_entry["type"] = _tool_call_type + return tool_calls + + + def _process_stream_response( + self, + stream_event: ResponseStreamEvent, + assistant_message: Message, + stream_data: MessageData, + tool_use: Dict[str, Any], + ) -> Tuple[Optional[ModelResponse], Dict[str, Any]]: + """ + Common handler for processing stream responses from Cohere. + + Args: + response: The streamed response from Cohere + assistant_message: The assistant message being built + stream_data: Data accumulated during streaming + tool_use: Current tool use data being built + + Returns: + Tuple containing the ModelResponse to yield and updated tool_use dict """ model_response = ModelResponse() - if response_delta.choices and len(response_delta.choices) > 0: - delta: ChoiceDelta = response_delta.choices[0].delta + + if stream_event.type == "response.created": + # Update metrics + if not assistant_message.metrics.time_to_first_token: + assistant_message.metrics.set_time_to_first_token() + + elif stream_event.type == "response.output_text.delta": # Add content - if delta.content is not None: - model_response.content = delta.content - - # Add tool calls - if delta.tool_calls is not None: - model_response.tool_calls = delta.tool_calls # type: ignore - - # Add audio if present - if hasattr(delta, "audio") and delta.audio is not None: - try: - if isinstance(delta.audio, dict): - model_response.audio = AudioResponse( - id=delta.audio.get("id"), - content=delta.audio.get("data"), - expires_at=delta.audio.get("expires_at"), - transcript=delta.audio.get("transcript"), - sample_rate=24000, - mime_type="pcm16", - ) - else: - model_response.audio = AudioResponse( - id=delta.audio.id, - content=delta.audio.data, - expires_at=delta.audio.expires_at, - transcript=delta.audio.transcript, - sample_rate=24000, - mime_type="pcm16", - ) - except Exception as e: - logger.warning(f"Error processing audio: {e}") - - # Add usage metrics if present - if response_delta.usage is not None: - model_response.response_usage = response_delta.usage + model_response.content = stream_event.delta + stream_data.response_content += stream_event.delta + + elif stream_event.type == "response.output_item.added": + item = stream_event.item + if item.type == "function_call": + tool_use = { + "id": item.id, + "name": item.name, + "arguments": item.arguments, + } - return model_response + elif stream_event.type == "response.function_call_arguments.delta": + tool_use["arguments"] += stream_event.delta + + elif stream_event.type == "response.output_item.done": + if assistant_message.tool_calls is None: + assistant_message.tool_calls = [] + assistant_message.tool_calls.append(tool_use) + tool_use = {} + + elif stream_event.type == "response.completed": + # Add usage metrics if present + if stream_event.response.usage is not None: + model_response.response_usage = stream_event.response.usage + + self._add_usage_metrics_to_assistant_message( + assistant_message=assistant_message, + response_usage=model_response.response_usage, + ) + + return model_response, tool_use + + def process_response_stream( + self, messages: List[Message], assistant_message: Message, stream_data: MessageData + ) -> Iterator[ModelResponse]: + """Process the synchronous response stream.""" + tool_use: Dict[str, Any] = {} + + for stream_event in self.invoke_stream(messages=messages): + model_response, tool_use = self._process_stream_response( + stream_event=stream_event, assistant_message=assistant_message, stream_data=stream_data, tool_use=tool_use + ) + if model_response is not None: + yield model_response + + async def aprocess_response_stream( + self, messages: List[Message], assistant_message: Message, stream_data: MessageData + ) -> AsyncIterator[ModelResponse]: + """Process the asynchronous response stream.""" + tool_use: Dict[str, Any] = {} + + async for stream_event in self.ainvoke_stream(messages=messages): + model_response, tool_use = self._process_stream_response( + stream_event=stream_event, assistant_message=assistant_message, stream_data=stream_data, tool_use=tool_use + ) + if model_response is not None: + yield model_response + + def parse_provider_response_delta(self, response: Any) -> ModelResponse: # type: ignore + pass \ No newline at end of file From 0e2174d8c47c5c786d1275cce23e14a2af9e2b4d Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Wed, 12 Mar 2025 14:03:15 +0200 Subject: [PATCH 03/13] Update --- cookbook/examples/apps/geobuddy/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbook/examples/apps/geobuddy/app.py b/cookbook/examples/apps/geobuddy/app.py index 5675d765fd..5e238d3efd 100644 --- a/cookbook/examples/apps/geobuddy/app.py +++ b/cookbook/examples/apps/geobuddy/app.py @@ -4,7 +4,7 @@ import streamlit as st from PIL import Image -from cookbook.use_cases.apps.geobuddy.geography_buddy import analyze_image +from geography_buddy import analyze_image # Streamlit App Configuration st.set_page_config( From 33ea7ca62c46cda64ac1a7d96e5e2b43e189bf82 Mon Sep 17 00:00:00 2001 From: Ayush <97244608+Ayush0054@users.noreply.github.com> Date: Wed, 12 Mar 2025 17:33:54 +0530 Subject: [PATCH 04/13] cookbook examples (#2374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description - **Summary of changes**: Describe the key changes in this PR and their purpose. - **Related issues**: Mention if this PR fixes or is connected to any issues. - **Motivation and context**: Explain the reason for the changes and the problem they solve. - **Environment or dependencies**: Specify any changes in dependencies or environment configurations required for this update. - **Impact on metrics**: (If applicable) Describe changes in any metrics or performance benchmarks. Fixes # (issue) --- ## Type of change Please check the options that are relevant: - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Model update (Addition or modification of models) - [ ] Other (please describe): --- ## Checklist - [ ] Adherence to standards: Code complies with Agno’s style guidelines and best practices. - [ ] Formatting and validation: You have run `./scripts/format.sh` and `./scripts/validate.sh` to ensure code is formatted and linted. - [ ] Self-review completed: A thorough review has been performed by the contributor(s). - [ ] Documentation: Docstrings and comments have been added or updated for any complex logic. - [ ] Examples and guides: Relevant cookbook examples have been included or updated (if applicable). - [ ] Tested in a clean environment: Changes have been tested in a clean environment to confirm expected behavior. - [ ] Tests (optional): Tests have been added or updated to cover any new or changed functionality. --- ## Additional Notes Include any deployment notes, performance implications, security considerations, or other relevant information (e.g., screenshots or logs if applicable). --- .../models/openai/responses/async_tool_use.py | 15 +++++ .../openai/responses/generate_images.py | 20 +++++++ .../models/openai/responses/image_agent.py | 20 +++++++ .../openai/responses/image_agent_bytes.py | 31 ++++++++++ .../responses/image_agent_with_memory.py | 23 ++++++++ cookbook/models/openai/responses/knowledge.py | 21 +++++++ cookbook/models/openai/responses/memory.py | 56 +++++++++++++++++++ cookbook/models/openai/responses/storage.py | 17 ++++++ .../openai/responses/structured_output.py | 54 ++++++++++++++++++ cookbook/models/openai/responses/tool_use.py | 13 +++++ .../openai/responses/tool_use_stream.py | 13 +++++ 11 files changed, 283 insertions(+) create mode 100644 cookbook/models/openai/responses/async_tool_use.py create mode 100644 cookbook/models/openai/responses/generate_images.py create mode 100644 cookbook/models/openai/responses/image_agent.py create mode 100644 cookbook/models/openai/responses/image_agent_bytes.py create mode 100644 cookbook/models/openai/responses/image_agent_with_memory.py create mode 100644 cookbook/models/openai/responses/knowledge.py create mode 100644 cookbook/models/openai/responses/memory.py create mode 100644 cookbook/models/openai/responses/storage.py create mode 100644 cookbook/models/openai/responses/structured_output.py create mode 100644 cookbook/models/openai/responses/tool_use.py create mode 100644 cookbook/models/openai/responses/tool_use_stream.py diff --git a/cookbook/models/openai/responses/async_tool_use.py b/cookbook/models/openai/responses/async_tool_use.py new file mode 100644 index 0000000000..ba7afd7116 --- /dev/null +++ b/cookbook/models/openai/responses/async_tool_use.py @@ -0,0 +1,15 @@ +"""Run `pip install duckduckgo-search` to install dependencies.""" + +import asyncio + +from agno.agent import Agent +from agno.models.openai import OpenAIResponses +from agno.tools.duckduckgo import DuckDuckGoTools + +agent = Agent( + model=OpenAIResponses(id="gpt-4o"), + tools=[DuckDuckGoTools()], + show_tool_calls=True, + markdown=True, +) +asyncio.run(agent.aprint_response("Whats happening in France?", stream=True)) diff --git a/cookbook/models/openai/responses/generate_images.py b/cookbook/models/openai/responses/generate_images.py new file mode 100644 index 0000000000..675a4e446b --- /dev/null +++ b/cookbook/models/openai/responses/generate_images.py @@ -0,0 +1,20 @@ +from agno.agent import Agent +from agno.models.openai import OpenAIResponses +from agno.tools.dalle import DalleTools + +image_agent = Agent( + model=OpenAIResponses(id="gpt-4o"), + tools=[DalleTools()], + description="You are an AI agent that can generate images using DALL-E.", + instructions="When the user asks you to create an image, use the `create_image` tool to create the image.", + markdown=True, + show_tool_calls=True, +) + +image_agent.print_response("Generate an image of a white siamese cat") + +images = image_agent.get_images() +if images and isinstance(images, list): + for image_response in images: + image_url = image_response.url + print(image_url) diff --git a/cookbook/models/openai/responses/image_agent.py b/cookbook/models/openai/responses/image_agent.py new file mode 100644 index 0000000000..406bb5cded --- /dev/null +++ b/cookbook/models/openai/responses/image_agent.py @@ -0,0 +1,20 @@ +from agno.agent import Agent +from agno.media import Image +from agno.models.openai import OpenAIResponses +from agno.tools.duckduckgo import DuckDuckGoTools + +agent = Agent( + model=OpenAIResponses(id="gpt-4o"), + tools=[DuckDuckGoTools()], + markdown=True, +) + +agent.print_response( + "Tell me about this image and give me the latest news about it.", + images=[ + Image( + url="https://upload.wikimedia.org/wikipedia/commons/0/0c/GoldenGateBridge-001.jpg" + ) + ], + stream=True, +) diff --git a/cookbook/models/openai/responses/image_agent_bytes.py b/cookbook/models/openai/responses/image_agent_bytes.py new file mode 100644 index 0000000000..a1593ab6e9 --- /dev/null +++ b/cookbook/models/openai/responses/image_agent_bytes.py @@ -0,0 +1,31 @@ +from pathlib import Path + +from agno.agent import Agent +from agno.media import Image +from agno.models.openai import OpenAIResponses +from agno.tools.duckduckgo import DuckDuckGoTools +from agno.utils.media import download_image + +agent = Agent( + model=OpenAIResponses(id="gpt-4o"), + tools=[DuckDuckGoTools()], + markdown=True, +) + +image_path = Path(__file__).parent.joinpath("sample.jpg") + +download_image( + url="https://upload.wikimedia.org/wikipedia/commons/0/0c/GoldenGateBridge-001.jpg", + output_path=str(image_path), +) + +# Read the image file content as bytes +image_bytes = image_path.read_bytes() + +agent.print_response( + "Tell me about this image and give me the latest news about it.", + images=[ + Image(content=image_bytes), + ], + stream=True, +) diff --git a/cookbook/models/openai/responses/image_agent_with_memory.py b/cookbook/models/openai/responses/image_agent_with_memory.py new file mode 100644 index 0000000000..682309c894 --- /dev/null +++ b/cookbook/models/openai/responses/image_agent_with_memory.py @@ -0,0 +1,23 @@ +from agno.agent import Agent +from agno.media import Image +from agno.models.openai import OpenAIResponses +from agno.tools.duckduckgo import DuckDuckGoTools + +agent = Agent( + model=OpenAIResponses(id="gpt-4o"), + tools=[DuckDuckGoTools()], + markdown=True, + add_history_to_messages=True, + num_history_responses=3, +) + +agent.print_response( + "Tell me about this image and give me the latest news about it.", + images=[ + Image( + url="https://upload.wikimedia.org/wikipedia/commons/0/0c/GoldenGateBridge-001.jpg" + ) + ], +) + +agent.print_response("Tell me where I can get more images?") diff --git a/cookbook/models/openai/responses/knowledge.py b/cookbook/models/openai/responses/knowledge.py new file mode 100644 index 0000000000..4649a1e449 --- /dev/null +++ b/cookbook/models/openai/responses/knowledge.py @@ -0,0 +1,21 @@ +"""Run `pip install duckduckgo-search sqlalchemy pgvector pypdf openai` to install dependencies.""" + +from agno.agent import Agent +from agno.knowledge.pdf_url import PDFUrlKnowledgeBase +from agno.models.openai import OpenAIResponses +from agno.vectordb.pgvector import PgVector + +db_url = "postgresql+psycopg://ai:ai@localhost:5532/ai" + +knowledge_base = PDFUrlKnowledgeBase( + urls=["https://agno-public.s3.amazonaws.com/recipes/ThaiRecipes.pdf"], + vector_db=PgVector(table_name="recipes", db_url=db_url), +) +knowledge_base.load(recreate=True) # Comment out after first run + +agent = Agent( + model=OpenAIResponses(id="gpt-4o"), + knowledge=knowledge_base, + show_tool_calls=True, +) +agent.print_response("How to make Thai curry?", markdown=True) diff --git a/cookbook/models/openai/responses/memory.py b/cookbook/models/openai/responses/memory.py new file mode 100644 index 0000000000..9f2c8e3ad5 --- /dev/null +++ b/cookbook/models/openai/responses/memory.py @@ -0,0 +1,56 @@ +""" +This recipe shows how to use personalized memories and summaries in an agent. +Steps: +1. Run: `./cookbook/scripts/run_pgvector.sh` to start a postgres container with pgvector +2. Run: `pip install openai sqlalchemy 'psycopg[binary]' pgvector` to install the dependencies +3. Run: `python cookbook/agents/personalized_memories_and_summaries.py` to run the agent +""" + +from agno.agent import Agent, AgentMemory +from agno.memory.db.postgres import PgMemoryDb +from agno.models.openai import OpenAIResponses +from agno.storage.agent.postgres import PostgresAgentStorage +from rich.pretty import pprint + +db_url = "postgresql+psycopg://ai:ai@localhost:5532/ai" +agent = Agent( + model=OpenAIResponses(id="gpt-4o"), + # Store the memories and summary in a database + memory=AgentMemory( + db=PgMemoryDb(table_name="agent_memory", db_url=db_url), + create_user_memories=True, + create_session_summary=True, + ), + # Store agent sessions in a database + storage=PostgresAgentStorage( + table_name="personalized_agent_sessions", db_url=db_url + ), + # Show debug logs so, you can see the memory being created + # debug_mode=True, +) + +# -*- Share personal information +agent.print_response("My name is john billings?", stream=True) +# -*- Print memories +pprint(agent.memory.memories) +# -*- Print summary +pprint(agent.memory.summary) + +# -*- Share personal information +agent.print_response("I live in nyc?", stream=True) +# -*- Print memories +pprint(agent.memory.memories) +# -*- Print summary +pprint(agent.memory.summary) + +# -*- Share personal information +agent.print_response("I'm going to a concert tomorrow?", stream=True) +# -*- Print memories +pprint(agent.memory.memories) +# -*- Print summary +pprint(agent.memory.summary) + +# Ask about the conversation +agent.print_response( + "What have we been talking about, do you know my name?", stream=True +) diff --git a/cookbook/models/openai/responses/storage.py b/cookbook/models/openai/responses/storage.py new file mode 100644 index 0000000000..5eb117966e --- /dev/null +++ b/cookbook/models/openai/responses/storage.py @@ -0,0 +1,17 @@ +"""Run `pip install duckduckgo-search sqlalchemy openai` to install dependencies.""" + +from agno.agent import Agent +from agno.models.openai import OpenAIResponses +from agno.storage.agent.postgres import PostgresAgentStorage +from agno.tools.duckduckgo import DuckDuckGoTools + +db_url = "postgresql+psycopg://ai:ai@localhost:5532/ai" + +agent = Agent( + model=OpenAIResponses(id="gpt-4o"), + storage=PostgresAgentStorage(table_name="agent_sessions", db_url=db_url), + tools=[DuckDuckGoTools()], + add_history_to_messages=True, +) +agent.print_response("How many people live in Canada?") +agent.print_response("What is their national anthem called?") diff --git a/cookbook/models/openai/responses/structured_output.py b/cookbook/models/openai/responses/structured_output.py new file mode 100644 index 0000000000..ec73938ec2 --- /dev/null +++ b/cookbook/models/openai/responses/structured_output.py @@ -0,0 +1,54 @@ +from typing import List + +from agno.agent import Agent, RunResponse # noqa +from agno.models.openai import OpenAIChat +from pydantic import BaseModel, Field +from rich.pretty import pprint + +from agno.models.openai.responses import OpenAIResponses # noqa + + +class MovieScript(BaseModel): + setting: str = Field( + ..., description="Provide a nice setting for a blockbuster movie." + ) + ending: str = Field( + ..., + description="Ending of the movie. If not available, provide a happy ending.", + ) + genre: str = Field( + ..., + description="Genre of the movie. If not available, select action, thriller or romantic comedy.", + ) + name: str = Field(..., description="Give a name to this movie") + characters: List[str] = Field(..., description="Name of characters for this movie.") + storyline: str = Field( + ..., description="3 sentence storyline for the movie. Make it exciting!" + ) + + +# Agent that uses JSON mode +json_mode_agent = Agent( + model=OpenAIResponses(id="gpt-4o" , response_format=MovieScript , structured_outputs=True), + description="You write movie scripts.", + # response_model=MovieScript, + structured_outputs=True, +) + +# Agent that uses structured outputs +structured_output_agent = Agent( + model=OpenAIResponses(id="gpt-4o-2024-08-06"), + description="You write movie scripts.", + # response_model=MovieScript, + # structured_outputs=True, +) + + +# Get the response in a variable +# json_mode_response: RunResponse = json_mode_agent.run("New York") +# pprint(json_mode_response.content) +# structured_output_response: RunResponse = structured_output_agent.run("New York") +# pprint(structured_output_response.content) + +json_mode_agent.print_response("New York") +structured_output_agent.print_response("New York") diff --git a/cookbook/models/openai/responses/tool_use.py b/cookbook/models/openai/responses/tool_use.py new file mode 100644 index 0000000000..cf3adc3f0b --- /dev/null +++ b/cookbook/models/openai/responses/tool_use.py @@ -0,0 +1,13 @@ +"""Run `pip install duckduckgo-search` to install dependencies.""" + +from agno.agent import Agent +from agno.models.openai import OpenAIResponses +from agno.tools.duckduckgo import DuckDuckGoTools + +agent = Agent( + model=OpenAIResponses(id="gpt-4o"), + tools=[DuckDuckGoTools()], + show_tool_calls=True, + markdown=True, +) +agent.print_response("Whats happening in France?") diff --git a/cookbook/models/openai/responses/tool_use_stream.py b/cookbook/models/openai/responses/tool_use_stream.py new file mode 100644 index 0000000000..02c62d1cd1 --- /dev/null +++ b/cookbook/models/openai/responses/tool_use_stream.py @@ -0,0 +1,13 @@ +"""Run `pip install duckduckgo-search` to install dependencies.""" + +from agno.agent import Agent +from agno.models.openai import OpenAIResponses +from agno.tools.duckduckgo import DuckDuckGoTools + +agent = Agent( + model=OpenAIResponses(id="gpt-4o"), + tools=[DuckDuckGoTools()], + show_tool_calls=True, + markdown=True, +) +agent.print_response("Whats happening in France?", stream=True) From 2acbbe6bfa8a6a7af8edb2e22d09509278fcb121 Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Wed, 12 Mar 2025 16:19:19 +0200 Subject: [PATCH 05/13] Update --- libs/agno/agno/agent/agent.py | 3 ++- libs/agno/agno/models/openai/responses.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/libs/agno/agno/agent/agent.py b/libs/agno/agno/agent/agent.py index ca47214418..d9f9989265 100644 --- a/libs/agno/agno/agent/agent.py +++ b/libs/agno/agno/agent/agent.py @@ -31,7 +31,6 @@ from agno.memory.agent import AgentMemory, AgentRun from agno.models.base import Model from agno.models.message import Message, MessageReferences -from agno.models.openai.like import OpenAILike from agno.models.response import ModelResponse, ModelResponseEvent from agno.reasoning.step import NextAction, ReasoningStep, ReasoningSteps from agno.run.messages import RunMessages @@ -2849,6 +2848,7 @@ def get_audio(self) -> Optional[List[AudioArtifact]]: ########################################################################### def reason(self, run_messages: RunMessages) -> Iterator[RunResponse]: + from agno.models.openai.like import OpenAILike # Yield a reasoning started event if self.stream_intermediate_steps: yield self.create_run_response(content="Reasoning started", event=RunEvent.reasoning_started) @@ -3030,6 +3030,7 @@ def reason(self, run_messages: RunMessages) -> Iterator[RunResponse]: ) async def areason(self, run_messages: RunMessages) -> Any: + from agno.models.openai.like import OpenAILike # Yield a reasoning started event if self.stream_intermediate_steps: yield self.create_run_response(content="Reasoning started", event=RunEvent.reasoning_started) diff --git a/libs/agno/agno/models/openai/responses.py b/libs/agno/agno/models/openai/responses.py index e64303fa70..5137a786ef 100644 --- a/libs/agno/agno/models/openai/responses.py +++ b/libs/agno/agno/models/openai/responses.py @@ -26,9 +26,9 @@ # Check version compatibility parsed_version = version.parse(openai_version) - if parsed_version.major == 0: + if parsed_version.major == 0 and parsed_version.minor < 66: import warnings - warnings.warn("OpenAI v1.x is recommended for the Responses API", UserWarning) + warnings.warn("OpenAI v1.66.0 or higher is recommended for the Responses API", UserWarning) except ImportError as e: # Handle different import error scenarios From 3ba0f7ad9acd8fb32275cd2d401bf30ac803e913 Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Wed, 12 Mar 2025 16:21:05 +0200 Subject: [PATCH 06/13] Merge --- .../openai/responses/generate_images.py | 20 ------------------- .../openai/responses/reasoning/__init__.py | 0 .../models/openai/responses/reasoning/o1.py | 8 ++++++++ .../responses/reasoning/o3_mini_stream.py | 7 +++++++ .../responses/reasoning/o3_mini_tool_use.py | 13 ++++++++++++ .../responses/reasoning/reasoning_effort.py | 13 ++++++++++++ 6 files changed, 41 insertions(+), 20 deletions(-) delete mode 100644 cookbook/models/openai/responses/generate_images.py create mode 100644 cookbook/models/openai/responses/reasoning/__init__.py create mode 100644 cookbook/models/openai/responses/reasoning/o1.py create mode 100644 cookbook/models/openai/responses/reasoning/o3_mini_stream.py create mode 100644 cookbook/models/openai/responses/reasoning/o3_mini_tool_use.py create mode 100644 cookbook/models/openai/responses/reasoning/reasoning_effort.py diff --git a/cookbook/models/openai/responses/generate_images.py b/cookbook/models/openai/responses/generate_images.py deleted file mode 100644 index 675a4e446b..0000000000 --- a/cookbook/models/openai/responses/generate_images.py +++ /dev/null @@ -1,20 +0,0 @@ -from agno.agent import Agent -from agno.models.openai import OpenAIResponses -from agno.tools.dalle import DalleTools - -image_agent = Agent( - model=OpenAIResponses(id="gpt-4o"), - tools=[DalleTools()], - description="You are an AI agent that can generate images using DALL-E.", - instructions="When the user asks you to create an image, use the `create_image` tool to create the image.", - markdown=True, - show_tool_calls=True, -) - -image_agent.print_response("Generate an image of a white siamese cat") - -images = image_agent.get_images() -if images and isinstance(images, list): - for image_response in images: - image_url = image_response.url - print(image_url) diff --git a/cookbook/models/openai/responses/reasoning/__init__.py b/cookbook/models/openai/responses/reasoning/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cookbook/models/openai/responses/reasoning/o1.py b/cookbook/models/openai/responses/reasoning/o1.py new file mode 100644 index 0000000000..a07a866ca6 --- /dev/null +++ b/cookbook/models/openai/responses/reasoning/o1.py @@ -0,0 +1,8 @@ +from agno.agent import Agent, RunResponse # noqa +from agno.models.openai import OpenAIResponses + +# This will only work if you have access to the o1 model from OpenAI +agent = Agent(model=OpenAIResponses(id="o1")) + +# Print the response in the terminal +agent.print_response("What is the closest galaxy to milky way?") diff --git a/cookbook/models/openai/responses/reasoning/o3_mini_stream.py b/cookbook/models/openai/responses/reasoning/o3_mini_stream.py new file mode 100644 index 0000000000..9bae5eb5e0 --- /dev/null +++ b/cookbook/models/openai/responses/reasoning/o3_mini_stream.py @@ -0,0 +1,7 @@ +from agno.agent import Agent +from agno.models.openai import OpenAIResponses + +agent = Agent(model=OpenAIResponses(id="o3-mini")) + +# Print the response in the terminal +agent.print_response("What is the closest galaxy to milky way?", stream=True) diff --git a/cookbook/models/openai/responses/reasoning/o3_mini_tool_use.py b/cookbook/models/openai/responses/reasoning/o3_mini_tool_use.py new file mode 100644 index 0000000000..914c35637a --- /dev/null +++ b/cookbook/models/openai/responses/reasoning/o3_mini_tool_use.py @@ -0,0 +1,13 @@ +from agno.agent import Agent +from agno.models.openai import OpenAIResponses +from agno.tools.duckduckgo import DuckDuckGoTools + +agent = Agent( + model=OpenAIResponses(id="o3-mini"), + tools=[DuckDuckGoTools()], + show_tool_calls=True, + markdown=True, +) + +# Print the response in the terminal +agent.print_response("Write a report on the latest news on o3-mini?", stream=True) diff --git a/cookbook/models/openai/responses/reasoning/reasoning_effort.py b/cookbook/models/openai/responses/reasoning/reasoning_effort.py new file mode 100644 index 0000000000..1d09346a74 --- /dev/null +++ b/cookbook/models/openai/responses/reasoning/reasoning_effort.py @@ -0,0 +1,13 @@ +from agno.agent import Agent +from agno.models.openai import OpenAIResponses +from agno.tools.yfinance import YFinanceTools + +agent = Agent( + model=OpenAIResponses(id="o3-mini", reasoning_effort="high"), + tools=[YFinanceTools(enable_all=True)], + show_tool_calls=True, + markdown=True, +) + +# Print the response in the terminal +agent.print_response("Write a report on the NVDA, is it a good buy?", stream=True) From 6c388cde3bbf127891b4be2a9c9ab5bf54eb1c12 Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Wed, 12 Mar 2025 21:01:26 +0200 Subject: [PATCH 07/13] Fix tests --- .../models/openai/responses/image_agent.py | 4 +- .../openai/responses/image_agent_bytes.py | 4 +- .../responses/image_agent_with_memory.py | 4 +- .../openai/responses/reasoning/__init__.py | 0 .../models/openai/responses/reasoning/o1.py | 8 - .../responses/reasoning/o3_mini_stream.py | 7 - .../responses/reasoning/o3_mini_tool_use.py | 13 - ...asoning_effort.py => reasoning_o3_mini.py} | 0 .../openai/responses/structured_output.py | 8 +- libs/agno/agno/models/openai/responses.py | 196 +++++++------ .../models/openai/{ => chat}/test_basic.py | 0 .../openai/{ => chat}/test_multimodal.py | 0 .../models/openai/{ => chat}/test_tool_use.py | 0 .../models/openai/responses/__init__.py | 1 + .../models/openai/responses/test_basic.py | 223 +++++++++++++++ .../openai/responses/test_multimodal.py | 49 ++++ .../models/openai/responses/test_tool_use.py | 262 ++++++++++++++++++ 17 files changed, 653 insertions(+), 126 deletions(-) delete mode 100644 cookbook/models/openai/responses/reasoning/__init__.py delete mode 100644 cookbook/models/openai/responses/reasoning/o1.py delete mode 100644 cookbook/models/openai/responses/reasoning/o3_mini_stream.py delete mode 100644 cookbook/models/openai/responses/reasoning/o3_mini_tool_use.py rename cookbook/models/openai/responses/{reasoning/reasoning_effort.py => reasoning_o3_mini.py} (100%) rename libs/agno/tests/integration/models/openai/{ => chat}/test_basic.py (100%) rename libs/agno/tests/integration/models/openai/{ => chat}/test_multimodal.py (100%) rename libs/agno/tests/integration/models/openai/{ => chat}/test_tool_use.py (100%) create mode 100644 libs/agno/tests/integration/models/openai/responses/__init__.py create mode 100644 libs/agno/tests/integration/models/openai/responses/test_basic.py create mode 100644 libs/agno/tests/integration/models/openai/responses/test_multimodal.py create mode 100644 libs/agno/tests/integration/models/openai/responses/test_tool_use.py diff --git a/cookbook/models/openai/responses/image_agent.py b/cookbook/models/openai/responses/image_agent.py index 406bb5cded..6e03fd58b5 100644 --- a/cookbook/models/openai/responses/image_agent.py +++ b/cookbook/models/openai/responses/image_agent.py @@ -1,11 +1,11 @@ from agno.agent import Agent from agno.media import Image from agno.models.openai import OpenAIResponses -from agno.tools.duckduckgo import DuckDuckGoTools +from agno.tools.googlesearch import GoogleSearchTools agent = Agent( model=OpenAIResponses(id="gpt-4o"), - tools=[DuckDuckGoTools()], + tools=[GoogleSearchTools()], markdown=True, ) diff --git a/cookbook/models/openai/responses/image_agent_bytes.py b/cookbook/models/openai/responses/image_agent_bytes.py index a1593ab6e9..8a612efbc6 100644 --- a/cookbook/models/openai/responses/image_agent_bytes.py +++ b/cookbook/models/openai/responses/image_agent_bytes.py @@ -3,12 +3,12 @@ from agno.agent import Agent from agno.media import Image from agno.models.openai import OpenAIResponses -from agno.tools.duckduckgo import DuckDuckGoTools +from agno.tools.googlesearch import GoogleSearchTools from agno.utils.media import download_image agent = Agent( model=OpenAIResponses(id="gpt-4o"), - tools=[DuckDuckGoTools()], + tools=[GoogleSearchTools()], markdown=True, ) diff --git a/cookbook/models/openai/responses/image_agent_with_memory.py b/cookbook/models/openai/responses/image_agent_with_memory.py index 682309c894..d4da56d25d 100644 --- a/cookbook/models/openai/responses/image_agent_with_memory.py +++ b/cookbook/models/openai/responses/image_agent_with_memory.py @@ -1,11 +1,11 @@ from agno.agent import Agent from agno.media import Image from agno.models.openai import OpenAIResponses -from agno.tools.duckduckgo import DuckDuckGoTools +from agno.tools.googlesearch import GoogleSearchTools agent = Agent( model=OpenAIResponses(id="gpt-4o"), - tools=[DuckDuckGoTools()], + tools=[GoogleSearchTools()], markdown=True, add_history_to_messages=True, num_history_responses=3, diff --git a/cookbook/models/openai/responses/reasoning/__init__.py b/cookbook/models/openai/responses/reasoning/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/cookbook/models/openai/responses/reasoning/o1.py b/cookbook/models/openai/responses/reasoning/o1.py deleted file mode 100644 index a07a866ca6..0000000000 --- a/cookbook/models/openai/responses/reasoning/o1.py +++ /dev/null @@ -1,8 +0,0 @@ -from agno.agent import Agent, RunResponse # noqa -from agno.models.openai import OpenAIResponses - -# This will only work if you have access to the o1 model from OpenAI -agent = Agent(model=OpenAIResponses(id="o1")) - -# Print the response in the terminal -agent.print_response("What is the closest galaxy to milky way?") diff --git a/cookbook/models/openai/responses/reasoning/o3_mini_stream.py b/cookbook/models/openai/responses/reasoning/o3_mini_stream.py deleted file mode 100644 index 9bae5eb5e0..0000000000 --- a/cookbook/models/openai/responses/reasoning/o3_mini_stream.py +++ /dev/null @@ -1,7 +0,0 @@ -from agno.agent import Agent -from agno.models.openai import OpenAIResponses - -agent = Agent(model=OpenAIResponses(id="o3-mini")) - -# Print the response in the terminal -agent.print_response("What is the closest galaxy to milky way?", stream=True) diff --git a/cookbook/models/openai/responses/reasoning/o3_mini_tool_use.py b/cookbook/models/openai/responses/reasoning/o3_mini_tool_use.py deleted file mode 100644 index 914c35637a..0000000000 --- a/cookbook/models/openai/responses/reasoning/o3_mini_tool_use.py +++ /dev/null @@ -1,13 +0,0 @@ -from agno.agent import Agent -from agno.models.openai import OpenAIResponses -from agno.tools.duckduckgo import DuckDuckGoTools - -agent = Agent( - model=OpenAIResponses(id="o3-mini"), - tools=[DuckDuckGoTools()], - show_tool_calls=True, - markdown=True, -) - -# Print the response in the terminal -agent.print_response("Write a report on the latest news on o3-mini?", stream=True) diff --git a/cookbook/models/openai/responses/reasoning/reasoning_effort.py b/cookbook/models/openai/responses/reasoning_o3_mini.py similarity index 100% rename from cookbook/models/openai/responses/reasoning/reasoning_effort.py rename to cookbook/models/openai/responses/reasoning_o3_mini.py diff --git a/cookbook/models/openai/responses/structured_output.py b/cookbook/models/openai/responses/structured_output.py index ec73938ec2..8d7a0f182b 100644 --- a/cookbook/models/openai/responses/structured_output.py +++ b/cookbook/models/openai/responses/structured_output.py @@ -29,9 +29,9 @@ class MovieScript(BaseModel): # Agent that uses JSON mode json_mode_agent = Agent( - model=OpenAIResponses(id="gpt-4o" , response_format=MovieScript , structured_outputs=True), + model=OpenAIResponses(id="gpt-4o" ), description="You write movie scripts.", - # response_model=MovieScript, + response_model=MovieScript, structured_outputs=True, ) @@ -39,8 +39,8 @@ class MovieScript(BaseModel): structured_output_agent = Agent( model=OpenAIResponses(id="gpt-4o-2024-08-06"), description="You write movie scripts.", - # response_model=MovieScript, - # structured_outputs=True, + response_model=MovieScript, + structured_outputs=True, ) diff --git a/libs/agno/agno/models/openai/responses.py b/libs/agno/agno/models/openai/responses.py index 5137a786ef..79b926c3f7 100644 --- a/libs/agno/agno/models/openai/responses.py +++ b/libs/agno/agno/models/openai/responses.py @@ -70,7 +70,6 @@ class OpenAIResponses(Model): metadata: Optional[Dict[str, Any]] = None store: Optional[bool] = None reasoning_effort: Optional[str] = None - reasoning_summary: Optional[str] = None # Built-in tools web_search: bool = False @@ -184,19 +183,20 @@ def request_kwargs(self) -> Dict[str, Any]: "metadata": self.metadata, "store": self.store, } - if self.reasoning_effort is not None or self.reasoning_summary is not None: + if self.reasoning_effort is not None: base_params["reasoning"] = { "effort": self.reasoning_effort, - "summary": self.reasoning_summary, } if self.response_format is not None: if self.structured_outputs: + schema = self.response_format.model_json_schema() + schema["additionalProperties"] = False base_params["text"] = { "format": { "type": "json_schema", - "name": self.response_model.__name__, - "schema": self.response_model.model_json_schema(), + "name": self.response_format.__name__, + "schema": schema, "strict": True, } } @@ -212,16 +212,20 @@ def request_kwargs(self) -> Dict[str, Any]: request_params["tools"].append({"type": "web_search_preview"}) # Add tools - if self._tools is not None and len(self._tools) > 0: + if self._functions is not None and len(self._functions) > 0: request_params.setdefault("tools", []) - request_params["tools"].extend(self._tools) - + for function in self._functions.values(): + function_dict = function.to_dict() + for prop in function_dict["parameters"]["properties"].values(): + if isinstance(prop["type"], list): + prop["type"] = prop["type"][0] + request_params["tools"].append({"type": "function", **function_dict}) if self.tool_choice is not None: request_params["tool_choice"] = self.tool_choice return request_params - def _format_message(self, message: Message) -> Dict[str, Any]: + def _format_messages(self, messages: List[Message]) -> List[Dict[str, Any]]: """ Format a message into the format expected by OpenAI. @@ -231,35 +235,56 @@ def _format_message(self, message: Message) -> Dict[str, Any]: Returns: Dict[str, Any]: The formatted message. """ - message_dict: Dict[str, Any] = { - "role": self.role_map[message.role], - "content": message.content, - } - message_dict = {k: v for k, v in message_dict.items() if v is not None} + formatted_messages: List[Dict[str, Any]] = [] + for message in messages: + + if message.role in ["user", "system"]: + message_dict: Dict[str, Any] = { + "role": self.role_map[message.role], + "content": message.content, + } + message_dict = {k: v for k, v in message_dict.items() if v is not None} - # Ignore non-string message content - # because we assume that the images/audio are already added to the message - if message.images is not None and len(message.images) > 0: - # Ignore non-string message content - # because we assume that the images/audio are already added to the message - if isinstance(message.content, str): - message_dict["content"] = [{"type": "input_text", "text": message.content}] - if message.images is not None: - message_dict["content"].extend(images_to_message(images=message.images)) + # Ignore non-string message content + # because we assume that the images/audio are already added to the message + if message.images is not None and len(message.images) > 0: + # Ignore non-string message content + # because we assume that the images/audio are already added to the message + if isinstance(message.content, str): + message_dict["content"] = [{"type": "input_text", "text": message.content}] + if message.images is not None: + message_dict["content"].extend(images_to_message(images=message.images)) - # TODO: File support + # TODO: File support - if message.audio is not None: - logger.warning("Audio input is currently unsupported.") + if message.audio is not None: + logger.warning("Audio input is currently unsupported.") - if message.videos is not None: - logger.warning("Video input is currently unsupported.") + if message.videos is not None: + logger.warning("Video input is currently unsupported.") - # OpenAI expects the tool_calls to be None if empty, not an empty list - if message.tool_calls is not None and len(message.tool_calls) == 0: - message_dict["tool_calls"] = None + formatted_messages.append(message_dict) - return message_dict + else: + # OpenAI expects the tool_calls to be None if empty, not an empty list + if message.tool_calls is not None and len(message.tool_calls) > 0: + for tool_call in message.tool_calls: + formatted_messages.append({ + "type": "function_call", + "id": tool_call["id"], + "call_id": tool_call["call_id"], + "name": tool_call["function"]["name"], + "arguments": tool_call["function"]["arguments"], + "status": "completed" + }) + + if message.role == "tool": + formatted_messages.append({ + "type": "function_call_output", + "call_id": message.tool_call_id, + "output": message.content + }) + return formatted_messages def invoke(self, messages: List[Message]) -> Response: """ @@ -275,7 +300,7 @@ def invoke(self, messages: List[Message]) -> Response: return self.get_client().responses.create( model=self.id, - input=[self._format_message(m) for m in messages], # type: ignore + input=self._format_messages(messages), # type: ignore **self.request_kwargs, ) except RateLimitError as e: @@ -327,7 +352,7 @@ async def ainvoke(self, messages: List[Message]) -> Response: return await self.get_async_client().responses.create( model=self.id, - input=[self._format_message(m) for m in messages], # type: ignore + input=self._format_messages(messages), # type: ignore **self.request_kwargs, ) except RateLimitError as e: @@ -376,9 +401,10 @@ def invoke_stream(self, messages: List[Message]) -> Iterator[ResponseStreamEvent Iterator[ResponseStreamEvent]: An iterator of response stream events. """ try: + yield from self.get_client().responses.create( model=self.id, - input=[self._format_message(m) for m in messages], # type: ignore + input=self._format_messages(messages), # type: ignore stream=True, **self.request_kwargs, ) # type: ignore @@ -430,7 +456,7 @@ async def ainvoke_stream(self, messages: List[Message]) -> AsyncIterator[Respons try: async_stream = await self.get_async_client().responses.create( model=self.id, - input=[self._format_message(m) for m in messages], # type: ignore + input=self._format_messages(messages), # type: ignore stream=True, **self.request_kwargs, ) @@ -470,6 +496,22 @@ async def ainvoke_stream(self, messages: List[Message]) -> AsyncIterator[Respons except Exception as e: logger.error(f"Error from OpenAI API: {e}") raise ModelProviderError(message=str(e), model_name=self.name, model_id=self.id) from e + + def format_function_call_results( + self, messages: List[Message], function_call_results: List[Message], tool_call_ids: List[str] + ) -> None: + """ + Handle the results of function calls. + + Args: + messages (List[Message]): The list of conversation messages. + function_call_results (List[Message]): The results of the function calls. + tool_ids (List[str]): The tool ids. + """ + if len(function_call_results) > 0: + for _fc_message_index, _fc_message in enumerate(function_call_results): + _fc_message.tool_call_id = tool_call_ids[_fc_message_index] + messages.append(_fc_message) def parse_provider_response(self, response: Response) -> ModelResponse: """ @@ -492,7 +534,6 @@ def parse_provider_response(self, response: Response) -> ModelResponse: # Add role model_response.role = "assistant" - for output in response.output: if output.type == "message": if model_response.content is None: @@ -507,11 +548,19 @@ def parse_provider_response(self, response: Response) -> ModelResponse: model_response.tool_calls.append( { "id": output.id, - "name": output.name, - "arguments": output.arguments, + "call_id": output.call_id, + "type": "function", + "function": { + "name": output.name, + "arguments": output.arguments, + } } ) + model_response.extra = model_response.extra or {} + model_response.extra.setdefault("tool_call_ids", []).append(output.call_id) + + # i.e. we asked for reasoning, so we need to add the reasoning content if self.reasoning_effort: model_response.reasoning_content = response.output_text @@ -522,49 +571,6 @@ def parse_provider_response(self, response: Response) -> ModelResponse: return model_response - - # Override base method - @staticmethod - def parse_tool_calls(tool_calls_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Build tool calls from streamed tool call data. - - Args: - tool_calls_data (List[Dict[str, Any]]): The tool call data to build from. - - Returns: - List[Dict[str, Any]]: The built tool calls. - """ - tool_calls: List[Dict[str, Any]] = [] - for _tool_call in tool_calls_data: - _index = _tool_call.index or 0 - _tool_call_id = _tool_call.id - _tool_call_type = _tool_call.type - _function_name = _tool_call.function.name if _tool_call.function else None - _function_arguments = _tool_call.function.arguments if _tool_call.function else None - - if len(tool_calls) <= _index: - tool_calls.extend([{}] * (_index - len(tool_calls) + 1)) - tool_call_entry = tool_calls[_index] - if not tool_call_entry: - tool_call_entry["id"] = _tool_call_id - tool_call_entry["type"] = _tool_call_type - tool_call_entry["function"] = { - "name": _function_name or "", - "arguments": _function_arguments or "", - } - else: - if _function_name: - tool_call_entry["function"]["name"] += _function_name - if _function_arguments: - tool_call_entry["function"]["arguments"] += _function_arguments - if _tool_call_id: - tool_call_entry["id"] = _tool_call_id - if _tool_call_type: - tool_call_entry["type"] = _tool_call_type - return tool_calls - - def _process_stream_response( self, stream_event: ResponseStreamEvent, @@ -584,7 +590,7 @@ def _process_stream_response( Returns: Tuple containing the ModelResponse to yield and updated tool_use dict """ - model_response = ModelResponse() + model_response = None if stream_event.type == "response.created": # Update metrics @@ -592,30 +598,44 @@ def _process_stream_response( assistant_message.metrics.set_time_to_first_token() elif stream_event.type == "response.output_text.delta": - + model_response = ModelResponse() # Add content model_response.content = stream_event.delta stream_data.response_content += stream_event.delta + if self.reasoning_effort: + model_response.reasoning_content = stream_event.delta + stream_data.response_thinking += stream_event.delta + elif stream_event.type == "response.output_item.added": item = stream_event.item if item.type == "function_call": tool_use = { "id": item.id, - "name": item.name, - "arguments": item.arguments, + "call_id": item.call_id, + "type": "function", + "function": { + "name": item.name, + "arguments": item.arguments, + } } elif stream_event.type == "response.function_call_arguments.delta": - tool_use["arguments"] += stream_event.delta + tool_use["function"]["arguments"] += stream_event.delta - elif stream_event.type == "response.output_item.done": + elif stream_event.type == "response.output_item.done" and tool_use: + model_response = ModelResponse() + model_response.tool_calls = tool_use if assistant_message.tool_calls is None: assistant_message.tool_calls = [] assistant_message.tool_calls.append(tool_use) + + stream_data.extra = stream_data.extra or {} + stream_data.extra.setdefault("tool_call_ids", []).append(tool_use["call_id"]) tool_use = {} elif stream_event.type == "response.completed": + model_response = ModelResponse() # Add usage metrics if present if stream_event.response.usage is not None: model_response.response_usage = stream_event.response.usage diff --git a/libs/agno/tests/integration/models/openai/test_basic.py b/libs/agno/tests/integration/models/openai/chat/test_basic.py similarity index 100% rename from libs/agno/tests/integration/models/openai/test_basic.py rename to libs/agno/tests/integration/models/openai/chat/test_basic.py diff --git a/libs/agno/tests/integration/models/openai/test_multimodal.py b/libs/agno/tests/integration/models/openai/chat/test_multimodal.py similarity index 100% rename from libs/agno/tests/integration/models/openai/test_multimodal.py rename to libs/agno/tests/integration/models/openai/chat/test_multimodal.py diff --git a/libs/agno/tests/integration/models/openai/test_tool_use.py b/libs/agno/tests/integration/models/openai/chat/test_tool_use.py similarity index 100% rename from libs/agno/tests/integration/models/openai/test_tool_use.py rename to libs/agno/tests/integration/models/openai/chat/test_tool_use.py diff --git a/libs/agno/tests/integration/models/openai/responses/__init__.py b/libs/agno/tests/integration/models/openai/responses/__init__.py new file mode 100644 index 0000000000..26677d3ea1 --- /dev/null +++ b/libs/agno/tests/integration/models/openai/responses/__init__.py @@ -0,0 +1 @@ +"""Integration tests for OpenAI Responses API.""" \ No newline at end of file diff --git a/libs/agno/tests/integration/models/openai/responses/test_basic.py b/libs/agno/tests/integration/models/openai/responses/test_basic.py new file mode 100644 index 0000000000..a1583d3cb8 --- /dev/null +++ b/libs/agno/tests/integration/models/openai/responses/test_basic.py @@ -0,0 +1,223 @@ +import pytest +from pydantic import BaseModel, Field + +from agno.agent import Agent, RunResponse # noqa +from agno.exceptions import ModelProviderError +from agno.memory import AgentMemory +from agno.memory.classifier import MemoryClassifier +from agno.memory.db.sqlite import SqliteMemoryDb +from agno.memory.manager import MemoryManager +from agno.memory.summarizer import MemorySummarizer +from agno.models.openai import OpenAIResponses +from agno.storage.agent.sqlite import SqliteAgentStorage +from agno.tools.duckduckgo import DuckDuckGoTools + + +def _assert_metrics(response: RunResponse): + """ + Assert that the response metrics are valid and consistent. + + Args: + response: The RunResponse to validate metrics for + """ + input_tokens = response.metrics.get("input_tokens", []) + output_tokens = response.metrics.get("output_tokens", []) + total_tokens = response.metrics.get("total_tokens", []) + + assert sum(input_tokens) > 0 + assert sum(output_tokens) > 0 + assert sum(total_tokens) > 0 + assert sum(total_tokens) == sum(input_tokens) + sum(output_tokens) + + +def test_basic(): + """Test basic functionality of the OpenAIResponses model.""" + agent = Agent(model=OpenAIResponses(id="gpt-4o-mini"), markdown=True, telemetry=False, monitoring=False) + + # Run a simple query + response: RunResponse = agent.run("Share a 2 sentence horror story") + + assert response.content is not None + assert len(response.messages) == 3 + assert [m.role for m in response.messages] == ["system", "user", "assistant"] + + _assert_metrics(response) + + +def test_basic_stream(): + """Test basic streaming functionality of the OpenAIResponses model.""" + agent = Agent(model=OpenAIResponses(id="gpt-4o-mini"), markdown=True, telemetry=False, monitoring=False) + + response_stream = agent.run("Share a 2 sentence horror story", stream=True) + + # Verify it's an iterator + assert hasattr(response_stream, "__iter__") + + responses = list(response_stream) + assert len(responses) > 0 + for response in responses: + assert isinstance(response, RunResponse) + assert response.content is not None + + _assert_metrics(agent.run_response) + + +@pytest.mark.asyncio +async def test_async_basic(): + """Test basic async functionality of the OpenAIResponses model.""" + agent = Agent(model=OpenAIResponses(id="gpt-4o-mini"), markdown=True, telemetry=False, monitoring=False) + + response = await agent.arun("Share a 2 sentence horror story") + + assert response.content is not None + assert len(response.messages) == 3 + assert [m.role for m in response.messages] == ["system", "user", "assistant"] + _assert_metrics(response) + + +@pytest.mark.asyncio +async def test_async_basic_stream(): + """Test basic async streaming functionality of the OpenAIResponses model.""" + agent = Agent(model=OpenAIResponses(id="gpt-4o-mini"), markdown=True, telemetry=False, monitoring=False) + + response_stream = await agent.arun("Share a 2 sentence horror story", stream=True) + + async for response in response_stream: + assert isinstance(response, RunResponse) + assert response.content is not None + _assert_metrics(agent.run_response) + + +def test_exception_handling(): + """Test proper error handling for invalid model IDs.""" + agent = Agent(model=OpenAIResponses(id="gpt-100"), markdown=True, telemetry=False, monitoring=False) + + with pytest.raises(ModelProviderError) as exc: + agent.run("Share a 2 sentence horror story") + + assert exc.value.model_name == "OpenAIResponses" + assert exc.value.model_id == "gpt-100" + assert exc.value.status_code == 400 + + +def test_with_memory(): + """Test that the model retains context from previous interactions.""" + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), + add_history_to_messages=True, + num_history_responses=5, + markdown=True, + telemetry=False, + monitoring=False, + ) + + # First interaction + response1 = agent.run("My name is John Smith") + assert response1.content is not None + + # Second interaction should remember the name + response2 = agent.run("What's my name?") + assert "John Smith" in response2.content + + # Verify memories were created + assert len(agent.memory.messages) == 5 + assert [m.role for m in agent.memory.messages] == ["system", "user", "assistant", "user", "assistant"] + + # Test metrics structure and types + _assert_metrics(response2) + + +def test_structured_output(): + """Test structured output with Pydantic models.""" + class MovieScript(BaseModel): + title: str = Field(..., description="Movie title") + genre: str = Field(..., description="Movie genre") + plot: str = Field(..., description="Brief plot summary") + + agent = Agent(model=OpenAIResponses(id="gpt-4o-mini"), response_model=MovieScript, telemetry=False, monitoring=False) + + response = agent.run("Create a movie about time travel") + + # Verify structured output + assert isinstance(response.content, MovieScript) + assert response.content.title is not None + assert response.content.genre is not None + assert response.content.plot is not None + + +def test_structured_output_native(): + """Test native structured output with the responses API.""" + class MovieScript(BaseModel): + title: str = Field(..., description="Movie title") + genre: str = Field(..., description="Movie genre") + plot: str = Field(..., description="Brief plot summary") + + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), + response_model=MovieScript, + structured_outputs=True, + telemetry=False, + monitoring=False, + ) + + response = agent.run("Create a movie about time travel") + + # Verify structured output + assert isinstance(response.content, MovieScript) + assert response.content.title is not None + assert response.content.genre is not None + assert response.content.plot is not None + + +def test_history(): + """Test conversation history in the agent.""" + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), + storage=SqliteAgentStorage(table_name="responses_agent_sessions", db_file="tmp/agent_storage.db"), + add_history_to_messages=True, + telemetry=False, + monitoring=False, + ) + agent.run("Hello") + assert len(agent.run_response.messages) == 2 + agent.run("Hello 2") + assert len(agent.run_response.messages) == 4 + agent.run("Hello 3") + assert len(agent.run_response.messages) == 6 + agent.run("Hello 4") + assert len(agent.run_response.messages) == 8 + + +def test_persistent_memory(): + """Test persistent memory with the Responses API.""" + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), + tools=[DuckDuckGoTools()], + markdown=True, + show_tool_calls=True, + telemetry=False, + monitoring=False, + instructions=[ + "You can search the internet with DuckDuckGo.", + ], + storage=SqliteAgentStorage(table_name="responses_agent", db_file="tmp/agent_storage.db"), + # Adds the current date and time to the instructions + add_datetime_to_instructions=True, + # Adds the history of the conversation to the messages + add_history_to_messages=True, + # Number of history responses to add to the messages + num_history_responses=15, + memory=AgentMemory( + db=SqliteMemoryDb(db_file="tmp/responses_agent_memory.db"), + create_user_memories=True, + create_session_summary=True, + update_user_memories_after_run=True, + update_session_summary_after_run=True, + classifier=MemoryClassifier(model=OpenAIResponses(id="gpt-4o-mini")), + summarizer=MemorySummarizer(model=OpenAIResponses(id="gpt-4o-mini")), + manager=MemoryManager(model=OpenAIResponses(id="gpt-4o-mini")), + ), + ) + + response = agent.run("What is current news in France?") + assert response.content is not None \ No newline at end of file diff --git a/libs/agno/tests/integration/models/openai/responses/test_multimodal.py b/libs/agno/tests/integration/models/openai/responses/test_multimodal.py new file mode 100644 index 0000000000..37014b30e3 --- /dev/null +++ b/libs/agno/tests/integration/models/openai/responses/test_multimodal.py @@ -0,0 +1,49 @@ +import requests + +from agno.agent.agent import Agent +from agno.media import Audio, Image +from agno.models.openai.responses import OpenAIResponses +from agno.tools.duckduckgo import DuckDuckGoTools + + +def test_image_input(): + """Test image input with the responses API.""" + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), + tools=[DuckDuckGoTools()], + markdown=True, + telemetry=False, + monitoring=False + ) + + response = agent.run( + "Tell me about this image and give me the latest news about it.", + images=[Image(url="https://upload.wikimedia.org/wikipedia/commons/0/0c/GoldenGateBridge-001.jpg")], + ) + + assert "golden" in response.content.lower() + assert "bridge" in response.content.lower() + assert "san francisco" in response.content.lower() + +def test_multimodal_with_tools(): + """Test multimodal input with tool use in the responses API.""" + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), + tools=[DuckDuckGoTools()], + show_tool_calls=True, + markdown=True, + telemetry=False, + monitoring=False, + ) + + response = agent.run( + "Tell me about this bridge and look up its current status.", + images=[Image(url="https://upload.wikimedia.org/wikipedia/commons/0/0c/GoldenGateBridge-001.jpg")], + ) + + # Verify content includes image analysis and tool usage + assert "golden" in response.content.lower() + assert "bridge" in response.content.lower() + + # Check for tool call + assert any(msg.tool_calls for msg in response.messages if hasattr(msg, 'tool_calls') and msg.tool_calls) \ No newline at end of file diff --git a/libs/agno/tests/integration/models/openai/responses/test_tool_use.py b/libs/agno/tests/integration/models/openai/responses/test_tool_use.py new file mode 100644 index 0000000000..164fa8229f --- /dev/null +++ b/libs/agno/tests/integration/models/openai/responses/test_tool_use.py @@ -0,0 +1,262 @@ +import pytest +from pydantic import BaseModel, Field + +from agno.agent import Agent, RunResponse # noqa +from agno.models.openai import OpenAIResponses +from agno.tools.duckduckgo import DuckDuckGoTools +from agno.tools.exa import ExaTools +from agno.tools.yfinance import YFinanceTools + + +def test_tool_use(): + """Test basic tool usage with the responses API.""" + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), + tools=[YFinanceTools()], + show_tool_calls=True, + markdown=True, + telemetry=False, + monitoring=False, + ) + + response = agent.run("What is the current price of TSLA?") + + # Verify tool usage + assert any(msg.tool_calls for msg in response.messages) + assert response.content is not None + assert "TSLA" in response.content + + +def test_tool_use_stream(): + """Test streaming with tool use in the responses API.""" + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), + tools=[YFinanceTools()], + show_tool_calls=True, + markdown=True, + telemetry=False, + monitoring=False, + ) + + response_stream = agent.run("What is the current price of TSLA?", stream=True) + + responses = [] + tool_call_seen = False + + for chunk in response_stream: + assert isinstance(chunk, RunResponse) + responses.append(chunk) + if chunk.tools: + if any(tc.get("tool_name") for tc in chunk.tools): + tool_call_seen = True + + assert len(responses) > 0 + assert tool_call_seen, "No tool calls observed in stream" + assert any("TSLA" in r.content for r in responses if r.content) + + +@pytest.mark.asyncio +async def test_async_tool_use(): + """Test async tool use with the responses API.""" + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), + tools=[YFinanceTools()], + show_tool_calls=True, + markdown=True, + telemetry=False, + monitoring=False, + ) + + response = await agent.arun("What is the current price of TSLA?") + + # Verify tool usage + assert any(msg.tool_calls for msg in response.messages if msg.role == "assistant") + assert response.content is not None + assert "TSLA" in response.content + + +@pytest.mark.asyncio +async def test_async_tool_use_stream(): + """Test async streaming with tool use in the responses API.""" + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), + tools=[YFinanceTools()], + show_tool_calls=True, + markdown=True, + telemetry=False, + monitoring=False, + ) + + response_stream = await agent.arun("What is the current price of TSLA?", stream=True) + + responses = [] + tool_call_seen = False + + async for chunk in response_stream: + assert isinstance(chunk, RunResponse) + responses.append(chunk) + if chunk.tools: + if any(tc.get("tool_name") for tc in chunk.tools): + tool_call_seen = True + + assert len(responses) > 0 + assert tool_call_seen, "No tool calls observed in stream" + assert any("TSLA" in r.content for r in responses if r.content) + + +def test_tool_use_with_native_structured_outputs(): + """Test native structured outputs with tool use in the responses API.""" + class StockPrice(BaseModel): + price: float = Field(..., description="The price of the stock") + currency: str = Field(..., description="The currency of the stock") + + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), + tools=[YFinanceTools()], + show_tool_calls=True, + markdown=True, + response_model=StockPrice, + structured_outputs=True, + telemetry=False, + monitoring=False, + ) + response = agent.run("What is the current price of TSLA?") + assert isinstance(response.content, StockPrice) + assert response.content is not None + assert response.content.price is not None + assert response.content.currency is not None + + +def test_parallel_tool_calls(): + """Test parallel tool calls with the responses API.""" + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), + tools=[YFinanceTools()], + show_tool_calls=True, + markdown=True, + telemetry=False, + monitoring=False, + ) + + response = agent.run("What is the current price of TSLA and AAPL?") + + # Verify tool usage + tool_calls = [msg.tool_calls for msg in response.messages if msg.tool_calls] + assert len(tool_calls) >= 1 # At least one message has tool calls + assert sum(len(calls) for calls in tool_calls) == 2 # Total of 2 tool calls made + assert response.content is not None + assert "TSLA" in response.content and "AAPL" in response.content + + +def test_multiple_tool_calls(): + """Test multiple different tool types with the responses API.""" + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), + tools=[YFinanceTools(), DuckDuckGoTools()], + show_tool_calls=True, + markdown=True, + telemetry=False, + monitoring=False, + ) + + response = agent.run("What is the current price of TSLA and what is the latest news about it?") + + # Verify tool usage + tool_calls = [msg.tool_calls for msg in response.messages if msg.tool_calls] + assert len(tool_calls) >= 1 # At least one message has tool calls + assert sum(len(calls) for calls in tool_calls) == 2 # Total of 2 tool calls made + assert response.content is not None + assert "TSLA" in response.content and "latest news" in response.content.lower() + + +def test_tool_call_custom_tool_no_parameters(): + """Test custom tool with no parameters with the responses API.""" + def get_the_weather(): + return "It is currently 70 degrees and cloudy in Tokyo" + + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), + tools=[get_the_weather], + show_tool_calls=True, + markdown=True, + telemetry=False, + monitoring=False, + ) + + response = agent.run("What is the weather in Tokyo?") + + # Verify tool usage + assert any(msg.tool_calls for msg in response.messages) + assert response.content is not None + assert "70" in response.content + + +def test_tool_call_list_parameters(): + """Test tool with list parameters with the responses API.""" + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), + tools=[ExaTools(answer=False, find_similar=False)], + instructions="Use a single tool call if possible", + show_tool_calls=True, + markdown=True, + telemetry=False, + monitoring=False, + ) + + response = agent.run( + "What are the papers at https://arxiv.org/pdf/2307.06435 and https://arxiv.org/pdf/2502.09601 about?" + ) + + # Verify tool usage + assert any(msg.tool_calls for msg in response.messages) + tool_calls = [] + for msg in response.messages: + if msg.tool_calls: + tool_calls.extend(msg.tool_calls) + for call in tool_calls: + assert call["function"]["name"] in ["get_contents", "exa_answer"] + assert response.content is not None + + +def test_web_search_built_in_tool(): + """Test the built-in web search tool in the Responses API.""" + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini", web_search=True), + show_tool_calls=True, + markdown=True, + telemetry=False, + monitoring=False, + ) + + response = agent.run("What was the most recent Olympic Games and who won the most medals?") + + assert response.content is not None + assert "medal" in response.content.lower() + # Check for typical web search result indicators + assert any(term in response.content.lower() for term in ["olympic", "games", "gold", "medal"]) + + +def test_web_search_built_in_tool_stream(): + """Test the built-in web search tool in the Responses API.""" + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini", web_search=True), + show_tool_calls=True, + markdown=True, + telemetry=False, + monitoring=False, + ) + + response_stream = agent.run("What was the most recent Olympic Games and who won the most medals?", stream=True) + + responses = [] + + responses = list(response_stream) + assert len(responses) > 0 + final_response = "" + for response in responses: + assert isinstance(response, RunResponse) + assert response.content is not None + final_response += response.content + + assert "medal" in final_response.lower() + assert any(term in final_response.lower() for term in ["olympic", "games", "gold", "medal"]) \ No newline at end of file From a26a1ebcc474bd9f34dc906be1c5b640fa1eb1fb Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Wed, 12 Mar 2025 21:03:03 +0200 Subject: [PATCH 08/13] Fix tests --- cookbook/examples/apps/geobuddy/app.py | 3 +- .../models/openai/responses/async_tool_use.py | 2 +- .../openai/responses/structured_output.py | 5 +- libs/agno/agno/agent/agent.py | 2 + libs/agno/agno/models/openai/responses.py | 76 +++++++++---------- libs/agno/agno/utils/openai_responses.py | 3 +- libs/agno/agno/workspace/settings.py | 2 +- .../models/openai/responses/__init__.py | 2 +- .../models/openai/responses/test_basic.py | 10 ++- .../openai/responses/test_multimodal.py | 19 +++-- .../models/openai/responses/test_tool_use.py | 6 +- 11 files changed, 64 insertions(+), 66 deletions(-) diff --git a/cookbook/examples/apps/geobuddy/app.py b/cookbook/examples/apps/geobuddy/app.py index 5e238d3efd..4a86f4d0d2 100644 --- a/cookbook/examples/apps/geobuddy/app.py +++ b/cookbook/examples/apps/geobuddy/app.py @@ -2,9 +2,8 @@ from pathlib import Path import streamlit as st -from PIL import Image - from geography_buddy import analyze_image +from PIL import Image # Streamlit App Configuration st.set_page_config( diff --git a/cookbook/models/openai/responses/async_tool_use.py b/cookbook/models/openai/responses/async_tool_use.py index ba7afd7116..1323848158 100644 --- a/cookbook/models/openai/responses/async_tool_use.py +++ b/cookbook/models/openai/responses/async_tool_use.py @@ -7,7 +7,7 @@ from agno.tools.duckduckgo import DuckDuckGoTools agent = Agent( - model=OpenAIResponses(id="gpt-4o"), + model=OpenAIResponses(id="gpt-4o"), tools=[DuckDuckGoTools()], show_tool_calls=True, markdown=True, diff --git a/cookbook/models/openai/responses/structured_output.py b/cookbook/models/openai/responses/structured_output.py index 8d7a0f182b..2082a00f34 100644 --- a/cookbook/models/openai/responses/structured_output.py +++ b/cookbook/models/openai/responses/structured_output.py @@ -2,11 +2,10 @@ from agno.agent import Agent, RunResponse # noqa from agno.models.openai import OpenAIChat +from agno.models.openai.responses import OpenAIResponses # noqa from pydantic import BaseModel, Field from rich.pretty import pprint -from agno.models.openai.responses import OpenAIResponses # noqa - class MovieScript(BaseModel): setting: str = Field( @@ -29,7 +28,7 @@ class MovieScript(BaseModel): # Agent that uses JSON mode json_mode_agent = Agent( - model=OpenAIResponses(id="gpt-4o" ), + model=OpenAIResponses(id="gpt-4o"), description="You write movie scripts.", response_model=MovieScript, structured_outputs=True, diff --git a/libs/agno/agno/agent/agent.py b/libs/agno/agno/agent/agent.py index d9f9989265..6794c75775 100644 --- a/libs/agno/agno/agent/agent.py +++ b/libs/agno/agno/agent/agent.py @@ -2849,6 +2849,7 @@ def get_audio(self) -> Optional[List[AudioArtifact]]: def reason(self, run_messages: RunMessages) -> Iterator[RunResponse]: from agno.models.openai.like import OpenAILike + # Yield a reasoning started event if self.stream_intermediate_steps: yield self.create_run_response(content="Reasoning started", event=RunEvent.reasoning_started) @@ -3031,6 +3032,7 @@ def reason(self, run_messages: RunMessages) -> Iterator[RunResponse]: async def areason(self, run_messages: RunMessages) -> Any: from agno.models.openai.like import OpenAILike + # Yield a reasoning started event if self.stream_intermediate_steps: yield self.create_run_response(content="Reasoning started", event=RunEvent.reasoning_started) diff --git a/libs/agno/agno/models/openai/responses.py b/libs/agno/agno/models/openai/responses.py index 79b926c3f7..a7ce245137 100644 --- a/libs/agno/agno/models/openai/responses.py +++ b/libs/agno/agno/models/openai/responses.py @@ -1,24 +1,20 @@ -from dataclasses import dataclass, field -from typing import Any, AsyncGenerator, AsyncIterator, Dict, Iterator, List, Optional, Tuple, Union -import asyncio -from agno.media import AudioResponse -from agno.models.response import ModelResponse -from agno.utils.openai_responses import images_to_message +from dataclasses import dataclass +from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Tuple, Union + import httpx from agno.exceptions import ModelProviderError from agno.models.base import MessageData, Model from agno.models.message import Message +from agno.models.response import ModelResponse from agno.utils.log import logger - +from agno.utils.openai_responses import images_to_message try: import importlib.metadata as metadata - from openai import AsyncOpenAI, OpenAI - from openai import APIConnectionError, APIStatusError, RateLimitError + from openai import APIConnectionError, APIStatusError, AsyncOpenAI, OpenAI, RateLimitError from openai.resources.responses.responses import Response, ResponseStreamEvent - from packaging import version # Get installed OpenAI version @@ -28,12 +24,13 @@ parsed_version = version.parse(openai_version) if parsed_version.major == 0 and parsed_version.minor < 66: import warnings + warnings.warn("OpenAI v1.66.0 or higher is recommended for the Responses API", UserWarning) except ImportError as e: # Handle different import error scenarios if "openai" in str(e): - raise ImportError("OpenAI not installed. Install with `pip install openai`") from e + raise ImportError("OpenAI not installed. Install with `pip install openai -U`") from e else: raise ImportError("Missing dependencies. Install with `pip install packaging importlib-metadata`") from e @@ -74,7 +71,6 @@ class OpenAIResponses(Model): # Built-in tools web_search: bool = False - # The role to map the message role to. role_map = { "system": "developer", @@ -83,7 +79,6 @@ class OpenAIResponses(Model): "tool": "tool", } - # OpenAI clients client: Optional[OpenAI] = None async_client: Optional[AsyncOpenAI] = None @@ -166,7 +161,6 @@ def get_async_client(self) -> AsyncOpenAI: self.async_client = AsyncOpenAI(**client_params) return self.async_client - @property def request_kwargs(self) -> Dict[str, Any]: """ @@ -202,7 +196,7 @@ def request_kwargs(self) -> Dict[str, Any]: } else: # JSON mode - base_params["text"] = {"format": { "type": "json_object" }} + base_params["text"] = {"format": {"type": "json_object"}} # Filter out None values request_params = {k: v for k, v in base_params.items() if v is not None} @@ -237,7 +231,6 @@ def _format_messages(self, messages: List[Message]) -> List[Dict[str, Any]]: """ formatted_messages: List[Dict[str, Any]] = [] for message in messages: - if message.role in ["user", "system"]: message_dict: Dict[str, Any] = { "role": self.role_map[message.role], @@ -269,21 +262,21 @@ def _format_messages(self, messages: List[Message]) -> List[Dict[str, Any]]: # OpenAI expects the tool_calls to be None if empty, not an empty list if message.tool_calls is not None and len(message.tool_calls) > 0: for tool_call in message.tool_calls: - formatted_messages.append({ - "type": "function_call", - "id": tool_call["id"], - "call_id": tool_call["call_id"], - "name": tool_call["function"]["name"], - "arguments": tool_call["function"]["arguments"], - "status": "completed" - }) + formatted_messages.append( + { + "type": "function_call", + "id": tool_call["id"], + "call_id": tool_call["call_id"], + "name": tool_call["function"]["name"], + "arguments": tool_call["function"]["arguments"], + "status": "completed", + } + ) if message.role == "tool": - formatted_messages.append({ - "type": "function_call_output", - "call_id": message.tool_call_id, - "output": message.content - }) + formatted_messages.append( + {"type": "function_call_output", "call_id": message.tool_call_id, "output": message.content} + ) return formatted_messages def invoke(self, messages: List[Message]) -> Response: @@ -297,7 +290,6 @@ def invoke(self, messages: List[Message]) -> Response: Response: The response from the API. """ try: - return self.get_client().responses.create( model=self.id, input=self._format_messages(messages), # type: ignore @@ -349,7 +341,6 @@ async def ainvoke(self, messages: List[Message]) -> Response: Response: The response from the API. """ try: - return await self.get_async_client().responses.create( model=self.id, input=self._format_messages(messages), # type: ignore @@ -401,7 +392,6 @@ def invoke_stream(self, messages: List[Message]) -> Iterator[ResponseStreamEvent Iterator[ResponseStreamEvent]: An iterator of response stream events. """ try: - yield from self.get_client().responses.create( model=self.id, input=self._format_messages(messages), # type: ignore @@ -496,7 +486,7 @@ async def ainvoke_stream(self, messages: List[Message]) -> AsyncIterator[Respons except Exception as e: logger.error(f"Error from OpenAI API: {e}") raise ModelProviderError(message=str(e), model_name=self.name, model_id=self.id) from e - + def format_function_call_results( self, messages: List[Message], function_call_results: List[Message], tool_call_ids: List[str] ) -> None: @@ -531,7 +521,7 @@ def parse_provider_response(self, response: Response) -> ModelResponse: model_name=self.name, model_id=self.id, ) - + # Add role model_response.role = "assistant" for output in response.output: @@ -553,13 +543,12 @@ def parse_provider_response(self, response: Response) -> ModelResponse: "function": { "name": output.name, "arguments": output.arguments, - } + }, } ) model_response.extra = model_response.extra or {} model_response.extra.setdefault("tool_call_ids", []).append(output.call_id) - # i.e. we asked for reasoning, so we need to add the reasoning content if self.reasoning_effort: @@ -570,7 +559,6 @@ def parse_provider_response(self, response: Response) -> ModelResponse: return model_response - def _process_stream_response( self, stream_event: ResponseStreamEvent, @@ -617,7 +605,7 @@ def _process_stream_response( "function": { "name": item.name, "arguments": item.arguments, - } + }, } elif stream_event.type == "response.function_call_arguments.delta": @@ -655,7 +643,10 @@ def process_response_stream( for stream_event in self.invoke_stream(messages=messages): model_response, tool_use = self._process_stream_response( - stream_event=stream_event, assistant_message=assistant_message, stream_data=stream_data, tool_use=tool_use + stream_event=stream_event, + assistant_message=assistant_message, + stream_data=stream_data, + tool_use=tool_use, ) if model_response is not None: yield model_response @@ -668,10 +659,13 @@ async def aprocess_response_stream( async for stream_event in self.ainvoke_stream(messages=messages): model_response, tool_use = self._process_stream_response( - stream_event=stream_event, assistant_message=assistant_message, stream_data=stream_data, tool_use=tool_use + stream_event=stream_event, + assistant_message=assistant_message, + stream_data=stream_data, + tool_use=tool_use, ) if model_response is not None: yield model_response def parse_provider_response_delta(self, response: Any) -> ModelResponse: # type: ignore - pass \ No newline at end of file + pass diff --git a/libs/agno/agno/utils/openai_responses.py b/libs/agno/agno/utils/openai_responses.py index 9e0da75d43..904c671804 100644 --- a/libs/agno/agno/utils/openai_responses.py +++ b/libs/agno/agno/utils/openai_responses.py @@ -1,11 +1,10 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Sequence, Union -from agno.media import Audio, Image +from agno.media import Image from agno.utils.log import logger - def _process_bytes_image(image: bytes) -> Dict[str, Any]: """Process bytes image data.""" import base64 diff --git a/libs/agno/agno/workspace/settings.py b/libs/agno/agno/workspace/settings.py index a4f6adc91d..a6aa5509f9 100644 --- a/libs/agno/agno/workspace/settings.py +++ b/libs/agno/agno/workspace/settings.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import List, Optional -from pydantic import field_validator, ValidationInfo +from pydantic import ValidationInfo, field_validator from pydantic_settings import BaseSettings, SettingsConfigDict from agno.api.schemas.workspace import WorkspaceSchema diff --git a/libs/agno/tests/integration/models/openai/responses/__init__.py b/libs/agno/tests/integration/models/openai/responses/__init__.py index 26677d3ea1..aabeb362b8 100644 --- a/libs/agno/tests/integration/models/openai/responses/__init__.py +++ b/libs/agno/tests/integration/models/openai/responses/__init__.py @@ -1 +1 @@ -"""Integration tests for OpenAI Responses API.""" \ No newline at end of file +"""Integration tests for OpenAI Responses API.""" diff --git a/libs/agno/tests/integration/models/openai/responses/test_basic.py b/libs/agno/tests/integration/models/openai/responses/test_basic.py index a1583d3cb8..a158fa02d9 100644 --- a/libs/agno/tests/integration/models/openai/responses/test_basic.py +++ b/libs/agno/tests/integration/models/openai/responses/test_basic.py @@ -16,7 +16,7 @@ def _assert_metrics(response: RunResponse): """ Assert that the response metrics are valid and consistent. - + Args: response: The RunResponse to validate metrics for """ @@ -129,12 +129,15 @@ def test_with_memory(): def test_structured_output(): """Test structured output with Pydantic models.""" + class MovieScript(BaseModel): title: str = Field(..., description="Movie title") genre: str = Field(..., description="Movie genre") plot: str = Field(..., description="Brief plot summary") - agent = Agent(model=OpenAIResponses(id="gpt-4o-mini"), response_model=MovieScript, telemetry=False, monitoring=False) + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini"), response_model=MovieScript, telemetry=False, monitoring=False + ) response = agent.run("Create a movie about time travel") @@ -147,6 +150,7 @@ class MovieScript(BaseModel): def test_structured_output_native(): """Test native structured output with the responses API.""" + class MovieScript(BaseModel): title: str = Field(..., description="Movie title") genre: str = Field(..., description="Movie genre") @@ -220,4 +224,4 @@ def test_persistent_memory(): ) response = agent.run("What is current news in France?") - assert response.content is not None \ No newline at end of file + assert response.content is not None diff --git a/libs/agno/tests/integration/models/openai/responses/test_multimodal.py b/libs/agno/tests/integration/models/openai/responses/test_multimodal.py index 37014b30e3..994ba2794b 100644 --- a/libs/agno/tests/integration/models/openai/responses/test_multimodal.py +++ b/libs/agno/tests/integration/models/openai/responses/test_multimodal.py @@ -1,7 +1,5 @@ -import requests - from agno.agent.agent import Agent -from agno.media import Audio, Image +from agno.media import Image from agno.models.openai.responses import OpenAIResponses from agno.tools.duckduckgo import DuckDuckGoTools @@ -9,11 +7,11 @@ def test_image_input(): """Test image input with the responses API.""" agent = Agent( - model=OpenAIResponses(id="gpt-4o-mini"), - tools=[DuckDuckGoTools()], - markdown=True, - telemetry=False, - monitoring=False + model=OpenAIResponses(id="gpt-4o-mini"), + tools=[DuckDuckGoTools()], + markdown=True, + telemetry=False, + monitoring=False, ) response = agent.run( @@ -25,6 +23,7 @@ def test_image_input(): assert "bridge" in response.content.lower() assert "san francisco" in response.content.lower() + def test_multimodal_with_tools(): """Test multimodal input with tool use in the responses API.""" agent = Agent( @@ -44,6 +43,6 @@ def test_multimodal_with_tools(): # Verify content includes image analysis and tool usage assert "golden" in response.content.lower() assert "bridge" in response.content.lower() - + # Check for tool call - assert any(msg.tool_calls for msg in response.messages if hasattr(msg, 'tool_calls') and msg.tool_calls) \ No newline at end of file + assert any(msg.tool_calls for msg in response.messages if hasattr(msg, "tool_calls") and msg.tool_calls) diff --git a/libs/agno/tests/integration/models/openai/responses/test_tool_use.py b/libs/agno/tests/integration/models/openai/responses/test_tool_use.py index 164fa8229f..4bc0d5b31b 100644 --- a/libs/agno/tests/integration/models/openai/responses/test_tool_use.py +++ b/libs/agno/tests/integration/models/openai/responses/test_tool_use.py @@ -106,6 +106,7 @@ async def test_async_tool_use_stream(): def test_tool_use_with_native_structured_outputs(): """Test native structured outputs with tool use in the responses API.""" + class StockPrice(BaseModel): price: float = Field(..., description="The price of the stock") currency: str = Field(..., description="The currency of the stock") @@ -171,6 +172,7 @@ def test_multiple_tool_calls(): def test_tool_call_custom_tool_no_parameters(): """Test custom tool with no parameters with the responses API.""" + def get_the_weather(): return "It is currently 70 degrees and cloudy in Tokyo" @@ -233,7 +235,7 @@ def test_web_search_built_in_tool(): assert response.content is not None assert "medal" in response.content.lower() # Check for typical web search result indicators - assert any(term in response.content.lower() for term in ["olympic", "games", "gold", "medal"]) + assert any(term in response.content.lower() for term in ["olympic", "games", "gold", "medal"]) def test_web_search_built_in_tool_stream(): @@ -259,4 +261,4 @@ def test_web_search_built_in_tool_stream(): final_response += response.content assert "medal" in final_response.lower() - assert any(term in final_response.lower() for term in ["olympic", "games", "gold", "medal"]) \ No newline at end of file + assert any(term in final_response.lower() for term in ["olympic", "games", "gold", "medal"]) From a18c7dc20921d211e53c0431566bcdc929f7ac18 Mon Sep 17 00:00:00 2001 From: Dirk Brand <51947788+dirkbrnd@users.noreply.github.com> Date: Wed, 12 Mar 2025 21:30:41 +0200 Subject: [PATCH 09/13] Update libs/agno/agno/models/openai/responses.py Co-authored-by: Yash Pratap Solanky <101447028+ysolanky@users.noreply.github.com> --- libs/agno/agno/models/openai/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/agno/agno/models/openai/responses.py b/libs/agno/agno/models/openai/responses.py index a7ce245137..99b280eeb6 100644 --- a/libs/agno/agno/models/openai/responses.py +++ b/libs/agno/agno/models/openai/responses.py @@ -40,7 +40,7 @@ class OpenAIResponses(Model): """ Implementation for the OpenAI Responses API using direct chat completions. - For more information, see: https://platform.openai.com/docs/api-reference/chat + For more information, see: https://platform.openai.com/docs/api-reference/responses """ id: str = "gpt-4o" From 65eede341b537478bbcc51e5358022edd870b762 Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Wed, 12 Mar 2025 21:35:13 +0200 Subject: [PATCH 10/13] update --- libs/agno/agno/models/openai/responses.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/libs/agno/agno/models/openai/responses.py b/libs/agno/agno/models/openai/responses.py index a7ce245137..812bcbdc31 100644 --- a/libs/agno/agno/models/openai/responses.py +++ b/libs/agno/agno/models/openai/responses.py @@ -94,11 +94,11 @@ def _get_client_params(self) -> Dict[str, Any]: Returns: Dict[str, Any]: Client parameters """ - import os + from os import getenv # Fetch API key from env if not already set if not self.api_key: - self.api_key = os.getenv("OPENAI_API_KEY") + self.api_key = getenv("OPENAI_API_KEY") if not self.api_key: logger.error("OPENAI_API_KEY not set. Please set the OPENAI_API_KEY environment variable.") @@ -526,12 +526,8 @@ def parse_provider_response(self, response: Response) -> ModelResponse: model_response.role = "assistant" for output in response.output: if output.type == "message": - if model_response.content is None: - model_response.content = "" # TODO: Support citations/annotations - for content in output.content: - if content.type == "output_text": - model_response.content += content.text + model_response.content = response.output_text elif output.type == "function_call": if model_response.tool_calls is None: model_response.tool_calls = [] From 7092ce82c1e4eacb1620f0a575260847aa0af976 Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Wed, 12 Mar 2025 21:40:49 +0200 Subject: [PATCH 11/13] update --- .../responses/websearch_builtin_tool.py | 10 ++++++++++ .../models/openai/responses/test_tool_use.py | 20 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 cookbook/models/openai/responses/websearch_builtin_tool.py diff --git a/cookbook/models/openai/responses/websearch_builtin_tool.py b/cookbook/models/openai/responses/websearch_builtin_tool.py new file mode 100644 index 0000000000..041690c164 --- /dev/null +++ b/cookbook/models/openai/responses/websearch_builtin_tool.py @@ -0,0 +1,10 @@ +"""Run `pip install duckduckgo-search` to install dependencies.""" + +from agno.agent import Agent +from agno.models.openai import OpenAIResponses + +agent = Agent( + model=OpenAIResponses(id="gpt-4o", web_search=True), + markdown=True, +) +agent.print_response("Whats happening in France?") diff --git a/libs/agno/tests/integration/models/openai/responses/test_tool_use.py b/libs/agno/tests/integration/models/openai/responses/test_tool_use.py index 4bc0d5b31b..2e043bb779 100644 --- a/libs/agno/tests/integration/models/openai/responses/test_tool_use.py +++ b/libs/agno/tests/integration/models/openai/responses/test_tool_use.py @@ -262,3 +262,23 @@ def test_web_search_built_in_tool_stream(): assert "medal" in final_response.lower() assert any(term in final_response.lower() for term in ["olympic", "games", "gold", "medal"]) + + +def test_web_search_built_in_tool_with_other_tools(): + """Test the built-in web search tool in the Responses API.""" + agent = Agent( + model=OpenAIResponses(id="gpt-4o-mini", web_search=True), + tools=[YFinanceTools()], + show_tool_calls=True, + markdown=True, + telemetry=False, + monitoring=False, + ) + + response = agent.run("What is the current price of TSLA and the latest news about it?") + + tool_calls = [msg.tool_calls for msg in response.messages if msg.tool_calls] + assert len(tool_calls) >= 1 # At least one message has tool calls + assert response.content is not None + assert "TSLA" in response.content + assert "news" in response.content.lower() From a12879a4a859507d7d4129e0fcb51b35a5c3872e Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Wed, 12 Mar 2025 21:43:07 +0200 Subject: [PATCH 12/13] update --- libs/agno/agno/models/openai/responses.py | 4 +++- .../integration/models/openai/responses/test_tool_use.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libs/agno/agno/models/openai/responses.py b/libs/agno/agno/models/openai/responses.py index da625e3b2a..8ff385c2f6 100644 --- a/libs/agno/agno/models/openai/responses.py +++ b/libs/agno/agno/models/openai/responses.py @@ -3,6 +3,8 @@ import httpx +from pydantic import BaseModel + from agno.exceptions import ModelProviderError from agno.models.base import MessageData, Model from agno.models.message import Message @@ -183,7 +185,7 @@ def request_kwargs(self) -> Dict[str, Any]: } if self.response_format is not None: - if self.structured_outputs: + if self.structured_outputs and isinstance(self.response_format, BaseModel): schema = self.response_format.model_json_schema() schema["additionalProperties"] = False base_params["text"] = { diff --git a/libs/agno/tests/integration/models/openai/responses/test_tool_use.py b/libs/agno/tests/integration/models/openai/responses/test_tool_use.py index 2e043bb779..5ba59d471b 100644 --- a/libs/agno/tests/integration/models/openai/responses/test_tool_use.py +++ b/libs/agno/tests/integration/models/openai/responses/test_tool_use.py @@ -280,5 +280,5 @@ def test_web_search_built_in_tool_with_other_tools(): tool_calls = [msg.tool_calls for msg in response.messages if msg.tool_calls] assert len(tool_calls) >= 1 # At least one message has tool calls assert response.content is not None - assert "TSLA" in response.content + assert "TSLA" in response.content assert "news" in response.content.lower() From e2376803149c29480929495ca4529bfa91ddf348 Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Wed, 12 Mar 2025 21:51:43 +0200 Subject: [PATCH 13/13] update --- libs/agno/agno/models/openai/responses.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/libs/agno/agno/models/openai/responses.py b/libs/agno/agno/models/openai/responses.py index 8ff385c2f6..8d6e589e1c 100644 --- a/libs/agno/agno/models/openai/responses.py +++ b/libs/agno/agno/models/openai/responses.py @@ -2,7 +2,6 @@ from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Tuple, Union import httpx - from pydantic import BaseModel from agno.exceptions import ModelProviderError @@ -201,15 +200,15 @@ def request_kwargs(self) -> Dict[str, Any]: base_params["text"] = {"format": {"type": "json_object"}} # Filter out None values - request_params = {k: v for k, v in base_params.items() if v is not None} + request_params: Dict[str, Any] = {k: v for k, v in base_params.items() if v is not None} if self.web_search: - request_params.setdefault("tools", []) + request_params.setdefault("tools", []) # type: ignore request_params["tools"].append({"type": "web_search_preview"}) # Add tools if self._functions is not None and len(self._functions) > 0: - request_params.setdefault("tools", []) + request_params.setdefault("tools", []) # type: ignore for function in self._functions.values(): function_dict = function.to_dict() for prop in function_dict["parameters"]["properties"].values(): @@ -452,7 +451,7 @@ async def ainvoke_stream(self, messages: List[Message]) -> AsyncIterator[Respons stream=True, **self.request_kwargs, ) - async for chunk in async_stream: + async for chunk in async_stream: # type: ignore yield chunk except RateLimitError as e: logger.error(f"Rate limit error from OpenAI API: {e}") @@ -519,7 +518,7 @@ def parse_provider_response(self, response: Response) -> ModelResponse: if response.error: raise ModelProviderError( - message=response.error.get("message", "Unknown model error"), + message=response.error.message, model_name=self.name, model_id=self.id, ) @@ -611,7 +610,7 @@ def _process_stream_response( elif stream_event.type == "response.output_item.done" and tool_use: model_response = ModelResponse() - model_response.tool_calls = tool_use + model_response.tool_calls = [tool_use] if assistant_message.tool_calls is None: assistant_message.tool_calls = [] assistant_message.tool_calls.append(tool_use)