diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py index d3cbebe2d82d..ca8a3a3345fe 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py @@ -15,8 +15,11 @@ """ import inspect import time -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Literal, Optional +from openai.types import CompletionUsage +from openai.types.chat import ChatCompletion, ChatCompletionMessage +from openai.types.chat.chat_completion import Choice from openai.types.responses import EasyInputMessageParam, ResponseInputParam @@ -87,38 +90,37 @@ def _map_params(kwargs: Dict[str, Any]) -> Dict[str, Any]: return mapped -class _Usage: - """chat.completions-shaped usage view over a Responses API usage object. +_FinishReason = Literal["stop", "length", "tool_calls", "content_filter", "function_call"] + + +def _to_usage(response_usage: Any) -> Optional[CompletionUsage]: + """Adapt a Responses API usage object to a chat.completions ``CompletionUsage``. The Responses API reports ``input_tokens`` / ``output_tokens`` / ``total_tokens``; judge/grader code (and the prompty response formatter) expects ``prompt_tokens`` / ``completion_tokens`` / - ``total_tokens``. This adapts the former to the latter. + ``total_tokens``. Returns ``None`` when the result carries no usage. """ - - def __init__(self, response_usage: Any) -> None: - self.prompt_tokens = getattr(response_usage, "input_tokens", 0) or 0 - self.completion_tokens = getattr(response_usage, "output_tokens", 0) or 0 - # Fall back to prompt + completion when the Responses usage omits total_tokens. - self.total_tokens = (getattr(response_usage, "total_tokens", 0) or 0) or ( - self.prompt_tokens + self.completion_tokens - ) - - -class _ChatMessage: - def __init__(self, content: str) -> None: - self.role = "assistant" - self.content = content - self.tool_calls = None + if response_usage is None: + return None + prompt_tokens = getattr(response_usage, "input_tokens", 0) or 0 + completion_tokens = getattr(response_usage, "output_tokens", 0) or 0 + # Fall back to prompt + completion when the Responses usage omits total_tokens. + total_tokens = (getattr(response_usage, "total_tokens", 0) or 0) or (prompt_tokens + completion_tokens) + return CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ) -def _finish_reason(response: Any) -> str: - """Map a Responses result's status to a chat-completions ``finish_reason``. +def _finish_reason(response: Any) -> _FinishReason: + """Map a Responses result's status to a chat.completions ``finish_reason``. A Responses call that stops early reports ``status="incomplete"`` with an - ``incomplete_details.reason`` (e.g. ``"max_output_tokens"`` / ``"content_filter"``). - Chat.completions callers expect ``"length"`` for truncation; surfacing it lets the prompty - formatter distinguish a truncated (invalid-JSON-prone) judge output from a complete one instead - of always seeing ``"stop"``. + ``incomplete_details.reason`` (``"max_output_tokens"`` / ``"content_filter"``). chat.completions + callers expect ``"length"`` for truncation; surfacing it lets the prompty formatter distinguish a + truncated (invalid-JSON-prone) judge output from a complete one. Anything else clamps to ``"stop"`` + to stay within the chat.completions ``finish_reason`` literal. """ if getattr(response, "status", None) != "incomplete": return "stop" @@ -126,33 +128,33 @@ def _finish_reason(response: Any) -> str: reason = getattr(details, "reason", None) if details is not None else None if reason == "max_output_tokens": return "length" - return reason or "stop" - - -class _Choice: - def __init__(self, content: str, finish_reason: str = "stop") -> None: - self.index = 0 - self.message = _ChatMessage(content) - self.finish_reason = finish_reason - - -class _ChatCompletion: - """Minimal chat.completions-shaped view over a Responses API result.""" - - def __init__(self, response: Any) -> None: - self.id = getattr(response, "id", None) - self.model = getattr(response, "model", None) - _usage = getattr(response, "usage", None) - self.usage = _Usage(_usage) if _usage is not None else None - self.object = "chat.completion" - # Server timestamp: Responses uses created_at (older/mocked shapes may use created); else now. - _created = getattr(response, "created_at", None) - if _created is None: - _created = getattr(response, "created", None) - self.created = ( - int(_created) if isinstance(_created, (int, float)) and not isinstance(_created, bool) else int(time.time()) - ) - self.choices = [_Choice(getattr(response, "output_text", "") or "", _finish_reason(response))] + if reason == "content_filter": + return "content_filter" + return "stop" + + +def _to_chat_completion(response: Any) -> ChatCompletion: + """Re-shape a Responses API result into the openai ``ChatCompletion`` the prompty judge consumes. + + The prompty judge path speaks the chat.completions contract, so the shim adapts each Responses + result into the real openai ``ChatCompletion`` / ``Choice`` / ``ChatCompletionMessage`` / + ``CompletionUsage`` types (rather than hand-written stand-ins) to stay in sync with the SDK schema. + """ + message = ChatCompletionMessage(role="assistant", content=getattr(response, "output_text", "") or "") + choice = Choice(index=0, finish_reason=_finish_reason(response), message=message) + # Server timestamp: Responses uses created_at (older/mocked shapes may use created); else now. + created = getattr(response, "created_at", None) + if created is None: + created = getattr(response, "created", None) + created = int(created) if isinstance(created, (int, float)) and not isinstance(created, bool) else int(time.time()) + return ChatCompletion( + id=getattr(response, "id", None) or "", + choices=[choice], + created=created, + model=getattr(response, "model", None) or "", + object="chat.completion", + usage=_to_usage(getattr(response, "usage", None)), + ) class _AsyncChatCompletions: @@ -161,10 +163,10 @@ def __init__(self, owner: "AsyncByoProjectResponsesClient") -> None: async def create( self, *, model: Optional[str] = None, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any - ) -> _ChatCompletion: + ) -> ChatCompletion: # ``model`` from the caller is ignored — the BYO model is fixed by the shim's config. response = await self._owner._responses_create(messages=messages, **kwargs) - return _ChatCompletion(response) + return _to_chat_completion(response) class _AsyncChat: diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py index e35eca66592e..cefbd5ed333e 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py @@ -14,8 +14,8 @@ _to_responses_input, _map_params, _map_response_format, - _ChatCompletion, - _Usage, + _to_chat_completion, + _to_usage, ) @@ -88,16 +88,16 @@ def test_map_params_omits_text_for_unrecognized_response_format(self): class TestUsageAdapter: def test_responses_usage_mapped_to_chat_shape(self): - usage = _Usage(MagicMock(input_tokens=5, output_tokens=7, total_tokens=12)) + usage = _to_usage(MagicMock(input_tokens=5, output_tokens=7, total_tokens=12)) assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (5, 7, 12) def test_missing_fields_default_to_zero(self): - usage = _Usage(object()) + usage = _to_usage(object()) assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (0, 0, 0) def test_total_tokens_falls_back_to_prompt_plus_completion(self): # Responses usage without total_tokens -> compute it from prompt + completion. - usage = _Usage(SimpleNamespace(input_tokens=5, output_tokens=7)) + usage = _to_usage(SimpleNamespace(input_tokens=5, output_tokens=7)) assert usage.total_tokens == 12 @@ -113,18 +113,18 @@ def _resp(**kwargs): return SimpleNamespace(**kwargs) def test_completed_is_stop(self): - assert _ChatCompletion(self._resp(status="completed")).choices[0].finish_reason == "stop" + assert _to_chat_completion(self._resp(status="completed")).choices[0].finish_reason == "stop" def test_missing_status_defaults_to_stop(self): - assert _ChatCompletion(self._resp()).choices[0].finish_reason == "stop" + assert _to_chat_completion(self._resp()).choices[0].finish_reason == "stop" def test_truncation_is_length(self): resp = self._resp(status="incomplete", incomplete_details=SimpleNamespace(reason="max_output_tokens")) - assert _ChatCompletion(resp).choices[0].finish_reason == "length" + assert _to_chat_completion(resp).choices[0].finish_reason == "length" def test_other_incomplete_reason_passes_through(self): resp = self._resp(status="incomplete", incomplete_details=SimpleNamespace(reason="content_filter")) - assert _ChatCompletion(resp).choices[0].finish_reason == "content_filter" + assert _to_chat_completion(resp).choices[0].finish_reason == "content_filter" class TestAsyncByoProjectResponsesClient: @@ -284,14 +284,16 @@ def test_per_request_extra_headers_merged_and_forwarded(self, mock_aipc): @patch("azure.ai.projects.aio.AIProjectClient") def test_preserves_server_created_timestamp(self, mock_aipc): # Responses exposes the server timestamp as created_at; the shim preserves it (not local time). - cc = _ChatCompletion(SimpleNamespace(output_text="ok", usage=None, id="r", model="m", created_at=1752694800)) + cc = _to_chat_completion( + SimpleNamespace(output_text="ok", usage=None, id="r", model="m", created_at=1752694800) + ) assert cc.created == 1752694800 def test_created_timestamp_falls_back_to_created_then_now(self): # Older/mocked shapes may only carry ``created``; that is honored as a fallback. - assert _ChatCompletion(SimpleNamespace(output_text="ok", created=1700000000)).created == 1700000000 + assert _to_chat_completion(SimpleNamespace(output_text="ok", created=1700000000)).created == 1700000000 # When neither is a real number, fall back to the local wall-clock time (a positive int). - assert _ChatCompletion(SimpleNamespace(output_text="ok")).created > 0 + assert _to_chat_completion(SimpleNamespace(output_text="ok")).created > 0 class TestValidateModelConfigByo: