diff --git a/.chronus/changes/wujin-voice_submodule-2026-7-12-16-36-34.md b/.chronus/changes/wujin-voice_submodule-2026-7-12-16-36-34.md new file mode 100644 index 000000000000..718110429490 --- /dev/null +++ b/.chronus/changes/wujin-voice_submodule-2026-7-12-16-36-34.md @@ -0,0 +1,7 @@ +--- +changeKind: feature +packages: + - azure-ai-agentserver-invocations +--- + +Added the experimental `azure.ai.agentserver.invocations.voice` typed event relay, a synchronous connection-termination signal for developer-owned task cancellation, and the `basic_voice_agent` full-duplex sample. \ No newline at end of file diff --git a/sdk/agentserver/azure-ai-agentserver-invocations/MANIFEST.in b/sdk/agentserver/azure-ai-agentserver-invocations/MANIFEST.in index cd83a6c13bfa..7021c00dddc3 100644 --- a/sdk/agentserver/azure-ai-agentserver-invocations/MANIFEST.in +++ b/sdk/agentserver/azure-ai-agentserver-invocations/MANIFEST.in @@ -1,7 +1,7 @@ include *.md include LICENSE recursive-include tests *.py -recursive-include samples *.py *.md +recursive-include samples *.py *.md *.txt *.yaml include azure/__init__.py include azure/ai/__init__.py include azure/ai/agentserver/__init__.py diff --git a/sdk/agentserver/azure-ai-agentserver-invocations/README.md b/sdk/agentserver/azure-ai-agentserver-invocations/README.md index dd2548c9ede8..c7518d70fdb4 100644 --- a/sdk/agentserver/azure-ai-agentserver-invocations/README.md +++ b/sdk/agentserver/azure-ai-agentserver-invocations/README.md @@ -4,6 +4,7 @@ The `azure-ai-agentserver-invocations` package provides the invocation protocol - **HTTP** (`invocations` protocol) — `POST /invocations`, `GET /invocations/{id}`, `POST /invocations/{id}/cancel`, `GET /invocations/docs/openapi.json`, `GET /invocations/docs/asyncapi.{json,yaml}`. - **WebSocket** (`invocations_ws` protocol) — full-duplex streaming at `/invocations_ws`, registered with `@app.ws_handler`. +- **Voice Live Bridge** — an experimental typed event relay in the `azure.ai.agentserver.invocations.voice` submodule, layered on `invocations_ws`. ## Getting started @@ -309,6 +310,102 @@ The handler receives a Starlette [`WebSocket`][starlette-ws] and returns `None`. [starlette-ws]: https://www.starlette.io/websockets/ +## Typed Voice Live Bridge submodule (preview) + +`VoiceAgentServerHost` provides typed `on_` decorators over the existing +`invocations_ws` transport. Each callback receives an immutable event and a +send-only `Session`: + +```python +from azure.ai.agentserver.invocations.voice import ( + ResponseCreated, + ResponseDone, + ResponseOutputTextDone, + Session, + SessionReady, + SessionRejected, + SessionStart, + UserMessage, + VoiceAgentServerHost, + new_item_id, + new_response_id, +) + +app = VoiceAgentServerHost() + + +@app.on_session_start +async def on_session_start(session: Session, event: SessionStart) -> None: + if event.protocol_version != "1.0": + await session.send( + SessionRejected(code="protocol_mismatch", retriable=False) + ) + return + # Restore durable application state here when event.reconnect is true. + await session.send(SessionReady()) + + +@app.on_user_message +async def on_user_message(session: Session, event: UserMessage) -> None: + response_id = new_response_id() + item_id = new_item_id() + await session.send( + ResponseCreated(response_id=response_id, in_reply_to=(event.item_id,)) + ) + await session.send( + ResponseOutputTextDone( + response_id=response_id, + item_id=item_id, + text="Hello from the hosted text agent.", + ) + ) + await session.send(ResponseDone(response_id=response_id)) +``` + +The submodule is deliberately a thin typed event relay. It decodes one inbound frame, +dispatches the corresponding callback, encodes explicit outbound messages, and +serializes concurrent WebSocket writes. It does **not** own pending responses, +terminal arbitration, timeout/cancel operations, generation tasks, history, or +reconnect state. + +When the peer or proxy closes the WebSocket, `@app.on_disconnect` receives a +local `SessionDisconnected` event. This callback represents only the observed +peer disconnect. + +`@app.on_connection_terminating` is the common cleanup signal for every +in-process exit from the connection handler, including peer disconnect, local +protocol close, callback failure, transport failure, and task cancellation. It +is synchronous so applications can promptly call `Task.cancel()` or set their +own stop signals without making WebSocket teardown wait for asynchronous +cleanup. The callback must be non-blocking and must not send frames. The SDK +invokes it once as each connection handler unwinds, and applications must keep +their signaling idempotent. The SDK does not retain, join, or guarantee +completion of application-owned tasks. + +For the Voice WebSocket relay, shutdown cancellation remains cancellation while +the SDK is awaiting WebSocket accept or receive, even if the ASGI transport +returns normally or translates the cancellation into a standard exception. This +guarantee requires the transport operation to eventually settle after receiving +cancellation; a transport that suppresses cancellation and never returns is +outside the contract. + +After repeated cancellation requests, the Voice endpoint is guaranteed to remain +cancelled. The exact nested `asyncio.CancelledError` instance or message selected +from a transport-defined exception graph is unspecified. + +Voice callback cancellation is cooperative. A callback that catches +`asyncio.CancelledError` must re-raise it after its own cleanup. If application +code catches cancellation and returns normally, recovery is outside the SDK +contract; the SDK does not forcibly terminate or retain that callback. + +For full-duplex streaming, the agent creates and owns a generation task, returns +from `on_user_message`, and cancels that task from `on_barge_in`, +`on_response_cancelled`, `on_response_timeout`, or +`on_connection_terminating`. Each task remains responsible for its own +asynchronous resource cleanup. See the complete +[`basic_voice_agent`](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/agentserver/azure-ai-agentserver-invocations/samples/basic_voice_agent) +sample. + ### Reference: configuration | Environment variable | Default | Description | @@ -340,6 +437,7 @@ Visit the [Samples](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/ | [async_invoke_agent](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/agentserver/azure-ai-agentserver-invocations/samples/async_invoke_agent/) | Long-running operations with polling and cancellation | | [ws_invoke_agent](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/agentserver/azure-ai-agentserver-invocations/samples/ws_invoke_agent/) | Combined `POST /invocations` (HTTP) and `/invocations_ws` (WebSocket) host | | [ws_bidirectional_streaming_agent](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/agentserver/azure-ai-agentserver-invocations/samples/ws_bidirectional_streaming_agent/) | Full-duplex `/invocations_ws` agent: concurrent token streams + mid-flight cancel (relies on the SDK's WS protocol Ping/Pong keep-alive, not application-level heartbeats) | +| [basic_voice_agent](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/agentserver/azure-ai-agentserver-invocations/samples/basic_voice_agent/) | Typed Voice Live Bridge callbacks with developer-owned full-duplex streaming and cancellation | ## Contributing diff --git a/sdk/agentserver/azure-ai-agentserver-invocations/api.md b/sdk/agentserver/azure-ai-agentserver-invocations/api.md index 8593e6499ec0..65952b6f5b13 100644 --- a/sdk/agentserver/azure-ai-agentserver-invocations/api.md +++ b/sdk/agentserver/azure-ai-agentserver-invocations/api.md @@ -6,11 +6,11 @@ namespace azure.ai.agentserver.invocations property ws_ping_interval: float # Read-only def __init__( - self, - *, - asyncapi_spec_json: Optional[dict[str, Any]] = ..., - asyncapi_spec_yaml: Optional[str] = ..., - openapi_spec: Optional[dict[str, Any]] = ..., + self, + *, + asyncapi_spec_json: Optional[dict[str, Any]] = ..., + asyncapi_spec_yaml: Optional[str] = ..., + openapi_spec: Optional[dict[str, Any]] = ..., **kwargs: Any ) -> None: ... @@ -29,4 +29,703 @@ namespace azure.ai.agentserver.invocations def ws_handler(self, fn: WSHandler) -> WSHandler: ... +namespace azure.ai.agentserver.invocations.voice + + @experimental + def azure.ai.agentserver.invocations.voice.new_item_id() -> str: ... + + + @experimental + def azure.ai.agentserver.invocations.voice.new_message_id() -> str: ... + + + @experimental + def azure.ai.agentserver.invocations.voice.new_response_id() -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.AgentError(_OutboundMessage): + code: str + id: str + item_id: Optional[str] + message: str + response_id: Optional[str] + ts: str + type: ClassVar[str] = error + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str = ..., + ts: str = ..., + code: str, + message: str, + response_id: str | None = None, + item_id: str | None = None + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.BargeIn(_InboundMessage): + heard_text: str + id: str + item_id: Optional[str] + response_id: str + ts: str + type: ClassVar[str] = barge_in + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str, + ts: str, + response_id: str, + heard_text: str, + item_id: str | None = None + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.EndCall(_OutboundMessage): + id: str + mode: EndCallMode = EndCallMode.DRAIN + reason: str + ts: str + type: ClassVar[str] = end_call + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str = ..., + ts: str = ..., + reason: str, + mode: EndCallMode = ... + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + class azure.ai.agentserver.invocations.voice.EndCallMode(str, Enum, metaclass=CaseInsensitiveEnumMeta): + DRAIN = "drain" + IMMEDIATE = "immediate" + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.InputTextPart: + text: str + type: Literal["input_text"] = input_text + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__(text: str) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.ResponseAccepted(_InboundMessage): + id: str + response_id: str + ts: str + type: ClassVar[str] = response.accepted + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str, + ts: str, + response_id: str + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.ResponseCancel(_OutboundMessage): + id: str + reason: Optional[str] + response_id: str + ts: str + type: ClassVar[str] = response.cancel + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str = ..., + ts: str = ..., + response_id: str, + reason: str | None = None + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.ResponseCancelled(_InboundMessage): + heard_text: str + id: str + item_id: Optional[str] + response_id: str + ts: str + type: ClassVar[str] = response.cancelled + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str, + ts: str, + response_id: str, + heard_text: str, + item_id: str | None = None + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.ResponseCreated(_OutboundMessage): + admission_timeout_ms: Optional[int] + id: str + in_reply_to: Optional[tuple[str, Ellipsis]] + response_id: str + supersede_key: Optional[str] + ts: str + type: ClassVar[str] = response.created + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str = ..., + ts: str = ..., + response_id: str, + in_reply_to: tuple = None, + admission_timeout_ms: int | None = None, + supersede_key: str | None = None + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.ResponseDone(_OutboundMessage): + id: str + response_id: str + ts: str + type: ClassVar[str] = response.done + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str = ..., + ts: str = ..., + response_id: str + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.ResponseDropped(_InboundMessage): + id: str + reason: str + response_id: str + ts: str + type: ClassVar[str] = response.dropped + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str, + ts: str, + response_id: str, + reason: str + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.ResponseNone(_OutboundMessage): + id: str + in_reply_to: tuple[str, Ellipsis] + reason: Optional[str] + ts: str + type: ClassVar[str] = response.none + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str = ..., + ts: str = ..., + in_reply_to: tuple, + reason: str | None = None + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.ResponseOutputTextDelta(_OutboundMessage): + delta: str + id: str + item_id: str + response_id: str + ts: str + type: ClassVar[str] = response.output_text.delta + voice: Optional[Mapping[str, Any]] + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str = ..., + ts: str = ..., + response_id: str, + item_id: str, + delta: str, + voice: Mapping = None + ) -> None: ... + + def __post_init__(self) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.ResponseOutputTextDone(_OutboundMessage): + id: str + item_id: str + response_id: str + text: str + ts: str + type: ClassVar[str] = response.output_text.done + voice: Optional[Mapping[str, Any]] + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str = ..., + ts: str = ..., + response_id: str, + item_id: str, + text: str, + voice: Mapping = None + ) -> None: ... + + def __post_init__(self) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.ResponseTimeout(_InboundMessage): + id: str + item_ids: Optional[tuple[str, Ellipsis]] + response_id: Optional[str] + stage: str + ts: str + type: ClassVar[str] = response.timeout + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str, + ts: str, + stage: str, + response_id: str | None = None, + item_ids: tuple = None + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.ResponseTimeouts: + first_output_ms: int + idle_ms: int + max_duration_ms: int + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + first_output_ms: int, + idle_ms: int, + max_duration_ms: int + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + class azure.ai.agentserver.invocations.voice.Session: + + def __init__(self) -> None: ... + + async def send(self, message: OutboundVoiceMessage) -> None: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.SessionDisconnected: + code: int + reason: Optional[str] + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__(code: int, reason: str | None = None) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.SessionEnd(_InboundMessage): + id: str + reason: str + ts: str + type: ClassVar[str] = session.end + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str, + ts: str, + reason: str + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.SessionReady(_OutboundMessage): + id: str + ts: str + type: ClassVar[str] = session.ready + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__(id: str = ..., ts: str = ...) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.SessionRejected(_OutboundMessage): + code: str + id: str + message: Optional[str] + retriable: bool + ts: str + type: ClassVar[str] = session.rejected + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str = ..., + ts: str = ..., + code: str, + retriable: bool, + message: str | None = None + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.SessionStart(_InboundMessage): + caller: Optional[Mapping[str, Any]] + greeting: Optional[str] + id: str + no_input_timeout_ms: Optional[int] + protocol_version: str + reconnect: bool + response_timeouts: ResponseTimeouts + ts: str + type: ClassVar[str] = session.start + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str, + ts: str, + protocol_version: str, + reconnect: bool, + response_timeouts: ResponseTimeouts, + greeting: str | None = None, + no_input_timeout_ms: int | None = None, + caller: Mapping = None + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.UserMessage(_InboundMessage): + content: tuple[InputTextPart, Ellipsis] + id: str + item_id: str + ts: str + type: ClassVar[str] = user.message + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str, + ts: str, + item_id: str, + content: tuple + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.UserNoInput(_InboundMessage): + count: int + id: str + item_id: str + ts: str + type: ClassVar[str] = user.no_input + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__( + id: str, + ts: str, + item_id: str, + count: int + ) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + @dataclass(frozen=True, kw_only=True, repr=False) + class azure.ai.agentserver.invocations.voice.UserSpeechStarted(_InboundMessage): + id: str + ts: str + type: ClassVar[str] = user.speech_started + + def __delattr__() -> None: ... + + def __eq__() -> None: ... + + def __hash__() -> None: ... + + def __init__(id: str, ts: str) -> None: ... + + def __setattr__() -> None: ... + + def _voice_model_repr(self: Any) -> str: ... + + + @experimental + class azure.ai.agentserver.invocations.voice.VoiceAgentServerHost(InvocationAgentServerHost): + property routes: list[BaseRoute] # Read-only + property ws_ping_interval: float # Read-only + + def __init__( + self, + *, + asyncapi_spec_json: dict[str, Any] | None = ..., + asyncapi_spec_yaml: str | None = ..., + openapi_spec: dict[str, Any] | None = ..., + **kwargs: Any + ) -> None: ... + + def cancel_invocation_handler(self, fn: Callable[[Request], Awaitable[Response]]) -> Callable[[Request], Awaitable[Response]]: ... + + def get_asyncapi_spec_json(self) -> Optional[dict[str, Any]]: ... + + def get_asyncapi_spec_yaml(self) -> Optional[str]: ... + + def get_invocation_handler(self, fn: Callable[[Request], Awaitable[Response]]) -> Callable[[Request], Awaitable[Response]]: ... + + def get_openapi_spec(self) -> Optional[dict[str, Any]]: ... + + def invoke_handler(self, fn: Callable[[Request], Awaitable[Response]]) -> Callable[[Request], Awaitable[Response]]: ... + + def on_barge_in(self, callback: BargeInCallback) -> BargeInCallback: ... + + def on_connection_terminating(self, callback: ConnectionTerminatingCallback) -> ConnectionTerminatingCallback: ... + + def on_disconnect(self, callback: DisconnectCallback) -> DisconnectCallback: ... + + def on_response_accepted(self, callback: ResponseAcceptedCallback) -> ResponseAcceptedCallback: ... + + def on_response_cancelled(self, callback: ResponseCancelledCallback) -> ResponseCancelledCallback: ... + + def on_response_dropped(self, callback: ResponseDroppedCallback) -> ResponseDroppedCallback: ... + + def on_response_timeout(self, callback: ResponseTimeoutCallback) -> ResponseTimeoutCallback: ... + + def on_session_end(self, callback: SessionEndCallback) -> SessionEndCallback: ... + + def on_session_start(self, callback: SessionStartCallback) -> SessionStartCallback: ... + + def on_user_message(self, callback: UserMessageCallback) -> UserMessageCallback: ... + + def on_user_no_input(self, callback: UserNoInputCallback) -> UserNoInputCallback: ... + + def on_user_speech_started(self, callback: UserSpeechStartedCallback) -> UserSpeechStartedCallback: ... + + def ws_handler(self, fn: Any) -> NoReturn: ... + + ``` \ No newline at end of file diff --git a/sdk/agentserver/azure-ai-agentserver-invocations/api.metadata.yml b/sdk/agentserver/azure-ai-agentserver-invocations/api.metadata.yml index b3a77c616f75..1ec41090eb4d 100644 --- a/sdk/agentserver/azure-ai-agentserver-invocations/api.metadata.yml +++ b/sdk/agentserver/azure-ai-agentserver-invocations/api.metadata.yml @@ -1,3 +1,3 @@ -apiMdSha256: 28f40971c3ed93df127abaf7faf714a62065ef4440aaf511b9269c584811520c +apiMdSha256: 8c81198c94f9b6d95391cc58b11d3769ec0de8850d93a2c5e736046e82d25f63 parserVersion: 0.3.31 pythonVersion: 3.11.15 diff --git a/sdk/agentserver/azure-ai-agentserver-invocations/azure/ai/agentserver/invocations/voice/__init__.py b/sdk/agentserver/azure-ai-agentserver-invocations/azure/ai/agentserver/invocations/voice/__init__.py new file mode 100644 index 000000000000..ec9cfc7169b3 --- /dev/null +++ b/sdk/agentserver/azure-ai-agentserver-invocations/azure/ai/agentserver/invocations/voice/__init__.py @@ -0,0 +1,98 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- +"""Typed Voice Live Bridge event relay over ``invocations_ws``.""" + +from ._models import ( + AgentError, + BargeIn, + EndCall, + EndCallMode, + InboundVoiceMessage, + InputTextPart, + OutboundVoiceMessage, + ResponseAccepted, + ResponseCancel, + ResponseCancelled, + ResponseCreated, + ResponseDone, + ResponseDropped, + ResponseNone, + ResponseOutputTextDelta, + ResponseOutputTextDone, + ResponseTimeout, + ResponseTimeouts, + SessionDisconnected, + SessionEnd, + SessionReady, + SessionRejected, + SessionStart, + UserMessage, + UserNoInput, + UserSpeechStarted, + new_item_id, + new_message_id, + new_response_id, +) +from ._session import Session +from ._voice_host import ( + BargeInCallback, + ConnectionTerminatingCallback, + DisconnectCallback, + ResponseAcceptedCallback, + ResponseCancelledCallback, + ResponseDroppedCallback, + ResponseTimeoutCallback, + SessionEndCallback, + SessionStartCallback, + UserMessageCallback, + UserNoInputCallback, + UserSpeechStartedCallback, + VoiceAgentServerHost, +) + +__all__ = [ + "AgentError", + "BargeIn", + "BargeInCallback", + "ConnectionTerminatingCallback", + "DisconnectCallback", + "EndCall", + "EndCallMode", + "InboundVoiceMessage", + "InputTextPart", + "OutboundVoiceMessage", + "ResponseAccepted", + "ResponseAcceptedCallback", + "ResponseCancel", + "ResponseCancelled", + "ResponseCancelledCallback", + "ResponseCreated", + "ResponseDone", + "ResponseDropped", + "ResponseDroppedCallback", + "ResponseNone", + "ResponseOutputTextDelta", + "ResponseOutputTextDone", + "ResponseTimeout", + "ResponseTimeoutCallback", + "ResponseTimeouts", + "Session", + "SessionDisconnected", + "SessionEnd", + "SessionEndCallback", + "SessionReady", + "SessionRejected", + "SessionStart", + "SessionStartCallback", + "UserMessage", + "UserMessageCallback", + "UserNoInput", + "UserNoInputCallback", + "UserSpeechStarted", + "UserSpeechStartedCallback", + "VoiceAgentServerHost", + "new_item_id", + "new_message_id", + "new_response_id", +] diff --git a/sdk/agentserver/azure-ai-agentserver-invocations/azure/ai/agentserver/invocations/voice/_codec.py b/sdk/agentserver/azure-ai-agentserver-invocations/azure/ai/agentserver/invocations/voice/_codec.py new file mode 100644 index 000000000000..067e54f17e36 --- /dev/null +++ b/sdk/agentserver/azure-ai-agentserver-invocations/azure/ai/agentserver/invocations/voice/_codec.py @@ -0,0 +1,731 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- +"""Context-free codec for the Voice Live Bridge text/control profile.""" + +from __future__ import annotations + +import datetime +import json +import math +import re +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Any, cast + +from azure.ai.agentserver.core import experimental + +from ._models import ( + AgentError, + BargeIn, + EndCall, + EndCallMode, + InboundVoiceMessage, + InputTextPart, + OutboundVoiceMessage, + ResponseAccepted, + ResponseCancel, + ResponseCancelled, + ResponseCreated, + ResponseDone, + ResponseDropped, + ResponseNone, + ResponseOutputTextDelta, + ResponseOutputTextDone, + ResponseTimeout, + ResponseTimeouts, + SessionEnd, + SessionReady, + SessionRejected, + SessionStart, + UserMessage, + UserNoInput, + UserSpeechStarted, +) + +MAX_FRAME_BYTES = 1_048_576 +MAX_IDENTIFIER_BYTES = 256 +MAX_INTEGER_DIGITS = 128 +MAX_JSON_DEPTH = 32 +MAX_JSON_NODES = 8_192 +MAX_ADMISSION_TIMEOUT_MS = 60_000 + +_CREDENTIAL_FIELD = re.compile( + r"(?:^|_)(?:authorization(?:_header)?|credentials?|password|passwd|pwd|secret(?:_value)?|" + r"api_(?:key|token)|auth_token|bearer_token|access_(?:key|token)|refresh_token|id_token|" + r"client_(?:assertion|secret)|private_key|connection_string|sas(?:_token|_url)?|account_key|" + r"subscription_key|shared_access_(?:key|signature))(?:_|$)" +) +_COMPACT_CREDENTIAL_FIELDS = frozenset( + { + "accesskey", + "accesstoken", + "accountkey", + "apikey", + "apitoken", + "authorization", + "authorizationheader", + "authtoken", + "azurepwd", + "azuresas", + "bearertoken", + "clientassertion", + "clientsecret", + "connectionstring", + "credential", + "credentials", + "idtoken", + "password", + "passwd", + "privatekey", + "pwd", + "refreshtoken", + "sas", + "sastoken", + "sasurl", + "secret", + "secretvalue", + "sharedaccesskey", + "sharedaccesssignature", + "subscriptionkey", + } +) +_COMPACT_CREDENTIAL_SUFFIXES = _COMPACT_CREDENTIAL_FIELDS - {"azurepwd", "azuresas", "pwd", "sas"} +_COMPACT_CREDENTIAL_MARKERS = _COMPACT_CREDENTIAL_SUFFIXES - {"secret"} +_VOICE_TYPE_ALIASES = {"azure-platform": "azure-standard", "custom": "azure-custom"} +_VOICE_TYPES = { + "openai", + "azure-standard", + "azure-custom", + "azure-personal", + "avatar-voice-sync", + "azure-realtime-native", +} +_VOICE_STRING_FIELDS = frozenset({"endpoint_id", "model", "name"}) +_VOICE_NON_EMPTY_STRING_FIELDS = frozenset({"endpoint_id", "name"}) +_VOICE_NULLABLE_STRING_FIELDS = frozenset( + { + "custom_lexicon_url", + "custom_text_normalization_url", + "locale", + "multi_talker_speaker_name", + "pitch", + "rate", + "style", + "volume", + } +) +_RFC3339 = re.compile( + r"^(?P[0-9]{4}-[0-9]{2}-[0-9]{2})T(?P