improved TTFB metrics for streaming TTS

.changeset/sour-gorillas-jam.md

@@ -0,0 +1,9 @@
+---
+"livekit-plugins-elevenlabs": patch
+"livekit-plugins-cartesia": patch
+"livekit-plugins-deepgram": patch
+"livekit-plugins-playai": patch
+"livekit-agents": patch
+---
+
+improved TTFB metrics for streaming TTS

livekit-agents/livekit/agents/tts/tts.py

@@ -246,6 +246,7 @@ def __init__(self, *, tts: TTS, conn_options: APIConnectOptions) -> None:
         self._task = asyncio.create_task(self._main_task(), name="TTS._main_task")
         self._task.add_done_callback(lambda _: self._event_ch.close())
         self._metrics_task: asyncio.Task | None = None  # started on first push
+        self._started_time: float = 0
 
         # used to track metrics
         self._mtc_pending_texts: list[str] = []
@@ -279,18 +280,26 @@ async def _main_task(self) -> None:
 
                 await asyncio.sleep(retry_interval)
 
+    def _mark_started(self) -> None:
+        # only set the started time once, it'll get reset after we emit metrics
+        if self._started_time == 0:
+            self._started_time = time.perf_counter()
+
     async def _metrics_monitor_task(
         self, event_aiter: AsyncIterable[SynthesizedAudio]
     ) -> None:
         """Task used to collect metrics"""
-        start_time = time.perf_counter()
         audio_duration = 0.0
         ttfb = -1.0
         request_id = ""
 
         def _emit_metrics():
-            nonlocal start_time, audio_duration, ttfb, request_id
-            duration = time.perf_counter() - start_time
+            nonlocal audio_duration, ttfb, request_id
+
+            if not self._started_time:
+                return
+
+            duration = time.perf_counter() - self._started_time
 
             if not self._mtc_pending_texts:
                 return
@@ -316,11 +325,11 @@ def _emit_metrics():
             audio_duration = 0.0
             ttfb = -1.0
             request_id = ""
-            start_time = time.perf_counter()
+            self._started_time = 0
 
         async for ev in event_aiter:
             if ttfb == -1.0:
-                ttfb = time.perf_counter() - start_time
+                ttfb = time.perf_counter() - self._started_time
 
             audio_duration += ev.frame.duration
             request_id = ev.request_id

livekit-plugins/livekit-plugins-cartesia/livekit/plugins/cartesia/tts.py

@@ -48,7 +48,7 @@
 API_VERSION = "2024-06-10"
 
 NUM_CHANNELS = 1
-BUFFERED_WORDS_COUNT = 8
+BUFFERED_WORDS_COUNT = 3
 
 
 @dataclass
@@ -267,6 +267,7 @@ async def _sentence_stream_task(ws: aiohttp.ClientWebSocketResponse):
                 token_pkt["context_id"] = request_id
                 token_pkt["transcript"] = ev.token + " "
                 token_pkt["continue"] = True
+                self._mark_started()
                 await ws.send_str(json.dumps(token_pkt))
 
             end_pkt = base_pkt.copy()

livekit-plugins/livekit-plugins-deepgram/livekit/plugins/deepgram/tts.py

@@ -279,6 +279,7 @@ async def _run_segments(ws: aiohttp.ClientWebSocketResponse):
             async for word_stream in self._segments_ch:
                 async for word in word_stream:
                     speak_msg = {"type": "Speak", "text": f"{word.token} "}
+                    self._mark_started()
                     await ws.send_str(json.dumps(speak_msg))
 
                 # Always flush after a segment

livekit-plugins/livekit-plugins-elevenlabs/livekit/plugins/elevenlabs/tts.py

@@ -431,6 +431,7 @@ async def send_task():
                     text=f"{text} ",  # must always end with a space
                     try_trigger_generation=False,
                 )
+                self._mark_started()
                 await ws_conn.send_str(json.dumps(data_pkt))
 
             if xml_content:

livekit-plugins/livekit-plugins-playai/livekit/plugins/playai/tts.py

@@ -277,6 +277,7 @@ async def _create_text_stream(self):
         async def text_stream():
             async for word_stream in self._segments_ch:
                 async for word in word_stream:
+                    self._mark_started()
                     yield word.token
 
         return text_stream()