Tune video encoder for low-delay streaming

examples/03_fps/fps_explore.py

@@ -40,6 +40,7 @@
 
 DEFAULT_RESOLUTION = 6
 DEFAULT_QUANTIZATION = 5  # 0 (high quality) - 63 (low quality)
+DEFAULT_BIT_RATE = 8  # in Mbps
 STATS_STYLES = """
     position: absolute;
     top: 1rem;
@@ -158,7 +159,7 @@ def build_ui(self):
                     v_if="display_mode === 'image'",
                 )
                 html.Span(
-                    "Quantization ({{quantization}}/63) - Codec ({{video_codec}})",
+                    "Bitrate ({{bit_rate}} Mbps) - Quant ({{quantization}}/63) - Codec ({{video_codec}})",
                     v_else=True,
                 )
                 v3.VSpacer()
@@ -205,7 +206,18 @@ def build_ui(self):
                     style="max-width: 300px",
                 )
                 v3.VSlider(
-                    v_else=True,
+                    v_if=(f"display_mode === '{VIDEO}'",),
+                    label="bitrate (Mbps)",
+                    v_model=("bit_rate", DEFAULT_BIT_RATE),
+                    min=1,
+                    max=100,
+                    step=1,
+                    hide_details=True,
+                    density="compact",
+                    style="max-width: 300px",
+                )
+                v3.VSlider(
+                    v_if=(f"display_mode === '{VIDEO}'",),
                     label="Quantization",
                     v_model=("quantization", DEFAULT_QUANTIZATION),
                     min=0,  # high quality
@@ -216,6 +228,7 @@ def build_ui(self):
                     style="max-width: 300px",
                 )
                 v3.VSlider(
+                    label="# cone faces",
                     v_model=("resolution", DEFAULT_RESOLUTION),
                     min=3,
                     max=60,
@@ -285,13 +298,15 @@ def update_cone(self, resolution, **kwargs):
     def update_quality(self, quality, **_):
         self.image_view_handler.update_quality(*quality)
 
-    @change("quantization")
-    def update_quantization(self, quantization, **_):
+    @change("bit_rate", "quantization")
+    def update_bit_rate(self, bit_rate, quantization, **_):
         scheduler = self.video_view_handler._scheduler
         if scheduler._rca_encoder.is_ready:
-            scheduler._rca_encoder.video_encoder.SetQuantizationParameter(
-                int(quantization)
-            )
+            scheduler._rca_encoder.encoder.quantization_parameter = quantization
+            scheduler._rca_encoder.encoder.bit_rate = bit_rate * 1_000_000
+            scheduler._rca_encoder.encoder.min_bit_rate = bit_rate * 1_000_000
+            scheduler._rca_encoder.encoder.max_bit_rate = bit_rate * 1_000_000
+            scheduler._rca_encoder.reset(self.render_window)
         self.video_view_handler.update()
 
     def update_reset_resolution(self):

src/trame_rca/encoders/video_encoder.py

@@ -36,6 +36,21 @@
     "vtkVpxEncoder": "libvpx",
 }
 
+# PLEASE DO NOT ALTER!
+# Video stream tuning for low-latency CBR (Constant Bit Rate) setup
+# - low-delay mode: P1 preset + ultra-low-latency tuning, infinite GOP (Group Of Pictures),
+#   0 B-frames, lookahead=False produces a packet per frame, nothing gets buffered;
+# - CBR at the chosen bitrate, with the frame rate told to the rate control
+#   so its per-frame budget is right (the default in vtk_streaming assumes 30 fps);
+# - QP (Quantization Parameter) floor so a static scene doesn't burn the budget on
+#   invisible detail.
+VIDEO_MAXIMUM_B_FRAMES = 0
+VIDEO_TARGET_FPS_DEFAULT = 30
+VIDEO_BITRATE_MBPS_DEFAULT = 8
+VIDEO_BITRATE_MBPS_MIN = 1
+VIDEO_BITRATE_MBPS_MAX = 100
+VIDEO_QP_FLOOR = 5  # H.264/HEVC QP scale; 5 is visually lossless
+
 
 def available_codecs() -> list[dict]:
     """Codecs this server can encode, server-preferred first."""
@@ -136,7 +151,12 @@ def is_ready(self) -> bool:
     def codecs(self) -> Optional[list[str]]:
         return self._codecs
 
-    def configure(self, codecs: Optional[list[str]]) -> dict:
+    def configure(
+        self,
+        codecs: Optional[list[str]],
+        target_fps: int = VIDEO_TARGET_FPS_DEFAULT,
+        target_bitrate_mbps: float = VIDEO_BITRATE_MBPS_DEFAULT,
+    ) -> dict:
         """(Re)create the encoder for the given codec ranking. Returns :func:`describe_encoder`."""
         with self._timer_configure:
             self.release()
@@ -149,7 +169,7 @@ def configure(self, codecs: Optional[list[str]]) -> dict:
             self.encoder.AddObserver(
                 vtkVideoEncoder.EncodedVideoChunkEvent, self._on_encoded_chunk
             )
-
+            self._tune(target_fps, target_bitrate_mbps)
             self._initialize(self._render_window)
         return self.describe()
 
@@ -162,19 +182,34 @@ def _set_size(self, render_window_size: tuple[int]):
         self.frame.SetHeight(self._window_size[1])
         self.frame.AllocateDataStore()
 
+    def _tune(self, target_fps: int, target_bitrate_mbps: float):
+        if self.encoder is None:
+            raise RuntimeError(
+                "Tune called but no encoder exists. Did you forget to call RcaVideoEncoder.configure(codecs)?"
+            )
+        bitrate = int(float(target_bitrate_mbps) * 1_000_000)
+        self.encoder.low_delay_mode = True
+        self.encoder.maximum_b_frames = VIDEO_MAXIMUM_B_FRAMES
+        self.encoder.time_base_start = 1
+        self.encoder.time_base_end = max(1, int(target_fps))
+        self.encoder.bit_rate_control_mode = vtkVideoEncoder.BRCType.CBR
+        self.encoder.quantization_parameter = VIDEO_QP_FLOOR
+        self.encoder.bit_rate = bitrate
+        self.encoder.min_bit_rate = bitrate
+        self.encoder.max_bit_rate = bitrate
+        # one IDR (Instantaneous Decoder Refresh) to start the (re)built context, then delta frames only.
+        self.encoder.force_i_frame = True  # we will turn this off in _on_encoded_chunk
+
     def _initialize(self, render_window: vtkRenderWindow):
         self.encoder.SetGraphicsContext(render_window)
         self.encoder.SetInputPixelFormat(VTKPF_IYUV)
-        self.encoder.SetBitRateControlMode(vtkVideoEncoder.BRCType.CQP)
-        self.encoder.SetQuantizationParameter(5)  # 0 (high quality) - 63 (low quality)
 
         self.frame = vtkOpenGLVideoFrame()
         self.frame.SetContext(render_window)
         self.frame.SetPixelFormat(VTKPF_IYUV)
 
         self._set_size(render_window.GetSize())
         self.encoder.Initialize()
-        self.encoder.ForceIFrameOn()
 
     def reset(self, render_window: vtkRenderWindow) -> None:
         if not self.is_ready:
@@ -192,6 +227,8 @@ def _on_encoded_chunk(
     ) -> None:
         now_ms = int(time_ns() / 1000000)
         content, meta, _ = encode(video_packet, now_ms)
+        if video_packet.is_key_frame and _encoder.force_i_frame:
+            _encoder.force_i_frame = False
         if self._push_callback is not None:
             self._push_callback(content, meta, now_ms)
 

src/trame_rca/schedulers/video_scheduler.py

@@ -76,7 +76,7 @@ def negotiate_video(self, codecs: list[str]) -> dict:
                 "label": "unavailable",
             }
         else:
-            info = self._rca_encoder.configure(codecs)
+            info = self._rca_encoder.configure(codecs, self.target_fps)
             self.schedule_render()
         if self.on_codec_changed is not None:
             self.on_codec_changed(info)