Implement dynamic framerate

zmerp · zmerp · commit 069b877331e4 · 2023-06-01T18:04:54.000+08:00
Reduce tracking frequency to head fps
diff --git a/alvr/client_core/src/c_api.rs b/alvr/client_core/src/c_api.rs
@@ -398,9 +398,14 @@ pub extern "C" fn alvr_get_tracker_prediction_offset_ns() -> u64 {
 }
 
 #[no_mangle]
-pub extern "C" fn alvr_report_submit(target_timestamp_ns: u64, vsync_queue_ns: u64) {
+pub extern "C" fn alvr_report_submit(
+    target_timestamp_ns: u64,
+    frame_interval_ns: u64,
+    vsync_queue_ns: u64,
+) {
     crate::report_submit(
         Duration::from_nanos(target_timestamp_ns),
+        Duration::from_nanos(frame_interval_ns),
         Duration::from_nanos(vsync_queue_ns),
     );
 }
diff --git a/alvr/client_core/src/lib.rs b/alvr/client_core/src/lib.rs
@@ -197,9 +197,9 @@ pub fn get_tracker_prediction_offset() -> Duration {
     }
 }
 
-pub fn report_submit(target_timestamp: Duration, vsync_queue: Duration) {
+pub fn report_submit(target_timestamp: Duration, frame_interval: Duration, vsync_queue: Duration) {
     if let Some(stats) = &mut *STATISTICS_MANAGER.lock() {
-        stats.report_submit(target_timestamp, vsync_queue);
+        stats.report_submit(target_timestamp, frame_interval, vsync_queue);
 
         if let Some(sender) = &*STATISTICS_SENDER.lock() {
             if let Some(stats) = stats.summary(target_timestamp) {
diff --git a/alvr/client_core/src/statistics.rs b/alvr/client_core/src/statistics.rs
@@ -95,8 +95,15 @@ impl StatisticsManager {
     }
 
     // vsync_queue is the latency between this call and the vsync. it cannot be measured by ALVR and
-    // should be reported by the VR runtime
-    pub fn report_submit(&mut self, target_timestamp: Duration, vsync_queue: Duration) {
+    // should be reported by the VR runtime.
+    // predicted_frame_interval is the frame interval returned by the runtime. this is more stable
+    // any any interval mearued by us.
+    pub fn report_submit(
+        &mut self,
+        target_timestamp: Duration,
+        predicted_frame_interval: Duration,
+        vsync_queue: Duration,
+    ) {
         let now = Instant::now();
 
         if let Some(frame) = self
@@ -118,6 +125,8 @@ impl StatisticsManager {
             let vsync = now + vsync_queue;
             frame.client_stats.frame_interval = vsync.saturating_duration_since(self.prev_vsync);
             self.prev_vsync = vsync;
+
+            frame.client_stats.predicted_frame_interval = predicted_frame_interval
         }
     }
 
diff --git a/alvr/client_mock/src/main.rs b/alvr/client_mock/src/main.rs
@@ -196,7 +196,7 @@ fn tracking_thread(streaming: Arc<RelaxedAtomic>, fps: f32, input: Arc<RwLock<Wi
 
         drop(input_lock);
 
-        loop_deadline += Duration::from_secs_f32(1.0 / fps / 3.0);
+        loop_deadline += Duration::from_secs_f32(1.0 / fps);
         thread::sleep(loop_deadline.saturating_duration_since(Instant::now()))
     }
 }
@@ -271,6 +271,7 @@ fn client_thread(
 
         alvr_client_core::report_submit(
             window_output.current_frame_timestamp,
+            Duration::from_secs_f32(1.0 / window_output.fps),
             Duration::from_millis(input_lock.emulated_vsync_ms),
         );
 
diff --git a/alvr/client_openxr/src/lib.rs b/alvr/client_openxr/src/lib.rs
@@ -463,6 +463,7 @@ pub fn entry_point() {
         let mut views_history = VecDeque::new();
 
         let (history_view_sender, history_view_receiver) = mpsc::channel();
+        let mut frame_interval_sender = None;
         let mut reference_space_sender = None::<mpsc::Sender<_>>;
 
         let default_view = xr::View {
@@ -692,10 +693,13 @@ pub fn entry_point() {
 
                         let (sender, reference_space_receiver) = mpsc::channel();
                         reference_space_sender = Some(sender);
+                        let (sender, frame_interval_receiver) = mpsc::channel();
+                        frame_interval_sender = Some(sender);
 
                         streaming_input_thread = Some(thread::spawn(move || {
                             let mut deadline = Instant::now();
-                            let frame_interval = Duration::from_secs_f32(1.0 / refresh_rate_hint);
+                            let mut frame_interval =
+                                Duration::from_secs_f32(1.0 / refresh_rate_hint);
 
                             while is_streaming.value() {
                                 update_streaming_input(&mut context);
@@ -704,7 +708,11 @@ pub fn entry_point() {
                                     context.reference_space = reference_space;
                                 }
 
-                                deadline += frame_interval / 3;
+                                if let Ok(interval) = frame_interval_receiver.try_recv() {
+                                    frame_interval = interval;
+                                }
+
+                                deadline += frame_interval;
                                 thread::sleep(deadline.saturating_duration_since(Instant::now()));
                             }
                         }));
@@ -832,6 +840,10 @@ pub fn entry_point() {
             let vsync_time =
                 Duration::from_nanos(frame_state.predicted_display_time.as_nanos() as _);
 
+            if let Some(sender) = &frame_interval_sender {
+                sender.send(frame_interval).ok();
+            }
+
             xr_frame_stream.begin().unwrap();
 
             if !frame_state.should_render {
@@ -901,7 +913,11 @@ pub fn entry_point() {
 
                 if !hardware_buffer.is_null() {
                     if let Some(now) = xr_runtime_now(&xr_instance) {
-                        alvr_client_core::report_submit(timestamp, vsync_time.saturating_sub(now));
+                        alvr_client_core::report_submit(
+                            timestamp,
+                            frame_interval,
+                            vsync_time.saturating_sub(now),
+                        );
                     }
                 }
 
diff --git a/alvr/packets/src/lib.rs b/alvr/packets/src/lib.rs
@@ -204,6 +204,7 @@ pub struct ClientStatistics {
     pub rendering: Duration,
     pub vsync_queue: Duration,
     pub total_pipeline_latency: Duration,
+    pub predicted_frame_interval: Duration,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
diff --git a/alvr/server/cpp/alvr_server/alvr_server.cpp b/alvr/server/cpp/alvr_server/alvr_server.cpp
@@ -227,8 +227,6 @@ void DeinitializeStreaming() {
     }
 }
 
-void SendVSync() { vr::VRServerDriverHost()->VsyncEvent(0.0); }
-
 void RequestIDR() {
     if (g_driver_provider.hmd && g_driver_provider.hmd->m_encoder) {
         g_driver_provider.hmd->m_encoder->InsertIDR();
diff --git a/alvr/server/cpp/alvr_server/bindings.h b/alvr/server/cpp/alvr_server/bindings.h
@@ -128,7 +128,6 @@ extern "C" void (*WaitForVSync)();
 extern "C" void *CppEntryPoint(const char *pInterfaceName, int *pReturnCode);
 extern "C" void InitializeStreaming();
 extern "C" void DeinitializeStreaming();
-extern "C" void SendVSync();
 extern "C" void RequestIDR();
 extern "C" void SetTracking(unsigned long long targetTimestampNs,
                             float controllerPoseTimeOffsetS,
diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs
@@ -2,7 +2,7 @@ use crate::{
     bitrate::BitrateManager,
     buttons::BUTTON_PATH_FROM_ID,
     face_tracking::FaceTrackingSink,
-    haptics,
+    haptics, openvr_props,
     sockets::WelcomeSocket,
     statistics::StatisticsManager,
     tracking::{self, TrackingManager},
@@ -24,7 +24,10 @@ use alvr_packets::{
     ButtonValue, ClientConnectionResult, ClientControlPacket, ClientListAction, ClientStatistics,
     ServerControlPacket, StreamConfigPacket, Tracking, AUDIO, HAPTICS, STATISTICS, TRACKING, VIDEO,
 };
-use alvr_session::{CodecType, ControllersEmulationMode, FrameSize, OpenvrConfig};
+use alvr_session::{
+    CodecType, ControllersEmulationMode, FrameSize, OpenvrConfig, OpenvrPropValue,
+    OpenvrPropertyKey,
+};
 use alvr_sockets::{
     spawn_cancelable, ControlSocketReceiver, ControlSocketSender, PeerType, ProtoControlSocket,
     StreamSocketBuilder, KEEPALIVE_INTERVAL,
@@ -637,8 +640,8 @@ async fn connection_pipeline(
                         crate::SetOpenvrProperty(
                             *alvr_common::HEAD_ID,
                             crate::openvr_props::to_ffi_openvr_prop(
-                                alvr_session::OpenvrPropertyKey::AudioDefaultPlaybackDeviceId,
-                                alvr_session::OpenvrPropValue::String(id),
+                                OpenvrPropertyKey::AudioDefaultPlaybackDeviceId,
+                                OpenvrPropValue::String(id),
                             ),
                         )
                     }
@@ -661,8 +664,8 @@ async fn connection_pipeline(
                         crate::SetOpenvrProperty(
                             *alvr_common::HEAD_ID,
                             crate::openvr_props::to_ffi_openvr_prop(
-                                alvr_session::OpenvrPropertyKey::AudioDefaultPlaybackDeviceId,
-                                alvr_session::OpenvrPropValue::String(id),
+                                OpenvrPropertyKey::AudioDefaultPlaybackDeviceId,
+                                OpenvrPropValue::String(id),
                             ),
                         )
                     }
@@ -686,8 +689,8 @@ async fn connection_pipeline(
                 crate::SetOpenvrProperty(
                     *alvr_common::HEAD_ID,
                     crate::openvr_props::to_ffi_openvr_prop(
-                        alvr_session::OpenvrPropertyKey::AudioDefaultRecordingDeviceId,
-                        alvr_session::OpenvrPropValue::String(id),
+                        OpenvrPropertyKey::AudioDefaultRecordingDeviceId,
+                        OpenvrPropValue::String(id),
                     ),
                 )
             }
@@ -925,6 +928,12 @@ async fn connection_pipeline(
                         network_latency,
                         decoder_latency,
                     );
+
+                    openvr_props::set_prop(
+                        *HEAD_ID,
+                        OpenvrPropertyKey::DisplayFrequency,
+                        OpenvrPropValue::Float(1.0 / stats.frame_interval_average().as_secs_f32()),
+                    );
                 }
             }
         }
diff --git a/alvr/server/src/lib.rs b/alvr/server/src/lib.rs
@@ -468,13 +468,8 @@ pub unsafe extern "C" fn HmdDriverFactory(
     }
 
     extern "C" fn wait_for_vsync() {
-        let wait_duration = STATISTICS_MANAGER
-            .lock()
-            .as_mut()
-            .map(|stats| stats.duration_until_next_vsync());
-
-        if let Some(duration) = wait_duration {
-            thread::sleep(duration);
+        if let Some(stats) = &mut *STATISTICS_MANAGER.lock() {
+            thread::sleep(stats.duration_until_next_vsync());
         }
     }
 
diff --git a/alvr/server/src/openvr_props.rs b/alvr/server/src/openvr_props.rs
@@ -2,7 +2,7 @@
 // todo: fill out more properties for headset and controllers
 // todo: add more emulation modes
 
-use crate::{FfiOpenvrProperty, FfiOpenvrPropertyValue, SERVER_DATA_MANAGER};
+use crate::{openvr_props, FfiOpenvrProperty, FfiOpenvrPropertyValue, SERVER_DATA_MANAGER};
 use alvr_common::{prelude::*, settings_schema::Switch, HEAD_ID, LEFT_HAND_ID, RIGHT_HAND_ID};
 use alvr_session::{
     ControllersEmulationMode, HeadsetEmulationMode, OpenvrPropValue,
@@ -13,6 +13,12 @@ use std::{
     ptr,
 };
 
+pub fn set_prop(device_id: u64, key: OpenvrPropertyKey, value: OpenvrPropValue) {
+    unsafe {
+        crate::SetOpenvrProperty(device_id, to_ffi_openvr_prop(key, value));
+    }
+}
+
 pub fn to_ffi_openvr_prop(key: OpenvrPropertyKey, value: OpenvrPropValue) -> FfiOpenvrProperty {
     let type_ = match value {
         OpenvrPropValue::Bool(_) => crate::FfiOpenvrPropertyType_Bool,
@@ -113,9 +119,7 @@ pub extern "C" fn set_device_openvr_props(device_id: u64) {
     if device_id == *HEAD_ID {
         fn set_prop(key: OpenvrPropertyKey, value: OpenvrPropValue) {
             info!("Setting head OpenVR prop: {key:?} => {value:?}");
-            unsafe {
-                crate::SetOpenvrProperty(*HEAD_ID, to_ffi_openvr_prop(key, value));
-            }
+            openvr_props::set_prop(*HEAD_ID, key, value);
         }
         fn set_string(key: OpenvrPropertyKey, value: &str) {
             set_prop(key, OpenvrPropValue::String(value.into()));
diff --git a/alvr/server/src/statistics.rs b/alvr/server/src/statistics.rs
@@ -11,6 +11,7 @@ const FULL_REPORT_INTERVAL: Duration = Duration::from_millis(500);
 pub struct HistoryFrame {
     target_timestamp: Duration,
     tracking_received: Instant,
+    // tracking_submitted: Instant,
     frame_present: Instant,
     frame_composed: Instant,
     frame_encoded: Instant,
@@ -23,6 +24,7 @@ impl Default for HistoryFrame {
         Self {
             target_timestamp: Duration::ZERO,
             tracking_received: now,
+            // tracking_submitted: now,
             frame_present: now,
             frame_composed: now,
             frame_encoded: now,
@@ -46,8 +48,8 @@ pub struct StatisticsManager {
     battery_gauges: HashMap<u64, f32>,
     steamvr_pipeline_latency: Duration,
     total_pipeline_latency_average: SlidingWindowAverage<Duration>,
+    predicted_frame_interval_average: SlidingWindowAverage<Duration>,
     last_vsync_time: Instant,
-    frame_interval: Duration,
 }
 
 impl StatisticsManager {
@@ -78,7 +80,10 @@ impl StatisticsManager {
                 max_history_size,
             ),
             last_vsync_time: Instant::now(),
-            frame_interval: nominal_server_frame_interval,
+            predicted_frame_interval_average: SlidingWindowAverage::new(
+                nominal_server_frame_interval,
+                max_history_size,
+            ),
         }
     }
 
@@ -100,6 +105,16 @@ impl StatisticsManager {
         }
     }
 
+    // pub fn report_tracking_submitted(&mut self, target_timestamp: Duration) {
+    //     if let Some(frame) = self
+    //         .history_buffer
+    //         .iter_mut()
+    //         .find(|frame| frame.target_timestamp == target_timestamp)
+    //     {
+    //         frame.tracking_submitted = Instant::now();
+    //     }
+    // }
+
     pub fn report_frame_present(&mut self, target_timestamp: Duration, offset: Duration) {
         if let Some(frame) = self
             .history_buffer
@@ -155,6 +170,9 @@ impl StatisticsManager {
     // Called every frame. Some statistics are reported once every frame
     // Returns network latency
     pub fn report_statistics(&mut self, client_stats: ClientStatistics) -> Duration {
+        self.predicted_frame_interval_average
+            .submit_sample(client_stats.predicted_frame_interval);
+
         if let Some(frame) = self
             .history_buffer
             .iter_mut()
@@ -275,15 +293,21 @@ impl StatisticsManager {
             .saturating_sub(self.total_pipeline_latency_average.get_average())
     }
 
+    pub fn frame_interval_average(&self) -> Duration {
+        self.predicted_frame_interval_average.get_average()
+    }
+
     // NB: this call is non-blocking, waiting should be done externally
     pub fn duration_until_next_vsync(&mut self) -> Duration {
         let now = Instant::now();
 
+        let frame_interval = self.predicted_frame_interval_average.get_average();
+
         // update the last vsync if it's too old
-        while self.last_vsync_time + self.frame_interval < now {
-            self.last_vsync_time += self.frame_interval;
+        while self.last_vsync_time + frame_interval < now {
+            self.last_vsync_time += frame_interval;
         }
 
-        (self.last_vsync_time + self.frame_interval).saturating_duration_since(now)
+        (self.last_vsync_time + frame_interval).saturating_duration_since(now)
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -197,9 +197,9 @@ pub fn get_tracker_prediction_offset() -> Duration {`
`197`	`197`	`}`
`198`	`198`	`}`
`199`	`199`
`200`		`-pub fn report_submit(target_timestamp: Duration, vsync_queue: Duration) {`
	`200`	`+pub fn report_submit(target_timestamp: Duration, frame_interval: Duration, vsync_queue: Duration) {`
`201`	`201`	`if let Some(stats) = &mut *STATISTICS_MANAGER.lock() {`
`202`		`- stats.report_submit(target_timestamp, vsync_queue);`
	`202`	`+ stats.report_submit(target_timestamp, frame_interval, vsync_queue);`
`203`	`203`
`204`	`204`	`if let Some(sender) = &*STATISTICS_SENDER.lock() {`
`205`	`205`	`if let Some(stats) = stats.summary(target_timestamp) {`
Original file line number	Diff line number	Diff line change
`@@ -196,7 +196,7 @@ fn tracking_thread(streaming: Arc<RelaxedAtomic>, fps: f32, input: Arc<RwLock<Wi`
`196`	`196`
`197`	`197`	`drop(input_lock);`
`198`	`198`
`199`		`- loop_deadline += Duration::from_secs_f32(1.0 / fps / 3.0);`
	`199`	`+ loop_deadline += Duration::from_secs_f32(1.0 / fps);`
`200`	`200`	`thread::sleep(loop_deadline.saturating_duration_since(Instant::now()))`
`201`	`201`	`}`
`202`	`202`	`}`
`@@ -271,6 +271,7 @@ fn client_thread(`
`271`	`271`
`272`	`272`	`alvr_client_core::report_submit(`
`273`	`273`	`window_output.current_frame_timestamp,`
	`274`	`+ Duration::from_secs_f32(1.0 / window_output.fps),`
`274`	`275`	`Duration::from_millis(input_lock.emulated_vsync_ms),`
`275`	`276`	`);`
`276`	`277`
Original file line number	Diff line number	Diff line change
`@@ -204,6 +204,7 @@ pub struct ClientStatistics {`
`204`	`204`	`pub rendering: Duration,`
`205`	`205`	`pub vsync_queue: Duration,`
`206`	`206`	`pub total_pipeline_latency: Duration,`
	`207`	`+ pub predicted_frame_interval: Duration,`
`207`	`208`	`}`
`208`	`209`
`209`	`210`	`#[derive(Serialize, Deserialize, Clone, Debug)]`
Original file line number	Diff line number	Diff line change
`@@ -227,8 +227,6 @@ void DeinitializeStreaming() {`
`227`	`227`	`}`
`228`	`228`	`}`
`229`	`229`
`230`		`-void SendVSync() { vr::VRServerDriverHost()->VsyncEvent(0.0); }`
`231`		`-`
`232`	`230`	`void RequestIDR() {`
`233`	`231`	`if (g_driver_provider.hmd && g_driver_provider.hmd->m_encoder) {`
`234`	`232`	`g_driver_provider.hmd->m_encoder->InsertIDR();`
Original file line number	Diff line number	Diff line change
`@@ -468,13 +468,8 @@ pub unsafe extern "C" fn HmdDriverFactory(`
`468`	`468`	`}`
`469`	`469`
`470`	`470`	`extern "C" fn wait_for_vsync() {`
`471`		`- let wait_duration = STATISTICS_MANAGER`
`472`		`- .lock()`
`473`		`- .as_mut()`
`474`		`- .map(\|stats\| stats.duration_until_next_vsync());`
`475`		`-`
`476`		`- if let Some(duration) = wait_duration {`
`477`		`- thread::sleep(duration);`
	`471`	`+ if let Some(stats) = &mut *STATISTICS_MANAGER.lock() {`
	`472`	`+ thread::sleep(stats.duration_until_next_vsync());`
`478`	`473`	`}`
`479`	`474`	`}`
`480`	`475`