Implement dynamic framerate

Reduce tracking frequency to head fps

Author: zmerp

alvr/client_core/src/c_api.rs

@@ -398,9 +398,14 @@ pub extern "C" fn alvr_get_tracker_prediction_offset_ns() -> u64 {
 }
 
 #[no_mangle]
-pub extern "C" fn alvr_report_submit(target_timestamp_ns: u64, vsync_queue_ns: u64) {
+pub extern "C" fn alvr_report_submit(
+    target_timestamp_ns: u64,
+    frame_interval_ns: u64,
+    vsync_queue_ns: u64,
+) {
     crate::report_submit(
         Duration::from_nanos(target_timestamp_ns),
+        Duration::from_nanos(frame_interval_ns),
         Duration::from_nanos(vsync_queue_ns),
     );
 }

alvr/client_core/src/lib.rs

@@ -197,9 +197,9 @@ pub fn get_tracker_prediction_offset() -> Duration {
     }
 }
 
-pub fn report_submit(target_timestamp: Duration, vsync_queue: Duration) {
+pub fn report_submit(target_timestamp: Duration, frame_interval: Duration, vsync_queue: Duration) {
     if let Some(stats) = &mut *STATISTICS_MANAGER.lock() {
-        stats.report_submit(target_timestamp, vsync_queue);
+        stats.report_submit(target_timestamp, frame_interval, vsync_queue);
 
         if let Some(sender) = &*STATISTICS_SENDER.lock() {
             if let Some(stats) = stats.summary(target_timestamp) {

alvr/client_core/src/statistics.rs

@@ -95,8 +95,15 @@ impl StatisticsManager {
     }
 
     // vsync_queue is the latency between this call and the vsync. it cannot be measured by ALVR and
-    // should be reported by the VR runtime
-    pub fn report_submit(&mut self, target_timestamp: Duration, vsync_queue: Duration) {
+    // should be reported by the VR runtime.
+    // predicted_frame_interval is the frame interval returned by the runtime. this is more stable
+    // any any interval mearued by us.
+    pub fn report_submit(
+        &mut self,
+        target_timestamp: Duration,
+        predicted_frame_interval: Duration,
+        vsync_queue: Duration,
+    ) {
         let now = Instant::now();
 
         if let Some(frame) = self
@@ -118,6 +125,8 @@ impl StatisticsManager {
             let vsync = now + vsync_queue;
             frame.client_stats.frame_interval = vsync.saturating_duration_since(self.prev_vsync);
             self.prev_vsync = vsync;
+
+            frame.client_stats.predicted_frame_interval = predicted_frame_interval
         }
     }
 

alvr/client_mock/src/main.rs

@@ -196,7 +196,7 @@ fn tracking_thread(streaming: Arc<RelaxedAtomic>, fps: f32, input: Arc<RwLock<Wi
 
         drop(input_lock);
 
-        loop_deadline += Duration::from_secs_f32(1.0 / fps / 3.0);
+        loop_deadline += Duration::from_secs_f32(1.0 / fps);
         thread::sleep(loop_deadline.saturating_duration_since(Instant::now()))
     }
 }
@@ -271,6 +271,7 @@ fn client_thread(
 
         alvr_client_core::report_submit(
             window_output.current_frame_timestamp,
+            Duration::from_secs_f32(1.0 / window_output.fps),
             Duration::from_millis(input_lock.emulated_vsync_ms),
         );
 

alvr/client_openxr/src/lib.rs

@@ -463,6 +463,7 @@ pub fn entry_point() {
         let mut views_history = VecDeque::new();
 
         let (history_view_sender, history_view_receiver) = mpsc::channel();
+        let mut frame_interval_sender = None;
         let mut reference_space_sender = None::<mpsc::Sender<_>>;
 
         let default_view = xr::View {
@@ -692,10 +693,13 @@ pub fn entry_point() {
 
                         let (sender, reference_space_receiver) = mpsc::channel();
                         reference_space_sender = Some(sender);
+                        let (sender, frame_interval_receiver) = mpsc::channel();
+                        frame_interval_sender = Some(sender);
 
                         streaming_input_thread = Some(thread::spawn(move || {
                             let mut deadline = Instant::now();
-                            let frame_interval = Duration::from_secs_f32(1.0 / refresh_rate_hint);
+                            let mut frame_interval =
+                                Duration::from_secs_f32(1.0 / refresh_rate_hint);
 
                             while is_streaming.value() {
                                 update_streaming_input(&mut context);
@@ -704,7 +708,11 @@ pub fn entry_point() {
                                     context.reference_space = reference_space;
                                 }
 
-                                deadline += frame_interval / 3;
+                                if let Ok(interval) = frame_interval_receiver.try_recv() {
+                                    frame_interval = interval;
+                                }
+
+                                deadline += frame_interval;
                                 thread::sleep(deadline.saturating_duration_since(Instant::now()));
                             }
                         }));
@@ -832,6 +840,10 @@ pub fn entry_point() {
             let vsync_time =
                 Duration::from_nanos(frame_state.predicted_display_time.as_nanos() as _);
 
+            if let Some(sender) = &frame_interval_sender {
+                sender.send(frame_interval).ok();
+            }
+
             xr_frame_stream.begin().unwrap();
 
             if !frame_state.should_render {
@@ -901,7 +913,11 @@ pub fn entry_point() {
 
                 if !hardware_buffer.is_null() {
                     if let Some(now) = xr_runtime_now(&xr_instance) {
-                        alvr_client_core::report_submit(timestamp, vsync_time.saturating_sub(now));
+                        alvr_client_core::report_submit(
+                            timestamp,
+                            frame_interval,
+                            vsync_time.saturating_sub(now),
+                        );
                     }
                 }
 

alvr/common/src/average.rs

@@ -1,4 +1,9 @@
-use std::{collections::VecDeque, time::Duration};
+use glam::Vec2;
+use std::{
+    collections::VecDeque,
+    f32::consts::PI,
+    time::{Duration, Instant},
+};
 
 pub struct SlidingWindowAverage<T> {
     history_buffer: VecDeque<T>,
@@ -33,3 +38,83 @@ impl SlidingWindowAverage<Duration> {
         self.history_buffer.iter().sum::<Duration>() / self.history_buffer.len() as u32
     }
 }
+
+pub enum DurationOffset {
+    Positive(Duration),
+    Negative(Duration),
+}
+
+// Calculate average time phase. The average is calulated in the complex domain and returned back as
+// the phase time offset. Only the phase samples are stored (as a complex number), not the frame
+// interval, since it's useless for these calculations.
+pub struct SlidingTimePhaseAverage {
+    last_time_sample: Instant,
+    last_sample: Vec2,
+    history_buffer: VecDeque<Vec2>,
+    max_history_size: usize,
+}
+
+impl SlidingTimePhaseAverage {
+    pub fn new(max_history_size: usize) -> Self {
+        Self {
+            last_time_sample: Instant::now(),
+            last_sample: Vec2::new(1.0, 0.0),
+            history_buffer: [].into_iter().collect(),
+            max_history_size,
+        }
+    }
+
+    // The sample is actually the time of this call.
+    pub fn submit_sample(&mut self, frame_interval: Duration) {
+        let time_sample = Instant::now();
+
+        let phase_sample = ((time_sample - self.last_time_sample).as_secs_f32()
+            / frame_interval.as_secs_f32())
+        .fract()
+            * 2.0
+            * PI;
+
+        let complex_sample = Vec2::new(f32::cos(phase_sample), f32::sin(phase_sample));
+
+        if self.history_buffer.len() >= self.max_history_size {
+            self.history_buffer.pop_front();
+        }
+
+        self.history_buffer.push_back(complex_sample);
+
+        self.last_time_sample = time_sample;
+        self.last_sample = complex_sample
+    }
+
+    // The reference frame of the phase average is an implementation detail. This method returns the
+    // phase offset wrt the time of this call.
+    pub fn get_average_diff(&self, frame_interval: Duration) -> DurationOffset {
+        let now = Instant::now();
+
+        let current_phase =
+            ((now - self.last_time_sample).as_secs_f32() / frame_interval.as_secs_f32()).fract()
+                * 2.0
+                * PI;
+
+        // let current_complex_phase = Vec2::new(f32::cos(current_phase), f32::sin(current_phase));
+
+        // Note: no need to normalize
+        let average_complex = self.history_buffer.iter().sum::<Vec2>();
+        let average_phase = f32::atan2(average_complex.y, average_complex.x);
+
+        let phase_diff = current_phase - average_phase;
+
+        // Nomalized between -PI and +PI
+        let normalized_phase_diff = (phase_diff + PI).rem_euclid(2.0 * PI) - PI;
+
+        if normalized_phase_diff.is_sign_positive() {
+            DurationOffset::Positive(Duration::from_secs_f32(
+                normalized_phase_diff * frame_interval.as_secs_f32(),
+            ))
+        } else {
+            DurationOffset::Negative(Duration::from_secs_f32(
+                -normalized_phase_diff * frame_interval.as_secs_f32(),
+            ))
+        }
+    }
+}

alvr/packets/src/lib.rs

@@ -204,6 +204,7 @@ pub struct ClientStatistics {
     pub rendering: Duration,
     pub vsync_queue: Duration,
     pub total_pipeline_latency: Duration,
+    pub predicted_frame_interval: Duration,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]

alvr/server/cpp/alvr_server/alvr_server.cpp

@@ -227,14 +227,17 @@ void DeinitializeStreaming() {
     }
 }
 
-void SendVSync() { vr::VRServerDriverHost()->VsyncEvent(0.0); }
-
 void RequestIDR() {
     if (g_driver_provider.hmd && g_driver_provider.hmd->m_encoder) {
         g_driver_provider.hmd->m_encoder->InsertIDR();
     }
 }
 
+// Linux-only
+void NotifyVsync() {
+    // todo
+}
+
 void SetTracking(unsigned long long targetTimestampNs,
                  float controllerPoseTimeOffsetS,
                  const FfiDeviceMotion *deviceMotions,

alvr/server/cpp/alvr_server/bindings.h

@@ -128,8 +128,8 @@ extern "C" void (*WaitForVSync)();
 extern "C" void *CppEntryPoint(const char *pInterfaceName, int *pReturnCode);
 extern "C" void InitializeStreaming();
 extern "C" void DeinitializeStreaming();
-extern "C" void SendVSync();
 extern "C" void RequestIDR();
+extern "C" void NotifyVsync();
 extern "C" void SetTracking(unsigned long long targetTimestampNs,
                             float controllerPoseTimeOffsetS,
                             const FfiDeviceMotion *deviceMotions,

alvr/server/src/connection.rs

@@ -2,13 +2,13 @@ use crate::{
     bitrate::BitrateManager,
     buttons::BUTTON_PATH_FROM_ID,
     face_tracking::FaceTrackingSink,
-    haptics,
+    haptics, openvr_props, phase_sync,
     sockets::WelcomeSocket,
     statistics::StatisticsManager,
     tracking::{self, TrackingManager},
     FfiButtonValue, FfiFov, FfiViewsConfig, VideoPacket, BITRATE_MANAGER, CONTROL_CHANNEL_SENDER,
-    DECODER_CONFIG, DISCONNECT_CLIENT_NOTIFIER, HAPTICS_SENDER, RESTART_NOTIFIER,
-    SERVER_DATA_MANAGER, STATISTICS_MANAGER, VIDEO_RECORDING_FILE, VIDEO_SENDER,
+    DECODER_CONFIG, DISCONNECT_CLIENT_NOTIFIER, HAPTICS_SENDER, PHASE_SYNC_MANAGER,
+    RESTART_NOTIFIER, SERVER_DATA_MANAGER, STATISTICS_MANAGER, VIDEO_RECORDING_FILE, VIDEO_SENDER,
 };
 use alvr_audio::AudioDevice;
 use alvr_common::{
@@ -24,7 +24,10 @@ use alvr_packets::{
     ButtonValue, ClientConnectionResult, ClientControlPacket, ClientListAction, ClientStatistics,
     ServerControlPacket, StreamConfigPacket, Tracking, AUDIO, HAPTICS, STATISTICS, TRACKING, VIDEO,
 };
-use alvr_session::{CodecType, ControllersEmulationMode, FrameSize, OpenvrConfig};
+use alvr_session::{
+    CodecType, ControllersEmulationMode, FrameSize, OpenvrConfig, OpenvrPropValue,
+    OpenvrPropertyKey,
+};
 use alvr_sockets::{
     spawn_cancelable, ControlSocketReceiver, ControlSocketSender, PeerType, ProtoControlSocket,
     StreamSocketBuilder, KEEPALIVE_INTERVAL,
@@ -637,8 +640,8 @@ async fn connection_pipeline(
                         crate::SetOpenvrProperty(
                             *alvr_common::HEAD_ID,
                             crate::openvr_props::to_ffi_openvr_prop(
-                                alvr_session::OpenvrPropertyKey::AudioDefaultPlaybackDeviceId,
-                                alvr_session::OpenvrPropValue::String(id),
+                                OpenvrPropertyKey::AudioDefaultPlaybackDeviceId,
+                                OpenvrPropValue::String(id),
                             ),
                         )
                     }
@@ -661,8 +664,8 @@ async fn connection_pipeline(
                         crate::SetOpenvrProperty(
                             *alvr_common::HEAD_ID,
                             crate::openvr_props::to_ffi_openvr_prop(
-                                alvr_session::OpenvrPropertyKey::AudioDefaultPlaybackDeviceId,
-                                alvr_session::OpenvrPropValue::String(id),
+                                OpenvrPropertyKey::AudioDefaultPlaybackDeviceId,
+                                OpenvrPropValue::String(id),
                             ),
                         )
                     }
@@ -686,8 +689,8 @@ async fn connection_pipeline(
                 crate::SetOpenvrProperty(
                     *alvr_common::HEAD_ID,
                     crate::openvr_props::to_ffi_openvr_prop(
-                        alvr_session::OpenvrPropertyKey::AudioDefaultRecordingDeviceId,
-                        alvr_session::OpenvrPropValue::String(id),
+                        OpenvrPropertyKey::AudioDefaultRecordingDeviceId,
+                        OpenvrPropValue::String(id),
                     ),
                 )
             }
@@ -917,6 +920,7 @@ async fn connection_pipeline(
                 if let Some(stats) = &mut *STATISTICS_MANAGER.lock() {
                     let timestamp = client_stats.target_timestamp;
                     let decoder_latency = client_stats.video_decode;
+                    let predicted_frame_interval = client_stats.predicted_frame_interval;
                     let network_latency = stats.report_statistics(client_stats);
 
                     BITRATE_MANAGER.lock().report_frame_latencies(
@@ -925,6 +929,17 @@ async fn connection_pipeline(
                         network_latency,
                         decoder_latency,
                     );
+
+                    let mut phase_sync_lock = PHASE_SYNC_MANAGER.lock();
+                    phase_sync_lock.report_predicted_frame_interval(predicted_frame_interval);
+
+                    openvr_props::set_prop(
+                        *HEAD_ID,
+                        OpenvrPropertyKey::DisplayFrequency,
+                        OpenvrPropValue::Float(
+                            1.0 / phase_sync_lock.frame_interval_average().as_secs_f32(),
+                        ),
+                    );
                 }
             }
         }