Glyphon text rendering

Cargo.toml

@@ -31,7 +31,7 @@ nokhwa = { git = "https://github.com/CapSoftware/nokhwa", rev = "e309b938ccd6",
 	"serialize",
 ] }
 nokhwa-bindings-macos = { git = "https://github.com/CapSoftware/nokhwa", rev = "e309b938ccd6" }
-wgpu = "22.1.0"
+wgpu = "24.0.1"
 flume = "0.11.0"
 thiserror = "1.0"
 sentry = { version = "0.34.0", features = [

crates/editor/src/editor.rs

@@ -4,9 +4,10 @@ use cap_media::{feeds::RawCameraFrame, frame_ws::WSFrame};
 use cap_project::{BackgroundSource, CursorEvents, RecordingMeta, StudioRecordingMeta, XY};
 use cap_rendering::{
     decoder::DecodedFrame, DecodedSegmentFrames, FrameRenderer, ProjectRecordings, ProjectUniforms,
-    RenderVideoConstants,
+    RenderVideoConstants, RenderVideoState,
 };
 use tokio::{
+    sync::Mutex,
     sync::{mpsc, oneshot},
     task::JoinHandle,
 };
@@ -27,6 +28,7 @@ pub struct Renderer {
     rx: mpsc::Receiver<RendererMessage>,
     frame_tx: flume::Sender<WSFrame>,
     render_constants: Arc<RenderVideoConstants>,
+    render_state: Mutex<RenderVideoState>,
     total_frames: u32,
 }
 
@@ -37,6 +39,7 @@ pub struct RendererHandle {
 impl Renderer {
     pub fn spawn(
         render_constants: Arc<RenderVideoConstants>,
+        render_state: Mutex<RenderVideoState>,
         frame_tx: flume::Sender<WSFrame>,
         recording_meta: &RecordingMeta,
         meta: &StudioRecordingMeta,
@@ -61,6 +64,7 @@ impl Renderer {
             rx,
             frame_tx,
             render_constants,
+            render_state,
             total_frames,
         };
 
@@ -72,7 +76,8 @@ impl Renderer {
     async fn run(mut self) {
         let mut frame_task: Option<JoinHandle<()>> = None;
 
-        let mut frame_renderer = FrameRenderer::new(&self.render_constants);
+        let mut mutables = self.render_state.lock().await;
+        let mut frame_renderer = FrameRenderer::new(&self.render_constants, &mut mutables);
 
         loop {
             while let Some(msg) = self.rx.recv().await {

crates/editor/src/editor_instance.rs

@@ -9,7 +9,7 @@ use cap_project::{CursorEvents, ProjectConfiguration, RecordingMeta, RecordingMe
 use cap_project::{RecordingConfig, StudioRecordingMeta};
 use cap_rendering::{
     get_duration, ProjectRecordings, ProjectUniforms, RecordingSegmentDecoders, RenderOptions,
-    RenderVideoConstants, SegmentVideoPaths,
+    RenderVideoConstants, RenderVideoState, SegmentVideoPaths,
 };
 use std::ops::Deref;
 use std::sync::Mutex as StdMutex;
@@ -102,9 +102,11 @@ impl EditorInstance {
                 .await
                 .unwrap(),
         );
+        let render_state = Mutex::new(RenderVideoState::new(&render_constants));
 
         let renderer = Arc::new(editor::Renderer::spawn(
             render_constants.clone(),
+            render_state,
             frame_tx,
             &recording_meta,
             meta,
@@ -278,6 +280,18 @@ impl EditorInstance {
                                 frame_number,
                                 fps,
                                 resolution_base,
+                                None,
+                                // Some(cap_rendering::Text {
+                                //     x: 500.0,
+                                //     y: 500.0,
+                                //     background_color: Some([0, 255, 0]),
+                                //     background_opacity: Some(10.0),
+                                //     padding: Some(10.0),
+                                //     text_color: Some([255, 0, 0]),
+                                //     font_size: Some(60.0),
+                                //     line_height: Some(82.0),
+                                //     content: "Preview".into(),
+                                // }),
                             ),
                             segment.cursor.clone(),
                         )

crates/editor/src/playback.rs

@@ -100,6 +100,7 @@ impl Playback {
                             frame_number,
                             fps,
                             resolution_base,
+                            None, // Some("Playback".into()),
                         );
 
                         self.renderer

crates/gpu-converters/src/nv12_rgba/mod.rs

@@ -12,7 +12,7 @@ pub struct NV12ToRGBA {
 impl NV12ToRGBA {
     pub async fn new() -> Self {
         println!("NV12ToRGBA");
-        let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::default());
+        let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor::default());
 
         // Get adapter for GPU
         let adapter = instance
@@ -85,7 +85,7 @@ impl NV12ToRGBA {
             label: Some("NV12 Converter Pipeline"),
             layout: Some(&pipeline_layout),
             module: &shader,
-            entry_point: "main",
+            entry_point: Some("main"),
             compilation_options: Default::default(),
             cache: None,
         });
@@ -209,15 +209,15 @@ impl NV12ToRGBA {
 
         // Copy texture to buffer
         encoder.copy_texture_to_buffer(
-            wgpu::ImageCopyTexture {
+            wgpu::TexelCopyTextureInfo {
                 texture: &output_texture,
                 mip_level: 0,
                 origin: wgpu::Origin3d::ZERO,
                 aspect: wgpu::TextureAspect::All,
             },
-            wgpu::ImageCopyBuffer {
+            wgpu::TexelCopyBufferInfo {
                 buffer: &output_buffer,
-                layout: wgpu::ImageDataLayout {
+                layout: wgpu::TexelCopyBufferLayout {
                     offset: 0,
                     bytes_per_row: Some(width * 4),
                     rows_per_image: Some(height),

crates/gpu-converters/src/util.rs

@@ -27,15 +27,15 @@ pub fn copy_texture_to_buffer_command(
     let bytes_per_row = texture.width() * bytes_per_px;
 
     encoder.copy_texture_to_buffer(
-        wgpu::ImageCopyTexture {
+        wgpu::TexelCopyTextureInfo {
             texture,
             mip_level: 0,
             origin: wgpu::Origin3d::ZERO,
             aspect: wgpu::TextureAspect::All,
         },
-        wgpu::ImageCopyBuffer {
+        wgpu::TexelCopyBufferInfo {
             buffer: &output_buffer,
-            layout: wgpu::ImageDataLayout {
+            layout: wgpu::TexelCopyBufferLayout {
                 offset: 0,
                 bytes_per_row: Some(bytes_per_row),
                 rows_per_image: Some(texture.height()),

crates/gpu-converters/src/uyvy_nv12/mod.rs

@@ -13,7 +13,7 @@ impl UYVYToNV12 {
     pub async fn new() -> Self {
         todo!("implement UV downsampling for UYVYToNV12");
 
-        let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::default());
+        let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor::default());
 
         let adapter = instance
             .request_adapter(&wgpu::RequestAdapterOptions {
@@ -99,7 +99,7 @@ impl UYVYToNV12 {
             label: Some("YUYV Converter Pipeline"),
             layout: Some(&pipeline_layout),
             module: &shader,
-            entry_point: "main",
+            entry_point: Some("main"),
             compilation_options: Default::default(),
             cache: None,
         });

crates/gpu-converters/src/uyvy_rgba/mod.rs

@@ -14,7 +14,7 @@ pub struct UYVYToRGBA {
 
 impl UYVYToRGBA {
     pub async fn new() -> Self {
-        let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::default());
+        let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor::default());
 
         let adapter = instance
             .request_adapter(&wgpu::RequestAdapterOptions {
@@ -74,7 +74,7 @@ impl UYVYToRGBA {
             label: Some("YUYV Converter Pipeline"),
             layout: Some(&pipeline_layout),
             module: &shader,
-            entry_point: "main",
+            entry_point: Some("main"),
             compilation_options: Default::default(),
             cache: None,
         });

crates/rendering/Cargo.toml

@@ -22,6 +22,7 @@ thiserror.workspace = true
 wgpu.workspace = true
 bezier_easing = "0.1.1"
 reactive_graph = "0.1.5"
+glyphon = "0.8.0"
 
 [target.'cfg(target_os = "macos")'.dependencies]
 cidre.workspace = true

crates/rendering/src/frame_pipeline.rs

@@ -1,7 +1,7 @@
 use futures_intrusive::channel::shared::oneshot_channel;
 use wgpu::COPY_BYTES_PER_ROW_ALIGNMENT;
 
-use crate::{get_either, ProjectUniforms, RenderVideoConstants, RenderingError};
+use crate::{get_either, ProjectUniforms, RenderVideoConstants, RenderVideoState, RenderingError};
 
 pub struct FramePipeline<'a, 'b> {
     pub state: &'a mut FramePipelineState<'b>,
@@ -10,6 +10,7 @@ pub struct FramePipeline<'a, 'b> {
 
 pub struct FramePipelineState<'a> {
     pub constants: &'a RenderVideoConstants,
+    pub state: &'a mut RenderVideoState,
     pub uniforms: &'a ProjectUniforms,
     pub textures: &'a (wgpu::Texture, wgpu::Texture),
     pub texture_views: (wgpu::TextureView, wgpu::TextureView),
@@ -19,6 +20,7 @@ pub struct FramePipelineState<'a> {
 impl<'a> FramePipelineState<'a> {
     pub fn new(
         constants: &'a RenderVideoConstants,
+        state: &'a mut RenderVideoState,
         uniforms: &'a ProjectUniforms,
         textures: &'a (wgpu::Texture, wgpu::Texture),
     ) -> Self {
@@ -33,6 +35,7 @@ impl<'a> FramePipelineState<'a> {
 
         Self {
             constants,
+            state,
             uniforms,
             textures,
             texture_views,
@@ -156,15 +159,15 @@ impl FramePipelineEncoder {
         );
 
         encoder.copy_texture_to_buffer(
-            wgpu::ImageCopyTexture {
+            wgpu::TexelCopyTextureInfo {
                 texture: state.get_current_texture(),
                 mip_level: 0,
                 origin: wgpu::Origin3d::ZERO,
                 aspect: wgpu::TextureAspect::All,
             },
-            wgpu::ImageCopyBuffer {
+            wgpu::TexelCopyBufferInfo {
                 buffer: &output_buffer,
-                layout: wgpu::ImageDataLayout {
+                layout: wgpu::TexelCopyBufferLayout {
                     offset: 0,
                     bytes_per_row: Some(padded_bytes_per_row),
                     rows_per_image: Some(state.uniforms.output_size.1),

crates/rendering/src/layers/background.rs

@@ -108,14 +108,14 @@ impl BackgroundLayer {
                         });
 
                         constants.queue.write_texture(
-                            wgpu::ImageCopyTexture {
+                            wgpu::TexelCopyTextureInfo {
                                 texture: &texture,
                                 mip_level: 0,
                                 origin: wgpu::Origin3d::ZERO,
                                 aspect: wgpu::TextureAspect::All,
                             },
                             &rgba,
-                            wgpu::ImageDataLayout {
+                            wgpu::TexelCopyBufferLayout {
                                 offset: 0,
                                 bytes_per_row: Some(4 * dimensions.0),
                                 rows_per_image: Some(dimensions.1),
@@ -304,17 +304,16 @@ impl ImageBackgroundPipeline {
             ),
             vertex: wgpu::VertexState {
                 module: &shader,
-                entry_point: "vs_main",
+                entry_point: Some("vs_main"),
                 buffers: &[],
                 compilation_options: wgpu::PipelineCompilationOptions {
                     constants: &empty_constants,
                     zero_initialize_workgroup_memory: false,
-                    vertex_pulling_transform: false,
                 },
             },
             fragment: Some(wgpu::FragmentState {
                 module: &shader,
-                entry_point: "fs_main",
+                entry_point: Some("fs_main"),
                 targets: &[Some(wgpu::ColorTargetState {
                     format: wgpu::TextureFormat::Rgba8UnormSrgb,
                     blend: Some(wgpu::BlendState::REPLACE),
@@ -323,7 +322,6 @@ impl ImageBackgroundPipeline {
                 compilation_options: wgpu::PipelineCompilationOptions {
                     constants: &empty_constants,
                     zero_initialize_workgroup_memory: false,
-                    vertex_pulling_transform: false,
                 },
             }),
             primitive: wgpu::PrimitiveState {
@@ -540,17 +538,16 @@ impl BackgroundBlurPipeline {
             layout: Some(&pipeline_layout),
             vertex: wgpu::VertexState {
                 module: &shader,
-                entry_point: "vs_main",
+                entry_point: Some("vs_main"),
                 buffers: &[],
                 compilation_options: wgpu::PipelineCompilationOptions {
                     constants: &empty_constants,
                     zero_initialize_workgroup_memory: false,
-                    vertex_pulling_transform: false,
                 },
             },
             fragment: Some(wgpu::FragmentState {
                 module: &shader,
-                entry_point: "fs_main",
+                entry_point: Some("fs_main"),
                 targets: &[Some(wgpu::ColorTargetState {
                     format: wgpu::TextureFormat::Rgba8UnormSrgb,
                     blend: Some(wgpu::BlendState::REPLACE),
@@ -559,7 +556,6 @@ impl BackgroundBlurPipeline {
                 compilation_options: wgpu::PipelineCompilationOptions {
                     constants: &empty_constants,
                     zero_initialize_workgroup_memory: false,
-                    vertex_pulling_transform: false,
                 },
             }),
             primitive: wgpu::PrimitiveState {

crates/rendering/src/layers/camera.rs

@@ -15,14 +15,14 @@ impl CameraLayer {
         let constants = pipeline.state.constants;
 
         constants.queue.write_texture(
-            wgpu::ImageCopyTexture {
+            wgpu::TexelCopyTextureInfo {
                 texture: &texture,
                 mip_level: 0,
                 origin: wgpu::Origin3d::ZERO,
                 aspect: wgpu::TextureAspect::All,
             },
             camera_frame,
-            wgpu::ImageDataLayout {
+            wgpu::TexelCopyBufferLayout {
                 offset: 0,
                 bytes_per_row: Some(camera_size.x * 4),
                 rows_per_image: None,

crates/rendering/src/layers/cursor.rs

@@ -67,17 +67,16 @@ impl CursorLayer {
             ),
             vertex: wgpu::VertexState {
                 module: &shader,
-                entry_point: "vs_main",
+                entry_point: Some("vs_main"),
                 buffers: &[],
                 compilation_options: wgpu::PipelineCompilationOptions {
                     constants: &empty_constants,
                     zero_initialize_workgroup_memory: false,
-                    vertex_pulling_transform: false,
                 },
             },
             fragment: Some(wgpu::FragmentState {
                 module: &shader,
-                entry_point: "fs_main",
+                entry_point: Some("fs_main"),
                 targets: &[Some(wgpu::ColorTargetState {
                     format: wgpu::TextureFormat::Rgba8UnormSrgb,
                     blend: Some(wgpu::BlendState {
@@ -97,7 +96,6 @@ impl CursorLayer {
                 compilation_options: wgpu::PipelineCompilationOptions {
                     constants: &empty_constants,
                     zero_initialize_workgroup_memory: false,
-                    vertex_pulling_transform: false,
                 },
             }),
             primitive: wgpu::PrimitiveState {

crates/rendering/src/layers/display.rs

@@ -11,14 +11,14 @@ impl DisplayLayer {
         let frame_size = pipeline.state.constants.options.screen_size;
 
         constants.queue.write_texture(
-            wgpu::ImageCopyTexture {
+            wgpu::TexelCopyTextureInfo {
                 texture: &constants.screen_frame.0,
                 mip_level: 0,
                 origin: wgpu::Origin3d::ZERO,
                 aspect: wgpu::TextureAspect::All,
             },
             &segment_frames.screen_frame,
-            wgpu::ImageDataLayout {
+            wgpu::TexelCopyBufferLayout {
                 offset: 0,
                 bytes_per_row: Some(constants.options.screen_size.x * 4),
                 rows_per_image: None,