feat(canvas): rasterize 2D on a render thread by default (#165)

* feat(canvas): rasterize 2D on a render thread by default

2D calls now record into pictures on the JS thread; a shared render thread replays them onto
the real GPU context and presents. On a busy screen Skia's flush took ~6 ms of main-thread CPU
per frame and starved every view animating beside the canvas. Opt out with
getContext('2d', { threaded: false }) or Canvas.threaded2D = false.

- Android and iOS: threaded creation, surface updates, readbacks routed to the render thread.
- Android: 2D keeps its contents across swaps (offscreen surface copied to the window).
- Video into a threaded canvas: HardwareBuffer (Android) and CVMetalTexture (iOS), zero-copy.
- putImageData honours the dirty rect; reset() drops clips and saves.
- Busy-screen benchmark: canvas-busy page, busy-bench.sh, busy-trace.sh.

* feat(canvas): rasterize 2D on the render thread on Windows

Windows now takes the threaded 2D path by default too (Canvas.threaded2D = true): the real
context lives on the shared render thread's Direct3D 12 device, which presents; the JS thread
only records. The render thread never waits on the UI thread.

- Creation: canvas_native_context_create_d3d_threaded; the render thread keeps contexts boxed,
  so register_d3d() gets their final address. Falls back to unthreaded D3D, then CPU.
- Swapchain panels: the swapchain is made on the render thread and bound on the UI thread.
- XAML SurfaceImageSources (every alpha canvas): XamlHandoff. The render thread begins, draws
  and suspends through ISurfaceImageSourceNativeWithD2D and posts the UI thread's message-only
  window, which ends the draw (EndDraw must run on the UI thread). SetDevice stays there too.
- Presents the display was not ready for are retried on the render thread.
- Device loss: lost flags from the render thread; restore releases what still shows lost
  canvases from the UI thread, then shows the canvas again and resets the recorder's state.
  __simulateD3DDeviceRemoval and __d3dAdapterInfo cover the render thread's device.
- drawImage(video): canvas-media's shared frame is opened on the rasterizing device, waited on
  (ready fence) and given back after Skia submits (release fence), zero-copy.
- Tests: threaded_d3d (panel, resize, shared video frame) and threaded_d3d_lost.

* chore: 3.0.0-alpha.22

Author: triniwiz

apps/demo/src/main-page.xml

@@ -7,6 +7,7 @@
       <StackLayout>
         <Button text="canvas-spec" tap="{{ viewDemo }}" class="btn btn-primary view-demo"/>
         <Button text="canvas-perf" tap="{{ viewDemo }}" class="btn btn-primary view-demo"/>
+        <Button text="canvas-busy" tap="{{ viewDemo }}" class="btn btn-primary view-demo"/>
         <Button text="canvas" tap="{{ viewDemo }}" class="btn btn-primary view-demo"/>
         <Button text="canvas-babylon" tap="{{ viewDemo }}" class="btn btn-primary view-demo"/>
         <Button text="canvas-media" tap="{{ viewDemo }}" class="btn btn-primary view-demo"/>

apps/demo/src/plugin-demos/canvas-busy.ts

@@ -0,0 +1,33 @@
+import { EventData, GridLayout, Observable, Page } from '@nativescript/core';
+import { runBusyBench } from '@demo/shared';
+import { launchArgs } from '../launch-args';
+
+// adb shell am start -n org.nativescript.plugindemo/com.tns.NativeScriptActivity --es demo canvas-busy --es suite 2d+views
+export function navigatingTo(args: EventData) {
+	const page = <Page>args.object;
+	page.bindingContext = new BusyModel();
+}
+
+class BusyModel extends Observable {
+	private ran = false;
+
+	constructor() {
+		super();
+		this.set('status', 'starting…');
+	}
+
+	hostLoaded(args: EventData) {
+		if (this.ran) {
+			return;
+		}
+		this.ran = true;
+		const scenario = launchArgs.suite ?? '2d+views';
+		this.set('status', `running ${scenario}…`);
+		runBusyBench(args.object as GridLayout, scenario)
+			.then(() => this.set('status', `${scenario} done — see logcat (BUSY|…)`))
+			.catch((e) => {
+				console.log(`BUSY|${scenario}|error|${e?.message ?? e}`);
+				this.set('status', `failed: ${e?.message ?? e}`);
+			});
+	}
+}

apps/demo/src/plugin-demos/canvas-busy.xml

@@ -0,0 +1,9 @@
+<Page xmlns="http://schemas.nativescript.org/tns.xsd" navigatingTo="navigatingTo" class="page">
+	<Page.actionBar>
+		<ActionBar title="canvas busy screen" icon="" class="action-bar" />
+	</Page.actionBar>
+	<GridLayout rows="*, auto">
+		<GridLayout row="0" id="host" loaded="{{ hostLoaded }}" />
+		<Label row="1" text="{{ status }}" textWrap="true" class="p-10" />
+	</GridLayout>
+</Page>

crates/canvas-2d/Cargo.toml

@@ -29,7 +29,7 @@ bitflags = "2.6.0"
 regex-lite = { workspace = true }
 
 [target.'cfg(target_os = "windows")'.dependencies]
-windows = { workspace = true, optional = true, features = ["Win32_Graphics_Direct3D11", "Win32_Graphics_Direct3D11on12", "Win32_Graphics_Direct3D12", "Win32_Graphics_Dxgi_Common"] }
+windows = { workspace = true, optional = true, features = ["Win32_Foundation", "Win32_Graphics_Direct3D11", "Win32_Graphics_Direct3D11on12", "Win32_Graphics_Direct3D12", "Win32_Graphics_Dxgi_Common"] }
 
 [target.'cfg(any(target_os = "ios", target_os="macos", target_os = "visionos", target_os = "tvos"))'.dependencies]
 foreign-types-shared = "0.3.1"

crates/canvas-2d/src/context/drawing_images/mod.rs

@@ -48,7 +48,7 @@ impl Context {
                 )
             } else if let Some((dimensions, bytes)) = bytes {
                 let (width, height) = dimensions;
-                if let Some(image) = crate::utils::image::from_image_slice_no_copy(
+                if let Some(image) = self.asset_image(
                     bytes,
                     width as i32,
                     height as i32,
@@ -83,7 +83,7 @@ impl Context {
                 self.draw_image(image, src_rect, dst_rect)
             } else if let Some((dimensions, bytes)) = bytes {
                 let (width, height) = dimensions;
-                if let Some(image) = crate::utils::image::from_image_slice_no_copy(
+                if let Some(image) = self.asset_image(
                     bytes,
                     width as i32,
                     height as i32,
@@ -107,7 +107,7 @@ impl Context {
                 self.draw_image_dx_dy(image, x, y)
             } else if let Some((dimensions, bytes)) = bytes {
                 let (width, height) = dimensions;
-                if let Some(image) = crate::utils::image::from_image_slice_no_copy(
+                if let Some(image) = self.asset_image(
                     bytes,
                     width as i32,
                     height as i32,
@@ -139,7 +139,7 @@ impl Context {
             } else if let Some((dimensions, bytes)) = bytes {
                 let (w, h) = dimensions;
                 if let Some(image) =
-                    crate::utils::image::from_image_slice_no_copy(bytes, w as i32, h as i32)
+                    self.asset_image(bytes, w as i32, h as i32)
                 {
                     self.draw_image_dx_dy_dw_dh(&image, x, y, width, height)
                 }
@@ -162,7 +162,7 @@ impl Context {
                 self.draw_image_with_rect(image, dst_rect)
             } else if let Some((dimensions, bytes)) = bytes {
                 let (width, height) = dimensions;
-                if let Some(image) = crate::utils::image::from_image_slice_no_copy(
+                if let Some(image) = self.asset_image(
                     bytes,
                     width as i32,
                     height as i32,

crates/canvas-2d/src/context/drawing_paths/mod.rs

@@ -155,24 +155,25 @@ impl Context {
 
     pub fn clip(&mut self, path: Option<&mut Path>, fill_rule: Option<FillRule>) {
         let fill_rule = fill_rule.unwrap_or(FillRule::NonZero);
-        match path {
-            Some(path) => {
-                let current = path.fill_type();
-                path.set_fill_type(fill_rule);
-                self.surface
-                    .canvas()
-                    .clip_path(path.path(), Some(ClipOp::Intersect), Some(true));
-                path.set_fill_type_raw(current);
-            }
-            None => {
-                let current = self.path.fill_type();
-                self.path.set_fill_type(fill_rule);
-                self.surface
-                    .canvas()
-                    .clip_path(self.path.path(), Some(ClipOp::Intersect), Some(true));
-                self.path.set_fill_type_raw(current);
-            }
-        }
+        let path = match path {
+            Some(path) => path,
+            None => &mut self.path,
+        };
+        let current = path.fill_type();
+        path.set_fill_type(fill_rule);
+        let clip = path.path().clone();
+        path.set_fill_type_raw(current);
+
+        let canvas = match self.recording.as_mut() {
+            Some(recording) => recording.canvas(),
+            None => self.surface.canvas(),
+        };
+        let matrix = canvas.local_to_device_as_3x3();
+        self.state.clips.push(crate::context::recording::DeviceClip {
+            path: clip.with_transform(&matrix),
+            anti_alias: true,
+        });
+        canvas.clip_path(&clip, Some(ClipOp::Intersect), Some(true));
     }
 
     /// The point is device-space per spec, paths are user-space. Note `state.matrix`
@@ -181,7 +182,7 @@ impl Context {
         if !x.is_finite() || !y.is_finite() {
             return None;
         }
-        let matrix = self.surface.canvas().local_to_device_as_3x3();
+        let matrix = self.canvas().local_to_device_as_3x3();
         if !is_invertible(&matrix) {
             return None;
         }

crates/canvas-2d/src/context/drawing_rectangles.rs

@@ -20,6 +20,7 @@ fn clear_paint() -> &'static Paint {
 impl Context {
     #[inline]
     pub fn clear_rect(&mut self, x: c_float, y: c_float, width: c_float, height: c_float) {
+        self.discard_if_cleared(&Rect::from_xywh(x, y, width, height));
         let paint = clear_paint();
         self.render_to_canvas(paint, |canvas, paint| {
             let rect = Rect::from_xywh(x, y, width, height);

crates/canvas-2d/src/context/mod.rs

@@ -6,9 +6,8 @@ use base64::Engine;
 use skia_safe::image::CachingHint;
 use skia_safe::BlendMode;
 pub use skia_safe::ColorType;
-use skia_safe::{
-    AlphaType, Color, EncodedImageFormat, IPoint, ISize, Image, ImageInfo, Point, Surface,
-};
+pub use skia_safe::Image;
+use skia_safe::{AlphaType, Color, EncodedImageFormat, IPoint, ISize, ImageInfo, Point, Surface};
 
 use compositing::composite_operation_type::CompositeOperationType;
 use fill_and_stroke_styles::paint::Paint;
@@ -32,6 +31,7 @@ pub mod drawing_images;
 pub mod drawing_text;
 pub mod fill_and_stroke_styles;
 pub mod paths;
+pub mod recording;
 pub mod pixel_manipulation;
 pub mod text_styles;
 
@@ -91,6 +91,9 @@ pub struct State {
     pub(crate) letter_spacing: f32,
     pub(crate) matrix: skia_safe::Matrix,
     pub(crate) clip: Option<Path>,
+    /// Device space; re-applied at picture boundaries.
+    pub(crate) clips: Vec<recording::DeviceClip>,
+    pub(crate) saved_matrix: skia_safe::M44,
 }
 
 impl Default for State {
@@ -127,6 +130,8 @@ impl Default for State {
             letter_spacing: 0.,
             matrix: skia_safe::Matrix::new_identity(),
             clip: None,
+            clips: Vec::new(),
+            saved_matrix: skia_safe::M44::new_identity(),
         }
     }
 }
@@ -213,6 +218,10 @@ pub struct Context {
     // native context first leaves GrDirectContext tearing itself down against
     // an already-destroyed device.
     pub(crate) surface: Surface,
+    /// Android: `surface` is then offscreen, as a swap leaves the window's next buffer undefined.
+    /// Must drop before `direct_context`.
+    #[cfg(feature = "gl")]
+    pub(crate) window_surface: Option<Surface>,
     pub(crate) surface_state: SurfaceState,
     #[cfg(any(feature = "gl", feature = "vulkan", feature = "metal", feature = "d3d"))]
     pub(crate) direct_context: Option<skia_safe::gpu::DirectContext>,
@@ -234,6 +243,7 @@ pub struct Context {
     pub(crate) state: State,
     pub(crate) state_stack: Vec<State>,
     pub(crate) font_color: Color,
+    pub(crate) recording: Option<recording::Recording>,
 }
 
 impl Drop for Context {
@@ -376,6 +386,13 @@ impl Context {
     /// Runs before every draw. Skia uploads a raster image the moment a draw records it (a pattern
     /// fill, say), so another canvas's GL context being current puts the texture in the wrong one.
     pub(crate) fn ensure_current(&mut self) {
+        if self.recording.is_none() {
+            self.bind_surface();
+        }
+    }
+
+    #[inline]
+    pub(crate) fn bind_surface(&mut self) {
         #[cfg(feature = "gl")]
         if let Some(ref context) = self.gl_context {
             context.make_current();
@@ -386,13 +403,21 @@ impl Context {
         }
     }
 
+    #[inline]
+    pub(crate) fn canvas(&mut self) -> &skia_safe::Canvas {
+        match self.recording.as_mut() {
+            Some(recording) => recording.canvas(),
+            None => self.surface.canvas(),
+        }
+    }
+
     #[inline]
     pub fn with_canvas<F>(&mut self, f: F)
     where
         F: FnOnce(&skia_safe::Canvas),
     {
         self.ensure_current();
-        f(self.surface.canvas());
+        f(self.canvas());
     }
 
     #[inline]
@@ -401,7 +426,11 @@ impl Context {
         F: FnOnce(&skia_safe::Canvas, &mut Path),
     {
         self.ensure_current();
-        f(self.surface.canvas(), &mut self.path);
+        let canvas = match self.recording.as_mut() {
+            Some(recording) => recording.canvas(),
+            None => self.surface.canvas(),
+        };
+        f(canvas, &mut self.path);
         self.surface_state = self.surface_state | SurfaceState::Pending;
     }
 
@@ -411,7 +440,7 @@ impl Context {
         F: FnOnce(&skia_safe::Canvas),
     {
         self.ensure_current();
-        f(self.surface.canvas());
+        f(self.canvas());
         self.surface_state = self.surface_state | SurfaceState::Pending;
     }
 
@@ -421,7 +450,11 @@ impl Context {
         F: FnOnce(&skia_safe::Canvas, &Paint),
     {
         self.ensure_current();
-        f(self.surface.canvas(), &self.state.paint);
+        let canvas = match self.recording.as_mut() {
+            Some(recording) => recording.canvas(),
+            None => self.surface.canvas(),
+        };
+        f(canvas, &self.state.paint);
         self.surface_state = self.surface_state | SurfaceState::Pending;
     }
 
@@ -438,7 +471,7 @@ impl Context {
         F: FnOnce(&skia_safe::Canvas),
     {
         self.ensure_current();
-        f(self.surface.canvas());
+        f(self.canvas());
         self.surface_state = self.surface_state | SurfaceState::Pending;
     }
 
@@ -483,7 +516,7 @@ impl Context {
     }
 
     pub fn draw_pixels(&mut self, image: &Image) {
-        let canvas = self.surface.canvas();
+        let canvas = self.canvas();
         canvas.save();
         canvas.reset_matrix();
         let mut paint = skia_safe::Paint::default();
@@ -634,7 +667,7 @@ impl Context {
         layer_paint.set_blend_mode(BlendMode::SrcOver);
         let mut layer_recorder = skia_safe::PictureRecorder::new();
         layer_recorder.begin_recording(self.surface_data.bounds, false);
-        let current_matrix = self.surface.canvas().local_to_device();
+        let current_matrix = self.canvas().local_to_device();
         if let Some(layer) = layer_recorder.recording_canvas() {
             layer.set_matrix(&current_matrix);
             f(layer, &layer_paint, &mut self.path);
@@ -643,7 +676,7 @@ impl Context {
         if let Some(pict) =
             layer_recorder.finish_recording_as_picture(Some(&self.surface_data.bounds))
         {
-            let canvas = self.surface.canvas();
+            let canvas = self.canvas();
             canvas.save();
             let mut blend_paint = skia_safe::Paint::default();
             blend_paint.set_anti_alias(true);
@@ -692,7 +725,7 @@ impl Context {
         layer_paint.set_blend_mode(BlendMode::SrcOver);
         let mut layer_recorder = skia_safe::PictureRecorder::new();
         layer_recorder.begin_recording(self.surface_data.bounds, false);
-        let current_matrix = self.surface.canvas().local_to_device();
+        let current_matrix = self.canvas().local_to_device();
         if let Some(layer) = layer_recorder.recording_canvas() {
             layer.set_matrix(&current_matrix);
             f(layer, &layer_paint);
@@ -701,7 +734,7 @@ impl Context {
         if let Some(pict) =
             layer_recorder.finish_recording_as_picture(Some(&self.surface_data.bounds))
         {
-            let canvas = self.surface.canvas();
+            let canvas = self.canvas();
             canvas.save();
             let mut blend_paint = skia_safe::Paint::default();
             blend_paint.set_anti_alias(true);
@@ -732,7 +765,7 @@ impl Context {
                 | BlendMode::DstATop
                 | BlendMode::Src
         ) {
-            f(self.surface.canvas(), paint);
+            f(self.canvas(), paint);
             self.surface_state = self.surface_state | SurfaceState::Pending;
             return;
         }
@@ -758,7 +791,7 @@ impl Context {
         if let Some(pict) =
             layer_recorder.finish_recording_as_picture(Some(&self.surface_data.bounds))
         {
-            let canvas = self.surface.canvas();
+            let canvas = self.canvas();
             canvas.save();
             let mut blend_paint = skia_safe::Paint::default();
             blend_paint.set_anti_alias(true);
@@ -786,6 +819,12 @@ impl Context {
         self.state = State::default();
         self.state_stack.clear();
         self.state.direction = direction;
+        let base = if self.recording.is_some() {
+            recording::RECORDING_BASE
+        } else {
+            1
+        };
+        self.canvas().restore_to_count(base);
     }
 
     pub fn clear_canvas(&mut self) {