feature: morph targets (#126)

* feature: morph targets

* devlog

DEVLOG.md

@@ -1,6 +1,26 @@
 ---
 title: devlog
 ---
+
+## Mon Aug 26, 2024
+
+I'm adding morph targets. I've done this once before so I expect it'll be pretty easy.
+
+...
+
+[It was pretty easy](https://github.com/schell/renderling/pull/126)! 
+
+Most of my time was spent reloading the concept into my brain.
+
+But now I notice that the morph-target stress test glTF file isn't displaying properly. 
+The morph targets/animation stuff is fine, it's the texturing on the base that is off.
+
+I think it might be the texture's repeat/wrapping. It also looks like it has ambient occlusion,
+oddly enough, and it seems the AO texture is getting read.
+
+[I created a ticket to track the MorphStressTest texture
+bug](https://github.com/schell/renderling/issues/127).
+
 ## Sun Aug 25, 2024
 
 ### Fox rigging bug

crates/renderling/src/slab/cpu.rs

@@ -345,7 +345,7 @@ impl<Buffer: IsBuffer> SlabAllocator<Buffer> {
         }
     }
 
-    fn allocate_array<T: SlabItem>(&self, len: usize) -> Array<T> {
+    pub(crate) fn allocate_array<T: SlabItem>(&self, len: usize) -> Array<T> {
         if len == 0 {
             return Array::default();
         }

crates/renderling/src/stage.rs

@@ -75,6 +75,22 @@ impl Skin {
     }
 }
 
+/// A displacement target.
+///
+/// Use to displace vertices using weights defined on the mesh.
+///
+/// For more info on morph targets, see
+/// <https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html#morph-targets>
+#[derive(Clone, Copy, Default, PartialEq, SlabItem)]
+#[cfg_attr(not(target_arch = "spirv"), derive(Debug))]
+pub struct MorphTarget {
+    pub position: Vec3,
+    pub normal: Vec3,
+    pub tangent: Vec3,
+    // TODO: Extend MorphTargets to include UV and Color.
+    // I think this would take a contribution to the `gltf` crate.
+}
+
 /// A vertex in a mesh.
 #[cfg_attr(not(target_arch = "spirv"), derive(Debug))]
 #[derive(Clone, Copy, PartialEq, SlabItem)]
@@ -186,6 +202,8 @@ pub struct Renderlet {
     pub transform_id: Id<Transform>,
     pub material_id: Id<Material>,
     pub skin_id: Id<Skin>,
+    pub morph_targets: Array<Array<MorphTarget>>,
+    pub morph_weights: Array<f32>,
     pub pbr_config_id: Id<PbrConfig>,
 }
 
@@ -199,11 +217,35 @@ impl Default for Renderlet {
             transform_id: Id::NONE,
             material_id: Id::NONE,
             skin_id: Id::NONE,
+            morph_targets: Array::default(),
+            morph_weights: Array::default(),
             pbr_config_id: Id::new(0),
         }
     }
 }
 
+impl Renderlet {
+    /// Retrieve the vertex from the slab, calculating any displacement due to morph targets.
+    pub fn get_vertex(&self, vertex_index: u32, slab: &[u32]) -> Vertex {
+        let index = if self.indices_array.is_null() {
+            vertex_index as usize
+        } else {
+            slab.read(self.indices_array.at(vertex_index as usize)) as usize
+        };
+        let vertex_id = self.vertices_array.at(index);
+        let mut vertex = slab.read_unchecked(vertex_id);
+        for i in 0..self.morph_targets.len() {
+            let morph_target_array = slab.read(self.morph_targets.at(i));
+            let morph_target = slab.read(morph_target_array.at(index));
+            let weight = slab.read(self.morph_weights.at(i));
+            vertex.position += weight * morph_target.position;
+            vertex.normal += weight * morph_target.normal;
+            vertex.tangent += weight * morph_target.tangent.extend(0.0);
+        }
+        vertex
+    }
+}
+
 #[cfg(feature = "renderlet_vertex")]
 /// Renderlet vertex shader.
 #[spirv(vertex)]
@@ -233,13 +275,7 @@ pub fn renderlet_vertex(
     *out_material = renderlet.material_id;
     *out_pbr_config = renderlet.pbr_config_id;
 
-    let index = if renderlet.indices_array.is_null() {
-        vertex_index as usize
-    } else {
-        slab.read(renderlet.indices_array.at(vertex_index as usize)) as usize
-    };
-    let vertex_id = renderlet.vertices_array.at(index);
-    let vertex = slab.read_unchecked(vertex_id);
+    let vertex = renderlet.get_vertex(vertex_index, slab);
     *out_color = vertex.color;
     *out_uv0 = vertex.uv0;
     *out_uv1 = vertex.uv1;

crates/renderling/src/stage/gltf_support.rs

@@ -14,7 +14,7 @@ use crate::{
         Material,
     },
     slab::*,
-    stage::{NestedTransform, Renderlet, Skin, Stage, Vertex},
+    stage::{MorphTarget, NestedTransform, Renderlet, Skin, Stage, Vertex},
     transform::Transform,
 };
 
@@ -268,6 +268,8 @@ pub struct GltfPrimitive {
     pub vertices: HybridArray<Vertex>,
     pub bounding_box: (Vec3, Vec3),
     pub material: Id<Material>,
+    pub morph_targets: Vec<HybridArray<MorphTarget>>,
+    pub morph_targets_array: HybridArray<Array<MorphTarget>>,
 }
 
 impl GltfPrimitive {
@@ -421,6 +423,55 @@ impl GltfPrimitive {
             })
             .chain(std::iter::repeat(UNWEIGHTED_WEIGHTS))
             .take(positions.len());
+
+        // See the GLTF spec on morph targets
+        // https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html#morph-targets
+        //
+        // TODO: Generate morph target normals and tangents if absent.
+        // Although the spec says we have to generate normals or tangents if not specified,
+        // we are explicitly *not* doing that here.
+        let morph_targets: Vec<Vec<MorphTarget>> = reader
+            .read_morph_targets()
+            .map(|(may_ps, may_ns, may_ts)| {
+                let ps = may_ps
+                    .into_iter()
+                    .flat_map(|iter| iter.map(Vec3::from_array))
+                    .chain(std::iter::repeat(Vec3::ZERO))
+                    .take(positions.len());
+
+                let ns = may_ns
+                    .into_iter()
+                    .flat_map(|iter| iter.map(Vec3::from_array))
+                    .chain(std::iter::repeat(Vec3::ZERO))
+                    .take(positions.len());
+
+                let ts = may_ts
+                    .into_iter()
+                    .flat_map(|iter| iter.map(Vec3::from_array))
+                    .chain(std::iter::repeat(Vec3::ZERO))
+                    .take(positions.len());
+
+                ps.zip(ns)
+                    .zip(ts)
+                    .map(|((position, normal), tangent)| MorphTarget {
+                        position,
+                        normal,
+                        tangent,
+                    })
+                    .collect()
+            })
+            .collect();
+        log::debug!(
+            "  {} morph_targets: {:?}",
+            morph_targets.len(),
+            morph_targets.iter().map(|mt| mt.len()).collect::<Vec<_>>()
+        );
+        let morph_targets = morph_targets
+            .into_iter()
+            .map(|verts| stage.new_array(verts))
+            .collect::<Vec<_>>();
+        let morph_targets_array = stage.new_array(morph_targets.iter().map(HybridArray::array));
+
         let vs = joints.into_iter().zip(weights);
         let vs = colors.zip(vs);
         let vs = tangents.into_iter().zip(vs);
@@ -457,13 +508,17 @@ impl GltfPrimitive {
             indices,
             material,
             bounding_box: (min, max),
+            morph_targets,
+            morph_targets_array,
         }
     }
 }
 
 #[derive(Debug)]
 pub struct GltfMesh {
+    /// Mesh primitives, aka meshlets
     pub primitives: Vec<GltfPrimitive>,
+    /// Morph target weights
     pub weights: HybridArray<f32>,
 }
 
@@ -572,18 +627,12 @@ pub struct GltfNode {
     ///
     /// Each element indexes into the `GltfDocument`'s `nodes` field.
     pub children: Vec<usize>,
-    /// Array of weights
+    /// Array of morph target weights
     pub weights: HybridArray<f32>,
     /// This node's transform.
     pub transform: NestedTransform,
 }
 
-impl From<&GltfNode> for (usize, NestedTransform) {
-    fn from(node: &GltfNode) -> Self {
-        (node.index, node.transform.clone())
-    }
-}
-
 impl GltfNode {
     pub fn global_transform(&self) -> Transform {
         self.transform.get_global_transform()
@@ -698,7 +747,7 @@ impl GltfDocument {
         let mut images = images.into_iter().map(AtlasImage::from).collect::<Vec<_>>();
 
         let num_textures = document.textures().len();
-        log::debug!("Preallocating an array of {num_textures} textures");
+        log::debug!("\nPreallocating an array of {num_textures} textures");
         let textures = stage.new_array(vec![AtlasTexture::default(); num_textures]);
 
         log::debug!("Creating materials");
@@ -738,6 +787,14 @@ impl GltfDocument {
             textures.set_item(i, texture);
         }
 
+        log::debug!("Loading meshes");
+        let mut meshes = vec![];
+        for mesh in document.meshes() {
+            let mesh = GltfMesh::from_gltf(stage, &buffer_data, &materials, mesh);
+            meshes.push(mesh);
+        }
+        log::trace!("  loaded {} meshes", meshes.len());
+
         log::debug!("Loading {} nodes", document.nodes().count());
         let mut nodes = vec![];
         let mut node_transforms = HashMap::<usize, NestedTransform>::new();
@@ -800,7 +857,21 @@ impl GltfDocument {
             let camera = node.camera().map(|camera| camera.index());
             let light = node.light().map(|light| light.index());
             let weights = node.weights().map(|w| w.to_vec()).unwrap_or_default();
-            let weights = stage.new_array(weights);
+            // From the glTF spec:
+            //
+            // A mesh with morph targets MAY also define an optional mesh.weights property
+            // that stores the default targets' weights. These weights MUST be used when
+            // node.weights is undefined. When mesh.weights is undefined, the default
+            // targets' weights are zeros.
+            let weights = if weights.is_empty() {
+                if let Some(mesh) = node.mesh() {
+                    meshes[mesh.index()].weights.clone()
+                } else {
+                    stage.new_array(weights)
+                }
+            } else {
+                stage.new_array(weights)
+            };
             let transform = transform_for_node(0, stage, &mut node_transforms, &node);
             nodes.push(GltfNode {
                 index: node.index(),
@@ -832,14 +903,6 @@ impl GltfDocument {
             cameras.push(GltfCamera::new(stage, camera, transform));
         }
 
-        log::debug!("Loading meshes");
-        let mut meshes = vec![];
-        for mesh in document.meshes() {
-            let mesh = GltfMesh::from_gltf(stage, &buffer_data, &materials, mesh);
-            meshes.push(mesh);
-        }
-        log::trace!("  loaded {} meshes", meshes.len());
-
         log::trace!("Loading lights");
         let mut lights = vec![];
         if let Some(gltf_lights) = document.lights() {
@@ -950,6 +1013,8 @@ impl GltfDocument {
                         material_id: prim.material,
                         camera_id,
                         skin_id,
+                        morph_targets: prim.morph_targets_array.array(),
+                        morph_weights: gltf_node.weights.array(),
                         ..Default::default()
                     });
                     log::debug!("    created renderlet {i}/{num_prims}: {:#?}", hybrid.get());