diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml new file mode 100644 index 0000000..e31d81c --- /dev/null +++ b/.github/workflows/jekyll-gh-pages.yml @@ -0,0 +1,51 @@ +# Sample workflow for building and deploying a Jekyll site to GitHub Pages +name: Deploy Jekyll with GitHub Pages dependencies preinstalled + +on: + # Runs on pushes targeting the default branch + push: + branches: ["main"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + # Build job + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Pages + uses: actions/configure-pages@v5 + - name: Build with Jekyll + uses: actions/jekyll-build-pages@v1 + with: + source: ./ + destination: ./_site + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + + # Deployment job + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/npm-grunt.yml b/.github/workflows/npm-grunt.yml new file mode 100644 index 0000000..5ef7a9a --- /dev/null +++ b/.github/workflows/npm-grunt.yml @@ -0,0 +1,28 @@ +name: NodeJS with Grunt + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + build: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [18.x, 20.x, 22.x] + + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + - name: Build + run: | + npm install + grunt diff --git a/README.md b/README.md index f99cdff..b0970bf 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,44 @@ # Project5-WebGPU-Gaussian-Splat-Viewer -**University of Pennsylvania, CIS 565: GPU Programming and Architecture, Project 4** +**University of Pennsylvania, CIS 565: GPU Programming and Architecture, Project 5** -* (TODO) YOUR NAME HERE -* Tested on: (TODO) **Google Chrome 222.2** on - Windows 22, i7-2222 @ 2.22GHz 22GB, GTX 222 222MB (Moore 2222 Lab) +* Maya Diaz Huizar +* Tested on: Google Chrome 132.0 on Windows 10, AMD Ryzen 9 5900X @ 3.7GHz 32GB RAM, Nvidia RTX 3080 10GB -### Live Demo +## Live Demo (requires Chrome and a WebGPU compatible GPU) +[Demo Link](https://aorus1.github.io/Project5-WebGPU-Gaussian-Splat-Viewer/) +[Sample Files](https://drive.google.com/drive/folders/1rwtArEbj7GfjMeK6mD3a4QN6ZFLKQoBB?usp=sharing) -[![](img/thumb.png)](http://TODO.github.io/Project4-WebGPU-Forward-Plus-and-Clustered-Deferred) +## Demo Video -### Demo Video/GIF +[![Demo Video]()](images/demo.mp4) -[![](img/video.mp4)](TODO) +## Project Overview -### (TODO: Your README) +The WebGPU Gaussian Splat Viewer is a 3D renderer for Gaussian splats, designed to display and visualize point cloud data using a Gaussian splatting technique. Gaussian splats allow for smooth, realistic rendering of point clouds with transparency effects. This viewer is built in WebGPU, featuring a Gaussian renderer that includes preprocessing and rendering pipelines, depth sorting, and spherical harmonics-based shading. -*DO NOT* leave the README to the last minute! It is a crucial part of the -project, and we will not be able to grade you without a good README. +## Features -This assignment has a considerable amount of performance analysis compared -to implementation work. Complete the implementation early to leave time! +### Gaussian Preprocessing Pipeline +The preprocessing pipeline handles 3D Gaussian data before rendering: +- **View-Frustum Culling**: Removes non-visible Gaussians outside the camera’s view. +- **3D to 2D Transformation**: Transforms Gaussian points into 2D screen space for rendering. +- **Covariance Calculation**: Computes a 2D conic for each Gaussian based on a covariance matrix derived from 3D rotation and scaling data. +- **Opacity Scaling**: Applies a sigmoid function to Gaussian opacities for realistic transparency. +- **Depth Sorting**: Uses a GPU-based radix sort to sort Gaussians by depth for accurate transparency effects. -### Credits +## Performance Analysis +- **Point Cloud vs. Gaussian Renderer**: Gaussian splatting improves rendering quality for point clouds, with spherical harmonics and Gaussian opacity scaling providing smoother, more realistic visuals. However, the Gaussian renderer is more computationally intensive than direct point cloud rendering due to shading and transparency effects. Performance is obviously and noticeably worse, but this is understandable. + +- **Effect of Workgroup Size on Performance**: Workgroup size significantly impacts performance in the Gaussian renderer. I found the default parameter given to be the most performant, which makes sense as its likely a nice middle ground. +- **View-Frustum Culling**: Culling offers a substantial performance boost by eliminating off-screen Gaussians from rendering, reducing the number of draw calls and unnecessary GPU computations. The speedup is substantial because a significant number of threads can be abandoned, leading to faster draws. + +- **Gaussian Count and Performance**: As the number of Gaussians increases, rendering performance decreases due to increased depth sorting and fragment processing requirements. Testing with varying Gaussian counts reveals the expected worse performance with increased Gaussian counts. + +## Credits - [Vite](https://vitejs.dev/) -- [tweakpane](https://tweakpane.github.io/docs//v3/monitor-bindings/) -- [stats.js](https://github.com/mrdoob/stats.js) +- [tweakpane](https://cocopon.github.io/tweakpane/) - [wgpu-matrix](https://github.com/greggman/wgpu-matrix) -- Special Thanks to: Shrek Shao (Google WebGPU team) & [Differential Guassian Renderer](https://github.com/graphdeco-inria/diff-gaussian-rasterization) +- [WebGPU Developer Tools](https://chrome.google.com/webstore/detail/webgpu-developer-tools/) +- Special thanks to Shrek Shao from Google WebGPU and [Differential Gaussian Renderer](https://github.com/graphdeco-inria/diff-gaussian-rasterization) for inspiration and resources. diff --git a/images/demo.mp4 b/images/demo.mp4 new file mode 100644 index 0000000..e5d5b16 Binary files /dev/null and b/images/demo.mp4 differ diff --git a/package-lock.json b/package-lock.json index 04843bd..694c409 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@loaders.gl/ply": "^4.2.2", "@petamoriken/float16": "^3.8.7", "tweakpane": "^3.1.8", + "tweakpane-plugin-file-import": "^0.2.0", "wgpu-matrix": "^3.2.0" }, "devDependencies": { diff --git a/src/camera/camera.ts b/src/camera/camera.ts index 47ea1dc..840ccc7 100644 --- a/src/camera/camera.ts +++ b/src/camera/camera.ts @@ -64,8 +64,8 @@ interface CameraPreset { rotation: Mat4, } -export async function load_camera_presets(file: string): Promise { - const blob = new Blob([file]); +export async function load_camera_presets(file: File): Promise { + const blob = new Blob([file], { type: file.type }); const arrayBuffer = await new Promise((resolve, reject) => { const reader = new FileReader(); diff --git a/src/renderers/gaussian-renderer.ts b/src/renderers/gaussian-renderer.ts index 1684523..5466164 100644 --- a/src/renderers/gaussian-renderer.ts +++ b/src/renderers/gaussian-renderer.ts @@ -1,14 +1,14 @@ import { PointCloud } from '../utils/load'; import preprocessWGSL from '../shaders/preprocess.wgsl'; import renderWGSL from '../shaders/gaussian.wgsl'; -import { get_sorter,c_histogram_block_rows,C } from '../sort/sort'; +import { get_sorter, c_histogram_block_rows, C } from '../sort/sort'; import { Renderer } from './renderer'; export interface GaussianRenderer extends Renderer { - + render_settings_buffer: GPUBuffer, } -// Utility to create GPU buffers +// Utility function to create GPU buffers with optional data initialization const createBuffer = ( device: GPUDevice, label: string, @@ -28,17 +28,16 @@ export default function get_renderer( camera_buffer: GPUBuffer, ): GaussianRenderer { + // Initialize sorter for GPU-based sorting of point cloud data const sorter = get_sorter(pc.num_points, device); - - // =============================================== - // Initialize GPU Buffers - // =============================================== + // Initialize GPU Buffers const nulling_data = new Uint32Array([0]); + const null_buffer = createBuffer(device, 'null buffer', 4, GPUBufferUsage.COPY_SRC | GPUBufferUsage.COPY_DST, nulling_data); + const splatBuffer = createBuffer(device, 'splat buffer', pc.num_points * 96, GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST); + const render_settings_buffer = createBuffer(device, 'render settings buffer', 4 * 2, GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, new Float32Array([1.0, pc.sh_deg])); - // =============================================== - // Create Compute Pipeline and Bind Groups - // =============================================== + // Create Compute Pipeline and Bind Groups const preprocess_pipeline = device.createComputePipeline({ label: 'preprocess', layout: 'auto', @@ -52,35 +51,101 @@ export default function get_renderer( }, }); - const sort_bind_group = device.createBindGroup({ - label: 'sort', - layout: preprocess_pipeline.getBindGroupLayout(2), + const preprocess_bind_group = device.createBindGroup({ + label: 'preprocess bind group', + layout: preprocess_pipeline.getBindGroupLayout(0), entries: [ { binding: 0, resource: { buffer: sorter.sort_info_buffer } }, { binding: 1, resource: { buffer: sorter.ping_pong[0].sort_depths_buffer } }, { binding: 2, resource: { buffer: sorter.ping_pong[0].sort_indices_buffer } }, { binding: 3, resource: { buffer: sorter.sort_dispatch_indirect_buffer } }, + { binding: 4, resource: { buffer: camera_buffer } }, + { binding: 5, resource: { buffer: pc.gaussian_3d_buffer } }, + { binding: 6, resource: { buffer: splatBuffer } }, + { binding: 7, resource: { buffer: render_settings_buffer } }, + { binding: 8, resource: { buffer: pc.sh_buffer } }, + ], + }); + + // Create Render Pipeline and Bind Groups + const render_pipeline = device.createRenderPipeline({ + label: 'render gaussian', + layout: 'auto', + vertex: { + module: device.createShaderModule({ code: renderWGSL }), + entryPoint: 'vs_main' + }, + fragment: { + module: device.createShaderModule({ code: renderWGSL }), + entryPoint: 'fs_main', + targets: [{ + format: presentation_format, + blend: { + color: { + srcFactor: 'one', + dstFactor: 'one-minus-src-alpha', + operation: 'add', + }, + alpha: { + srcFactor: 'one', + dstFactor: 'one-minus-src-alpha', + operation: 'add', + }, + }, + }], + } + }); + + const render_bind_group = device.createBindGroup({ + layout: render_pipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: { buffer: splatBuffer } }, + { binding: 1, resource: { buffer: sorter.ping_pong[0].sort_indices_buffer } }, + { binding: 2, resource: { buffer: camera_buffer } }, ], }); + const indirect_draw_data = new Uint32Array([6, 0, 0, 0]); + const indirect_draw_buffer = createBuffer(device, 'indirect draw buffer', 4 * 4, GPUBufferUsage.INDIRECT | GPUBufferUsage.COPY_DST, indirect_draw_data); - // =============================================== - // Create Render Pipeline and Bind Groups - // =============================================== - + // Command Encoder Functions + const preprocess = (encoder: GPUCommandEncoder) => { + const preprocess_pass = encoder.beginComputePass(); + preprocess_pass.setPipeline(preprocess_pipeline); + preprocess_pass.setBindGroup(0, preprocess_bind_group); + preprocess_pass.dispatchWorkgroups(Math.ceil(pc.num_points / C.histogram_wg_size), 1, 1); + preprocess_pass.end(); + }; - // =============================================== - // Command Encoder Functions - // =============================================== - + const render = (encoder: GPUCommandEncoder, texture_view: GPUTextureView) => { + const pass = encoder.beginRenderPass({ + label: "render gaussian pass", + colorAttachments: [{ + view: texture_view, + loadOp: 'clear', + storeOp: 'store', + }], + }); + pass.setPipeline(render_pipeline); + pass.setBindGroup(0, render_bind_group); + pass.drawIndirect(indirect_draw_buffer, 0); + pass.end(); + }; - // =============================================== - // Return Render Object - // =============================================== + // Return Render Object with Frame Function return { frame: (encoder: GPUCommandEncoder, texture_view: GPUTextureView) => { + encoder.copyBufferToBuffer(null_buffer, 0, sorter.sort_info_buffer, 0, 4); + encoder.copyBufferToBuffer(null_buffer, 0, sorter.sort_dispatch_indirect_buffer, 0, 4); + + preprocess(encoder); sorter.sort(encoder); + + encoder.copyBufferToBuffer(sorter.sort_info_buffer, 0, indirect_draw_buffer, 4, 4); + + render(encoder, texture_view); }, camera_buffer, + render_settings_buffer, }; } diff --git a/src/renderers/point-cloud-renderer.ts b/src/renderers/point-cloud-renderer.ts index 36a2e8e..89ac6da 100644 --- a/src/renderers/point-cloud-renderer.ts +++ b/src/renderers/point-cloud-renderer.ts @@ -6,8 +6,13 @@ export default function get_renderer( pc: PointCloud, device: GPUDevice, presentation_format: GPUTextureFormat, - camera_buffer: GPUBuffer): Renderer { - const render_shader = device.createShaderModule({code: pointcloud_wgsl}); + camera_buffer: GPUBuffer +): Renderer { + + // Create Shader Module + const render_shader = device.createShaderModule({ code: pointcloud_wgsl }); + + // Create Render Pipeline const render_pipeline = device.createRenderPipeline({ label: 'render', layout: 'auto', @@ -25,44 +30,41 @@ export default function get_renderer( }, }); + // Create Bind Groups const camera_bind_group = device.createBindGroup({ label: 'point cloud camera', layout: render_pipeline.getBindGroupLayout(0), - entries: [{binding: 0, resource: { buffer: camera_buffer }}], + entries: [{ binding: 0, resource: { buffer: camera_buffer } }], }); const gaussian_bind_group = device.createBindGroup({ label: 'point cloud gaussians', layout: render_pipeline.getBindGroupLayout(1), - entries: [ - {binding: 0, resource: { buffer: pc.gaussian_3d_buffer }}, - ], + entries: [{ binding: 0, resource: { buffer: pc.gaussian_3d_buffer } }], }); + // Render Function const render = (encoder: GPUCommandEncoder, texture_view: GPUTextureView) => { const pass = encoder.beginRenderPass({ label: 'point cloud render', - colorAttachments: [ - { - view: texture_view, - loadOp: 'clear', - storeOp: 'store', - } - ], + colorAttachments: [{ + view: texture_view, + loadOp: 'clear', + storeOp: 'store', + }], }); pass.setPipeline(render_pipeline); pass.setBindGroup(0, camera_bind_group); pass.setBindGroup(1, gaussian_bind_group); - pass.draw(pc.num_points); pass.end(); }; + // Return Renderer Object return { frame: (encoder: GPUCommandEncoder, texture_view: GPUTextureView) => { render(encoder, texture_view); }, - camera_buffer, }; } \ No newline at end of file diff --git a/src/renderers/renderer.ts b/src/renderers/renderer.ts index ffdf9ba..4de4edd 100644 --- a/src/renderers/renderer.ts +++ b/src/renderers/renderer.ts @@ -3,7 +3,7 @@ import { Pane } from 'tweakpane'; import * as TweakpaneFileImportPlugin from 'tweakpane-plugin-file-import'; import { default as get_renderer_gaussian, GaussianRenderer } from './gaussian-renderer'; import { default as get_renderer_pointcloud } from './point-cloud-renderer'; -import { Camera, load_camera_presets} from '../camera/camera'; +import { Camera, load_camera_presets } from '../camera/camera'; import { CameraControl } from '../camera/camera-control'; import { time, timeReturn } from '../utils/simple-console'; @@ -17,34 +17,41 @@ export default async function init( context: GPUCanvasContext, device: GPUDevice ) { + // Track loading status of files let ply_file_loaded = false; let cam_file_loaded = false; + + // Renderer objects for point cloud and Gaussian rendering let renderers: { pointcloud?: Renderer, gaussian?: Renderer } = {}; let gaussian_renderer: GaussianRenderer | undefined; let pointcloud_renderer: Renderer | undefined; let renderer: Renderer | undefined; let cameras; - + + // Initialize camera and controls const camera = new Camera(canvas, device); const control = new CameraControl(camera); + // Set up canvas resize handling const observer = new ResizeObserver(() => { canvas.width = canvas.clientWidth; canvas.height = canvas.clientHeight; - camera.on_update_canvas(); }); observer.observe(canvas); - + + // Configure canvas context for rendering const presentation_format = navigator.gpu.getPreferredCanvasFormat(); context.configure({ device, format: presentation_format, alphaMode: 'opaque', }); - - // Tweakpane: easily adding tweak control for parameters. + // =============================================== + // Tweakpane for Interactive Parameter Control + // =============================================== + const params = { fps: 0.0, gaussian_multiplier: 1, @@ -58,96 +65,88 @@ export default async function init( expanded: true, }); pane.registerPlugin(TweakpaneFileImportPlugin); - { - pane.addMonitor(params, 'fps', { - readonly:true - }); - } - { - pane.addInput(params, 'renderer', { - options: { - pointcloud: 'pointcloud', - gaussian: 'gaussian', - } - }).on('change', (e) => { - renderer = renderers[e.value]; - }); - } - { - pane.addInput(params, 'ply_file', { - view: 'file-input', - lineCount: 3, - filetypes: ['.ply'], - invalidFiletypeMessage: "We can't accept those filetypes!" - }) - .on('change', async (file) => { - const uploadedFile = file.value; - if (uploadedFile) { - const pc = await load(uploadedFile, device); - pointcloud_renderer = get_renderer_pointcloud(pc, device, presentation_format, camera.uniform_buffer); - gaussian_renderer = get_renderer_gaussian(pc, device, presentation_format, camera.uniform_buffer); - renderers = { - pointcloud: pointcloud_renderer, - gaussian: gaussian_renderer, - }; - renderer = renderers[params.renderer]; - ply_file_loaded = true; - }else{ - ply_file_loaded = false; - } - }); - } - { - pane.addInput(params, 'cam_file', { - view: 'file-input', - lineCount: 3, - filetypes: ['.json'], - invalidFiletypeMessage: "We can't accept those filetypes!" - }) - .on('change', async (file) => { - const uploadedFile = file.value; - if (uploadedFile) { - cameras=await load_camera_presets(file.value); - camera.set_preset(cameras[0]); - cam_file_loaded = true; - }else{ - cam_file_loaded = false; - } - }); - } - { - pane.addInput( - params, - 'gaussian_multiplier', - {min: 0, max: 1.5} - ).on('change', (e) => { - //TODO: Bind constants to the gaussian renderer. - }); - } + + // Monitor frame rate + pane.addMonitor(params, 'fps', { readonly: true }); + + // Renderer selection (pointcloud or gaussian) + pane.addInput(params, 'renderer', { + options: { pointcloud: 'pointcloud', gaussian: 'gaussian' } + }).on('change', (e) => { + renderer = renderers[e.value]; + }); + + // Load PLY file for point cloud + pane.addInput(params, 'ply_file', { + view: 'file-input', + lineCount: 3, + filetypes: ['.ply'], + invalidFiletypeMessage: "We can't accept those filetypes!" + }).on('change', async (file) => { + const uploadedFile = file.value as unknown as File; + if (uploadedFile) { + const pc = await load(uploadedFile, device); + pointcloud_renderer = get_renderer_pointcloud(pc, device, presentation_format, camera.uniform_buffer); + gaussian_renderer = get_renderer_gaussian(pc, device, presentation_format, camera.uniform_buffer); + renderers = { pointcloud: pointcloud_renderer, gaussian: gaussian_renderer }; + renderer = renderers[params.renderer]; + ply_file_loaded = true; + } else { + ply_file_loaded = false; + } + }); + + // Load camera presets from JSON file + pane.addInput(params, 'cam_file', { + view: 'file-input', + lineCount: 3, + filetypes: ['.json'], + invalidFiletypeMessage: "We can't accept those filetypes!" + }).on('change', async (file) => { + const uploadedFile = file.value as unknown as File; + if (uploadedFile) { + cameras = await load_camera_presets(uploadedFile); + camera.set_preset(cameras[0]); + cam_file_loaded = true; + } else { + cam_file_loaded = false; + } + }); + + // Control Gaussian multiplier + pane.addInput(params, 'gaussian_multiplier', { min: 0, max: 1.5 }).on('change', (e) => { + if (gaussian_renderer) { + device.queue.writeBuffer( + gaussian_renderer.render_settings_buffer, + 0, + new Float32Array([e.value]) + ); + } + }); + + // =============================================== + // Camera Preset Shortcuts (0-9 Keys) + // =============================================== document.addEventListener('keydown', (event) => { - switch(event.key) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - const i = parseInt(event.key); - console.log(`set to camera preset ${i}`); - camera.set_preset(cameras[i]); - break; + const key = parseInt(event.key); + if (!isNaN(key) && key >= 0 && key <= 9) { + console.log(`Set to camera preset ${key}`); + camera.set_preset(cameras[key]); } }); + // =============================================== + // Render Loop with Frame Timing + // =============================================== + function frame() { if (ply_file_loaded && cam_file_loaded) { - params.fps=1.0/timeReturn()*1000.0; + // Update FPS display + params.fps = 1.0 / timeReturn() * 1000.0; time(); + + // Create command encoder and render the current frame const encoder = device.createCommandEncoder(); const texture_view = context.getCurrentTexture().createView(); renderer.frame(encoder, texture_view); diff --git a/src/shaders/gaussian.wgsl b/src/shaders/gaussian.wgsl index 759226d..f7224f0 100644 --- a/src/shaders/gaussian.wgsl +++ b/src/shaders/gaussian.wgsl @@ -1,22 +1,88 @@ +// Data structure for vertex shader output struct VertexOutput { @builtin(position) position: vec4, - //TODO: information passed from vertex shader to fragment shader + @location(0) color: vec4, + @location(1) opacity: f32, + @location(2) conic: vec3, + @location(3) splatCenter: vec2, }; +// Structure for packed splat data struct Splat { - //TODO: information defined in preprocess compute shader + packedPos: u32, + packedSize: u32, + packedColor: array, + packedConicAndOpacity: array, }; +// Camera uniforms, including view and projection matrices +struct CameraUniforms { + view: mat4x4, + view_inv: mat4x4, + proj: mat4x4, + proj_inv: mat4x4, + viewport: vec2, + focal: vec2 +}; + +// Bindings for storage buffers and camera data +@group(0) @binding(0) var splats: array; +@group(0) @binding(1) var sort_indices: array; +@group(0) @binding(2) var camera: CameraUniforms; + @vertex fn vs_main( + @builtin(instance_index) instanceIdx: u32, + @builtin(vertex_index) vertIdx: u32 ) -> VertexOutput { - //TODO: reconstruct 2D quad based on information from splat, pass var out: VertexOutput; - out.position = vec4(1. ,1. , 0., 1.); + + // Retrieve the correct splat index and data from sorted arrays + let splatIdx = sort_indices[instanceIdx]; + let splat = splats[splatIdx]; + + // Unpack splat data + let pos = unpack2x16float(splat.packedPos); + let size = unpack2x16float(splat.packedSize); + let color = vec4f(unpack2x16float(splat.packedColor[0]), unpack2x16float(splat.packedColor[1])); + let conic = vec3f(unpack2x16float(splat.packedConicAndOpacity[0]), unpack2x16float(splat.packedConicAndOpacity[1]).x); + let opacity = unpack2x16float(splat.packedConicAndOpacity[1]).y; + + // Define quad vertices for each splat instance + let quadVerts = array( + vec2f(pos.x - size.x, pos.y + size.y), + vec2f(pos.x - size.x, pos.y - size.y), + vec2f(pos.x + size.x, pos.y - size.y), + vec2f(pos.x + size.x, pos.y - size.y), + vec2f(pos.x + size.x, pos.y + size.y), + vec2f(pos.x - size.x, pos.y + size.y) + ); + + // Set vertex output attributes + out.position = vec4f(quadVerts[vertIdx], 0.0, 1.0); + out.color = color; + out.opacity = opacity; + out.conic = conic; + out.splatCenter = (pos.xy * vec2f(0.5, -0.5) + vec2f(0.5, 0.5)) * camera.viewport; // Convert to pixel space for fragment shader usage return out; } @fragment -fn fs_main(in: VertexOutput) -> @location(0) vec4 { - return vec4(1.); -} \ No newline at end of file +fn fs_main( + in: VertexOutput +) -> @location(0) vec4 { + // Calculate distance to center for Gaussian falloff + let distToCenter = in.splatCenter - in.position.xy; + let power = -0.5 * (in.conic.x * pow(distToCenter.x, 2.0) + + in.conic.z * pow(distToCenter.y, 2.0)) + - in.conic.y * distToCenter.x * distToCenter.y; + + // Check if outside Gaussian influence range + if (power > 0.0) { + return vec4f(0.0, 0.0, 0.0, 0.0); + } + + // Apply opacity with Gaussian falloff + let alpha = min(0.99f, in.opacity * exp(power)); + return in.color * alpha; +} diff --git a/src/shaders/point_cloud.wgsl b/src/shaders/point_cloud.wgsl index 01dded1..f197202 100644 --- a/src/shaders/point_cloud.wgsl +++ b/src/shaders/point_cloud.wgsl @@ -8,7 +8,7 @@ struct CameraUniforms { }; struct Gaussian { - pos_opacity: array, + pos_opa: array, rot: array, scale: array } @@ -30,12 +30,11 @@ fn vs_main( var out: VertexOutput; let vertex = gaussians[in_vertex_index]; - let a = unpack2x16float(vertex.pos_opacity[0]); - let b = unpack2x16float(vertex.pos_opacity[1]); - let pos = vec4(a.x, a.y, b.x, 1.); + let xy = unpack2x16float(vertex.pos_opa[0]); + let zo = unpack2x16float(vertex.pos_opa[1]); + let pos = vec4(xy.x, xy.y, zo.x, 1.); - // TODO: MVP calculations - out.position = pos; + out.position = camera.proj * camera.view * pos; return out; } diff --git a/src/shaders/preprocess.wgsl b/src/shaders/preprocess.wgsl index bbc63f5..b85e420 100644 --- a/src/shaders/preprocess.wgsl +++ b/src/shaders/preprocess.wgsl @@ -1,13 +1,14 @@ +// Spherical Harmonic constants used in shading computations const SH_C0: f32 = 0.28209479177387814; -const SH_C1 = 0.4886025119029199; -const SH_C2 = array( +const SH_C1: f32 = 0.4886025119029199; +const SH_C2: array = array( 1.0925484305920792, -1.0925484305920792, 0.31539156525252005, -1.0925484305920792, 0.5462742152960396 ); -const SH_C3 = array( +const SH_C3: array = array( -0.5900435899266435, 2.890611442640554, -0.4570457994644658, @@ -17,102 +18,275 @@ const SH_C3 = array( -0.5900435899266435 ); -override workgroupSize: u32; -override sortKeyPerThread: u32; +// Overridable constants for workgroup configuration +override workgroupSize: u32; // Number of threads per workgroup +override sortKeyPerThread: u32; // Number of sort keys processed per thread +// Structures for dispatching compute shader workgroups and sorting information struct DispatchIndirect { - dispatch_x: atomic, + dispatch_x: atomic, // Atomic counter for workgroups in the X dimension dispatch_y: u32, dispatch_z: u32, } struct SortInfos { - keys_size: atomic, // instance_count in DrawIndirect - //data below is for info inside radix sort - padded_size: u32, - passes: u32, + keys_size: atomic, // Total number of keys to sort (instance count) + padded_size: u32, // Padded size of the keys array for radix sort + passes: u32, // Number of passes in radix sort even_pass: u32, odd_pass: u32, } +// Uniforms for camera transformations and parameters struct CameraUniforms { - view: mat4x4, - view_inv: mat4x4, - proj: mat4x4, - proj_inv: mat4x4, - viewport: vec2, - focal: vec2 -}; + view: mat4x4, // View matrix + view_inv: mat4x4, // Inverse view matrix + proj: mat4x4, // Projection matrix + proj_inv: mat4x4, // Inverse projection matrix + viewport: vec2, // Viewport dimensions + focal: vec2 // Focal lengths in x and y directions +} +// Uniforms for rendering settings struct RenderSettings { - gaussian_scaling: f32, - sh_deg: f32, + gaussian_scaling: f32, // Scaling factor for Gaussian splats + sh_deg: f32, // Degree of spherical harmonics } +// Structures representing input Gaussians and output splats struct Gaussian { - pos_opacity: array, - rot: array, - scale: array -}; + pos_opa: array, // Packed position and opacity + rot: array, // Packed rotation quaternion + scale: array // Packed scale factors in log-space +} struct Splat { - //TODO: store information for 2D splat rendering -}; + pos: u32, // Packed NDC position + size: u32, // Packed size (radius) + color: array, // Packed RGB color + conic_opa: array // Packed conic coefficients and opacity +} -//TODO: bind your data here -@group(2) @binding(0) +// Shader resource bindings +@group(0) @binding(0) var sort_infos: SortInfos; -@group(2) @binding(1) -var sort_depths : array; -@group(2) @binding(2) -var sort_indices : array; -@group(2) @binding(3) +@group(0) @binding(1) +var sort_depths: array; +@group(0) @binding(2) +var sort_indices: array; +@group(0) @binding(3) var sort_dispatch: DispatchIndirect; +@group(0) @binding(4) +var camera: CameraUniforms; +@group(0) @binding(5) +var gaussians: array; +@group(0) @binding(6) +var splats: array; +@group(0) @binding(7) +var render_settings: RenderSettings; +@group(0) @binding(8) +var colors: array; -/// reads the ith sh coef from the storage buffer -fn sh_coef(splat_idx: u32, c_idx: u32) -> vec3 { - //TODO: access your binded sh_coeff, see load.ts for how it is stored - return vec3(0.0); -} +/// Reads the spherical harmonic coefficient for a given splat index and coefficient index +fn readSHCoefficient(splat_idx: u32, coeff_idx: u32) -> vec3 { + // Calculate the base index in the color array + let base_index = splat_idx * 24 + (coeff_idx / 2) * 3 + coeff_idx % 2; + let color_part1 = unpack2x16float(colors[base_index + 0]); + let color_part2 = unpack2x16float(colors[base_index + 1]); -// spherical harmonics evaluation with Condon–Shortley phase -fn computeColorFromSH(dir: vec3, v_idx: u32, sh_deg: u32) -> vec3 { - var result = SH_C0 * sh_coef(v_idx, 0u); + // Return the appropriate color vector based on the coefficient index + if (coeff_idx % 2 == 0) { + return vec3f(color_part1.x, color_part1.y, color_part2.x); + } - if sh_deg > 0u { + return vec3f(color_part1.y, color_part2.x, color_part2.y); +} - let x = dir.x; - let y = dir.y; - let z = dir.z; +// Computes the color from spherical harmonics using the Condon–Shortley phase +fn computeColorFromSH(direction: vec3, splat_idx: u32, sh_degree: u32) -> vec3 { + var result = SH_C0 * readSHCoefficient(splat_idx, 0u); - result += - SH_C1 * y * sh_coef(v_idx, 1u) + SH_C1 * z * sh_coef(v_idx, 2u) - SH_C1 * x * sh_coef(v_idx, 3u); + if sh_degree > 0u { + let x = direction.x; + let y = direction.y; + let z = direction.z; - if sh_deg > 1u { + result += -SH_C1 * y * readSHCoefficient(splat_idx, 1u) + + SH_C1 * z * readSHCoefficient(splat_idx, 2u) + - SH_C1 * x * readSHCoefficient(splat_idx, 3u); - let xx = dir.x * dir.x; - let yy = dir.y * dir.y; - let zz = dir.z * dir.z; - let xy = dir.x * dir.y; - let yz = dir.y * dir.z; - let xz = dir.x * dir.z; + if sh_degree > 1u { + let xx = x * x; + let yy = y * y; + let zz = z * z; + let xy = x * y; + let yz = y * z; + let xz = x * z; - result += SH_C2[0] * xy * sh_coef(v_idx, 4u) + SH_C2[1] * yz * sh_coef(v_idx, 5u) + SH_C2[2] * (2.0 * zz - xx - yy) * sh_coef(v_idx, 6u) + SH_C2[3] * xz * sh_coef(v_idx, 7u) + SH_C2[4] * (xx - yy) * sh_coef(v_idx, 8u); + result += SH_C2[0] * xy * readSHCoefficient(splat_idx, 4u) + + SH_C2[1] * yz * readSHCoefficient(splat_idx, 5u) + + SH_C2[2] * (2.0 * zz - xx - yy) * readSHCoefficient(splat_idx, 6u) + + SH_C2[3] * xz * readSHCoefficient(splat_idx, 7u) + + SH_C2[4] * (xx - yy) * readSHCoefficient(splat_idx, 8u); - if sh_deg > 2u { - result += SH_C3[0] * y * (3.0 * xx - yy) * sh_coef(v_idx, 9u) + SH_C3[1] * xy * z * sh_coef(v_idx, 10u) + SH_C3[2] * y * (4.0 * zz - xx - yy) * sh_coef(v_idx, 11u) + SH_C3[3] * z * (2.0 * zz - 3.0 * xx - 3.0 * yy) * sh_coef(v_idx, 12u) + SH_C3[4] * x * (4.0 * zz - xx - yy) * sh_coef(v_idx, 13u) + SH_C3[5] * z * (xx - yy) * sh_coef(v_idx, 14u) + SH_C3[6] * x * (xx - 3.0 * yy) * sh_coef(v_idx, 15u); + if sh_degree > 2u { + result += SH_C3[0] * y * (3.0 * xx - yy) * readSHCoefficient(splat_idx, 9u) + + SH_C3[1] * xy * z * readSHCoefficient(splat_idx, 10u) + + SH_C3[2] * y * (4.0 * zz - xx - yy) * readSHCoefficient(splat_idx, 11u) + + SH_C3[3] * z * (2.0 * zz - 3.0 * xx - 3.0 * yy) * readSHCoefficient(splat_idx, 12u) + + SH_C3[4] * x * (4.0 * zz - xx - yy) * readSHCoefficient(splat_idx, 13u) + + SH_C3[5] * z * (xx - yy) * readSHCoefficient(splat_idx, 14u) + + SH_C3[6] * x * (xx - 3.0 * yy) * readSHCoefficient(splat_idx, 15u); } } } - result += 0.5; + result += 0.5; // Offset to adjust the color range - return max(vec3(0.), result); + return max(vec3(0.0), result); } -@compute @workgroup_size(workgroupSize,1,1) -fn preprocess(@builtin(global_invocation_id) gid: vec3, @builtin(num_workgroups) wgs: vec3) { - let idx = gid.x; - //TODO: set up pipeline as described in instruction +@compute @workgroup_size(workgroupSize, 1, 1) +fn preprocess(@builtin(global_invocation_id) global_id: vec3, @builtin(num_workgroups) num_workgroups: vec3) { + let splat_idx = global_id.x; + if (splat_idx >= arrayLength(&gaussians)) { + return; + } + + // Retrieve Gaussian data + let gaussian = gaussians[splat_idx]; + let pos_xy = unpack2x16float(gaussian.pos_opa[0]); + let pos_z_opacity = unpack2x16float(gaussian.pos_opa[1]); + let position = vec3(pos_xy, pos_z_opacity.x); + + // Decode opacity using a sigmoid function + let opacity = 1.0 / (1.0 + exp(-pos_z_opacity.y)); + + // Transform position from world space to Normalized Device Coordinates (NDC) + let view_pos = camera.view * vec4(position, 1.0); + var pos_ndc = camera.proj * view_pos; + pos_ndc /= pos_ndc.w; + + // Frustum culling with a slight margin to avoid clipping edge splats + if (pos_ndc.x < -1.2 || pos_ndc.x > 1.2 || + pos_ndc.y < -1.2 || pos_ndc.y > 1.2 || + view_pos.z < 0.0) { + return; + } + + // Unpack rotation quaternion components + let rot_wx = unpack2x16float(gaussian.rot[0]); + let rot_yz = unpack2x16float(gaussian.rot[1]); + + // Unpack and exponentiate scale factors (stored in log-space) + let scale_xy = exp(unpack2x16float(gaussian.scale[0])); + let scale_zw = exp(unpack2x16float(gaussian.scale[1])); + + // Construct rotation matrix from quaternion + let rotation_matrix = mat3x3( + 1.0 - 2.0 * (rot_yz.x * rot_yz.x + rot_yz.y * rot_yz.y), + 2.0 * (rot_wx.y * rot_yz.x - rot_wx.x * rot_yz.y), + 2.0 * (rot_wx.y * rot_yz.y + rot_wx.x * rot_yz.x), + 2.0 * (rot_wx.y * rot_yz.x + rot_wx.x * rot_yz.y), + 1.0 - 2.0 * (rot_wx.y * rot_wx.y + rot_yz.y * rot_yz.y), + 2.0 * (rot_yz.x * rot_yz.y - rot_wx.x * rot_wx.y), + 2.0 * (rot_wx.y * rot_yz.y - rot_wx.x * rot_yz.x), + 2.0 * (rot_yz.x * rot_yz.y + rot_wx.x * rot_wx.y), + 1.0 - 2.0 * (rot_wx.y * rot_wx.y + rot_yz.x * rot_yz.x) + ); + + // Create scale matrix + let scale_matrix = mat3x3( + render_settings.gaussian_scaling * scale_xy.x, 0.0, 0.0, + 0.0, render_settings.gaussian_scaling * scale_xy.y, 0.0, + 0.0, 0.0, render_settings.gaussian_scaling * scale_zw.x + ); + + // Compute 3D covariance matrix for the Gaussian + let covariance_matrix_3d = transpose(scale_matrix * rotation_matrix) * scale_matrix * rotation_matrix; + + // Compute the Jacobian matrix for screen space transformation + let jacobian_j = mat3x3( + camera.focal.x / view_pos.z, 0.0, -camera.focal.x * view_pos.x / (view_pos.z * view_pos.z), + 0.0, camera.focal.y / view_pos.z, -camera.focal.y * view_pos.y / (view_pos.z * view_pos.z), + 0.0, 0.0, 0.0 + ); - let keys_per_dispatch = workgroupSize * sortKeyPerThread; - // increment DispatchIndirect.dispatchx each time you reach limit for one dispatch of keys -} \ No newline at end of file + // Extract rotation part of the camera view matrix + let camera_rotation = transpose(mat3x3( + camera.view[0].xyz, + camera.view[1].xyz, + camera.view[2].xyz + )); + + // Compute transformation matrix + let transformation_t = camera_rotation * jacobian_j; + + // Construct symmetric covariance matrix + let covariance_v = mat3x3( + covariance_matrix_3d[0][0], covariance_matrix_3d[0][1], covariance_matrix_3d[0][2], + covariance_matrix_3d[0][1], covariance_matrix_3d[1][1], covariance_matrix_3d[1][2], + covariance_matrix_3d[0][2], covariance_matrix_3d[1][2], covariance_matrix_3d[2][2] + ); + + // Compute 2D covariance matrix in screen space + var covariance_matrix_2d = transpose(transformation_t) * transpose(covariance_v) * transformation_t; + + // Apply minimum variance to prevent undersized Gaussians + covariance_matrix_2d[0][0] += 0.3; + covariance_matrix_2d[1][1] += 0.3; + + // Extract unique elements from the symmetric covariance matrix + let covariance_2d = vec3( + covariance_matrix_2d[0][0], + covariance_matrix_2d[0][1], + covariance_matrix_2d[1][1] + ); + + // Calculate the determinant of the covariance matrix + var determinant = covariance_2d.x * covariance_2d.z - (covariance_2d.y * covariance_2d.y); + if (determinant == 0.0) { + return; + } + + // Compute eigenvalues to determine the radius + let mid = (covariance_2d.x + covariance_2d.z) * 0.5; + let sqrt_term = sqrt(max(0.1, mid * mid - determinant)); + let lambda1 = mid + sqrt_term; + let lambda2 = mid - sqrt_term; + let radius = ceil(3.0 * sqrt(max(lambda1, lambda2))); + + // Calculate the viewing direction for shading + let camera_position = -camera.view[3].xyz; + let view_direction = normalize(position - camera_position); + let color = computeColorFromSH(view_direction, splat_idx, u32(render_settings.sh_deg)); + + // Compute conic coefficients for fragment coverage determination + let determinant_inv = 1.0 / determinant; + let conic_coefficients = vec3( + covariance_2d.z * determinant_inv, + -covariance_2d.y * determinant_inv, + covariance_2d.x * determinant_inv + ); + + // Get index to store the computed splat + let splat_store_index = atomicAdd(&sort_infos.keys_size, 1); + + // Store splat data for rendering + splats[splat_store_index].pos = pack2x16float(pos_ndc.xy); + splats[splat_store_index].size = pack2x16float(vec2(radius, radius) / camera.viewport); + splats[splat_store_index].color[0] = pack2x16float(color.rg); + splats[splat_store_index].color[1] = pack2x16float(vec2(color.b, 1.0)); + splats[splat_store_index].conic_opa[0] = pack2x16float(conic_coefficients.xy); + splats[splat_store_index].conic_opa[1] = pack2x16float(vec2(conic_coefficients.z, opacity)); + + // Store depth for sorting (bitcast as u32 for radix sort) + sort_depths[splat_store_index] = bitcast(100.0 - view_pos.z); + sort_indices[splat_store_index] = splat_store_index; + + let keys_per_dispatch = workgroupSize * sortKeyPerThread; + + // Increment dispatch count if necessary + if (splat_store_index % keys_per_dispatch == 0) { + atomicAdd(&sort_dispatch.dispatch_x, 1); + } +} diff --git a/src/utils/load.ts b/src/utils/load.ts index 030b857..18247b0 100644 --- a/src/utils/load.ts +++ b/src/utils/load.ts @@ -13,8 +13,8 @@ const c_size_3d_gaussian = export type PointCloud = Awaited>; -export async function load(file: string, device: GPUDevice) { - const blob = new Blob([file]); +export async function load(file: File, device: GPUDevice) { + const blob = new Blob([file], { type: file.type }); const arrayBuffer = await new Promise((resolve, reject) => { const reader = new FileReader(); @@ -118,4 +118,4 @@ export async function load(file: string, device: GPUDevice) { gaussian_3d_buffer, sh_buffer, }; -} +} \ No newline at end of file