Skip to content

convert raymarch to use 3d texture instead of atlas. #370

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
toloudis wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

convert raymarch to use 3d texture instead of atlas. #370

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9b10374
convert raymarch to use 3d texture instead of atlas.
toloudis Jan 30, 2026
474a6eb
Merge branch 'main' into feature/raymarch-3d-texture
toloudis Mar 1, 2026
720cddb
keep scratch buffer around
toloudis Mar 1, 2026
212f070
dry refactor and fix time playback
toloudis Mar 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 13 additions & 44 deletions src/PathTracedVolume.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import type { VolumeRenderImpl } from "./VolumeRenderImpl.js";
import { VolumeRenderSettings, SettingsFlags } from "./VolumeRenderSettings.js";
import Channel from "./Channel.js";
import RenderToBuffer from "./RenderToBuffer.js";
import { VolumeDataUpdater } from "./utils/VolumeDataUpdater.js";

export default class PathTracedVolume implements VolumeRenderImpl {
private settings: VolumeRenderSettings;
Expand All @@ -40,6 +41,7 @@ export default class PathTracedVolume implements VolumeRenderImpl {
private viewChannels: number[]; // should have 4 or less elements

private volumeTexture: Data3DTexture;
private volumeDataUpdater: VolumeDataUpdater;

private cameraIsMoving: boolean;
private sampleCounter: number;
Expand Down Expand Up @@ -81,6 +83,7 @@ export default class PathTracedVolume implements VolumeRenderImpl {
this.volumeTexture.generateMipmaps = false;

this.volumeTexture.needsUpdate = true;
this.volumeDataUpdater = new VolumeDataUpdater();

// create Lut textures
// empty array
Expand Down Expand Up @@ -258,6 +261,9 @@ export default class PathTracedVolume implements VolumeRenderImpl {
// Update channel and alpha mask if they have changed
this.updateVolumeData4();
}
if (dirtyFlags & SettingsFlags.MASK_DATA) {
this.updateVolumeData4();
}
if (dirtyFlags & SettingsFlags.VIEW) {
this.pathTracingUniforms.gCamera.value.mIsOrtho = this.settings.isOrtho ? 1 : 0;
}
Expand Down Expand Up @@ -436,50 +442,13 @@ export default class PathTracedVolume implements VolumeRenderImpl {
}

updateVolumeData4(): void {
const { x: sx, y: sy, z: sz } = this.volume.imageInfo.subregionSize;

const data = new Uint8Array(sx * sy * sz * 4);
data.fill(0);

for (let i = 0; i < 4; ++i) {
const ch = this.viewChannels[i];
if (ch === -1) {
continue;
}

const volumeChannel = this.volume.getChannel(ch);
for (let iz = 0; iz < sz; ++iz) {
for (let iy = 0; iy < sy; ++iy) {
for (let ix = 0; ix < sx; ++ix) {
// TODO expand to 16-bpp raw intensities?
data[i + ix * 4 + iy * 4 * sx + iz * 4 * sx * sy] =
255 * volumeChannel.normalizeRaw(volumeChannel.getIntensity(ix, iy, iz));
}
}
}
if (this.settings.maskChannelIndex !== -1 && this.settings.maskAlpha < 1.0) {
const maskChannel = this.volume.getChannel(this.settings.maskChannelIndex);
// const maskMax = maskChannel.getHistogram().dataMax;
let maskVal = 1.0;
const maskAlpha = this.settings.maskAlpha;
for (let iz = 0; iz < sz; ++iz) {
for (let iy = 0; iy < sy; ++iy) {
for (let ix = 0; ix < sx; ++ix) {
// nonbinary masking
// maskVal = maskChannel.getIntensity(ix,iy,iz) * maskAlpha / maskMax;

// binary masking
maskVal = maskChannel.getIntensity(ix, iy, iz) > 0 ? 1.0 : maskAlpha;

data[i + ix * 4 + iy * 4 * sx + iz * 4 * sx * sy] *= maskVal;
}
}
}
}
}
// defaults to rgba and unsignedbytetype so dont need to supply format this time.
this.volumeTexture.image.data.set(data);
this.volumeTexture.needsUpdate = true;
this.volumeDataUpdater.update(
this.volume,
this.viewChannels,
this.settings.maskChannelIndex,
this.settings.maskAlpha,
this.volumeTexture
);
}

updateLuts(channelColors: FuseChannel[], channelData: Channel[]): void {
Expand Down
128 changes: 96 additions & 32 deletions src/RayMarchedAtlasVolume.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,24 +5,27 @@ import {
BufferAttribute,
BufferGeometry,
Color,
Data3DTexture,
DataTexture,
DepthTexture,
Group,
LineBasicMaterial,
LineSegments,
LinearFilter,
Material,
Matrix4,
Mesh,
NearestFilter,
OrthographicCamera,
PerspectiveCamera,
RGBAFormat,
ShaderMaterial,
Texture,
Vector2,
UnsignedByteType,
Vector3,
WebGLRenderer,
} from "three";

import FusedChannelData from "./FusedChannelData.js";
import {
rayMarchingVertexShaderSrc,
rayMarchingFragmentShaderSrc,
Expand All @@ -32,8 +35,10 @@ import { Volume } from "./index.js";
import Channel from "./Channel.js";
import type { VolumeRenderImpl } from "./VolumeRenderImpl.js";

import type { FuseChannel } from "./types.js";
import { FUSE_DISABLED_RGB_COLOR, type FuseChannel } from "./types.js";
import { VolumeRenderSettings, SettingsFlags } from "./VolumeRenderSettings.js";
import { LUT_ARRAY_LENGTH } from "./Lut.js";
import { VolumeDataUpdater } from "./utils/VolumeDataUpdater.js";

const BOUNDING_BOX_DEFAULT_COLOR = new Color(0xffff00);

Expand All @@ -47,9 +52,13 @@ export default class RayMarchedAtlasVolume implements VolumeRenderImpl {
private tickMarksMesh: LineSegments;
private geometryTransformNode: Group;
private uniforms: ReturnType<typeof rayMarchingShaderUniforms>;
private channelData!: FusedChannelData;
private emptyPositionTex: DataTexture;

private volumeTexture: Data3DTexture;
private lutTexture: DataTexture;
private viewChannels: number[]; // should have 4 or less elements
private volumeDataUpdater: VolumeDataUpdater;

/**
* Creates a new RayMarchedAtlasVolume.
* @param volume The volume that this renderer should render data from.
Expand All @@ -58,6 +67,7 @@ export default class RayMarchedAtlasVolume implements VolumeRenderImpl {
*/
constructor(volume: Volume, settings: VolumeRenderSettings = new VolumeRenderSettings(volume)) {
this.volume = volume;
this.viewChannels = [-1, -1, -1, -1];

this.uniforms = rayMarchingShaderUniforms();
[this.geometry, this.geometryMesh] = this.createGeometry(this.uniforms);
Expand All @@ -80,6 +90,28 @@ export default class RayMarchedAtlasVolume implements VolumeRenderImpl {

this.emptyPositionTex = new DataTexture(new Uint8Array(Array(16).fill(0)), 2, 2);

// create volume texture (Data3DTexture)
const { x: sx, y: sy, z: sz } = volume.imageInfo.subregionSize;
const data = new Uint8Array(sx * sy * sz * 4).fill(0);
this.volumeTexture = new Data3DTexture(data, sx, sy, sz);
this.volumeTexture.format = RGBAFormat;
this.volumeTexture.type = UnsignedByteType;
this.volumeTexture.minFilter = LinearFilter;
this.volumeTexture.magFilter = LinearFilter;
this.volumeTexture.generateMipmaps = false;
this.volumeTexture.needsUpdate = true;
this.volumeDataUpdater = new VolumeDataUpdater();

// create LUT texture (256x4, each row is a channel's LUT)
const lutData = new Uint8Array(LUT_ARRAY_LENGTH * 4).fill(255);
this.lutTexture = new DataTexture(lutData, 256, 4, RGBAFormat, UnsignedByteType);
this.lutTexture.minFilter = LinearFilter;
this.lutTexture.magFilter = LinearFilter;
this.lutTexture.needsUpdate = true;

this.setUniform("volumeTexture", this.volumeTexture);
this.setUniform("gLutTexture", this.lutTexture);

this.settings = settings;
this.updateSettings(settings, SettingsFlags.ALL);
// TODO this is doing *more* redundant work! Fix?
Expand All @@ -97,21 +129,6 @@ export default class RayMarchedAtlasVolume implements VolumeRenderImpl {
this.boxHelper.box.set(fullRegionScale.clone().multiplyScalar(-0.5), fullRegionScale.clone().multiplyScalar(0.5));
this.tickMarksMesh.scale.copy(fullRegionScale);
this.settings && this.updateSettings(this.settings, SettingsFlags.ROI);

// Set atlas dimension uniforms
const { atlasTileDims, subregionSize } = this.volume.imageInfo;
const atlasSize = new Vector2(subregionSize.x, subregionSize.y).multiply(atlasTileDims);

this.setUniform("ATLAS_DIMS", atlasTileDims);

this.setUniform("textureRes", atlasSize);
this.setUniform("SLICES", this.volume.imageInfo.volumeSize.z);

// (re)create channel data
if (!this.channelData || this.channelData.width !== atlasSize.x || this.channelData.height !== atlasSize.y) {
this.channelData?.cleanup();
this.channelData = new FusedChannelData(atlasSize.x, atlasSize.y);
}
}

public viewpointMoved(): void {
Expand Down Expand Up @@ -193,18 +210,21 @@ export default class RayMarchedAtlasVolume implements VolumeRenderImpl {
}

if (dirtyFlags & SettingsFlags.SAMPLING) {
this.setUniform("interpolationEnabled", this.settings.useInterpolation);
this.volumeTexture.minFilter = this.volumeTexture.magFilter = this.settings.useInterpolation
? LinearFilter
: NearestFilter;
this.volumeTexture.needsUpdate = true;
this.setUniform("iResolution", this.settings.resolution);
}
Comment on lines 212 to 218
Copy link

Copilot AI Mar 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sampling interpolation is now controlled via volumeTexture.minFilter/magFilter, but the shader uniform interpolationEnabled is still declared and included in the uniforms object. Since it’s no longer used in raymarch.frag, consider removing the uniform from both the TS uniforms and GLSL to avoid dead/unreferenced state and reduce confusion.

Copilot uses AI. Check for mistakes.

if (dirtyFlags & SettingsFlags.MASK_ALPHA) {
this.setUniform("maskAlpha", this.settings.maskChannelIndex < 0 ? 1.0 : this.settings.maskAlpha);
Copy link

Copilot AI Mar 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maskAlpha is now applied on the CPU when building volumeTexture, but the shader uniform is still being set here. Since the fragment shader no longer uses maskAlpha, keeping this uniform update is misleading and can be removed (or, if runtime maskAlpha changes are expected without rebuilding the 3D texture, the shader should use the uniform again).

Suggested change
this.setUniform("maskAlpha", this.settings.maskChannelIndex < 0 ? 1.0 : this.settings.maskAlpha);

Copilot uses AI. Check for mistakes.
// Re-update volume data with new mask settings
this.updateVolumeData4();
}
if (dirtyFlags & SettingsFlags.MASK_DATA) {
this.channelData.setChannelAsMask(
this.settings.maskChannelIndex,
this.volume.getChannel(this.settings.maskChannelIndex)
);
// Re-update volume data with new mask settings
this.updateVolumeData4();
}
}

Expand Down Expand Up @@ -307,7 +327,8 @@ export default class RayMarchedAtlasVolume implements VolumeRenderImpl {
this.geometry.dispose();
this.geometryMesh.material.dispose();

this.channelData.cleanup();
this.volumeTexture.dispose();
this.lutTexture.dispose();
}

public doRender(
Expand All @@ -325,9 +346,6 @@ export default class RayMarchedAtlasVolume implements VolumeRenderImpl {
this.setUniform("CLIP_NEAR", camera.near);
this.setUniform("CLIP_FAR", camera.far);

Comment on lines 334 to 348
Copy link

Copilot AI Mar 1, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Raymarch now relies on Data3DTexture/sampler3D, which requires WebGL2. This renderer currently doesn’t guard against running under the WebGL1 fallback path (see ThreeJsPanel’s non-WebGL2 branch), so in WebGL1 this will fail shader compilation / rendering. Please add an explicit WebGL2 capability check (e.g., renderer.capabilities.isWebGL2) and either (a) fall back to the old atlas path or (b) disable raymarch with a clear error message when WebGL2 is unavailable.

Copilot uses AI. Check for mistakes.
this.channelData.gpuFuse(renderer);
this.setUniform("textureAtlas", this.channelData.getFusedTexture());

this.geometryTransformNode.updateMatrixWorld(true);

const mvm = new Matrix4();
Expand Down Expand Up @@ -355,13 +373,59 @@ export default class RayMarchedAtlasVolume implements VolumeRenderImpl {
this.uniforms[name].value = value;
}

/**
* Update the 3D volume texture with data from up to 4 active channels.
*/
private updateVolumeData4(): void {
this.volumeDataUpdater.update(
this.volume,
this.viewChannels,
this.settings.maskChannelIndex,
this.settings.maskAlpha,
this.volumeTexture
);
}

/**
* Update LUT textures for the active channels.
*/
private updateLuts(channelColors: FuseChannel[], channelData: Channel[]): void {
for (let i = 0; i < this.uniforms.gNChannels.value; ++i) {
const channel = this.viewChannels[i];
const combinedLut = channelData[channel].combineLuts(channelColors[channel].rgbColor);

this.lutTexture.image.data.set(combinedLut, i * LUT_ARRAY_LENGTH);
}
this.lutTexture.needsUpdate = true;
Comment on lines +392 to +399
Copy link

Copilot AI Mar 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

updateLuts() calls Channel.combineLuts() without providing the optional out buffer, which allocates a new Uint8Array every time LUTs are updated. This can create noticeable GC churn when users adjust colors/LUTs frequently. Consider reusing a preallocated Uint8Array per active slot (and passing it as out) so LUT updates are in-place.

Copilot uses AI. Check for mistakes.
}

// channelcolors is array of {rgbColor, lut} and channeldata is volume.channels
public updateActiveChannels(channelcolors: FuseChannel[], channeldata: Channel[]): void {
this.channelData.fuse(channelcolors, channeldata);
const ch = [-1, -1, -1, -1];
let activeChannel = 0;
const NC = this.volume.imageInfo.numChannels;
const maxch = 4;
for (let i = 0; i < NC && activeChannel < maxch; ++i) {
// check that channel is not disabled and is loaded
if (channelcolors[i].rgbColor !== FUSE_DISABLED_RGB_COLOR && channeldata[i].loaded) {
ch[activeChannel] = i;
activeChannel++;
}
}

const unchanged = ch.every((elem, index) => elem === this.viewChannels[index], this);
if (unchanged) {
// Still update LUTs in case colors changed
this.updateLuts(channelcolors, channeldata);
return;
}

this.setUniform("gNChannels", activeChannel);

// update to fused texture
this.setUniform("textureAtlas", this.channelData.getFusedTexture());
this.setUniform("textureAtlasMask", this.channelData.maskTexture);
this.viewChannels = ch;
// update volume data according to channels selected.
this.updateVolumeData4();
this.updateLuts(channelcolors, channeldata);
}

public setRenderUpdateListener(_listener?: ((iteration: number) => void) | undefined) {
Expand Down
3 changes: 3 additions & 0 deletions src/VolumeDrawable.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,9 @@ export default class VolumeDrawable {
}
}

this.volumeRendering.updateActiveChannels(this.fusion, this.volume.channels);
this.volumeRendering.updateSettings(this.settings, SettingsFlags.MASK_DATA);

// let the outside world have a chance
this.onChannelDataReadyCallback?.();
}
Expand Down
Loading