Skip to content

Commit

Permalink
add multimodal sample
Browse files Browse the repository at this point in the history
  • Loading branch information
@cabanier
cabanier committed Nov 11, 2024
1 parent 9a7a523 commit 17f04e1
Showing 1 changed file with 120 additions and 37 deletions.
157 changes: 120 additions & 37 deletions immersive-hands.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
<details open>
<summary>Immersive VR Session with hands</summary>
<p>
This sample demonstrates a simple vr session that shows the user's hands.
This sample demonstrates a simple VR session that shows the user's hands.
If your device supports this feature, you will see your hands using a set
of cubes. Each cube represents a joint in your hand.
<a class="back" href="./">Back</a>
Expand All @@ -62,6 +62,12 @@
import {vec3} from './js/render/math/gl-matrix.js';
import {Ray} from './js/render/math/ray.js';

// This library matches XRInputSource profiles to available controller models for us.
import { fetchProfile } from 'https://cdn.jsdelivr.net/npm/@webxr-input-profiles/[email protected]/dist/motion-controllers.module.js';

// The path of the CDN the sample will fetch controller models from.
const DEFAULT_PROFILES_PATH = 'https://cdn.jsdelivr.net/npm/@webxr-input-profiles/[email protected]/dist/profiles';

// XR globals.
let xrButton = null;
let xrRefSpace = null;
Expand All @@ -72,8 +78,12 @@
// Boxes
let boxes_left = [];
let boxes_right = [];
let boxes = { left: boxes_left, right: boxes_right };
let indexFingerBoxes = { left: null, right: null };
let boxes_none = [];
let tracked_boxes_left = [];
let tracked_boxes_right = [];
let tracked_boxes_none = [];
let boxes = { input_left: boxes_left, input_right: boxes_right, input_none: boxes_none, tracked_left: tracked_boxes_left, tracked_right: tracked_boxes_right, tracked_none: tracked_boxes_none };
let indexFingerBoxes = { input_left: null, input_right: null, tracked_left: null, tracked_right: null };
const defaultBoxColor = {r: 0.5, g: 0.5, b: 0.5};
const leftBoxColor = {r: 1, g: 0, b: 1};
const rightBoxColor = {r: 0, g: 1, b: 1};
Expand Down Expand Up @@ -115,24 +125,34 @@
}
boxes_left = [];
boxes_right = [];
boxes = { left: boxes_left, right: boxes_right };
boxes = { input_left: boxes_left, input_right: boxes_right, input_none: boxes_none, tracked_left: tracked_boxes_left, tracked_right: tracked_boxes_right, tracked_none: tracked_boxes_none };
if (typeof XRHand !== 'undefined') {
for (let i = 0; i <= 24; i++) {
const r = .6 + Math.random() * .4;
const g = .6 + Math.random() * .4;
const b = .6 + Math.random() * .4;
boxes_left.push(addBox(0, 0, 0, r, g, b));
boxes_right.push(addBox(0, 0, 0, r, g, b));
tracked_boxes_left.push(addBox(0, 0, 0, r, g, b));
tracked_boxes_right.push(addBox(0, 0, 0, r, g, b));
}
}
if (indexFingerBoxes.left) {
if (indexFingerBoxes.input_left) {
scene.removeNode(indexFingerBoxes.left);
}
if (indexFingerBoxes.right) {
scene.removeNode(indexFingerBoxes.right);
if (indexFingerBoxes.input_right) {
scene.removeNode(indexFingerBoxes.input_right);
}
if (indexFingerBoxes.tracked_left) {
scene.removeNode(indexFingerBoxes.tracked_left);
}
if (indexFingerBoxes.tracked_right) {
scene.removeNode(indexFingerBoxes.tracked_right);
}
indexFingerBoxes.left = addBox(0, 0, 0, leftBoxColor.r, leftBoxColor.g, leftBoxColor.b);
indexFingerBoxes.right = addBox(0, 0, 0, rightBoxColor.r, rightBoxColor.g, rightBoxColor.b);
indexFingerBoxes.input_left = addBox(0, 0, 0, leftBoxColor.r, leftBoxColor.g, leftBoxColor.b);
indexFingerBoxes.input_right = addBox(0, 0, 0, rightBoxColor.r, rightBoxColor.g, rightBoxColor.b);
indexFingerBoxes.tracked_left = addBox(0, 0, 0, leftBoxColor.r, leftBoxColor.g, leftBoxColor.b);
indexFingerBoxes.tracked_right = addBox(0, 0, 0, rightBoxColor.r, rightBoxColor.g, rightBoxColor.b);
}

// Checks to see if WebXR is available and, if so, queries a list of
Expand Down Expand Up @@ -178,14 +198,23 @@
// Listen for the sessions 'end' event so we can respond if the user
// or UA ends the session for any reason.
session.addEventListener('end', onSessionEnded);
session.addEventListener('inputsourceschange', onInputSourcesChange);
// trackedSources are still experimental. Don't rely on this feature yet.
session.addEventListener('trackedsourceschange', onTrackedSourcesChange);

session.addEventListener('visibilitychange', e => {
// remove hand controller while blurred
if(e.session.visibilityState === 'visible-blurred') {
for (const box of boxes['left']) {
for (const box of boxes['input_left']) {
scene.removeNode(box);
}
for (const box of boxes['input_right']) {
scene.removeNode(box);
}
for (const box of boxes['tracked_left']) {
scene.removeNode(box);
}
for (const box of boxes['right']) {
for (const box of boxes['tracked_right']) {
scene.removeNode(box);
}
}
Expand Down Expand Up @@ -240,41 +269,94 @@
renderer = null;
}

function onInputSourcesChange(event) {
onSourcesChange(event, "input_");
}

function onTrackedSourcesChange(event) {
onSourcesChange(event, "tracked_");
}

function onSourcesChange(event, type) {
console.log("onSourcesChange " + type + " added " + event.added.length + " removed " + event.removed.length);
// As input sources are connected if they are tracked-pointer devices
// look up which meshes should be associated with their profile and
// load as the controller model for that hand.
for (let inputSource of event.added) {
if (inputSource.targetRayMode == 'tracked-pointer') {
// Use the fetchProfile method from the motionControllers library
// to find the appropriate glTF mesh path for this controller.
fetchProfile(inputSource, DEFAULT_PROFILES_PATH).then(({profile, assetPath}) => {
// Typically if you wanted to animate the controllers in response
// to device inputs you'd create a new MotionController() instance
// here to handle the animation, but this sample will skip that
// and only display a static mesh for simplicity.

scene.inputRenderer.setControllerMesh(new Gltf2Node({url: assetPath}), inputSource.handedness, inputSource.profiles[0]);
});
}
}
}

function updateInputSources(session, frame, refSpace) {
updateSources(session, frame, refSpace, session.inputSources, "input_");
}

function updateTrackedSources(session, frame, refSpace) {
// session.trackedSources are still experimental. Don't rely on this feature yet.
if (session.trackedSources) {
updateSources(session, frame, refSpace, session.trackedSources, "tracked_");
}
}

function updateSources(session, frame, refSpace, sources, type) {
if(session.visibilityState === 'visible-blurred') {
return;
}
for (let inputSource of session.inputSources) {
let targetRayPose = frame.getPose(inputSource.targetRaySpace, refSpace);
if (targetRayPose) {
if (inputSource.targetRayMode == 'tracked-pointer') {
scene.inputRenderer.addLaserPointer(targetRayPose.transform);
for (let inputSource of sources) {
let hand_type = type + inputSource.handedness;
if (type == "input_") {
let targetRayPose = frame.getPose(inputSource.targetRaySpace, refSpace);
if (targetRayPose) {
if (inputSource.targetRayMode == 'tracked-pointer') {
scene.inputRenderer.addLaserPointer(targetRayPose.transform);
}

let targetRay = new Ray(targetRayPose.transform);
let cursorDistance = 2.0;
let cursorPos = vec3.fromValues(
targetRay.origin.x,
targetRay.origin.y,
targetRay.origin.z
);
vec3.add(cursorPos, cursorPos, [
targetRay.direction.x * cursorDistance,
targetRay.direction.y * cursorDistance,
targetRay.direction.z * cursorDistance,
]);

scene.inputRenderer.addCursor(cursorPos);
}
}

let targetRay = new Ray(targetRayPose.transform);
let cursorDistance = 2.0;
let cursorPos = vec3.fromValues(
targetRay.origin.x,
targetRay.origin.y,
targetRay.origin.z
);
vec3.add(cursorPos, cursorPos, [
targetRay.direction.x * cursorDistance,
targetRay.direction.y * cursorDistance,
targetRay.direction.z * cursorDistance,
]);

scene.inputRenderer.addCursor(cursorPos);
if (!inputSource.hand && inputSource.gripSpace) {
let gripPose = frame.getPose(inputSource.gripSpace, refSpace);
if (gripPose) {
scene.inputRenderer.addController(gripPose.transform.matrix, inputSource.handedness, inputSource.profiles[0]);
} else {
scene.inputRenderer.hideController(hand_type);
}
}

let offset = 0;
if (!inputSource.hand) {
continue;
} else {
for (const box of boxes[inputSource.handedness]) {
for (const box of boxes[hand_type]) {
scene.removeNode(box);
}
scene.removeNode(indexFingerBoxes[hand_type]);

continue;
} else {
let pose = frame.getPose(inputSource.targetRaySpace, refSpace);
if (pose === undefined) {
console.log("no pose");
Expand All @@ -288,7 +370,7 @@
console.log("no fillPoses");
continue;
}
for (const box of boxes[inputSource.handedness]) {
for (const box of boxes[hand_type]) {
scene.addNode(box);
let matrix = positions.slice(offset * 16, (offset + 1) * 16);
let jointRadius = radii[offset];
Expand All @@ -299,7 +381,7 @@
}

// Render a special box for each index finger on each hand
const indexFingerBox = indexFingerBoxes[inputSource.handedness];
const indexFingerBox = indexFingerBoxes[hand_type];
scene.addNode(indexFingerBox);
let joint = inputSource.hand.get('index-finger-tip');
let jointPose = frame.getJointPose(joint, xrRefSpace);
Expand Down Expand Up @@ -343,9 +425,9 @@
const interactionDistance = interactionBox.scale[0];
leftInteractionBox.visible = false;
rightInteractionBox.visible = false;
if (Distance(indexFingerBoxes.left, interactionBox) < interactionDistance) {
if (Distance(indexFingerBoxes.input_left, interactionBox) < interactionDistance) {
leftInteractionBox.visible = true;
} else if (Distance(indexFingerBoxes.right, interactionBox) < interactionDistance) {
} else if (Distance(indexFingerBoxes.input_right, interactionBox) < interactionDistance) {
rightInteractionBox.visible = true;
}
interactionBox.visible = !(leftInteractionBox.visible || rightInteractionBox.visible);
Expand All @@ -367,6 +449,7 @@
session.requestAnimationFrame(onXRFrame);

updateInputSources(session, frame, xrRefSpace);
updateTrackedSources(session, frame, xrRefSpace);
UpdateInteractables(t);

// Get the XRDevice pose relative to the Frame of Reference we created
Expand Down

0 comments on commit 17f04e1

Please sign in to comment.