Feature/hololens ar

.github/workflows/nx_affected.yml

@@ -99,3 +99,29 @@ jobs:
           publish_branch: main
           publish_dir: ./dist
           force_orphan: true
+
+  deploy-ar-demo:
+    runs-on: ubuntu-latest
+    needs: build
+    #if: success() && github.ref == 'refs/heads/develop'
+
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: ar-demo-build
+          path: dist
+      - name: Check if build exists
+        id: check_files
+        uses: andstor/file-existence-action@v1
+        with:
+          files: "dist/index.html"
+      - name: Deploy to production
+        uses: peaceiris/actions-gh-pages@v3
+        if: steps.check_files.outputs.files_exists == 'true'
+        with:
+          cname: ar.visian.org
+          deploy_key: ${{ secrets.AR_REPO_DEPLOY_KEY }}
+          external_repository: HealthML/visian-ar
+          publish_branch: main
+          publish_dir: ./dist
+          force_orphan: true

apps/ar-demo/src/constants.ts

@@ -0,0 +1,2 @@
+// TODO: Auto-detect if `hit-test` feature is available
+export const USE_HIT_TEST = false;

apps/ar-demo/src/lib/renderer/helpers/scanNavigator.ts

@@ -194,21 +194,12 @@ export default class ScanNavigator implements IDisposable {
   };
 
   private handleTransformMove = () => {
-    this.workingVector.copy(this.transformObject.position);
-
-    this.workingVector.x =
-      Math.round(this.workingVector.x / this.voxelDimensions.x) *
-      this.voxelDimensions.x;
-    this.workingVector.y =
-      Math.round(this.workingVector.y / this.voxelDimensions.y) *
-      this.voxelDimensions.y;
-    this.workingVector.z =
-      Math.round(this.workingVector.z / this.voxelDimensions.z) *
-      this.voxelDimensions.z;
-
-    this.workingVector.divide(this.voxelDimensions);
-    this.workingVector.max(this.minSelectedVoxel);
-    this.workingVector.min(this.maxSelectedVoxel);
+    this.workingVector
+      .copy(this.transformObject.position)
+      .divide(this.voxelDimensions)
+      .round()
+      .max(this.minSelectedVoxel)
+      .min(this.maxSelectedVoxel);
 
     // x is inverted...
     this.workingVector.x = SCAN.voxelCount.x - this.workingVector.x - 1;

apps/ar-demo/src/lib/renderer/helpers/spriteHandler.ts

@@ -22,11 +22,11 @@ export default class SpriteHandler {
 
   private cameraOctant?: number;
 
-  public selectedVoxel: Voxel = {
-    x: Math.floor(SCAN.voxelCount.x / 2),
-    y: Math.floor(SCAN.voxelCount.y / 2),
-    z: Math.floor(SCAN.voxelCount.z / 2),
-  };
+  public selectedVoxel = new THREE.Vector3(
+    Math.floor(SCAN.voxelCount.x / 2),
+    Math.floor(SCAN.voxelCount.y / 2),
+    Math.floor(SCAN.voxelCount.z / 2),
+  );
 
   constructor(private renderer: Renderer) {
     const loader = new THREE.TextureLoader();
@@ -57,7 +57,7 @@ export default class SpriteHandler {
       contrast: { value: 1 },
       brightness: { value: 1 },
       blueTint: { value: true },
-      opacity: { value: 0.5 },
+      opacity: { value: 0.7 },
     };
 
     this.materials = viewTypes.map(
@@ -151,7 +151,7 @@ export default class SpriteHandler {
   };
 
   public setSelectedVoxel = (voxel: Voxel) => {
-    this.selectedVoxel = voxel;
+    this.selectedVoxel.set(voxel.x, voxel.y, voxel.z);
 
     this.materials.forEach((material) => {
       // eslint-disable-next-line no-param-reassign

apps/ar-demo/src/lib/renderer/renderer.ts

@@ -1,6 +1,7 @@
 import * as THREE from "three";
 
 import { IDisposable } from "..";
+import { USE_HIT_TEST } from "../../constants";
 import * as SCAN from "../staticScan";
 import {
   defaultStructureColor,
@@ -66,6 +67,21 @@ export default class Renderer implements IDisposable {
   private scanBaseRotation = Math.PI;
   private acceptARSelect = true;
 
+  private controller1?: THREE.Group;
+  private controller2?: THREE.Group;
+
+  private grabbedDimension?: number;
+  private startPosition = new THREE.Vector3();
+  private startSlice?: number;
+
+  private helperBall = new THREE.Mesh(
+    new THREE.SphereBufferGeometry(0.003),
+    new THREE.MeshBasicMaterial(),
+  );
+
+  private workingVector1 = new THREE.Vector3();
+  private workingVector2 = new THREE.Vector3();
+
   constructor(private canvas: HTMLCanvasElement, public updateUI: () => void) {
     // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
     this.domOverlay = document.getElementById("ar-overlay")!;
@@ -116,13 +132,11 @@ export default class Renderer implements IDisposable {
       this.cameraNavigator,
     );
 
-    this.camera.position.copy(
-      this.scanOffsetGroup.localToWorld(
-        new THREE.Vector3(
-          -0.25 * SCAN.scanSize.x,
-          1.25 * SCAN.scanSize.y,
-          1.25 * SCAN.scanSize.z,
-        ),
+    this.scanOffsetGroup.localToWorld(
+      this.camera.position.set(
+        -0.25 * SCAN.scanSize.x,
+        1.25 * SCAN.scanSize.y,
+        1.25 * SCAN.scanSize.z,
       ),
     );
     const target = this.scanOffsetGroup.localToWorld(
@@ -190,6 +204,32 @@ export default class Renderer implements IDisposable {
       if (frame) {
         this.reticle.update(frame);
       }
+
+      if (
+        this.controller1 &&
+        this.grabbedDimension !== undefined &&
+        this.startSlice !== undefined
+      ) {
+        const offset = this.workingVector1.copy(this.controller1.position);
+        this.scanOffsetGroup.worldToLocal(offset);
+        offset.sub(this.startPosition);
+        offset.divide(SCAN.voxelDimensions).round();
+        offset.x *= -1;
+
+        const sliceOffset = offset.getComponent(this.grabbedDimension);
+        const newSlice = Math.max(
+          0,
+          Math.min(
+            SCAN.voxelCount.getComponent(this.grabbedDimension) - 1,
+            this.startSlice + sliceOffset,
+          ),
+        );
+
+        const newSelectedVoxel = this.workingVector1
+          .copy(this.spriteHandler.selectedVoxel)
+          .setComponent(this.grabbedDimension, newSlice);
+        this.spriteHandler.setSelectedVoxel(newSelectedVoxel);
+      }
     }
 
     if (this.renderDirty || this.arActive) this.forceRender();
@@ -216,8 +256,8 @@ export default class Renderer implements IDisposable {
     this.domOverlay.style.display = "";
 
     const sessionInit = {
-      requiredFeatures: ["hit-test"],
-      optionalFeatures: ["dom-overlay"],
+      requiredFeatures: [],
+      optionalFeatures: ["hit-test", "dom-overlay"],
       domOverlay: { root: this.domOverlay },
     };
 
@@ -232,15 +272,31 @@ export default class Renderer implements IDisposable {
         this.renderer.xr.setReferenceSpaceType("local");
         this.renderer.xr.setSession(session);
 
-        this.reticle.activate();
+        if (USE_HIT_TEST) {
+          this.reticle.activate();
+        }
 
-        this.scanContainer.visible = false;
+        this.scanContainer.visible = !USE_HIT_TEST;
 
         this.updateUI();
 
-        const controller = this.renderer.xr.getController(0);
-        controller.addEventListener("select", this.onARSelect);
-        this.scene.add(controller);
+        // TODO: The HoloLens sadly does not stay consistent in its controller enumeration.
+        // Instead, the primary (right) hand is controller 0, as long as it is visible.
+        // If only the other (left) hand is visible, it becomes controller 0
+        // until the primary hand becomes visible (again).
+        // This has to be accounted for when trying to ensure continous drag & drop interactions.
+        this.controller1 = this.renderer.xr.getController(0);
+        this.controller1.addEventListener("selectstart", this.onARSelect);
+        this.controller1.addEventListener("selectend", this.onARDeselect);
+
+        this.controller2 = this.renderer.xr.getController(1);
+        this.controller2.addEventListener("selectstart", this.onARSelect);
+        this.controller2.addEventListener("selectend", this.onARDeselect);
+
+        this.controller1.add(this.helperBall);
+
+        this.scene.add(this.controller1);
+        this.scene.add(this.controller2);
       })
       .catch((e) => {
         // eslint-disable-next-line no-console
@@ -264,11 +320,17 @@ export default class Renderer implements IDisposable {
         // The XR session hides everything else. So we have to show it again.
         document.getElementById("root")?.setAttribute("style", "");
 
-        const controller = this.renderer.xr.getController(0);
-        controller.removeEventListener("select", this.onARSelect);
+        const controller1 = this.renderer.xr.getController(0);
+        controller1.removeEventListener("selectstart", this.onARSelect);
+        controller1.removeEventListener("selectend", this.onARDeselect);
+
+        const controller2 = this.renderer.xr.getController(1);
+        controller2.removeEventListener("selectstart", this.onARSelect);
+        controller2.removeEventListener("selectend", this.onARDeselect);
 
         this.reticle.hide();
 
+        // TODO: Fix camera reset
         if (this.oldCameraPosition) {
           this.camera.position.copy(this.oldCameraPosition);
           this.oldCameraPosition = undefined;
@@ -298,19 +360,80 @@ export default class Renderer implements IDisposable {
       });
   };
 
-  private onARSelect = () => {
+  // Controller Interaction
+  protected startGrab = (controller: THREE.Group) => {
+    const controllerPosition = this.workingVector2;
+    controller.getWorldPosition(controllerPosition);
+    this.scanOffsetGroup.worldToLocal(controllerPosition);
+    controllerPosition.divide(SCAN.voxelDimensions);
+    controllerPosition.x = SCAN.voxelCount.x - controllerPosition.x - 1;
+    controllerPosition.sub(this.spriteHandler.selectedVoxel);
+    let index = 0;
+    let distance = Infinity;
+    controllerPosition.toArray().forEach((d, i) => {
+      const absD = Math.abs(d);
+      if (absD < distance) {
+        distance = absD;
+        index = i;
+      }
+    });
+
+    const maxDistance = Math.max(...controllerPosition.toArray());
+
+    if (distance < 50 && maxDistance < 300) {
+      this.grabbedDimension = index;
+      this.startPosition.copy(controller.position);
+      this.scanOffsetGroup.worldToLocal(this.startPosition);
+      this.startSlice = [
+        this.spriteHandler.selectedVoxel.x,
+        this.spriteHandler.selectedVoxel.y,
+        this.spriteHandler.selectedVoxel.z,
+      ][index];
+    } else {
+      controller.attach(this.scanContainer);
+      this.scanContainer.userData.selections =
+        (this.scanContainer.userData.selections || 0) + 1;
+      controller.userData.selected = this.scanContainer;
+    }
+  };
+  protected endGrab = (controller: THREE.Group) => {
+    this.grabbedDimension = undefined;
+    this.startSlice = undefined;
+    if (controller.userData.selected !== undefined) {
+      const object = controller.userData.selected;
+      object.userData.selections = (object.userData.selections || 1) - 1;
+      controller.userData.selected = undefined;
+      if (!object.userData.selections) {
+        this.scene.attach(object);
+      }
+    }
+  };
+
+  private onARSelect = (event: THREE.Event) => {
     if (!this.acceptARSelect) return;
 
-    this.scanContainer.visible = true;
+    if (USE_HIT_TEST) {
+      this.scanContainer.visible = true;
 
-    if (this.reticle.active) {
-      if (this.reticle.visible) {
-        this.scanContainer.position.setFromMatrixPosition(this.reticle.matrix);
+      if (this.reticle.active) {
+        if (this.reticle.visible) {
+          this.scanContainer.position.setFromMatrixPosition(
+            this.reticle.matrix,
+          );
 
-        this.reticle.activate(false);
+          this.reticle.activate(false);
+        }
+      } else {
+        this.reticle.activate();
       }
     } else {
-      this.reticle.activate();
+      this.startGrab(event.target);
+    }
+  };
+
+  private onARDeselect = (event: THREE.Event) => {
+    if (!USE_HIT_TEST) {
+      this.endGrab(event.target);
     }
   };
 

apps/ar-demo/src/lib/staticScan/index.ts

@@ -1,31 +1,21 @@
 import * as THREE from "three";
 
-import { Pixel, Voxel } from "../types";
 import preGeneratedGeometries from "./preGeneratedGeometries";
 
-export const voxelCount: Voxel = {
-  x: 170,
-  y: 244,
-  z: 216,
-};
+export const voxelCount = new THREE.Vector3(170, 244, 216);
 
 // in meters
-export const voxelDimensions: Voxel = {
-  x: 0.0009999985694885254,
-  y: 0.001,
-  z: 0.001,
-};
+export const voxelDimensions = new THREE.Vector3(
+  0.0009999985694885254,
+  0.001,
+  0.001,
+);
 
-export const scanSize = {
-  x: voxelCount.x * voxelDimensions.x,
-  y: voxelCount.y * voxelDimensions.y,
-  z: voxelCount.z * voxelDimensions.z,
-};
+export const scanSize = new THREE.Vector3()
+  .copy(voxelCount)
+  .multiply(voxelDimensions);
 
-export const atlasGrid: Pixel = {
-  x: 18,
-  y: 12,
-};
+export const atlasGrid = new THREE.Vector2(18, 12);
 
 export const getConnectedStructureGeometries: () => Promise<THREE.BufferGeometry>[] = () => {
   const geometryLoader = new THREE.BufferGeometryLoader();

libs/rendering/src/lib/volume-renderer/xr-manager.ts

@@ -16,8 +16,7 @@ export class XRManager implements IXRManager {
   protected endGrab = (controller: THREE.Group) => {
     if (controller.userData.selected !== undefined) {
       const object = controller.userData.selected;
-      object.userData.selections =
-        (this.renderer.volume.userData.selections || 1) - 1;
+      object.userData.selections = (object.userData.selections || 1) - 1;
       controller.userData.selected = undefined;
       if (!object.userData.selections) {
         this.renderer.scene.attach(object);