diff --git a/apps/editor/src/assets/de.json b/apps/editor/src/assets/de.json index 280a835ed..9432b2b08 100644 --- a/apps/editor/src/assets/de.json +++ b/apps/editor/src/assets/de.json @@ -121,6 +121,13 @@ "export-tooltip": "Export (Strg + E)", "exporting": "Exportieren", + "annotations": "Annotationen", + "unify-annotations": "Annotationen vereinigen", + "annotation-consensus": "Annotationskonsens", + "show-annotation-density": "Annotationsdichte anzeigen", + + "annotation-time": "Annotationszeit", + "show-delta": "Delta anzeigen", "confirm-task-annotation-tooltip": "Bestätigen", "skip-task-annotation-tooltip": "Überspringen", diff --git a/apps/editor/src/assets/en.json b/apps/editor/src/assets/en.json index 124fbf133..5ce2390f1 100644 --- a/apps/editor/src/assets/en.json +++ b/apps/editor/src/assets/en.json @@ -121,6 +121,13 @@ "export-tooltip": "Export (Ctrl + E)", "exporting": "Exporting", + "annotations": "Annotations", + "unify-annotations": "Unify annotations", + "annotation-consensus": "Annotation consensus", + "show-annotation-density": "Show annotation density", + + "annotation-time": "Annotation Time", + "show-delta": "Show delta", "confirm-task-annotation-tooltip": "Confirm", "skip-task-annotation-tooltip": "Skip", diff --git a/apps/editor/src/components/editor/ai-bar/ai-bar.tsx b/apps/editor/src/components/editor/ai-bar/ai-bar.tsx index 691248e94..8c5425eda 100644 --- a/apps/editor/src/components/editor/ai-bar/ai-bar.tsx +++ b/apps/editor/src/components/editor/ai-bar/ai-bar.tsx @@ -1,4 +1,5 @@ import { + AnnotationStatus, BlueButtonParam, color, fontSize, @@ -20,7 +21,6 @@ import styled from "styled-components"; import { useStore } from "../../../app/root-store"; import { whoHome } from "../../../constants"; -import { AnnotationStatus } from "../../../models/who/annotation"; import { AnnotationData } from "../../../models/who/annotationData"; const AIBarSheet = styled(Sheet)` diff --git a/apps/editor/src/components/editor/layers/layers.tsx b/apps/editor/src/components/editor/layers/layers.tsx index 12b5fc4c4..1c843a295 100644 --- a/apps/editor/src/components/editor/layers/layers.tsx +++ b/apps/editor/src/components/editor/layers/layers.tsx @@ -15,6 +15,7 @@ import { stopPropagation, styledScrollbarMixin, SubtleText, + TaskType, useDelay, useDoubleTap, useForwardEvent, @@ -22,7 +23,7 @@ import { useShortTap, useTranslation, } from "@visian/ui-shared"; -import { Pixel } from "@visian/utils"; +import { isFromWHO, Pixel } from "@visian/utils"; import { Observer, observer } from "mobx-react-lite"; import React, { useCallback, useEffect, useRef, useState } from "react"; import { @@ -365,6 +366,8 @@ export const Layers: React.FC = observer(() => { const layerCount = layers?.length; const activeLayer = store?.editor.activeDocument?.activeLayer; const activeLayerIndex = layers?.findIndex((layer) => layer === activeLayer); + const isSupervisorMode = + isFromWHO() && store?.currentTask?.kind === TaskType.Review; return ( <> { tooltipTx="add-annotation-layer" isDisabled={ !layerCount || - layerCount >= (store?.editor.activeDocument?.maxLayers || 0) + layerCount >= (store?.editor.activeDocument?.maxLayers || 0) || + isSupervisorMode } onPointerDown={ store?.editor.activeDocument?.addNewAnnotationLayer diff --git a/apps/editor/src/components/editor/reviewer-panel/index.ts b/apps/editor/src/components/editor/reviewer-panel/index.ts new file mode 100644 index 000000000..6e087e3bf --- /dev/null +++ b/apps/editor/src/components/editor/reviewer-panel/index.ts @@ -0,0 +1 @@ +export * from "./reviewer-panel"; diff --git a/apps/editor/src/components/editor/reviewer-panel/reviewer-panel.tsx b/apps/editor/src/components/editor/reviewer-panel/reviewer-panel.tsx new file mode 100644 index 000000000..94d4a4191 --- /dev/null +++ b/apps/editor/src/components/editor/reviewer-panel/reviewer-panel.tsx @@ -0,0 +1,77 @@ +import { + BooleanParam, + Divider, + Modal, + NumberParam, + TaskType, +} from "@visian/ui-shared"; +import { observer } from "mobx-react-lite"; +import React, { useCallback, useState } from "react"; +import styled, { keyframes } from "styled-components"; +import { useStore } from "../../../app/root-store"; + +const scaleAnimationIn = keyframes` + 0% { + opacity: 0; + animation-timing-function: ease-in; + } + 100% { + opacity: 1; + } +`; + +const UnificationOptionsContainer = styled.div` + display: flex; + flex-direction: column; + animation: ${scaleAnimationIn} 0.5s; +`; + +export const ReviewerPanel = observer(() => { + const store = useStore(); + if (!(store?.currentTask?.kind === TaskType.Correct)) return <>; + + const [shouldUnifyAnnotations, setShouldUnifyAnnotations] = useState(false); + const [ + shouldShowAnnotationDensity, + setShouldShowAnnotationDensity, + ] = useState(false); + const setAnnotationConsensus = useCallback( + (value: number) => { + store?.editor.activeDocument?.setAnnotationConsensusCount(value); + }, + [store?.editor.activeDocument], + ); + + return ( + + { + setShouldUnifyAnnotations(!shouldUnifyAnnotations); + }} + /> + {shouldUnifyAnnotations && ( + + + + + { + setShouldShowAnnotationDensity(!shouldShowAnnotationDensity); + }} + /> + + )} + + ); +}); diff --git a/apps/editor/src/components/editor/supervisor-panel/index.ts b/apps/editor/src/components/editor/supervisor-panel/index.ts new file mode 100644 index 000000000..8fbba3d89 --- /dev/null +++ b/apps/editor/src/components/editor/supervisor-panel/index.ts @@ -0,0 +1 @@ +export * from "./supervisor-panel"; diff --git a/apps/editor/src/components/editor/supervisor-panel/supervisor-panel.tsx b/apps/editor/src/components/editor/supervisor-panel/supervisor-panel.tsx new file mode 100644 index 000000000..1102acd1d --- /dev/null +++ b/apps/editor/src/components/editor/supervisor-panel/supervisor-panel.tsx @@ -0,0 +1,150 @@ +import { + BooleanParam, + color, + Divider, + fontSize, + Icon, + Modal, + TaskType, + Text, + UserRole, +} from "@visian/ui-shared"; +import { observer } from "mobx-react-lite"; +import React, { useState } from "react"; +import styled from "styled-components"; +import { useStore } from "../../../app/root-store"; + +interface AnnotatorSectionProps { + annotatorRole: UserRole; + annotatorName: string; + annotationTime?: string; + colorAddition: string; + colorDeletion?: string; + isLast?: boolean; +} + +const SectionContainer = styled.div<{ isLast?: boolean }>` + display: flex; + flex-direction: column; + margin-bottom: ${(props) => (props.isLast ? "0px" : "12px")}; + width: 100%; +`; + +const AnnotatorInformationContainer = styled.div` + display: flex; + flex-direction: row; + justify-content: space-between; + align-items: center; +`; + +const AnnotationTimeContainer = styled.div` + display: flex; + flex-direction: column; + margin-top: 8px; +`; + +const TypeText = styled(Text)` + color: ${color("lightText")}; + font-size: ${fontSize("small")}; +`; + +const InformationText = styled(Text)` + font-size: 18px; +`; + +const EditCircle = styled.div<{ circleColor: string }>` + width: 17px; + height: 17px; + background-color: ${(props) => + color(props.circleColor as any) || props.circleColor}; + border-radius: 50%; +`; +const CircleContainer = styled.div` + display: flex; + flex-direction: row; + min-width: 40px; + justify-content: space-between; +`; + +const AnnotatorSection: React.FC = ({ + annotatorRole, + annotatorName, + annotationTime, + colorAddition, + colorDeletion, + isLast = false, +}) => ( + <> + + + + + + + + + + {colorDeletion && ( + + + + )} + + + {annotationTime && ( + + + + + )} + + +); + +export const SupervisorPanel = observer(() => { + const store = useStore(); + if (!(store?.currentTask?.kind === TaskType.Review)) return <>; + const annotationCount = store.currentTask.annotations.length; + const annotations = store.currentTask.annotations.sort( + (firstAnnotation, secondAnnotation) => + new Date(firstAnnotation.submittedAt).getTime() - + new Date(secondAnnotation.submittedAt).getTime(), + ); + + const [shouldShowDelta, setShouldShowDelta] = useState(false); + + return ( + + {/* TODO: Set delta options */} + { + setShouldShowDelta(!shouldShowDelta); + }} + /> + {annotations.map((annotation, index) => { + const isLast = index === annotationCount - 1; + // TODO: Make coloring work properly + const correspondingLayer = store.editor.activeDocument?.getLayer( + annotation.data[0].correspondingLayerId, + ); + const annotationColor = correspondingLayer?.color || "yellow"; + return ( + + ); + })} + + ); +}); diff --git a/apps/editor/src/components/editor/ui-overlay/ui-overlay.tsx b/apps/editor/src/components/editor/ui-overlay/ui-overlay.tsx index 899c38ca6..ae2b2f430 100644 --- a/apps/editor/src/components/editor/ui-overlay/ui-overlay.tsx +++ b/apps/editor/src/components/editor/ui-overlay/ui-overlay.tsx @@ -4,6 +4,7 @@ import { FloatingUIButton, Notification, Spacer, + TaskType, Text, } from "@visian/ui-shared"; import { isFromWHO } from "@visian/utils"; @@ -37,6 +38,8 @@ import { ViewSettings } from "../view-settings"; import { UIOverlayProps } from "./ui-overlay.props"; import { SettingsPopUp } from "../settings-popup"; import { MeasurementPopUp } from "../measurement-popup"; +import { SupervisorPanel } from "../supervisor-panel"; +import { ReviewerPanel } from "../reviewer-panel"; const Container = styled(AbsoluteCover)` align-items: stretch; @@ -245,6 +248,14 @@ export const UIOverlay = observer( + {isFromWHO() && + store?.currentTask?.kind === TaskType.Review && ( + + )} + {isFromWHO() && + store?.currentTask?.kind === TaskType.Correct && ( + + )} diff --git a/apps/editor/src/models/editor/document.ts b/apps/editor/src/models/editor/document.ts index 4cec3c9e5..8a0af1e2c 100644 --- a/apps/editor/src/models/editor/document.ts +++ b/apps/editor/src/models/editor/document.ts @@ -13,6 +13,8 @@ import { ErrorNotification, ValueType, PerformanceMode, + ITask, + BlendGroup, } from "@visian/ui-shared"; import { handlePromiseSettledResult, @@ -86,7 +88,7 @@ export interface DocumentSnapshot { export class Document implements IDocument, ISerializable, IDisposable { - public readonly excludeFromSnapshotTracking = ["editor"]; + public readonly excludeFromSnapshotTracking = ["editor", "customBlendGroups"]; public id: string; protected titleOverride?: string; @@ -96,6 +98,8 @@ export class Document protected layerMap: { [key: string]: Layer }; protected layerIds: string[]; + public customBlendGroups: BlendGroup[] = []; + public measurementType: MeasurementType = "volume"; public history: History; @@ -115,6 +119,8 @@ export class Document public useExclusiveSegmentations = false; + public annotationConsensusCount = 1; + constructor( snapshot: DocumentSnapshot | undefined, protected editor: IEditor, @@ -159,6 +165,7 @@ export class Document measurementDisplayLayerId: observable, layerMap: observable, layerIds: observable, + customBlendGroups: observable, measurementType: observable, history: observable, viewSettings: observable, @@ -168,6 +175,7 @@ export class Document showLayerMenu: observable, trackingData: observable, useExclusiveSegmentations: observable, + annotationConsensusCount: observable, title: computed, activeLayer: computed, @@ -187,12 +195,15 @@ export class Document moveLayer: action, deleteLayer: action, toggleTypeAndRepositionLayer: action, + addBlendGroup: action, + deleteBlendGroup: action, importImage: action, importAnnotation: action, importTrackingLog: action, setShowLayerMenu: action, toggleLayerMenu: action, setUseExclusiveSegmentations: action, + setAnnotationConsensusCount: action, applySnapshot: action, }); @@ -417,6 +428,16 @@ export class Document return Object.values(this.layerMap).some((layer) => layer.is3DLayer); } + public addBlendGroup(group: BlendGroup): void { + this.customBlendGroups.push(group); + } + + public deleteBlendGroup(group: BlendGroup): void { + this.customBlendGroups = this.customBlendGroups.filter( + (blendGroup) => blendGroup !== group, + ); + } + // I/O public exportZip = async (limitToAnnotations?: boolean) => { const zip = new Zip(); @@ -799,6 +820,10 @@ export class Document ) as unknown) as IImageLayer[]; } + public setAnnotationConsensusCount = (value = 1) => { + this.annotationConsensusCount = value; + }; + // Proxies public get sliceRenderer(): ISliceRenderer | undefined { return this.editor.sliceRenderer; @@ -824,6 +849,10 @@ export class Document return this.editor.performanceMode; } + public get currentTask(): ITask | undefined { + return this.context?.getCurrentTask(); + } + // Serialization public toJSON(): DocumentSnapshot { return { diff --git a/apps/editor/src/models/editor/editor.ts b/apps/editor/src/models/editor/editor.ts index c0f389615..ec2f1a1de 100644 --- a/apps/editor/src/models/editor/editor.ts +++ b/apps/editor/src/models/editor/editor.ts @@ -7,6 +7,7 @@ import { ColorMode, i18n, IEditor, + IImageLayer, ISliceRenderer, IVolumeRenderer, PerformanceMode, @@ -86,6 +87,18 @@ export class Editor }); this.applySnapshot(snapshot); + + // TODO: Remove this debug function. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (window as any).addDebugGroup = () => { + this.activeDocument?.addBlendGroup({ + mode: "COMPARE", + layers: [ + this.activeDocument.layers[0] as IImageLayer, + this.activeDocument.layers[1] as IImageLayer, + ], + }); + }; } public dispose(): void { diff --git a/apps/editor/src/models/editor/layers/layer-group.ts b/apps/editor/src/models/editor/layers/layer-group.ts index 3dcae272f..843c1d61b 100644 --- a/apps/editor/src/models/editor/layers/layer-group.ts +++ b/apps/editor/src/models/editor/layers/layer-group.ts @@ -1,6 +1,13 @@ -import { IDocument, ILayer, ILayerGroup } from "@visian/ui-shared"; +import { + BlendGroup, + BlendMode, + IDocument, + IImageLayer, + ILayer, + ILayerGroup, +} from "@visian/ui-shared"; import { ISerializable } from "@visian/utils"; -import { action, makeObservable, observable, toJS } from "mobx"; +import { action, computed, makeObservable, observable, toJS } from "mobx"; import { Layer, LayerSnapshot } from "./layer"; @@ -17,17 +24,23 @@ export class LayerGroup protected layerIds: string[] = []; + protected blendMode?: BlendMode; + constructor( snapshot: Partial | undefined, protected document: IDocument, ) { super(snapshot, document); - makeObservable(this, { + makeObservable(this, { layerIds: observable, + blendMode: observable, + + blendGroup: computed, addLayer: action, removeLayer: action, + setBlendMode: action, }); } @@ -36,6 +49,33 @@ export class LayerGroup return this.layerIds.map((id) => this.document.getLayer(id)!); } + public get blendGroup(): BlendGroup | undefined { + if (!this.blendMode) return undefined; + + const imageLayers = this.layers.filter( + (layer) => layer.kind === "image", + ) as IImageLayer[]; + + if (this.blendMode === "COMPARE") { + if (imageLayers.length !== 2) return undefined; + + return { + mode: "COMPARE", + layers: imageLayers as [IImageLayer, IImageLayer], + }; + } + + if (this.blendMode === "MAJORITY_VOTE") { + return { + mode: "MAJORITY_VOTE", + layers: imageLayers, + majority: Math.ceil(imageLayers.length / 2), + }; + } + + return undefined; + } + public addLayer(idOrLayer: string | ILayer) { if (typeof idOrLayer === "string") { this.layerIds.push(idOrLayer); @@ -60,6 +100,10 @@ export class LayerGroup idOrLayer.setParent(); } + public setBlendMode = (value?: BlendMode) => { + this.blendMode = value; + }; + // Serialization public toJSON(): LayerGroupSnapshot { return { diff --git a/apps/editor/src/models/editor/layers/layer.ts b/apps/editor/src/models/editor/layers/layer.ts index f3a30b455..389cfaf1c 100644 --- a/apps/editor/src/models/editor/layers/layer.ts +++ b/apps/editor/src/models/editor/layers/layer.ts @@ -1,10 +1,4 @@ -import { - BlendMode, - color, - IDocument, - ILayer, - MarkerConfig, -} from "@visian/ui-shared"; +import { color, IDocument, ILayer, MarkerConfig } from "@visian/ui-shared"; import { ISerializable, ViewType } from "@visian/utils"; import { action, computed, makeObservable, observable } from "mobx"; import { Matrix4 } from "three"; @@ -26,7 +20,6 @@ export interface LayerSnapshot { titleOverride?: string; parentId?: string; - blendMode: BlendMode; color?: string; isVisible: boolean; opacityOverride?: number; @@ -46,7 +39,6 @@ export class Layer implements ILayer, ISerializable { protected titleOverride?: string; protected parentId?: string; - public blendMode!: BlendMode; public color?: string; public isVisible!: boolean; protected opacityOverride?: number; @@ -68,7 +60,6 @@ export class Layer implements ILayer, ISerializable { id: observable, titleOverride: observable, parentId: observable, - blendMode: observable, color: observable, isVisible: observable, opacityOverride: observable, @@ -81,7 +72,6 @@ export class Layer implements ILayer, ISerializable { setParent: action, setIsAnnotation: action, setTitle: action, - setBlendMode: action, setColor: action, setIsVisible: action, setOpacity: action, @@ -125,10 +115,6 @@ export class Layer implements ILayer, ISerializable { : undefined; } - public setBlendMode = (value?: BlendMode): void => { - this.blendMode = value || "NORMAL"; - }; - public setColor = (value?: string): void => { this.color = value; }; @@ -148,7 +134,6 @@ export class Layer implements ILayer, ISerializable { }; public resetSettings = (): void => { - this.setBlendMode(); this.setColor(this.isAnnotation ? defaultAnnotationColor : undefined); this.setOpacity(); }; @@ -195,7 +180,6 @@ export class Layer implements ILayer, ISerializable { id: this.id, titleOverride: this.titleOverride, parentId: this.parentId, - blendMode: this.blendMode, color: this.color, isVisible: this.isVisible, opacityOverride: this.opacityOverride, @@ -211,7 +195,6 @@ export class Layer implements ILayer, ISerializable { this.setIsAnnotation(snapshot?.isAnnotation); this.setTitle(snapshot?.titleOverride); this.setParent(snapshot?.parentId); - this.setBlendMode(snapshot?.blendMode); this.setColor(snapshot?.color); this.setIsVisible(snapshot?.isVisible); this.setOpacity(snapshot?.opacityOverride); diff --git a/apps/editor/src/models/editor/tools/tool.ts b/apps/editor/src/models/editor/tools/tool.ts index 8b2f28a94..dcee16aca 100644 --- a/apps/editor/src/models/editor/tools/tool.ts +++ b/apps/editor/src/models/editor/tools/tool.ts @@ -3,9 +3,10 @@ import { IconType, IDocument, ITool, + TaskType, ViewMode, } from "@visian/ui-shared"; -import { ISerializable } from "@visian/utils"; +import { ISerializable, isFromWHO } from "@visian/utils"; import { makeObservable, observable } from "mobx"; import { Parameter, ParameterSnapshot } from "../parameters"; @@ -96,11 +97,17 @@ export class Tool } public canActivate(): boolean { + const isSupervisorMode = + isFromWHO() && this.document.currentTask?.kind === TaskType.Review; return Boolean( - (!this.supportedViewModes || - this.supportedViewModes.includes( - this.document.viewSettings.viewMode, - )) && + !( + isSupervisorMode && + (this.isDrawingTool || this.supportAnnotationsOnly) + ) && + (!this.supportedViewModes || + this.supportedViewModes.includes( + this.document.viewSettings.viewMode, + )) && (!this.supportedLayerKinds || (this.document.activeLayer && this.supportedLayerKinds.includes( diff --git a/apps/editor/src/models/root.ts b/apps/editor/src/models/root.ts index eb6dc3ec7..662d97c0f 100644 --- a/apps/editor/src/models/root.ts +++ b/apps/editor/src/models/root.ts @@ -6,6 +6,8 @@ import { IStorageBackend, Tab, ErrorNotification, + TaskType, + ITask, } from "@visian/ui-shared"; import { createFileFromBase64, @@ -21,7 +23,7 @@ import { DICOMWebServer } from "./dicomweb-server"; import { Editor, EditorSnapshot } from "./editor"; import { Tracker } from "./tracking"; import { ProgressNotification } from "./types"; -import { Task, TaskType } from "./who"; +import { Task } from "./who"; export interface RootSnapshot { editor: EditorSnapshot; @@ -54,7 +56,7 @@ export class RootStore implements ISerializable, IDisposable { public refs: { [key: string]: React.RefObject } = {}; public pointerDispatch?: IDispatch; - public currentTask?: Task; + public currentTask?: ITask; public tracker?: Tracker; @@ -96,6 +98,7 @@ export class RootStore implements ISerializable, IDisposable { setError: this.setError, getTracker: () => this.tracker, getColorMode: () => this.colorMode, + getCurrentTask: () => this.currentTask, }); deepObserve(this.editor, this.persist, { @@ -194,24 +197,31 @@ export class RootStore implements ISerializable, IDisposable { } else { // Task Type is Correct or Review await Promise.all( - whoTask.annotations.map(async (annotation, index) => { - const title = - whoTask.samples[index].title || - whoTask.samples[0].title || - `annotation_${index}`; + whoTask.annotations.map(async (annotation, annotationIndex) => { + let baseTitle = + whoTask.samples[annotationIndex]?.title || + whoTask.samples[0]?.title || + // TODO: Translation for unnamed + "unnamed.nii"; + // TODO: Handle base title without ".nii" ending + baseTitle = baseTitle.replace( + ".nii", + `_annotation_${annotationIndex}`, + ); await Promise.all( - annotation.data.map(async (annotationData) => { + annotation.data.map(async (annotationData, dataIndex) => { + const title = `${baseTitle}_${dataIndex}`; const createdLayerId = await this.editor.activeDocument?.importFiles( createFileFromBase64( - title.replace(".nii", "_annotation").concat(".nii"), + title.concat(".nii"), annotationData.data, ), - title.replace(".nii", "_annotation"), + title, true, ); if (createdLayerId) - annotationData.correspondingLayerId = createdLayerId; + annotationData.setCorrespondingLayerId(createdLayerId); }), ); }), diff --git a/apps/editor/src/models/types.ts b/apps/editor/src/models/types.ts index d3de07a79..9a84a62ba 100644 --- a/apps/editor/src/models/types.ts +++ b/apps/editor/src/models/types.ts @@ -1,5 +1,10 @@ import type React from "react"; -import type { ColorMode, ErrorNotification, Theme } from "@visian/ui-shared"; +import type { + ColorMode, + ErrorNotification, + ITask, + Theme, +} from "@visian/ui-shared"; import type { Tracker } from "./tracking"; @@ -27,6 +32,8 @@ export interface StoreContext { getTracker(): Tracker | undefined; getColorMode(): ColorMode; + + getCurrentTask(): ITask | undefined; } export interface ProgressNotification { diff --git a/apps/editor/src/models/who/annotation.ts b/apps/editor/src/models/who/annotation.ts index f0c8a69e1..fe586caa4 100644 --- a/apps/editor/src/models/who/annotation.ts +++ b/apps/editor/src/models/who/annotation.ts @@ -1,21 +1,12 @@ -import { AnnotationData, AnnotationDataSnapshot } from "./annotationData"; -import { User, UserSnapshot } from "./user"; +import { + AnnotationSnapshot, + AnnotationStatus, + IAnnotation, +} from "@visian/ui-shared"; +import { AnnotationData } from "./annotationData"; +import { User } from "./user"; -export enum AnnotationStatus { - Pending = "PENDING", - Completed = "COMPLETED", - Rejected = "REJECTED", -} - -export interface AnnotationSnapshot { - annotationUUID: string; - status: AnnotationStatus; - annotationDataList: AnnotationDataSnapshot[]; - annotator: UserSnapshot; - submittedAt: string; -} - -export class Annotation { +export class Annotation implements IAnnotation { public annotationUUID: string; public status: AnnotationStatus; public data: AnnotationData[]; @@ -23,6 +14,7 @@ export class Annotation { public submittedAt: string; // TODO: Properly type API response data + // TODO: Make observable constructor(annotation: any) { this.annotationUUID = annotation.annotationUUID; this.status = annotation.status; diff --git a/apps/editor/src/models/who/annotationData.ts b/apps/editor/src/models/who/annotationData.ts index 904af2c1f..669db1463 100644 --- a/apps/editor/src/models/who/annotationData.ts +++ b/apps/editor/src/models/who/annotationData.ts @@ -1,9 +1,7 @@ -export interface AnnotationDataSnapshot { - annotationDataUUID: string; - data: string; -} +import { AnnotationDataSnapshot, IAnnotationData } from "@visian/ui-shared"; +import { action, makeObservable, observable } from "mobx"; -export class AnnotationData { +export class AnnotationData implements IAnnotationData { public annotationDataUUID: string; public data: string; public correspondingLayerId = ""; @@ -12,6 +10,18 @@ export class AnnotationData { constructor(annotationData: any) { this.annotationDataUUID = annotationData.annotationDataUUID; this.data = annotationData.data; + + makeObservable(this, { + annotationDataUUID: observable, + data: observable, + correspondingLayerId: observable, + + setCorrespondingLayerId: action, + }); + } + + public setCorrespondingLayerId(layerId: string): void { + this.correspondingLayerId = layerId; } public toJSON(): AnnotationDataSnapshot { diff --git a/apps/editor/src/models/who/annotationTask.ts b/apps/editor/src/models/who/annotationTask.ts index 953c2f2ed..0b60d0164 100644 --- a/apps/editor/src/models/who/annotationTask.ts +++ b/apps/editor/src/models/who/annotationTask.ts @@ -1,24 +1,17 @@ -export interface AnnotationTaskSnapshot { - annotationTaskUUID: string; - kind: string; - title: string; - description: string; -} - -export enum AnnotationTaskType { - Classification = "classification", - ObjectDetection = "object_detection", - SemanticSegmentation = "semantic_segmentation", - InstanceSegmentation = "instance_segmentation", -} +import { + AnnotationTaskSnapshot, + AnnotationTaskType, + IAnnotationTask, +} from "@visian/ui-shared"; -export class AnnotationTask { +export class AnnotationTask implements IAnnotationTask { public annotationTaskUUID: string; public kind: AnnotationTaskType; public title: string; public description: string; // TODO: Properly type API response data + // TODO: Make observable constructor(annotationTask: any) { this.annotationTaskUUID = annotationTask.annotationTaskUUID; this.kind = annotationTask.kind; diff --git a/apps/editor/src/models/who/annotator.ts b/apps/editor/src/models/who/annotator.ts index 0492fb40b..a6c481ed9 100644 --- a/apps/editor/src/models/who/annotator.ts +++ b/apps/editor/src/models/who/annotator.ts @@ -1,15 +1,6 @@ -export interface AnnotatorSnapshot { - annotatorUUID: string; - expertise: string; - yearsInPractice: number; - expectedSalary: number; - workCountry: string; - studyCountry: string; - selfAssessment: number; - degree: string; -} +import { AnnotatorSnapshot, IAnnotator } from "@visian/ui-shared"; -export class Annotator { +export class Annotator implements IAnnotator { public annotatorUUID: string; public expertise: string; public yearsInPractice: number; @@ -20,6 +11,7 @@ export class Annotator { public degree: string; // TODO: Properly type API response data + // TODO: Make observable constructor(annotator: any) { this.annotatorUUID = annotator.annotatorUUID; this.expertise = annotator.expertise; diff --git a/apps/editor/src/models/who/campaign.ts b/apps/editor/src/models/who/campaign.ts new file mode 100644 index 000000000..8966f0487 --- /dev/null +++ b/apps/editor/src/models/who/campaign.ts @@ -0,0 +1,44 @@ +import { CampaignSnapshot, ICampaign } from "@visian/ui-shared"; +import { User } from "./user"; + +export class Campaign implements ICampaign { + public campaignUUID: string; + public name: string; + public description: string; + public status: string; + public datasets: string[]; + public annotators: User[]; + public reviewers: User[]; + + // TODO: Properly type API response data + // TODO: Make observable + constructor(campaign: any) { + this.campaignUUID = campaign.campaignUUID; + this.name = campaign.name; + this.description = campaign.description; + this.status = campaign.status; + this.datasets = campaign.datasets; + this.annotators = campaign.annotators.map( + (annotator: any) => new User(annotator), + ); + this.reviewers = campaign.reviewers.map( + (reviewer: any) => new User(reviewer), + ); + } + + public toJSON(): CampaignSnapshot { + return { + campaignUUID: this.campaignUUID, + name: this.name, + description: this.description, + status: this.status, + datasets: this.datasets, + annotators: Object.values(this.annotators).map((annotator) => + annotator.toJSON(), + ), + reviewers: Object.values(this.reviewers).map((reviewer) => + reviewer.toJSON(), + ), + }; + } +} diff --git a/apps/editor/src/models/who/reviewer.ts b/apps/editor/src/models/who/reviewer.ts index ee3b4feb0..270b5bdb2 100644 --- a/apps/editor/src/models/who/reviewer.ts +++ b/apps/editor/src/models/who/reviewer.ts @@ -1,11 +1,10 @@ -export interface ReviewerSnapshot { - reviewerUUID: string; -} +import { IReviewer, ReviewerSnapshot } from "@visian/ui-shared"; -export class Reviewer { +export class Reviewer implements IReviewer { public reviewerUUID: string; // TODO: Properly type API response data + // TODO: Make observable constructor(reviewer: any) { this.reviewerUUID = reviewer.reviewerUUID; } diff --git a/apps/editor/src/models/who/sample.ts b/apps/editor/src/models/who/sample.ts index fd18647e6..936ad6292 100644 --- a/apps/editor/src/models/who/sample.ts +++ b/apps/editor/src/models/who/sample.ts @@ -1,15 +1,12 @@ -export interface SampleSnapshot { - sampleUUID: string; - title: string; - data: string; -} +import { ISample, SampleSnapshot } from "@visian/ui-shared"; -export class Sample { +export class Sample implements ISample { public sampleUUID: string; public title: string; public data: string; // TODO: Properly type API response data + // TODO: Make observable constructor(sample: any) { this.sampleUUID = sample.sampleUUID; this.title = sample.title; diff --git a/apps/editor/src/models/who/task.ts b/apps/editor/src/models/who/task.ts index 5745705cf..fc1e481b3 100644 --- a/apps/editor/src/models/who/task.ts +++ b/apps/editor/src/models/who/task.ts @@ -1,24 +1,16 @@ -import { Annotation, AnnotationSnapshot, AnnotationStatus } from "./annotation"; -import { AnnotationTask, AnnotationTaskSnapshot } from "./annotationTask"; +import { + AnnotationStatus, + ITask, + TaskSnapshot, + TaskType, +} from "@visian/ui-shared"; +import { Annotation } from "./annotation"; +import { AnnotationTask } from "./annotationTask"; +import { Campaign } from "./campaign"; import { Sample } from "./sample"; -import { User, UserSnapshot } from "./user"; +import { User } from "./user"; -export interface TaskSnapshot { - taskUUID: string; - kind: string; - readOnly: boolean; - annotationTasks: AnnotationTaskSnapshot[]; - annotations?: AnnotationSnapshot[]; - assignee: UserSnapshot; -} - -export enum TaskType { - Create = "create", - Correct = "correct", - Review = "review", -} - -export class Task { +export class Task implements ITask { public taskUUID: string; public kind: TaskType; public readOnly: boolean; @@ -26,8 +18,10 @@ export class Task { public samples: Sample[]; public annotations: Annotation[]; public assignee: User; + public campaign?: Campaign; // TODO: Properly type API response data + // TODO: Make observable constructor(task: any) { this.taskUUID = task.taskUUID; this.kind = task.kind; @@ -40,6 +34,9 @@ export class Task { (annotation: any) => new Annotation(annotation), ); this.assignee = new User(task.assignee); + if (task.campaign && Object.keys(task.campaign).length > 1) { + this.campaign = new Campaign(task.campaign); + } } public addNewAnnotation(): void { @@ -65,6 +62,7 @@ export class Task { annotation.toJSON(), ), assignee: this.assignee.toJSON(), + campaign: this.campaign ? this.campaign.toJSON() : {}, }; } } diff --git a/apps/editor/src/models/who/user.ts b/apps/editor/src/models/who/user.ts index e14818349..31b6126a1 100644 --- a/apps/editor/src/models/who/user.ts +++ b/apps/editor/src/models/who/user.ts @@ -1,18 +1,8 @@ -import { Annotator, AnnotatorSnapshot } from "./annotator"; -import { Reviewer, ReviewerSnapshot } from "./reviewer"; +import { IUser, UserRole, UserSnapshot } from "@visian/ui-shared"; +import { Annotator } from "./annotator"; +import { Reviewer } from "./reviewer"; -export interface UserSnapshot { - userUUID: string; - idpID: string; - username: string; - birthdate: string; - timezone: string; - email: string; - annotatorRole: AnnotatorSnapshot | Record; - reviewerRole: ReviewerSnapshot | Record; -} - -export class User { +export class User implements IUser { public userUUID: string; public idpID: string; public username: string; @@ -23,6 +13,7 @@ export class User { public reviewerRole?: Reviewer; // TODO: Properly type API response data + // TODO: Make observable constructor(user: any) { this.userUUID = user.userUUID; this.idpID = user.idpID; @@ -30,14 +21,21 @@ export class User { this.birthdate = user.birthdate; this.timezone = user.timezone; this.email = user.email; - if (user.annotatorRole && Object.keys(user.annotatorRole).length > 1) { + if (user.annotatorRole && "annotatorUUID" in user.annotatorRole) { this.annotatorRole = new Annotator(user.annotatorRole); } - if (user.reviewerRole && Object.keys(user.reviewerRole).length > 1) { + if (user.reviewerRole && "reviewerUUID" in user.reviewerRole) { this.reviewerRole = new Reviewer(user.reviewerRole); } } + // TODO: Return actual role name + public getRoleName(): UserRole { + if (this.reviewerRole) return "Reviewer"; + if (this.annotatorRole) return "Annotator"; + return "Supervisor"; + } + public toJSON(): UserSnapshot { return { userUUID: this.userUUID, diff --git a/libs/rendering/src/lib/shaders/compose-layered-shader.ts b/libs/rendering/src/lib/shaders/compose-layered-shader.ts index dff58da44..a043c37fb 100644 --- a/libs/rendering/src/lib/shaders/compose-layered-shader.ts +++ b/libs/rendering/src/lib/shaders/compose-layered-shader.ts @@ -1,5 +1,7 @@ /* eslint-disable max-len */ +import { BlendGroup, IImageLayer } from "@visian/ui-shared"; + /** * Generates the GLSL code for the `reduceLayerStack` macro which blends the * image data of all layers, taking into account their layer settings. @@ -77,72 +79,60 @@ const generateReduceLayerStack = ( }; /** - * Generates the GLSL code for the `reduceEnhancedLayerStack` macro which blends the - * image data of all layers, taking into account their layer settings, after - * applying an enhancement function to the non-annotation layers. + * Generates the GLSL code for the `reduceEnhancedLayerStack` macro which processes one single layer. * - * @param layerCount The number of layers. - * @param outputName The output variable to assign the blended color to. + * @param layerIndex The index of the layers. + * @param image The variable name for the result in the shader code. + * @param activeLayer The variable name for the active layer in the shader code. + * @param accumulatedAnnotations The variable name for the accumulated annotations in the shader code. * @param volumeCoords The name of the variable holding the current UV coordinates. * @param enhancementFunctionName The optional name of the function which is * applied to every non-annotation layer before blending. * @returns The generated GLSL code. */ -const generateReduceEnhancedLayerStack = ( - layerCount: number, - outputName = "imageValue", +const simpleLayerProcessing = ( + layerIndex: number, + image: string, + activeLayer: string, + accumulatedAnnotations: string, volumeCoords = "volumeCoords", activeLayerMergeName?: string, enhancementFunctionName?: string, ) => { - const image = `_image${Math.floor(Math.random() * 1000)}`; - const activeLayer = `_activeLayer${Math.floor(Math.random() * 1000)}`; - const oldAlpha = `_oldAlpha${Math.floor(Math.random() * 1000)}`; - const accumulatedAnnotations = `_accumulatedAnnotations${Math.floor( - Math.random() * 1000, - )}`; - let fragment = ` - vec4 ${image} = vec4(0.0); - vec4 ${activeLayer} = texture(uActiveLayerData, ${volumeCoords}); - float ${oldAlpha} = 0.0; - float ${accumulatedAnnotations} = 0.0; - `; - - for (let i = 0; i < layerCount; i++) { - fragment += `${image} = texture(uLayerData[${i}], ${volumeCoords}); + let fragment = `${image} = texture(uLayerData[${layerIndex}], ${volumeCoords}); `; - if (activeLayerMergeName) { - fragment += `${image} = mix( + if (activeLayerMergeName) { + fragment += `${image} = mix( ${image}, uToolPreviewMerge == 1 ? max(${image}, ${activeLayerMergeName}) : clamp(${image} - ${activeLayerMergeName}, 0.0, 1.0), - float(${i} == uActiveLayerIndex && uLayerAnnotationStatuses[${i}]) + float(${layerIndex} == uActiveLayerIndex && uLayerAnnotationStatuses[${layerIndex}]) ); `; - } + } - if (i === 0) { - // Region growing preview - fragment += ` + if (layerIndex === -1) { + // Region growing preview + fragment += ` ${image}.rgb = step(uRegionGrowingThreshold, ${image}.rgb); ${image}.rgb *= vec3(1.0) - step(0.001, ${activeLayer}.rgb); `; - } + } - fragment += ` - if(uComponents < 3 || uLayerAnnotationStatuses[${i}]) { + fragment += ` + if(uComponents < 3 || uLayerAnnotationStatuses[${layerIndex}]) { ${image}.a = ${image}.x; - ${image}.rgb = uLayerColors[${i}]; + ${image}.rgb = uLayerColors[${layerIndex}]; } - if(uUseExclusiveSegmentations && uLayerAnnotationStatuses[${i}]) { + if(uUseExclusiveSegmentations && uLayerAnnotationStatuses[${layerIndex}]) { ${image}.a = mix(${image}.a, 0.0, step(0.001, ${accumulatedAnnotations})); - ${accumulatedAnnotations} = mix(${accumulatedAnnotations}, 1.0, step(0.001, ${image}.a * step(0.001, uLayerOpacities[${i}]))); + ${accumulatedAnnotations} = mix(${accumulatedAnnotations}, 1.0, step(0.001, ${image}.a * step(0.001, uLayerOpacities[${layerIndex}]))); } - if(uLayerAnnotationStatuses[${i}]) { + if(uLayerAnnotationStatuses[${layerIndex}]) { ${image}.a = step(0.01, ${image}.a); } ${ enhancementFunctionName @@ -152,8 +142,166 @@ const generateReduceEnhancedLayerStack = ( : "" } - ${image}.a *= uLayerOpacities[${i}]; - + ${image}.a *= uLayerOpacities[${layerIndex}]; + `; + + return fragment; +}; + +/** + * Generates the GLSL code for the `reduceEnhancedLayerStack` macro which compares two annotation layers. + * + * @param layerIndizes The indizes of the layers to compare. + * @param image The variable name for the result in the shader code. + * @param activeLayer The variable name for the active layer in the shader code. + * @param accumulatedAnnotations The variable name for the accumulated annotations in the shader code. + * @param volumeCoords The name of the variable holding the current UV coordinates. + * @param enhancementFunctionName The optional name of the function which is + * applied to every non-annotation layer before blending. + * @returns The generated GLSL code. + */ +const compareLayers = ( + layerIndizes: [number, number], + image: string, + activeLayer: string, + accumulatedAnnotations: string, + volumeCoords = "volumeCoords", + activeLayerMergeName?: string, + enhancementFunctionName?: string, +) => { + const layer1 = `_layer1${Math.floor(Math.random() * 1000)}`; + const layer2 = `_layer2${Math.floor(Math.random() * 1000)}`; + const mixedColor = `_mixedColor${Math.floor(Math.random() * 1000)}`; + + let fragment = ` + vec4 ${layer1} = vec4(0.0); + vec4 ${layer2} = vec4(0.0); + vec4 ${mixedColor} = mix( + vec4(uLayerColors[${layerIndizes[0]}], uLayerOpacities[${layerIndizes[0]}]), + vec4(uLayerColors[${layerIndizes[1]}], uLayerOpacities[${layerIndizes[1]}]), + 0.5 + ); + `; + + fragment += simpleLayerProcessing( + layerIndizes[0], + layer1, + activeLayer, + accumulatedAnnotations, + volumeCoords, + activeLayerMergeName, + enhancementFunctionName, + ); + fragment += simpleLayerProcessing( + layerIndizes[1], + layer2, + activeLayer, + accumulatedAnnotations, + volumeCoords, + activeLayerMergeName, + enhancementFunctionName, + ); + + fragment += ` + ${image} = mix( + ${layer2}, + mix( + ${layer1}, + ${mixedColor}, + step(0.0001, ${layer2}.a) + ), + step(0.0001, ${layer1}.a) + ); + `; + + return fragment; +}; + +/** + * Generates the GLSL code for the `reduceEnhancedLayerStack` macro which blends the + * image data of all layers, taking into account their layer settings, after + * applying an enhancement function to the non-annotation layers. + * + * @param layers The layers. + * @param blendGroups The blend groups. + * @param outputName The output variable to assign the blended color to. + * @param volumeCoords The name of the variable holding the current UV coordinates. + * @param enhancementFunctionName The optional name of the function which is + * applied to every non-annotation layer before blending. + * @returns The generated GLSL code. + */ +const generateReduceEnhancedLayerStack = ( + layers: IImageLayer[], + blendGroups: BlendGroup[], + outputName = "imageValue", + volumeCoords = "volumeCoords", + activeLayerMergeName?: string, + enhancementFunctionName?: string, +) => { + const areBlendGroupsUsed = blendGroups.map((_) => false); + + const image = `_image${Math.floor(Math.random() * 1000)}`; + const activeLayer = `_activeLayer${Math.floor(Math.random() * 1000)}`; + const oldAlpha = `_oldAlpha${Math.floor(Math.random() * 1000)}`; + const accumulatedAnnotations = `_accumulatedAnnotations${Math.floor( + Math.random() * 1000, + )}`; + let fragment = ` + vec4 ${image} = vec4(0.0); + vec4 ${activeLayer} = texture(uActiveLayerData, ${volumeCoords}); + float ${oldAlpha} = 0.0; + float ${accumulatedAnnotations} = 0.0; + `; + + for (let i = -1; i < layers.length; i++) { + if ( + i === -1 || + !blendGroups.find((group) => group.layers.includes(layers[i])) + ) { + fragment += simpleLayerProcessing( + i + 1, + image, + activeLayer, + accumulatedAnnotations, + volumeCoords, + activeLayerMergeName, + enhancementFunctionName, + ); + } else { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const blendGroup = blendGroups.find((group) => + group.layers.includes(layers[i]), + )!; + const blendGroupIndex = blendGroups.indexOf(blendGroup); + const isBlendGroupUsed = areBlendGroupsUsed[blendGroupIndex]; + + // eslint-disable-next-line no-continue + if (isBlendGroupUsed) continue; + + areBlendGroupsUsed[blendGroupIndex] = true; + + if (blendGroup.mode === "COMPARE") { + fragment += compareLayers( + blendGroup.layers.map((layer) => layers.indexOf(layer) + 1) as [ + number, + number, + ], + image, + activeLayer, + accumulatedAnnotations, + volumeCoords, + activeLayerMergeName, + enhancementFunctionName, + ); + } else if (blendGroup.mode === "MAJORITY_VOTE") { + // TODO: Implement majority rendering + } else { + throw new Error("unexpected-blend-mode"); + } + } + + // Blend result behind previous layers + fragment += ` ${oldAlpha} = ${outputName}.a; ${outputName}.a = mix(${image}.a, 1.0, ${oldAlpha}); ${outputName}.rgb = mix( @@ -205,11 +353,19 @@ const reduceRawImagesRegex = /{{reduceRawImages\((\w+),\s*(\w+)\)}}/g; * procedurally generated GLSL code. * * @param shader The shader string. - * @param layerCount The number of layers the shader should be instantiated for. + * @param layers The layers the shader should be instantiated for. + * @param blendGroups The blend groups the shader should use. * @returns The processed shader string. */ -export const composeLayeredShader = (shader: string, layerCount: number) => - shader +export const composeLayeredShader = ( + shader: string, + layers: IImageLayer[], + blendGroups: BlendGroup[], +) => { + // Addtional layer for tool preview. + const layerCount = layers.length + 1; + + return shader .replace(layerCountRegex, `${layerCount}`) .replace( reduceLayerStackRegex, @@ -240,7 +396,8 @@ export const composeLayeredShader = (shader: string, layerCount: number) => enhancementFunctionName, ) => generateReduceEnhancedLayerStack( - layerCount, + layers, + blendGroups, outputName, uvName, activeLayerMergeName, @@ -250,3 +407,4 @@ export const composeLayeredShader = (shader: string, layerCount: number) => .replace(reduceRawImagesRegex, (_match, outputName, uvName) => generateReduceRawImages(layerCount, outputName, uvName), ); +}; diff --git a/libs/rendering/src/lib/slice-renderer/slice-material.ts b/libs/rendering/src/lib/slice-renderer/slice-material.ts index da2660c4e..d3db01667 100644 --- a/libs/rendering/src/lib/slice-renderer/slice-material.ts +++ b/libs/rendering/src/lib/slice-renderer/slice-material.ts @@ -1,5 +1,11 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { color, IEditor, IImageLayer, MergeFunction } from "@visian/ui-shared"; +import { + BlendGroup, + color, + IEditor, + IImageLayer, + MergeFunction, +} from "@visian/ui-shared"; import { IDisposable, IDisposer, ViewType } from "@visian/utils"; import { autorun, reaction } from "mobx"; import * as THREE from "three"; @@ -83,16 +89,24 @@ export class SliceMaterial extends THREE.ShaderMaterial implements IDisposable { }, ), reaction( - () => editor.volumeRenderer?.renderedImageLayerCount || 1, - (layerCount: number) => { + () => + [ + editor.activeDocument?.imageLayers || [], + editor.sliceRenderer?.renderBlendGroups || [], + ] as [IImageLayer[], BlendGroup[]], + ([layers, blendGroups]) => { this.fragmentShader = composeLayeredShader( sliceFragmentShader, - layerCount, + layers, + blendGroups, ); this.needsUpdate = true; + + editor.sliceRenderer?.lazyRender(); }, { fireImmediately: true }, ), + reaction( () => Boolean(editor.activeDocument?.mainImageLayer?.is3DLayer), (is3D: boolean) => { diff --git a/libs/rendering/src/lib/slice-renderer/slice-renderer.ts b/libs/rendering/src/lib/slice-renderer/slice-renderer.ts index 408f2ce55..d7b5bfaf2 100644 --- a/libs/rendering/src/lib/slice-renderer/slice-renderer.ts +++ b/libs/rendering/src/lib/slice-renderer/slice-renderer.ts @@ -5,8 +5,14 @@ import { ViewType, viewTypes, } from "@visian/utils"; -import { IEditor, IImageLayer, ISliceRenderer } from "@visian/ui-shared"; -import { autorun, reaction } from "mobx"; +import { + BlendGroup, + IEditor, + IImageLayer, + ILayerGroup, + ISliceRenderer, +} from "@visian/ui-shared"; +import { autorun, computed, makeObservable, reaction } from "mobx"; import * as THREE from "three"; import { Slice } from "./slice"; @@ -123,6 +129,10 @@ export class SliceRenderer implements ISliceRenderer { ), autorun(this.updateMainBrushCursor), ); + + makeObservable(this, { + renderBlendGroups: computed, + }); } public dispose() { @@ -132,6 +142,20 @@ export class SliceRenderer implements ISliceRenderer { window.removeEventListener("resize", this.resize); } + public get renderBlendGroups(): BlendGroup[] { + return [ + ...(this.editor.activeDocument?.customBlendGroups || []), + ...(this.editor.activeDocument?.layers + .filter( + (layer) => + layer.kind === "group" && + Boolean((layer as ILayerGroup).blendGroup), + ) + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + .map((group) => (group as ILayerGroup).blendGroup!) || []), + ]; + } + private get canvas() { return this.renderer.domElement; } diff --git a/libs/rendering/src/lib/volume-renderer/utils/clipping-plane-material.ts b/libs/rendering/src/lib/volume-renderer/utils/clipping-plane-material.ts index a69275376..ce2f15107 100644 --- a/libs/rendering/src/lib/volume-renderer/utils/clipping-plane-material.ts +++ b/libs/rendering/src/lib/volume-renderer/utils/clipping-plane-material.ts @@ -1,4 +1,4 @@ -import { IEditor, IImageLayer } from "@visian/ui-shared"; +import { BlendGroup, IEditor, IImageLayer } from "@visian/ui-shared"; import { IDisposer } from "@visian/utils"; import { autorun, reaction } from "mobx"; import * as THREE from "three"; @@ -30,16 +30,24 @@ export class ClippingPlaneMaterial extends THREE.ShaderMaterial { this.disposers.push( reaction( - () => editor.volumeRenderer?.renderedImageLayerCount || 1, - (layerCount: number) => { + () => + [ + editor.activeDocument?.imageLayers || [], + editor.sliceRenderer?.renderBlendGroups || [], + ] as [IImageLayer[], BlendGroup[]], + ([layers, blendGroups]) => { this.fragmentShader = composeLayeredShader( clippingPlaneFragmentShader, - layerCount, + layers, + blendGroups, ); this.needsUpdate = true; + + editor.volumeRenderer?.lazyRender(); }, { fireImmediately: true }, ), + autorun(() => { const layers = editor.activeDocument?.imageLayers || []; diff --git a/libs/rendering/src/lib/volume-renderer/utils/gradient-computer/gradient-material.ts b/libs/rendering/src/lib/volume-renderer/utils/gradient-computer/gradient-material.ts index dd52958a1..cbed03d88 100644 --- a/libs/rendering/src/lib/volume-renderer/utils/gradient-computer/gradient-material.ts +++ b/libs/rendering/src/lib/volume-renderer/utils/gradient-computer/gradient-material.ts @@ -1,5 +1,5 @@ import { Texture3DMaterial } from "@visian/rendering"; -import { IEditor } from "@visian/ui-shared"; +import { BlendGroup, IEditor, IImageLayer } from "@visian/ui-shared"; import { IDisposer } from "@visian/utils"; import { reaction } from "mobx"; import * as THREE from "three"; @@ -42,11 +42,16 @@ export class GradientMaterial extends Texture3DMaterial { this.disposers = [ reaction( - () => editor.volumeRenderer?.renderedImageLayerCount || 1, - (layerCount: number) => { + () => + [ + editor.activeDocument?.imageLayers || [], + editor.sliceRenderer?.renderBlendGroups || [], + ] as [IImageLayer[], BlendGroup[]], + ([layers, blendGroups]) => { this.fragmentShader = composeLayeredShader( gradientFragmentShader, - layerCount, + layers, + blendGroups, ); this.needsUpdate = true; }, diff --git a/libs/rendering/src/lib/volume-renderer/utils/lao-computer/lao-material.ts b/libs/rendering/src/lib/volume-renderer/utils/lao-computer/lao-material.ts index 7758360e0..55a65be0a 100644 --- a/libs/rendering/src/lib/volume-renderer/utils/lao-computer/lao-material.ts +++ b/libs/rendering/src/lib/volume-renderer/utils/lao-computer/lao-material.ts @@ -1,5 +1,5 @@ import { Texture3DMaterial } from "@visian/rendering"; -import { IEditor } from "@visian/ui-shared"; +import { BlendGroup, IEditor, IImageLayer } from "@visian/ui-shared"; import { IDisposer } from "@visian/utils"; import { autorun, reaction } from "mobx"; import * as THREE from "three"; @@ -43,11 +43,16 @@ export class LAOMaterial extends Texture3DMaterial { this.disposers.push( reaction( - () => editor.volumeRenderer?.renderedImageLayerCount || 1, - (layerCount: number) => { + () => + [ + editor.activeDocument?.imageLayers || [], + editor.sliceRenderer?.renderBlendGroups || [], + ] as [IImageLayer[], BlendGroup[]], + ([layers, blendGroups]) => { this.fragmentShader = composeLayeredShader( laoFragmentShader, - layerCount, + layers, + blendGroups, ); this.needsUpdate = true; }, diff --git a/libs/rendering/src/lib/volume-renderer/volume-material.ts b/libs/rendering/src/lib/volume-renderer/volume-material.ts index dc2e1a4e2..fb7eb3269 100644 --- a/libs/rendering/src/lib/volume-renderer/volume-material.ts +++ b/libs/rendering/src/lib/volume-renderer/volume-material.ts @@ -1,4 +1,4 @@ -import { IEditor } from "@visian/ui-shared"; +import { BlendGroup, IEditor, IImageLayer } from "@visian/ui-shared"; import { IDisposable, IDisposer } from "@visian/utils"; import { autorun, reaction } from "mobx"; import * as THREE from "three"; @@ -51,11 +51,16 @@ export class VolumeMaterial this.disposers.push( reaction( - () => editor.volumeRenderer?.renderedImageLayerCount || 1, - (layerCount: number) => { + () => + [ + editor.activeDocument?.imageLayers || [], + editor.sliceRenderer?.renderBlendGroups || [], + ] as [IImageLayer[], BlendGroup[]], + ([layers, blendGroups]) => { this.fragmentShader = composeLayeredShader( volumeFragmentShader, - layerCount, + layers, + blendGroups, ); this.needsUpdate = true; }, diff --git a/libs/rendering/src/lib/volume-renderer/volume-renderer.ts b/libs/rendering/src/lib/volume-renderer/volume-renderer.ts index 7e71e3718..cafd8baaa 100644 --- a/libs/rendering/src/lib/volume-renderer/volume-renderer.ts +++ b/libs/rendering/src/lib/volume-renderer/volume-renderer.ts @@ -1,6 +1,6 @@ import { DragPoint, IEditor, IVolumeRenderer } from "@visian/ui-shared"; import { IDisposer, Vector, ViewType, Voxel } from "@visian/utils"; -import { autorun, computed, makeObservable, reaction } from "mobx"; +import { autorun, reaction } from "mobx"; import * as THREE from "three"; import { OrbitControls } from "three/examples/jsm/controls/OrbitControls"; import Stats from "three/examples/jsm/libs/stats.module"; @@ -261,10 +261,6 @@ export class VolumeRenderer implements IVolumeRenderer { { fireImmediately: true }, ), ); - - makeObservable(this, { - renderedImageLayerCount: computed, - }); } public dispose = () => { @@ -293,11 +289,6 @@ export class VolumeRenderer implements IVolumeRenderer { this.axesConvention.dispose(); }; - public get renderedImageLayerCount() { - // additional layer for 3d region growing preview - return (this.editor.activeDocument?.imageLayers.length || 0) + 1; - } - public resetScene(hardReset = false) { // Position the volume in a reasonable height for XR. this.volume.resetRotation(); diff --git a/libs/ui-shared/src/lib/types/editor/document.ts b/libs/ui-shared/src/lib/types/editor/document.ts index 8641392e5..01fef5f9f 100644 --- a/libs/ui-shared/src/lib/types/editor/document.ts +++ b/libs/ui-shared/src/lib/types/editor/document.ts @@ -1,5 +1,5 @@ import type * as THREE from "three"; -import { ITrackingData, TrackingLog } from "@visian/ui-shared"; +import { ITask, ITrackingData, TrackingLog } from "@visian/ui-shared"; import type { ISliceRenderer, IVolumeRenderer } from "../rendering"; import type { IHistory } from "./history"; @@ -11,7 +11,7 @@ import type { IMarkers } from "./markers"; import type { IClipboard } from "./clipboard"; import type { ErrorNotification } from "../error-notification"; import { Theme } from "../../theme"; -import { MeasurementType, PerformanceMode } from "."; +import { BlendGroup, MeasurementType, PerformanceMode } from "."; /** A VISIAN document, consisting of up to multiple editable layers. */ export interface IDocument { @@ -52,6 +52,8 @@ export interface IDocument { */ mainImageLayer?: Reference; + customBlendGroups: BlendGroup[]; + /** The document's history. */ history: IHistory; @@ -74,6 +76,8 @@ export interface IDocument { theme: Theme; performanceMode: PerformanceMode; + currentTask: ITask | undefined; + /** Indicates wether the layer menu is open. */ showLayerMenu: boolean; @@ -101,6 +105,12 @@ export interface IDocument { /** Deletes a layer from the document. */ deleteLayer(idOrLayer: string | ILayer): void; + /** Adds a blend group to the document. */ + addBlendGroup(group: BlendGroup): void; + + /** Deletes a blend group from the document. */ + deleteBlendGroup(group: BlendGroup): void; + /** Returns the first color that is not yet used to color any layer. */ getFirstUnusedColor(): string; /** Returns the color to be used for, e.g., 3D region growing preview. */ diff --git a/libs/ui-shared/src/lib/types/editor/layers.ts b/libs/ui-shared/src/lib/types/editor/layers.ts index 0aa9d2481..036b62bad 100644 --- a/libs/ui-shared/src/lib/types/editor/layers.ts +++ b/libs/ui-shared/src/lib/types/editor/layers.ts @@ -1,26 +1,27 @@ import type { Image, Vector, ViewType, Voxel } from "@visian/utils"; import type { Matrix4 } from "three"; -import { Histogram } from "./types"; +import { Histogram, Reference } from "./types"; import { MarkerConfig } from "./markers"; -/** - * The supported layer blending modes - * @see https://helpx.adobe.com/photoshop/using/blending-modes.html - */ -export type BlendMode = - | "COLOR" - | "DARKEN" - | "DIFFERENCE" - | "DIVIDE" - | "HUE" - | "LIGHTEN" - | "LUMINOSITY" - | "MULTIPLY" - | "NORMAL" - | "OVERLAY" - | "SATURATION" - | "SCREEN" - | "SUBTRACT"; +/** The supported layer blending modes. */ +export type BlendMode = "COMPARE" | "MAJORITY_VOTE"; + +export interface IBaseBlendGroup { + mode: BlendMode; + layers: Reference[]; +} + +export interface ICompareBlendGroup extends IBaseBlendGroup { + mode: "COMPARE"; + layers: [Reference, Reference]; +} + +export interface IMajorityBlendGroup extends IBaseBlendGroup { + mode: "MAJORITY_VOTE"; + majority: number; +} + +export type BlendGroup = ICompareBlendGroup | IMajorityBlendGroup; /** A generic layer. */ export interface ILayer { @@ -50,11 +51,6 @@ export interface ILayer { */ parent?: ILayer; - /** - * The blend mode used to combine this layer on top of the ones below. - * Defaults to `"NORMAL"`. - */ - blendMode?: BlendMode; /** * The color used to render layers without intrinsic color information, * provided as a theme key or CSS color string. @@ -83,7 +79,6 @@ export interface ILayer { setIsAnnotation(value?: boolean): void; - setBlendMode(blendMode?: BlendMode): void; setColor(value?: string): void; setIsVisible(value?: boolean): void; setOpacity(value?: number): void; @@ -164,9 +159,13 @@ export interface ILayerGroup extends ILayer { /** All layers in the group. */ layers: ILayer[]; + blendGroup?: BlendGroup; + /** Adds a layer to the group. */ addLayer(idOrlayer: string | ILayer): void; /** Removes a layer from the document (but keeps it in the document). */ removeLayer(idOrLayer: string | ILayer): void; + + setBlendMode(blendMode?: BlendMode): void; } diff --git a/libs/ui-shared/src/lib/types/index.ts b/libs/ui-shared/src/lib/types/index.ts index 49fae9e15..42ede455a 100644 --- a/libs/ui-shared/src/lib/types/index.ts +++ b/libs/ui-shared/src/lib/types/index.ts @@ -3,3 +3,4 @@ export * from "./error-notification"; export * from "./rendering"; export * from "./tracking"; export * from "./translation"; +export * from "./who"; diff --git a/libs/ui-shared/src/lib/types/rendering/slice-renderer.ts b/libs/ui-shared/src/lib/types/rendering/slice-renderer.ts index dfcb2b08e..9d17e7f55 100644 --- a/libs/ui-shared/src/lib/types/rendering/slice-renderer.ts +++ b/libs/ui-shared/src/lib/types/rendering/slice-renderer.ts @@ -1,4 +1,5 @@ import type { IDisposable, Pixel, ViewType } from "@visian/utils"; +import { BlendGroup } from ".."; import type { IOutline } from "./outline"; @@ -19,4 +20,6 @@ export interface ISliceRenderer extends IDisposable { getWebGLPosition(screenPosition: Pixel, viewType?: ViewType): Pixel; resetCrosshairOffset(): void; + + renderBlendGroups: BlendGroup[]; } diff --git a/libs/ui-shared/src/lib/types/rendering/volume-renderer.ts b/libs/ui-shared/src/lib/types/rendering/volume-renderer.ts index 3f611bbed..3690054db 100644 --- a/libs/ui-shared/src/lib/types/rendering/volume-renderer.ts +++ b/libs/ui-shared/src/lib/types/rendering/volume-renderer.ts @@ -15,7 +15,6 @@ export interface IVolumeRenderer extends IDisposable { scene: THREE.Scene; xr: IXRManager; volume: THREE.Mesh; - renderedImageLayerCount: number; animate(): void; diff --git a/libs/ui-shared/src/lib/types/who/annotation.ts b/libs/ui-shared/src/lib/types/who/annotation.ts new file mode 100644 index 000000000..e88a5e5e7 --- /dev/null +++ b/libs/ui-shared/src/lib/types/who/annotation.ts @@ -0,0 +1,26 @@ +import { AnnotationDataSnapshot, IAnnotationData } from "./annotationData"; +import { IUser, UserSnapshot } from "./user"; + +export enum AnnotationStatus { + Pending = "PENDING", + Completed = "COMPLETED", + Rejected = "REJECTED", +} + +export interface AnnotationSnapshot { + annotationUUID: string; + status: AnnotationStatus; + annotationDataList: AnnotationDataSnapshot[]; + annotator: UserSnapshot; + submittedAt: string; +} + +export interface IAnnotation { + annotationUUID: string; + status: AnnotationStatus; + data: IAnnotationData[]; + annotator: IUser; + submittedAt: string; + + toJSON(): AnnotationSnapshot; +} diff --git a/libs/ui-shared/src/lib/types/who/annotationData.ts b/libs/ui-shared/src/lib/types/who/annotationData.ts new file mode 100644 index 000000000..46f03b103 --- /dev/null +++ b/libs/ui-shared/src/lib/types/who/annotationData.ts @@ -0,0 +1,13 @@ +export interface AnnotationDataSnapshot { + annotationDataUUID: string; + data: string; +} + +export interface IAnnotationData { + annotationDataUUID: string; + data: string; + correspondingLayerId: string; + + setCorrespondingLayerId(layerId: string): void; + toJSON(): AnnotationDataSnapshot; +} diff --git a/libs/ui-shared/src/lib/types/who/annotationTask.ts b/libs/ui-shared/src/lib/types/who/annotationTask.ts new file mode 100644 index 000000000..6ffeacbf3 --- /dev/null +++ b/libs/ui-shared/src/lib/types/who/annotationTask.ts @@ -0,0 +1,22 @@ +export interface AnnotationTaskSnapshot { + annotationTaskUUID: string; + kind: string; + title: string; + description: string; +} + +export enum AnnotationTaskType { + Classification = "classification", + ObjectDetection = "object_detection", + SemanticSegmentation = "semantic_segmentation", + InstanceSegmentation = "instance_segmentation", +} + +export interface IAnnotationTask { + annotationTaskUUID: string; + kind: AnnotationTaskType; + title: string; + description: string; + + toJSON(): AnnotationTaskSnapshot; +} diff --git a/libs/ui-shared/src/lib/types/who/annotator.ts b/libs/ui-shared/src/lib/types/who/annotator.ts new file mode 100644 index 000000000..6cb3952e9 --- /dev/null +++ b/libs/ui-shared/src/lib/types/who/annotator.ts @@ -0,0 +1,23 @@ +export interface AnnotatorSnapshot { + annotatorUUID: string; + expertise: string; + yearsInPractice: number; + expectedSalary: number; + workCountry: string; + studyCountry: string; + selfAssessment: number; + degree: string; +} + +export interface IAnnotator { + annotatorUUID: string; + expertise: string; + yearsInPractice: number; + expectedSalary: number; + workCountry: string; + studyCountry: string; + selfAssessment: number; + degree: string; + + toJSON(): AnnotatorSnapshot; +} diff --git a/libs/ui-shared/src/lib/types/who/campaign.ts b/libs/ui-shared/src/lib/types/who/campaign.ts new file mode 100644 index 000000000..33f93fefe --- /dev/null +++ b/libs/ui-shared/src/lib/types/who/campaign.ts @@ -0,0 +1,23 @@ +import { IUser, UserSnapshot } from "./user"; + +export interface CampaignSnapshot { + campaignUUID: string; + name: string; + description: string; + status: string; + datasets: string[]; + annotators: UserSnapshot[]; + reviewers: UserSnapshot[]; +} + +export interface ICampaign { + campaignUUID: string; + name: string; + description: string; + status: string; + datasets: string[]; + annotators: IUser[]; + reviewers: IUser[]; + + toJSON(): CampaignSnapshot; +} diff --git a/libs/ui-shared/src/lib/types/who/index.ts b/libs/ui-shared/src/lib/types/who/index.ts new file mode 100644 index 000000000..25c73fa3e --- /dev/null +++ b/libs/ui-shared/src/lib/types/who/index.ts @@ -0,0 +1,9 @@ +export * from "./annotation"; +export * from "./annotationData"; +export * from "./annotationTask"; +export * from "./annotator"; +export * from "./campaign"; +export * from "./reviewer"; +export * from "./sample"; +export * from "./task"; +export * from "./user"; diff --git a/libs/ui-shared/src/lib/types/who/reviewer.ts b/libs/ui-shared/src/lib/types/who/reviewer.ts new file mode 100644 index 000000000..0a9b07464 --- /dev/null +++ b/libs/ui-shared/src/lib/types/who/reviewer.ts @@ -0,0 +1,9 @@ +export interface ReviewerSnapshot { + reviewerUUID: string; +} + +export interface IReviewer { + reviewerUUID: string; + + toJSON(): ReviewerSnapshot; +} diff --git a/libs/ui-shared/src/lib/types/who/sample.ts b/libs/ui-shared/src/lib/types/who/sample.ts new file mode 100644 index 000000000..1dbca7e0f --- /dev/null +++ b/libs/ui-shared/src/lib/types/who/sample.ts @@ -0,0 +1,13 @@ +export interface SampleSnapshot { + sampleUUID: string; + title: string; + data: string; +} + +export interface ISample { + sampleUUID: string; + title: string; + data: string; + + toJSON(): SampleSnapshot; +} diff --git a/libs/ui-shared/src/lib/types/who/task.ts b/libs/ui-shared/src/lib/types/who/task.ts new file mode 100644 index 000000000..25aa0ffc2 --- /dev/null +++ b/libs/ui-shared/src/lib/types/who/task.ts @@ -0,0 +1,35 @@ +import { AnnotationSnapshot, IAnnotation } from "./annotation"; +import { AnnotationTaskSnapshot, IAnnotationTask } from "./annotationTask"; +import { CampaignSnapshot, ICampaign } from "./campaign"; +import { ISample } from "./sample"; +import { IUser, UserSnapshot } from "./user"; + +export interface TaskSnapshot { + taskUUID: string; + kind: string; + readOnly: boolean; + annotationTasks: AnnotationTaskSnapshot[]; + annotations?: AnnotationSnapshot[]; + assignee: UserSnapshot; + campaign: CampaignSnapshot | Record; +} + +export enum TaskType { + Create = "create", + Correct = "correct", + Review = "review", +} + +export interface ITask { + taskUUID: string; + kind: TaskType; + readOnly: boolean; + annotationTasks: IAnnotationTask[]; + samples: ISample[]; + annotations: IAnnotation[]; + assignee: IUser; + campaign?: ICampaign; + + addNewAnnotation(): void; + toJSON(): TaskSnapshot; +} diff --git a/libs/ui-shared/src/lib/types/who/user.ts b/libs/ui-shared/src/lib/types/who/user.ts new file mode 100644 index 000000000..59ba140e3 --- /dev/null +++ b/libs/ui-shared/src/lib/types/who/user.ts @@ -0,0 +1,29 @@ +import { AnnotatorSnapshot, IAnnotator } from "./annotator"; +import { IReviewer, ReviewerSnapshot } from "./reviewer"; + +export type UserRole = "Annotator" | "Reviewer" | "Supervisor"; + +export interface UserSnapshot { + userUUID: string; + idpID: string; + username: string; + birthdate: string; + timezone: string; + email: string; + annotatorRole: AnnotatorSnapshot | Record; + reviewerRole: ReviewerSnapshot | Record; +} + +export interface IUser { + userUUID: string; + idpID: string; + username: string; + birthdate: string; + timezone: string; + email: string; + annotatorRole?: IAnnotator; + reviewerRole?: IReviewer; + + getRoleName(): UserRole; + toJSON(): UserSnapshot; +}