From e7f4438809c19821bea78c428018825687ceac2c Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 9 May 2022 08:59:52 +0200
Subject: [PATCH 001/122] [WIP] editable mappings

---
 .../annotation/AnnotationUploadService.scala     |  5 +----
 .../annotation/WKRemoteTracingStoreClient.scala  |  1 -
 webknossos-datastore/proto/VolumeTracing.proto   |  1 +
 .../tracings/volume/VolumeUpdateActions.scala    | 16 ++++++++++++++++
 4 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/app/models/annotation/AnnotationUploadService.scala b/app/models/annotation/AnnotationUploadService.scala
index 84f5bb44a59..ae3ec12ca8f 100644
--- a/app/models/annotation/AnnotationUploadService.scala
+++ b/app/models/annotation/AnnotationUploadService.scala
@@ -15,15 +15,12 @@ import net.liftweb.util.Helpers.tryo
 import oxalis.files.TempFileService
 import play.api.i18n.MessagesProvider
 
-import scala.concurrent.ExecutionContext
-
 case class UploadedVolumeLayer(tracing: VolumeTracing, dataZipLocation: String, name: Option[String]) {
   def getDataZipFrom(otherFiles: Map[String, File]): Option[File] =
     otherFiles.get(dataZipLocation)
 }
 
-class AnnotationUploadService @Inject()(tempFileService: TempFileService)(implicit ec: ExecutionContext)
-    extends LazyLogging {
+class AnnotationUploadService @Inject()(tempFileService: TempFileService)() extends LazyLogging {
 
   private def extractFromNml(file: File, name: String, overwritingDataSetName: Option[String], isTaskUpload: Boolean)(
       implicit m: MessagesProvider): NmlParseResult =
diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala
index debea369190..3e6ef631f87 100644
--- a/app/models/annotation/WKRemoteTracingStoreClient.scala
+++ b/app/models/annotation/WKRemoteTracingStoreClient.scala
@@ -9,7 +9,6 @@ import com.scalableminds.util.tools.Fox.bool2Fox
 import com.scalableminds.util.tools.JsonHelper.{boxFormat, optionFormat}
 import com.scalableminds.webknossos.datastore.SkeletonTracing.{SkeletonTracing, SkeletonTracings}
 import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracings}
-import com.scalableminds.webknossos.datastore.models.datasource.DataSourceLike
 import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.scalableminds.webknossos.tracingstore.tracings.TracingSelector
 import com.scalableminds.webknossos.tracingstore.tracings.volume.ResolutionRestrictions
diff --git a/webknossos-datastore/proto/VolumeTracing.proto b/webknossos-datastore/proto/VolumeTracing.proto
index e8892623878..56f95717b2c 100644
--- a/webknossos-datastore/proto/VolumeTracing.proto
+++ b/webknossos-datastore/proto/VolumeTracing.proto
@@ -36,6 +36,7 @@ message VolumeTracing {
     optional string organizationName = 14; // to identify the dataset (may differ from annotation orga)
     repeated Vec3IntProto resolutions = 15;
     repeated Segment segments = 16;
+    optional string mappingName = 17; // either a mapping present in the fallback layer, or an editable mapping on the tracingstore
 }
 
 message VolumeTracingOpt {
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala
index bb33fe0169a..16a924b6761 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala
@@ -242,6 +242,21 @@ object DeleteSegmentVolumeAction {
   implicit val jsonFormat: OFormat[DeleteSegmentVolumeAction] = Json.format[DeleteSegmentVolumeAction]
 }
 
+case class UpdateMappingNameAction(mappingName: String, actionTimestamp: Option[Long]) extends ApplyableVolumeAction {
+  override def addTimestamp(timestamp: Long): VolumeUpdateAction =
+    this.copy(actionTimestamp = Some(timestamp))
+
+  override def transformToCompact: UpdateAction[VolumeTracing] =
+    CompactVolumeUpdateAction("updateMappingName", actionTimestamp, Json.obj("mappingName" -> mappingName))
+
+  override def applyOn(tracing: VolumeTracing): VolumeTracing =
+    tracing.withMappingName(mappingName)
+}
+
+object UpdateMappingNameAction {
+  implicit val jsonFormat: OFormat[UpdateMappingNameAction] = Json.format[UpdateMappingNameAction]
+}
+
 case class CompactVolumeUpdateAction(name: String, actionTimestamp: Option[Long], value: JsObject)
     extends VolumeUpdateAction
 
@@ -275,6 +290,7 @@ object VolumeUpdateAction {
         case "createSegment"                   => (json \ "value").validate[CreateSegmentVolumeAction]
         case "updateSegment"                   => (json \ "value").validate[UpdateSegmentVolumeAction]
         case "deleteSegment"                   => (json \ "value").validate[DeleteSegmentVolumeAction]
+        case "updateMappingName"               => (json \ "value").validate[UpdateMappingNameAction]
         case unknownAction: String             => JsError(s"Invalid update action s'$unknownAction'")
       }
 

From 44234b89fa0f8072bdd480d1947fe0cabf32bb94 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 9 May 2022 11:16:10 +0200
Subject: [PATCH 002/122] query agglomerate skeleton

---
 conf/application.conf                         |  3 ++
 conf/messages                                 |  1 +
 .../TSRemoteDatastoreClient.scala             | 28 +++++++++++++++
 .../tracingstore/TracingStoreConfig.scala     |  3 ++
 .../controllers/VolumeTracingController.scala | 35 ++++++++++++++++++-
 .../EditableMappingService.scala              | 17 +++++++++
 ...alableminds.webknossos.tracingstore.routes |  1 +
 .../conf/standalone-tracingstore.conf         |  3 ++
 8 files changed, 90 insertions(+), 1 deletion(-)
 create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
 create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala

diff --git a/conf/application.conf b/conf/application.conf
index dd8005e5c40..48c22ef767a 100644
--- a/conf/application.conf
+++ b/conf/application.conf
@@ -103,6 +103,9 @@ tracingstore {
   webKnossos {
     uri = ${http.uri}
   }
+  datastore {
+    uri = ${http.uri}
+  }
   fossildb {
     address = "localhost"
     port = 7155
diff --git a/conf/messages b/conf/messages
index c9e26edcfa7..5d7b3bdf94a 100644
--- a/conf/messages
+++ b/conf/messages
@@ -276,6 +276,7 @@ annotation.reopen.failed=Failed to reopen the annotation.
 annotation.sandbox.skeletonOnly=Sandbox annotations are currently available as skeleton only.
 annotation.multiLayers.skeleton.notImplemented=This feature is not implemented for annotations with more than one skeleton layer
 annotation.multiLayers.volume.notImplemented=This feature is not implemented for annotations with more than one volume layer
+annotation.noMappingSet=No mapping is pinned for this annotation, cannot generate agglomerate skeleton.
 
 mesh.notFound=Mesh couldn’t be found
 mesh.write.failed=Failed to convert mesh info to json
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
new file mode 100644
index 00000000000..9775ab23fbd
--- /dev/null
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
@@ -0,0 +1,28 @@
+package com.scalableminds.webknossos.tracingstore
+
+import com.google.inject.Inject
+import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.datastore.rpc.RPC
+import com.typesafe.scalalogging.LazyLogging
+import play.api.inject.ApplicationLifecycle
+
+class TSRemoteDatastoreClient @Inject()(
+    rpc: RPC,
+    config: TracingStoreConfig,
+    val lifecycle: ApplicationLifecycle
+) extends LazyLogging {
+
+  private val datastoreUrl: String = config.Tracingstore.WebKnossos.uri
+
+  def getAgglomerateSkeleton(userToken: Option[String],
+                             organizationName: String,
+                             dataSetName: String,
+                             dataLayerName: String,
+                             mappingName: String,
+                             agglomerateId: Long): Fox[Array[Byte]] =
+    rpc(
+      s"$datastoreUrl/data/datasets/$organizationName/$dataSetName/layers/$dataLayerName/agglomerates/$mappingName/skeleton/$agglomerateId")
+      .addQueryStringOptional("token", userToken)
+      .getWithBytesResponse
+
+}
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreConfig.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreConfig.scala
index 05dadc402d3..af086596a4b 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreConfig.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreConfig.scala
@@ -17,6 +17,9 @@ class TracingStoreConfig @Inject()(configuration: Configuration) extends ConfigR
     object WebKnossos {
       val uri: String = get[String]("tracingstore.webKnossos.uri")
     }
+    object Datastore {
+      val uri: String = get[String]("tracingstore.datastore.uri")
+    }
     object Fossildb {
       val address: String = get[String]("tracingstore.fossildb.address")
       val port: Int = get[Int]("tracingstore.fossildb.port")
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index be11832d7b4..f536903bd8c 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -12,8 +12,13 @@ import com.scalableminds.webknossos.datastore.models.{WebKnossosDataRequest, Web
 import com.scalableminds.webknossos.datastore.services.UserAccessRequest
 import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings}
 import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService
+import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService
 import com.scalableminds.webknossos.tracingstore.tracings.volume.{ResolutionRestrictions, VolumeTracingService}
-import com.scalableminds.webknossos.tracingstore.{TracingStoreAccessTokenService, TSRemoteWebKnossosClient}
+import com.scalableminds.webknossos.tracingstore.{
+  TSRemoteDatastoreClient,
+  TSRemoteWebKnossosClient,
+  TracingStoreAccessTokenService
+}
 import play.api.i18n.Messages
 import play.api.libs.Files.TemporaryFile
 import play.api.libs.iteratee.Enumerator
@@ -25,6 +30,8 @@ import scala.concurrent.ExecutionContext
 
 class VolumeTracingController @Inject()(val tracingService: VolumeTracingService,
                                         val remoteWebKnossosClient: TSRemoteWebKnossosClient,
+                                        remoteDatastoreClient: TSRemoteDatastoreClient,
+                                        editableMappingService: EditableMappingService,
                                         val accessTokenService: TracingStoreAccessTokenService,
                                         val slackNotificationService: TSSlackNotificationService)(
     implicit val ec: ExecutionContext,
@@ -219,4 +226,30 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
     }
   }
 
+  def agglomerateSkeleton(token: Option[String], tracingId: String, agglomerateId: Long): Action[AnyContent] =
+    Action.async { implicit request =>
+      accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) {
+        for {
+          tracing <- tracingService.find(tracingId)
+          mappingName <- tracing.mappingName ?~> "annotation.agglomerateSkeleton.noMappingSet"
+          organizationName <- tracing.organizationName ?~> "annotation.agglomerateSkeleton.noOrganizationNameKnown"
+          fallbackLayer <- tracing.fallbackLayer ?~> "annotation.agglomerateSkeleton.noFallbackLayer"
+          // TODO: if editable mapping of this name exists, use that.
+          agglomerateSkeletonBytes <- remoteDatastoreClient.getAgglomerateSkeleton(token,
+                                                                                   organizationName,
+                                                                                   tracing.dataSetName,
+                                                                                   fallbackLayer,
+                                                                                   mappingName,
+                                                                                   agglomerateId)
+        } yield Ok(agglomerateSkeletonBytes)
+      }
+    }
+
+  def createEditableMapping(token: Option[String]): Action[AnyContent] =
+    Action.async { implicit request =>
+      for {
+        id <- editableMappingService.create
+      } yield Ok(id)
+    }
+
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
new file mode 100644
index 00000000000..3aa4870ef8f
--- /dev/null
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -0,0 +1,17 @@
+package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
+
+import java.util.UUID
+
+import com.google.inject.Inject
+import com.scalableminds.util.tools.Fox
+
+import scala.concurrent.ExecutionContext
+
+class EditableMappingService @Inject()() {
+  def generateId: String = UUID.randomUUID.toString
+
+  def create(implicit ec: ExecutionContext): Fox[String] =
+    Fox.successful(generateId)
+}
+
+case class EditableMappingUpdateAction()
diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
index 64436444460..068aaaee253 100644
--- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
+++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
@@ -20,6 +20,7 @@ GET         /volume/:tracingId/updateActionLog                  @com.scalablemin
 POST        /volume/:tracingId/isosurface                       @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.requestIsosurface(token: Option[String], tracingId: String)
 POST        /volume/:tracingId/importVolumeData                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.importVolumeData(token: Option[String], tracingId: String)
 GET         /volume/:tracingId/findData                         @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.findData(token: Option[String], tracingId: String)
+GET         /volume/:tracingId/agglomerateSkeleton              @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateSkeleton(token: Option[String], tracingId: String, agglomerateId: Long)
 POST        /volume/getMultiple                                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getMultiple(token: Option[String])
 POST        /volume/mergedFromIds                               @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromIds(token: Option[String], persist: Boolean)
 POST        /volume/mergedFromContents                          @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromContents(token: Option[String], persist: Boolean)
diff --git a/webknossos-tracingstore/conf/standalone-tracingstore.conf b/webknossos-tracingstore/conf/standalone-tracingstore.conf
index d29775a44d5..b0d51d327e5 100644
--- a/webknossos-tracingstore/conf/standalone-tracingstore.conf
+++ b/webknossos-tracingstore/conf/standalone-tracingstore.conf
@@ -37,6 +37,9 @@ tracingstore {
   webKnossos {
     uri = "http://localhost:9000"
   }
+  datastore {
+    uri = "http://localhost:9000"
+  }
   fossildb {
     address = "localhost"
     port = 7155

From d9f1d18f158e39b157cf88ca4719b675d6989b56 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 9 May 2022 11:43:42 +0200
Subject: [PATCH 003/122] first update actions

---
 .../controllers/VolumeTracingController.scala | 12 ++++++----
 .../tracings/TracingDataStore.scala           |  6 +++++
 .../editablemapping/EditableMapping.scala     | 11 +++++++++
 .../EditableMappingService.scala              | 23 ++++++++++++++-----
 .../EditableMappingUpdateActions.scala        | 21 +++++++++++++++++
 5 files changed, 63 insertions(+), 10 deletions(-)
 create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
 create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index f536903bd8c..3086cb91509 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -245,11 +245,15 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
       }
     }
 
-  def createEditableMapping(token: Option[String]): Action[AnyContent] =
+  def createEditableMapping(token: Option[String], tracingId: String): Action[AnyContent] =
     Action.async { implicit request =>
-      for {
-        id <- editableMappingService.create
-      } yield Ok(id)
+      accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) {
+        for {
+          tracing <- tracingService.find(tracingId)
+          tracingMappingName <- tracing.mappingName ?~> "annotation.noMappingSet"
+          id <- editableMappingService.create(baseMappingName = tracingMappingName)
+        } yield Ok(id)
+      }
     }
 
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala
index aaa7ec020d0..3d8cbad0f18 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala
@@ -27,6 +27,10 @@ class TracingDataStore @Inject()(config: TracingStoreConfig,
 
   lazy val volumeUpdates = new FossilDBClient("volumeUpdates", config, slackNotificationService)
 
+  lazy val editableMappings = new FossilDBClient("editableMappings", config, slackNotificationService)
+
+  lazy val editableMappingUpdates = new FossilDBClient("editableMappingUpdates", config, slackNotificationService)
+
   def shutdown(): Unit = {
     healthClient.shutdown()
     skeletons.shutdown()
@@ -34,6 +38,8 @@ class TracingDataStore @Inject()(config: TracingStoreConfig,
     volumes.shutdown()
     volumeData.shutdown()
     volumeUpdates.shutdown()
+    editableMappings.shutdown()
+    editableMappingUpdates.shutdown()
     ()
   }
 
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
new file mode 100644
index 00000000000..f8168d007f5
--- /dev/null
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
@@ -0,0 +1,11 @@
+package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
+
+case class EditableMapping(
+    baseMappingName: String,
+    segmentToAgglomerate: Map[Long, Long],
+    agglomerateToSegments: Map[Long, List[Long]],
+    agglomerateToEdges: Map[Long, List[(Long, Long)]],
+    agglomerateToAffinities: Map[Long, List[Long]]
+) {
+  def toBytes: Array[Byte] = ???
+}
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 3aa4870ef8f..165d16406f2 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -4,14 +4,25 @@ import java.util.UUID
 
 import com.google.inject.Inject
 import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.tracingstore.tracings.TracingDataStore
 
-import scala.concurrent.ExecutionContext
+class EditableMappingService @Inject()(
+    val tracingDataStore: TracingDataStore
+) {
 
-class EditableMappingService @Inject()() {
   def generateId: String = UUID.randomUUID.toString
 
-  def create(implicit ec: ExecutionContext): Fox[String] =
-    Fox.successful(generateId)
+  def create(baseMappingName: String): Fox[String] = {
+    val newId = generateId
+    val newEditableMapping = EditableMapping(
+      baseMappingName,
+      Map(),
+      Map(),
+      Map(),
+      Map(),
+    )
+    for {
+      _ <- tracingDataStore.editableMappings.put(newId, 0L, newEditableMapping.toBytes)
+    } yield newId
+  }
 }
-
-case class EditableMappingUpdateAction()
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
new file mode 100644
index 00000000000..b751e8b8a60
--- /dev/null
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
@@ -0,0 +1,21 @@
+package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
+
+import com.scalableminds.util.geometry.Vec3Int
+
+trait EditableMappingUpdateAction {
+  def applyOn(editableMapping: EditableMapping): EditableMapping
+}
+
+case class SplitAgglomerateUpdateAction(agglomerateId: Long, segmentId1: Long, segmentId2: Long)
+    extends EditableMappingUpdateAction {
+  def applyOn(editableMapping: EditableMapping): EditableMapping = ???
+}
+
+case class MergeAgglomerateUpdateAction(agglomerateId1: Long,
+                                        agglomerateId2: Long,
+                                        segmentId1: Long,
+                                        segmentPosition2: Vec3Int)
+    extends EditableMappingUpdateAction {
+
+  def applyOn(editableMapping: EditableMapping): EditableMapping = ???
+}

From fbf1731b447a18d9ee68c690f6c540e6135770b2 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 9 May 2022 12:30:04 +0200
Subject: [PATCH 004/122] column families

---
 docker-compose.yml | 2 +-
 fossildb/run.sh    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 674a6db8713..c426488dfdf 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -265,7 +265,7 @@ services:
     command:
       - fossildb
       - -c
-      - skeletons,skeletonUpdates,volumes,volumeData,volumeUpdates
+      - skeletons,skeletonUpdates,volumes,volumeData,volumeUpdates,editableMappings,editableMappingUpdates
     user: ${USER_UID:-fossildb}:${USER_GID:-fossildb}
 
   fossildb-persisted:
diff --git a/fossildb/run.sh b/fossildb/run.sh
index 5ebda2e57fc..c13429c2a06 100755
--- a/fossildb/run.sh
+++ b/fossildb/run.sh
@@ -14,6 +14,6 @@ if [ ! -f "$JAR" ] || [ ! "$CURRENT_VERSION" == "$VERSION" ]; then
   wget -q --show-progress -O "$JAR" "$URL"
 fi
 
-COLLECTIONS="skeletons,skeletonUpdates,volumes,volumeData,volumeUpdates"
+COLLECTIONS="skeletons,skeletonUpdates,volumes,volumeData,volumeUpdates,editableMappings,editableMappingUpdates"
 
 exec java -jar "$JAR" -c "$COLLECTIONS" -d "$FOSSILDB_HOME/data" -b "$FOSSILDB_HOME/backup"

From a6b5c9271fd030acbcdc61f80ab38e6c45ea71b1 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 9 May 2022 13:12:19 +0200
Subject: [PATCH 005/122] pipe update actions to fossildb

---
 .../controllers/VolumeTracingController.scala | 37 +++++++++++--------
 .../EditableMappingService.scala              | 16 +++++++-
 .../EditableMappingUpdateActions.scala        | 31 ++++++++++++++++
 ...alableminds.webknossos.tracingstore.routes |  1 +
 4 files changed, 68 insertions(+), 17 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index 3086cb91509..71315423047 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -12,13 +12,9 @@ import com.scalableminds.webknossos.datastore.models.{WebKnossosDataRequest, Web
 import com.scalableminds.webknossos.datastore.services.UserAccessRequest
 import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings}
 import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService
-import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService
+import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{EditableMappingService, EditableMappingUpdateAction}
 import com.scalableminds.webknossos.tracingstore.tracings.volume.{ResolutionRestrictions, VolumeTracingService}
-import com.scalableminds.webknossos.tracingstore.{
-  TSRemoteDatastoreClient,
-  TSRemoteWebKnossosClient,
-  TracingStoreAccessTokenService
-}
+import com.scalableminds.webknossos.tracingstore.{TSRemoteDatastoreClient, TSRemoteWebKnossosClient, TracingStoreAccessTokenService}
 import play.api.i18n.Messages
 import play.api.libs.Files.TemporaryFile
 import play.api.libs.iteratee.Enumerator
@@ -158,10 +154,10 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
             dataSetBoundingBox = request.body.asJson.flatMap(_.validateOpt[BoundingBox].asOpt.flatten)
             resolutionRestrictions = ResolutionRestrictions(minResolution, maxResolution)
             (newId, newTracing) <- tracingService.duplicate(tracingId,
-                                                            tracing,
-                                                            fromTask.getOrElse(false),
-                                                            dataSetBoundingBox,
-                                                            resolutionRestrictions)
+              tracing,
+              fromTask.getOrElse(false),
+              dataSetBoundingBox,
+              resolutionRestrictions)
             _ <- Fox.runIfOptionTrue(downsample)(tracingService.downsample(newId, newTracing))
           } yield Ok(Json.toJson(newId))
         }
@@ -236,11 +232,11 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
           fallbackLayer <- tracing.fallbackLayer ?~> "annotation.agglomerateSkeleton.noFallbackLayer"
           // TODO: if editable mapping of this name exists, use that.
           agglomerateSkeletonBytes <- remoteDatastoreClient.getAgglomerateSkeleton(token,
-                                                                                   organizationName,
-                                                                                   tracing.dataSetName,
-                                                                                   fallbackLayer,
-                                                                                   mappingName,
-                                                                                   agglomerateId)
+            organizationName,
+            tracing.dataSetName,
+            fallbackLayer,
+            mappingName,
+            agglomerateId)
         } yield Ok(agglomerateSkeletonBytes)
       }
     }
@@ -256,4 +252,15 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
       }
     }
 
+  def updateEditableMapping(token: Option[String], tracingId: String, version: Long): Action[EditableMappingUpdateAction] =
+    Action.async(validateJson[EditableMappingUpdateAction]) { implicit request =>
+      accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) {
+        for {
+          tracing <- tracingService.find(tracingId)
+          mappingName <- tracing.mappingName.toFox
+          _ <- editableMappingService.assertExists(mappingName)
+          _ <- editableMappingService.update(mappingName, request.body, version)
+        } yield Ok
+      }
+    }
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 165d16406f2..52c6baa7dd0 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -4,11 +4,11 @@ import java.util.UUID
 
 import com.google.inject.Inject
 import com.scalableminds.util.tools.Fox
-import com.scalableminds.webknossos.tracingstore.tracings.TracingDataStore
+import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore}
 
 class EditableMappingService @Inject()(
     val tracingDataStore: TracingDataStore
-) {
+) extends KeyValueStoreImplicits {
 
   def generateId: String = UUID.randomUUID.toString
 
@@ -25,4 +25,16 @@ class EditableMappingService @Inject()(
       _ <- tracingDataStore.editableMappings.put(newId, 0L, newEditableMapping.toBytes)
     } yield newId
   }
+
+  def assertExists(editableMappingId: String): Fox[Unit] =
+    for {
+      _ <- tracingDataStore.editableMappings.getVersion(editableMappingId, mayBeEmpty = Some(true), version = Some(0L))
+    } yield ()
+
+
+  def update(editableMappingId: String, updateAction: EditableMappingUpdateAction, version: Long): Fox[Unit] = {
+    for {
+      _ <- tracingDataStore.editableMappingUpdates.put(editableMappingId, version, updateAction)
+    } yield ()
+  }
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
index b751e8b8a60..96dd824f8a7 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
@@ -1,6 +1,8 @@
 package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
 
 import com.scalableminds.util.geometry.Vec3Int
+import play.api.libs.json.Format.GenericFormat
+import play.api.libs.json.{Format, JsError, JsResult, JsValue, Json, OFormat}
 
 trait EditableMappingUpdateAction {
   def applyOn(editableMapping: EditableMapping): EditableMapping
@@ -11,6 +13,10 @@ case class SplitAgglomerateUpdateAction(agglomerateId: Long, segmentId1: Long, s
   def applyOn(editableMapping: EditableMapping): EditableMapping = ???
 }
 
+object SplitAgglomerateUpdateAction {
+  implicit val jsonFormat: OFormat[SplitAgglomerateUpdateAction] = Json.format[SplitAgglomerateUpdateAction]
+}
+
 case class MergeAgglomerateUpdateAction(agglomerateId1: Long,
                                         agglomerateId2: Long,
                                         segmentId1: Long,
@@ -19,3 +25,28 @@ case class MergeAgglomerateUpdateAction(agglomerateId1: Long,
 
   def applyOn(editableMapping: EditableMapping): EditableMapping = ???
 }
+
+object MergeAgglomerateUpdateAction {
+  implicit val jsonFormat: OFormat[MergeAgglomerateUpdateAction] = Json.format[MergeAgglomerateUpdateAction]
+}
+
+
+object EditableMappingUpdateAction {
+
+  implicit object editableMappingUpdateActionFormat extends Format[EditableMappingUpdateAction] {
+    override def reads(json: JsValue): JsResult[EditableMappingUpdateAction] =
+      (json \ "name").validate[String].flatMap {
+        case "mergeAgglomerate" => (json \ "value").validate[MergeAgglomerateUpdateAction]
+        case "splitAgglomerate" => (json \ "value").validate[SplitAgglomerateUpdateAction]
+        case unknownAction: String => JsError(s"Invalid update action s'$unknownAction'")
+      }
+
+    override def writes(o: EditableMappingUpdateAction): JsValue = o match {
+      case s: SplitAgglomerateUpdateAction =>
+        Json.obj("name" -> "splitAgglomerate", "value" -> Json.toJson(s)(SplitAgglomerateUpdateAction.jsonFormat))
+      case s: MergeAgglomerateUpdateAction =>
+        Json.obj("name" -> "mergeAgglomerate", "value" -> Json.toJson(s)(MergeAgglomerateUpdateAction.jsonFormat))
+    }
+  }
+
+}
diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
index 068aaaee253..53ba108a64b 100644
--- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
+++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
@@ -21,6 +21,7 @@ POST        /volume/:tracingId/isosurface                       @com.scalablemin
 POST        /volume/:tracingId/importVolumeData                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.importVolumeData(token: Option[String], tracingId: String)
 GET         /volume/:tracingId/findData                         @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.findData(token: Option[String], tracingId: String)
 GET         /volume/:tracingId/agglomerateSkeleton              @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateSkeleton(token: Option[String], tracingId: String, agglomerateId: Long)
+POST        /volume/:tracingId/updateEditableMapping            @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.updateEditableMapping(token: Option[String], tracingId: String, version: Long)
 POST        /volume/getMultiple                                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getMultiple(token: Option[String])
 POST        /volume/mergedFromIds                               @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromIds(token: Option[String], persist: Boolean)
 POST        /volume/mergedFromContents                          @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromContents(token: Option[String], persist: Boolean)

From 673eca264755ac79f107ed581860cf2c7814c074 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 9 May 2022 13:53:58 +0200
Subject: [PATCH 006/122] logic for applying updates on load

---
 .../controllers/VolumeTracingController.scala | 39 +++++----
 .../editablemapping/EditableMapping.scala     |  4 +
 .../EditableMappingService.scala              | 83 +++++++++++++++++--
 .../EditableMappingUpdateActions.scala        |  5 +-
 .../skeleton/SkeletonTracingService.scala     |  8 +-
 5 files changed, 108 insertions(+), 31 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index 71315423047..20372df6d4b 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -12,9 +12,16 @@ import com.scalableminds.webknossos.datastore.models.{WebKnossosDataRequest, Web
 import com.scalableminds.webknossos.datastore.services.UserAccessRequest
 import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings}
 import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService
-import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{EditableMappingService, EditableMappingUpdateAction}
+import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{
+  EditableMappingService,
+  EditableMappingUpdateAction
+}
 import com.scalableminds.webknossos.tracingstore.tracings.volume.{ResolutionRestrictions, VolumeTracingService}
-import com.scalableminds.webknossos.tracingstore.{TSRemoteDatastoreClient, TSRemoteWebKnossosClient, TracingStoreAccessTokenService}
+import com.scalableminds.webknossos.tracingstore.{
+  TSRemoteDatastoreClient,
+  TSRemoteWebKnossosClient,
+  TracingStoreAccessTokenService
+}
 import play.api.i18n.Messages
 import play.api.libs.Files.TemporaryFile
 import play.api.libs.iteratee.Enumerator
@@ -154,10 +161,10 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
             dataSetBoundingBox = request.body.asJson.flatMap(_.validateOpt[BoundingBox].asOpt.flatten)
             resolutionRestrictions = ResolutionRestrictions(minResolution, maxResolution)
             (newId, newTracing) <- tracingService.duplicate(tracingId,
-              tracing,
-              fromTask.getOrElse(false),
-              dataSetBoundingBox,
-              resolutionRestrictions)
+                                                            tracing,
+                                                            fromTask.getOrElse(false),
+                                                            dataSetBoundingBox,
+                                                            resolutionRestrictions)
             _ <- Fox.runIfOptionTrue(downsample)(tracingService.downsample(newId, newTracing))
           } yield Ok(Json.toJson(newId))
         }
@@ -230,13 +237,13 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
           mappingName <- tracing.mappingName ?~> "annotation.agglomerateSkeleton.noMappingSet"
           organizationName <- tracing.organizationName ?~> "annotation.agglomerateSkeleton.noOrganizationNameKnown"
           fallbackLayer <- tracing.fallbackLayer ?~> "annotation.agglomerateSkeleton.noFallbackLayer"
-          // TODO: if editable mapping of this name exists, use that.
+          isEditableMapping <- editableMappingService.exists(mappingName)
           agglomerateSkeletonBytes <- remoteDatastoreClient.getAgglomerateSkeleton(token,
-            organizationName,
-            tracing.dataSetName,
-            fallbackLayer,
-            mappingName,
-            agglomerateId)
+                                                                                   organizationName,
+                                                                                   tracing.dataSetName,
+                                                                                   fallbackLayer,
+                                                                                   mappingName,
+                                                                                   agglomerateId)
         } yield Ok(agglomerateSkeletonBytes)
       }
     }
@@ -252,13 +259,17 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
       }
     }
 
-  def updateEditableMapping(token: Option[String], tracingId: String, version: Long): Action[EditableMappingUpdateAction] =
+  def updateEditableMapping(token: Option[String],
+                            tracingId: String,
+                            version: Long): Action[EditableMappingUpdateAction] =
     Action.async(validateJson[EditableMappingUpdateAction]) { implicit request =>
       accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) {
         for {
           tracing <- tracingService.find(tracingId)
           mappingName <- tracing.mappingName.toFox
-          _ <- editableMappingService.assertExists(mappingName)
+          _ <- Fox.assertTrue(editableMappingService.exists(mappingName))
+          currentVersion <- editableMappingService.currentVersion(mappingName)
+          _ <- bool2Fox(version == currentVersion + 1) ?~> "version mismatch"
           _ <- editableMappingService.update(mappingName, request.body, version)
         } yield Ok
       }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
index f8168d007f5..92faf46c201 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
@@ -9,3 +9,7 @@ case class EditableMapping(
 ) {
   def toBytes: Array[Byte] = ???
 }
+
+object EditableMapping {
+  def fromBytes(bytes: Array[Byte]): EditableMapping = ???
+}
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 52c6baa7dd0..fbb30ceca1b 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -4,13 +4,23 @@ import java.util.UUID
 
 import com.google.inject.Inject
 import com.scalableminds.util.tools.Fox
-import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore}
+import com.scalableminds.webknossos.tracingstore.tracings.{
+  KeyValueStoreImplicits,
+  TracingDataStore,
+  VersionedKeyValuePair
+}
+
+import scala.concurrent.ExecutionContext
 
 class EditableMappingService @Inject()(
     val tracingDataStore: TracingDataStore
-) extends KeyValueStoreImplicits {
+)(implicit ec: ExecutionContext)
+    extends KeyValueStoreImplicits {
+
+  private def generateId: String = UUID.randomUUID.toString
 
-  def generateId: String = UUID.randomUUID.toString
+  def currentVersion(editableMappingId: String): Fox[Long] =
+    tracingDataStore.editableMappings.getVersion(editableMappingId, mayBeEmpty = Some(true), emptyFallback = Some(0L))
 
   def create(baseMappingName: String): Fox[String] = {
     val newId = generateId
@@ -26,15 +36,72 @@ class EditableMappingService @Inject()(
     } yield newId
   }
 
-  def assertExists(editableMappingId: String): Fox[Unit] =
+  def exists(editableMappingId: String): Fox[Boolean] =
     for {
-      _ <- tracingDataStore.editableMappings.getVersion(editableMappingId, mayBeEmpty = Some(true), version = Some(0L))
-    } yield ()
+      versionOrMinusOne: Long <- tracingDataStore.editableMappings.getVersion(editableMappingId,
+                                                                              mayBeEmpty = Some(true),
+                                                                              version = Some(0L),
+                                                                              emptyFallback = Some(-1L))
+    } yield versionOrMinusOne >= 0
+
+  def get(editableMappingId: String, version: Option[Long] = None): Fox[EditableMapping] =
+    for {
+      closestMaterializedVersion: VersionedKeyValuePair[Array[Byte]] <- tracingDataStore.editableMappings
+        .get(editableMappingId, version)
+      materialized <- applyPendingUpdates(editableMappingId,
+                                          EditableMapping.fromBytes(closestMaterializedVersion.value),
+                                          closestMaterializedVersion.version,
+                                          version)
+    } yield materialized
 
+  private def findDesiredOrNewestPossibleVersion(existingMaterializedVersion: Long,
+                                                 editableMappingId: String,
+                                                 desiredVersion: Option[Long]): Fox[Long] =
+    /*
+     * Determines the newest saved version from the updates column.
+     * if there are no updates at all, assume mapping is brand new,
+     * hence the emptyFallbck tracing.version)
+     */
+    for {
+      newestUpdateVersion <- tracingDataStore.editableMappingUpdates.getVersion(editableMappingId,
+                                                                                mayBeEmpty = Some(true),
+                                                                                emptyFallback =
+                                                                                  Some(existingMaterializedVersion))
+    } yield {
+      desiredVersion match {
+        case None              => newestUpdateVersion
+        case Some(desiredSome) => math.min(desiredSome, newestUpdateVersion)
+      }
+    }
+
+  private def applyPendingUpdates(editableMappingId: String,
+                                  existingEditableMapping: EditableMapping,
+                                  existingVersion: Long,
+                                  requestedVersion: Option[Long]): Fox[EditableMapping] =
+    for {
+      desiredVersion <- findDesiredOrNewestPossibleVersion(existingVersion, editableMappingId, requestedVersion)
+      pendingUpdates <- findPendingUpdates(editableMappingId, existingVersion, desiredVersion)
+      appliedEditableMapping <- applyUpdates(existingEditableMapping, existingVersion, desiredVersion, pendingUpdates)
+    } yield appliedEditableMapping
 
-  def update(editableMappingId: String, updateAction: EditableMappingUpdateAction, version: Long): Fox[Unit] = {
+  private def applyUpdates(existingEditableMapping: EditableMapping,
+                           existingVersion: Long,
+                           desiredVersion: Long,
+                           pendingUpdates: List[EditableMappingUpdateAction]): Fox[EditableMapping] = ???
+
+  private def findPendingUpdates(editableMappingId: String, existingVersion: Long, desiredVersion: Long)(
+      implicit ec: ExecutionContext): Fox[List[EditableMappingUpdateAction]] =
+    if (desiredVersion == existingVersion) Fox.successful(List())
+    else {
+      tracingDataStore.editableMappingUpdates.getMultipleVersions(
+        editableMappingId,
+        Some(desiredVersion),
+        Some(existingVersion + 1)
+      )(fromJson[EditableMappingUpdateAction])
+    }
+
+  def update(editableMappingId: String, updateAction: EditableMappingUpdateAction, version: Long): Fox[Unit] =
     for {
       _ <- tracingDataStore.editableMappingUpdates.put(editableMappingId, version, updateAction)
     } yield ()
-  }
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
index 96dd824f8a7..ff95361e5de 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
@@ -30,14 +30,13 @@ object MergeAgglomerateUpdateAction {
   implicit val jsonFormat: OFormat[MergeAgglomerateUpdateAction] = Json.format[MergeAgglomerateUpdateAction]
 }
 
-
 object EditableMappingUpdateAction {
 
   implicit object editableMappingUpdateActionFormat extends Format[EditableMappingUpdateAction] {
     override def reads(json: JsValue): JsResult[EditableMappingUpdateAction] =
       (json \ "name").validate[String].flatMap {
-        case "mergeAgglomerate" => (json \ "value").validate[MergeAgglomerateUpdateAction]
-        case "splitAgglomerate" => (json \ "value").validate[SplitAgglomerateUpdateAction]
+        case "mergeAgglomerate"    => (json \ "value").validate[MergeAgglomerateUpdateAction]
+        case "splitAgglomerate"    => (json \ "value").validate[SplitAgglomerateUpdateAction]
         case unknownAction: String => JsError(s"Invalid update action s'$unknownAction'")
       }
 
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala
index 036af9c7d38..dd3e8a719e5 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala
@@ -64,9 +64,7 @@ class SkeletonTracingService @Inject()(
           pendingUpdates <- findPendingUpdates(tracingId, existingVersion, newVersion)
           updatedTracing <- update(tracing, tracingId, pendingUpdates, newVersion)
           _ <- save(updatedTracing, Some(tracingId), newVersion)
-        } yield {
-          updatedTracing
-        }
+        } yield updatedTracing
       } else {
         Full(tracing)
       }
@@ -102,9 +100,7 @@ class SkeletonTracingService @Inject()(
           tracingId,
           Some(desiredVersion),
           Some(existingVersion + 1))(fromJson[List[SkeletonUpdateAction]])
-      } yield {
-        updateActionGroups.reverse.flatten
-      }
+      } yield updateActionGroups.reverse.flatten
     }
 
   private def update(tracing: SkeletonTracing,

From 13a9ceedf5d80f7e3675d313c6ffaa6b05900b77 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 10 May 2022 10:47:43 +0200
Subject: [PATCH 007/122] some logic for gathering relevant ampping

---
 .../TSRemoteDatastoreClient.scala             |  5 ++
 .../controllers/VolumeTracingController.scala |  5 +-
 .../EditableMappingService.scala              | 59 ++++++++++++++++++-
 3 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
index 9775ab23fbd..81b7992f578 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
@@ -2,7 +2,9 @@ package com.scalableminds.webknossos.tracingstore
 
 import com.google.inject.Inject
 import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.datastore.models.DataRequestCollection.DataRequestCollection
 import com.scalableminds.webknossos.datastore.rpc.RPC
+import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.RemoteFallbackLayer
 import com.typesafe.scalalogging.LazyLogging
 import play.api.inject.ApplicationLifecycle
 
@@ -25,4 +27,7 @@ class TSRemoteDatastoreClient @Inject()(
       .addQueryStringOptional("token", userToken)
       .getWithBytesResponse
 
+  def getData(remoteFallbackLayer: RemoteFallbackLayer,
+              dataRequests: DataRequestCollection): Fox[(Array[Byte], List[Int])] = ???
+
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index 20372df6d4b..70a1fde22c0 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -135,7 +135,10 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
         accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) {
           for {
             tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound")
-            (data, indices) <- tracingService.data(tracingId, tracing, request.body)
+            hasEditableMapping <- Fox.runOptional(tracing.mappingName)(editableMappingService.exists)
+            (data, indices) <- if (hasEditableMapping.getOrElse(false))
+              tracingService.data(tracingId, tracing, request.body)
+            else editableMappingService.volumeData(tracingId, tracing, request.body)
           } yield Ok(data).withHeaders(getMissingBucketsHeaders(indices): _*)
         }
       }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index fbb30ceca1b..e6077e8b826 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -4,6 +4,11 @@ import java.util.UUID
 
 import com.google.inject.Inject
 import com.scalableminds.util.tools.Fox
+import com.scalableminds.util.tools.Fox.option2Fox
+import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing
+import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing.ElementClass
+import com.scalableminds.webknossos.datastore.models.DataRequestCollection.DataRequestCollection
+import com.scalableminds.webknossos.tracingstore.TSRemoteDatastoreClient
 import com.scalableminds.webknossos.tracingstore.tracings.{
   KeyValueStoreImplicits,
   TracingDataStore,
@@ -13,7 +18,8 @@ import com.scalableminds.webknossos.tracingstore.tracings.{
 import scala.concurrent.ExecutionContext
 
 class EditableMappingService @Inject()(
-    val tracingDataStore: TracingDataStore
+    val tracingDataStore: TracingDataStore,
+    remoteDatastoreClient: TSRemoteDatastoreClient
 )(implicit ec: ExecutionContext)
     extends KeyValueStoreImplicits {
 
@@ -104,4 +110,55 @@ class EditableMappingService @Inject()(
     for {
       _ <- tracingDataStore.editableMappingUpdates.put(editableMappingId, version, updateAction)
     } yield ()
+
+  def volumeData(tracingId: String,
+                 tracing: VolumeTracing,
+                 dataRequests: DataRequestCollection): Fox[(Array[Byte], List[Int])] =
+    for {
+      editableMappingId <- tracing.mappingName.toFox
+      editableMapping <- get(editableMappingId)
+      remoteFallbackLayer <- remoteFallbackLayer(tracing)
+      (unmappedData, indices) <- getUnmappedDataFromDatastore(remoteFallbackLayer, dataRequests)
+      segmentIds = collectSegmentIds(unmappedData, indices, tracing.elementClass)
+      relevantMapping <- generateCombinedMappingSubset(segmentIds, editableMapping, remoteFallbackLayer)
+      mappedData <- mapData(unmappedData, indices, relevantMapping)
+    } yield (mappedData, indices)
+
+  private def generateCombinedMappingSubset(segmentIds: Set[Long],
+                                            editableMapping: EditableMapping,
+                                            remoteFallbackLayer: RemoteFallbackLayer): Fox[Map[Long, Long]] = {
+    val segmentIdsInEditableMapping: Set[Long] = segmentIds.intersect(editableMapping.segmentToAgglomerate.keySet)
+    val segmentIdsInBaseMapping: Set[Long] = segmentIds.diff(segmentIdsInEditableMapping)
+    val editableMappingSubset =
+      editableMapping.segmentToAgglomerate.filterKeys(key => segmentIdsInEditableMapping.contains(key))
+    for {
+      baseMappingSubset <- getBaseSegmentToAgglomeate(editableMapping.baseMappingName,
+                                                      segmentIdsInBaseMapping,
+                                                      remoteFallbackLayer)
+    } yield editableMappingSubset ++ baseMappingSubset
+  }
+
+  private def getBaseSegmentToAgglomeate(mappingName: String,
+                                         segmentIds: Set[Long],
+                                         remoteFallbackLayer: RemoteFallbackLayer): Fox[Map[Long, Long]] = ???
+
+  private def getUnmappedDataFromDatastore(remoteFallbackLayer: RemoteFallbackLayer,
+                                           dataRequests: DataRequestCollection): Fox[(Array[Byte], List[Int])] =
+    for {
+      (data, indices) <- remoteDatastoreClient.getData(remoteFallbackLayer, dataRequests)
+    } yield (data, indices)
+
+  private def collectSegmentIds(data: Array[Byte], indices: List[Int], elementClass: ElementClass): Set[Long] = ???
+
+  private def remoteFallbackLayer(tracing: VolumeTracing): Fox[RemoteFallbackLayer] =
+    for {
+      layerName <- tracing.fallbackLayer.toFox
+      organizationName <- tracing.organizationName.toFox
+    } yield RemoteFallbackLayer(organizationName, tracing.dataSetName, layerName)
+
+  private def mapData(unmappedData: Array[Byte],
+                      indices: List[Int],
+                      relevantMapping: Map[Long, Long]): Fox[Array[Byte]] = ???
 }
+
+case class RemoteFallbackLayer(organizationName: String, dataSetName: String, layerName: String)

From 8b85df7290484830ca70866afa256e2f697a3173 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 10 May 2022 11:45:54 +0200
Subject: [PATCH 008/122] some logic for generating skeleton

---
 .../TSRemoteDatastoreClient.scala             | 13 ++--
 .../controllers/VolumeTracingController.scala | 17 ++--
 .../editablemapping/EditableMapping.scala     |  3 +
 .../EditableMappingService.scala              | 78 ++++++++++++++++---
 .../editablemapping/RemoteFallbackLayer.scala |  3 +
 5 files changed, 90 insertions(+), 24 deletions(-)
 create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/RemoteFallbackLayer.scala

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
index 81b7992f578..046a49cf117 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
@@ -17,17 +17,20 @@ class TSRemoteDatastoreClient @Inject()(
   private val datastoreUrl: String = config.Tracingstore.WebKnossos.uri
 
   def getAgglomerateSkeleton(userToken: Option[String],
-                             organizationName: String,
-                             dataSetName: String,
-                             dataLayerName: String,
+                             remoteFallbackLayer: RemoteFallbackLayer,
                              mappingName: String,
                              agglomerateId: Long): Fox[Array[Byte]] =
-    rpc(
-      s"$datastoreUrl/data/datasets/$organizationName/$dataSetName/layers/$dataLayerName/agglomerates/$mappingName/skeleton/$agglomerateId")
+    rpc(s"${remoteLayerUri(remoteFallbackLayer)}/agglomerates/$mappingName/skeleton/$agglomerateId")
       .addQueryStringOptional("token", userToken)
       .getWithBytesResponse
 
   def getData(remoteFallbackLayer: RemoteFallbackLayer,
               dataRequests: DataRequestCollection): Fox[(Array[Byte], List[Int])] = ???
 
+  def getAgglomerateIdsForSegmentIds(remoteFallbackLayer: RemoteFallbackLayer,
+                                     mappingName: String,
+                                     segmentIdsOrdered: List[Long]): Fox[List[Long]] = ???
+
+  private def remoteLayerUri(remoteLayer: RemoteFallbackLayer): String =
+    s"$datastoreUrl/data/datasets/${remoteLayer.organizationName}/${remoteLayer.dataSetName}/layers/${remoteLayer.layerName}"
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index 70a1fde22c0..44d5ab8c052 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -138,7 +138,7 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
             hasEditableMapping <- Fox.runOptional(tracing.mappingName)(editableMappingService.exists)
             (data, indices) <- if (hasEditableMapping.getOrElse(false))
               tracingService.data(tracingId, tracing, request.body)
-            else editableMappingService.volumeData(tracingId, tracing, request.body)
+            else editableMappingService.volumeData(tracing, request.body)
           } yield Ok(data).withHeaders(getMissingBucketsHeaders(indices): _*)
         }
       }
@@ -238,15 +238,14 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
         for {
           tracing <- tracingService.find(tracingId)
           mappingName <- tracing.mappingName ?~> "annotation.agglomerateSkeleton.noMappingSet"
-          organizationName <- tracing.organizationName ?~> "annotation.agglomerateSkeleton.noOrganizationNameKnown"
-          fallbackLayer <- tracing.fallbackLayer ?~> "annotation.agglomerateSkeleton.noFallbackLayer"
+          remoteFallbackLayer <- editableMappingService.remoteFallbackLayer(tracing)
           isEditableMapping <- editableMappingService.exists(mappingName)
-          agglomerateSkeletonBytes <- remoteDatastoreClient.getAgglomerateSkeleton(token,
-                                                                                   organizationName,
-                                                                                   tracing.dataSetName,
-                                                                                   fallbackLayer,
-                                                                                   mappingName,
-                                                                                   agglomerateId)
+          agglomerateSkeletonBytes <- if (isEditableMapping)
+            editableMappingService.getAgglomerateSkeletonWithFallback(token,
+                                                                      mappingName,
+                                                                      remoteFallbackLayer,
+                                                                      agglomerateId)
+          else remoteDatastoreClient.getAgglomerateSkeleton(token, remoteFallbackLayer, mappingName, agglomerateId)
         } yield Ok(agglomerateSkeletonBytes)
       }
     }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
index 92faf46c201..b0f796681a2 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
@@ -1,10 +1,13 @@
 package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
 
+import com.scalableminds.util.geometry.Vec3Int
+
 case class EditableMapping(
     baseMappingName: String,
     segmentToAgglomerate: Map[Long, Long],
     agglomerateToSegments: Map[Long, List[Long]],
     agglomerateToEdges: Map[Long, List[(Long, Long)]],
+    agglomerateToPositions: Map[Long, List[Vec3Int]],
     agglomerateToAffinities: Map[Long, List[Long]]
 ) {
   def toBytes: Array[Byte] = ???
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index e6077e8b826..904da3bdd50 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -5,8 +5,10 @@ import java.util.UUID
 import com.google.inject.Inject
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.util.tools.Fox.option2Fox
+import com.scalableminds.webknossos.datastore.SkeletonTracing.{Edge, Tree}
 import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing
 import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing.ElementClass
+import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, ProtoGeometryImplicits, SkeletonTracingDefaults}
 import com.scalableminds.webknossos.datastore.models.DataRequestCollection.DataRequestCollection
 import com.scalableminds.webknossos.tracingstore.TSRemoteDatastoreClient
 import com.scalableminds.webknossos.tracingstore.tracings.{
@@ -21,7 +23,8 @@ class EditableMappingService @Inject()(
     val tracingDataStore: TracingDataStore,
     remoteDatastoreClient: TSRemoteDatastoreClient
 )(implicit ec: ExecutionContext)
-    extends KeyValueStoreImplicits {
+    extends KeyValueStoreImplicits
+    with ProtoGeometryImplicits {
 
   private def generateId: String = UUID.randomUUID.toString
 
@@ -36,6 +39,7 @@ class EditableMappingService @Inject()(
       Map(),
       Map(),
       Map(),
+      Map()
     )
     for {
       _ <- tracingDataStore.editableMappings.put(newId, 0L, newEditableMapping.toBytes)
@@ -111,9 +115,7 @@ class EditableMappingService @Inject()(
       _ <- tracingDataStore.editableMappingUpdates.put(editableMappingId, version, updateAction)
     } yield ()
 
-  def volumeData(tracingId: String,
-                 tracing: VolumeTracing,
-                 dataRequests: DataRequestCollection): Fox[(Array[Byte], List[Int])] =
+  def volumeData(tracing: VolumeTracing, dataRequests: DataRequestCollection): Fox[(Array[Byte], List[Int])] =
     for {
       editableMappingId <- tracing.mappingName.toFox
       editableMapping <- get(editableMappingId)
@@ -121,7 +123,7 @@ class EditableMappingService @Inject()(
       (unmappedData, indices) <- getUnmappedDataFromDatastore(remoteFallbackLayer, dataRequests)
       segmentIds = collectSegmentIds(unmappedData, indices, tracing.elementClass)
       relevantMapping <- generateCombinedMappingSubset(segmentIds, editableMapping, remoteFallbackLayer)
-      mappedData <- mapData(unmappedData, indices, relevantMapping)
+      mappedData <- mapData(unmappedData, indices, relevantMapping, tracing.elementClass)
     } yield (mappedData, indices)
 
   private def generateCombinedMappingSubset(segmentIds: Set[Long],
@@ -138,9 +140,66 @@ class EditableMappingService @Inject()(
     } yield editableMappingSubset ++ baseMappingSubset
   }
 
+  def getAgglomerateSkeletonWithFallback(userToken: Option[String],
+                                         editableMappingId: String,
+                                         remoteFallbackLayer: RemoteFallbackLayer,
+                                         agglomerateId: Long): Fox[Array[Byte]] =
+    for {
+      editableMapping <- get(editableMappingId)
+      agglomerateIdIsPresent = editableMapping.agglomerateToSegments.contains(agglomerateId)
+      skeletonBytes <- if (agglomerateIdIsPresent)
+        getAgglomerateSkeleton(editableMappingId, editableMapping, remoteFallbackLayer, agglomerateId)
+      else
+        remoteDatastoreClient.getAgglomerateSkeleton(userToken,
+                                                     remoteFallbackLayer,
+                                                     editableMapping.baseMappingName,
+                                                     agglomerateId)
+    } yield skeletonBytes
+
+  private def getAgglomerateSkeleton(editableMappingId: String,
+                                     editableMapping: EditableMapping,
+                                     remoteFallbackLayer: RemoteFallbackLayer,
+                                     agglomerateId: Long): Fox[Array[Byte]] =
+    for {
+      positions <- editableMapping.agglomerateToPositions.get(agglomerateId)
+      nodes = positions.zipWithIndex.map {
+        case (pos, idx) =>
+          NodeDefaults.createInstance.copy(
+            id = idx,
+            position = pos
+          )
+      }
+      edges <- editableMapping.agglomerateToEdges.get(agglomerateId)
+      skeletonEdges = edges.map { e =>
+        Edge(source = e._1.toInt, target = e._2.toInt)
+      }
+
+      trees = Seq(
+        Tree(
+          treeId = agglomerateId.toInt,
+          createdTimestamp = System.currentTimeMillis(),
+          nodes = nodes,
+          edges = skeletonEdges,
+          name = s"agglomerate $agglomerateId ($editableMappingId)"
+        ))
+
+      skeleton = SkeletonTracingDefaults.createInstance.copy(
+        dataSetName = remoteFallbackLayer.dataSetName,
+        trees = trees,
+        organizationName = Some(remoteFallbackLayer.organizationName)
+      )
+    } yield skeleton.toByteArray
+
   private def getBaseSegmentToAgglomeate(mappingName: String,
                                          segmentIds: Set[Long],
-                                         remoteFallbackLayer: RemoteFallbackLayer): Fox[Map[Long, Long]] = ???
+                                         remoteFallbackLayer: RemoteFallbackLayer): Fox[Map[Long, Long]] = {
+    val segmentIdsOrdered = segmentIds.toList
+    for {
+      agglomerateIdsOrdered <- remoteDatastoreClient.getAgglomerateIdsForSegmentIds(remoteFallbackLayer,
+                                                                                    mappingName,
+                                                                                    segmentIdsOrdered)
+    } yield segmentIdsOrdered.zip(agglomerateIdsOrdered).toMap
+  }
 
   private def getUnmappedDataFromDatastore(remoteFallbackLayer: RemoteFallbackLayer,
                                            dataRequests: DataRequestCollection): Fox[(Array[Byte], List[Int])] =
@@ -150,7 +209,7 @@ class EditableMappingService @Inject()(
 
   private def collectSegmentIds(data: Array[Byte], indices: List[Int], elementClass: ElementClass): Set[Long] = ???
 
-  private def remoteFallbackLayer(tracing: VolumeTracing): Fox[RemoteFallbackLayer] =
+  def remoteFallbackLayer(tracing: VolumeTracing): Fox[RemoteFallbackLayer] =
     for {
       layerName <- tracing.fallbackLayer.toFox
       organizationName <- tracing.organizationName.toFox
@@ -158,7 +217,6 @@ class EditableMappingService @Inject()(
 
   private def mapData(unmappedData: Array[Byte],
                       indices: List[Int],
-                      relevantMapping: Map[Long, Long]): Fox[Array[Byte]] = ???
+                      relevantMapping: Map[Long, Long],
+                      elementClass: ElementClass): Fox[Array[Byte]] = ???
 }
-
-case class RemoteFallbackLayer(organizationName: String, dataSetName: String, layerName: String)
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/RemoteFallbackLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/RemoteFallbackLayer.scala
new file mode 100644
index 00000000000..feb953c99f6
--- /dev/null
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/RemoteFallbackLayer.scala
@@ -0,0 +1,3 @@
+package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
+
+case class RemoteFallbackLayer(organizationName: String, dataSetName: String, layerName: String)

From 6bf30badcefb8dc63b6b6c107b181153f945ffa8 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 10 May 2022 14:01:09 +0200
Subject: [PATCH 009/122] logic for applying update actions in order

---
 .../EditableMappingService.scala              | 58 ++++++++++++++++---
 .../editablemapping/RemoteFallbackLayer.scala |  7 ++-
 2 files changed, 56 insertions(+), 9 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 904da3bdd50..335cad1a827 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -3,6 +3,7 @@ package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
 import java.util.UUID
 
 import com.google.inject.Inject
+import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.util.tools.Fox.option2Fox
 import com.scalableminds.webknossos.datastore.SkeletonTracing.{Edge, Tree}
@@ -16,6 +17,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.{
   TracingDataStore,
   VersionedKeyValuePair
 }
+import net.liftweb.common.{Empty, Full}
 
 import scala.concurrent.ExecutionContext
 
@@ -54,12 +56,15 @@ class EditableMappingService @Inject()(
                                                                               emptyFallback = Some(-1L))
     } yield versionOrMinusOne >= 0
 
-  def get(editableMappingId: String, version: Option[Long] = None): Fox[EditableMapping] =
+  def get(editableMappingId: String,
+          remoteFallbackLayer: RemoteFallbackLayer,
+          version: Option[Long] = None): Fox[EditableMapping] =
     for {
       closestMaterializedVersion: VersionedKeyValuePair[Array[Byte]] <- tracingDataStore.editableMappings
         .get(editableMappingId, version)
       materialized <- applyPendingUpdates(editableMappingId,
                                           EditableMapping.fromBytes(closestMaterializedVersion.value),
+                                          remoteFallbackLayer,
                                           closestMaterializedVersion.version,
                                           version)
     } yield materialized
@@ -86,18 +91,55 @@ class EditableMappingService @Inject()(
 
   private def applyPendingUpdates(editableMappingId: String,
                                   existingEditableMapping: EditableMapping,
+                                  remoteFallbackLayer: RemoteFallbackLayer,
                                   existingVersion: Long,
                                   requestedVersion: Option[Long]): Fox[EditableMapping] =
     for {
       desiredVersion <- findDesiredOrNewestPossibleVersion(existingVersion, editableMappingId, requestedVersion)
       pendingUpdates <- findPendingUpdates(editableMappingId, existingVersion, desiredVersion)
-      appliedEditableMapping <- applyUpdates(existingEditableMapping, existingVersion, desiredVersion, pendingUpdates)
+      appliedEditableMapping <- applyUpdates(existingEditableMapping, pendingUpdates, remoteFallbackLayer)
     } yield appliedEditableMapping
 
   private def applyUpdates(existingEditableMapping: EditableMapping,
-                           existingVersion: Long,
-                           desiredVersion: Long,
-                           pendingUpdates: List[EditableMappingUpdateAction]): Fox[EditableMapping] = ???
+                           updates: List[EditableMappingUpdateAction],
+                           remoteFallbackLayer: RemoteFallbackLayer): Fox[EditableMapping] = {
+    def updateIter(mappingFox: Fox[EditableMapping],
+                   remainingUpdates: List[EditableMappingUpdateAction]): Fox[EditableMapping] =
+      mappingFox.futureBox.flatMap {
+        case Empty => Fox.empty
+        case Full(mapping) =>
+          remainingUpdates match {
+            case List() => Fox.successful(mapping)
+            case head :: tail =>
+              updateIter(applyOneUpdate(mapping, head, remoteFallbackLayer), tail)
+          }
+        case _ => mappingFox
+      }
+
+    updateIter(Some(existingEditableMapping), updates)
+  }
+
+  private def applyOneUpdate(mapping: EditableMapping,
+                             update: EditableMappingUpdateAction,
+                             remoteFallbackLayer: RemoteFallbackLayer): Fox[EditableMapping] =
+    update match {
+      case splitAction: SplitAgglomerateUpdateAction => applySplitAction(mapping, splitAction, remoteFallbackLayer)
+      case mergeAction: MergeAgglomerateUpdateAction => applyMergeAction(mapping, mergeAction, remoteFallbackLayer)
+    }
+
+  private def applySplitAction(mapping: EditableMapping,
+                               update: SplitAgglomerateUpdateAction,
+                               remoteFallbackLayer: RemoteFallbackLayer): Fox[EditableMapping] = ???
+
+  private def applyMergeAction(mapping: EditableMapping,
+                               update: MergeAgglomerateUpdateAction,
+                               remoteFallbackLayer: RemoteFallbackLayer): Fox[EditableMapping] =
+    for {
+      segmentId2 <- findSegmentIdAtPosition(update.segmentPosition2, remoteFallbackLayer)
+      // TODO
+    } yield mapping
+
+  private def findSegmentIdAtPosition(pos: Vec3Int, remoteFallbackLayer: RemoteFallbackLayer): Fox[Long] = ???
 
   private def findPendingUpdates(editableMappingId: String, existingVersion: Long, desiredVersion: Long)(
       implicit ec: ExecutionContext): Fox[List[EditableMappingUpdateAction]] =
@@ -118,8 +160,8 @@ class EditableMappingService @Inject()(
   def volumeData(tracing: VolumeTracing, dataRequests: DataRequestCollection): Fox[(Array[Byte], List[Int])] =
     for {
       editableMappingId <- tracing.mappingName.toFox
-      editableMapping <- get(editableMappingId)
       remoteFallbackLayer <- remoteFallbackLayer(tracing)
+      editableMapping <- get(editableMappingId, remoteFallbackLayer)
       (unmappedData, indices) <- getUnmappedDataFromDatastore(remoteFallbackLayer, dataRequests)
       segmentIds = collectSegmentIds(unmappedData, indices, tracing.elementClass)
       relevantMapping <- generateCombinedMappingSubset(segmentIds, editableMapping, remoteFallbackLayer)
@@ -145,7 +187,7 @@ class EditableMappingService @Inject()(
                                          remoteFallbackLayer: RemoteFallbackLayer,
                                          agglomerateId: Long): Fox[Array[Byte]] =
     for {
-      editableMapping <- get(editableMappingId)
+      editableMapping <- get(editableMappingId, remoteFallbackLayer)
       agglomerateIdIsPresent = editableMapping.agglomerateToSegments.contains(agglomerateId)
       skeletonBytes <- if (agglomerateIdIsPresent)
         getAgglomerateSkeleton(editableMappingId, editableMapping, remoteFallbackLayer, agglomerateId)
@@ -213,7 +255,7 @@ class EditableMappingService @Inject()(
     for {
       layerName <- tracing.fallbackLayer.toFox
       organizationName <- tracing.organizationName.toFox
-    } yield RemoteFallbackLayer(organizationName, tracing.dataSetName, layerName)
+    } yield RemoteFallbackLayer(organizationName, tracing.dataSetName, layerName, tracing.elementClass)
 
   private def mapData(unmappedData: Array[Byte],
                       indices: List[Int],
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/RemoteFallbackLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/RemoteFallbackLayer.scala
index feb953c99f6..bb29d447a19 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/RemoteFallbackLayer.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/RemoteFallbackLayer.scala
@@ -1,3 +1,8 @@
 package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
 
-case class RemoteFallbackLayer(organizationName: String, dataSetName: String, layerName: String)
+import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing.ElementClass
+
+case class RemoteFallbackLayer(organizationName: String,
+                               dataSetName: String,
+                               layerName: String,
+                               elementClass: ElementClass)

From 905ac1fa5733cfc499003099eeea7be52d12fc36 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 10 May 2022 14:25:06 +0200
Subject: [PATCH 010/122] resolve segment position to segment id

---
 .../TSRemoteDatastoreClient.scala             | 16 ++++++++++++
 .../EditableMappingService.scala              | 26 +++++++++++++++----
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
index 046a49cf117..835dba44d3c 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
@@ -1,6 +1,7 @@
 package com.scalableminds.webknossos.tracingstore
 
 import com.google.inject.Inject
+import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.models.DataRequestCollection.DataRequestCollection
 import com.scalableminds.webknossos.datastore.rpc.RPC
@@ -31,6 +32,21 @@ class TSRemoteDatastoreClient @Inject()(
                                      mappingName: String,
                                      segmentIdsOrdered: List[Long]): Fox[List[Long]] = ???
 
+  def getVoxelAtPosition(userToken: Option[String],
+                         remoteFallbackLayer: RemoteFallbackLayer,
+                         pos: Vec3Int,
+                         mag: Vec3Int): Fox[Array[Byte]] =
+    rpc(s"${remoteLayerUri(remoteFallbackLayer)}/data")
+      .addQueryStringOptional("token", userToken)
+      .addQueryString("x" -> pos.x.toString)
+      .addQueryString("y" -> pos.y.toString)
+      .addQueryString("z" -> pos.z.toString)
+      .addQueryString("width" -> "1")
+      .addQueryString("height" -> "1")
+      .addQueryString("depth" -> "1")
+      .addQueryString("mag" -> mag.toMagLiteral())
+      .getWithBytesResponse
+
   private def remoteLayerUri(remoteLayer: RemoteFallbackLayer): String =
     s"$datastoreUrl/data/datasets/${remoteLayer.organizationName}/${remoteLayer.dataSetName}/layers/${remoteLayer.layerName}"
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 335cad1a827..601a353f766 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -135,11 +135,20 @@ class EditableMappingService @Inject()(
                                update: MergeAgglomerateUpdateAction,
                                remoteFallbackLayer: RemoteFallbackLayer): Fox[EditableMapping] =
     for {
-      segmentId2 <- findSegmentIdAtPosition(update.segmentPosition2, remoteFallbackLayer)
+      segmentId2 <- findSegmentIdAtPosition(remoteFallbackLayer, update.segmentPosition2)
       // TODO
     } yield mapping
 
-  private def findSegmentIdAtPosition(pos: Vec3Int, remoteFallbackLayer: RemoteFallbackLayer): Fox[Long] = ???
+  private def findSegmentIdAtPosition(remoteFallbackLayer: RemoteFallbackLayer, pos: Vec3Int): Fox[Long] =
+    for {
+      voxelAsBytes: Array[Byte] <- remoteDatastoreClient.getVoxelAtPosition(Some("TODO pass token here"),
+                                                                            remoteFallbackLayer,
+                                                                            pos,
+                                                                            mag = Vec3Int(1, 1, 1))
+      voxelAsLongList: List[Long] = bytesToLongList(voxelAsBytes, remoteFallbackLayer.elementClass)
+      _ <- Fox.bool2Fox(voxelAsLongList.length == 1) ?~> s"Expected one, got ${voxelAsLongList.length} segment id values for voxel."
+      voxelAsLong <- voxelAsLongList.headOption
+    } yield voxelAsLong
 
   private def findPendingUpdates(editableMappingId: String, existingVersion: Long, desiredVersion: Long)(
       implicit ec: ExecutionContext): Fox[List[EditableMappingUpdateAction]] =
@@ -249,16 +258,23 @@ class EditableMappingService @Inject()(
       (data, indices) <- remoteDatastoreClient.getData(remoteFallbackLayer, dataRequests)
     } yield (data, indices)
 
-  private def collectSegmentIds(data: Array[Byte], indices: List[Int], elementClass: ElementClass): Set[Long] = ???
+  private def collectSegmentIds(data: Array[Byte], indices: List[Int], elementClass: ElementClass): Set[Long] =
+    // TODO do we need to skip something, using the indices?
+    bytesToLongList(data, elementClass).toSet
 
   def remoteFallbackLayer(tracing: VolumeTracing): Fox[RemoteFallbackLayer] =
     for {
-      layerName <- tracing.fallbackLayer.toFox
-      organizationName <- tracing.organizationName.toFox
+      layerName <- tracing.fallbackLayer.toFox ?~> "This feature is only defined on volume annotations with fallback segmentation layer."
+      organizationName <- tracing.organizationName.toFox ?~> "This feature is only implemented for volume annotations with an explicit organization name tag, not for legacy volume annotations."
     } yield RemoteFallbackLayer(organizationName, tracing.dataSetName, layerName, tracing.elementClass)
 
   private def mapData(unmappedData: Array[Byte],
                       indices: List[Int],
                       relevantMapping: Map[Long, Long],
                       elementClass: ElementClass): Fox[Array[Byte]] = ???
+
+  private def bytesToLongList(bytes: Array[Byte], elementClass: ElementClass): List[Long] = ???
+
+  private def longListToBytes(longs: List[Long], elementClass: ElementClass): Array[Byte] = ???
+
 }

From ac304851c1cd0be399cec5b9cf1418a5e02ab112 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 10 May 2022 14:49:43 +0200
Subject: [PATCH 011/122] mapData

---
 .../editablemapping/EditableMappingService.scala   | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 601a353f766..67a8827c60d 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -172,9 +172,9 @@ class EditableMappingService @Inject()(
       remoteFallbackLayer <- remoteFallbackLayer(tracing)
       editableMapping <- get(editableMappingId, remoteFallbackLayer)
       (unmappedData, indices) <- getUnmappedDataFromDatastore(remoteFallbackLayer, dataRequests)
-      segmentIds = collectSegmentIds(unmappedData, indices, tracing.elementClass)
+      segmentIds = collectSegmentIds(unmappedData, tracing.elementClass)
       relevantMapping <- generateCombinedMappingSubset(segmentIds, editableMapping, remoteFallbackLayer)
-      mappedData <- mapData(unmappedData, indices, relevantMapping, tracing.elementClass)
+      mappedData = mapData(unmappedData, relevantMapping, tracing.elementClass)
     } yield (mappedData, indices)
 
   private def generateCombinedMappingSubset(segmentIds: Set[Long],
@@ -258,8 +258,7 @@ class EditableMappingService @Inject()(
       (data, indices) <- remoteDatastoreClient.getData(remoteFallbackLayer, dataRequests)
     } yield (data, indices)
 
-  private def collectSegmentIds(data: Array[Byte], indices: List[Int], elementClass: ElementClass): Set[Long] =
-    // TODO do we need to skip something, using the indices?
+  private def collectSegmentIds(data: Array[Byte], elementClass: ElementClass): Set[Long] =
     bytesToLongList(data, elementClass).toSet
 
   def remoteFallbackLayer(tracing: VolumeTracing): Fox[RemoteFallbackLayer] =
@@ -269,9 +268,12 @@ class EditableMappingService @Inject()(
     } yield RemoteFallbackLayer(organizationName, tracing.dataSetName, layerName, tracing.elementClass)
 
   private def mapData(unmappedData: Array[Byte],
-                      indices: List[Int],
                       relevantMapping: Map[Long, Long],
-                      elementClass: ElementClass): Fox[Array[Byte]] = ???
+                      elementClass: ElementClass): Array[Byte] = {
+    val unmappedDataLongs = bytesToLongList(unmappedData, elementClass)
+    val mappedDataLongs = unmappedDataLongs.map(relevantMapping)
+    longListToBytes(mappedDataLongs, elementClass)
+  }
 
   private def bytesToLongList(bytes: Array[Byte], elementClass: ElementClass): List[Long] = ???
 

From 08861f3ab354e476f2e3f7c1961ffc90f762494b Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 11 May 2022 10:26:01 +0200
Subject: [PATCH 012/122] fetch unmapped data from datastore

---
 .../controllers/BinaryDataController.scala    | 16 ++++------
 .../helpers/MissingBucketHeaders.scala        | 29 +++++++++++++++++++
 .../webknossos/datastore/rpc/RPCRequest.scala |  5 ++++
 .../TSRemoteDatastoreClient.scala             | 26 ++++++++++++-----
 .../EditableMappingService.scala              |  7 ++++-
 5 files changed, 65 insertions(+), 18 deletions(-)
 create mode 100644 webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/MissingBucketHeaders.scala

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala
index fd3eae5da07..fe1296f1926 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala
@@ -8,6 +8,7 @@ import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.image.{ImageCreator, ImageCreatorParameters, JPEGWriter}
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.DataStoreConfig
+import com.scalableminds.webknossos.datastore.helpers.MissingBucketHeaders
 import com.scalableminds.webknossos.datastore.models.DataRequestCollection._
 import com.scalableminds.webknossos.datastore.models.datasource._
 import com.scalableminds.webknossos.datastore.models.requests.{
@@ -44,7 +45,8 @@ class BinaryDataController @Inject()(
     isosurfaceServiceHolder: IsosurfaceServiceHolder,
     findDataService: FindDataService,
 )(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers)
-    extends Controller {
+    extends Controller
+    with MissingBucketHeaders {
 
   override def allowRemoteOrigin: Boolean = true
 
@@ -77,17 +79,11 @@ class BinaryDataController @Inject()(
                 + s"  dataLayer: $dataLayerName\n"
                 + s"  requestCount: ${request.body.size}"
                 + s"  requestHead: ${request.body.headOption}")
-        } yield Ok(data).withHeaders(getMissingBucketsHeaders(indices): _*)
+        } yield Ok(data).withHeaders(createMissingBucketsHeaders(indices): _*)
       }
     }
   }
 
-  private def getMissingBucketsHeaders(indices: List[Int]): Seq[(String, String)] =
-    List("MISSING-BUCKETS" -> formatMissingBucketList(indices), "Access-Control-Expose-Headers" -> "MISSING-BUCKETS")
-
-  private def formatMissingBucketList(indices: List[Int]): String =
-    "[" + indices.mkString(", ") + "]"
-
   /**
     * Handles requests for raw binary data via HTTP GET.
     */
@@ -130,7 +126,7 @@ class BinaryDataController @Inject()(
           DataServiceRequestSettings(halfByte = halfByte)
         )
         (data, indices) <- requestData(dataSource, dataLayer, request)
-      } yield Ok(data).withHeaders(getMissingBucketsHeaders(indices): _*)
+      } yield Ok(data).withHeaders(createMissingBucketsHeaders(indices): _*)
     }
   }
 
@@ -163,7 +159,7 @@ class BinaryDataController @Inject()(
           cubeSize
         )
         (data, indices) <- requestData(dataSource, dataLayer, request)
-      } yield Ok(data).withHeaders(getMissingBucketsHeaders(indices): _*)
+      } yield Ok(data).withHeaders(createMissingBucketsHeaders(indices): _*)
     }
   }
 
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/MissingBucketHeaders.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/MissingBucketHeaders.scala
new file mode 100644
index 00000000000..00a33f180be
--- /dev/null
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/MissingBucketHeaders.scala
@@ -0,0 +1,29 @@
+package com.scalableminds.webknossos.datastore.helpers
+
+import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import net.liftweb.common.Box.tryo
+
+import scala.concurrent.ExecutionContext
+
+trait MissingBucketHeaders extends FoxImplicits {
+
+  protected lazy val missingBucketsHeader: String = "MISSING-BUCKETS"
+
+  protected def createMissingBucketsHeaders(indices: List[Int]): Seq[(String, String)] =
+    List(missingBucketsHeader -> formatMissingBucketList(indices),
+         "Access-Control-Expose-Headers" -> missingBucketsHeader)
+
+  protected def formatMissingBucketList(indices: List[Int]): String =
+    "[" + indices.mkString(", ") + "]"
+
+  protected def parseMissingBucketHeader(headerLiteralOpt: Option[String])(
+      implicit ec: ExecutionContext): Fox[List[Int]] =
+    for {
+      headerLiteral: String <- headerLiteralOpt.toFox
+      headerLiteralTrim = headerLiteral.trim
+      _ <- bool2Fox(headerLiteralTrim.startsWith("[") && headerLiteralTrim.endsWith("]"))
+      indicesStr = headerLiteralTrim.drop(1).dropRight(1).split(",").toList
+      indices <- Fox.serialCombined(indicesStr)(indexStr => tryo(indexStr.trim.toInt))
+    } yield indices
+
+}
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala
index 89b46f0e417..5ddd3796163 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala
@@ -78,6 +78,11 @@ class RPCRequest(val id: Int, val url: String, wsClient: WSClient) extends FoxIm
     performRequest
   }
 
+  def postWithBytesResponse: Fox[Array[Byte]] = {
+    request = request.withMethod("POST")
+    extractBytesResponse(performRequest)
+  }
+
   def postWithJsonResponse[T: Reads]: Fox[T] = {
     request = request.withMethod("POST")
     parseJsonResponse(performRequest)
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
index 835dba44d3c..6255aea828b 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
@@ -3,17 +3,23 @@ package com.scalableminds.webknossos.tracingstore
 import com.google.inject.Inject
 import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.tools.Fox
-import com.scalableminds.webknossos.datastore.models.DataRequestCollection.DataRequestCollection
+import com.scalableminds.webknossos.datastore.helpers.MissingBucketHeaders
+import com.scalableminds.webknossos.datastore.models.WebKnossosDataRequest
 import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.RemoteFallbackLayer
 import com.typesafe.scalalogging.LazyLogging
+import play.api.http.Status
 import play.api.inject.ApplicationLifecycle
 
+import scala.concurrent.ExecutionContext
+
 class TSRemoteDatastoreClient @Inject()(
     rpc: RPC,
     config: TracingStoreConfig,
     val lifecycle: ApplicationLifecycle
-) extends LazyLogging {
+)(implicit ec: ExecutionContext)
+    extends LazyLogging
+    with MissingBucketHeaders {
 
   private val datastoreUrl: String = config.Tracingstore.WebKnossos.uri
 
@@ -26,11 +32,13 @@ class TSRemoteDatastoreClient @Inject()(
       .getWithBytesResponse
 
   def getData(remoteFallbackLayer: RemoteFallbackLayer,
-              dataRequests: DataRequestCollection): Fox[(Array[Byte], List[Int])] = ???
-
-  def getAgglomerateIdsForSegmentIds(remoteFallbackLayer: RemoteFallbackLayer,
-                                     mappingName: String,
-                                     segmentIdsOrdered: List[Long]): Fox[List[Long]] = ???
+              dataRequests: List[WebKnossosDataRequest]): Fox[(Array[Byte], List[Int])] =
+    for {
+      response <- rpc(s"${remoteLayerUri(remoteFallbackLayer)}/").post(dataRequests)
+      _ <- bool2Fox(Status.isSuccessful(response.status))
+      bytes = response.bodyAsBytes.toArray
+      indices <- parseMissingBucketHeader(response.header(missingBucketsHeader)) ?~> "failed to parse missing bucket header"
+    } yield (bytes, indices)
 
   def getVoxelAtPosition(userToken: Option[String],
                          remoteFallbackLayer: RemoteFallbackLayer,
@@ -47,6 +55,10 @@ class TSRemoteDatastoreClient @Inject()(
       .addQueryString("mag" -> mag.toMagLiteral())
       .getWithBytesResponse
 
+  def getAgglomerateIdsForSegmentIds(remoteFallbackLayer: RemoteFallbackLayer,
+                                     mappingName: String,
+                                     segmentIdsOrdered: List[Long]): Fox[List[Long]] = ???
+
   private def remoteLayerUri(remoteLayer: RemoteFallbackLayer): String =
     s"$datastoreUrl/data/datasets/${remoteLayer.organizationName}/${remoteLayer.dataSetName}/layers/${remoteLayer.layerName}"
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 67a8827c60d..a81dfd24c02 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -11,6 +11,7 @@ import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing
 import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing.ElementClass
 import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, ProtoGeometryImplicits, SkeletonTracingDefaults}
 import com.scalableminds.webknossos.datastore.models.DataRequestCollection.DataRequestCollection
+import com.scalableminds.webknossos.datastore.models.WebKnossosDataRequest
 import com.scalableminds.webknossos.tracingstore.TSRemoteDatastoreClient
 import com.scalableminds.webknossos.tracingstore.tracings.{
   KeyValueStoreImplicits,
@@ -255,7 +256,11 @@ class EditableMappingService @Inject()(
   private def getUnmappedDataFromDatastore(remoteFallbackLayer: RemoteFallbackLayer,
                                            dataRequests: DataRequestCollection): Fox[(Array[Byte], List[Int])] =
     for {
-      (data, indices) <- remoteDatastoreClient.getData(remoteFallbackLayer, dataRequests)
+      dataRequestsTyped <- Fox.serialCombined(dataRequests) {
+        case r: WebKnossosDataRequest => Fox.successful(r)
+        case _                        => Fox.failure("Editable Mappings currently only work for webKnossos data requests")
+      }
+      (data, indices) <- remoteDatastoreClient.getData(remoteFallbackLayer, dataRequestsTyped)
     } yield (data, indices)
 
   private def collectSegmentIds(data: Array[Byte], elementClass: ElementClass): Set[Long] =

From fff5f2ade9176bea5f3dcf425c666190707e5b17 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 11 May 2022 11:21:25 +0200
Subject: [PATCH 013/122] bytes conversion

---
 .../datastore/models/UnsignedInteger.scala    |  12 +-
 .../controllers/VolumeTracingController.scala |   8 +-
 .../EditableMappingService.scala              | 108 +++++++++++-------
 3 files changed, 83 insertions(+), 45 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/UnsignedInteger.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/UnsignedInteger.scala
index b7665553409..28212cb5f6b 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/UnsignedInteger.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/UnsignedInteger.scala
@@ -1,6 +1,6 @@
 package com.scalableminds.webknossos.datastore.models
 
-import java.nio.{Buffer, ByteBuffer, ByteOrder, IntBuffer, LongBuffer, ShortBuffer}
+import java.nio._
 
 import com.scalableminds.webknossos.datastore.models.UnsignedInteger.wrongElementClass
 import com.scalableminds.webknossos.datastore.models.datasource.ElementClass
@@ -13,6 +13,7 @@ trait UnsignedInteger {
   def increment: UnsignedInteger
   def isZero: Boolean
   def toSignedLong: Long
+  def toUnsignedLong: Long
 }
 
 object UInt8 { @inline final def apply(n: Byte): UInt8 = new UInt8(n) }
@@ -47,6 +48,8 @@ class UInt8(val signed: Byte) extends UnsignedInteger {
   def increment: UInt8 = UInt8((signed + 1).toByte)
   def isZero: Boolean = signed == 0
   override def toSignedLong: Long = signed.toLong
+  override def toUnsignedLong: Long =
+    if (signed >= 0) toSignedLong else toSignedLong + Byte.MaxValue // TODO double-check these
   override def toString = s"UInt8($signed)"
   override def hashCode: Int = signed.hashCode
   override def equals(that: Any): Boolean = that match {
@@ -54,10 +57,12 @@ class UInt8(val signed: Byte) extends UnsignedInteger {
     case _           => false
   }
 }
+
 class UInt16(val signed: Short) extends UnsignedInteger {
   def increment: UInt16 = UInt16((signed + 1).toShort)
   def isZero: Boolean = signed == 0
   override def toSignedLong: Long = signed.toLong
+  override def toUnsignedLong: Long = if (signed >= 0) toSignedLong else toSignedLong + Short.MaxValue
   override def toString = s"UInt16($signed)"
   override def hashCode: Int = signed.hashCode
   override def equals(that: Any): Boolean = that match {
@@ -65,10 +70,12 @@ class UInt16(val signed: Short) extends UnsignedInteger {
     case _            => false
   }
 }
+
 class UInt32(val signed: Int) extends UnsignedInteger {
   def increment: UInt32 = UInt32(signed + 1)
   def isZero: Boolean = signed == 0
   override def toSignedLong: Long = signed.toLong
+  override def toUnsignedLong: Long = if (signed >= 0) toSignedLong else toSignedLong + Int.MaxValue
   override def toString = s"UInt32($signed)"
   override def hashCode: Int = signed.hashCode
   override def equals(that: Any): Boolean = that match {
@@ -76,10 +83,13 @@ class UInt32(val signed: Int) extends UnsignedInteger {
     case _            => false
   }
 }
+
 class UInt64(val signed: Long) extends UnsignedInteger {
   def increment: UInt64 = UInt64(signed + 1)
   def isZero: Boolean = signed == 0L
   override def toSignedLong: Long = signed
+  override def toUnsignedLong: Long =
+    if (signed >= 0) signed else throw new Exception("Cannot convert UInt64 with value > 2**32 to Long")
   override def toString = s"UInt64($signed)"
   override def hashCode: Int = signed.hashCode
   override def equals(that: Any): Boolean = that match {
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index 44d5ab8c052..6459d882e85 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -138,7 +138,7 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
             hasEditableMapping <- Fox.runOptional(tracing.mappingName)(editableMappingService.exists)
             (data, indices) <- if (hasEditableMapping.getOrElse(false))
               tracingService.data(tracingId, tracing, request.body)
-            else editableMappingService.volumeData(tracing, request.body)
+            else editableMappingService.volumeData(tracing, request.body, token)
           } yield Ok(data).withHeaders(getMissingBucketsHeaders(indices): _*)
         }
       }
@@ -241,10 +241,10 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
           remoteFallbackLayer <- editableMappingService.remoteFallbackLayer(tracing)
           isEditableMapping <- editableMappingService.exists(mappingName)
           agglomerateSkeletonBytes <- if (isEditableMapping)
-            editableMappingService.getAgglomerateSkeletonWithFallback(token,
-                                                                      mappingName,
+            editableMappingService.getAgglomerateSkeletonWithFallback(mappingName,
                                                                       remoteFallbackLayer,
-                                                                      agglomerateId)
+                                                                      agglomerateId,
+                                                                      token)
           else remoteDatastoreClient.getAgglomerateSkeleton(token, remoteFallbackLayer, mappingName, agglomerateId)
         } yield Ok(agglomerateSkeletonBytes)
       }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index a81dfd24c02..bed6aa7491f 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -4,20 +4,20 @@ import java.util.UUID
 
 import com.google.inject.Inject
 import com.scalableminds.util.geometry.Vec3Int
-import com.scalableminds.util.tools.Fox
-import com.scalableminds.util.tools.Fox.option2Fox
+import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.SkeletonTracing.{Edge, Tree}
 import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing
 import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing.ElementClass
 import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, ProtoGeometryImplicits, SkeletonTracingDefaults}
 import com.scalableminds.webknossos.datastore.models.DataRequestCollection.DataRequestCollection
-import com.scalableminds.webknossos.datastore.models.WebKnossosDataRequest
+import com.scalableminds.webknossos.datastore.models.{UnsignedInteger, UnsignedIntegerArray, WebKnossosDataRequest}
 import com.scalableminds.webknossos.tracingstore.TSRemoteDatastoreClient
 import com.scalableminds.webknossos.tracingstore.tracings.{
   KeyValueStoreImplicits,
   TracingDataStore,
   VersionedKeyValuePair
 }
+import net.liftweb.common.Box.tryo
 import net.liftweb.common.{Empty, Full}
 
 import scala.concurrent.ExecutionContext
@@ -27,6 +27,7 @@ class EditableMappingService @Inject()(
     remoteDatastoreClient: TSRemoteDatastoreClient
 )(implicit ec: ExecutionContext)
     extends KeyValueStoreImplicits
+    with FoxImplicits
     with ProtoGeometryImplicits {
 
   private def generateId: String = UUID.randomUUID.toString
@@ -59,15 +60,19 @@ class EditableMappingService @Inject()(
 
   def get(editableMappingId: String,
           remoteFallbackLayer: RemoteFallbackLayer,
+          userToken: Option[String],
           version: Option[Long] = None): Fox[EditableMapping] =
     for {
       closestMaterializedVersion: VersionedKeyValuePair[Array[Byte]] <- tracingDataStore.editableMappings
         .get(editableMappingId, version)
-      materialized <- applyPendingUpdates(editableMappingId,
-                                          EditableMapping.fromBytes(closestMaterializedVersion.value),
-                                          remoteFallbackLayer,
-                                          closestMaterializedVersion.version,
-                                          version)
+      materialized <- applyPendingUpdates(
+        editableMappingId,
+        EditableMapping.fromBytes(closestMaterializedVersion.value),
+        remoteFallbackLayer,
+        closestMaterializedVersion.version,
+        version,
+        userToken
+      )
     } yield materialized
 
   private def findDesiredOrNewestPossibleVersion(existingMaterializedVersion: Long,
@@ -94,16 +99,18 @@ class EditableMappingService @Inject()(
                                   existingEditableMapping: EditableMapping,
                                   remoteFallbackLayer: RemoteFallbackLayer,
                                   existingVersion: Long,
-                                  requestedVersion: Option[Long]): Fox[EditableMapping] =
+                                  requestedVersion: Option[Long],
+                                  userToken: Option[String]): Fox[EditableMapping] =
     for {
       desiredVersion <- findDesiredOrNewestPossibleVersion(existingVersion, editableMappingId, requestedVersion)
       pendingUpdates <- findPendingUpdates(editableMappingId, existingVersion, desiredVersion)
-      appliedEditableMapping <- applyUpdates(existingEditableMapping, pendingUpdates, remoteFallbackLayer)
+      appliedEditableMapping <- applyUpdates(existingEditableMapping, pendingUpdates, remoteFallbackLayer, userToken)
     } yield appliedEditableMapping
 
   private def applyUpdates(existingEditableMapping: EditableMapping,
                            updates: List[EditableMappingUpdateAction],
-                           remoteFallbackLayer: RemoteFallbackLayer): Fox[EditableMapping] = {
+                           remoteFallbackLayer: RemoteFallbackLayer,
+                           userToken: Option[String]): Fox[EditableMapping] = {
     def updateIter(mappingFox: Fox[EditableMapping],
                    remainingUpdates: List[EditableMappingUpdateAction]): Fox[EditableMapping] =
       mappingFox.futureBox.flatMap {
@@ -112,7 +119,7 @@ class EditableMappingService @Inject()(
           remainingUpdates match {
             case List() => Fox.successful(mapping)
             case head :: tail =>
-              updateIter(applyOneUpdate(mapping, head, remoteFallbackLayer), tail)
+              updateIter(applyOneUpdate(mapping, head, remoteFallbackLayer, userToken), tail)
           }
         case _ => mappingFox
       }
@@ -122,33 +129,40 @@ class EditableMappingService @Inject()(
 
   private def applyOneUpdate(mapping: EditableMapping,
                              update: EditableMappingUpdateAction,
-                             remoteFallbackLayer: RemoteFallbackLayer): Fox[EditableMapping] =
+                             remoteFallbackLayer: RemoteFallbackLayer,
+                             userToken: Option[String]): Fox[EditableMapping] =
     update match {
-      case splitAction: SplitAgglomerateUpdateAction => applySplitAction(mapping, splitAction, remoteFallbackLayer)
-      case mergeAction: MergeAgglomerateUpdateAction => applyMergeAction(mapping, mergeAction, remoteFallbackLayer)
+      case splitAction: SplitAgglomerateUpdateAction =>
+        applySplitAction(mapping, splitAction, remoteFallbackLayer, userToken)
+      case mergeAction: MergeAgglomerateUpdateAction =>
+        applyMergeAction(mapping, mergeAction, remoteFallbackLayer, userToken)
     }
 
   private def applySplitAction(mapping: EditableMapping,
                                update: SplitAgglomerateUpdateAction,
-                               remoteFallbackLayer: RemoteFallbackLayer): Fox[EditableMapping] = ???
+                               remoteFallbackLayer: RemoteFallbackLayer,
+                               userToken: Option[String]): Fox[EditableMapping] = ???
 
   private def applyMergeAction(mapping: EditableMapping,
                                update: MergeAgglomerateUpdateAction,
-                               remoteFallbackLayer: RemoteFallbackLayer): Fox[EditableMapping] =
+                               remoteFallbackLayer: RemoteFallbackLayer,
+                               userToken: Option[String]): Fox[EditableMapping] =
     for {
-      segmentId2 <- findSegmentIdAtPosition(remoteFallbackLayer, update.segmentPosition2)
+      segmentId2 <- findSegmentIdAtPosition(remoteFallbackLayer, update.segmentPosition2, userToken)
       // TODO
     } yield mapping
 
-  private def findSegmentIdAtPosition(remoteFallbackLayer: RemoteFallbackLayer, pos: Vec3Int): Fox[Long] =
+  private def findSegmentIdAtPosition(remoteFallbackLayer: RemoteFallbackLayer,
+                                      pos: Vec3Int,
+                                      userToken: Option[String]): Fox[Long] =
     for {
-      voxelAsBytes: Array[Byte] <- remoteDatastoreClient.getVoxelAtPosition(Some("TODO pass token here"),
+      voxelAsBytes: Array[Byte] <- remoteDatastoreClient.getVoxelAtPosition(userToken,
                                                                             remoteFallbackLayer,
                                                                             pos,
                                                                             mag = Vec3Int(1, 1, 1))
-      voxelAsLongList: List[Long] = bytesToLongList(voxelAsBytes, remoteFallbackLayer.elementClass)
-      _ <- Fox.bool2Fox(voxelAsLongList.length == 1) ?~> s"Expected one, got ${voxelAsLongList.length} segment id values for voxel."
-      voxelAsLong <- voxelAsLongList.headOption
+      voxelAsLongArray: Array[Long] <- bytesToLongs(voxelAsBytes, remoteFallbackLayer.elementClass)
+      _ <- Fox.bool2Fox(voxelAsLongArray.length == 1) ?~> s"Expected one, got ${voxelAsLongArray.length} segment id values for voxel."
+      voxelAsLong <- voxelAsLongArray.headOption
     } yield voxelAsLong
 
   private def findPendingUpdates(editableMappingId: String, existingVersion: Long, desiredVersion: Long)(
@@ -167,15 +181,17 @@ class EditableMappingService @Inject()(
       _ <- tracingDataStore.editableMappingUpdates.put(editableMappingId, version, updateAction)
     } yield ()
 
-  def volumeData(tracing: VolumeTracing, dataRequests: DataRequestCollection): Fox[(Array[Byte], List[Int])] =
+  def volumeData(tracing: VolumeTracing,
+                 dataRequests: DataRequestCollection,
+                 userToken: Option[String]): Fox[(Array[Byte], List[Int])] =
     for {
       editableMappingId <- tracing.mappingName.toFox
       remoteFallbackLayer <- remoteFallbackLayer(tracing)
-      editableMapping <- get(editableMappingId, remoteFallbackLayer)
+      editableMapping <- get(editableMappingId, remoteFallbackLayer, userToken)
       (unmappedData, indices) <- getUnmappedDataFromDatastore(remoteFallbackLayer, dataRequests)
-      segmentIds = collectSegmentIds(unmappedData, tracing.elementClass)
+      segmentIds <- collectSegmentIds(unmappedData, tracing.elementClass)
       relevantMapping <- generateCombinedMappingSubset(segmentIds, editableMapping, remoteFallbackLayer)
-      mappedData = mapData(unmappedData, relevantMapping, tracing.elementClass)
+      mappedData <- mapData(unmappedData, relevantMapping, tracing.elementClass)
     } yield (mappedData, indices)
 
   private def generateCombinedMappingSubset(segmentIds: Set[Long],
@@ -192,12 +208,12 @@ class EditableMappingService @Inject()(
     } yield editableMappingSubset ++ baseMappingSubset
   }
 
-  def getAgglomerateSkeletonWithFallback(userToken: Option[String],
-                                         editableMappingId: String,
+  def getAgglomerateSkeletonWithFallback(editableMappingId: String,
                                          remoteFallbackLayer: RemoteFallbackLayer,
-                                         agglomerateId: Long): Fox[Array[Byte]] =
+                                         agglomerateId: Long,
+                                         userToken: Option[String]): Fox[Array[Byte]] =
     for {
-      editableMapping <- get(editableMappingId, remoteFallbackLayer)
+      editableMapping <- get(editableMappingId, remoteFallbackLayer, userToken)
       agglomerateIdIsPresent = editableMapping.agglomerateToSegments.contains(agglomerateId)
       skeletonBytes <- if (agglomerateIdIsPresent)
         getAgglomerateSkeleton(editableMappingId, editableMapping, remoteFallbackLayer, agglomerateId)
@@ -263,8 +279,10 @@ class EditableMappingService @Inject()(
       (data, indices) <- remoteDatastoreClient.getData(remoteFallbackLayer, dataRequestsTyped)
     } yield (data, indices)
 
-  private def collectSegmentIds(data: Array[Byte], elementClass: ElementClass): Set[Long] =
-    bytesToLongList(data, elementClass).toSet
+  private def collectSegmentIds(data: Array[Byte], elementClass: ElementClass): Fox[Set[Long]] =
+    for {
+      dataAsLongs <- bytesToLongs(data, elementClass)
+    } yield dataAsLongs.toSet
 
   def remoteFallbackLayer(tracing: VolumeTracing): Fox[RemoteFallbackLayer] =
     for {
@@ -274,14 +292,24 @@ class EditableMappingService @Inject()(
 
   private def mapData(unmappedData: Array[Byte],
                       relevantMapping: Map[Long, Long],
-                      elementClass: ElementClass): Array[Byte] = {
-    val unmappedDataLongs = bytesToLongList(unmappedData, elementClass)
-    val mappedDataLongs = unmappedDataLongs.map(relevantMapping)
-    longListToBytes(mappedDataLongs, elementClass)
-  }
+                      elementClass: ElementClass): Fox[Array[Byte]] =
+    for {
+      unmappedDataLongs <- bytesToLongs(unmappedData, elementClass)
+      mappedDataLongs = unmappedDataLongs.map(relevantMapping)
+      bytes <- longsToBytes(mappedDataLongs, elementClass)
+    } yield bytes
 
-  private def bytesToLongList(bytes: Array[Byte], elementClass: ElementClass): List[Long] = ???
+  private def bytesToLongs(bytes: Array[Byte], elementClass: ElementClass): Fox[Array[Long]] =
+    for {
+      _ <- bool2Fox(!elementClass.isuint64)
+      unsignedIntArray <- tryo(UnsignedIntegerArray.fromByteArray(bytes, elementClass)).toFox
+    } yield unsignedIntArray.map(_.toUnsignedLong)
 
-  private def longListToBytes(longs: List[Long], elementClass: ElementClass): Array[Byte] = ???
+  private def longsToBytes(longs: Array[Long], elementClass: ElementClass): Fox[Array[Byte]] =
+    for {
+      _ <- bool2Fox(!elementClass.isuint64)
+      unsignedIntArray: Array[UnsignedInteger] = longs.map(UnsignedInteger.fromLongWithElementClass(_, elementClass))
+      bytes = UnsignedIntegerArray.toByteArray(unsignedIntArray, elementClass)
+    } yield bytes
 
 }

From e13e4747ed61ed0a416589b2253e0051efc29499 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 11 May 2022 11:41:31 +0200
Subject: [PATCH 014/122] fix unsigned int to long

---
 .../datastore/models/UnsignedInteger.scala    | 19 ++++++++-----------
 .../EditableMappingService.scala              |  2 +-
 .../volume/VolumeTracingService.scala         | 19 ++++++++++---------
 3 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/UnsignedInteger.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/UnsignedInteger.scala
index 28212cb5f6b..bc2d07ceaf7 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/UnsignedInteger.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/UnsignedInteger.scala
@@ -12,8 +12,7 @@ import scala.reflect.ClassTag
 trait UnsignedInteger {
   def increment: UnsignedInteger
   def isZero: Boolean
-  def toSignedLong: Long
-  def toUnsignedLong: Long
+  def toPositiveLong: Long
 }
 
 object UInt8 { @inline final def apply(n: Byte): UInt8 = new UInt8(n) }
@@ -47,9 +46,8 @@ object UnsignedInteger {
 class UInt8(val signed: Byte) extends UnsignedInteger {
   def increment: UInt8 = UInt8((signed + 1).toByte)
   def isZero: Boolean = signed == 0
-  override def toSignedLong: Long = signed.toLong
-  override def toUnsignedLong: Long =
-    if (signed >= 0) toSignedLong else toSignedLong + Byte.MaxValue // TODO double-check these
+  override def toPositiveLong: Long =
+    if (signed >= 0) signed.toLong else signed.toLong + Byte.MaxValue.toLong + Byte.MaxValue.toLong + 2L
   override def toString = s"UInt8($signed)"
   override def hashCode: Int = signed.hashCode
   override def equals(that: Any): Boolean = that match {
@@ -61,8 +59,8 @@ class UInt8(val signed: Byte) extends UnsignedInteger {
 class UInt16(val signed: Short) extends UnsignedInteger {
   def increment: UInt16 = UInt16((signed + 1).toShort)
   def isZero: Boolean = signed == 0
-  override def toSignedLong: Long = signed.toLong
-  override def toUnsignedLong: Long = if (signed >= 0) toSignedLong else toSignedLong + Short.MaxValue
+  override def toPositiveLong: Long =
+    if (signed >= 0) signed.toLong else signed.toLong + Short.MaxValue.toLong + Short.MaxValue.toLong + 2L
   override def toString = s"UInt16($signed)"
   override def hashCode: Int = signed.hashCode
   override def equals(that: Any): Boolean = that match {
@@ -74,8 +72,8 @@ class UInt16(val signed: Short) extends UnsignedInteger {
 class UInt32(val signed: Int) extends UnsignedInteger {
   def increment: UInt32 = UInt32(signed + 1)
   def isZero: Boolean = signed == 0
-  override def toSignedLong: Long = signed.toLong
-  override def toUnsignedLong: Long = if (signed >= 0) toSignedLong else toSignedLong + Int.MaxValue
+  override def toPositiveLong: Long =
+    if (signed >= 0) signed.toLong else signed.toLong + Int.MaxValue.toLong + Int.MaxValue.toLong + 2L
   override def toString = s"UInt32($signed)"
   override def hashCode: Int = signed.hashCode
   override def equals(that: Any): Boolean = that match {
@@ -87,8 +85,7 @@ class UInt32(val signed: Int) extends UnsignedInteger {
 class UInt64(val signed: Long) extends UnsignedInteger {
   def increment: UInt64 = UInt64(signed + 1)
   def isZero: Boolean = signed == 0L
-  override def toSignedLong: Long = signed
-  override def toUnsignedLong: Long =
+  override def toPositiveLong: Long =
     if (signed >= 0) signed else throw new Exception("Cannot convert UInt64 with value > 2**32 to Long")
   override def toString = s"UInt64($signed)"
   override def hashCode: Int = signed.hashCode
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index bed6aa7491f..0748b8f04aa 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -303,7 +303,7 @@ class EditableMappingService @Inject()(
     for {
       _ <- bool2Fox(!elementClass.isuint64)
       unsignedIntArray <- tryo(UnsignedIntegerArray.fromByteArray(bytes, elementClass)).toFox
-    } yield unsignedIntArray.map(_.toUnsignedLong)
+    } yield unsignedIntArray.map(_.toPositiveLong)
 
   private def longsToBytes(longs: Array[Long], elementClass: ElementClass): Fox[Array[Byte]] =
     for {
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
index 05dab2f856a..09bd42e0153 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
@@ -476,16 +476,17 @@ class VolumeTracingService @Inject()(
       _ <- mergedVolume.withMergedBuckets { (bucketPosition, bucketBytes) =>
         saveBucket(volumeLayer, bucketPosition, bucketBytes, tracing.version + 1)
       }
-      updateGroup = UpdateActionGroup[VolumeTracing](tracing.version + 1,
-                                                     System.currentTimeMillis(),
-                                                     List(ImportVolumeData(mergedVolume.largestSegmentId.toSignedLong)),
-                                                     None,
-                                                     None,
-                                                     None,
-                                                     None,
-                                                     None)
+      updateGroup = UpdateActionGroup[VolumeTracing](
+        tracing.version + 1,
+        System.currentTimeMillis(),
+        List(ImportVolumeData(mergedVolume.largestSegmentId.toPositiveLong)),
+        None,
+        None,
+        None,
+        None,
+        None)
       _ <- handleUpdateGroup(tracingId, updateGroup, tracing.version)
-    } yield mergedVolume.largestSegmentId.toSignedLong
+    } yield mergedVolume.largestSegmentId.toPositiveLong
   }
 
   def dummyTracing: VolumeTracing = ???

From 2657d2b87c7aa4798742a9dcf0a9087a2c5b9968 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 11 May 2022 14:54:06 +0200
Subject: [PATCH 015/122] apply merge update action

---
 .../TSRemoteDatastoreClient.scala             |  6 +-
 .../controllers/VolumeTracingController.scala | 12 ++-
 .../editablemapping/AgglomerateGraph.scala    | 15 ++++
 .../editablemapping/EditableMapping.scala     | 13 +---
 .../EditableMappingService.scala              | 76 +++++++++++++------
 .../EditableMappingUpdateActions.scala        | 15 +---
 ...alableminds.webknossos.tracingstore.routes |  1 +
 7 files changed, 87 insertions(+), 51 deletions(-)
 create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/AgglomerateGraph.scala

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
index 6255aea828b..af23fe2792f 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
@@ -6,7 +6,7 @@ import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.helpers.MissingBucketHeaders
 import com.scalableminds.webknossos.datastore.models.WebKnossosDataRequest
 import com.scalableminds.webknossos.datastore.rpc.RPC
-import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.RemoteFallbackLayer
+import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{AgglomerateGraph, RemoteFallbackLayer}
 import com.typesafe.scalalogging.LazyLogging
 import play.api.http.Status
 import play.api.inject.ApplicationLifecycle
@@ -61,4 +61,8 @@ class TSRemoteDatastoreClient @Inject()(
 
   private def remoteLayerUri(remoteLayer: RemoteFallbackLayer): String =
     s"$datastoreUrl/data/datasets/${remoteLayer.organizationName}/${remoteLayer.dataSetName}/layers/${remoteLayer.layerName}"
+
+  def getAgglomerateGraph(remoteFallbackLayer: RemoteFallbackLayer,
+                          agglomerateId: Long,
+                          userToken: Option[String]): Fox[AgglomerateGraph] = ???
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index 6459d882e85..73d31e7ec0c 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -239,13 +239,11 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
           tracing <- tracingService.find(tracingId)
           mappingName <- tracing.mappingName ?~> "annotation.agglomerateSkeleton.noMappingSet"
           remoteFallbackLayer <- editableMappingService.remoteFallbackLayer(tracing)
-          isEditableMapping <- editableMappingService.exists(mappingName)
-          agglomerateSkeletonBytes <- if (isEditableMapping)
-            editableMappingService.getAgglomerateSkeletonWithFallback(mappingName,
-                                                                      remoteFallbackLayer,
-                                                                      agglomerateId,
-                                                                      token)
-          else remoteDatastoreClient.getAgglomerateSkeleton(token, remoteFallbackLayer, mappingName, agglomerateId)
+          _ <- Fox.assertTrue(editableMappingService.exists(mappingName)) ?~> "Cannot query agglomerate skeleton for volume annotation"
+          agglomerateSkeletonBytes <- editableMappingService.getAgglomerateSkeletonWithFallback(mappingName,
+                                                                                                remoteFallbackLayer,
+                                                                                                agglomerateId,
+                                                                                                token)
         } yield Ok(agglomerateSkeletonBytes)
       }
     }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/AgglomerateGraph.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/AgglomerateGraph.scala
new file mode 100644
index 00000000000..681cfe06fed
--- /dev/null
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/AgglomerateGraph.scala
@@ -0,0 +1,15 @@
+package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
+
+import com.scalableminds.util.geometry.Vec3Int
+import play.api.libs.json.{Json, OFormat}
+
+case class AgglomerateGraph(segments: List[Long],
+                            edges: List[(Long, Long)],
+                            positions: List[Vec3Int],
+                            affinities: List[Long])
+
+object AgglomerateGraph {
+  implicit val jsonFormat: OFormat[AgglomerateGraph] = Json.format[AgglomerateGraph]
+
+  def empty: AgglomerateGraph = AgglomerateGraph(List.empty, List.empty, List.empty, List.empty)
+}
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
index b0f796681a2..f47bf5ae383 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
@@ -1,18 +1,13 @@
 package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
 
-import com.scalableminds.util.geometry.Vec3Int
+import play.api.libs.json.{Json, OFormat}
 
 case class EditableMapping(
     baseMappingName: String,
     segmentToAgglomerate: Map[Long, Long],
-    agglomerateToSegments: Map[Long, List[Long]],
-    agglomerateToEdges: Map[Long, List[(Long, Long)]],
-    agglomerateToPositions: Map[Long, List[Vec3Int]],
-    agglomerateToAffinities: Map[Long, List[Long]]
-) {
-  def toBytes: Array[Byte] = ???
-}
+    agglomerateToGraph: Map[Long, AgglomerateGraph],
+)
 
 object EditableMapping {
-  def fromBytes(bytes: Array[Byte]): EditableMapping = ???
+  implicit val jsonFormat: OFormat[EditableMapping] = Json.format[EditableMapping]
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 0748b8f04aa..3225880287b 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -38,15 +38,12 @@ class EditableMappingService @Inject()(
   def create(baseMappingName: String): Fox[String] = {
     val newId = generateId
     val newEditableMapping = EditableMapping(
-      baseMappingName,
-      Map(),
-      Map(),
-      Map(),
-      Map(),
-      Map()
+      baseMappingName = baseMappingName,
+      segmentToAgglomerate = Map(),
+      agglomerateToGraph = Map()
     )
     for {
-      _ <- tracingDataStore.editableMappings.put(newId, 0L, newEditableMapping.toBytes)
+      _ <- tracingDataStore.editableMappings.put(newId, 0L, newEditableMapping)
     } yield newId
   }
 
@@ -58,21 +55,21 @@ class EditableMappingService @Inject()(
                                                                               emptyFallback = Some(-1L))
     } yield versionOrMinusOne >= 0
 
-  def get(editableMappingId: String,
-          remoteFallbackLayer: RemoteFallbackLayer,
-          userToken: Option[String],
-          version: Option[Long] = None): Fox[EditableMapping] =
+  private def get(editableMappingId: String,
+                  remoteFallbackLayer: RemoteFallbackLayer,
+                  userToken: Option[String],
+                  version: Option[Long] = None): Fox[EditableMapping] =
     for {
-      closestMaterializedVersion: VersionedKeyValuePair[Array[Byte]] <- tracingDataStore.editableMappings
-        .get(editableMappingId, version)
+      closestMaterializedVersion: VersionedKeyValuePair[EditableMapping] <- tracingDataStore.editableMappings
+        .get(editableMappingId, version)(fromJson[EditableMapping])
       materialized <- applyPendingUpdates(
         editableMappingId,
-        EditableMapping.fromBytes(closestMaterializedVersion.value),
+        closestMaterializedVersion.value,
         remoteFallbackLayer,
         closestMaterializedVersion.version,
         version,
         userToken
-      )
+      ) // TODO store materialized in cache or db
     } yield materialized
 
   private def findDesiredOrNewestPossibleVersion(existingMaterializedVersion: Long,
@@ -149,8 +146,42 @@ class EditableMappingService @Inject()(
                                userToken: Option[String]): Fox[EditableMapping] =
     for {
       segmentId2 <- findSegmentIdAtPosition(remoteFallbackLayer, update.segmentPosition2, userToken)
-      // TODO
-    } yield mapping
+      agglomerateGraph1 <- agglomerateGraphForId(mapping, update.agglomerateId1, remoteFallbackLayer, userToken)
+      agglomerateGraph2 <- agglomerateGraphForId(mapping, update.agglomerateId2, remoteFallbackLayer, userToken)
+      mergedGraph = mergeGraph(agglomerateGraph1, agglomerateGraph2, update.segmentId1, segmentId2)
+      _ <- bool2Fox(agglomerateGraph2.segments.contains(segmentId2)) ?~> "segment as queried by position is not contained in fetched agglomerate graph"
+      mergedSegmentToAgglomerate: Map[Long, Long] = agglomerateGraph2.segments.map(s => s -> update.agglomerateId1)
+    } yield
+      EditableMapping(
+        baseMappingName = mapping.baseMappingName,
+        segmentToAgglomerate = mapping.segmentToAgglomerate ++ mergedSegmentToAgglomerate,
+        agglomerateToGraph = mapping.agglomerateToGraph ++ Map(update.agglomerateId1 -> mergedGraph,
+                                                               update.agglomerateId2 -> AgglomerateGraph.empty)
+      )
+
+  private def mergeGraph(agglomerateGraph1: AgglomerateGraph,
+                         agglomerateGraph2: AgglomerateGraph,
+                         segmentId1: Long,
+                         segmentId2: Long): AgglomerateGraph = {
+    val newEdge = (segmentId1, segmentId2)
+    val newEdgeAffinity = 255L
+    AgglomerateGraph(
+      segments = agglomerateGraph1.segments ++ agglomerateGraph2.segments,
+      edges = newEdge :: (agglomerateGraph1.edges ++ agglomerateGraph2.edges),
+      affinities = newEdgeAffinity :: (agglomerateGraph1.affinities ++ agglomerateGraph2.affinities),
+      positions = agglomerateGraph1.positions ++ agglomerateGraph2.positions
+    )
+  }
+
+  private def agglomerateGraphForId(mapping: EditableMapping,
+                                    agglomerateId: Long,
+                                    remoteFallbackLayer: RemoteFallbackLayer,
+                                    userToken: Option[String]): Fox[AgglomerateGraph] =
+    if (mapping.agglomerateToGraph.contains(agglomerateId)) {
+      Fox.successful(mapping.agglomerateToGraph(agglomerateId))
+    } else {
+      remoteDatastoreClient.getAgglomerateGraph(remoteFallbackLayer, agglomerateId, userToken)
+    }
 
   private def findSegmentIdAtPosition(remoteFallbackLayer: RemoteFallbackLayer,
                                       pos: Vec3Int,
@@ -214,7 +245,7 @@ class EditableMappingService @Inject()(
                                          userToken: Option[String]): Fox[Array[Byte]] =
     for {
       editableMapping <- get(editableMappingId, remoteFallbackLayer, userToken)
-      agglomerateIdIsPresent = editableMapping.agglomerateToSegments.contains(agglomerateId)
+      agglomerateIdIsPresent = editableMapping.agglomerateToGraph.contains(agglomerateId)
       skeletonBytes <- if (agglomerateIdIsPresent)
         getAgglomerateSkeleton(editableMappingId, editableMapping, remoteFallbackLayer, agglomerateId)
       else
@@ -229,16 +260,15 @@ class EditableMappingService @Inject()(
                                      remoteFallbackLayer: RemoteFallbackLayer,
                                      agglomerateId: Long): Fox[Array[Byte]] =
     for {
-      positions <- editableMapping.agglomerateToPositions.get(agglomerateId)
-      nodes = positions.zipWithIndex.map {
+      graph <- editableMapping.agglomerateToGraph.get(agglomerateId)
+      nodes = graph.positions.zipWithIndex.map {
         case (pos, idx) =>
           NodeDefaults.createInstance.copy(
             id = idx,
             position = pos
           )
       }
-      edges <- editableMapping.agglomerateToEdges.get(agglomerateId)
-      skeletonEdges = edges.map { e =>
+      skeletonEdges = graph.edges.map { e =>
         Edge(source = e._1.toInt, target = e._2.toInt)
       }
 
@@ -273,7 +303,7 @@ class EditableMappingService @Inject()(
                                            dataRequests: DataRequestCollection): Fox[(Array[Byte], List[Int])] =
     for {
       dataRequestsTyped <- Fox.serialCombined(dataRequests) {
-        case r: WebKnossosDataRequest => Fox.successful(r)
+        case r: WebKnossosDataRequest => Fox.successful(r.copy(applyAgglomerate = None))
         case _                        => Fox.failure("Editable Mappings currently only work for webKnossos data requests")
       }
       (data, indices) <- remoteDatastoreClient.getData(remoteFallbackLayer, dataRequestsTyped)
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
index ff95361e5de..e7875e31637 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
@@ -4,14 +4,10 @@ import com.scalableminds.util.geometry.Vec3Int
 import play.api.libs.json.Format.GenericFormat
 import play.api.libs.json.{Format, JsError, JsResult, JsValue, Json, OFormat}
 
-trait EditableMappingUpdateAction {
-  def applyOn(editableMapping: EditableMapping): EditableMapping
-}
+trait EditableMappingUpdateAction {}
 
 case class SplitAgglomerateUpdateAction(agglomerateId: Long, segmentId1: Long, segmentId2: Long)
-    extends EditableMappingUpdateAction {
-  def applyOn(editableMapping: EditableMapping): EditableMapping = ???
-}
+    extends EditableMappingUpdateAction {}
 
 object SplitAgglomerateUpdateAction {
   implicit val jsonFormat: OFormat[SplitAgglomerateUpdateAction] = Json.format[SplitAgglomerateUpdateAction]
@@ -20,11 +16,8 @@ object SplitAgglomerateUpdateAction {
 case class MergeAgglomerateUpdateAction(agglomerateId1: Long,
                                         agglomerateId2: Long,
                                         segmentId1: Long,
-                                        segmentPosition2: Vec3Int)
-    extends EditableMappingUpdateAction {
-
-  def applyOn(editableMapping: EditableMapping): EditableMapping = ???
-}
+                                        segmentPosition2: Vec3Int) // TODO: needs mag
+    extends EditableMappingUpdateAction {}
 
 object MergeAgglomerateUpdateAction {
   implicit val jsonFormat: OFormat[MergeAgglomerateUpdateAction] = Json.format[MergeAgglomerateUpdateAction]
diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
index 53ba108a64b..85bed33bbe5 100644
--- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
+++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
@@ -21,6 +21,7 @@ POST        /volume/:tracingId/isosurface                       @com.scalablemin
 POST        /volume/:tracingId/importVolumeData                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.importVolumeData(token: Option[String], tracingId: String)
 GET         /volume/:tracingId/findData                         @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.findData(token: Option[String], tracingId: String)
 GET         /volume/:tracingId/agglomerateSkeleton              @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateSkeleton(token: Option[String], tracingId: String, agglomerateId: Long)
+POST        /volume/:tracingId/createEditableMapping            @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.createEditableMapping(token: Option[String], tracingId: String)
 POST        /volume/:tracingId/updateEditableMapping            @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.updateEditableMapping(token: Option[String], tracingId: String, version: Long)
 POST        /volume/getMultiple                                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getMultiple(token: Option[String])
 POST        /volume/mergedFromIds                               @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromIds(token: Option[String], persist: Boolean)

From 0a18d3e13595ca929545825f1e03e61bdfb81646 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 11 May 2022 15:42:35 +0200
Subject: [PATCH 016/122] split update action

---
 .../TSRemoteDatastoreClient.scala             |  2 +
 .../EditableMappingService.scala              | 86 ++++++++++++++++++-
 2 files changed, 86 insertions(+), 2 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
index af23fe2792f..9386102329e 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
@@ -65,4 +65,6 @@ class TSRemoteDatastoreClient @Inject()(
   def getAgglomerateGraph(remoteFallbackLayer: RemoteFallbackLayer,
                           agglomerateId: Long,
                           userToken: Option[String]): Fox[AgglomerateGraph] = ???
+
+  def getLargestAgglomerateId(remoteFallbackLayer: RemoteFallbackLayer, mappingName: String): Fox[Long]
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 3225880287b..4e227f6f7e0 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -20,6 +20,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.{
 import net.liftweb.common.Box.tryo
 import net.liftweb.common.{Empty, Full}
 
+import scala.collection.mutable
 import scala.concurrent.ExecutionContext
 
 class EditableMappingService @Inject()(
@@ -138,7 +139,86 @@ class EditableMappingService @Inject()(
   private def applySplitAction(mapping: EditableMapping,
                                update: SplitAgglomerateUpdateAction,
                                remoteFallbackLayer: RemoteFallbackLayer,
-                               userToken: Option[String]): Fox[EditableMapping] = ???
+                               userToken: Option[String]): Fox[EditableMapping] =
+    for {
+      agglomerateGraph <- agglomerateGraphForId(mapping, update.agglomerateId, remoteFallbackLayer, userToken)
+      largestExistingAgglomerateId <- largestAgglomerateId(mapping, remoteFallbackLayer, userToken)
+      agglomerateId2 = largestExistingAgglomerateId + 1L
+      (graph1, graph2) = splitGraph(agglomerateGraph, update.segmentId1, update.segmentId2)
+      splitSegmentToAgglomerate = graph2.segments.map(_ -> agglomerateId2).toMap
+    } yield
+      EditableMapping(
+        mapping.baseMappingName,
+        segmentToAgglomerate = mapping.segmentToAgglomerate ++ splitSegmentToAgglomerate,
+        agglomerateToGraph = mapping.agglomerateToGraph ++ Map(update.agglomerateId -> graph1, agglomerateId2 -> graph2)
+      )
+
+  private def splitGraph(agglomerateGraph: AgglomerateGraph,
+                         segmentId1: Long,
+                         segmentId2: Long): (AgglomerateGraph, AgglomerateGraph) = {
+    val edgesMinusOne = agglomerateGraph.edges.filter {
+      case (from, to) =>
+        ((from == segmentId1 && to == segmentId2) || (from == segmentId2 && to == segmentId1))
+    }
+    val graph1Nodes: Set[Long] = computeConnectedComponent(startNode = segmentId1, edgesMinusOne)
+    val graph1NodesWithPositions = agglomerateGraph.segments.zip(agglomerateGraph.positions).filter {
+      case (seg, _) => graph1Nodes.contains(seg)
+    }
+    val graph1EdgesWithAffinities = agglomerateGraph.edges.zip(agglomerateGraph.affinities).filter {
+      case (e, _) => graph1Nodes.contains(e._1) && graph1Nodes.contains(e._2)
+    }
+    val graph1 = AgglomerateGraph(
+      segments = graph1NodesWithPositions.map(_._1),
+      edges = graph1EdgesWithAffinities.map(_._1),
+      positions = graph1NodesWithPositions.map(_._2),
+      affinities = graph1EdgesWithAffinities.map(_._2),
+    )
+
+    val graph2Nodes: Set[Long] = agglomerateGraph.segments.toSet.diff(graph2Nodes)
+    val graph2NodesWithPositions = agglomerateGraph.segments.zip(agglomerateGraph.positions).filter {
+      case (seg, _) => graph2Nodes.contains(seg)
+    }
+    val graph2EdgesWithAffinities = agglomerateGraph.edges.zip(agglomerateGraph.affinities).filter {
+      case (e, _) => graph2Nodes.contains(e._1) && graph2Nodes.contains(e._2)
+    }
+    val graph2 = AgglomerateGraph(
+      segments = graph2NodesWithPositions.map(_._1),
+      edges = graph2EdgesWithAffinities.map(_._1),
+      positions = graph2NodesWithPositions.map(_._2),
+      affinities = graph2EdgesWithAffinities.map(_._2),
+    )
+    (graph1, graph2)
+  }
+
+  private def computeConnectedComponent(startNode: Long, edges: List[(Long, Long)]): Set[Long] = {
+    val neighborsByNode =
+      mutable.HashMap[Long, mutable.MutableList[Long]]().withDefaultValue(mutable.MutableList[Long]())
+    edges.foreach {
+      case (from, to) =>
+        neighborsByNode(from) += to
+        neighborsByNode(to) += from
+    }
+    val nodesToVisit = mutable.HashSet[Long](startNode)
+    val visitedNodes = mutable.HashSet[Long]()
+    while (nodesToVisit.nonEmpty) {
+      val node = nodesToVisit.head
+      nodesToVisit -= node
+      if (!visitedNodes.contains(node)) {
+        visitedNodes += node
+        nodesToVisit ++= neighborsByNode(node)
+      }
+    }
+    visitedNodes.toSet
+  }
+
+  private def largestAgglomerateId(mapping: EditableMapping,
+                                   remoteFallbackLayer: RemoteFallbackLayer,
+                                   userToken: Option[String]): Fox[Long] =
+    for {
+      largestBaseAgglomerateId <- remoteDatastoreClient.getLargestAgglomerateId(remoteFallbackLayer,
+                                                                                mapping.baseMappingName)
+      keySet = mapping.agglomerateToGraph.keySet
+    } yield math.max(if (keySet.isEmpty) 0L else keySet.max, largestBaseAgglomerateId)
 
   private def applyMergeAction(mapping: EditableMapping,
                                update: MergeAgglomerateUpdateAction,
@@ -150,7 +230,9 @@ class EditableMappingService @Inject()(
       agglomerateGraph2 <- agglomerateGraphForId(mapping, update.agglomerateId2, remoteFallbackLayer, userToken)
       mergedGraph = mergeGraph(agglomerateGraph1, agglomerateGraph2, update.segmentId1, segmentId2)
       _ <- bool2Fox(agglomerateGraph2.segments.contains(segmentId2)) ?~> "segment as queried by position is not contained in fetched agglomerate graph"
-      mergedSegmentToAgglomerate: Map[Long, Long] = agglomerateGraph2.segments.map(s => s -> update.agglomerateId1)
+      mergedSegmentToAgglomerate: Map[Long, Long] = agglomerateGraph2.segments
+        .map(s => s -> update.agglomerateId1)
+        .toMap
     } yield
       EditableMapping(
         baseMappingName = mapping.baseMappingName,

From 5b4497eabab0cc6447900df6a1f1e09c3c80c208 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Wed, 11 May 2022 16:47:23 +0200
Subject: [PATCH 017/122] enable bounding box tool in skeleton-only annotations

---
 .../oxalis/model/reducers/reducer_helpers.ts  |  4 +--
 .../oxalis/view/action-bar/toolbar_view.tsx   | 32 +++++++++----------
 2 files changed, 16 insertions(+), 20 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/reducers/reducer_helpers.ts b/frontend/javascripts/oxalis/model/reducers/reducer_helpers.ts
index 32267684d0b..0fc5291673d 100644
--- a/frontend/javascripts/oxalis/model/reducers/reducer_helpers.ts
+++ b/frontend/javascripts/oxalis/model/reducers/reducer_helpers.ts
@@ -122,7 +122,7 @@ export function convertServerAnnotationToFrontendAnnotation(annotation: APIAnnot
 }
 export function getNextTool(state: OxalisState): AnnotationTool | null {
   const disabledToolInfo = getDisabledInfoForTools(state);
-  const tools = Object.keys(AnnotationToolEnum);
+  const tools = Object.keys(AnnotationToolEnum) as AnnotationTool[];
   const currentToolIndex = tools.indexOf(state.uiInformation.activeTool);
 
   // Search for the next tool which is not disabled.
@@ -133,9 +133,7 @@ export function getNextTool(state: OxalisState): AnnotationTool | null {
   ) {
     const newTool = tools[newToolIndex % tools.length];
 
-    // @ts-expect-error ts-migrate(7053) FIXME: Element implicitly has an 'any' type because expre... Remove this comment to see the full error message
     if (!disabledToolInfo[newTool].isDisabled) {
-      // @ts-expect-error ts-migrate(2322) FIXME: Type 'string' is not assignable to type '"TRACE" |... Remove this comment to see the full error message
       return newTool;
     }
   }
diff --git a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
index 7f55596c23f..8500090beab 100644
--- a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
+++ b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
@@ -663,25 +663,23 @@ export default function ToolbarView() {
                 }}
               />
             </RadioButtonWithTooltip>
-            <RadioButtonWithTooltip
-              title="Bounding Box Tool - Create, resize and modify bounding boxes."
-              disabled={false}
-              style={narrowButtonStyle}
-              value={AnnotationToolEnum.BOUNDING_BOX}
-            >
-              <img
-                src="/assets/images/bounding-box.svg"
-                alt="Bounding Box Icon"
-                style={{
-                  opacity: disabledInfosForTools[AnnotationToolEnum.BOUNDING_BOX].isDisabled
-                    ? 0.5
-                    : 1,
-                  ...imgStyleForSpaceyIcons,
-                }}
-              />
-            </RadioButtonWithTooltip>
           </React.Fragment>
         ) : null}
+        <RadioButtonWithTooltip
+          title="Bounding Box Tool - Create, resize and modify bounding boxes."
+          disabled={false}
+          style={narrowButtonStyle}
+          value={AnnotationToolEnum.BOUNDING_BOX}
+        >
+          <img
+            src="/assets/images/bounding-box.svg"
+            alt="Bounding Box Icon"
+            style={{
+              opacity: disabledInfosForTools[AnnotationToolEnum.BOUNDING_BOX].isDisabled ? 0.5 : 1,
+              ...imgStyleForSpaceyIcons,
+            }}
+          />
+        </RadioButtonWithTooltip>
       </Radio.Group>
 
       <ToolSpecificSettings

From 1728beae8f3f44b2577b33d4f2db18b624bda720 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Wed, 11 May 2022 20:13:56 +0200
Subject: [PATCH 018/122] avoid loading agglomerate skeletons twice

---
 .../oxalis/model/sagas/skeletontracing_saga.ts        | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
index 063e7b8a500..83aad9ecafd 100644
--- a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
@@ -309,6 +309,17 @@ function* loadAgglomerateSkeletonWithId(action: LoadAgglomerateSkeletonAction):
     return;
   }
 
+  const treeName = getTreeNameForAgglomerateSkeleton(agglomerateId, mappingName);
+  const trees = yield* select((state) => enforceSkeletonTracing(state.tracing).trees);
+  const maybeTree = findTreeByName(trees, treeName).getOrElse(null);
+
+  if (maybeTree != null) {
+    Toast.info(
+      `Skeleton for agglomerate ${agglomerateId} with mapping ${mappingName} is already loaded. Its tree name is "${treeName}".`,
+    );
+    return;
+  }
+
   const progressCallback = createProgressCallback({
     pauseDelay: 100,
     successMessageDelay: 2000,

From d02e74ab3f33bdc412a96928554b16277c03a756 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Wed, 11 May 2022 20:28:15 +0200
Subject: [PATCH 019/122] add proofread tool, add merge and split agglomerate
 update actions, clean up update actions and add missing descriptions for
 version view, add first version of proofreading saga

---
 frontend/javascripts/oxalis/constants.ts      |   1 +
 .../controller/combinations/tool_controls.ts  |  25 +++++
 .../controller/viewmodes/plane_controller.tsx |   3 +
 .../oxalis/model/accessors/tool_accessor.ts   |   8 ++
 .../model/actions/skeletontracing_actions.tsx |   4 +-
 .../oxalis/model/sagas/proofread_saga.ts      |  63 +++++++++++
 .../oxalis/model/sagas/update_actions.ts      | 106 +++++++++++++-----
 .../oxalis/model/sagas/volumetracing_saga.tsx |  16 +--
 .../oxalis/view/action-bar/toolbar_view.tsx   |  53 ++++++---
 .../javascripts/oxalis/view/input_catcher.tsx |   1 +
 .../javascripts/oxalis/view/version_entry.tsx |  52 ++++++++-
 11 files changed, 271 insertions(+), 61 deletions(-)
 create mode 100644 frontend/javascripts/oxalis/model/sagas/proofread_saga.ts

diff --git a/frontend/javascripts/oxalis/constants.ts b/frontend/javascripts/oxalis/constants.ts
index e6a28110b5e..a9ed16065af 100644
--- a/frontend/javascripts/oxalis/constants.ts
+++ b/frontend/javascripts/oxalis/constants.ts
@@ -180,6 +180,7 @@ export enum AnnotationToolEnum {
   FILL_CELL = "FILL_CELL",
   PICK_CELL = "PICK_CELL",
   BOUNDING_BOX = "BOUNDING_BOX",
+  PROOFREAD = "PROOFREAD",
 }
 export const VolumeTools: Array<keyof typeof AnnotationToolEnum> = [
   AnnotationToolEnum.BRUSH,
diff --git a/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts b/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts
index cd31205b2e2..940d9d6be7a 100644
--- a/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts
+++ b/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts
@@ -617,10 +617,35 @@ export class BoundingBoxTool {
     getSceneController().highlightUserBoundingBox(null);
   }
 }
+export class ProofreadTool {
+  static getPlaneMouseControls(_planeId: OrthoView): any {
+    return {
+      leftClick: (pos: Point2, _plane: OrthoView, _event: MouseEvent) => {
+        handleAgglomerateSkeletonAtClick(pos);
+      },
+    };
+  }
+
+  static getActionDescriptors(
+    _activeTool: AnnotationTool,
+    _useLegacyBindings: boolean,
+    _shiftKey: boolean,
+    _ctrlKey: boolean,
+    _altKey: boolean,
+  ): ActionDescriptor {
+    return {
+      leftClick: "Select Segment to Proofread",
+      rightClick: "Context Menu",
+    };
+  }
+
+  static onToolDeselected() {}
+}
 const toolToToolClass = {
   [AnnotationToolEnum.MOVE]: MoveTool,
   [AnnotationToolEnum.SKELETON]: SkeletonTool,
   [AnnotationToolEnum.BOUNDING_BOX]: BoundingBoxTool,
+  [AnnotationToolEnum.PROOFREAD]: ProofreadTool,
   [AnnotationToolEnum.BRUSH]: DrawTool,
   [AnnotationToolEnum.TRACE]: DrawTool,
   [AnnotationToolEnum.ERASE_TRACE]: EraseTool,
diff --git a/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx b/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx
index baa6124a534..b7e5b646d34 100644
--- a/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx
+++ b/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx
@@ -41,6 +41,7 @@ import {
   PickCellTool,
   FillCellTool,
   BoundingBoxTool,
+  ProofreadTool,
 } from "oxalis/controller/combinations/tool_controls";
 import type {
   ShowContextMenuFunction,
@@ -301,6 +302,7 @@ class PlaneController extends React.PureComponent<Props> {
       this.planeView,
       this.props.showContextMenuAt,
     );
+    const proofreadControls = ProofreadTool.getPlaneMouseControls(planeId);
 
     const allControlKeys = _.union(
       Object.keys(moveControls),
@@ -326,6 +328,7 @@ class PlaneController extends React.PureComponent<Props> {
         [AnnotationToolEnum.PICK_CELL]: pickCellControls[controlKey],
         [AnnotationToolEnum.FILL_CELL]: fillCellControls[controlKey],
         [AnnotationToolEnum.BOUNDING_BOX]: boundingBoxControls[controlKey],
+        [AnnotationToolEnum.PROOFREAD]: proofreadControls[controlKey],
       });
     }
 
diff --git a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
index 8a2b6c4d9bb..f1c175d9bd4 100644
--- a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
+++ b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
@@ -82,6 +82,10 @@ function _getDisabledInfoWhenVolumeIsDisabled(
     [AnnotationToolEnum.FILL_CELL]: disabledInfo,
     [AnnotationToolEnum.PICK_CELL]: disabledInfo,
     [AnnotationToolEnum.BOUNDING_BOX]: notDisabledInfo,
+    [AnnotationToolEnum.PROOFREAD]: {
+      isDisabled: !hasSkeleton,
+      explanation: disabledSkeletonExplanation,
+    },
   };
 }
 
@@ -131,6 +135,10 @@ function _getDisabledInfoFromArgs(
       isDisabled: false,
       explanation: disabledSkeletonExplanation,
     },
+    [AnnotationToolEnum.PROOFREAD]: {
+      isDisabled: !hasSkeleton,
+      explanation: disabledSkeletonExplanation,
+    },
   };
 }
 
diff --git a/frontend/javascripts/oxalis/model/actions/skeletontracing_actions.tsx b/frontend/javascripts/oxalis/model/actions/skeletontracing_actions.tsx
index c08942fccda..fb1fd2cfb43 100644
--- a/frontend/javascripts/oxalis/model/actions/skeletontracing_actions.tsx
+++ b/frontend/javascripts/oxalis/model/actions/skeletontracing_actions.tsx
@@ -33,7 +33,7 @@ type DeleteNodeAction = {
   treeId?: number;
   timestamp: number;
 };
-type DeleteEdgeAction = {
+export type DeleteEdgeAction = {
   type: "DELETE_EDGE";
   sourceNodeId: number;
   targetNodeId: number;
@@ -133,7 +133,7 @@ type SetActiveGroupAction = {
 type DeselectActiveGroupAction = {
   type: "DESELECT_ACTIVE_GROUP";
 };
-type MergeTreesAction = {
+export type MergeTreesAction = {
   type: "MERGE_TREES";
   sourceNodeId: number;
   targetNodeId: number;
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
new file mode 100644
index 00000000000..348060d2b50
--- /dev/null
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -0,0 +1,63 @@
+import type { Saga } from "oxalis/model/sagas/effect-generators";
+import { takeEvery, put } from "typed-redux-saga";
+import { select, take } from "oxalis/model/sagas/effect-generators";
+import { AnnotationToolEnum } from "oxalis/constants";
+import Toast from "libs/toast";
+import type {
+  DeleteEdgeAction,
+  MergeTreesAction,
+} from "oxalis/model/actions/skeletontracing_actions";
+import {
+  enforceSkeletonTracing,
+  findTreeByNodeId,
+} from "oxalis/model/accessors/skeletontracing_accessor";
+import { pushSaveQueueTransaction } from "oxalis/model/actions/save_actions";
+import { splitAgglomerate, mergeAgglomerates } from "oxalis/model/sagas/update_actions";
+
+export default function* proofreadMapping(): Saga<any> {
+  yield* take("INITIALIZE_SKELETONTRACING");
+  yield* take("WK_READY");
+  yield* takeEvery(["DELETE_EDGE", "MERGE_TREES"], splitOrMergeAgglomerates);
+}
+
+function* splitOrMergeAgglomerates(action: MergeTreesAction | DeleteEdgeAction) {
+  const allowUpdate = yield* select((state) => state.tracing.restrictions.allowUpdate);
+  if (!allowUpdate) return;
+
+  const activeTool = yield* select((state) => state.uiInformation.activeTool);
+  if (activeTool !== AnnotationToolEnum.PROOFREAD) return;
+
+  const { sourceNodeId, targetNodeId } = action;
+
+  const skeletonTracing = yield* select((state) => enforceSkeletonTracing(state.tracing));
+
+  const { trees, tracingId, type: tracingType } = skeletonTracing;
+  const sourceTree = findTreeByNodeId(trees, sourceNodeId).getOrElse(null);
+  const targetTree = findTreeByNodeId(trees, targetNodeId).getOrElse(null);
+
+  if (sourceTree == null || targetTree == null) {
+    return;
+  }
+
+  const sourceNodePosition = sourceTree.nodes.get(sourceNodeId).position;
+  const targetNodePosition = targetTree.nodes.get(targetNodeId).position;
+
+  const items = [];
+  if (action.type === "MERGE_TREES") {
+    if (sourceTree === targetTree) {
+      Toast.error("Segments that should be merged need to be in different agglomerates.");
+      return;
+    }
+    items.push(mergeAgglomerates(sourceNodePosition, targetNodePosition));
+  } else if (action.type === "DELETE_EDGE") {
+    if (sourceTree !== targetTree) {
+      Toast.error("Segments that should be split need to be in the same agglomerate.");
+      return;
+    }
+    items.push(splitAgglomerate(sourceNodePosition, targetNodePosition));
+  }
+
+  if (items.length) {
+    yield* put(pushSaveQueueTransaction(items, tracingType, tracingId));
+  }
+}
diff --git a/frontend/javascripts/oxalis/model/sagas/update_actions.ts b/frontend/javascripts/oxalis/model/sagas/update_actions.ts
index 542f7efb9f2..c1470906a12 100644
--- a/frontend/javascripts/oxalis/model/sagas/update_actions.ts
+++ b/frontend/javascripts/oxalis/model/sagas/update_actions.ts
@@ -34,7 +34,7 @@ export type DeleteTreeUpdateAction = {
     id: number;
   };
 };
-type MoveTreeComponentUpdateAction = {
+export type MoveTreeComponentUpdateAction = {
   name: "moveTreeComponent";
   value: {
     sourceId: number;
@@ -42,7 +42,7 @@ type MoveTreeComponentUpdateAction = {
     nodeIds: Array<number>;
   };
 };
-type MergeTreeUpdateAction = {
+export type MergeTreeUpdateAction = {
   name: "mergeTree";
   value: {
     sourceId: number;
@@ -113,7 +113,7 @@ type UpdateVolumeTracingUpdateAction = {
     zoomLevel: number;
   };
 };
-type CreateSegmentVolumeAction = {
+export type CreateSegmentUpdateAction = {
   name: "createSegment";
   value: {
     id: number;
@@ -122,7 +122,7 @@ type CreateSegmentVolumeAction = {
     creationTime: number | null | undefined;
   };
 };
-type UpdateSegmentVolumeAction = {
+export type UpdateSegmentUpdateAction = {
   name: "updateSegment";
   value: {
     id: number;
@@ -131,13 +131,13 @@ type UpdateSegmentVolumeAction = {
     creationTime: number | null | undefined;
   };
 };
-type DeleteSegmentVolumeAction = {
+export type DeleteSegmentUpdateAction = {
   name: "deleteSegment";
   value: {
     id: number;
   };
 };
-type UpdateUserBoundingBoxesAction = {
+type UpdateUserBoundingBoxesUpdateAction = {
   name: "updateUserBoundingBoxes";
   value: {
     boundingBoxes: Array<UserBoundingBoxToServer>;
@@ -163,14 +163,28 @@ export type RevertToVersionUpdateAction = {
 };
 // This action is not dispatched by our code, anymore,
 // but we still need to keep it for backwards compatibility.
-export type RemoveFallbackLayerAction = {
+export type RemoveFallbackLayerUpdateAction = {
   name: "removeFallbackLayer";
   value: {};
 };
-export type UpdateTdCameraAction = {
+export type UpdateTdCameraUpdateAction = {
   name: "updateTdCamera";
   value: {};
 };
+export type SplitAgglomerateUpdateAction = {
+  name: "splitAgglomerate";
+  value: {
+    segment_1_position: Vector3;
+    segment_2_position: Vector3;
+  };
+};
+export type MergeAgglomeratesUpdateAction = {
+  name: "mergeAgglomerates";
+  value: {
+    segment_1_position: Vector3;
+    segment_2_position: Vector3;
+  };
+};
 export type UpdateAction =
   | UpdateTreeUpdateAction
   | DeleteTreeUpdateAction
@@ -183,22 +197,31 @@ export type UpdateAction =
   | DeleteEdgeUpdateAction
   | UpdateSkeletonTracingUpdateAction
   | UpdateVolumeTracingUpdateAction
-  | UpdateUserBoundingBoxesAction
-  | CreateSegmentVolumeAction
-  | UpdateSegmentVolumeAction
-  | DeleteSegmentVolumeAction
+  | UpdateUserBoundingBoxesUpdateAction
+  | CreateSegmentUpdateAction
+  | UpdateSegmentUpdateAction
+  | DeleteSegmentUpdateAction
   | UpdateBucketUpdateAction
   | UpdateTreeVisibilityUpdateAction
   | UpdateTreeGroupVisibilityUpdateAction
   | RevertToVersionUpdateAction
   | UpdateTreeGroupsUpdateAction
-  | RemoveFallbackLayerAction
-  | UpdateTdCameraAction;
+  | RemoveFallbackLayerUpdateAction
+  | UpdateTdCameraUpdateAction
+  | SplitAgglomerateUpdateAction
+  | MergeAgglomeratesUpdateAction;
 // This update action is only created in the frontend for display purposes
 type CreateTracingUpdateAction = {
   name: "createTracing";
   value: {};
 };
+// This update action is only created by the backend
+type ImportVolumeTracingUpdateAction = {
+  name: "importVolumeTracing";
+  value: {
+    largestSegmentId: number;
+  };
+};
 type AddServerValuesFn<T extends { value: any }> = (arg0: T) => T & {
   value: T["value"] & {
     actionTimestamp: number;
@@ -220,18 +243,21 @@ export type ServerUpdateAction =
   | AsServerAction<DeleteEdgeUpdateAction>
   | AsServerAction<UpdateSkeletonTracingUpdateAction>
   | AsServerAction<UpdateVolumeTracingUpdateAction>
-  | AsServerAction<UpdateUserBoundingBoxesAction>
-  | AsServerAction<CreateSegmentVolumeAction>
-  | AsServerAction<UpdateSegmentVolumeAction>
-  | AsServerAction<DeleteSegmentVolumeAction>
+  | AsServerAction<UpdateUserBoundingBoxesUpdateAction>
+  | AsServerAction<CreateSegmentUpdateAction>
+  | AsServerAction<UpdateSegmentUpdateAction>
+  | AsServerAction<DeleteSegmentUpdateAction>
   | AsServerAction<UpdateBucketUpdateAction>
   | AsServerAction<UpdateTreeVisibilityUpdateAction>
   | AsServerAction<UpdateTreeGroupVisibilityUpdateAction>
   | AsServerAction<RevertToVersionUpdateAction>
   | AsServerAction<UpdateTreeGroupsUpdateAction>
   | AsServerAction<CreateTracingUpdateAction>
-  | AsServerAction<RemoveFallbackLayerAction>
-  | AsServerAction<UpdateTdCameraAction>;
+  | AsServerAction<RemoveFallbackLayerUpdateAction>
+  | AsServerAction<UpdateTdCameraUpdateAction>
+  | AsServerAction<ImportVolumeTracingUpdateAction>
+  | AsServerAction<SplitAgglomerateUpdateAction>
+  | AsServerAction<MergeAgglomeratesUpdateAction>;
 export function createTree(tree: Tree): UpdateTreeUpdateAction {
   return {
     name: "createTree",
@@ -407,7 +433,7 @@ export function updateVolumeTracing(
 }
 export function updateUserBoundingBoxes(
   userBoundingBoxes: Array<UserBoundingBox>,
-): UpdateUserBoundingBoxesAction {
+): UpdateUserBoundingBoxesUpdateAction {
   return {
     name: "updateUserBoundingBoxes",
     value: {
@@ -415,12 +441,12 @@ export function updateUserBoundingBoxes(
     },
   };
 }
-export function createSegmentVolumeAction(
+export function createSegmentUpdateAction(
   id: number,
   anchorPosition: Vector3 | null | undefined,
   name: string | null | undefined,
   creationTime: number | null | undefined,
-): CreateSegmentVolumeAction {
+): CreateSegmentUpdateAction {
   return {
     name: "createSegment",
     value: {
@@ -431,12 +457,12 @@ export function createSegmentVolumeAction(
     },
   };
 }
-export function updateSegmentVolumeAction(
+export function updateSegmentUpdateAction(
   id: number,
   anchorPosition: Vector3 | null | undefined,
   name: string | null | undefined,
   creationTime: number | null | undefined,
-): UpdateSegmentVolumeAction {
+): UpdateSegmentUpdateAction {
   return {
     name: "updateSegment",
     value: {
@@ -447,7 +473,7 @@ export function updateSegmentVolumeAction(
     },
   };
 }
-export function deleteSegmentVolumeAction(id: number) {
+export function deleteSegmentUpdateAction(id: number) {
   return {
     name: "deleteSegment",
     value: {
@@ -482,13 +508,13 @@ export function revertToVersion(version: number): RevertToVersionUpdateAction {
     },
   };
 }
-export function removeFallbackLayer(): RemoveFallbackLayerAction {
+export function removeFallbackLayer(): RemoveFallbackLayerUpdateAction {
   return {
     name: "removeFallbackLayer",
     value: {},
   };
 }
-export function updateTdCamera(): UpdateTdCameraAction {
+export function updateTdCamera(): UpdateTdCameraUpdateAction {
   return {
     name: "updateTdCamera",
     value: {},
@@ -502,3 +528,27 @@ export function serverCreateTracing(timestamp: number) {
     },
   };
 }
+export function splitAgglomerate(
+  segment_1_position: Vector3,
+  segment_2_position: Vector3,
+): SplitAgglomerateUpdateAction {
+  return {
+    name: "splitAgglomerate",
+    value: {
+      segment_1_position,
+      segment_2_position,
+    },
+  };
+}
+export function mergeAgglomerates(
+  segment_1_position: Vector3,
+  segment_2_position: Vector3,
+): MergeAgglomeratesUpdateAction {
+  return {
+    name: "mergeAgglomerates",
+    value: {
+      segment_1_position,
+      segment_2_position,
+    },
+  };
+}
diff --git a/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx b/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx
index 7a72f9adae7..2430b3dcd24 100644
--- a/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx
+++ b/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx
@@ -15,9 +15,9 @@ import type { UpdateAction } from "oxalis/model/sagas/update_actions";
 import {
   updateVolumeTracing,
   updateUserBoundingBoxes,
-  createSegmentVolumeAction,
-  updateSegmentVolumeAction,
-  deleteSegmentVolumeAction,
+  createSegmentUpdateAction,
+  updateSegmentUpdateAction,
+  deleteSegmentUpdateAction,
   removeFallbackLayer,
 } from "oxalis/model/sagas/update_actions";
 import type { UpdateTemporarySettingAction } from "oxalis/model/actions/settings_actions";
@@ -880,13 +880,13 @@ export function* diffSegmentLists(
 
   for (const segmentId of deletedSegmentIds) {
     // @ts-expect-error ts-migrate(2322) FIXME: Type '{ name: string; value: { id: number; }; }' i... Remove this comment to see the full error message
-    yield deleteSegmentVolumeAction(segmentId);
+    yield deleteSegmentUpdateAction(segmentId);
   }
 
   for (const segmentId of addedSegmentIds) {
     const segment = newSegments.get(segmentId);
     // @ts-expect-error ts-migrate(2554) FIXME: Expected 4 arguments, but got 3.
-    yield createSegmentVolumeAction(segment.id, segment.somePosition, segment.name);
+    yield createSegmentUpdateAction(segment.id, segment.somePosition, segment.name);
   }
 
   for (const segmentId of bothSegmentIds) {
@@ -894,7 +894,7 @@ export function* diffSegmentLists(
     const prevSegment = prevSegments.get(segmentId);
 
     if (segment !== prevSegment) {
-      yield updateSegmentVolumeAction(
+      yield updateSegmentUpdateAction(
         segment.id,
         segment.somePosition,
         segment.name,
@@ -1107,8 +1107,8 @@ function* ensureValidBrushSize(): Saga<void> {
   yield* takeLatest(
     [
       "WK_READY",
-      // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'action' implicitly has an 'any' type.
-      (action) => action.type === "UPDATE_LAYER_SETTING" && action.propertyName === "isDisabled",
+      (action: Action) =>
+        action.type === "UPDATE_LAYER_SETTING" && action.propertyName === "isDisabled",
     ],
     maybeClampBrushSize,
   );
diff --git a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
index 8500090beab..df042e362df 100644
--- a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
+++ b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
@@ -519,31 +519,49 @@ export default function ToolbarView() {
         </RadioButtonWithTooltip>
 
         {hasSkeleton ? (
-          <RadioButtonWithTooltip
-            title={skeletonToolDescription}
-            disabledTitle=""
-            disabled={disabledInfosForTools[AnnotationToolEnum.SKELETON].isDisabled}
-            style={narrowButtonStyle}
-            value={AnnotationToolEnum.SKELETON}
-          >
-            {/*
+          <>
+            <RadioButtonWithTooltip
+              title={skeletonToolDescription}
+              disabledTitle=""
+              disabled={disabledInfosForTools[AnnotationToolEnum.SKELETON].isDisabled}
+              style={narrowButtonStyle}
+              value={AnnotationToolEnum.SKELETON}
+            >
+              {/*
            When visible changes to false, the tooltip fades out in an animation. However, skeletonToolHint
            will be null, too, which means the tooltip text would immediately change to an empty string.
            To avoid this, we fallback to previousSkeletonToolHint.
           */}
-            <Tooltip
-              title={skeletonToolHint || previousSkeletonToolHint}
-              visible={skeletonToolHint != null}
+              <Tooltip
+                title={skeletonToolHint || previousSkeletonToolHint}
+                visible={skeletonToolHint != null}
+              >
+                <i
+                  style={{
+                    paddingLeft: 4,
+                    opacity: disabledInfosForTools[AnnotationToolEnum.SKELETON].isDisabled
+                      ? 0.5
+                      : 1,
+                  }}
+                  className="fas fa-project-diagram"
+                />
+              </Tooltip>
+            </RadioButtonWithTooltip>
+            <RadioButtonWithTooltip
+              title="Proofreading Tool - Modify an agglomerated segmentation."
+              disabledTitle={disabledInfosForTools[AnnotationToolEnum.PROOFREAD].explanation}
+              disabled={disabledInfosForTools[AnnotationToolEnum.PROOFREAD].isDisabled}
+              style={narrowButtonStyle}
+              value={AnnotationToolEnum.PROOFREAD}
             >
               <i
+                className="fas fa-clipboard-check"
                 style={{
-                  paddingLeft: 4,
-                  opacity: disabledInfosForTools[AnnotationToolEnum.SKELETON].isDisabled ? 0.5 : 1,
+                  opacity: disabledInfosForTools[AnnotationToolEnum.PROOFREAD].isDisabled ? 0.5 : 1,
                 }}
-                className="fas fa-project-diagram"
               />
-            </Tooltip>
-          </RadioButtonWithTooltip>
+            </RadioButtonWithTooltip>
+          </>
         ) : null}
 
         {isVolumeSupported ? (
@@ -667,7 +685,8 @@ export default function ToolbarView() {
         ) : null}
         <RadioButtonWithTooltip
           title="Bounding Box Tool - Create, resize and modify bounding boxes."
-          disabled={false}
+          disabledTitle={disabledInfosForTools[AnnotationToolEnum.BOUNDING_BOX].explanation}
+          disabled={disabledInfosForTools[AnnotationToolEnum.BOUNDING_BOX].isDisabled}
           style={narrowButtonStyle}
           value={AnnotationToolEnum.BOUNDING_BOX}
         >
diff --git a/frontend/javascripts/oxalis/view/input_catcher.tsx b/frontend/javascripts/oxalis/view/input_catcher.tsx
index 176e984211d..0331d3aaea1 100644
--- a/frontend/javascripts/oxalis/view/input_catcher.tsx
+++ b/frontend/javascripts/oxalis/view/input_catcher.tsx
@@ -94,6 +94,7 @@ const cursorForTool = {
   FILL_CELL: "url(/assets/images/fill-pointed-solid-border.svg) 0 16,auto",
   PICK_CELL: "url(/assets/images/eye-dropper-solid-border.svg) 0 12,auto",
   BOUNDING_BOX: "move",
+  PROOFREAD: "crosshair",
 };
 
 function InputCatcher({
diff --git a/frontend/javascripts/oxalis/view/version_entry.tsx b/frontend/javascripts/oxalis/view/version_entry.tsx
index ea908e3fee0..74b1753e59a 100644
--- a/frontend/javascripts/oxalis/view/version_entry.tsx
+++ b/frontend/javascripts/oxalis/view/version_entry.tsx
@@ -28,6 +28,13 @@ import type {
   UpdateTreeGroupVisibilityUpdateAction,
   CreateEdgeUpdateAction,
   DeleteEdgeUpdateAction,
+  SplitAgglomerateUpdateAction,
+  MergeAgglomeratesUpdateAction,
+  CreateSegmentUpdateAction,
+  UpdateSegmentUpdateAction,
+  DeleteSegmentUpdateAction,
+  MoveTreeComponentUpdateAction,
+  MergeTreeUpdateAction,
 } from "oxalis/model/sagas/update_actions";
 import FormattedDate from "components/formatted_date";
 import { MISSING_GROUP_ID } from "oxalis/view/right-border-tabs/tree_hierarchy_view_helpers";
@@ -35,11 +42,15 @@ type Description = {
   description: string;
   icon: React.ReactNode;
 };
+const updateTracingDescription = {
+  description: "Modified the annotation.",
+  icon: <EditOutlined />,
+};
 // The order in which the update actions are added to this object,
 // determines the order in which update actions are checked
 // to describe an update action batch. See also the comment
 // of the `getDescriptionForBatch` function.
-const descriptionFns = {
+const descriptionFns: Record<ServerUpdateAction["name"], (...args: any) => Description> = {
   importVolumeTracing: (): Description => ({
     description: "Imported a volume tracing.",
     icon: <PlusOutlined />,
@@ -56,6 +67,14 @@ const descriptionFns = {
     description: "Removed the segmentation fallback layer.",
     icon: <DeleteOutlined />,
   }),
+  splitAgglomerate: (action: SplitAgglomerateUpdateAction): Description => ({
+    description: `Split an agglomerate by separating the segments at position ${action.value.segment_1_position} and ${action.value.segment_2_position}.`,
+    icon: <DeleteOutlined />,
+  }),
+  mergeAgglomerates: (action: MergeAgglomeratesUpdateAction): Description => ({
+    description: `Merged two agglomerates by combining the segments at position ${action.value.segment_1_position} and ${action.value.segment_2_position}.`,
+    icon: <PlusOutlined />,
+  }),
   deleteTree: (action: DeleteTreeUpdateAction, count: number): Description => ({
     description:
       count > 1 ? `Deleted ${count} trees.` : `Deleted the tree with id ${action.value.id}.`,
@@ -116,6 +135,31 @@ const descriptionFns = {
     description: "Updated the 3D view.",
     icon: <CodeSandboxOutlined />,
   }),
+  createSegment: (action: CreateSegmentUpdateAction): Description => ({
+    description: `Added the segment with id ${action.value.id} to the segments list.`,
+    icon: <PlusOutlined />,
+  }),
+  updateSegment: (action: UpdateSegmentUpdateAction): Description => ({
+    description: `Updated the segment with id ${action.value.id} in the segments list.`,
+    icon: <EditOutlined />,
+  }),
+  deleteSegment: (action: DeleteSegmentUpdateAction): Description => ({
+    description: `Deleted the segment with id ${action.value.id} from the segments list.`,
+    icon: <DeleteOutlined />,
+  }),
+  // This should never be shown since currently this update action can only be triggered
+  // by merging or splitting trees which is recognized separately, before this description
+  // is accessed.
+  moveTreeComponent: (action: MoveTreeComponentUpdateAction): Description => ({
+    description: `Moved ${action.value.nodeIds.length} nodes from tree with id ${action.value.sourceId} to tree with id ${action.value.targetId}.`,
+    icon: <EditOutlined />,
+  }),
+  // This should never be shown since currently this update action is never dispatched.
+  mergeTree: (action: MergeTreeUpdateAction): Description => ({
+    description: `Merged the trees with id ${action.value.sourceId} and ${action.value.targetId}.`,
+    icon: <EditOutlined />,
+  }),
+  updateTracing: (): Description => updateTracingDescription,
 };
 
 function getDescriptionForSpecificBatch(
@@ -128,7 +172,6 @@ function getDescriptionForSpecificBatch(
     throw new Error("Flow constraint violated");
   }
 
-  // @ts-expect-error ts-migrate(7053) FIXME: Element implicitly has an 'any' type because expre... Remove this comment to see the full error message
   return descriptionFns[type](firstAction, actions.length);
 }
 
@@ -198,10 +241,7 @@ function getDescriptionForBatch(actions: Array<ServerUpdateAction>): Description
   }
 
   // Catch-all for other update actions, currently updateTracing.
-  return {
-    description: "Modified the annotation.",
-    icon: <EditOutlined />,
-  };
+  return updateTracingDescription;
 }
 
 type Props = {

From 043c6de13110086dd479a34d086626f049ac49d3 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Thu, 12 May 2022 12:18:16 +0200
Subject: [PATCH 020/122] fix json serialization, create stubs for querying
 agglomerate graph + largest agglomerate id

---
 .../scalableminds/util/tools/JsonHelper.scala | 109 +++++++++++-------
 .../controllers/DataSourceController.scala    |  45 ++++++++
 .../datastore/models}/AgglomerateGraph.scala  |   2 +-
 .../services/AgglomerateService.scala         |  26 +++++
 ....scalableminds.webknossos.datastore.routes |   2 +
 .../TSRemoteDatastoreClient.scala             |  17 ++-
 .../controllers/VolumeTracingController.scala |   9 +-
 .../editablemapping/EditableMapping.scala     |   4 +-
 .../EditableMappingService.scala              |  47 +++++---
 9 files changed, 188 insertions(+), 73 deletions(-)
 rename {webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping => webknossos-datastore/app/com/scalableminds/webknossos/datastore/models}/AgglomerateGraph.scala (87%)

diff --git a/util/src/main/scala/com/scalableminds/util/tools/JsonHelper.scala b/util/src/main/scala/com/scalableminds/util/tools/JsonHelper.scala
index b703b108137..5b512ef6b02 100644
--- a/util/src/main/scala/com/scalableminds/util/tools/JsonHelper.scala
+++ b/util/src/main/scala/com/scalableminds/util/tools/JsonHelper.scala
@@ -7,15 +7,80 @@ import com.scalableminds.util.io.FileIO
 import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common._
 import play.api.i18n.Messages
-import play.api.libs.json._
 import play.api.libs.json.Reads._
 import play.api.libs.json.Writes._
-import scala.concurrent.ExecutionContext.Implicits._
+import play.api.libs.json._
 
+import scala.concurrent.ExecutionContext.Implicits._
 import scala.concurrent.duration._
 import scala.io.{BufferedSource, Source}
 
-object JsonHelper extends BoxImplicits with LazyLogging {
+trait AdditionalJsonFormats {
+  implicit def boxFormat[T: Format]: Format[Box[T]] = new Format[Box[T]] {
+    override def reads(json: JsValue): JsResult[Box[T]] =
+      (json \ "status").validate[String].flatMap {
+        case "Full"    => (json \ "value").validate[T].map(Full(_))
+        case "Empty"   => JsSuccess(Empty)
+        case "Failure" => (json \ "value").validate[String].map(Failure(_))
+        case _         => JsError("invalid status")
+      }
+
+    override def writes(o: Box[T]): JsValue = o match {
+      case Full(t)    => Json.obj("status" -> "Full", "value" -> Json.toJson(t))
+      case Empty      => Json.obj("status" -> "Empty")
+      case f: Failure => Json.obj("status" -> "Failure", "value" -> f.msg)
+    }
+  }
+
+  def oFormat[T](format: Format[T]): OFormat[T] = {
+    val oFormat: OFormat[T] = new OFormat[T]() {
+      override def writes(o: T): JsObject = format.writes(o).as[JsObject]
+      override def reads(json: JsValue): JsResult[T] = format.reads(json)
+    }
+    oFormat
+  }
+
+  implicit object FiniteDurationFormat extends Format[FiniteDuration] {
+    def reads(json: JsValue): JsResult[FiniteDuration] = LongReads.reads(json).map(_.seconds)
+    def writes(o: FiniteDuration): JsValue = LongWrites.writes(o.toSeconds)
+  }
+
+  implicit def optionFormat[T: Format]: Format[Option[T]] = new Format[Option[T]] {
+    override def reads(json: JsValue): JsResult[Option[T]] = json.validateOpt[T]
+
+    override def writes(o: Option[T]): JsValue = o match {
+      case Some(t) ⇒ implicitly[Writes[T]].writes(t)
+      case None ⇒ JsNull
+    }
+  }
+
+  implicit def longMapFormat[T: Format]: Format[Map[Long, T]] = new Format[Map[Long, T]] {
+    override def reads(jsValue: JsValue): JsResult[Map[Long, T]] =
+      jsValue match {
+        case JsObject(map) =>
+          val mapProcessed = map.map {
+            case (k, v: JsValue) => k.toLong -> v.validate[T]
+          }.toMap
+          if (mapProcessed.forall {
+                case (_, _: JsSuccess[T]) => true
+                case _                    => false
+              }) {
+            JsSuccess(mapProcessed.flatMap {
+              case (k, v: JsSuccess[T]) => Some(k -> v.value)
+              case _                    => None
+            })
+          } else JsError()
+        case _ => JsError()
+      }
+
+    override def writes(m: Map[Long, T]): JsValue =
+      JsObject(m.map {
+        case (k, v) => k.toString -> Json.toJson(v)
+      })
+  }
+}
+
+object JsonHelper extends BoxImplicits with LazyLogging with AdditionalJsonFormats {
 
   def jsonToFile[A: Writes](path: Path, value: A) =
     FileIO.printToFile(path.toFile) { printer =>
@@ -64,44 +129,6 @@ object JsonHelper extends BoxImplicits with LazyLogging {
         s"Error at json path '$path': $errorStr."
     }.mkString("\n")
 
-  implicit def boxFormat[T: Format]: Format[Box[T]] = new Format[Box[T]] {
-    override def reads(json: JsValue): JsResult[Box[T]] =
-      (json \ "status").validate[String].flatMap {
-        case "Full"    => (json \ "value").validate[T].map(Full(_))
-        case "Empty"   => JsSuccess(Empty)
-        case "Failure" => (json \ "value").validate[String].map(Failure(_))
-        case _         => JsError("invalid status")
-      }
-
-    override def writes(o: Box[T]): JsValue = o match {
-      case Full(t)    => Json.obj("status" -> "Full", "value" -> Json.toJson(t))
-      case Empty      => Json.obj("status" -> "Empty")
-      case f: Failure => Json.obj("status" -> "Failure", "value" -> f.msg)
-    }
-  }
-
-  def oFormat[T](format: Format[T]): OFormat[T] = {
-    val oFormat: OFormat[T] = new OFormat[T]() {
-      override def writes(o: T): JsObject = format.writes(o).as[JsObject]
-      override def reads(json: JsValue): JsResult[T] = format.reads(json)
-    }
-    oFormat
-  }
-
-  implicit object FiniteDurationFormat extends Format[FiniteDuration] {
-    def reads(json: JsValue): JsResult[FiniteDuration] = LongReads.reads(json).map(_.seconds)
-    def writes(o: FiniteDuration): JsValue = LongWrites.writes(o.toSeconds)
-  }
-
-  implicit def optionFormat[T: Format]: Format[Option[T]] = new Format[Option[T]] {
-    override def reads(json: JsValue): JsResult[Option[T]] = json.validateOpt[T]
-
-    override def writes(o: Option[T]): JsValue = o match {
-      case Some(t) ⇒ implicitly[Writes[T]].writes(t)
-      case None ⇒ JsNull
-    }
-  }
-
   def parseJsonToFox[T: Reads](s: String): Box[T] =
     Json.parse(s).validate[T] match {
       case JsSuccess(parsed, _) =>
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
index 21736241c3b..ecd55aaa29a 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
@@ -324,6 +324,51 @@ Expects:
     }
   }
 
+  @ApiOperation(hidden = true, value = "")
+  def agglomerateGraph(
+      token: Option[String],
+      organizationName: String,
+      dataSetName: String,
+      dataLayerName: String,
+      mappingName: String,
+      agglomerateId: Long
+  ): Action[AnyContent] = Action.async { implicit request =>
+    accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationName)),
+                                      token) {
+      for {
+        agglomerateGraph <- binaryDataServiceHolder.binaryDataService.agglomerateService.generateAgglomerateGraph(
+          organizationName,
+          dataSetName,
+          dataLayerName,
+          mappingName,
+          agglomerateId) ?~> "agglomerateGraph.failed"
+      } yield Ok(Json.toJson(agglomerateGraph))
+    }
+  }
+
+  @ApiOperation(hidden = true, value = "")
+  def largestAgglomerateId(
+      token: Option[String],
+      organizationName: String,
+      dataSetName: String,
+      dataLayerName: String,
+      mappingName: String
+  ): Action[AnyContent] = Action.async { implicit request =>
+    accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationName)),
+                                      token) {
+      for {
+        largestAgglomerateId: Long <- binaryDataServiceHolder.binaryDataService.agglomerateService
+          .largestAgglomerateId(
+            organizationName,
+            dataSetName,
+            dataLayerName,
+            mappingName,
+          )
+          .toFox
+      } yield Ok(Json.toJson(largestAgglomerateId))
+    }
+  }
+
   @ApiOperation(hidden = true, value = "")
   def listMeshFiles(token: Option[String],
                     organizationName: String,
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/AgglomerateGraph.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala
similarity index 87%
rename from webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/AgglomerateGraph.scala
rename to webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala
index 681cfe06fed..c1d6162e19f 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/AgglomerateGraph.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala
@@ -1,4 +1,4 @@
-package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
+package com.scalableminds.webknossos.datastore.models
 
 import com.scalableminds.util.geometry.Vec3Int
 import play.api.libs.json.{Json, OFormat}
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 21a0f954e31..6cf8f078c62 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -9,6 +9,7 @@ import com.scalableminds.webknossos.datastore.DataStoreConfig
 import com.scalableminds.webknossos.datastore.SkeletonTracing.{Edge, SkeletonTracing, Tree}
 import com.scalableminds.webknossos.datastore.geometry.Vec3IntProto
 import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, SkeletonTracingDefaults}
+import com.scalableminds.webknossos.datastore.models.AgglomerateGraph
 import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataRequest
 import com.scalableminds.webknossos.datastore.storage._
 import com.typesafe.scalalogging.LazyLogging
@@ -189,4 +190,29 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
     } catch {
       case e: Exception => Failure(e.getMessage)
     }
+
+  def largestAgglomerateId(organizationName: String,
+                           dataSetName: String,
+                           dataLayerName: String,
+                           mappingName: String): Box[Long] = {
+    val hdfFile =
+      dataBaseDir
+        .resolve(organizationName)
+        .resolve(dataSetName)
+        .resolve(dataLayerName)
+        .resolve(agglomerateDir)
+        .resolve(s"$mappingName.$agglomerateFileExtension")
+        .toFile
+
+    val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
+    Full(0L)
+  }
+
+  def generateAgglomerateGraph(organizationName: String,
+                               dataSetName: String,
+                               dataLayerName: String,
+                               mappingName: String,
+                               agglomerateId: Long): Box[AgglomerateGraph] =
+    Full(AgglomerateGraph.empty)
+
 }
diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
index 915e239a9c3..fa09588d53b 100644
--- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
+++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
@@ -33,6 +33,8 @@ GET           /datasets/:organizationName/:dataSetName/layers/:dataLayerName/map
 # Agglomerate files
 GET           /datasets/:organizationName/:dataSetName/layers/:dataLayerName/agglomerates                                           @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listAgglomerates(token: Option[String], organizationName: String, dataSetName: String, dataLayerName: String)
 GET           /datasets/:organizationName/:dataSetName/layers/:dataLayerName/agglomerates/:mappingName/skeleton/:agglomerateId      @com.scalableminds.webknossos.datastore.controllers.DataSourceController.generateAgglomerateSkeleton(token: Option[String], organizationName: String, dataSetName: String, dataLayerName: String, mappingName: String, agglomerateId: Long)
+GET           /datasets/:organizationName/:dataSetName/layers/:dataLayerName/agglomerates/:mappingName/agglomerateGraph/:agglomerateId      @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateGraph(token: Option[String], organizationName: String, dataSetName: String, dataLayerName: String, mappingName: String, agglomerateId: Long)
+GET           /datasets/:organizationName/:dataSetName/layers/:dataLayerName/agglomerates/:mappingName/largestAgglomerateId         @com.scalableminds.webknossos.datastore.controllers.DataSourceController.largestAgglomerateId(token: Option[String], organizationName: String, dataSetName: String, dataLayerName: String, mappingName: String)
 
 # Mesh files
 GET           /datasets/:organizationName/:dataSetName/layers/:dataLayerName/meshes                                                 @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listMeshFiles(token: Option[String], organizationName: String, dataSetName: String, dataLayerName: String)
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
index 9386102329e..bfa4817814e 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
@@ -4,9 +4,9 @@ import com.google.inject.Inject
 import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.helpers.MissingBucketHeaders
-import com.scalableminds.webknossos.datastore.models.WebKnossosDataRequest
+import com.scalableminds.webknossos.datastore.models.{AgglomerateGraph, WebKnossosDataRequest}
 import com.scalableminds.webknossos.datastore.rpc.RPC
-import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{AgglomerateGraph, RemoteFallbackLayer}
+import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.RemoteFallbackLayer
 import com.typesafe.scalalogging.LazyLogging
 import play.api.http.Status
 import play.api.inject.ApplicationLifecycle
@@ -63,8 +63,17 @@ class TSRemoteDatastoreClient @Inject()(
     s"$datastoreUrl/data/datasets/${remoteLayer.organizationName}/${remoteLayer.dataSetName}/layers/${remoteLayer.layerName}"
 
   def getAgglomerateGraph(remoteFallbackLayer: RemoteFallbackLayer,
+                          baseMappingName: String,
                           agglomerateId: Long,
-                          userToken: Option[String]): Fox[AgglomerateGraph] = ???
+                          userToken: Option[String]): Fox[AgglomerateGraph] =
+    rpc(s"${remoteLayerUri(remoteFallbackLayer)}/agglomerates/$baseMappingName/agglomerateGraph/$agglomerateId")
+      .addQueryStringOptional("token", userToken)
+      .getWithJsonResponse[AgglomerateGraph]
 
-  def getLargestAgglomerateId(remoteFallbackLayer: RemoteFallbackLayer, mappingName: String): Fox[Long]
+  def getLargestAgglomerateId(remoteFallbackLayer: RemoteFallbackLayer,
+                              mappingName: String,
+                              userToken: Option[String]): Fox[Long] =
+    rpc(s"${remoteLayerUri(remoteFallbackLayer)}/agglomerates/$mappingName/largestAgglomerateId")
+      .addQueryStringOptional("token", userToken)
+      .getWithJsonResponse[Long]
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index 73d31e7ec0c..fb499ef55df 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -8,20 +8,16 @@ import com.google.inject.Inject
 import com.scalableminds.util.geometry.{BoundingBox, Vec3Int}
 import com.scalableminds.util.tools.ExtendedTypes.ExtendedString
 import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings}
 import com.scalableminds.webknossos.datastore.models.{WebKnossosDataRequest, WebKnossosIsosurfaceRequest}
 import com.scalableminds.webknossos.datastore.services.UserAccessRequest
-import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings}
 import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService
 import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{
   EditableMappingService,
   EditableMappingUpdateAction
 }
 import com.scalableminds.webknossos.tracingstore.tracings.volume.{ResolutionRestrictions, VolumeTracingService}
-import com.scalableminds.webknossos.tracingstore.{
-  TSRemoteDatastoreClient,
-  TSRemoteWebKnossosClient,
-  TracingStoreAccessTokenService
-}
+import com.scalableminds.webknossos.tracingstore.{TSRemoteWebKnossosClient, TracingStoreAccessTokenService}
 import play.api.i18n.Messages
 import play.api.libs.Files.TemporaryFile
 import play.api.libs.iteratee.Enumerator
@@ -33,7 +29,6 @@ import scala.concurrent.ExecutionContext
 
 class VolumeTracingController @Inject()(val tracingService: VolumeTracingService,
                                         val remoteWebKnossosClient: TSRemoteWebKnossosClient,
-                                        remoteDatastoreClient: TSRemoteDatastoreClient,
                                         editableMappingService: EditableMappingService,
                                         val accessTokenService: TracingStoreAccessTokenService,
                                         val slackNotificationService: TSSlackNotificationService)(
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
index f47bf5ae383..9121b835084 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
@@ -1,5 +1,7 @@
 package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
 
+import com.scalableminds.util.tools.AdditionalJsonFormats
+import com.scalableminds.webknossos.datastore.models.AgglomerateGraph
 import play.api.libs.json.{Json, OFormat}
 
 case class EditableMapping(
@@ -8,6 +10,6 @@ case class EditableMapping(
     agglomerateToGraph: Map[Long, AgglomerateGraph],
 )
 
-object EditableMapping {
+object EditableMapping extends AdditionalJsonFormats {
   implicit val jsonFormat: OFormat[EditableMapping] = Json.format[EditableMapping]
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 4e227f6f7e0..e921e6f5c1d 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -10,7 +10,12 @@ import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing
 import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing.ElementClass
 import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, ProtoGeometryImplicits, SkeletonTracingDefaults}
 import com.scalableminds.webknossos.datastore.models.DataRequestCollection.DataRequestCollection
-import com.scalableminds.webknossos.datastore.models.{UnsignedInteger, UnsignedIntegerArray, WebKnossosDataRequest}
+import com.scalableminds.webknossos.datastore.models.{
+  AgglomerateGraph,
+  UnsignedInteger,
+  UnsignedIntegerArray,
+  WebKnossosDataRequest
+}
 import com.scalableminds.webknossos.tracingstore.TSRemoteDatastoreClient
 import com.scalableminds.webknossos.tracingstore.tracings.{
   KeyValueStoreImplicits,
@@ -63,16 +68,25 @@ class EditableMappingService @Inject()(
     for {
       closestMaterializedVersion: VersionedKeyValuePair[EditableMapping] <- tracingDataStore.editableMappings
         .get(editableMappingId, version)(fromJson[EditableMapping])
+      desiredVersion <- findDesiredOrNewestPossibleVersion(closestMaterializedVersion.version,
+                                                           editableMappingId,
+                                                           version)
       materialized <- applyPendingUpdates(
         editableMappingId,
+        desiredVersion,
         closestMaterializedVersion.value,
         remoteFallbackLayer,
         closestMaterializedVersion.version,
-        version,
         userToken
-      ) // TODO store materialized in cache or db
+      )
+      _ <- Fox.runIf(shouldPersistMaterialized(closestMaterializedVersion.version, desiredVersion)) {
+        tracingDataStore.editableMappings.put(editableMappingId, desiredVersion, materialized)
+      }
     } yield materialized
 
+  private def shouldPersistMaterialized(previouslyMaterializedVersion: Long, newVersion: Long): Boolean =
+    newVersion > previouslyMaterializedVersion && newVersion % 10 == 5
+
   private def findDesiredOrNewestPossibleVersion(existingMaterializedVersion: Long,
                                                  editableMappingId: String,
                                                  desiredVersion: Option[Long]): Fox[Long] =
@@ -94,21 +108,12 @@ class EditableMappingService @Inject()(
     }
 
   private def applyPendingUpdates(editableMappingId: String,
+                                  desiredVersion: Long,
                                   existingEditableMapping: EditableMapping,
                                   remoteFallbackLayer: RemoteFallbackLayer,
                                   existingVersion: Long,
-                                  requestedVersion: Option[Long],
-                                  userToken: Option[String]): Fox[EditableMapping] =
-    for {
-      desiredVersion <- findDesiredOrNewestPossibleVersion(existingVersion, editableMappingId, requestedVersion)
-      pendingUpdates <- findPendingUpdates(editableMappingId, existingVersion, desiredVersion)
-      appliedEditableMapping <- applyUpdates(existingEditableMapping, pendingUpdates, remoteFallbackLayer, userToken)
-    } yield appliedEditableMapping
+                                  userToken: Option[String]): Fox[EditableMapping] = {
 
-  private def applyUpdates(existingEditableMapping: EditableMapping,
-                           updates: List[EditableMappingUpdateAction],
-                           remoteFallbackLayer: RemoteFallbackLayer,
-                           userToken: Option[String]): Fox[EditableMapping] = {
     def updateIter(mappingFox: Fox[EditableMapping],
                    remainingUpdates: List[EditableMappingUpdateAction]): Fox[EditableMapping] =
       mappingFox.futureBox.flatMap {
@@ -122,7 +127,10 @@ class EditableMappingService @Inject()(
         case _ => mappingFox
       }
 
-    updateIter(Some(existingEditableMapping), updates)
+    for {
+      pendingUpdates <- findPendingUpdates(editableMappingId, existingVersion, desiredVersion)
+      appliedEditableMapping <- updateIter(Some(existingEditableMapping), pendingUpdates)
+    } yield appliedEditableMapping
   }
 
   private def applyOneUpdate(mapping: EditableMapping,
@@ -158,7 +166,7 @@ class EditableMappingService @Inject()(
                          segmentId2: Long): (AgglomerateGraph, AgglomerateGraph) = {
     val edgesMinusOne = agglomerateGraph.edges.filter {
       case (from, to) =>
-        ((from == segmentId1 && to == segmentId2) || (from == segmentId2 && to == segmentId1))
+        (from == segmentId1 && to == segmentId2) || (from == segmentId2 && to == segmentId1)
     }
     val graph1Nodes: Set[Long] = computeConnectedComponent(startNode = segmentId1, edgesMinusOne)
     val graph1NodesWithPositions = agglomerateGraph.segments.zip(agglomerateGraph.positions).filter {
@@ -174,7 +182,7 @@ class EditableMappingService @Inject()(
       affinities = graph1EdgesWithAffinities.map(_._2),
     )
 
-    val graph2Nodes: Set[Long] = agglomerateGraph.segments.toSet.diff(graph2Nodes)
+    val graph2Nodes: Set[Long] = agglomerateGraph.segments.toSet.diff(graph1Nodes)
     val graph2NodesWithPositions = agglomerateGraph.segments.zip(agglomerateGraph.positions).filter {
       case (seg, _) => graph2Nodes.contains(seg)
     }
@@ -216,7 +224,8 @@ class EditableMappingService @Inject()(
                                    userToken: Option[String]): Fox[Long] =
     for {
       largestBaseAgglomerateId <- remoteDatastoreClient.getLargestAgglomerateId(remoteFallbackLayer,
-                                                                                mapping.baseMappingName)
+                                                                                mapping.baseMappingName,
+                                                                                userToken)
       keySet = mapping.agglomerateToGraph.keySet
     } yield math.max(if (keySet.isEmpty) 0L else keySet.max, largestBaseAgglomerateId)
 
@@ -262,7 +271,7 @@ class EditableMappingService @Inject()(
     if (mapping.agglomerateToGraph.contains(agglomerateId)) {
       Fox.successful(mapping.agglomerateToGraph(agglomerateId))
     } else {
-      remoteDatastoreClient.getAgglomerateGraph(remoteFallbackLayer, agglomerateId, userToken)
+      remoteDatastoreClient.getAgglomerateGraph(remoteFallbackLayer, mapping.baseMappingName, agglomerateId, userToken)
     }
 
   private def findSegmentIdAtPosition(remoteFallbackLayer: RemoteFallbackLayer,

From 6bb4dcce2e4e23f4c61cbe97601d96d07f846053 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Thu, 12 May 2022 15:14:03 +0200
Subject: [PATCH 021/122] fix some naming, enable proofread saga

---
 .../oxalis/model/sagas/proofread_saga.ts      | 27 +++++++++----
 .../oxalis/model/sagas/root_saga.ts           |  2 +
 .../oxalis/model/sagas/update_actions.ts      | 38 +++++++++----------
 .../javascripts/oxalis/view/version_entry.tsx |  8 ++--
 4 files changed, 44 insertions(+), 31 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 348060d2b50..d52ff926a41 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -1,5 +1,5 @@
 import type { Saga } from "oxalis/model/sagas/effect-generators";
-import { takeEvery, put } from "typed-redux-saga";
+import { takeEvery, put, call } from "typed-redux-saga";
 import { select, take } from "oxalis/model/sagas/effect-generators";
 import { AnnotationToolEnum } from "oxalis/constants";
 import Toast from "libs/toast";
@@ -12,21 +12,28 @@ import {
   findTreeByNodeId,
 } from "oxalis/model/accessors/skeletontracing_accessor";
 import { pushSaveQueueTransaction } from "oxalis/model/actions/save_actions";
-import { splitAgglomerate, mergeAgglomerates } from "oxalis/model/sagas/update_actions";
+import { splitAgglomerate, mergeAgglomerate } from "oxalis/model/sagas/update_actions";
+import Model from "oxalis/model";
+import api from "oxalis/api/internal_api";
+import { getActiveSegmentationTracing } from "oxalis/model/accessors/volumetracing_accessor";
 
 export default function* proofreadMapping(): Saga<any> {
   yield* take("INITIALIZE_SKELETONTRACING");
   yield* take("WK_READY");
-  yield* takeEvery(["DELETE_EDGE", "MERGE_TREES"], splitOrMergeAgglomerates);
+  yield* takeEvery(["DELETE_EDGE", "MERGE_TREES"], splitOrMergeAgglomerate);
 }
 
-function* splitOrMergeAgglomerates(action: MergeTreesAction | DeleteEdgeAction) {
+function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
   const allowUpdate = yield* select((state) => state.tracing.restrictions.allowUpdate);
   if (!allowUpdate) return;
 
   const activeTool = yield* select((state) => state.uiInformation.activeTool);
   if (activeTool !== AnnotationToolEnum.PROOFREAD) return;
 
+  const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
+  if (volumeTracing == null) return;
+
+  const layerName = volumeTracing.tracingId;
   const { sourceNodeId, targetNodeId } = action;
 
   const skeletonTracing = yield* select((state) => enforceSkeletonTracing(state.tracing));
@@ -48,7 +55,7 @@ function* splitOrMergeAgglomerates(action: MergeTreesAction | DeleteEdgeAction)
       Toast.error("Segments that should be merged need to be in different agglomerates.");
       return;
     }
-    items.push(mergeAgglomerates(sourceNodePosition, targetNodePosition));
+    items.push(mergeAgglomerate(sourceNodePosition, targetNodePosition));
   } else if (action.type === "DELETE_EDGE") {
     if (sourceTree !== targetTree) {
       Toast.error("Segments that should be split need to be in the same agglomerate.");
@@ -57,7 +64,11 @@ function* splitOrMergeAgglomerates(action: MergeTreesAction | DeleteEdgeAction)
     items.push(splitAgglomerate(sourceNodePosition, targetNodePosition));
   }
 
-  if (items.length) {
-    yield* put(pushSaveQueueTransaction(items, tracingType, tracingId));
-  }
+  if (items.length === 0) return;
+
+  // TODO: Will there be a separate end point for these update actions?
+  yield* put(pushSaveQueueTransaction(items, tracingType, tracingId));
+  yield* call([Model, Model.ensureSavedState]);
+
+  yield* call([api.data, api.data.reloadBuckets], layerName);
 }
diff --git a/frontend/javascripts/oxalis/model/sagas/root_saga.ts b/frontend/javascripts/oxalis/model/sagas/root_saga.ts
index 746f4c6c87a..002855e8f0f 100644
--- a/frontend/javascripts/oxalis/model/sagas/root_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/root_saga.ts
@@ -16,6 +16,7 @@ import watchTasksAsync, { warnAboutMagRestriction } from "oxalis/model/sagas/tas
 import loadHistogramDataSaga from "oxalis/model/sagas/load_histogram_data_saga";
 import listenToClipHistogramSaga from "oxalis/model/sagas/clip_histogram_saga";
 import MappingSaga from "oxalis/model/sagas/mapping_saga";
+import ProofreadSaga from "oxalis/model/sagas/proofread_saga";
 let rootSagaCrashed = false;
 export default function* rootSaga(): Saga<void> {
   while (true) {
@@ -44,6 +45,7 @@ function* restartableSaga(): Saga<void> {
       call(watchMaximumRenderableLayers),
       call(MappingSaga),
       call(watchToolDeselection),
+      call(ProofreadSaga),
       ...AnnotationSagas.map((saga) => call(saga)),
       ...SaveSagas.map((saga) => call(saga)),
       ...VolumetracingSagas.map((saga) => call(saga)),
diff --git a/frontend/javascripts/oxalis/model/sagas/update_actions.ts b/frontend/javascripts/oxalis/model/sagas/update_actions.ts
index c1470906a12..010c1d16124 100644
--- a/frontend/javascripts/oxalis/model/sagas/update_actions.ts
+++ b/frontend/javascripts/oxalis/model/sagas/update_actions.ts
@@ -174,15 +174,15 @@ export type UpdateTdCameraUpdateAction = {
 export type SplitAgglomerateUpdateAction = {
   name: "splitAgglomerate";
   value: {
-    segment_1_position: Vector3;
-    segment_2_position: Vector3;
+    segmentPosition1: Vector3;
+    segmentPosition2: Vector3;
   };
 };
-export type MergeAgglomeratesUpdateAction = {
-  name: "mergeAgglomerates";
+export type MergeAgglomerateUpdateAction = {
+  name: "mergeAgglomerate";
   value: {
-    segment_1_position: Vector3;
-    segment_2_position: Vector3;
+    segmentPosition1: Vector3;
+    segmentPosition2: Vector3;
   };
 };
 export type UpdateAction =
@@ -209,7 +209,7 @@ export type UpdateAction =
   | RemoveFallbackLayerUpdateAction
   | UpdateTdCameraUpdateAction
   | SplitAgglomerateUpdateAction
-  | MergeAgglomeratesUpdateAction;
+  | MergeAgglomerateUpdateAction;
 // This update action is only created in the frontend for display purposes
 type CreateTracingUpdateAction = {
   name: "createTracing";
@@ -257,7 +257,7 @@ export type ServerUpdateAction =
   | AsServerAction<UpdateTdCameraUpdateAction>
   | AsServerAction<ImportVolumeTracingUpdateAction>
   | AsServerAction<SplitAgglomerateUpdateAction>
-  | AsServerAction<MergeAgglomeratesUpdateAction>;
+  | AsServerAction<MergeAgglomerateUpdateAction>;
 export function createTree(tree: Tree): UpdateTreeUpdateAction {
   return {
     name: "createTree",
@@ -529,26 +529,26 @@ export function serverCreateTracing(timestamp: number) {
   };
 }
 export function splitAgglomerate(
-  segment_1_position: Vector3,
-  segment_2_position: Vector3,
+  segmentPosition1: Vector3,
+  segmentPosition2: Vector3,
 ): SplitAgglomerateUpdateAction {
   return {
     name: "splitAgglomerate",
     value: {
-      segment_1_position,
-      segment_2_position,
+      segmentPosition1,
+      segmentPosition2,
     },
   };
 }
-export function mergeAgglomerates(
-  segment_1_position: Vector3,
-  segment_2_position: Vector3,
-): MergeAgglomeratesUpdateAction {
+export function mergeAgglomerate(
+  segmentPosition1: Vector3,
+  segmentPosition2: Vector3,
+): MergeAgglomerateUpdateAction {
   return {
-    name: "mergeAgglomerates",
+    name: "mergeAgglomerate",
     value: {
-      segment_1_position,
-      segment_2_position,
+      segmentPosition1,
+      segmentPosition2,
     },
   };
 }
diff --git a/frontend/javascripts/oxalis/view/version_entry.tsx b/frontend/javascripts/oxalis/view/version_entry.tsx
index 74b1753e59a..bc35143419a 100644
--- a/frontend/javascripts/oxalis/view/version_entry.tsx
+++ b/frontend/javascripts/oxalis/view/version_entry.tsx
@@ -29,7 +29,7 @@ import type {
   CreateEdgeUpdateAction,
   DeleteEdgeUpdateAction,
   SplitAgglomerateUpdateAction,
-  MergeAgglomeratesUpdateAction,
+  MergeAgglomerateUpdateAction,
   CreateSegmentUpdateAction,
   UpdateSegmentUpdateAction,
   DeleteSegmentUpdateAction,
@@ -68,11 +68,11 @@ const descriptionFns: Record<ServerUpdateAction["name"], (...args: any) => Descr
     icon: <DeleteOutlined />,
   }),
   splitAgglomerate: (action: SplitAgglomerateUpdateAction): Description => ({
-    description: `Split an agglomerate by separating the segments at position ${action.value.segment_1_position} and ${action.value.segment_2_position}.`,
+    description: `Split an agglomerate by separating the segments at position ${action.value.segmentPosition1} and ${action.value.segmentPosition2}.`,
     icon: <DeleteOutlined />,
   }),
-  mergeAgglomerates: (action: MergeAgglomeratesUpdateAction): Description => ({
-    description: `Merged two agglomerates by combining the segments at position ${action.value.segment_1_position} and ${action.value.segment_2_position}.`,
+  mergeAgglomerate: (action: MergeAgglomerateUpdateAction): Description => ({
+    description: `Merged two agglomerates by combining the segments at position ${action.value.segmentPosition1} and ${action.value.segmentPosition2}.`,
     icon: <PlusOutlined />,
   }),
   deleteTree: (action: DeleteTreeUpdateAction, count: number): Description => ({

From 6c0882166287e10f4f9b2be911633742489472a6 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 16 May 2022 10:56:24 +0200
Subject: [PATCH 022/122] in update actions, use positions and mag

---
 .../TSRemoteDatastoreClient.scala             |  6 +++-
 .../EditableMappingService.scala              | 29 +++++++++++--------
 .../EditableMappingUpdateActions.scala        | 10 +++++--
 3 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
index bfa4817814e..58db13585e1 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
@@ -57,7 +57,11 @@ class TSRemoteDatastoreClient @Inject()(
 
   def getAgglomerateIdsForSegmentIds(remoteFallbackLayer: RemoteFallbackLayer,
                                      mappingName: String,
-                                     segmentIdsOrdered: List[Long]): Fox[List[Long]] = ???
+                                     segmentIdsOrdered: List[Long],
+                                     userToken: Option[String]): Fox[List[Long]] =
+    rpc(s"${remoteLayerUri(remoteFallbackLayer)}/agglomerates/$mappingName/agglomeratesForSegments")
+      .addQueryStringOptional("token", userToken)
+      .postJsonWithJsonResponse[List[Long], List[Long]](segmentIdsOrdered)
 
   private def remoteLayerUri(remoteLayer: RemoteFallbackLayer): String =
     s"$datastoreUrl/data/datasets/${remoteLayer.organizationName}/${remoteLayer.dataSetName}/layers/${remoteLayer.layerName}"
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index e921e6f5c1d..befa5ec1062 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -150,9 +150,11 @@ class EditableMappingService @Inject()(
                                userToken: Option[String]): Fox[EditableMapping] =
     for {
       agglomerateGraph <- agglomerateGraphForId(mapping, update.agglomerateId, remoteFallbackLayer, userToken)
+      segmentId1 <- findSegmentIdAtPosition(remoteFallbackLayer, update.segmentPosition1, update.mag, userToken)
+      segmentId2 <- findSegmentIdAtPosition(remoteFallbackLayer, update.segmentPosition2, update.mag, userToken)
       largestExistingAgglomerateId <- largestAgglomerateId(mapping, remoteFallbackLayer, userToken)
       agglomerateId2 = largestExistingAgglomerateId + 1L
-      (graph1, graph2) = splitGraph(agglomerateGraph, update.segmentId1, update.segmentId2)
+      (graph1, graph2) = splitGraph(agglomerateGraph, segmentId1, segmentId2)
       splitSegmentToAgglomerate = graph2.segments.map(_ -> agglomerateId2).toMap
     } yield
       EditableMapping(
@@ -234,10 +236,11 @@ class EditableMappingService @Inject()(
                                remoteFallbackLayer: RemoteFallbackLayer,
                                userToken: Option[String]): Fox[EditableMapping] =
     for {
-      segmentId2 <- findSegmentIdAtPosition(remoteFallbackLayer, update.segmentPosition2, userToken)
+      segmentId1 <- findSegmentIdAtPosition(remoteFallbackLayer, update.segmentPosition1, update.mag, userToken)
+      segmentId2 <- findSegmentIdAtPosition(remoteFallbackLayer, update.segmentPosition2, update.mag, userToken)
       agglomerateGraph1 <- agglomerateGraphForId(mapping, update.agglomerateId1, remoteFallbackLayer, userToken)
       agglomerateGraph2 <- agglomerateGraphForId(mapping, update.agglomerateId2, remoteFallbackLayer, userToken)
-      mergedGraph = mergeGraph(agglomerateGraph1, agglomerateGraph2, update.segmentId1, segmentId2)
+      mergedGraph = mergeGraph(agglomerateGraph1, agglomerateGraph2, segmentId1, segmentId2)
       _ <- bool2Fox(agglomerateGraph2.segments.contains(segmentId2)) ?~> "segment as queried by position is not contained in fetched agglomerate graph"
       mergedSegmentToAgglomerate: Map[Long, Long] = agglomerateGraph2.segments
         .map(s => s -> update.agglomerateId1)
@@ -276,12 +279,10 @@ class EditableMappingService @Inject()(
 
   private def findSegmentIdAtPosition(remoteFallbackLayer: RemoteFallbackLayer,
                                       pos: Vec3Int,
+                                      mag: Vec3Int,
                                       userToken: Option[String]): Fox[Long] =
     for {
-      voxelAsBytes: Array[Byte] <- remoteDatastoreClient.getVoxelAtPosition(userToken,
-                                                                            remoteFallbackLayer,
-                                                                            pos,
-                                                                            mag = Vec3Int(1, 1, 1))
+      voxelAsBytes: Array[Byte] <- remoteDatastoreClient.getVoxelAtPosition(userToken, remoteFallbackLayer, pos, mag)
       voxelAsLongArray: Array[Long] <- bytesToLongs(voxelAsBytes, remoteFallbackLayer.elementClass)
       _ <- Fox.bool2Fox(voxelAsLongArray.length == 1) ?~> s"Expected one, got ${voxelAsLongArray.length} segment id values for voxel."
       voxelAsLong <- voxelAsLongArray.headOption
@@ -312,13 +313,14 @@ class EditableMappingService @Inject()(
       editableMapping <- get(editableMappingId, remoteFallbackLayer, userToken)
       (unmappedData, indices) <- getUnmappedDataFromDatastore(remoteFallbackLayer, dataRequests)
       segmentIds <- collectSegmentIds(unmappedData, tracing.elementClass)
-      relevantMapping <- generateCombinedMappingSubset(segmentIds, editableMapping, remoteFallbackLayer)
+      relevantMapping <- generateCombinedMappingSubset(segmentIds, editableMapping, remoteFallbackLayer, userToken)
       mappedData <- mapData(unmappedData, relevantMapping, tracing.elementClass)
     } yield (mappedData, indices)
 
   private def generateCombinedMappingSubset(segmentIds: Set[Long],
                                             editableMapping: EditableMapping,
-                                            remoteFallbackLayer: RemoteFallbackLayer): Fox[Map[Long, Long]] = {
+                                            remoteFallbackLayer: RemoteFallbackLayer,
+                                            userToken: Option[String]): Fox[Map[Long, Long]] = {
     val segmentIdsInEditableMapping: Set[Long] = segmentIds.intersect(editableMapping.segmentToAgglomerate.keySet)
     val segmentIdsInBaseMapping: Set[Long] = segmentIds.diff(segmentIdsInEditableMapping)
     val editableMappingSubset =
@@ -326,7 +328,8 @@ class EditableMappingService @Inject()(
     for {
       baseMappingSubset <- getBaseSegmentToAgglomeate(editableMapping.baseMappingName,
                                                       segmentIdsInBaseMapping,
-                                                      remoteFallbackLayer)
+                                                      remoteFallbackLayer,
+                                                      userToken)
     } yield editableMappingSubset ++ baseMappingSubset
   }
 
@@ -381,12 +384,14 @@ class EditableMappingService @Inject()(
 
   private def getBaseSegmentToAgglomeate(mappingName: String,
                                          segmentIds: Set[Long],
-                                         remoteFallbackLayer: RemoteFallbackLayer): Fox[Map[Long, Long]] = {
+                                         remoteFallbackLayer: RemoteFallbackLayer,
+                                         userToken: Option[String]): Fox[Map[Long, Long]] = {
     val segmentIdsOrdered = segmentIds.toList
     for {
       agglomerateIdsOrdered <- remoteDatastoreClient.getAgglomerateIdsForSegmentIds(remoteFallbackLayer,
                                                                                     mappingName,
-                                                                                    segmentIdsOrdered)
+                                                                                    segmentIdsOrdered,
+                                                                                    userToken)
     } yield segmentIdsOrdered.zip(agglomerateIdsOrdered).toMap
   }
 
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
index e7875e31637..ae6087e4d86 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
@@ -6,7 +6,10 @@ import play.api.libs.json.{Format, JsError, JsResult, JsValue, Json, OFormat}
 
 trait EditableMappingUpdateAction {}
 
-case class SplitAgglomerateUpdateAction(agglomerateId: Long, segmentId1: Long, segmentId2: Long)
+case class SplitAgglomerateUpdateAction(agglomerateId: Long,
+                                        segmentPosition1: Vec3Int,
+                                        segmentPosition2: Vec3Int,
+                                        mag: Vec3Int)
     extends EditableMappingUpdateAction {}
 
 object SplitAgglomerateUpdateAction {
@@ -15,8 +18,9 @@ object SplitAgglomerateUpdateAction {
 
 case class MergeAgglomerateUpdateAction(agglomerateId1: Long,
                                         agglomerateId2: Long,
-                                        segmentId1: Long,
-                                        segmentPosition2: Vec3Int) // TODO: needs mag
+                                        segmentPosition1: Vec3Int,
+                                        segmentPosition2: Vec3Int,
+                                        mag: Vec3Int)
     extends EditableMappingUpdateAction {}
 
 object MergeAgglomerateUpdateAction {

From 744b2efd16df0349a54a713e7118338571f6f05f Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 16 May 2022 11:38:24 +0200
Subject: [PATCH 023/122] fetch agglomerate ids for segment ids

---
 .../controllers/DataSourceController.scala    | 42 ++++++++---
 .../services/AgglomerateService.scala         | 71 ++++++++++---------
 .../services/BinaryDataService.scala          |  4 +-
 .../storage/AgglomerateFileCache.scala        | 43 ++++++-----
 ....scalableminds.webknossos.datastore.routes |  2 +
 5 files changed, 101 insertions(+), 61 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
index ecd55aaa29a..7e950d9ca4f 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
@@ -16,6 +16,7 @@ import play.api.libs.json.Json
 import play.api.mvc.{Action, AnyContent, MultipartFormData, PlayBodyParsers}
 import java.io.File
 
+import com.scalableminds.webknossos.datastore.storage.AgglomerateFileKey
 import io.swagger.annotations.{Api, ApiImplicitParam, ApiImplicitParams, ApiOperation, ApiResponse, ApiResponses}
 import play.api.libs.Files
 
@@ -337,10 +338,7 @@ Expects:
                                       token) {
       for {
         agglomerateGraph <- binaryDataServiceHolder.binaryDataService.agglomerateService.generateAgglomerateGraph(
-          organizationName,
-          dataSetName,
-          dataLayerName,
-          mappingName,
+          AgglomerateFileKey(organizationName, dataSetName, dataLayerName, mappingName),
           agglomerateId) ?~> "agglomerateGraph.failed"
       } yield Ok(Json.toJson(agglomerateGraph))
     }
@@ -359,16 +357,44 @@ Expects:
       for {
         largestAgglomerateId: Long <- binaryDataServiceHolder.binaryDataService.agglomerateService
           .largestAgglomerateId(
-            organizationName,
-            dataSetName,
-            dataLayerName,
-            mappingName,
+            AgglomerateFileKey(
+              organizationName,
+              dataSetName,
+              dataLayerName,
+              mappingName
+            )
           )
           .toFox
       } yield Ok(Json.toJson(largestAgglomerateId))
     }
   }
 
+  @ApiOperation(hidden = true, value = "")
+  def agglomerateIdsForSegmentIds(
+      token: Option[String],
+      organizationName: String,
+      dataSetName: String,
+      dataLayerName: String,
+      mappingName: String
+  ): Action[List[Long]] = Action.async(validateJson[List[Long]]) { implicit request =>
+    accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationName)),
+                                      token) {
+      for {
+        agglomerateIds: List[Long] <- binaryDataServiceHolder.binaryDataService.agglomerateService
+          .agglomerateIdsForSegmentIds(
+            AgglomerateFileKey(
+              organizationName,
+              dataSetName,
+              dataLayerName,
+              mappingName
+            ),
+            request.body
+          )
+          .toFox
+      } yield Ok(Json.toJson(agglomerateIds))
+    }
+  }
+
   @ApiOperation(hidden = true, value = "")
   def listMeshFiles(token: Option[String],
                     organizationName: String,
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 6cf8f078c62..4674dcdce97 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -14,6 +14,7 @@ import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataReq
 import com.scalableminds.webknossos.datastore.storage._
 import com.typesafe.scalalogging.LazyLogging
 import javax.inject.Inject
+import net.liftweb.common.Box.tryo
 import net.liftweb.common.{Box, Failure, Full}
 import org.apache.commons.io.FilenameUtils
 import spire.math.{UByte, UInt, ULong, UShort}
@@ -45,10 +46,13 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
     def intFunc(buf: ByteBuffer, lon: Long) = buf putInt lon.toInt
     def longFunc(buf: ByteBuffer, lon: Long) = buf putLong lon
 
+    val agglomerateFileKey = AgglomerateFileKey.fromDataRequest(request)
+
     def convertToAgglomerate(input: Array[ULong],
                              numBytes: Int,
                              bufferFunc: (ByteBuffer, Long) => ByteBuffer): Array[Byte] = {
-      val cachedAgglomerateFile = agglomerateFileCache.withCache(request)(initHDFReader)
+
+      val cachedAgglomerateFile = agglomerateFileCache.withCache(agglomerateFileKey)(initHDFReader)
 
       val agglomerateIds = cachedAgglomerateFile.cache match {
         case Left(agglomerateIdCache) =>
@@ -86,36 +90,31 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
   // In this array, the agglomerate id is found by using the segment id as index.
   // There are two ways of how we prevent a file lookup for every input element. When present, we use the cumsum.json to initialize a BoundingBoxCache (see comment there).
   // Otherwise, we read configurable sized blocks from the agglomerate file and save them in a LRU cache.
-  private def initHDFReader(request: DataServiceDataRequest) = {
+  private def initHDFReader(agglomerateFileKey: AgglomerateFileKey) = {
     val hdfFile =
-      dataBaseDir
-        .resolve(request.dataSource.id.team)
-        .resolve(request.dataSource.id.name)
-        .resolve(request.dataLayer.name)
-        .resolve(agglomerateDir)
-        .resolve(s"${request.settings.appliedAgglomerate.get}.$agglomerateFileExtension")
-        .toFile
+      agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
 
     val cumsumPath =
       dataBaseDir
-        .resolve(request.dataSource.id.team)
-        .resolve(request.dataSource.id.name)
-        .resolve(request.dataLayer.name)
+        .resolve(agglomerateFileKey.organizationName)
+        .resolve(agglomerateFileKey.dataSetName)
+        .resolve(agglomerateFileKey.layerName)
         .resolve(agglomerateDir)
         .resolve(cumsumFileName)
 
     val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
 
-    val cache: Either[AgglomerateIdCache, BoundingBoxCache] =
+    val agglomerateIdCache = new AgglomerateIdCache(config.Datastore.Cache.AgglomerateFile.maxSegmentIdEntries,
+                                                    config.Datastore.Cache.AgglomerateFile.blockSize)
+
+    val defaultCache: Either[AgglomerateIdCache, BoundingBoxCache] =
       if (Files.exists(cumsumPath)) {
         Right(CumsumParser.parse(cumsumPath.toFile, ULong(config.Datastore.Cache.AgglomerateFile.cumsumMaxReaderRange)))
       } else {
-        Left(
-          new AgglomerateIdCache(config.Datastore.Cache.AgglomerateFile.maxSegmentIdEntries,
-                                 config.Datastore.Cache.AgglomerateFile.blockSize))
+        Left(agglomerateIdCache)
       }
 
-    CachedAgglomerateFile(reader, reader.`object`().openDataSet(datasetName), cache)
+    CachedAgglomerateFile(reader, reader.`object`().openDataSet(datasetName), agglomerateIdCache, defaultCache)
   }
 
   def generateSkeleton(organizationName: String,
@@ -191,28 +190,32 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
       case e: Exception => Failure(e.getMessage)
     }
 
-  def largestAgglomerateId(organizationName: String,
-                           dataSetName: String,
-                           dataLayerName: String,
-                           mappingName: String): Box[Long] = {
-    val hdfFile =
-      dataBaseDir
-        .resolve(organizationName)
-        .resolve(dataSetName)
-        .resolve(dataLayerName)
-        .resolve(agglomerateDir)
-        .resolve(s"$mappingName.$agglomerateFileExtension")
-        .toFile
+  def largestAgglomerateId(agglomerateFileKey: AgglomerateFileKey): Box[Long] = {
+    val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
 
     val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
     Full(0L)
   }
 
-  def generateAgglomerateGraph(organizationName: String,
-                               dataSetName: String,
-                               dataLayerName: String,
-                               mappingName: String,
-                               agglomerateId: Long): Box[AgglomerateGraph] =
+  def agglomerateIdsForSegmentIds(agglomerateFileKey: AgglomerateFileKey, segmentIds: List[Long]): Box[List[Long]] = {
+    val cachedAgglomerateFile = agglomerateFileCache.withCache(agglomerateFileKey)(initHDFReader)
+
+    tryo {
+      val agglomerateIds = segmentIds.map { segmentId =>
+        cachedAgglomerateFile.agglomerateIdCache.withCache(ULong(segmentId),
+                                                           cachedAgglomerateFile.reader,
+                                                           cachedAgglomerateFile.dataset)(readHDF)
+      }
+      cachedAgglomerateFile.finishAccess()
+      agglomerateIds
+    }
+
+  }
+
+  def generateAgglomerateGraph(agglomerateFileKey: AgglomerateFileKey, agglomerateId: Long): Box[AgglomerateGraph] = {
+    val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
+
     Full(AgglomerateGraph.empty)
+  }
 
 }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
index 4d67790d4bc..421bf55d9fb 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
@@ -180,8 +180,8 @@ class BinaryDataService(val dataBaseDir: Path, maxCacheSize: Int, val agglomerat
         _ == cubeKey.dataLayerName)
 
     def agglomerateFileMatchPredicate(agglomerateKey: AgglomerateFileKey) =
-      agglomerateKey.dataSourceName == dataSetName && agglomerateKey.organization == organizationName && layerName
-        .forall(_ == agglomerateKey.dataLayerName)
+      agglomerateKey.dataSetName == dataSetName && agglomerateKey.organizationName == organizationName && layerName
+        .forall(_ == agglomerateKey.layerName)
 
     val closedAgglomerateFileHandleCount = agglomerateService.agglomerateFileCache.clear(agglomerateFileMatchPredicate)
     val closedDataCubeHandleCount = cache.clear(dataCubeMatchPredicate)
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala
index 76fac47d0f3..b99468f3a4a 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala
@@ -1,50 +1,59 @@
 package com.scalableminds.webknossos.datastore.storage
 
+import java.nio.file.Path
 import java.util
+
 import ch.systemsx.cisd.hdf5.{HDF5DataSet, IHDF5Reader}
 import com.scalableminds.util.cache.LRUConcurrentCache
 import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.webknossos.datastore.dataformats.SafeCachable
-import com.scalableminds.webknossos.datastore.models.requests.{Cuboid, DataServiceDataRequest}
-import com.scalableminds.webknossos.datastore.storage
-import spire.math.{ULong, min, max}
 import com.scalableminds.webknossos.datastore.models.VoxelPosition
+import com.scalableminds.webknossos.datastore.models.requests.{Cuboid, DataServiceDataRequest}
 import com.typesafe.scalalogging.LazyLogging
+import spire.math.{ULong, max, min}
 
 import scala.collection.mutable
 
 case class CachedAgglomerateFile(reader: IHDF5Reader,
                                  dataset: HDF5DataSet,
+                                 agglomerateIdCache: AgglomerateIdCache,
                                  cache: Either[AgglomerateIdCache, BoundingBoxCache])
     extends SafeCachable {
   override protected def onFinalize(): Unit = { dataset.close(); reader.close() }
 }
 
 case class AgglomerateFileKey(
-    organization: String,
-    dataSourceName: String,
-    dataLayerName: String,
-    agglomerateName: String
-)
+    organizationName: String,
+    dataSetName: String,
+    layerName: String,
+    mappingName: String
+) {
+  def path(dataBaseDir: Path, agglomerateDir: String, agglomerateFileExtension: String): Path =
+    dataBaseDir
+      .resolve(organizationName)
+      .resolve(dataSetName)
+      .resolve(layerName)
+      .resolve(agglomerateDir)
+      .resolve(s"$mappingName.$agglomerateFileExtension")
+}
 
 object AgglomerateFileKey {
-  def from(dataRequest: DataServiceDataRequest): AgglomerateFileKey =
-    storage.AgglomerateFileKey(dataRequest.dataSource.id.team,
-                               dataRequest.dataSource.id.name,
-                               dataRequest.dataLayer.name,
-                               dataRequest.settings.appliedAgglomerate.get)
+  def fromDataRequest(dataRequest: DataServiceDataRequest): AgglomerateFileKey =
+    AgglomerateFileKey(dataRequest.dataSource.id.team,
+                       dataRequest.dataSource.id.name,
+                       dataRequest.dataLayer.name,
+                       dataRequest.settings.appliedAgglomerate.get)
 }
 
 class AgglomerateFileCache(val maxEntries: Int) extends LRUConcurrentCache[AgglomerateFileKey, CachedAgglomerateFile] {
   override def onElementRemoval(key: AgglomerateFileKey, value: CachedAgglomerateFile): Unit =
     value.scheduleForRemoval()
 
-  def withCache(dataRequest: DataServiceDataRequest)(
-      loadFn: DataServiceDataRequest => CachedAgglomerateFile): CachedAgglomerateFile = {
-    val agglomerateFileKey = AgglomerateFileKey.from(dataRequest)
+  def withCache(agglomerateFileKey: AgglomerateFileKey)(
+      loadFn: AgglomerateFileKey => CachedAgglomerateFile): CachedAgglomerateFile = {
 
     def handleUncachedAgglomerateFile() = {
-      val agglomerateFile = loadFn(dataRequest)
+      val agglomerateFile = loadFn(agglomerateFileKey)
       // We don't need to check the return value of the `tryAccess` call as we just created the agglomerate file and use it only to increase the access counter.
       agglomerateFile.tryAccess()
       put(agglomerateFileKey, agglomerateFile)
diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
index 05cc54a98b2..459b448b930 100644
--- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
+++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
@@ -35,6 +35,8 @@ GET           /datasets/:organizationName/:dataSetName/layers/:dataLayerName/agg
 GET           /datasets/:organizationName/:dataSetName/layers/:dataLayerName/agglomerates/:mappingName/skeleton/:agglomerateId      @com.scalableminds.webknossos.datastore.controllers.DataSourceController.generateAgglomerateSkeleton(token: Option[String], organizationName: String, dataSetName: String, dataLayerName: String, mappingName: String, agglomerateId: Long)
 GET           /datasets/:organizationName/:dataSetName/layers/:dataLayerName/agglomerates/:mappingName/agglomerateGraph/:agglomerateId      @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateGraph(token: Option[String], organizationName: String, dataSetName: String, dataLayerName: String, mappingName: String, agglomerateId: Long)
 GET           /datasets/:organizationName/:dataSetName/layers/:dataLayerName/agglomerates/:mappingName/largestAgglomerateId         @com.scalableminds.webknossos.datastore.controllers.DataSourceController.largestAgglomerateId(token: Option[String], organizationName: String, dataSetName: String, dataLayerName: String, mappingName: String)
+POST          /datasets/:organizationName/:dataSetName/layers/:dataLayerName/agglomerates/:mappingName/agglomeratesForSegments      @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateIdsForSegmentIds(token: Option[String], organizationName: String, dataSetName: String, dataLayerName: String, mappingName: String)
+
 
 # Mesh files
 GET           /datasets/:organizationName/:dataSetName/layers/:dataLayerName/meshes                                                 @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listMeshFiles(token: Option[String], organizationName: String, dataSetName: String, dataLayerName: String)

From f7c58f084cf5685d3ff3750975bb46a25c9564f5 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 16 May 2022 11:48:58 +0200
Subject: [PATCH 024/122] generate AgglomerateGraph from hdf file

---
 .../services/AgglomerateService.scala         | 38 ++++++++++++++++++-
 1 file changed, 36 insertions(+), 2 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 4674dcdce97..1b48104b477 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -4,6 +4,7 @@ import java.nio._
 import java.nio.file.{Files, Paths}
 
 import ch.systemsx.cisd.hdf5._
+import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.io.PathUtils
 import com.scalableminds.webknossos.datastore.DataStoreConfig
 import com.scalableminds.webknossos.datastore.SkeletonTracing.{Edge, SkeletonTracing, Tree}
@@ -194,6 +195,7 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
     val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
 
     val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
+    
     Full(0L)
   }
 
@@ -213,9 +215,41 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
   }
 
   def generateAgglomerateGraph(agglomerateFileKey: AgglomerateFileKey, agglomerateId: Long): Box[AgglomerateGraph] = {
-    val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
+    tryo {
+      val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
+
+      val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
+
+      val positionsRange: Array[Long] =
+        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments_offsets", 2, agglomerateId)
+      val edgesRange: Array[Long] =
+        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_edges_offsets", 2, agglomerateId)
 
-    Full(AgglomerateGraph.empty)
+      val nodeCount = positionsRange(1) - positionsRange(0)
+      val edgeCount = edgesRange(1) - edgesRange(0)
+      val edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
+      if (nodeCount > edgeLimit) {
+        throw new Exception(s"Agglomerate has too many nodes ($nodeCount > $edgeLimit)")
+      }
+      if (edgeCount > edgeLimit) {
+        throw new Exception(s"Agglomerate has too many edges ($edgeCount > $edgeLimit)")
+      }
+      val segmentIds: Array[Long] =
+        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments", nodeCount.toInt, positionsRange(0))
+      val positions: Array[Array[Long]] =
+        reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_positions", nodeCount.toInt, 3, positionsRange(0), 0)
+      val edges: Array[Array[Long]] =
+        reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_edges", edgeCount.toInt, 2, edgesRange(0), 0)
+      val affinities: Array[Long] =
+        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_affinities", edgeCount.toInt, edgesRange(0))
+
+      AgglomerateGraph(
+        segments = segmentIds.toList,
+        edges = edges.toList.map(e => (e(0), e(1))),
+        positions = positions.toList.map(pos => Vec3Int(pos(0).toInt, pos(1).toInt, pos(2).toInt)),
+        affinities = affinities.toList
+      )
+    }
   }
 
 }

From 1679d919cfdd7edf114a75941af3ec4162f6bf12 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Mon, 16 May 2022 14:04:31 +0200
Subject: [PATCH 025/122] also send mag and agglomerate ids in editable mapping
 update actions

---
 .../oxalis/model/sagas/proofread_saga.ts      | 48 +++++++++++++++----
 .../oxalis/model/sagas/update_actions.ts      | 15 ++++++
 .../javascripts/oxalis/view/version_entry.tsx |  4 +-
 3 files changed, 57 insertions(+), 10 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index d52ff926a41..010d66b5d39 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -15,7 +15,8 @@ import { pushSaveQueueTransaction } from "oxalis/model/actions/save_actions";
 import { splitAgglomerate, mergeAgglomerate } from "oxalis/model/sagas/update_actions";
 import Model from "oxalis/model";
 import api from "oxalis/api/internal_api";
-import { getActiveSegmentationTracing } from "oxalis/model/accessors/volumetracing_accessor";
+import { getActiveSegmentationTracingLayer } from "oxalis/model/accessors/volumetracing_accessor";
+import { getResolutionInfo } from "oxalis/model/accessors/dataset_accessor";
 
 export default function* proofreadMapping(): Saga<any> {
   yield* take("INITIALIZE_SKELETONTRACING");
@@ -30,10 +31,14 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
   const activeTool = yield* select((state) => state.uiInformation.activeTool);
   if (activeTool !== AnnotationToolEnum.PROOFREAD) return;
 
-  const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
-  if (volumeTracing == null) return;
+  const volumeTracingLayer = yield* select((state) => getActiveSegmentationTracingLayer(state));
+  if (volumeTracingLayer == null) return;
 
-  const layerName = volumeTracing.tracingId;
+  const layerName = volumeTracingLayer.name;
+  const resolutionInfo = getResolutionInfo(volumeTracingLayer.resolutions);
+  // The mag the agglomerate skeleton corresponds to should be the finest available mag of the volume tracing layer
+  const agglomerateFileMag = resolutionInfo.getHighestResolution();
+  const agglomerateFileZoomstep = resolutionInfo.getHighestResolutionPowerOf2();
   const { sourceNodeId, targetNodeId } = action;
 
   const skeletonTracing = yield* select((state) => enforceSkeletonTracing(state.tracing));
@@ -48,20 +53,47 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
 
   const sourceNodePosition = sourceTree.nodes.get(sourceNodeId).position;
   const targetNodePosition = targetTree.nodes.get(targetNodeId).position;
+  const sourceNodeAgglomerateId = yield* call(
+    [api.data, api.data.getDataValue],
+    layerName,
+    sourceNodePosition,
+    agglomerateFileZoomstep,
+  );
+  const targetNodeAgglomerateId = yield* call(
+    [api.data, api.data.getDataValue],
+    layerName,
+    targetNodePosition,
+    agglomerateFileZoomstep,
+  );
 
   const items = [];
   if (action.type === "MERGE_TREES") {
-    if (sourceTree === targetTree) {
+    if (sourceTree === targetTree || sourceNodeAgglomerateId === targetNodeAgglomerateId) {
       Toast.error("Segments that should be merged need to be in different agglomerates.");
       return;
     }
-    items.push(mergeAgglomerate(sourceNodePosition, targetNodePosition));
+    items.push(
+      mergeAgglomerate(
+        sourceNodeAgglomerateId,
+        targetNodeAgglomerateId,
+        sourceNodePosition,
+        targetNodePosition,
+        agglomerateFileMag,
+      ),
+    );
   } else if (action.type === "DELETE_EDGE") {
-    if (sourceTree !== targetTree) {
+    if (sourceTree !== targetTree || sourceNodeAgglomerateId !== targetNodeAgglomerateId) {
       Toast.error("Segments that should be split need to be in the same agglomerate.");
       return;
     }
-    items.push(splitAgglomerate(sourceNodePosition, targetNodePosition));
+    items.push(
+      splitAgglomerate(
+        sourceNodeAgglomerateId,
+        sourceNodePosition,
+        targetNodePosition,
+        agglomerateFileMag,
+      ),
+    );
   }
 
   if (items.length === 0) return;
diff --git a/frontend/javascripts/oxalis/model/sagas/update_actions.ts b/frontend/javascripts/oxalis/model/sagas/update_actions.ts
index e4a1523716f..2ca84aae493 100644
--- a/frontend/javascripts/oxalis/model/sagas/update_actions.ts
+++ b/frontend/javascripts/oxalis/model/sagas/update_actions.ts
@@ -174,15 +174,20 @@ export type UpdateTdCameraUpdateAction = {
 export type SplitAgglomerateUpdateAction = {
   name: "splitAgglomerate";
   value: {
+    agglomerateId: number;
     segmentPosition1: Vector3;
     segmentPosition2: Vector3;
+    mag: Vector3;
   };
 };
 export type MergeAgglomerateUpdateAction = {
   name: "mergeAgglomerate";
   value: {
+    agglomerateId1: number;
+    agglomerateId2: number;
     segmentPosition1: Vector3;
     segmentPosition2: Vector3;
+    mag: Vector3;
   };
 };
 export type UpdateAction =
@@ -529,26 +534,36 @@ export function serverCreateTracing(timestamp: number) {
   };
 }
 export function splitAgglomerate(
+  agglomerateId: number,
   segmentPosition1: Vector3,
   segmentPosition2: Vector3,
+  mag: Vector3,
 ): SplitAgglomerateUpdateAction {
   return {
     name: "splitAgglomerate",
     value: {
+      agglomerateId,
       segmentPosition1,
       segmentPosition2,
+      mag,
     },
   };
 }
 export function mergeAgglomerate(
+  agglomerateId1: number,
+  agglomerateId2: number,
   segmentPosition1: Vector3,
   segmentPosition2: Vector3,
+  mag: Vector3,
 ): MergeAgglomerateUpdateAction {
   return {
     name: "mergeAgglomerate",
     value: {
+      agglomerateId1,
+      agglomerateId2,
       segmentPosition1,
       segmentPosition2,
+      mag,
     },
   };
 }
diff --git a/frontend/javascripts/oxalis/view/version_entry.tsx b/frontend/javascripts/oxalis/view/version_entry.tsx
index bc35143419a..bc526f5fd4a 100644
--- a/frontend/javascripts/oxalis/view/version_entry.tsx
+++ b/frontend/javascripts/oxalis/view/version_entry.tsx
@@ -68,11 +68,11 @@ const descriptionFns: Record<ServerUpdateAction["name"], (...args: any) => Descr
     icon: <DeleteOutlined />,
   }),
   splitAgglomerate: (action: SplitAgglomerateUpdateAction): Description => ({
-    description: `Split an agglomerate by separating the segments at position ${action.value.segmentPosition1} and ${action.value.segmentPosition2}.`,
+    description: `Split agglomerate ${action.value.agglomerateId} by separating the segments at position ${action.value.segmentPosition1} and ${action.value.segmentPosition2}.`,
     icon: <DeleteOutlined />,
   }),
   mergeAgglomerate: (action: MergeAgglomerateUpdateAction): Description => ({
-    description: `Merged two agglomerates by combining the segments at position ${action.value.segmentPosition1} and ${action.value.segmentPosition2}.`,
+    description: `Merged agglomerates ${action.value.agglomerateId1} and ${action.value.agglomerateId2} by combining the segments at position ${action.value.segmentPosition1} and ${action.value.segmentPosition2}.`,
     icon: <PlusOutlined />,
   }),
   deleteTree: (action: DeleteTreeUpdateAction, count: number): Description => ({

From 0fd6a246a9517aef44053a5b6fadff376d616121 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 16 May 2022 14:13:51 +0200
Subject: [PATCH 026/122] include agglomerate id in skeleton uri, directly set
 editable mapping in route, add bool mappingIsEditable

---
 .../services/AgglomerateService.scala         | 10 ++---
 .../proto/VolumeTracing.proto                 |  1 +
 .../controllers/VolumeTracingController.scala | 40 +++++++++++++++++--
 .../EditableMappingService.scala              | 15 +++++++
 .../tracings/volume/VolumeUpdateActions.scala |  5 ++-
 ...alableminds.webknossos.tracingstore.routes |  5 ++-
 6 files changed, 64 insertions(+), 12 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 1b48104b477..8b74e1d0211 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -194,9 +194,10 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
   def largestAgglomerateId(agglomerateFileKey: AgglomerateFileKey): Box[Long] = {
     val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
 
-    val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
-    
-    Full(0L)
+    tryo {
+      val reader = HDF5FactoryProvider.get.openForReading(hdfFile)
+      reader.`object`().getNumberOfElements("/agglomerate_to_segments_offsets") - 1L
+    }
   }
 
   def agglomerateIdsForSegmentIds(agglomerateFileKey: AgglomerateFileKey, segmentIds: List[Long]): Box[List[Long]] = {
@@ -214,7 +215,7 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
 
   }
 
-  def generateAgglomerateGraph(agglomerateFileKey: AgglomerateFileKey, agglomerateId: Long): Box[AgglomerateGraph] = {
+  def generateAgglomerateGraph(agglomerateFileKey: AgglomerateFileKey, agglomerateId: Long): Box[AgglomerateGraph] =
     tryo {
       val hdfFile = agglomerateFileKey.path(dataBaseDir, agglomerateDir, agglomerateFileExtension).toFile
 
@@ -250,6 +251,5 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
         affinities = affinities.toList
       )
     }
-  }
 
 }
diff --git a/webknossos-datastore/proto/VolumeTracing.proto b/webknossos-datastore/proto/VolumeTracing.proto
index 56f95717b2c..e99eb211326 100644
--- a/webknossos-datastore/proto/VolumeTracing.proto
+++ b/webknossos-datastore/proto/VolumeTracing.proto
@@ -37,6 +37,7 @@ message VolumeTracing {
     repeated Vec3IntProto resolutions = 15;
     repeated Segment segments = 16;
     optional string mappingName = 17; // either a mapping present in the fallback layer, or an editable mapping on the tracingstore
+    optional bool mappingIsEditable = 18;
 }
 
 message VolumeTracingOpt {
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index fb499ef55df..45e155878c9 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -12,11 +12,16 @@ import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, Volu
 import com.scalableminds.webknossos.datastore.models.{WebKnossosDataRequest, WebKnossosIsosurfaceRequest}
 import com.scalableminds.webknossos.datastore.services.UserAccessRequest
 import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService
+import com.scalableminds.webknossos.tracingstore.tracings.UpdateActionGroup
 import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{
   EditableMappingService,
   EditableMappingUpdateAction
 }
-import com.scalableminds.webknossos.tracingstore.tracings.volume.{ResolutionRestrictions, VolumeTracingService}
+import com.scalableminds.webknossos.tracingstore.tracings.volume.{
+  ResolutionRestrictions,
+  UpdateMappingNameAction,
+  VolumeTracingService
+}
 import com.scalableminds.webknossos.tracingstore.{TSRemoteWebKnossosClient, TracingStoreAccessTokenService}
 import play.api.i18n.Messages
 import play.api.libs.Files.TemporaryFile
@@ -243,14 +248,29 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
       }
     }
 
-  def createEditableMapping(token: Option[String], tracingId: String): Action[AnyContent] =
+  def makeMappingEditable(token: Option[String], tracingId: String): Action[AnyContent] =
     Action.async { implicit request =>
       accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) {
         for {
           tracing <- tracingService.find(tracingId)
           tracingMappingName <- tracing.mappingName ?~> "annotation.noMappingSet"
           id <- editableMappingService.create(baseMappingName = tracingMappingName)
-        } yield Ok(id)
+          volumeUpdate = UpdateMappingNameAction(id,
+                                                 isEditable = Some(true),
+                                                 actionTimestamp = Some(System.currentTimeMillis()))
+          _ <- tracingService.handleUpdateGroup(
+            tracingId,
+            UpdateActionGroup[VolumeTracing](tracing.version + 1,
+                                             System.currentTimeMillis(),
+                                             List(volumeUpdate),
+                                             None,
+                                             None,
+                                             None,
+                                             None,
+                                             None),
+            tracing.version
+          )
+        } yield Ok
       }
     }
 
@@ -269,4 +289,18 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
         } yield Ok
       }
     }
+
+  def editableMappingUpdateActionLog(token: Option[String], tracingId: String): Action[AnyContent] = Action.async {
+    implicit request =>
+      log() {
+        accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) {
+          for {
+            tracing <- tracingService.find(tracingId)
+            mappingName <- tracing.mappingName.toFox
+            _ <- Fox.assertTrue(editableMappingService.exists(mappingName))
+            updateLog <- editableMappingService.updateActionLog(mappingName)
+          } yield Ok(updateLog)
+        }
+      }
+  }
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index befa5ec1062..ff2c042fd34 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -24,6 +24,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.{
 }
 import net.liftweb.common.Box.tryo
 import net.liftweb.common.{Empty, Full}
+import play.api.libs.json.{JsObject, JsValue, Json}
 
 import scala.collection.mutable
 import scala.concurrent.ExecutionContext
@@ -61,6 +62,20 @@ class EditableMappingService @Inject()(
                                                                               emptyFallback = Some(-1L))
     } yield versionOrMinusOne >= 0
 
+  def updateActionLog(editableMappingId: String): Fox[JsValue] = {
+    def versionedTupleToJson(tuple: (Long, List[EditableMappingUpdateAction])): JsObject =
+      Json.obj(
+        "version" -> tuple._1,
+        "value" -> Json.toJson(tuple._2)
+      )
+
+    for {
+      updates <- tracingDataStore.editableMappingUpdates.getMultipleVersionsAsVersionValueTuple(editableMappingId)(
+        fromJson[List[EditableMappingUpdateAction]])
+      updateActionGroupsJs = updates.map(versionedTupleToJson)
+    } yield Json.toJson(updateActionGroupsJs)
+  }
+
   private def get(editableMappingId: String,
                   remoteFallbackLayer: RemoteFallbackLayer,
                   userToken: Option[String],
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala
index 16a924b6761..3edff995a77 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala
@@ -242,7 +242,8 @@ object DeleteSegmentVolumeAction {
   implicit val jsonFormat: OFormat[DeleteSegmentVolumeAction] = Json.format[DeleteSegmentVolumeAction]
 }
 
-case class UpdateMappingNameAction(mappingName: String, actionTimestamp: Option[Long]) extends ApplyableVolumeAction {
+case class UpdateMappingNameAction(mappingName: String, isEditable: Option[Boolean], actionTimestamp: Option[Long])
+    extends ApplyableVolumeAction {
   override def addTimestamp(timestamp: Long): VolumeUpdateAction =
     this.copy(actionTimestamp = Some(timestamp))
 
@@ -250,7 +251,7 @@ case class UpdateMappingNameAction(mappingName: String, actionTimestamp: Option[
     CompactVolumeUpdateAction("updateMappingName", actionTimestamp, Json.obj("mappingName" -> mappingName))
 
   override def applyOn(tracing: VolumeTracing): VolumeTracing =
-    tracing.withMappingName(mappingName)
+    tracing.withMappingName(mappingName).withMappingIsEditable(isEditable.getOrElse(false))
 }
 
 object UpdateMappingNameAction {
diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
index 85bed33bbe5..6eca5bddb46 100644
--- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
+++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
@@ -20,9 +20,10 @@ GET         /volume/:tracingId/updateActionLog                  @com.scalablemin
 POST        /volume/:tracingId/isosurface                       @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.requestIsosurface(token: Option[String], tracingId: String)
 POST        /volume/:tracingId/importVolumeData                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.importVolumeData(token: Option[String], tracingId: String)
 GET         /volume/:tracingId/findData                         @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.findData(token: Option[String], tracingId: String)
-GET         /volume/:tracingId/agglomerateSkeleton              @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateSkeleton(token: Option[String], tracingId: String, agglomerateId: Long)
-POST        /volume/:tracingId/createEditableMapping            @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.createEditableMapping(token: Option[String], tracingId: String)
+GET         /volume/:tracingId/agglomerateSkeleton/:agglomerateId     @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateSkeleton(token: Option[String], tracingId: String, agglomerateId: Long)
+POST        /volume/:tracingId/makeMappingEditable              @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.makeMappingEditable(token: Option[String], tracingId: String)
 POST        /volume/:tracingId/updateEditableMapping            @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.updateEditableMapping(token: Option[String], tracingId: String, version: Long)
+GET         /volume/:tracingId/editableMappingUpdateActionLog   @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingUpdateActionLog(token: Option[String], tracingId: String)
 POST        /volume/getMultiple                                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getMultiple(token: Option[String])
 POST        /volume/mergedFromIds                               @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromIds(token: Option[String], persist: Boolean)
 POST        /volume/mergedFromContents                          @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromContents(token: Option[String], persist: Boolean)

From 0ca70390e4c815b26c0c1f0cdfa6554a6179fe60 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Mon, 16 May 2022 16:01:24 +0200
Subject: [PATCH 027/122] persist mappingName in volume annotation and restore
 mapping on load

---
 .../model/actions/volumetracing_actions.ts    |  3 ++
 .../model/reducers/volumetracing_reducer.ts   | 28 ++++++++++++++++++-
 .../reducers/volumetracing_reducer_helpers.ts | 19 +++++++++++--
 .../oxalis/model/sagas/mapping_saga.ts        |  2 +-
 .../oxalis/model/sagas/save_saga.ts           |  6 ++++
 .../oxalis/model/sagas/update_actions.ts      | 25 ++++++++++++++---
 .../oxalis/model/sagas/volumetracing_saga.tsx |  5 ++++
 .../oxalis/model_initialization.ts            | 23 +++++++++++++++
 frontend/javascripts/oxalis/store.ts          |  2 ++
 .../javascripts/oxalis/view/version_entry.tsx |  8 ++++++
 frontend/javascripts/types/api_flow_types.ts  |  2 ++
 11 files changed, 115 insertions(+), 8 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts b/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts
index 9195e9b9120..aecda03e823 100644
--- a/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts
+++ b/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts
@@ -132,6 +132,9 @@ export const VolumeTracingSaveRelevantActions = [
   "UPDATE_SEGMENT",
   "SET_SEGMENTS",
   ...AllUserBoundingBoxActions,
+  // Note that the following two actions are defined in settings_actions.ts
+  "SET_MAPPING",
+  "SET_MAPPING_ENABLED",
 ];
 export const VolumeTracingUndoRelevantActions = ["START_EDITING", "COPY_SEGMENTATION_LAYER"];
 export const initializeVolumeTracingAction = (
diff --git a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
index b3276f806ce..57359b10a62 100644
--- a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
+++ b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
@@ -24,11 +24,14 @@ import {
   setContourTracingModeReducer,
   setMaxCellReducer,
   updateVolumeTracing,
+  setMappingNameReducer,
 } from "oxalis/model/reducers/volumetracing_reducer_helpers";
 import { updateKey2 } from "oxalis/model/helpers/deep_update";
 import DiffableMap from "libs/diffable_map";
 import * as Utils from "libs/utils";
 import type { ServerVolumeTracing } from "types/api_flow_types";
+import { SetMappingAction, SetMappingEnabledAction } from "../actions/settings_actions";
+import { getMappingInfo } from "../accessors/dataset_accessor";
 type SegmentUpdateInfo =
   | {
       readonly type: "UPDATE_VOLUME_TRACING";
@@ -171,12 +174,17 @@ export function serverVolumeToClientVolumeTracing(tracing: ServerVolumeTracing):
     boundingBox: convertServerBoundingBoxToFrontend(tracing.boundingBox),
     fallbackLayer: tracing.fallbackLayer,
     userBoundingBoxes,
+    mappingName: tracing.mappingName,
+    mappingIsEditable: tracing.mappingIsEditable,
   };
   // @ts-expect-error ts-migrate(2322) FIXME: Type '{ createdTimestamp: number; type: string; se... Remove this comment to see the full error message
   return volumeTracing;
 }
 
-function VolumeTracingReducer(state: OxalisState, action: VolumeTracingAction): OxalisState {
+function VolumeTracingReducer(
+  state: OxalisState,
+  action: VolumeTracingAction | SetMappingAction | SetMappingEnabledAction,
+): OxalisState {
   switch (action.type) {
     case "INITIALIZE_VOLUMETRACING": {
       const volumeTracing = serverVolumeToClientVolumeTracing(action.tracing);
@@ -258,6 +266,24 @@ function VolumeTracingReducer(state: OxalisState, action: VolumeTracingAction):
       return setMaxCellReducer(state, volumeTracing, Math.max(activeCellId, maxCellId));
     }
 
+    case "SET_MAPPING": {
+      return setMappingNameReducer(state, volumeTracing, action.mappingName, action.mappingType);
+    }
+
+    case "SET_MAPPING_ENABLED": {
+      const { mappingName, mappingType } = getMappingInfo(
+        state.temporaryConfiguration.activeMappingByLayer,
+        action.layerName,
+      );
+      return setMappingNameReducer(
+        state,
+        volumeTracing,
+        mappingName,
+        mappingType,
+        action.isMappingEnabled,
+      );
+    }
+
     default:
       return state;
   }
diff --git a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts
index cf42e0d319d..84fb2029d8e 100644
--- a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts
+++ b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts
@@ -1,6 +1,6 @@
 import update from "immutability-helper";
-import type { ContourMode, Vector3 } from "oxalis/constants";
-import type { OxalisState, VolumeTracing } from "oxalis/store";
+import { ContourMode, Vector3 } from "oxalis/constants";
+import type { MappingType, OxalisState, VolumeTracing } from "oxalis/store";
 import { isVolumeAnnotationDisallowedForZoom } from "oxalis/model/accessors/volumetracing_accessor";
 import { setDirectionReducer } from "oxalis/model/reducers/flycam_reducer";
 import { updateKey } from "oxalis/model/helpers/deep_update";
@@ -106,3 +106,18 @@ export function setMaxCellReducer(state: OxalisState, volumeTracing: VolumeTraci
     maxCellId: id,
   });
 }
+export function setMappingNameReducer(
+  state: OxalisState,
+  volumeTracing: VolumeTracing,
+  mappingName: string | null | undefined,
+  mappingType: MappingType,
+  isMappingEnabled: boolean = true,
+) {
+  // Only HDF5 mappings are persisted in volume annotations for now
+  if (mappingType !== "HDF5" || !isMappingEnabled) {
+    mappingName = null;
+  }
+  return updateVolumeTracing(state, volumeTracing.tracingId, {
+    mappingName,
+  });
+}
diff --git a/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts b/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts
index 19709179f44..a1cb62993c2 100644
--- a/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts
@@ -191,7 +191,7 @@ function* getLargestSegmentId(layerName: string): Saga<number> {
   const segmentationLayer = getLayerByName(dataset, layerName);
 
   if (segmentationLayer.category !== "segmentation") {
-    throw new Error("Mappings class must be instantiated with a segmentation layer.");
+    throw new Error("Mappings only exist for segmentation layers.");
   }
 
   return segmentationLayer.largestSegmentId;
diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
index 72885e9df90..3b2aae1563b 100644
--- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
@@ -102,6 +102,10 @@ import compactUpdateActions from "oxalis/model/helpers/compaction/compact_update
 import createProgressCallback from "libs/progress_callback";
 import messages from "messages";
 import window, { alert, document, location } from "libs/window";
+import type {
+  SetMappingAction,
+  SetMappingEnabledAction,
+} from "oxalis/model/actions/settings_actions";
 import { enforceSkeletonTracing } from "../accessors/skeletontracing_accessor";
 
 // This function is needed so that Flow is satisfied
@@ -1015,6 +1019,8 @@ export function* saveTracingTypeAsync(
         ...SkeletonTracingSaveRelevantActions,
         ...FlycamActions,
         ...ViewModeSaveRelevantActions,
+        // SET_TRACING is not included in SkeletonTracingSaveRelevantActions, because it is used by Undo/Redo and
+        // should not create its own Undo/Redo stack entry
         "SET_TRACING",
       ]);
     } else {
diff --git a/frontend/javascripts/oxalis/model/sagas/update_actions.ts b/frontend/javascripts/oxalis/model/sagas/update_actions.ts
index 2ca84aae493..d6b4d738c27 100644
--- a/frontend/javascripts/oxalis/model/sagas/update_actions.ts
+++ b/frontend/javascripts/oxalis/model/sagas/update_actions.ts
@@ -171,6 +171,12 @@ export type UpdateTdCameraUpdateAction = {
   name: "updateTdCamera";
   value: {};
 };
+export type UpdateMappingNameUpdateAction = {
+  name: "updateMappingName";
+  value: {
+    mappingName: string | null | undefined;
+  };
+};
 export type SplitAgglomerateUpdateAction = {
   name: "splitAgglomerate";
   value: {
@@ -213,6 +219,7 @@ export type UpdateAction =
   | UpdateTreeGroupsUpdateAction
   | RemoveFallbackLayerUpdateAction
   | UpdateTdCameraUpdateAction
+  | UpdateMappingNameUpdateAction
   | SplitAgglomerateUpdateAction
   | MergeAgglomerateUpdateAction;
 // This update action is only created in the frontend for display purposes
@@ -234,7 +241,7 @@ type AddServerValuesFn<T extends { value: any }> = (arg0: T) => T & {
 };
 
 type AsServerAction<A extends { value: any }> = ReturnType<AddServerValuesFn<A>>;
-// Since flow does not provide ways to perform type transformations on the
+// Since typescript does not provide ways to perform type transformations on the
 // single parts of a union, we need to write this out manually.
 export type ServerUpdateAction =
   | AsServerAction<UpdateTreeUpdateAction>
@@ -257,12 +264,14 @@ export type ServerUpdateAction =
   | AsServerAction<UpdateTreeGroupVisibilityUpdateAction>
   | AsServerAction<RevertToVersionUpdateAction>
   | AsServerAction<UpdateTreeGroupsUpdateAction>
-  | AsServerAction<CreateTracingUpdateAction>
   | AsServerAction<RemoveFallbackLayerUpdateAction>
   | AsServerAction<UpdateTdCameraUpdateAction>
-  | AsServerAction<ImportVolumeTracingUpdateAction>
+  | AsServerAction<UpdateMappingNameUpdateAction>
   | AsServerAction<SplitAgglomerateUpdateAction>
-  | AsServerAction<MergeAgglomerateUpdateAction>;
+  | AsServerAction<MergeAgglomerateUpdateAction>
+  // These two actions are never sent by the frontend and, therefore, don't exist in the UpdateAction type
+  | AsServerAction<ImportVolumeTracingUpdateAction>
+  | AsServerAction<CreateTracingUpdateAction>;
 export function createTree(tree: Tree): UpdateTreeUpdateAction {
   return {
     name: "createTree",
@@ -533,6 +542,14 @@ export function serverCreateTracing(timestamp: number) {
     },
   };
 }
+export function updateMappingName(
+  mappingName: string | null | undefined,
+): UpdateMappingNameUpdateAction {
+  return {
+    name: "updateMappingName",
+    value: { mappingName },
+  };
+}
 export function splitAgglomerate(
   agglomerateId: number,
   segmentPosition1: Vector3,
diff --git a/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx b/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx
index 1dfcc6797d0..6dd000f1d7d 100644
--- a/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx
+++ b/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx
@@ -84,6 +84,7 @@ import {
   updateSegmentVolumeAction,
   updateUserBoundingBoxes,
   updateVolumeTracing,
+  updateMappingName,
 } from "oxalis/model/sagas/update_actions";
 import VolumeLayer, { getFast3DCoordinateHelper } from "oxalis/model/volumetracing/volumelayer";
 import { applyVoxelMap } from "oxalis/model/volumetracing/volume_annotation_sampling";
@@ -710,6 +711,10 @@ export function* diffVolumeTracing(
   if (prevVolumeTracing.fallbackLayer != null && volumeTracing.fallbackLayer == null) {
     yield removeFallbackLayer();
   }
+
+  if (prevVolumeTracing.mappingName !== volumeTracing.mappingName) {
+    yield updateMappingName(volumeTracing.mappingName);
+  }
 }
 
 function* ensureSegmentExists(
diff --git a/frontend/javascripts/oxalis/model_initialization.ts b/frontend/javascripts/oxalis/model_initialization.ts
index f6dd7340148..84bf60da740 100644
--- a/frontend/javascripts/oxalis/model_initialization.ts
+++ b/frontend/javascripts/oxalis/model_initialization.ts
@@ -552,6 +552,7 @@ function determineDefaultState(
     zoomStep: urlStateZoomStep,
     rotation: urlStateRotation,
     activeNode: urlStateActiveNode,
+    stateByLayer: urlStateByLayer,
     ...rest
   } = urlState;
   // If there is no editPosition (e.g. when viewing a dataset) and
@@ -594,12 +595,34 @@ function determineDefaultState(
     rotation = urlStateRotation;
   }
 
+  const stateByLayer = urlStateByLayer ?? {};
+
+  const volumeTracings = tracings.filter(
+    (tracing) => tracing.typ === "Volume",
+  ) as ServerVolumeTracing[];
+  for (const volumeTracing of volumeTracings) {
+    const { id: layerName, mappingName } = volumeTracing;
+
+    if (mappingName == null) continue;
+
+    if (!(layerName in stateByLayer)) {
+      stateByLayer[layerName] = {};
+    }
+    if (stateByLayer[layerName].mappingInfo == null) {
+      stateByLayer[layerName].mappingInfo = {
+        mappingName,
+        mappingType: "HDF5",
+      };
+    }
+  }
+
   const activeNode = urlStateActiveNode;
   return {
     position,
     zoomStep,
     rotation,
     activeNode,
+    stateByLayer,
     ...rest,
   };
 }
diff --git a/frontend/javascripts/oxalis/store.ts b/frontend/javascripts/oxalis/store.ts
index 6edaba29f9c..0a162e9d018 100644
--- a/frontend/javascripts/oxalis/store.ts
+++ b/frontend/javascripts/oxalis/store.ts
@@ -222,6 +222,8 @@ export type VolumeTracing = TracingBase & {
   // Stores points of the currently drawn region in global coordinates
   readonly contourList: Array<Vector3>;
   readonly fallbackLayer?: string;
+  readonly mappingName?: string | null | undefined;
+  readonly mappingIsEditable?: boolean;
 };
 export type ReadOnlyTracing = TracingBase & {
   readonly type: "readonly";
diff --git a/frontend/javascripts/oxalis/view/version_entry.tsx b/frontend/javascripts/oxalis/view/version_entry.tsx
index bc526f5fd4a..02372ec3012 100644
--- a/frontend/javascripts/oxalis/view/version_entry.tsx
+++ b/frontend/javascripts/oxalis/view/version_entry.tsx
@@ -35,6 +35,7 @@ import type {
   DeleteSegmentUpdateAction,
   MoveTreeComponentUpdateAction,
   MergeTreeUpdateAction,
+  UpdateMappingNameUpdateAction,
 } from "oxalis/model/sagas/update_actions";
 import FormattedDate from "components/formatted_date";
 import { MISSING_GROUP_ID } from "oxalis/view/right-border-tabs/tree_hierarchy_view_helpers";
@@ -67,6 +68,13 @@ const descriptionFns: Record<ServerUpdateAction["name"], (...args: any) => Descr
     description: "Removed the segmentation fallback layer.",
     icon: <DeleteOutlined />,
   }),
+  updateMappingName: (action: UpdateMappingNameUpdateAction): Description => ({
+    description:
+      action.value.mappingName != null
+        ? `Activated mapping ${action.value.mappingName}.`
+        : "Deactivated the active mapping.",
+    icon: <EyeOutlined />,
+  }),
   splitAgglomerate: (action: SplitAgglomerateUpdateAction): Description => ({
     description: `Split agglomerate ${action.value.agglomerateId} by separating the segments at position ${action.value.segmentPosition1} and ${action.value.segmentPosition2}.`,
     icon: <DeleteOutlined />,
diff --git a/frontend/javascripts/types/api_flow_types.ts b/frontend/javascripts/types/api_flow_types.ts
index a12ffb20872..c81e4a96595 100644
--- a/frontend/javascripts/types/api_flow_types.ts
+++ b/frontend/javascripts/types/api_flow_types.ts
@@ -608,6 +608,8 @@ export type ServerVolumeTracing = ServerTracingBase & {
   // https://github.com/scalableminds/webknossos/pull/4755
   resolutions?: Array<Point3>;
   organizationName?: string;
+  mappingName?: string | null | undefined;
+  mappingIsEditable?: boolean;
 };
 export type ServerTracing = ServerSkeletonTracing | ServerVolumeTracing;
 export type APIMeshFile = {

From c78490fe70c35bb37a63bda490de1608410f8ac9 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Mon, 16 May 2022 17:29:55 +0200
Subject: [PATCH 028/122] make mapping editable, lazily (before first
 proofreading action)

---
 frontend/javascripts/admin/admin_rest_api.ts  | 11 ++++
 .../model/actions/volumetracing_actions.ts    |  9 +++-
 .../model/reducers/volumetracing_reducer.ts   |  6 +++
 .../oxalis/model/sagas/proofread_saga.ts      | 53 ++++++++++++++++---
 4 files changed, 71 insertions(+), 8 deletions(-)

diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts
index d1ef2a428ec..664a95cf970 100644
--- a/frontend/javascripts/admin/admin_rest_api.ts
+++ b/frontend/javascripts/admin/admin_rest_api.ts
@@ -1562,6 +1562,17 @@ export function fetchMapping(
   );
 }
 
+export function makeMappingEditable(tracingStoreUrl: string, tracingId: string): Promise<void> {
+  return doWithToken((token) =>
+    Request.triggerRequest(
+      `${tracingStoreUrl}/tracings/volume/${tracingId}/makeMappingEditable?token=${token}`,
+      {
+        method: "POST",
+      },
+    ),
+  );
+}
+
 export async function getAgglomeratesForDatasetLayer(
   datastoreUrl: string,
   datasetId: APIDatasetId,
diff --git a/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts b/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts
index aecda03e823..50f2ac0b017 100644
--- a/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts
+++ b/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts
@@ -103,6 +103,9 @@ export type UpdateSegmentAction = {
   layerName: string;
   timestamp: number;
 };
+export type SetMappingIsEditableAction = {
+  type: "SET_MAPPING_IS_EDITABLE";
+};
 export type VolumeTracingAction =
   | InitializeVolumeTracingAction
   | CreateCellAction
@@ -124,7 +127,8 @@ export type VolumeTracingAction =
   | UpdateSegmentAction
   | AddBucketToUndoAction
   | ImportVolumeTracingAction
-  | SetMaxCellAction;
+  | SetMaxCellAction
+  | SetMappingIsEditableAction;
 export const VolumeTracingSaveRelevantActions = [
   "CREATE_CELL",
   "SET_ACTIVE_CELL",
@@ -266,3 +270,6 @@ export const dispatchFloodfillAsync = async (
   dispatch(action);
   await readyDeferred.promise();
 };
+export const setMappingisEditableAction = (): SetMappingIsEditableAction => ({
+  type: "SET_MAPPING_IS_EDITABLE",
+});
diff --git a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
index 57359b10a62..153664b4256 100644
--- a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
+++ b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
@@ -284,6 +284,12 @@ function VolumeTracingReducer(
       );
     }
 
+    case "SET_MAPPING_IS_EDITABLE": {
+      return updateVolumeTracing(state, volumeTracing.tracingId, {
+        mappingIsEditable: true,
+      });
+    }
+
     default:
       return state;
   }
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 010d66b5d39..518f413626e 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -1,22 +1,30 @@
 import type { Saga } from "oxalis/model/sagas/effect-generators";
 import { takeEvery, put, call } from "typed-redux-saga";
 import { select, take } from "oxalis/model/sagas/effect-generators";
-import { AnnotationToolEnum } from "oxalis/constants";
+import { AnnotationToolEnum, MappingStatusEnum } from "oxalis/constants";
 import Toast from "libs/toast";
 import type {
   DeleteEdgeAction,
   MergeTreesAction,
 } from "oxalis/model/actions/skeletontracing_actions";
+import { setMappingisEditableAction } from "oxalis/model/actions/volumetracing_actions";
 import {
   enforceSkeletonTracing,
   findTreeByNodeId,
 } from "oxalis/model/accessors/skeletontracing_accessor";
-import { pushSaveQueueTransaction } from "oxalis/model/actions/save_actions";
+import {
+  pushSaveQueueTransaction,
+  setVersionNumberAction,
+} from "oxalis/model/actions/save_actions";
 import { splitAgglomerate, mergeAgglomerate } from "oxalis/model/sagas/update_actions";
 import Model from "oxalis/model";
 import api from "oxalis/api/internal_api";
-import { getActiveSegmentationTracingLayer } from "oxalis/model/accessors/volumetracing_accessor";
-import { getResolutionInfo } from "oxalis/model/accessors/dataset_accessor";
+import {
+  getActiveSegmentationTracingLayer,
+  getActiveSegmentationTracing,
+} from "oxalis/model/accessors/volumetracing_accessor";
+import { getMappingInfo, getResolutionInfo } from "oxalis/model/accessors/dataset_accessor";
+import { makeMappingEditable } from "admin/admin_rest_api";
 
 export default function* proofreadMapping(): Saga<any> {
   yield* take("INITIALIZE_SKELETONTRACING");
@@ -33,17 +41,48 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
 
   const volumeTracingLayer = yield* select((state) => getActiveSegmentationTracingLayer(state));
   if (volumeTracingLayer == null) return;
+  const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
+  if (volumeTracing == null) return;
+  const { tracingId: volumeTracingId } = volumeTracing;
 
   const layerName = volumeTracingLayer.name;
+  const activeMappingByLayer = yield* select(
+    (state) => state.temporaryConfiguration.activeMappingByLayer,
+  );
+  const mappingInfo = getMappingInfo(activeMappingByLayer, layerName);
+  const { mappingName, mappingType, mappingStatus } = mappingInfo;
+  if (
+    mappingName == null ||
+    mappingType !== "HDF5" ||
+    mappingStatus === MappingStatusEnum.DISABLED
+  ) {
+    Toast.error("An HDF5 mapping needs to be enabled to use the proofreading tool.");
+  }
+
+  if (!volumeTracing.mappingIsEditable) {
+    const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url);
+    // Save before making the mapping editable to make sure the correct mapping is activated in the backend
+    yield* call([Model, Model.ensureSavedState]);
+    // Get volume tracing again to make sure the version is up to date
+    const upToDateVolumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
+    if (upToDateVolumeTracing == null) return;
+
+    yield* call(makeMappingEditable, tracingStoreUrl, volumeTracingId);
+    yield* put(setMappingisEditableAction());
+    yield* put(
+      setVersionNumberAction(upToDateVolumeTracing.version + 1, "volume", volumeTracingId),
+    );
+  }
+
   const resolutionInfo = getResolutionInfo(volumeTracingLayer.resolutions);
   // The mag the agglomerate skeleton corresponds to should be the finest available mag of the volume tracing layer
   const agglomerateFileMag = resolutionInfo.getHighestResolution();
-  const agglomerateFileZoomstep = resolutionInfo.getHighestResolutionPowerOf2();
+  const agglomerateFileZoomstep = resolutionInfo.getHighestResolutionIndex();
   const { sourceNodeId, targetNodeId } = action;
 
   const skeletonTracing = yield* select((state) => enforceSkeletonTracing(state.tracing));
 
-  const { trees, tracingId, type: tracingType } = skeletonTracing;
+  const { trees } = skeletonTracing;
   const sourceTree = findTreeByNodeId(trees, sourceNodeId).getOrElse(null);
   const targetTree = findTreeByNodeId(trees, targetNodeId).getOrElse(null);
 
@@ -99,7 +138,7 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
   if (items.length === 0) return;
 
   // TODO: Will there be a separate end point for these update actions?
-  yield* put(pushSaveQueueTransaction(items, tracingType, tracingId));
+  yield* put(pushSaveQueueTransaction(items, "editableMapping", volumeTracingId));
   yield* call([Model, Model.ensureSavedState]);
 
   yield* call([api.data, api.data.reloadBuckets], layerName);

From 146085267f79d75ea6acbe3f04ac02216559903c Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Mon, 16 May 2022 18:24:53 +0200
Subject: [PATCH 029/122] add new save queue for mappings and improve save
 queue typing

---
 frontend/javascripts/admin/admin_rest_api.ts  |  5 +-
 frontend/javascripts/oxalis/default_state.ts  |  3 +
 .../oxalis/model/accessors/save_accessor.ts   | 20 ++++---
 .../oxalis/model/actions/save_actions.ts      | 34 +++++------
 .../oxalis/model/reducers/save_reducer.ts     | 17 +++---
 .../oxalis/model/sagas/proofread_saga.ts      |  2 +-
 .../oxalis/model/sagas/save_saga.ts           | 56 +++++++++----------
 frontend/javascripts/oxalis/store.ts          |  8 +--
 .../javascripts/oxalis/view/version_list.tsx  | 13 +++--
 .../javascripts/oxalis/view/version_view.tsx  |  4 +-
 10 files changed, 85 insertions(+), 77 deletions(-)

diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts
index 664a95cf970..0366e5194cb 100644
--- a/frontend/javascripts/admin/admin_rest_api.ts
+++ b/frontend/javascripts/admin/admin_rest_api.ts
@@ -80,6 +80,7 @@ import Toast from "libs/toast";
 import * as Utils from "libs/utils";
 import messages from "messages";
 import window, { location } from "libs/window";
+import { SaveQueueType } from "oxalis/model/actions/save_actions";
 
 const MAX_SERVER_ITEMS_PER_RESPONSE = 1000;
 
@@ -835,11 +836,11 @@ export async function getTracingForAnnotationType(
 export function getUpdateActionLog(
   tracingStoreUrl: string,
   tracingId: string,
-  tracingType: "skeleton" | "volume",
+  versionedObjectType: SaveQueueType,
 ): Promise<Array<APIUpdateActionBatch>> {
   return doWithToken((token) =>
     Request.receiveJSON(
-      `${tracingStoreUrl}/tracings/${tracingType}/${tracingId}/updateActionLog?token=${token}`,
+      `${tracingStoreUrl}/tracings/${versionedObjectType}/${tracingId}/updateActionLog?token=${token}`,
     ),
   );
 }
diff --git a/frontend/javascripts/oxalis/default_state.ts b/frontend/javascripts/oxalis/default_state.ts
index 9528008a993..b373740fd7b 100644
--- a/frontend/javascripts/oxalis/default_state.ts
+++ b/frontend/javascripts/oxalis/default_state.ts
@@ -153,14 +153,17 @@ const defaultState: OxalisState = {
     queue: {
       skeleton: [],
       volumes: {},
+      mappings: {},
     },
     isBusyInfo: {
       skeleton: false,
       volume: false,
+      mapping: false,
     },
     lastSaveTimestamp: {
       skeleton: 0,
       volumes: {},
+      mappings: {},
     },
     progressInfo: {
       processedActionCount: 0,
diff --git a/frontend/javascripts/oxalis/model/accessors/save_accessor.ts b/frontend/javascripts/oxalis/model/accessors/save_accessor.ts
index 8eeb68a7c17..299ec12f6ca 100644
--- a/frontend/javascripts/oxalis/model/accessors/save_accessor.ts
+++ b/frontend/javascripts/oxalis/model/accessors/save_accessor.ts
@@ -1,15 +1,21 @@
-import type { IsBusyInfo, OxalisState } from "oxalis/store";
+import type { IsBusyInfo, OxalisState, SaveQueueEntry } from "oxalis/store";
+import type { SaveQueueType } from "oxalis/model/actions/save_actions";
 export function isBusy(isBusyInfo: IsBusyInfo): boolean {
   return isBusyInfo.skeleton || isBusyInfo.volume;
 }
 export function selectQueue(
   state: OxalisState,
-  tracingType: "skeleton" | "volume",
+  saveQueueType: SaveQueueType,
   tracingId: string,
-) {
-  if (tracingType === "skeleton") {
-    return state.save.queue.skeleton;
+): Array<SaveQueueEntry> {
+  switch (saveQueueType) {
+    case "skeleton":
+      return state.save.queue.skeleton;
+    case "volume":
+      return state.save.queue.volumes[tracingId];
+    case "mapping":
+      return state.save.queue.mappings[tracingId];
+    default:
+      throw new Error(`Unknown save queue type: ${saveQueueType}`);
   }
-
-  return state.save.queue.volumes[tracingId];
 }
diff --git a/frontend/javascripts/oxalis/model/actions/save_actions.ts b/frontend/javascripts/oxalis/model/actions/save_actions.ts
index 7fa0cbf49ab..3aa7026f17a 100644
--- a/frontend/javascripts/oxalis/model/actions/save_actions.ts
+++ b/frontend/javascripts/oxalis/model/actions/save_actions.ts
@@ -2,11 +2,11 @@ import type { UpdateAction } from "oxalis/model/sagas/update_actions";
 import { getUid } from "libs/uid_generator";
 import Date from "libs/date";
 import Deferred from "libs/deferred";
-type TracingType = "skeleton" | "volume";
+export type SaveQueueType = "skeleton" | "volume" | "mapping";
 export type PushSaveQueueTransaction = {
   type: "PUSH_SAVE_QUEUE_TRANSACTION";
   items: Array<UpdateAction>;
-  tracingType: TracingType;
+  saveQueueType: SaveQueueType;
   tracingId: string;
   transactionId: string;
 };
@@ -16,7 +16,7 @@ type SaveNowAction = {
 export type ShiftSaveQueueAction = {
   type: "SHIFT_SAVE_QUEUE";
   count: number;
-  tracingType: TracingType;
+  saveQueueType: SaveQueueType;
   tracingId: string;
 };
 type DiscardSaveQueuesAction = {
@@ -25,18 +25,18 @@ type DiscardSaveQueuesAction = {
 type SetSaveBusyAction = {
   type: "SET_SAVE_BUSY";
   isBusy: boolean;
-  tracingType: TracingType;
+  saveQueueType: SaveQueueType;
 };
-type SetLastSaveTimestampAction = {
+export type SetLastSaveTimestampAction = {
   type: "SET_LAST_SAVE_TIMESTAMP";
   timestamp: number;
-  tracingType: TracingType;
+  saveQueueType: SaveQueueType;
   tracingId: string;
 };
 export type SetVersionNumberAction = {
   type: "SET_VERSION_NUMBER";
   version: number;
-  tracingType: TracingType;
+  saveQueueType: SaveQueueType;
   tracingId: string;
 };
 export type UndoAction = {
@@ -63,13 +63,13 @@ export type SaveAction =
   | DisableSavingAction;
 export const pushSaveQueueTransaction = (
   items: Array<UpdateAction>,
-  tracingType: TracingType,
+  saveQueueType: SaveQueueType,
   tracingId: string,
   transactionId: string = getUid(),
 ): PushSaveQueueTransaction => ({
   type: "PUSH_SAVE_QUEUE_TRANSACTION",
   items,
-  tracingType,
+  saveQueueType,
   tracingId,
   transactionId,
 });
@@ -78,12 +78,12 @@ export const saveNowAction = (): SaveNowAction => ({
 });
 export const shiftSaveQueueAction = (
   count: number,
-  tracingType: TracingType,
+  saveQueueType: SaveQueueType,
   tracingId: string,
 ): ShiftSaveQueueAction => ({
   type: "SHIFT_SAVE_QUEUE",
   count,
-  tracingType,
+  saveQueueType,
   tracingId,
 });
 export const discardSaveQueuesAction = (): DiscardSaveQueuesAction => ({
@@ -91,29 +91,29 @@ export const discardSaveQueuesAction = (): DiscardSaveQueuesAction => ({
 });
 export const setSaveBusyAction = (
   isBusy: boolean,
-  tracingType: TracingType,
+  saveQueueType: SaveQueueType,
 ): SetSaveBusyAction => ({
   type: "SET_SAVE_BUSY",
   isBusy,
-  tracingType,
+  saveQueueType,
 });
 export const setLastSaveTimestampAction = (
-  tracingType: TracingType,
+  saveQueueType: SaveQueueType,
   tracingId: string,
 ): SetLastSaveTimestampAction => ({
   type: "SET_LAST_SAVE_TIMESTAMP",
   timestamp: Date.now(),
-  tracingType,
+  saveQueueType,
   tracingId,
 });
 export const setVersionNumberAction = (
   version: number,
-  tracingType: TracingType,
+  saveQueueType: SaveQueueType,
   tracingId: string,
 ): SetVersionNumberAction => ({
   type: "SET_VERSION_NUMBER",
   version,
-  tracingType,
+  saveQueueType,
   tracingId,
 });
 export const undoAction = (callback?: () => void): UndoAction => ({
diff --git a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts
index 864ee01f90f..1f6ffd05a2b 100644
--- a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts
+++ b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts
@@ -6,6 +6,7 @@ import type {
   PushSaveQueueTransaction,
   SetVersionNumberAction,
   ShiftSaveQueueAction,
+  SetLastSaveTimestampAction,
 } from "oxalis/model/actions/save_actions";
 import { getActionLog } from "oxalis/model/helpers/action_logger_middleware";
 import { getStats } from "oxalis/model/accessors/skeletontracing_accessor";
@@ -21,7 +22,7 @@ function updateQueueObj(
   oldQueueObj: SaveState["queue"],
   newQueue: any,
 ): SaveState["queue"] {
-  if (action.tracingType === "skeleton") {
+  if (action.saveQueueType === "skeleton") {
     return { ...oldQueueObj, skeleton: newQueue };
   }
 
@@ -36,7 +37,7 @@ export function getTotalSaveQueueLength(queueObj: SaveState["queue"]) {
 }
 
 function updateVersion(state: OxalisState, action: SetVersionNumberAction) {
-  if (action.tracingType === "skeleton" && state.tracing.skeleton != null) {
+  if (action.saveQueueType === "skeleton" && state.tracing.skeleton != null) {
     return updateKey2(state, "tracing", "skeleton", {
       version: action.version,
     });
@@ -47,9 +48,8 @@ function updateVersion(state: OxalisState, action: SetVersionNumberAction) {
   });
 }
 
-// @ts-expect-error ts-migrate(7006) FIXME: Parameter 'state' implicitly has an 'any' type.
-function updateLastSaveTimestamp(state, action) {
-  if (action.tracingType === "skeleton") {
+function updateLastSaveTimestamp(state: OxalisState, action: SetLastSaveTimestampAction) {
+  if (action.saveQueueType === "skeleton") {
     return updateKey2(state, "save", "lastSaveTimestamp", {
       skeleton: action.timestamp,
     });
@@ -86,7 +86,7 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState {
 
         const transactionGroupCount = updateActionChunks.length;
         const actionLogInfo = JSON.stringify(getActionLog().slice(-10));
-        const oldQueue = selectQueue(state, action.tracingType, action.tracingId);
+        const oldQueue = selectQueue(state, action.saveQueueType, action.tracingId);
         const newQueue = oldQueue.concat(
           updateActionChunks.map((actions, transactionGroupIndex) => ({
             // Placeholder, the version number will be updated before sending to the server
@@ -123,7 +123,7 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState {
       const { count } = action;
 
       if (count > 0) {
-        const queue = selectQueue(state, action.tracingType, action.tracingId);
+        const queue = selectQueue(state, action.saveQueueType, action.tracingId);
 
         const processedQueueActionCount = _.sumBy(
           queue.slice(0, count),
@@ -163,6 +163,7 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState {
             $set: {
               skeleton: [],
               volumes: _.mapValues(state.save.queue.volumes, () => []),
+              mappings: _.mapValues(state.save.queue.mappings, () => []),
             },
           },
           progressInfo: {
@@ -181,7 +182,7 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState {
       return update(state, {
         save: {
           isBusyInfo: {
-            [action.tracingType]: {
+            [action.saveQueueType]: {
               $set: action.isBusy,
             },
           },
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 518f413626e..cfbafa47e22 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -138,7 +138,7 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
   if (items.length === 0) return;
 
   // TODO: Will there be a separate end point for these update actions?
-  yield* put(pushSaveQueueTransaction(items, "editableMapping", volumeTracingId));
+  yield* put(pushSaveQueueTransaction(items, "mapping", volumeTracingId));
   yield* call([Model, Model.ensureSavedState]);
 
   yield* call([api.data, api.data.reloadBuckets], layerName);
diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
index 3b2aae1563b..8b4b7603acb 100644
--- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
@@ -45,7 +45,7 @@ import {
   centerActiveNodeAction,
   setTracingAction,
 } from "oxalis/model/actions/skeletontracing_actions";
-import type { UndoAction, RedoAction } from "oxalis/model/actions/save_actions";
+import type { UndoAction, RedoAction, SaveQueueType } from "oxalis/model/actions/save_actions";
 import {
   shiftSaveQueueAction,
   setSaveBusyAction,
@@ -717,12 +717,9 @@ function* applyAndGetRevertingVolumeBatch(
   };
 }
 
-export function* pushTracingTypeAsync(
-  tracingType: "skeleton" | "volume",
-  tracingId: string,
-): Saga<void> {
+export function* pushSaveQueueAsync(saveQueueType: SaveQueueType, tracingId: string): Saga<void> {
   yield* take("WK_READY");
-  yield* put(setLastSaveTimestampAction(tracingType, tracingId));
+  yield* put(setLastSaveTimestampAction(saveQueueType, tracingId));
   let loopCounter = 0;
 
   while (true) {
@@ -730,7 +727,7 @@ export function* pushTracingTypeAsync(
     let saveQueue;
     // Check whether the save queue is actually empty, the PUSH_SAVE_QUEUE_TRANSACTION action
     // could have been triggered during the call to sendRequestToServer
-    saveQueue = yield* select((state) => selectQueue(state, tracingType, tracingId));
+    saveQueue = yield* select((state) => selectQueue(state, saveQueueType, tracingId));
 
     if (saveQueue.length === 0) {
       if (loopCounter % 100 === 0) {
@@ -747,21 +744,21 @@ export function* pushTracingTypeAsync(
       timeout: delay(PUSH_THROTTLE_TIME),
       forcePush: take("SAVE_NOW"),
     });
-    yield* put(setSaveBusyAction(true, tracingType));
+    yield* put(setSaveBusyAction(true, saveQueueType));
 
     if (forcePush) {
       while (true) {
         // Send batches to the server until the save queue is empty.
-        saveQueue = yield* select((state) => selectQueue(state, tracingType, tracingId));
+        saveQueue = yield* select((state) => selectQueue(state, saveQueueType, tracingId));
 
         if (saveQueue.length > 0) {
-          yield* call(sendRequestToServer, tracingType, tracingId);
+          yield* call(sendRequestToServer, saveQueueType, tracingId);
         } else {
           break;
         }
       }
     } else {
-      saveQueue = yield* select((state) => selectQueue(state, tracingType, tracingId));
+      saveQueue = yield* select((state) => selectQueue(state, saveQueueType, tracingId));
 
       if (saveQueue.length > 0) {
         // Saving the tracing automatically (via timeout) only saves the current state.
@@ -769,11 +766,11 @@ export function* pushTracingTypeAsync(
         // important when the auto-saving happens during continuous movements.
         // Always draining the save queue completely would mean that save
         // requests are sent as long as the user moves.
-        yield* call(sendRequestToServer, tracingType, tracingId);
+        yield* call(sendRequestToServer, saveQueueType, tracingId);
       }
     }
 
-    yield* put(setSaveBusyAction(false, tracingType));
+    yield* put(setSaveBusyAction(false, saveQueueType));
   }
 }
 export function sendRequestWithToken(
@@ -808,14 +805,13 @@ function getRetryWaitTime(retryCount: number) {
   return Math.min(2 ** retryCount * SAVE_RETRY_WAITING_TIME, MAX_SAVE_RETRY_WAITING_TIME);
 }
 
-export function* sendRequestToServer(
-  tracingType: "skeleton" | "volume",
-  tracingId: string,
-): Saga<void> {
-  const fullSaveQueue = yield* select((state) => selectQueue(state, tracingType, tracingId));
+export function* sendRequestToServer(saveQueueType: SaveQueueType, tracingId: string): Saga<void> {
+  const fullSaveQueue = yield* select((state) => selectQueue(state, saveQueueType, tracingId));
   const saveQueue = sliceAppropriateBatchCount(fullSaveQueue);
   let compactedSaveQueue = compactSaveQueue(saveQueue);
-  const { version, type } = yield* select((state) => selectTracing(state, tracingType, tracingId));
+  const { version, type } = yield* select((state) =>
+    selectTracing(state, saveQueueType, tracingId),
+  );
   const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url);
   compactedSaveQueue = addVersionNumbers(compactedSaveQueue, version);
   let retryCount = 0;
@@ -846,12 +842,12 @@ export function* sendRequestToServer(
       }
 
       yield* put(
-        setVersionNumberAction(version + compactedSaveQueue.length, tracingType, tracingId),
+        setVersionNumberAction(version + compactedSaveQueue.length, saveQueueType, tracingId),
       );
-      yield* put(setLastSaveTimestampAction(tracingType, tracingId));
-      yield* put(shiftSaveQueueAction(saveQueue.length, tracingType, tracingId));
+      yield* put(setLastSaveTimestampAction(saveQueueType, tracingId));
+      yield* put(shiftSaveQueueAction(saveQueue.length, saveQueueType, tracingId));
 
-      if (tracingType === "volume") {
+      if (saveQueueType === "volume") {
         try {
           yield* call(markBucketsAsNotDirty, compactedSaveQueue, tracingId);
         } catch (error) {
@@ -1002,19 +998,19 @@ export function* saveTracingTypeAsync(
   /*
     Listen to changes to the annotation and derive UpdateActions from the
     old and new state.
-     The actual push to the server is done by the forked pushTracingTypeAsync saga.
+     The actual push to the server is done by the forked pushSaveQueueAsync saga.
   */
-  const tracingType =
+  const saveQueueType =
     initializeAction.type === "INITIALIZE_SKELETONTRACING" ? "skeleton" : "volume";
   const tracingId = initializeAction.tracing.id;
-  yield* fork(pushTracingTypeAsync, tracingType, tracingId);
-  let prevTracing = yield* select((state) => selectTracing(state, tracingType, tracingId));
+  yield* fork(pushSaveQueueAsync, saveQueueType, tracingId);
+  let prevTracing = yield* select((state) => selectTracing(state, saveQueueType, tracingId));
   let prevFlycam = yield* select((state) => state.flycam);
   let prevTdCamera = yield* select((state) => state.viewModeData.plane.tdCamera);
   yield* take("WK_READY");
 
   while (true) {
-    if (tracingType === "skeleton") {
+    if (saveQueueType === "skeleton") {
       yield* take([
         ...SkeletonTracingSaveRelevantActions,
         ...FlycamActions,
@@ -1036,7 +1032,7 @@ export function* saveTracingTypeAsync(
       (state) => state.tracing.restrictions.allowUpdate && state.tracing.restrictions.allowSave,
     );
     if (!allowUpdate) return;
-    const tracing = yield* select((state) => selectTracing(state, tracingType, tracingId));
+    const tracing = yield* select((state) => selectTracing(state, saveQueueType, tracingId));
     const flycam = yield* select((state) => state.flycam);
     const tdCamera = yield* select((state) => state.viewModeData.plane.tdCamera);
     const items = compactUpdateActions(
@@ -1055,7 +1051,7 @@ export function* saveTracingTypeAsync(
     );
 
     if (items.length > 0) {
-      yield* put(pushSaveQueueTransaction(items, tracingType, tracingId));
+      yield* put(pushSaveQueueTransaction(items, saveQueueType, tracingId));
     }
 
     prevTracing = tracing;
diff --git a/frontend/javascripts/oxalis/store.ts b/frontend/javascripts/oxalis/store.ts
index 0a162e9d018..5da4e3e528e 100644
--- a/frontend/javascripts/oxalis/store.ts
+++ b/frontend/javascripts/oxalis/store.ts
@@ -63,6 +63,7 @@ import overwriteActionMiddleware from "oxalis/model/helpers/overwrite_action_mid
 import reduceReducers from "oxalis/model/helpers/reduce_reducers";
 import rootSaga from "oxalis/model/sagas/root_saga";
 import ConnectomeReducer from "oxalis/model/reducers/connectome_reducer";
+import { SaveQueueType } from "./model/actions/save_actions";
 export type MutableCommentType = {
   content: string;
   nodeId: number;
@@ -367,19 +368,18 @@ export type ProgressInfo = {
   readonly processedActionCount: number;
   readonly totalActionCount: number;
 };
-export type IsBusyInfo = {
-  readonly skeleton: boolean;
-  readonly volume: boolean;
-};
+export type IsBusyInfo = Record<SaveQueueType, boolean>;
 export type SaveState = {
   readonly isBusyInfo: IsBusyInfo;
   readonly queue: {
     readonly skeleton: Array<SaveQueueEntry>;
     readonly volumes: Record<string, Array<SaveQueueEntry>>;
+    readonly mappings: Record<string, Array<SaveQueueEntry>>;
   };
   readonly lastSaveTimestamp: {
     readonly skeleton: number;
     readonly volumes: Record<string, number>;
+    readonly mappings: Record<string, number>;
   };
   readonly progressInfo: ProgressInfo;
 };
diff --git a/frontend/javascripts/oxalis/view/version_list.tsx b/frontend/javascripts/oxalis/view/version_list.tsx
index c77fad6ffdf..e3fac94b764 100644
--- a/frontend/javascripts/oxalis/view/version_list.tsx
+++ b/frontend/javascripts/oxalis/view/version_list.tsx
@@ -9,6 +9,7 @@ import { getUpdateActionLog, downloadNml } from "admin/admin_rest_api";
 import { handleGenericError } from "libs/error_handling";
 import {
   pushSaveQueueTransaction,
+  SaveQueueType,
   setVersionNumberAction,
 } from "oxalis/model/actions/save_actions";
 import { revertToVersion, serverCreateTracing } from "oxalis/model/sagas/update_actions";
@@ -20,7 +21,7 @@ import Store from "oxalis/store";
 import VersionEntryGroup from "oxalis/view/version_entry_group";
 import api from "oxalis/api/internal_api";
 type Props = {
-  tracingType: "skeleton" | "volume";
+  versionedObjectType: SaveQueueType;
   tracing: SkeletonTracing | VolumeTracing;
   allowUpdate: boolean;
 };
@@ -89,7 +90,7 @@ class VersionList extends React.Component<Props, State> {
       const updateActionLog = await getUpdateActionLog(
         tracingStoreUrl,
         tracingId,
-        this.props.tracingType,
+        this.props.versionedObjectType,
       );
       // Insert version 0
       updateActionLog.push({
@@ -118,14 +119,14 @@ class VersionList extends React.Component<Props, State> {
       Store.dispatch(
         setVersionNumberAction(
           this.getNewestVersion(),
-          this.props.tracingType,
+          this.props.versionedObjectType,
           this.props.tracing.tracingId,
         ),
       );
       Store.dispatch(
         pushSaveQueueTransaction(
           [revertToVersion(version)],
-          this.props.tracingType,
+          this.props.versionedObjectType,
           this.props.tracing.tracingId,
         ),
       );
@@ -136,13 +137,13 @@ class VersionList extends React.Component<Props, State> {
       const { annotationType, annotationId, volumes } = Store.getState().tracing;
       const includesVolumeFallbackData = volumes.some((volume) => volume.fallbackLayer != null);
       downloadNml(annotationId, annotationType, includesVolumeFallbackData, {
-        [this.props.tracingType]: version,
+        [this.props.versionedObjectType]: version,
       });
     }
   };
 
   handlePreviewVersion = (version: number) => {
-    if (this.props.tracingType === "skeleton") {
+    if (this.props.versionedObjectType === "skeleton") {
       return previewVersion({
         skeleton: version,
       });
diff --git a/frontend/javascripts/oxalis/view/version_view.tsx b/frontend/javascripts/oxalis/view/version_view.tsx
index 19701d41b16..1c120f61e51 100644
--- a/frontend/javascripts/oxalis/view/version_view.tsx
+++ b/frontend/javascripts/oxalis/view/version_view.tsx
@@ -113,7 +113,7 @@ class VersionView extends React.Component<Props, State> {
             {this.props.tracing.skeleton != null ? (
               <TabPane tab="Skeleton" key="skeleton">
                 <VersionList
-                  tracingType="skeleton"
+                  versionedObjectType="skeleton"
                   tracing={this.props.tracing.skeleton}
                   allowUpdate={this.state.initialAllowUpdate}
                 />
@@ -125,7 +125,7 @@ class VersionView extends React.Component<Props, State> {
                 key={volumeTracing.tracingId}
               >
                 <VersionList
-                  tracingType="volume"
+                  versionedObjectType="volume"
                   tracing={volumeTracing}
                   allowUpdate={this.state.initialAllowUpdate}
                 />

From cc06911425297403098a093128e5350ef6905faa Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 17 May 2022 11:14:11 +0200
Subject: [PATCH 030/122] restructure routes for compatibility

---
 .../controllers/VolumeTracingController.scala | 36 +++++++++++++------
 .../EditableMappingService.scala              | 10 ++++++
 .../EditableMappingUpdateActions.scala        | 12 ++++++-
 .../tracings/volume/VolumeUpdateActions.scala |  6 ++--
 ...alableminds.webknossos.tracingstore.routes |  7 ++--
 5 files changed, 55 insertions(+), 16 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index 45e155878c9..59c4cc244c9 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -15,7 +15,7 @@ import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotifi
 import com.scalableminds.webknossos.tracingstore.tracings.UpdateActionGroup
 import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{
   EditableMappingService,
-  EditableMappingUpdateAction
+  EditableMappingUpdateActionGroup
 }
 import com.scalableminds.webknossos.tracingstore.tracings.volume.{
   ResolutionRestrictions,
@@ -254,8 +254,8 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
         for {
           tracing <- tracingService.find(tracingId)
           tracingMappingName <- tracing.mappingName ?~> "annotation.noMappingSet"
-          id <- editableMappingService.create(baseMappingName = tracingMappingName)
-          volumeUpdate = UpdateMappingNameAction(id,
+          editableMappingId <- editableMappingService.create(baseMappingName = tracingMappingName)
+          volumeUpdate = UpdateMappingNameAction(Some(editableMappingId),
                                                  isEditable = Some(true),
                                                  actionTimestamp = Some(System.currentTimeMillis()))
           _ <- tracingService.handleUpdateGroup(
@@ -270,22 +270,23 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
                                              None),
             tracing.version
           )
-        } yield Ok
+          infoJson <- editableMappingService.infoJson(tracingId = tracingId, editableMappingId = editableMappingId)
+        } yield Ok(infoJson)
       }
     }
 
-  def updateEditableMapping(token: Option[String],
-                            tracingId: String,
-                            version: Long): Action[EditableMappingUpdateAction] =
-    Action.async(validateJson[EditableMappingUpdateAction]) { implicit request =>
-      accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) {
+  def updateEditableMapping(token: Option[String], tracingId: String): Action[EditableMappingUpdateActionGroup] =
+    Action.async(validateJson[EditableMappingUpdateActionGroup]) { implicit request =>
+      accessTokenService.validateAccess(UserAccessRequest.writeTracing(tracingId), token) {
         for {
           tracing <- tracingService.find(tracingId)
           mappingName <- tracing.mappingName.toFox
           _ <- Fox.assertTrue(editableMappingService.exists(mappingName))
           currentVersion <- editableMappingService.currentVersion(mappingName)
-          _ <- bool2Fox(version == currentVersion + 1) ?~> "version mismatch"
-          _ <- editableMappingService.update(mappingName, request.body, version)
+          _ <- bool2Fox(request.body.version == currentVersion + 1) ?~> "version mismatch"
+          _ <- bool2Fox(request.body.actions.length == 1) ?~> "Editable mapping update group must contain exactly one update action"
+          updateAction <- request.body.actions.headOption.toFox
+          _ <- editableMappingService.update(mappingName, updateAction, request.body.version)
         } yield Ok
       }
     }
@@ -303,4 +304,17 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
         }
       }
   }
+
+  def editableMappingInfo(token: Option[String], tracingId: String): Action[AnyContent] = Action.async {
+    implicit request =>
+      log() {
+        accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) {
+          for {
+            tracing <- tracingService.find(tracingId)
+            mappingName <- tracing.mappingName.toFox
+            infoJson <- editableMappingService.infoJson(tracingId = tracingId, editableMappingId = mappingName)
+          } yield Ok(infoJson)
+        }
+      }
+  }
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index ff2c042fd34..e6bcebed898 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -42,6 +42,16 @@ class EditableMappingService @Inject()(
   def currentVersion(editableMappingId: String): Fox[Long] =
     tracingDataStore.editableMappings.getVersion(editableMappingId, mayBeEmpty = Some(true), emptyFallback = Some(0L))
 
+  def infoJson(tracingId: String, editableMappingId: String): Fox[JsObject] =
+    for {
+      version <- currentVersion(editableMappingId)
+    } yield
+      Json.obj(
+        "mappingName" -> editableMappingId,
+        "version" -> version,
+        "tracingId" -> tracingId
+      )
+
   def create(baseMappingName: String): Fox[String] = {
     val newId = generateId
     val newEditableMapping = EditableMapping(
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
index ae6087e4d86..0e20e12fa7d 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
@@ -2,7 +2,7 @@ package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
 
 import com.scalableminds.util.geometry.Vec3Int
 import play.api.libs.json.Format.GenericFormat
-import play.api.libs.json.{Format, JsError, JsResult, JsValue, Json, OFormat}
+import play.api.libs.json._
 
 trait EditableMappingUpdateAction {}
 
@@ -46,3 +46,13 @@ object EditableMappingUpdateAction {
   }
 
 }
+
+case class EditableMappingUpdateActionGroup(
+    version: Long,
+    timestamp: Long,
+    actions: List[EditableMappingUpdateAction]
+)
+
+object EditableMappingUpdateActionGroup {
+  implicit val jsonFormat: OFormat[EditableMappingUpdateActionGroup] = Json.format[EditableMappingUpdateActionGroup]
+}
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala
index 3edff995a77..5b2816d614a 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala
@@ -242,7 +242,9 @@ object DeleteSegmentVolumeAction {
   implicit val jsonFormat: OFormat[DeleteSegmentVolumeAction] = Json.format[DeleteSegmentVolumeAction]
 }
 
-case class UpdateMappingNameAction(mappingName: String, isEditable: Option[Boolean], actionTimestamp: Option[Long])
+case class UpdateMappingNameAction(mappingName: Option[String],
+                                   isEditable: Option[Boolean],
+                                   actionTimestamp: Option[Long])
     extends ApplyableVolumeAction {
   override def addTimestamp(timestamp: Long): VolumeUpdateAction =
     this.copy(actionTimestamp = Some(timestamp))
@@ -251,7 +253,7 @@ case class UpdateMappingNameAction(mappingName: String, isEditable: Option[Boole
     CompactVolumeUpdateAction("updateMappingName", actionTimestamp, Json.obj("mappingName" -> mappingName))
 
   override def applyOn(tracing: VolumeTracing): VolumeTracing =
-    tracing.withMappingName(mappingName).withMappingIsEditable(isEditable.getOrElse(false))
+    tracing.copy(mappingName = mappingName, mappingIsEditable = Some(isEditable.getOrElse(false)))
 }
 
 object UpdateMappingNameAction {
diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
index 6eca5bddb46..1b2fb43cc18 100644
--- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
+++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
@@ -22,12 +22,15 @@ POST        /volume/:tracingId/importVolumeData                 @com.scalablemin
 GET         /volume/:tracingId/findData                         @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.findData(token: Option[String], tracingId: String)
 GET         /volume/:tracingId/agglomerateSkeleton/:agglomerateId     @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateSkeleton(token: Option[String], tracingId: String, agglomerateId: Long)
 POST        /volume/:tracingId/makeMappingEditable              @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.makeMappingEditable(token: Option[String], tracingId: String)
-POST        /volume/:tracingId/updateEditableMapping            @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.updateEditableMapping(token: Option[String], tracingId: String, version: Long)
-GET         /volume/:tracingId/editableMappingUpdateActionLog   @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingUpdateActionLog(token: Option[String], tracingId: String)
 POST        /volume/getMultiple                                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getMultiple(token: Option[String])
 POST        /volume/mergedFromIds                               @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromIds(token: Option[String], persist: Boolean)
 POST        /volume/mergedFromContents                          @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromContents(token: Option[String], persist: Boolean)
 
+# Editable Mappings
+POST        /mapping/:tracingId/update                          @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.updateEditableMapping(token: Option[String], tracingId: String)
+GET         /mapping/:tracingId/updateActionLog                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingUpdateActionLog(token: Option[String], tracingId: String)
+GET         /mapping/:tracingId                                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingInfo(token: Option[String], tracingId: String)
+
 # Skeleton tracings
 POST        /skeleton/save                                      @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.save(token: Option[String])
 POST        /skeleton/saveMultiple                              @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.saveMultiple(token: Option[String])

From c7192774de1c9440d868d1b05436d7636e5a4167 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 17 May 2022 14:28:01 +0200
Subject: [PATCH 031/122] implement separate save queues for editable mappings,
 initialize and push update actions for editable mappings

---
 frontend/javascripts/admin/admin_rest_api.ts  | 17 ++++-
 frontend/javascripts/oxalis/default_state.ts  |  1 +
 .../model/accessors/tracing_accessor.ts       | 39 ++++++++----
 .../model/actions/volumetracing_actions.ts    | 15 ++++-
 .../oxalis/model/reducers/save_reducer.ts     | 62 +++++++++++++++----
 .../model/reducers/volumetracing_reducer.ts   | 11 +++-
 .../reducers/volumetracing_reducer_helpers.ts | 18 +++++-
 .../oxalis/model/sagas/mapping_saga.ts        |  8 ++-
 .../oxalis/model/sagas/proofread_saga.ts      | 12 +++-
 .../oxalis/model/sagas/save_saga.ts           | 39 +++++++++---
 .../oxalis/model_initialization.ts            | 37 +++++++++--
 frontend/javascripts/oxalis/store.ts          |  8 +++
 frontend/javascripts/types/api_flow_types.ts  |  6 ++
 13 files changed, 225 insertions(+), 48 deletions(-)

diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts
index 0366e5194cb..b8c2b820816 100644
--- a/frontend/javascripts/admin/admin_rest_api.ts
+++ b/frontend/javascripts/admin/admin_rest_api.ts
@@ -54,6 +54,7 @@ import type {
   ServerTracing,
   TracingType,
   WkConnectDatasetConfig,
+  ServerEditableMapping,
 } from "types/api_flow_types";
 import { APIAnnotationTypeEnum } from "types/api_flow_types";
 import type { Vector3, Vector6 } from "oxalis/constants";
@@ -1563,9 +1564,12 @@ export function fetchMapping(
   );
 }
 
-export function makeMappingEditable(tracingStoreUrl: string, tracingId: string): Promise<void> {
+export function makeMappingEditable(
+  tracingStoreUrl: string,
+  tracingId: string,
+): Promise<ServerEditableMapping> {
   return doWithToken((token) =>
-    Request.triggerRequest(
+    Request.receiveJSON(
       `${tracingStoreUrl}/tracings/volume/${tracingId}/makeMappingEditable?token=${token}`,
       {
         method: "POST",
@@ -1574,6 +1578,15 @@ export function makeMappingEditable(tracingStoreUrl: string, tracingId: string):
   );
 }
 
+export function getEditableMapping(
+  tracingStoreUrl: string,
+  tracingId: string,
+): Promise<ServerEditableMapping> {
+  return doWithToken((token) =>
+    Request.receiveJSON(`${tracingStoreUrl}/tracings/mapping/${tracingId}?token=${token}`),
+  );
+}
+
 export async function getAgglomeratesForDatasetLayer(
   datastoreUrl: string,
   datasetId: APIDatasetId,
diff --git a/frontend/javascripts/oxalis/default_state.ts b/frontend/javascripts/oxalis/default_state.ts
index b373740fd7b..e3b43d9e836 100644
--- a/frontend/javascripts/oxalis/default_state.ts
+++ b/frontend/javascripts/oxalis/default_state.ts
@@ -145,6 +145,7 @@ const defaultState: OxalisState = {
       tracingId: "",
     },
     volumes: [],
+    mappings: [],
     skeleton: null,
     user: null,
     annotationLayers: [],
diff --git a/frontend/javascripts/oxalis/model/accessors/tracing_accessor.ts b/frontend/javascripts/oxalis/model/accessors/tracing_accessor.ts
index b707208bfc3..cc0b5f51226 100644
--- a/frontend/javascripts/oxalis/model/accessors/tracing_accessor.ts
+++ b/frontend/javascripts/oxalis/model/accessors/tracing_accessor.ts
@@ -1,4 +1,5 @@
 import type {
+  EditableMapping,
   OxalisState,
   ReadOnlyTracing,
   SkeletonTracing,
@@ -8,6 +9,8 @@ import type {
 } from "oxalis/store";
 import type { ServerTracing, TracingType } from "types/api_flow_types";
 import { TracingTypeEnum } from "types/api_flow_types";
+import { SaveQueueType } from "oxalis/model/actions/save_actions";
+
 export function maybeGetSomeTracing(
   tracing: Tracing,
 ): SkeletonTracing | VolumeTracing | ReadOnlyTracing | null {
@@ -52,24 +55,36 @@ export function getTracingType(tracing: Tracing): TracingType {
 }
 export function selectTracing(
   state: OxalisState,
-  tracingType: "skeleton" | "volume",
+  tracingType: SaveQueueType,
   tracingId: string,
-): SkeletonTracing | VolumeTracing {
-  if (tracingType === "skeleton") {
-    if (state.tracing.skeleton == null) {
-      throw new Error(`Skeleton tracing with id ${tracingId} not found`);
-    }
+): SkeletonTracing | VolumeTracing | EditableMapping {
+  let tracing;
 
-    return state.tracing.skeleton;
+  switch (tracingType) {
+    case "skeleton": {
+      tracing = state.tracing.skeleton;
+      break;
+    }
+    case "volume": {
+      tracing = state.tracing.volumes.find(
+        (volumeTracing) => volumeTracing.tracingId === tracingId,
+      );
+      break;
+    }
+    case "mapping": {
+      tracing = state.tracing.mappings.find((mapping) => mapping.tracingId === tracingId);
+      break;
+    }
+    default: {
+      throw new Error(`Unknown tracing type: ${tracingType}`);
+    }
   }
 
-  const volumeTracing = state.tracing.volumes.find((tracing) => tracing.tracingId === tracingId);
-
-  if (volumeTracing == null) {
-    throw new Error(`Volume tracing with id ${tracingId} not found`);
+  if (tracing == null) {
+    throw new Error(`${tracingType} object with id ${tracingId} not found`);
   }
 
-  return volumeTracing;
+  return tracing;
 }
 export const getUserBoundingBoxesFromState = (state: OxalisState): Array<UserBoundingBox> => {
   const maybeSomeTracing = maybeGetSomeTracing(state.tracing);
diff --git a/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts b/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts
index 50f2ac0b017..916645ad09a 100644
--- a/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts
+++ b/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts
@@ -1,4 +1,4 @@
-import type { ServerVolumeTracing } from "types/api_flow_types";
+import type { ServerEditableMapping, ServerVolumeTracing } from "types/api_flow_types";
 import type { Vector2, Vector3, Vector4, OrthoView, ContourMode } from "oxalis/constants";
 import type { BucketDataArray } from "oxalis/model/bucket_data_handling/bucket";
 import type { Segment, SegmentMap } from "oxalis/store";
@@ -9,6 +9,10 @@ export type InitializeVolumeTracingAction = {
   type: "INITIALIZE_VOLUMETRACING";
   tracing: ServerVolumeTracing;
 };
+export type InitializeEditableMappingAction = {
+  type: "INITIALIZE_EDITABLE_MAPPING";
+  mapping: ServerEditableMapping;
+};
 type CreateCellAction = {
   type: "CREATE_CELL";
 };
@@ -128,7 +132,8 @@ export type VolumeTracingAction =
   | AddBucketToUndoAction
   | ImportVolumeTracingAction
   | SetMaxCellAction
-  | SetMappingIsEditableAction;
+  | SetMappingIsEditableAction
+  | InitializeEditableMappingAction;
 export const VolumeTracingSaveRelevantActions = [
   "CREATE_CELL",
   "SET_ACTIVE_CELL",
@@ -147,6 +152,12 @@ export const initializeVolumeTracingAction = (
   type: "INITIALIZE_VOLUMETRACING",
   tracing,
 });
+export const initializeEditableMappingAction = (
+  mapping: ServerEditableMapping,
+): InitializeEditableMappingAction => ({
+  type: "INITIALIZE_EDITABLE_MAPPING",
+  mapping,
+});
 export const createCellAction = (): CreateCellAction => ({
   type: "CREATE_CELL",
 });
diff --git a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts
index 1f6ffd05a2b..67c3e70fecc 100644
--- a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts
+++ b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts
@@ -13,7 +13,10 @@ import { getStats } from "oxalis/model/accessors/skeletontracing_accessor";
 import { maximumActionCountPerBatch } from "oxalis/model/sagas/save_saga_constants";
 import { selectQueue } from "oxalis/model/accessors/save_accessor";
 import { updateKey2 } from "oxalis/model/helpers/deep_update";
-import { updateVolumeTracing } from "oxalis/model/reducers/volumetracing_reducer_helpers";
+import {
+  updateEditableMapping,
+  updateVolumeTracing,
+} from "oxalis/model/reducers/volumetracing_reducer_helpers";
 import Date from "libs/date";
 import * as Utils from "libs/utils";
 
@@ -24,15 +27,24 @@ function updateQueueObj(
 ): SaveState["queue"] {
   if (action.saveQueueType === "skeleton") {
     return { ...oldQueueObj, skeleton: newQueue };
+  } else if (action.saveQueueType === "volume") {
+    return { ...oldQueueObj, volumes: { ...oldQueueObj.volumes, [action.tracingId]: newQueue } };
+  } else if (action.saveQueueType === "mapping") {
+    return { ...oldQueueObj, mappings: { ...oldQueueObj.mappings, [action.tracingId]: newQueue } };
   }
 
-  return { ...oldQueueObj, volumes: { ...oldQueueObj.volumes, [action.tracingId]: newQueue } };
+  return oldQueueObj;
 }
 
 export function getTotalSaveQueueLength(queueObj: SaveState["queue"]) {
   return (
     queueObj.skeleton.length +
-    _.sum(Utils.values(queueObj.volumes).map((volumeQueue: SaveQueueEntry[]) => volumeQueue.length))
+    _.sum(
+      Utils.values(queueObj.volumes).map((volumeQueue: SaveQueueEntry[]) => volumeQueue.length),
+    ) +
+    _.sum(
+      Utils.values(queueObj.mappings).map((mappingQueue: SaveQueueEntry[]) => mappingQueue.length),
+    )
   );
 }
 
@@ -41,11 +53,17 @@ function updateVersion(state: OxalisState, action: SetVersionNumberAction) {
     return updateKey2(state, "tracing", "skeleton", {
       version: action.version,
     });
+  } else if (action.saveQueueType === "volume") {
+    return updateVolumeTracing(state, action.tracingId, {
+      version: action.version,
+    });
+  } else if (action.saveQueueType === "mapping") {
+    return updateEditableMapping(state, action.tracingId, {
+      version: action.version,
+    });
   }
 
-  return updateVolumeTracing(state, action.tracingId, {
-    version: action.version,
-  });
+  return state;
 }
 
 function updateLastSaveTimestamp(state: OxalisState, action: SetLastSaveTimestampAction) {
@@ -53,15 +71,25 @@ function updateLastSaveTimestamp(state: OxalisState, action: SetLastSaveTimestam
     return updateKey2(state, "save", "lastSaveTimestamp", {
       skeleton: action.timestamp,
     });
+  } else if (action.saveQueueType === "volume") {
+    const newVolumesDict = {
+      ...state.save.lastSaveTimestamp.volumes,
+      [action.tracingId]: action.timestamp,
+    };
+    return updateKey2(state, "save", "lastSaveTimestamp", {
+      volumes: newVolumesDict,
+    });
+  } else if (action.saveQueueType === "mapping") {
+    const newMappingsDict = {
+      ...state.save.lastSaveTimestamp.mappings,
+      [action.tracingId]: action.timestamp,
+    };
+    return updateKey2(state, "save", "lastSaveTimestamp", {
+      mappings: newMappingsDict,
+    });
   }
 
-  const newVolumesDict = {
-    ...state.save.lastSaveTimestamp.volumes,
-    [action.tracingId]: action.timestamp,
-  };
-  return updateKey2(state, "save", "lastSaveTimestamp", {
-    volumes: newVolumesDict,
-  });
+  return state;
 }
 
 function SaveReducer(state: OxalisState, action: Action): OxalisState {
@@ -74,6 +102,14 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState {
       });
     }
 
+    case "INITIALIZE_EDITABLE_MAPPING": {
+      // Set up empty save queue array for editable mapping
+      const newMappingsQueue = { ...state.save.queue.volumes, [action.mapping.tracingId]: [] };
+      return updateKey2(state, "save", "queue", {
+        mappings: newMappingsQueue,
+      });
+    }
+
     case "PUSH_SAVE_QUEUE_TRANSACTION": {
       // Only report tracing statistics, if a "real" update to the tracing happened
       const stats = _.some(action.items, (ua) => ua.name !== "updateTracing")
diff --git a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
index 153664b4256..fbcbc7a91f7 100644
--- a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
+++ b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
@@ -1,6 +1,6 @@
 import update from "immutability-helper";
 import { ContourModeEnum } from "oxalis/constants";
-import type { OxalisState, VolumeTracing } from "oxalis/store";
+import type { EditableMapping, OxalisState, VolumeTracing } from "oxalis/store";
 import type {
   VolumeTracingAction,
   UpdateSegmentAction,
@@ -24,6 +24,7 @@ import {
   setContourTracingModeReducer,
   setMaxCellReducer,
   updateVolumeTracing,
+  updateEditableMapping,
   setMappingNameReducer,
 } from "oxalis/model/reducers/volumetracing_reducer_helpers";
 import { updateKey2 } from "oxalis/model/helpers/deep_update";
@@ -202,6 +203,14 @@ function VolumeTracingReducer(
       return createCellReducer(newState, volumeTracing, action.tracing.activeSegmentId);
     }
 
+    case "INITIALIZE_EDITABLE_MAPPING": {
+      const mapping: EditableMapping = {
+        type: "mapping",
+        ...action.mapping,
+      };
+      return updateEditableMapping(state, action.mapping.tracingId, mapping);
+    }
+
     case "SET_SEGMENTS": {
       return handleSetSegments(state, action);
     }
diff --git a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts
index 84fb2029d8e..9383d971286 100644
--- a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts
+++ b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts
@@ -1,6 +1,6 @@
 import update from "immutability-helper";
 import { ContourMode, Vector3 } from "oxalis/constants";
-import type { MappingType, OxalisState, VolumeTracing } from "oxalis/store";
+import type { EditableMapping, MappingType, OxalisState, VolumeTracing } from "oxalis/store";
 import { isVolumeAnnotationDisallowedForZoom } from "oxalis/model/accessors/volumetracing_accessor";
 import { setDirectionReducer } from "oxalis/model/reducers/flycam_reducer";
 import { updateKey } from "oxalis/model/helpers/deep_update";
@@ -20,6 +20,22 @@ export function updateVolumeTracing(
     volumes: newVolumes,
   });
 }
+export function updateEditableMapping(
+  state: OxalisState,
+  volumeTracingId: string,
+  shape: Partial<EditableMapping>,
+) {
+  const newMappings = state.tracing.mappings.map((mapping) => {
+    if (mapping.tracingId === volumeTracingId) {
+      return { ...mapping, ...shape };
+    } else {
+      return mapping;
+    }
+  });
+  return updateKey(state, "tracing", {
+    mappings: newMappings,
+  });
+}
 export function setActiveCellReducer(state: OxalisState, volumeTracing: VolumeTracing, id: number) {
   return updateVolumeTracing(state, volumeTracing.tracingId, {
     activeCellId: id,
diff --git a/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts b/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts
index a1cb62993c2..98abb3b79c8 100644
--- a/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts
@@ -103,12 +103,18 @@ function* maybeFetchMapping(
       ? layerInfo.fallbackLayer
       : layerInfo.name,
   ];
-  const [jsonMappings, hdf5Mappings] = yield* all([
+  const [jsonMappings, serverHdf5Mappings] = yield* all([
     // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
     call(getMappingsForDatasetLayer, ...params),
     // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
     call(getAgglomeratesForDatasetLayer, ...params),
   ]);
+  const editableMappings = yield* select((state) =>
+    state.tracing.volumes
+      .filter((volumeTracing) => volumeTracing.mappingIsEditable)
+      .map((volumeTracing) => volumeTracing.mappingName),
+  );
+  const hdf5Mappings = [...serverHdf5Mappings, ...editableMappings];
   const mappingsWithCorrectType = mappingType === "JSON" ? jsonMappings : hdf5Mappings;
 
   if (!mappingsWithCorrectType.includes(mappingName)) {
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index cfbafa47e22..035c7535a44 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -7,7 +7,10 @@ import type {
   DeleteEdgeAction,
   MergeTreesAction,
 } from "oxalis/model/actions/skeletontracing_actions";
-import { setMappingisEditableAction } from "oxalis/model/actions/volumetracing_actions";
+import {
+  initializeEditableMappingAction,
+  setMappingisEditableAction,
+} from "oxalis/model/actions/volumetracing_actions";
 import {
   enforceSkeletonTracing,
   findTreeByNodeId,
@@ -67,8 +70,13 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
     const upToDateVolumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
     if (upToDateVolumeTracing == null) return;
 
-    yield* call(makeMappingEditable, tracingStoreUrl, volumeTracingId);
+    const serverEditableMapping = yield* call(
+      makeMappingEditable,
+      tracingStoreUrl,
+      volumeTracingId,
+    );
     yield* put(setMappingisEditableAction());
+    yield* put(initializeEditableMappingAction(serverEditableMapping));
     yield* put(
       setVersionNumberAction(upToDateVolumeTracing.version + 1, "volume", volumeTracingId),
     );
diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
index 8b4b7603acb..c7327f44d94 100644
--- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
@@ -8,6 +8,7 @@ import type {
   MaybeUnmergedBucketLoadedPromise,
   UpdateSegmentAction,
   InitializeVolumeTracingAction,
+  InitializeEditableMappingAction,
 } from "oxalis/model/actions/volumetracing_actions";
 import {
   VolumeTracingSaveRelevantActions,
@@ -102,10 +103,6 @@ import compactUpdateActions from "oxalis/model/helpers/compaction/compact_update
 import createProgressCallback from "libs/progress_callback";
 import messages from "messages";
 import window, { alert, document, location } from "libs/window";
-import type {
-  SetMappingAction,
-  SetMappingEnabledAction,
-} from "oxalis/model/actions/settings_actions";
 import { enforceSkeletonTracing } from "../accessors/skeletontracing_accessor";
 
 // This function is needed so that Flow is satisfied
@@ -717,8 +714,13 @@ function* applyAndGetRevertingVolumeBatch(
   };
 }
 
-export function* pushSaveQueueAsync(saveQueueType: SaveQueueType, tracingId: string): Saga<void> {
-  yield* take("WK_READY");
+export function* pushSaveQueueAsync(
+  saveQueueType: SaveQueueType,
+  tracingId: string,
+  isWkReady: boolean = false,
+): Saga<void> {
+  if (!isWkReady) yield* take("WK_READY");
+
   yield* put(setLastSaveTimestampAction(saveQueueType, tracingId));
   let loopCounter = 0;
 
@@ -991,6 +993,23 @@ export function performDiffTracing(
 export function* saveTracingAsync(): Saga<void> {
   yield* takeEvery("INITIALIZE_SKELETONTRACING", saveTracingTypeAsync);
   yield* takeEvery("INITIALIZE_VOLUMETRACING", saveTracingTypeAsync);
+  yield* takeEvery("INITIALIZE_EDITABLE_MAPPING", saveEditableMappingAsync);
+}
+
+let isWkReady = false;
+
+function setWkReady() {
+  isWkReady = true;
+}
+
+export function* saveEditableMappingAsync(
+  initializeAction: InitializeEditableMappingAction,
+): Saga<void> {
+  // No diffing needs to be done for editable mappings as the saga pushes update actions
+  // to the respective save queues, itself
+  const volumeTracingId = initializeAction.mapping.tracingId;
+  yield* takeEvery("WK_READY", setWkReady);
+  yield* fork(pushSaveQueueAsync, "mapping", volumeTracingId, isWkReady);
 }
 export function* saveTracingTypeAsync(
   initializeAction: InitializeSkeletonTracingAction | InitializeVolumeTracingAction,
@@ -1004,7 +1023,9 @@ export function* saveTracingTypeAsync(
     initializeAction.type === "INITIALIZE_SKELETONTRACING" ? "skeleton" : "volume";
   const tracingId = initializeAction.tracing.id;
   yield* fork(pushSaveQueueAsync, saveQueueType, tracingId);
-  let prevTracing = yield* select((state) => selectTracing(state, saveQueueType, tracingId));
+  let prevTracing = (yield* select((state) => selectTracing(state, saveQueueType, tracingId))) as
+    | VolumeTracing
+    | SkeletonTracing;
   let prevFlycam = yield* select((state) => state.flycam);
   let prevTdCamera = yield* select((state) => state.viewModeData.plane.tdCamera);
   yield* take("WK_READY");
@@ -1032,7 +1053,9 @@ export function* saveTracingTypeAsync(
       (state) => state.tracing.restrictions.allowUpdate && state.tracing.restrictions.allowSave,
     );
     if (!allowUpdate) return;
-    const tracing = yield* select((state) => selectTracing(state, saveQueueType, tracingId));
+    const tracing = (yield* select((state) => selectTracing(state, saveQueueType, tracingId))) as
+      | VolumeTracing
+      | SkeletonTracing;
     const flycam = yield* select((state) => state.flycam);
     const tdCamera = yield* select((state) => state.viewModeData.plane.tdCamera);
     const items = compactUpdateActions(
diff --git a/frontend/javascripts/oxalis/model_initialization.ts b/frontend/javascripts/oxalis/model_initialization.ts
index 84bf60da740..72d331f560e 100644
--- a/frontend/javascripts/oxalis/model_initialization.ts
+++ b/frontend/javascripts/oxalis/model_initialization.ts
@@ -7,6 +7,7 @@ import type {
   APIDataLayer,
   ServerVolumeTracing,
   ServerTracing,
+  ServerEditableMapping,
 } from "types/api_flow_types";
 import type { Versions } from "oxalis/view/version_view";
 import {
@@ -38,6 +39,7 @@ import {
   getSharingToken,
   getUserConfiguration,
   getDatasetViewConfiguration,
+  getEditableMapping,
 } from "admin/admin_rest_api";
 import {
   initializeAnnotationAction,
@@ -50,7 +52,10 @@ import {
   setViewModeAction,
   setMappingAction,
 } from "oxalis/model/actions/settings_actions";
-import { initializeVolumeTracingAction } from "oxalis/model/actions/volumetracing_actions";
+import {
+  initializeEditableMappingAction,
+  initializeVolumeTracingAction,
+} from "oxalis/model/actions/volumetracing_actions";
 import {
   setActiveNodeAction,
   initializeSkeletonTracingAction,
@@ -147,9 +152,8 @@ export async function initialize(
     Record<string, any>,
     Array<ServerTracing>,
   ] = await fetchParallel(annotation, datasetId, versions);
-  const displayedVolumeTracings = getServerVolumeTracings(serverTracings).map(
-    (volumeTracing) => volumeTracing.id,
-  );
+  const serverVolumeTracings = getServerVolumeTracings(serverTracings);
+  const displayedVolumeTracings = serverVolumeTracings.map((volumeTracing) => volumeTracing.id);
   initializeDataset(initialFetch, dataset, serverTracings);
   const initialDatasetSettings = await getDatasetViewConfiguration(
     dataset,
@@ -178,7 +182,11 @@ export async function initialize(
 
   // There is no need to initialize the tracing if there is no tracing (View mode).
   if (annotation != null) {
-    initializeTracing(annotation, serverTracings);
+    const editableMappings = await fetchEditableMappings(
+      annotation.tracingStore.url,
+      serverVolumeTracings,
+    );
+    initializeTracing(annotation, serverTracings, editableMappings);
   } else {
     // In view only tracings we need to set the view mode too.
     const { allowedModes } = determineAllowedModes(dataset);
@@ -209,6 +217,16 @@ async function fetchParallel(
   ]);
 }
 
+async function fetchEditableMappings(
+  tracingStoreUrl: string,
+  serverVolumeTracings: ServerVolumeTracing[],
+): Promise<ServerEditableMapping[]> {
+  const promises = serverVolumeTracings
+    .filter((tracing) => tracing.mappingIsEditable)
+    .map((tracing) => getEditableMapping(tracingStoreUrl, tracing.id));
+  return Promise.all(promises);
+}
+
 function validateSpecsForLayers(dataset: APIDataset, requiredBucketCapacity: number): any {
   const layers = dataset.dataSource.dataLayers;
   const specs = getSupportedTextureSpecs();
@@ -242,7 +260,11 @@ function maybeWarnAboutUnsupportedLayers(layers: Array<APIDataLayer>): void {
   }
 }
 
-function initializeTracing(_annotation: APIAnnotation, serverTracings: Array<ServerTracing>) {
+function initializeTracing(
+  _annotation: APIAnnotation,
+  serverTracings: Array<ServerTracing>,
+  editableMappings: Array<ServerEditableMapping>,
+) {
   // This method is not called for the View mode
   const { dataset } = Store.getState();
   let annotation = _annotation;
@@ -279,6 +301,9 @@ function initializeTracing(_annotation: APIAnnotation, serverTracings: Array<Ser
       );
       Store.dispatch(initializeVolumeTracingAction(volumeTracing));
     });
+
+    editableMappings.map((mapping) => Store.dispatch(initializeEditableMappingAction(mapping)));
+
     const skeletonTracing = getNullableSkeletonTracing(serverTracings);
 
     if (skeletonTracing != null) {
diff --git a/frontend/javascripts/oxalis/store.ts b/frontend/javascripts/oxalis/store.ts
index 5da4e3e528e..cc08d6b07bf 100644
--- a/frontend/javascripts/oxalis/store.ts
+++ b/frontend/javascripts/oxalis/store.ts
@@ -229,10 +229,18 @@ export type VolumeTracing = TracingBase & {
 export type ReadOnlyTracing = TracingBase & {
   readonly type: "readonly";
 };
+export type EditableMapping = {
+  readonly type: "mapping";
+  readonly version: number;
+  readonly mappingName: string;
+  // The id of the volume tracing the editable mapping belongs to
+  readonly tracingId: string;
+};
 export type HybridTracing = Annotation & {
   readonly skeleton: SkeletonTracing | null | undefined;
   readonly volumes: Array<VolumeTracing>;
   readonly readOnly: ReadOnlyTracing | null | undefined;
+  readonly mappings: Array<EditableMapping>;
 };
 export type Tracing = HybridTracing;
 export type TraceOrViewCommand =
diff --git a/frontend/javascripts/types/api_flow_types.ts b/frontend/javascripts/types/api_flow_types.ts
index c81e4a96595..8e12902df95 100644
--- a/frontend/javascripts/types/api_flow_types.ts
+++ b/frontend/javascripts/types/api_flow_types.ts
@@ -612,6 +612,12 @@ export type ServerVolumeTracing = ServerTracingBase & {
   mappingIsEditable?: boolean;
 };
 export type ServerTracing = ServerSkeletonTracing | ServerVolumeTracing;
+export type ServerEditableMapping = {
+  version: number;
+  mappingName: string;
+  // The id of the volume tracing the editable mapping belongs to
+  tracingId: string;
+};
 export type APIMeshFile = {
   meshFileName: string;
   mappingName?: string | null | undefined;

From 100b1226cd9e2933a3d8f30430df96800746120a Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 17 May 2022 14:53:17 +0200
Subject: [PATCH 032/122] fix route, token and missing-buckets header parsing

---
 .../datastore/helpers/MissingBucketHeaders.scala           | 2 +-
 .../webknossos/tracingstore/TSRemoteDatastoreClient.scala  | 7 +++++--
 .../tracingstore/controllers/VolumeTracingController.scala | 4 ++--
 .../tracings/editablemapping/EditableMappingService.scala  | 7 ++++---
 4 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/MissingBucketHeaders.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/MissingBucketHeaders.scala
index 00a33f180be..d4cddd89f9a 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/MissingBucketHeaders.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/MissingBucketHeaders.scala
@@ -22,7 +22,7 @@ trait MissingBucketHeaders extends FoxImplicits {
       headerLiteral: String <- headerLiteralOpt.toFox
       headerLiteralTrim = headerLiteral.trim
       _ <- bool2Fox(headerLiteralTrim.startsWith("[") && headerLiteralTrim.endsWith("]"))
-      indicesStr = headerLiteralTrim.drop(1).dropRight(1).split(",").toList
+      indicesStr = headerLiteralTrim.drop(1).dropRight(1).split(",").toList.filter(_.nonEmpty)
       indices <- Fox.serialCombined(indicesStr)(indexStr => tryo(indexStr.trim.toInt))
     } yield indices
 
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
index 58db13585e1..a41c3082c51 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
@@ -32,9 +32,12 @@ class TSRemoteDatastoreClient @Inject()(
       .getWithBytesResponse
 
   def getData(remoteFallbackLayer: RemoteFallbackLayer,
-              dataRequests: List[WebKnossosDataRequest]): Fox[(Array[Byte], List[Int])] =
+              dataRequests: List[WebKnossosDataRequest],
+              userToken: Option[String]): Fox[(Array[Byte], List[Int])] =
     for {
-      response <- rpc(s"${remoteLayerUri(remoteFallbackLayer)}/").post(dataRequests)
+      response <- rpc(s"${remoteLayerUri(remoteFallbackLayer)}/data")
+        .addQueryStringOptional("token", userToken)
+        .post(dataRequests)
       _ <- bool2Fox(Status.isSuccessful(response.status))
       bytes = response.bodyAsBytes.toArray
       indices <- parseMissingBucketHeader(response.header(missingBucketsHeader)) ?~> "failed to parse missing bucket header"
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index 59c4cc244c9..b28c57b2a68 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -137,8 +137,8 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
             tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound")
             hasEditableMapping <- Fox.runOptional(tracing.mappingName)(editableMappingService.exists)
             (data, indices) <- if (hasEditableMapping.getOrElse(false))
-              tracingService.data(tracingId, tracing, request.body)
-            else editableMappingService.volumeData(tracing, request.body, token)
+              editableMappingService.volumeData(tracing, request.body, token)
+            else tracingService.data(tracingId, tracing, request.body)
           } yield Ok(data).withHeaders(getMissingBucketsHeaders(indices): _*)
         }
       }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index e6bcebed898..a2388a8005b 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -336,7 +336,7 @@ class EditableMappingService @Inject()(
       editableMappingId <- tracing.mappingName.toFox
       remoteFallbackLayer <- remoteFallbackLayer(tracing)
       editableMapping <- get(editableMappingId, remoteFallbackLayer, userToken)
-      (unmappedData, indices) <- getUnmappedDataFromDatastore(remoteFallbackLayer, dataRequests)
+      (unmappedData, indices) <- getUnmappedDataFromDatastore(remoteFallbackLayer, dataRequests, userToken)
       segmentIds <- collectSegmentIds(unmappedData, tracing.elementClass)
       relevantMapping <- generateCombinedMappingSubset(segmentIds, editableMapping, remoteFallbackLayer, userToken)
       mappedData <- mapData(unmappedData, relevantMapping, tracing.elementClass)
@@ -421,13 +421,14 @@ class EditableMappingService @Inject()(
   }
 
   private def getUnmappedDataFromDatastore(remoteFallbackLayer: RemoteFallbackLayer,
-                                           dataRequests: DataRequestCollection): Fox[(Array[Byte], List[Int])] =
+                                           dataRequests: DataRequestCollection,
+                                           userToken: Option[String]): Fox[(Array[Byte], List[Int])] =
     for {
       dataRequestsTyped <- Fox.serialCombined(dataRequests) {
         case r: WebKnossosDataRequest => Fox.successful(r.copy(applyAgglomerate = None))
         case _                        => Fox.failure("Editable Mappings currently only work for webKnossos data requests")
       }
-      (data, indices) <- remoteDatastoreClient.getData(remoteFallbackLayer, dataRequestsTyped)
+      (data, indices) <- remoteDatastoreClient.getData(remoteFallbackLayer, dataRequestsTyped, userToken)
     } yield (data, indices)
 
   private def collectSegmentIds(data: Array[Byte], elementClass: ElementClass): Fox[Set[Long]] =

From 2a4d6b7c14cdde282f282e29f03194d9ec8cffc4 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 17 May 2022 14:54:40 +0200
Subject: [PATCH 033/122] fix editable mapping initialization

---
 .../oxalis/model/reducers/volumetracing_reducer.ts  | 13 +++++++++++--
 .../javascripts/oxalis/model/sagas/save_saga.ts     |  3 ++-
 frontend/javascripts/oxalis/view/version_list.tsx   |  4 ++--
 frontend/javascripts/oxalis/view/version_view.tsx   | 12 ++++++++++++
 4 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
index fbcbc7a91f7..9997f4011ff 100644
--- a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
+++ b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
@@ -24,7 +24,6 @@ import {
   setContourTracingModeReducer,
   setMaxCellReducer,
   updateVolumeTracing,
-  updateEditableMapping,
   setMappingNameReducer,
 } from "oxalis/model/reducers/volumetracing_reducer_helpers";
 import { updateKey2 } from "oxalis/model/helpers/deep_update";
@@ -208,7 +207,17 @@ function VolumeTracingReducer(
         type: "mapping",
         ...action.mapping,
       };
-      return updateEditableMapping(state, action.mapping.tracingId, mapping);
+      const newMappings = state.tracing.mappings.filter(
+        (tracing) => tracing.tracingId !== mapping.tracingId,
+      );
+      newMappings.push(mapping);
+      return update(state, {
+        tracing: {
+          mappings: {
+            $set: newMappings,
+          },
+        },
+      });
     }
 
     case "SET_SEGMENTS": {
diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
index c7327f44d94..b0c21466bce 100644
--- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
@@ -829,7 +829,8 @@ export function* sendRequestToServer(saveQueueType: SaveQueueType, tracingId: st
         {
           method: "POST",
           data: compactedSaveQueue,
-          compress: true,
+          // TODO: Switch back to true before merging
+          compress: false,
         },
       );
       const endTime = Date.now();
diff --git a/frontend/javascripts/oxalis/view/version_list.tsx b/frontend/javascripts/oxalis/view/version_list.tsx
index e3fac94b764..f021d526fb0 100644
--- a/frontend/javascripts/oxalis/view/version_list.tsx
+++ b/frontend/javascripts/oxalis/view/version_list.tsx
@@ -16,13 +16,13 @@ import { revertToVersion, serverCreateTracing } from "oxalis/model/sagas/update_
 import { setAnnotationAllowUpdateAction } from "oxalis/model/actions/annotation_actions";
 import { setVersionRestoreVisibilityAction } from "oxalis/model/actions/ui_actions";
 import Model from "oxalis/model";
-import type { SkeletonTracing, VolumeTracing } from "oxalis/store";
+import type { EditableMapping, SkeletonTracing, VolumeTracing } from "oxalis/store";
 import Store from "oxalis/store";
 import VersionEntryGroup from "oxalis/view/version_entry_group";
 import api from "oxalis/api/internal_api";
 type Props = {
   versionedObjectType: SaveQueueType;
-  tracing: SkeletonTracing | VolumeTracing;
+  tracing: SkeletonTracing | VolumeTracing | EditableMapping;
   allowUpdate: boolean;
 };
 type State = {
diff --git a/frontend/javascripts/oxalis/view/version_view.tsx b/frontend/javascripts/oxalis/view/version_view.tsx
index 1c120f61e51..2942fb3d920 100644
--- a/frontend/javascripts/oxalis/view/version_view.tsx
+++ b/frontend/javascripts/oxalis/view/version_view.tsx
@@ -131,6 +131,18 @@ class VersionView extends React.Component<Props, State> {
                 />
               </TabPane>
             ))}
+            {this.props.tracing.mappings.map((mapping) => (
+              <TabPane
+                tab={getReadableNameByVolumeTracingId(this.props.tracing, mapping.tracingId)}
+                key={`${mapping.tracingId}-${mapping.mappingName}`}
+              >
+                <VersionList
+                  versionedObjectType="mapping"
+                  tracing={mapping}
+                  allowUpdate={this.state.initialAllowUpdate}
+                />
+              </TabPane>
+            ))}
           </Tabs>
         </div>
       </div>

From 2e549958871429fb4514f96dc3ac6eaf0722a26f Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 17 May 2022 15:14:01 +0200
Subject: [PATCH 034/122] updateEditableMapping takes list of groups

---
 .../controllers/VolumeTracingController.scala      | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index b28c57b2a68..0d53b3e03ff 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -275,18 +275,20 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
       }
     }
 
-  def updateEditableMapping(token: Option[String], tracingId: String): Action[EditableMappingUpdateActionGroup] =
-    Action.async(validateJson[EditableMappingUpdateActionGroup]) { implicit request =>
+  def updateEditableMapping(token: Option[String], tracingId: String): Action[List[EditableMappingUpdateActionGroup]] =
+    Action.async(validateJson[List[EditableMappingUpdateActionGroup]]) { implicit request =>
       accessTokenService.validateAccess(UserAccessRequest.writeTracing(tracingId), token) {
         for {
           tracing <- tracingService.find(tracingId)
           mappingName <- tracing.mappingName.toFox
           _ <- Fox.assertTrue(editableMappingService.exists(mappingName))
           currentVersion <- editableMappingService.currentVersion(mappingName)
-          _ <- bool2Fox(request.body.version == currentVersion + 1) ?~> "version mismatch"
-          _ <- bool2Fox(request.body.actions.length == 1) ?~> "Editable mapping update group must contain exactly one update action"
-          updateAction <- request.body.actions.headOption.toFox
-          _ <- editableMappingService.update(mappingName, updateAction, request.body.version)
+          _ <- bool2Fox(request.body.length == 1) ?~> "Editable mapping update group must contain exactly one update group"
+          updateGroup <- request.body.headOption.toFox
+          _ <- bool2Fox(updateGroup.version == currentVersion + 1) ?~> "version mismatch"
+          _ <- bool2Fox(updateGroup.actions.length == 1) ?~> "Editable mapping update group must contain exactly one update action"
+          updateAction <- updateGroup.actions.headOption.toFox
+          _ <- editableMappingService.update(mappingName, updateAction, updateGroup.version)
         } yield Ok
       }
     }

From b87c73d0b1248549e3e07acbfac3a74e4adb45e2 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 17 May 2022 15:23:09 +0200
Subject: [PATCH 035/122] affinities are float

---
 .../webknossos/datastore/models/AgglomerateGraph.scala        | 2 +-
 .../webknossos/datastore/services/AgglomerateService.scala    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala
index c1d6162e19f..c1a623c3ad4 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala
@@ -6,7 +6,7 @@ import play.api.libs.json.{Json, OFormat}
 case class AgglomerateGraph(segments: List[Long],
                             edges: List[(Long, Long)],
                             positions: List[Vec3Int],
-                            affinities: List[Long])
+                            affinities: List[Float])
 
 object AgglomerateGraph {
   implicit val jsonFormat: OFormat[AgglomerateGraph] = Json.format[AgglomerateGraph]
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 8b74e1d0211..d03a8777bea 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -241,8 +241,8 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
         reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_positions", nodeCount.toInt, 3, positionsRange(0), 0)
       val edges: Array[Array[Long]] =
         reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_edges", edgeCount.toInt, 2, edgesRange(0), 0)
-      val affinities: Array[Long] =
-        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_affinities", edgeCount.toInt, edgesRange(0))
+      val affinities: Array[Float] =
+        reader.float32().readArrayBlockWithOffset("/agglomerate_to_affinities", edgeCount.toInt, edgesRange(0))
 
       AgglomerateGraph(
         segments = segmentIds.toList,

From 8cc7b785a17851204e6e534fe7ead3a9144a906a Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 17 May 2022 15:45:01 +0200
Subject: [PATCH 036/122] load editable agglomerate skeleton from tracingstore

---
 frontend/javascripts/admin/admin_rest_api.ts  | 18 +++++++++
 .../oxalis/model/sagas/save_saga.ts           | 12 +++---
 .../model/sagas/skeletontracing_saga.ts       | 40 +++++++++++++------
 3 files changed, 52 insertions(+), 18 deletions(-)

diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts
index b8c2b820816..724821032f4 100644
--- a/frontend/javascripts/admin/admin_rest_api.ts
+++ b/frontend/javascripts/admin/admin_rest_api.ts
@@ -1951,6 +1951,24 @@ export function getAgglomerateSkeleton(
   );
 }
 
+export function getEditableAgglomerateSkeleton(
+  tracingStoreUrl: string,
+  tracingId: string,
+  agglomerateId: number,
+): Promise<ArrayBuffer> {
+  return doWithToken((token) =>
+    Request.receiveArraybuffer(
+      `${tracingStoreUrl}/tracings/volume/${tracingId}/agglomerateSkeleton/${agglomerateId}?token=${token}`,
+      // The webworker code cannot do proper error handling and always expects an array buffer from the server.
+      // In this case, the server sends an error json instead of an array buffer sometimes. Therefore, don't use the webworker code.
+      {
+        useWebworkerForArrayBuffer: false,
+        showErrorToast: false,
+      },
+    ),
+  );
+}
+
 export function getMeshfilesForDatasetLayer(
   dataStoreUrl: string,
   datasetId: APIDatasetId,
diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
index b0c21466bce..2574e74ba71 100644
--- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
@@ -991,17 +991,18 @@ export function performDiffTracing(
 
   return actions;
 }
-export function* saveTracingAsync(): Saga<void> {
-  yield* takeEvery("INITIALIZE_SKELETONTRACING", saveTracingTypeAsync);
-  yield* takeEvery("INITIALIZE_VOLUMETRACING", saveTracingTypeAsync);
-  yield* takeEvery("INITIALIZE_EDITABLE_MAPPING", saveEditableMappingAsync);
-}
 
 let isWkReady = false;
 
 function setWkReady() {
   isWkReady = true;
 }
+export function* saveTracingAsync(): Saga<void> {
+  yield* takeEvery("WK_READY", setWkReady);
+  yield* takeEvery("INITIALIZE_SKELETONTRACING", saveTracingTypeAsync);
+  yield* takeEvery("INITIALIZE_VOLUMETRACING", saveTracingTypeAsync);
+  yield* takeEvery("INITIALIZE_EDITABLE_MAPPING", saveEditableMappingAsync);
+}
 
 export function* saveEditableMappingAsync(
   initializeAction: InitializeEditableMappingAction,
@@ -1009,7 +1010,6 @@ export function* saveEditableMappingAsync(
   // No diffing needs to be done for editable mappings as the saga pushes update actions
   // to the respective save queues, itself
   const volumeTracingId = initializeAction.mapping.tracingId;
-  yield* takeEvery("WK_READY", setWkReady);
   yield* fork(pushSaveQueueAsync, "mapping", volumeTracingId, isWkReady);
 }
 export function* saveTracingTypeAsync(
diff --git a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
index 83aad9ecafd..f0a183b8945 100644
--- a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
@@ -70,7 +70,7 @@ import * as Utils from "libs/utils";
 import api from "oxalis/api/internal_api";
 import messages from "messages";
 import { getLayerByName } from "oxalis/model/accessors/dataset_accessor";
-import { getAgglomerateSkeleton } from "admin/admin_rest_api";
+import { getAgglomerateSkeleton, getEditableAgglomerateSkeleton } from "admin/admin_rest_api";
 import { parseProtoTracing } from "oxalis/model/helpers/proto_helpers";
 import createProgressCallback from "libs/progress_callback";
 import {
@@ -232,20 +232,36 @@ function* getAgglomerateSkeletonTracing(
   agglomerateId: number,
 ): Saga<ServerSkeletonTracing> {
   const dataset = yield* select((state) => state.dataset);
+  const annotation = yield* select((state) => state.tracing);
   const layerInfo = getLayerByName(dataset, layerName);
-  // If there is a fallbackLayer, request the agglomerate for that instead of the tracing segmentation layer
-  // @ts-expect-error ts-migrate(2339) FIXME: Property 'fallbackLayer' does not exist on type 'A... Remove this comment to see the full error message
-  const effectiveLayerName = layerInfo.fallbackLayer != null ? layerInfo.fallbackLayer : layerName;
+
+  const editableMapping = annotation.mappings.find(
+    (mapping) => mapping.mappingName === mappingName,
+  );
 
   try {
-    const nmlProtoBuffer = yield* call(
-      getAgglomerateSkeleton,
-      dataset.dataStore.url,
-      dataset,
-      effectiveLayerName,
-      mappingName,
-      agglomerateId,
-    );
+    let nmlProtoBuffer;
+    if (editableMapping == null) {
+      // If there is a fallbackLayer, request the agglomerate for that instead of the tracing segmentation layer
+      const effectiveLayerName =
+        // @ts-expect-error ts-migrate(2339) FIXME: Property 'fallbackLayer' does not exist on type 'A... Remove this comment to see the full error message
+        layerInfo.fallbackLayer != null ? layerInfo.fallbackLayer : layerName;
+      nmlProtoBuffer = yield* call(
+        getAgglomerateSkeleton,
+        dataset.dataStore.url,
+        dataset,
+        effectiveLayerName,
+        mappingName,
+        agglomerateId,
+      );
+    } else {
+      nmlProtoBuffer = yield* call(
+        getEditableAgglomerateSkeleton,
+        annotation.tracingStore.url,
+        editableMapping.tracingId,
+        agglomerateId,
+      );
+    }
     const parsedTracing = parseProtoTracing(nmlProtoBuffer, "skeleton");
 
     if (!("trees" in parsedTracing)) {

From 3b70b58d5ef88da8fd8fd0334d4a51b9517431b6 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 17 May 2022 16:28:56 +0200
Subject: [PATCH 037/122] fix that source and target were in different trees

---
 frontend/javascripts/oxalis/model/sagas/proofread_saga.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 035c7535a44..4758670cf7c 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -115,7 +115,7 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
 
   const items = [];
   if (action.type === "MERGE_TREES") {
-    if (sourceTree === targetTree || sourceNodeAgglomerateId === targetNodeAgglomerateId) {
+    if (sourceNodeAgglomerateId === targetNodeAgglomerateId) {
       Toast.error("Segments that should be merged need to be in different agglomerates.");
       return;
     }
@@ -129,7 +129,7 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
       ),
     );
   } else if (action.type === "DELETE_EDGE") {
-    if (sourceTree !== targetTree || sourceNodeAgglomerateId !== targetNodeAgglomerateId) {
+    if (sourceNodeAgglomerateId !== targetNodeAgglomerateId) {
       Toast.error("Segments that should be split need to be in the same agglomerate.");
       return;
     }

From 89c85c493ee495d11bb98a2ded46e910fd1b55f1 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 17 May 2022 17:16:24 +0200
Subject: [PATCH 038/122] some backend fixes

---
 .../services/AgglomerateService.scala         |  2 +-
 .../controllers/VolumeTracingController.scala |  2 +-
 .../EditableMappingService.scala              | 29 ++++++++-----------
 3 files changed, 14 insertions(+), 19 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index d03a8777bea..4892e224936 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -246,7 +246,7 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
 
       AgglomerateGraph(
         segments = segmentIds.toList,
-        edges = edges.toList.map(e => (e(0), e(1))),
+        edges = edges.toList.map(e => (segmentIds(e(0).toInt), segmentIds(e(1).toInt))),
         positions = positions.toList.map(pos => Vec3Int(pos(0).toInt, pos(1).toInt, pos(2).toInt)),
         affinities = affinities.toList
       )
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index 0d53b3e03ff..d4e4bf63107 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -282,7 +282,7 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
           tracing <- tracingService.find(tracingId)
           mappingName <- tracing.mappingName.toFox
           _ <- Fox.assertTrue(editableMappingService.exists(mappingName))
-          currentVersion <- editableMappingService.currentVersion(mappingName)
+          currentVersion <- editableMappingService.newestMaterializableVersion(mappingName)
           _ <- bool2Fox(request.body.length == 1) ?~> "Editable mapping update group must contain exactly one update group"
           updateGroup <- request.body.headOption.toFox
           _ <- bool2Fox(updateGroup.version == currentVersion + 1) ?~> "version mismatch"
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index a2388a8005b..78c419089f9 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -39,12 +39,14 @@ class EditableMappingService @Inject()(
 
   private def generateId: String = UUID.randomUUID.toString
 
-  def currentVersion(editableMappingId: String): Fox[Long] =
-    tracingDataStore.editableMappings.getVersion(editableMappingId, mayBeEmpty = Some(true), emptyFallback = Some(0L))
+  def newestMaterializableVersion(editableMappingId: String): Fox[Long] =
+    tracingDataStore.editableMappingUpdates.getVersion(editableMappingId,
+                                                       mayBeEmpty = Some(true),
+                                                       emptyFallback = Some(0L))
 
   def infoJson(tracingId: String, editableMappingId: String): Fox[JsObject] =
     for {
-      version <- currentVersion(editableMappingId)
+      version <- newestMaterializableVersion(editableMappingId)
     } yield
       Json.obj(
         "mappingName" -> editableMappingId,
@@ -93,9 +95,7 @@ class EditableMappingService @Inject()(
     for {
       closestMaterializedVersion: VersionedKeyValuePair[EditableMapping] <- tracingDataStore.editableMappings
         .get(editableMappingId, version)(fromJson[EditableMapping])
-      desiredVersion <- findDesiredOrNewestPossibleVersion(closestMaterializedVersion.version,
-                                                           editableMappingId,
-                                                           version)
+      desiredVersion <- findDesiredOrNewestPossibleVersion(editableMappingId, version)
       materialized <- applyPendingUpdates(
         editableMappingId,
         desiredVersion,
@@ -112,19 +112,14 @@ class EditableMappingService @Inject()(
   private def shouldPersistMaterialized(previouslyMaterializedVersion: Long, newVersion: Long): Boolean =
     newVersion > previouslyMaterializedVersion && newVersion % 10 == 5
 
-  private def findDesiredOrNewestPossibleVersion(existingMaterializedVersion: Long,
-                                                 editableMappingId: String,
-                                                 desiredVersion: Option[Long]): Fox[Long] =
+  private def findDesiredOrNewestPossibleVersion(editableMappingId: String, desiredVersion: Option[Long]): Fox[Long] =
     /*
      * Determines the newest saved version from the updates column.
      * if there are no updates at all, assume mapping is brand new,
      * hence the emptyFallbck tracing.version)
      */
     for {
-      newestUpdateVersion <- tracingDataStore.editableMappingUpdates.getVersion(editableMappingId,
-                                                                                mayBeEmpty = Some(true),
-                                                                                emptyFallback =
-                                                                                  Some(existingMaterializedVersion))
+      newestUpdateVersion <- newestMaterializableVersion(editableMappingId)
     } yield {
       desiredVersion match {
         case None              => newestUpdateVersion
@@ -191,7 +186,7 @@ class EditableMappingService @Inject()(
   private def splitGraph(agglomerateGraph: AgglomerateGraph,
                          segmentId1: Long,
                          segmentId2: Long): (AgglomerateGraph, AgglomerateGraph) = {
-    val edgesMinusOne = agglomerateGraph.edges.filter {
+    val edgesMinusOne = agglomerateGraph.edges.filterNot {
       case (from, to) =>
         (from == segmentId1 && to == segmentId2) || (from == segmentId2 && to == segmentId1)
     }
@@ -227,11 +222,11 @@ class EditableMappingService @Inject()(
 
   private def computeConnectedComponent(startNode: Long, edges: List[(Long, Long)]): Set[Long] = {
     val neighborsByNode =
-      mutable.HashMap[Long, mutable.MutableList[Long]]().withDefaultValue(mutable.MutableList[Long]())
+      mutable.HashMap[Long, List[Long]]().withDefaultValue(List[Long]())
     edges.foreach {
       case (from, to) =>
-        neighborsByNode(from) += to
-        neighborsByNode(to) += from
+        neighborsByNode(from) = to :: neighborsByNode(from)
+        neighborsByNode(to) = from :: neighborsByNode(to)
     }
     val nodesToVisit = mutable.HashSet[Long](startNode)
     val visitedNodes = mutable.HashSet[Long]()

From 1ff671698c09c41f25db3888084bd9085c518bf3 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 18 May 2022 09:57:45 +0200
Subject: [PATCH 039/122] Add logging, fix update action order, store as lists
 in fossildb

---
 .../datastore/models/AgglomerateGraph.scala   |  5 +++-
 .../controllers/VolumeTracingController.scala |  3 +-
 .../editablemapping/EditableMapping.scala     |  4 ++-
 .../EditableMappingService.scala              | 29 ++++++++++++++-----
 .../EditableMappingUpdateActions.scala        | 18 ++++++++----
 5 files changed, 42 insertions(+), 17 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala
index c1a623c3ad4..c607f5a9762 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala
@@ -6,7 +6,10 @@ import play.api.libs.json.{Json, OFormat}
 case class AgglomerateGraph(segments: List[Long],
                             edges: List[(Long, Long)],
                             positions: List[Vec3Int],
-                            affinities: List[Float])
+                            affinities: List[Float]) {
+  override def toString: String =
+    f"AgglomerateGraph(${segments.length} segments, ${edges.length} edges)"
+}
 
 object AgglomerateGraph {
   implicit val jsonFormat: OFormat[AgglomerateGraph] = Json.format[AgglomerateGraph]
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index d4e4bf63107..cf3c9a6d6c5 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -287,8 +287,7 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
           updateGroup <- request.body.headOption.toFox
           _ <- bool2Fox(updateGroup.version == currentVersion + 1) ?~> "version mismatch"
           _ <- bool2Fox(updateGroup.actions.length == 1) ?~> "Editable mapping update group must contain exactly one update action"
-          updateAction <- updateGroup.actions.headOption.toFox
-          _ <- editableMappingService.update(mappingName, updateAction, updateGroup.version)
+          _ <- editableMappingService.update(mappingName, updateGroup, updateGroup.version)
         } yield Ok
       }
     }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
index 9121b835084..75fb625e0e6 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
@@ -8,7 +8,9 @@ case class EditableMapping(
     baseMappingName: String,
     segmentToAgglomerate: Map[Long, Long],
     agglomerateToGraph: Map[Long, AgglomerateGraph],
-)
+) {
+  override def toString: String = f"EditableMapping(agglomerates:${agglomerateToGraph.keySet})"
+}
 
 object EditableMapping extends AdditionalJsonFormats {
   implicit val jsonFormat: OFormat[EditableMapping] = Json.format[EditableMapping]
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 78c419089f9..a9d6cc5fd76 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -22,6 +22,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.{
   TracingDataStore,
   VersionedKeyValuePair
 }
+import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common.Box.tryo
 import net.liftweb.common.{Empty, Full}
 import play.api.libs.json.{JsObject, JsValue, Json}
@@ -35,7 +36,8 @@ class EditableMappingService @Inject()(
 )(implicit ec: ExecutionContext)
     extends KeyValueStoreImplicits
     with FoxImplicits
-    with ProtoGeometryImplicits {
+    with ProtoGeometryImplicits
+    with LazyLogging {
 
   private def generateId: String = UUID.randomUUID.toString
 
@@ -96,6 +98,8 @@ class EditableMappingService @Inject()(
       closestMaterializedVersion: VersionedKeyValuePair[EditableMapping] <- tracingDataStore.editableMappings
         .get(editableMappingId, version)(fromJson[EditableMapping])
       desiredVersion <- findDesiredOrNewestPossibleVersion(editableMappingId, version)
+      _ = logger.info(
+        f"Loading mapping version $desiredVersion, closest materialized is version ${closestMaterializedVersion.version} (${closestMaterializedVersion.value})")
       materialized <- applyPendingUpdates(
         editableMappingId,
         desiredVersion,
@@ -104,6 +108,7 @@ class EditableMappingService @Inject()(
         closestMaterializedVersion.version,
         userToken
       )
+      _ = logger.info(s"Materialized mapping: $materialized")
       _ <- Fox.runIf(shouldPersistMaterialized(closestMaterializedVersion.version, desiredVersion)) {
         tracingDataStore.editableMappings.put(editableMappingId, desiredVersion, materialized)
       }
@@ -149,6 +154,7 @@ class EditableMappingService @Inject()(
 
     for {
       pendingUpdates <- findPendingUpdates(editableMappingId, existingVersion, desiredVersion)
+      _ = logger.info(s"Applying ${pendingUpdates.length} mapping updates: $pendingUpdates...")
       appliedEditableMapping <- updateIter(Some(existingEditableMapping), pendingUpdates)
     } yield appliedEditableMapping
   }
@@ -170,11 +176,15 @@ class EditableMappingService @Inject()(
                                userToken: Option[String]): Fox[EditableMapping] =
     for {
       agglomerateGraph <- agglomerateGraphForId(mapping, update.agglomerateId, remoteFallbackLayer, userToken)
+      _ = logger.info(
+        s"Applying one split action on agglomerate ${update.agglomerateId} (previously $agglomerateGraph)...")
       segmentId1 <- findSegmentIdAtPosition(remoteFallbackLayer, update.segmentPosition1, update.mag, userToken)
       segmentId2 <- findSegmentIdAtPosition(remoteFallbackLayer, update.segmentPosition2, update.mag, userToken)
       largestExistingAgglomerateId <- largestAgglomerateId(mapping, remoteFallbackLayer, userToken)
       agglomerateId2 = largestExistingAgglomerateId + 1L
       (graph1, graph2) = splitGraph(agglomerateGraph, segmentId1, segmentId2)
+      _ = logger.info(
+        s"Graphs after split: Agglomerate ${update.agglomerateId}: $graph1, Aggloemrate $agglomerateId2: $graph2")
       splitSegmentToAgglomerate = graph2.segments.map(_ -> agglomerateId2).toMap
     } yield
       EditableMapping(
@@ -312,16 +322,19 @@ class EditableMappingService @Inject()(
       implicit ec: ExecutionContext): Fox[List[EditableMappingUpdateAction]] =
     if (desiredVersion == existingVersion) Fox.successful(List())
     else {
-      tracingDataStore.editableMappingUpdates.getMultipleVersions(
-        editableMappingId,
-        Some(desiredVersion),
-        Some(existingVersion + 1)
-      )(fromJson[EditableMappingUpdateAction])
+      for {
+        updates <- tracingDataStore.editableMappingUpdates.getMultipleVersions(
+          editableMappingId,
+          Some(desiredVersion),
+          Some(existingVersion + 1)
+        )(fromJson[List[EditableMappingUpdateAction]])
+      } yield updates.reverse.flatten
     }
 
-  def update(editableMappingId: String, updateAction: EditableMappingUpdateAction, version: Long): Fox[Unit] =
+  def update(editableMappingId: String, updateActionGroup: EditableMappingUpdateActionGroup, version: Long): Fox[Unit] =
     for {
-      _ <- tracingDataStore.editableMappingUpdates.put(editableMappingId, version, updateAction)
+      actionsWithTimestamp <- Fox.successful(updateActionGroup.actions.map(_.addTimestamp(updateActionGroup.timestamp)))
+      _ <- tracingDataStore.editableMappingUpdates.put(editableMappingId, version, actionsWithTimestamp)
     } yield ()
 
   def volumeData(tracing: VolumeTracing,
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
index 0e20e12fa7d..041746c22c0 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala
@@ -4,13 +4,18 @@ import com.scalableminds.util.geometry.Vec3Int
 import play.api.libs.json.Format.GenericFormat
 import play.api.libs.json._
 
-trait EditableMappingUpdateAction {}
+trait EditableMappingUpdateAction {
+  def addTimestamp(timestamp: Long): EditableMappingUpdateAction
+}
 
 case class SplitAgglomerateUpdateAction(agglomerateId: Long,
                                         segmentPosition1: Vec3Int,
                                         segmentPosition2: Vec3Int,
-                                        mag: Vec3Int)
-    extends EditableMappingUpdateAction {}
+                                        mag: Vec3Int,
+                                        actionTimestamp: Option[Long] = None)
+    extends EditableMappingUpdateAction {
+  override def addTimestamp(timestamp: Long): EditableMappingUpdateAction = this.copy(actionTimestamp = Some(timestamp))
+}
 
 object SplitAgglomerateUpdateAction {
   implicit val jsonFormat: OFormat[SplitAgglomerateUpdateAction] = Json.format[SplitAgglomerateUpdateAction]
@@ -20,8 +25,11 @@ case class MergeAgglomerateUpdateAction(agglomerateId1: Long,
                                         agglomerateId2: Long,
                                         segmentPosition1: Vec3Int,
                                         segmentPosition2: Vec3Int,
-                                        mag: Vec3Int)
-    extends EditableMappingUpdateAction {}
+                                        mag: Vec3Int,
+                                        actionTimestamp: Option[Long] = None)
+    extends EditableMappingUpdateAction {
+  override def addTimestamp(timestamp: Long): EditableMappingUpdateAction = this.copy(actionTimestamp = Some(timestamp))
+}
 
 object MergeAgglomerateUpdateAction {
   implicit val jsonFormat: OFormat[MergeAgglomerateUpdateAction] = Json.format[MergeAgglomerateUpdateAction]

From 5a364652a7ffece8faef640fb73ee55900441da8 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 18 May 2022 11:30:07 +0200
Subject: [PATCH 040/122] agglomerate skeleton node ids start at one

---
 .../webknossos/datastore/services/AgglomerateService.scala  | 6 ++++--
 .../tracings/editablemapping/EditableMappingService.scala   | 5 +++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 4892e224936..65b4efa8222 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -154,16 +154,18 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
       val edges: Array[Array[Long]] =
         reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_edges", edgeCount.toInt, 2, edgesRange(0), 0)
 
+      val nodeIdStartAtOneOffset = 1
+
       val nodes = positions.zipWithIndex.map {
         case (pos, idx) =>
           NodeDefaults.createInstance.copy(
-            id = idx,
+            id = idx + nodeIdStartAtOneOffset,
             position = Vec3IntProto(pos(0).toInt, pos(1).toInt, pos(2).toInt)
           )
       }
 
       val skeletonEdges = edges.map { e =>
-        Edge(source = e(0).toInt, target = e(1).toInt)
+        Edge(source = e(0).toInt + nodeIdStartAtOneOffset, target = e(1).toInt + nodeIdStartAtOneOffset)
       }
 
       val trees = Seq(
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index a9d6cc5fd76..aa91c57795b 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -388,15 +388,16 @@ class EditableMappingService @Inject()(
                                      agglomerateId: Long): Fox[Array[Byte]] =
     for {
       graph <- editableMapping.agglomerateToGraph.get(agglomerateId)
+      nodeIdStartAtOneOffset = 1
       nodes = graph.positions.zipWithIndex.map {
         case (pos, idx) =>
           NodeDefaults.createInstance.copy(
-            id = idx,
+            id = idx + nodeIdStartAtOneOffset,
             position = pos
           )
       }
       skeletonEdges = graph.edges.map { e =>
-        Edge(source = e._1.toInt, target = e._2.toInt)
+        Edge(source = e._1.toInt + nodeIdStartAtOneOffset, target = e._2.toInt + nodeIdStartAtOneOffset)
       }
 
       trees = Seq(

From c04e9bf9d3732d2abcbed0b2a9b91a144f176006 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Wed, 18 May 2022 12:05:49 +0200
Subject: [PATCH 041/122] remap tree and node ids if there ids below the
 minimum

---
 frontend/javascripts/oxalis/constants.ts        |  2 ++
 .../reducers/skeletontracing_reducer_helpers.ts | 17 ++++++++++++-----
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/frontend/javascripts/oxalis/constants.ts b/frontend/javascripts/oxalis/constants.ts
index 79436f117bf..e86e8fc4903 100644
--- a/frontend/javascripts/oxalis/constants.ts
+++ b/frontend/javascripts/oxalis/constants.ts
@@ -295,6 +295,8 @@ const Constants = {
   DEFAULT_NODE_RADIUS: 1.0,
   RESIZE_THROTTLE_TIME: 50,
   MIN_TREE_ID: 1,
+  // TreeIds > 1024^2 break webKnossos, see https://github.com/scalableminds/webknossos/issues/5009
+  MAX_TREE_ID: 1048576,
   MIN_NODE_ID: 1,
   // Maximum of how many buckets will be held in RAM (per layer)
   MAXIMUM_BUCKET_COUNT_PER_LAYER: 5000,
diff --git a/frontend/javascripts/oxalis/model/reducers/skeletontracing_reducer_helpers.ts b/frontend/javascripts/oxalis/model/reducers/skeletontracing_reducer_helpers.ts
index 9fe193c4ca6..9429bde5cb3 100644
--- a/frontend/javascripts/oxalis/model/reducers/skeletontracing_reducer_helpers.ts
+++ b/frontend/javascripts/oxalis/model/reducers/skeletontracing_reducer_helpers.ts
@@ -59,8 +59,13 @@ export function generateTreeName(state: OxalisState, timestamp: number, treeId:
 
   return `${prefix}${Utils.zeroPad(treeId, 3)}`;
 }
+function getMinimumNodeId(trees: TreeMap | MutableTreeMap): number {
+  const minNodeId = _.min(_.flatMap(trees, (tree) => tree.nodes.map((n) => n.id)));
+
+  return minNodeId != null ? minNodeId : Constants.MIN_NODE_ID;
+}
 export function getMaximumNodeId(trees: TreeMap | MutableTreeMap): number {
-  const newMaxNodeId = _.max(_.flatMap(trees, (__) => __.nodes.map((n) => n.id)));
+  const newMaxNodeId = _.max(_.flatMap(trees, (tree) => tree.nodes.map((n) => n.id)));
 
   return newMaxNodeId != null ? newMaxNodeId : Constants.MIN_NODE_ID - 1;
 }
@@ -438,7 +443,7 @@ export function deleteBranchPoint(
   const { branchPointsAllowed } = restrictions;
   const { trees } = skeletonTracing;
 
-  const hasBranchPoints = _.some(_.map(trees, (__) => !_.isEmpty(__.branchPoints)));
+  const hasBranchPoints = _.some(_.map(trees, (tree) => !_.isEmpty(tree.branchPoints)));
 
   if (!branchPointsAllowed || !hasBranchPoints) return Maybe.Nothing();
 
@@ -534,10 +539,12 @@ export function addTreesAndGroups(
   treeGroups: Array<MutableTreeGroup>,
 ): Maybe<[MutableTreeMap, Array<TreeGroup>, number]> {
   const treeIds = Object.keys(trees).map((treeId) => Number(treeId));
-  // TreeIds > 1024^2 break webKnossos, see https://github.com/scalableminds/webknossos/issues/5009
-  const hasTreeIdsLargerThanMaximum = treeIds.some((treeId) => treeId > 1048576);
+  const hasInvalidTreeIds = treeIds.some(
+    (treeId) => treeId < Constants.MIN_TREE_ID || treeId > Constants.MAX_TREE_ID,
+  );
+  const hasInvalidNodeIds = getMinimumNodeId(trees) < Constants.MIN_NODE_ID;
   const needsReassignedIds =
-    Object.keys(skeletonTracing.trees).length > 0 || hasTreeIdsLargerThanMaximum;
+    Object.keys(skeletonTracing.trees).length > 0 || hasInvalidTreeIds || hasInvalidNodeIds;
 
   if (!needsReassignedIds) {
     // Without reassigning ids, the code is considerably faster.

From 1233b60bd2eb1f7d13cdf70f752c9966418051b8 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 18 May 2022 14:15:50 +0200
Subject: [PATCH 042/122] localize node ids in agglomerate skeleton

---
 .../tracings/editablemapping/EditableMappingService.scala     | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index aa91c57795b..6f456d8f1d9 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -396,8 +396,10 @@ class EditableMappingService @Inject()(
             position = pos
           )
       }
+      segmentIdToNodeIdMinusOne: Map[Long, Int] = graph.segments.zipWithIndex.toMap
       skeletonEdges = graph.edges.map { e =>
-        Edge(source = e._1.toInt + nodeIdStartAtOneOffset, target = e._2.toInt + nodeIdStartAtOneOffset)
+        Edge(source = segmentIdToNodeIdMinusOne(e._1) + nodeIdStartAtOneOffset,
+             target = segmentIdToNodeIdMinusOne(e._2) + nodeIdStartAtOneOffset)
       }
 
       trees = Seq(

From 9ff2ec977c129db64de345fdeaf4d70fd1da6460 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Wed, 18 May 2022 15:02:55 +0200
Subject: [PATCH 043/122] correctly set new mapping name after creating an
 editable mapping

---
 .../oxalis/model/actions/settings_actions.ts    | 14 ++++++++++++++
 .../oxalis/model/reducers/settings_reducer.ts   |  5 +++++
 .../model/reducers/volumetracing_reducer.ts     | 17 ++++++++++++++---
 .../oxalis/model/sagas/proofread_saga.ts        |  8 +++++---
 4 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/actions/settings_actions.ts b/frontend/javascripts/oxalis/model/actions/settings_actions.ts
index 8c9bfecb112..841ce299234 100644
--- a/frontend/javascripts/oxalis/model/actions/settings_actions.ts
+++ b/frontend/javascripts/oxalis/model/actions/settings_actions.ts
@@ -85,6 +85,11 @@ export type SetMappingAction = {
   layerName: string;
   showLoadingIndicator: boolean | null | undefined;
 };
+export type SetMappingNameAction = {
+  type: "SET_MAPPING_NAME";
+  mappingName: string;
+  layerName: string;
+};
 type SetHideUnmappedIdsAction = {
   type: "SET_HIDE_UNMAPPED_IDS";
   hideUnmappedIds: boolean;
@@ -102,6 +107,7 @@ export type SettingAction =
   | SetControlModeAction
   | SetMappingEnabledAction
   | SetMappingAction
+  | SetMappingNameAction
   | SetHideUnmappedIdsAction
   | SetHistogramDataAction
   | InitializeGpuSetupAction;
@@ -233,6 +239,14 @@ export const setMappingAction = (
   hideUnmappedIds,
   showLoadingIndicator,
 });
+export const setMappingNameAction = (
+  layerName: string,
+  mappingName: string,
+): SetMappingNameAction => ({
+  type: "SET_MAPPING_NAME",
+  layerName,
+  mappingName,
+});
 export const setHideUnmappedIdsAction = (
   layerName: string,
   hideUnmappedIds: boolean,
diff --git a/frontend/javascripts/oxalis/model/reducers/settings_reducer.ts b/frontend/javascripts/oxalis/model/reducers/settings_reducer.ts
index 915ae11b007..9523668666b 100644
--- a/frontend/javascripts/oxalis/model/reducers/settings_reducer.ts
+++ b/frontend/javascripts/oxalis/model/reducers/settings_reducer.ts
@@ -252,6 +252,11 @@ function SettingsReducer(state: OxalisState, action: Action): OxalisState {
       );
     }
 
+    case "SET_MAPPING_NAME": {
+      const { mappingName, layerName } = action;
+      return updateActiveMapping(state, { mappingName }, layerName);
+    }
+
     default: // pass;
   }
 
diff --git a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
index 9997f4011ff..c7a6e51fb7e 100644
--- a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
+++ b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
@@ -30,8 +30,12 @@ import { updateKey2 } from "oxalis/model/helpers/deep_update";
 import DiffableMap from "libs/diffable_map";
 import * as Utils from "libs/utils";
 import type { ServerVolumeTracing } from "types/api_flow_types";
-import { SetMappingAction, SetMappingEnabledAction } from "../actions/settings_actions";
-import { getMappingInfo } from "../accessors/dataset_accessor";
+import {
+  SetMappingAction,
+  SetMappingEnabledAction,
+  SetMappingNameAction,
+} from "oxalis/model/actions/settings_actions";
+import { getMappingInfo } from "oxalis/model/accessors/dataset_accessor";
 type SegmentUpdateInfo =
   | {
       readonly type: "UPDATE_VOLUME_TRACING";
@@ -183,7 +187,7 @@ export function serverVolumeToClientVolumeTracing(tracing: ServerVolumeTracing):
 
 function VolumeTracingReducer(
   state: OxalisState,
-  action: VolumeTracingAction | SetMappingAction | SetMappingEnabledAction,
+  action: VolumeTracingAction | SetMappingAction | SetMappingEnabledAction | SetMappingNameAction,
 ): OxalisState {
   switch (action.type) {
     case "INITIALIZE_VOLUMETRACING": {
@@ -302,6 +306,13 @@ function VolumeTracingReducer(
       );
     }
 
+    case "SET_MAPPING_NAME": {
+      const { mappingName, layerName } = action;
+      return updateVolumeTracing(state, volumeTracing.tracingId, {
+        mappingName,
+      });
+    }
+
     case "SET_MAPPING_IS_EDITABLE": {
       return updateVolumeTracing(state, volumeTracing.tracingId, {
         mappingIsEditable: true,
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 4758670cf7c..5fa61c68b5b 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -28,6 +28,7 @@ import {
 } from "oxalis/model/accessors/volumetracing_accessor";
 import { getMappingInfo, getResolutionInfo } from "oxalis/model/accessors/dataset_accessor";
 import { makeMappingEditable } from "admin/admin_rest_api";
+import { setMappingAction, setMappingNameAction } from "oxalis/model/actions/settings_actions";
 
 export default function* proofreadMapping(): Saga<any> {
   yield* take("INITIALIZE_SKELETONTRACING");
@@ -75,11 +76,13 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
       tracingStoreUrl,
       volumeTracingId,
     );
-    yield* put(setMappingisEditableAction());
-    yield* put(initializeEditableMappingAction(serverEditableMapping));
+    // The server increments the volume tracing's version by 1 when switching the mapping to an editable one
     yield* put(
       setVersionNumberAction(upToDateVolumeTracing.version + 1, "volume", volumeTracingId),
     );
+    yield* put(setMappingNameAction(layerName, serverEditableMapping.mappingName));
+    yield* put(setMappingisEditableAction());
+    yield* put(initializeEditableMappingAction(serverEditableMapping));
   }
 
   const resolutionInfo = getResolutionInfo(volumeTracingLayer.resolutions);
@@ -145,7 +148,6 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
 
   if (items.length === 0) return;
 
-  // TODO: Will there be a separate end point for these update actions?
   yield* put(pushSaveQueueTransaction(items, "mapping", volumeTracingId));
   yield* call([Model, Model.ensureSavedState]);
 

From 3941a95de0fda3f370f5736d3d3f16098fa6ee19 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Wed, 18 May 2022 15:49:20 +0200
Subject: [PATCH 044/122] fix tests

---
 .../model/reducers/volumetracing_reducer.ts   |  2 +-
 .../oxalis/model/sagas/proofread_saga.ts      |  2 +-
 .../oxalis/model/sagas/save_saga.ts           |  3 +--
 .../binary/layers/wkstore_adapter.spec.ts     |  2 +-
 .../test/reducers/save_reducer.spec.ts        | 13 ++++++++++---
 .../test/sagas/annotation_tool_saga.spec.ts   | 19 ++++++++++++++++---
 .../javascripts/test/sagas/save_saga.spec.ts  |  8 ++++----
 .../test/sagas/skeletontracing_saga.spec.ts   |  4 ++--
 .../volumetracing/volumetracing_saga.spec.ts  |  4 ++--
 9 files changed, 38 insertions(+), 19 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
index c7a6e51fb7e..ab001d8fe25 100644
--- a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
+++ b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
@@ -307,7 +307,7 @@ function VolumeTracingReducer(
     }
 
     case "SET_MAPPING_NAME": {
-      const { mappingName, layerName } = action;
+      const { mappingName } = action;
       return updateVolumeTracing(state, volumeTracing.tracingId, {
         mappingName,
       });
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 5fa61c68b5b..caf47262fd1 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -28,7 +28,7 @@ import {
 } from "oxalis/model/accessors/volumetracing_accessor";
 import { getMappingInfo, getResolutionInfo } from "oxalis/model/accessors/dataset_accessor";
 import { makeMappingEditable } from "admin/admin_rest_api";
-import { setMappingAction, setMappingNameAction } from "oxalis/model/actions/settings_actions";
+import { setMappingNameAction } from "oxalis/model/actions/settings_actions";
 
 export default function* proofreadMapping(): Saga<any> {
   yield* take("INITIALIZE_SKELETONTRACING");
diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
index 2574e74ba71..786c8ef07b1 100644
--- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
@@ -829,8 +829,7 @@ export function* sendRequestToServer(saveQueueType: SaveQueueType, tracingId: st
         {
           method: "POST",
           data: compactedSaveQueue,
-          // TODO: Switch back to true before merging
-          compress: false,
+          compress: true,
         },
       );
       const endTime = Date.now();
diff --git a/frontend/javascripts/test/model/binary/layers/wkstore_adapter.spec.ts b/frontend/javascripts/test/model/binary/layers/wkstore_adapter.spec.ts
index 3084805bb22..8aefc3b4731 100644
--- a/frontend/javascripts/test/model/binary/layers/wkstore_adapter.spec.ts
+++ b/frontend/javascripts/test/model/binary/layers/wkstore_adapter.spec.ts
@@ -250,7 +250,7 @@ test.serial("sendToStore: Request Handling should send the correct request param
       },
     ],
     transactionId: "dummyRequestId",
-    tracingType: "volume",
+    saveQueueType: "volume",
     tracingId,
   };
   return sendToStore(batch, tracingId).then(() => {
diff --git a/frontend/javascripts/test/reducers/save_reducer.spec.ts b/frontend/javascripts/test/reducers/save_reducer.spec.ts
index 3ff8725f61c..e20b5a6ef4c 100644
--- a/frontend/javascripts/test/reducers/save_reducer.spec.ts
+++ b/frontend/javascripts/test/reducers/save_reducer.spec.ts
@@ -2,6 +2,7 @@ import Maybe from "data.maybe";
 import mockRequire from "mock-require";
 import test from "ava";
 import "test/reducers/save_reducer.mock";
+import type { SaveState } from "oxalis/store";
 import { createSaveQueueFromUpdateActions } from "../helpers/saveHelpers";
 const TIMESTAMP = 1494695001688;
 const DateMock = {
@@ -15,16 +16,22 @@ mockRequire("oxalis/model/accessors/skeletontracing_accessor", AccessorMock);
 const SaveActions = mockRequire.reRequire("oxalis/model/actions/save_actions");
 const SaveReducer = mockRequire.reRequire("oxalis/model/reducers/save_reducer").default;
 const { createEdge } = mockRequire.reRequire("oxalis/model/sagas/update_actions");
-const initialState = {
+const initialState: { save: SaveState } = {
   save: {
-    isBusy: false,
+    isBusyInfo: {
+      skeleton: false,
+      volume: false,
+      mapping: false,
+    },
     queue: {
       skeleton: [],
       volumes: {},
+      mappings: {},
     },
     lastSaveTimestamp: {
       skeleton: 0,
-      volume: {},
+      volumes: {},
+      mappings: {},
     },
     progressInfo: {
       processedActionCount: 0,
diff --git a/frontend/javascripts/test/sagas/annotation_tool_saga.spec.ts b/frontend/javascripts/test/sagas/annotation_tool_saga.spec.ts
index d3b9d71056a..cc80fe97b2c 100644
--- a/frontend/javascripts/test/sagas/annotation_tool_saga.spec.ts
+++ b/frontend/javascripts/test/sagas/annotation_tool_saga.spec.ts
@@ -13,8 +13,16 @@ Object.values(AnnotationToolEnum).forEach((annotationTool) => {
 mockRequire("oxalis/model/accessors/tool_accessor", {
   getDisabledInfoForTools: () => disabledInfoMock,
 });
-const { MoveTool, SkeletonTool, BoundingBoxTool, DrawTool, EraseTool, FillCellTool, PickCellTool } =
-  mockRequire.reRequire("oxalis/controller/combinations/tool_controls");
+const {
+  MoveTool,
+  SkeletonTool,
+  BoundingBoxTool,
+  DrawTool,
+  EraseTool,
+  FillCellTool,
+  PickCellTool,
+  ProofreadTool,
+} = mockRequire.reRequire("oxalis/controller/combinations/tool_controls");
 const UiReducer = mockRequire.reRequire("oxalis/model/reducers/ui_reducer").default;
 const { wkReadyAction } = mockRequire.reRequire("oxalis/model/actions/actions");
 const { cycleToolAction, setToolAction } = mockRequire.reRequire("oxalis/model/actions/ui_actions");
@@ -27,6 +35,7 @@ const allTools = [
   EraseTool,
   FillCellTool,
   PickCellTool,
+  ProofreadTool,
 ];
 const spies = allTools.map((tool) => sinon.spy(tool, "onToolDeselected"));
 test.beforeEach(() => {
@@ -67,6 +76,8 @@ test.serial(
     cycleTool();
     t.true(BoundingBoxTool.onToolDeselected.calledOnce);
     cycleTool();
+    t.true(ProofreadTool.onToolDeselected.calledOnce);
+    cycleTool();
     t.true(MoveTool.onToolDeselected.calledTwice);
   },
 );
@@ -101,8 +112,10 @@ test.serial("Selecting another tool should trigger a deselection of the previous
   t.true(FillCellTool.onToolDeselected.calledOnce);
   cycleTool(AnnotationToolEnum.BOUNDING_BOX);
   t.true(PickCellTool.onToolDeselected.calledOnce);
-  cycleTool(AnnotationToolEnum.MOVE);
+  cycleTool(AnnotationToolEnum.PROOFREAD);
   t.true(BoundingBoxTool.onToolDeselected.calledOnce);
+  cycleTool(AnnotationToolEnum.MOVE);
+  t.true(ProofreadTool.onToolDeselected.calledOnce);
   cycleTool(AnnotationToolEnum.SKELETON);
   t.true(MoveTool.onToolDeselected.calledTwice);
 });
diff --git a/frontend/javascripts/test/sagas/save_saga.spec.ts b/frontend/javascripts/test/sagas/save_saga.spec.ts
index ce804785268..1f5e593a687 100644
--- a/frontend/javascripts/test/sagas/save_saga.spec.ts
+++ b/frontend/javascripts/test/sagas/save_saga.spec.ts
@@ -19,7 +19,7 @@ const UpdateActions = mockRequire.reRequire("oxalis/model/sagas/update_actions")
 const SaveActions = mockRequire.reRequire("oxalis/model/actions/save_actions");
 const { take, call, put } = mockRequire.reRequire("redux-saga/effects");
 const {
-  pushTracingTypeAsync,
+  pushSaveQueueAsync,
   sendRequestToServer,
   toggleErrorHighlighting,
   addVersionNumbers,
@@ -79,7 +79,7 @@ test("SaveSaga should compact multiple updateTracing update actions", (t) => {
 test("SaveSaga should send update actions", (t) => {
   const updateActions = [UpdateActions.createEdge(1, 0, 1), UpdateActions.createEdge(1, 1, 2)];
   const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP);
-  const saga = pushTracingTypeAsync(TRACING_TYPE, tracingId);
+  const saga = pushSaveQueueAsync(TRACING_TYPE, tracingId);
   expectValueDeepEqual(t, saga.next(), take(INIT_ACTION));
   saga.next(); // setLastSaveTimestampAction
 
@@ -194,7 +194,7 @@ test("SaveSaga should escalate on permanent client error update actions", (t) =>
 test("SaveSaga should send update actions right away and try to reach a state where all updates are saved", (t) => {
   const updateActions = [UpdateActions.createEdge(1, 0, 1), UpdateActions.createEdge(1, 1, 2)];
   const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP);
-  const saga = pushTracingTypeAsync(TRACING_TYPE, tracingId);
+  const saga = pushSaveQueueAsync(TRACING_TYPE, tracingId);
   expectValueDeepEqual(t, saga.next(), take(INIT_ACTION));
   saga.next();
   saga.next(); // select state
@@ -217,7 +217,7 @@ test("SaveSaga should send update actions right away and try to reach a state wh
 test("SaveSaga should not try to reach state with all actions being saved when saving is triggered by a timeout", (t) => {
   const updateActions = [UpdateActions.createEdge(1, 0, 1), UpdateActions.createEdge(1, 1, 2)];
   const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP);
-  const saga = pushTracingTypeAsync(TRACING_TYPE, tracingId);
+  const saga = pushSaveQueueAsync(TRACING_TYPE, tracingId);
   expectValueDeepEqual(t, saga.next(), take(INIT_ACTION));
   saga.next();
   saga.next(); // select state
diff --git a/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts b/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts
index 6a763c8005d..5cba4a0f111 100644
--- a/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts
+++ b/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts
@@ -112,7 +112,7 @@ test("SkeletonTracingSaga shouldn't do anything if unchanged (saga test)", (t) =
   const saga = saveTracingTypeAsync(
     SkeletonTracingActions.initializeSkeletonTracingAction(skeletonTracing),
   );
-  saga.next(); // forking pushTracingTypeAsync
+  saga.next(); // forking pushSaveQueueAsync
 
   saga.next();
   saga.next(initialState.tracing.skeleton);
@@ -132,7 +132,7 @@ test("SkeletonTracingSaga should do something if changed (saga test)", (t) => {
   const saga = saveTracingTypeAsync(
     SkeletonTracingActions.initializeSkeletonTracingAction(skeletonTracing),
   );
-  saga.next(); // forking pushTracingTypeAsync
+  saga.next(); // forking pushSaveQueueAsync
 
   saga.next();
   saga.next(initialState.tracing.skeleton);
diff --git a/frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.spec.ts b/frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.spec.ts
index 2bf3573bfaa..c39e8bad6d7 100644
--- a/frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.spec.ts
+++ b/frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.spec.ts
@@ -119,7 +119,7 @@ test("VolumeTracingSaga shouldn't do anything if unchanged (saga test)", (t) =>
   const saga = saveTracingTypeAsync(
     VolumeTracingActions.initializeVolumeTracingAction(serverVolumeTracing),
   );
-  saga.next(); // forking pushTracingTypeAsync
+  saga.next(); // forking pushSaveQueueAsync
 
   saga.next();
   saga.next(initialState.tracing.volumes[0]);
@@ -139,7 +139,7 @@ test("VolumeTracingSaga should do something if changed (saga test)", (t) => {
   const saga = saveTracingTypeAsync(
     VolumeTracingActions.initializeVolumeTracingAction(serverVolumeTracing),
   );
-  saga.next(); // forking pushTracingTypeAsync
+  saga.next(); // forking pushSaveQueueAsync
 
   saga.next();
   saga.next(initialState.tracing.volumes[0]);

From b8e664b8ce3178220d5a3885482f6990217f591c Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Wed, 18 May 2022 16:00:03 +0200
Subject: [PATCH 045/122] allow to select nodes when proofread tool is active

---
 .../oxalis/controller/combinations/tool_controls.ts    | 10 +++++++---
 .../oxalis/controller/viewmodes/plane_controller.tsx   |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts b/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts
index 940d9d6be7a..4e401b81ce7 100644
--- a/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts
+++ b/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts
@@ -618,10 +618,14 @@ export class BoundingBoxTool {
   }
 }
 export class ProofreadTool {
-  static getPlaneMouseControls(_planeId: OrthoView): any {
+  static getPlaneMouseControls(_planeId: OrthoView, planeView: PlaneView): any {
     return {
-      leftClick: (pos: Point2, _plane: OrthoView, _event: MouseEvent) => {
-        handleAgglomerateSkeletonAtClick(pos);
+      leftClick: (pos: Point2, plane: OrthoView, _event: MouseEvent, isTouch: boolean) => {
+        const didSelectNode = SkeletonHandlers.handleSelectNode(planeView, pos, plane, isTouch);
+
+        if (!didSelectNode) {
+          handleAgglomerateSkeletonAtClick(pos);
+        }
       },
     };
   }
diff --git a/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx b/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx
index b7e5b646d34..846210805ef 100644
--- a/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx
+++ b/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx
@@ -302,7 +302,7 @@ class PlaneController extends React.PureComponent<Props> {
       this.planeView,
       this.props.showContextMenuAt,
     );
-    const proofreadControls = ProofreadTool.getPlaneMouseControls(planeId);
+    const proofreadControls = ProofreadTool.getPlaneMouseControls(planeId, this.planeView);
 
     const allControlKeys = _.union(
       Object.keys(moveControls),

From 4d1faac31b6f105f5142d6e48c0b1b9acc76ac9c Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Thu, 19 May 2022 15:29:48 +0200
Subject: [PATCH 046/122] cache materialized editable mappings

---
 .../EditableMappingService.scala              | 47 +++++++++++++++++--
 1 file changed, 44 insertions(+), 3 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 6f456d8f1d9..4efcd737cbf 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -2,6 +2,8 @@ package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
 
 import java.util.UUID
 
+import akka.http.caching.LfuCache
+import akka.http.caching.scaladsl.{Cache, CachingSettings}
 import com.google.inject.Inject
 import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
@@ -24,9 +26,10 @@ import com.scalableminds.webknossos.tracingstore.tracings.{
 }
 import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common.Box.tryo
-import net.liftweb.common.{Empty, Full}
+import net.liftweb.common.{Box, Empty, Full}
 import play.api.libs.json.{JsObject, JsValue, Json}
 
+import scala.concurrent.duration._
 import scala.collection.mutable
 import scala.concurrent.ExecutionContext
 
@@ -41,6 +44,21 @@ class EditableMappingService @Inject()(
 
   private def generateId: String = UUID.randomUUID.toString
 
+  private lazy val materializedEditableMappingCache: Cache[String, Box[EditableMapping]] = {
+    val maxEntries = 10
+    val defaultCachingSettings = CachingSettings("")
+    val lfuCacheSettings =
+      defaultCachingSettings.lfuCacheSettings
+        .withInitialCapacity(maxEntries)
+        .withMaxCapacity(maxEntries)
+        .withTimeToLive(2 hours)
+        .withTimeToIdle(1 hour)
+    val cachingSettings =
+      defaultCachingSettings.withLfuCacheSettings(lfuCacheSettings)
+    val lfuCache: Cache[String, Box[EditableMapping]] = LfuCache(cachingSettings)
+    lfuCache
+  }
+
   def newestMaterializableVersion(editableMappingId: String): Fox[Long] =
     tracingDataStore.editableMappingUpdates.getVersion(editableMappingId,
                                                        mayBeEmpty = Some(true),
@@ -95,9 +113,32 @@ class EditableMappingService @Inject()(
                   userToken: Option[String],
                   version: Option[Long] = None): Fox[EditableMapping] =
     for {
-      closestMaterializedVersion: VersionedKeyValuePair[EditableMapping] <- tracingDataStore.editableMappings
-        .get(editableMappingId, version)(fromJson[EditableMapping])
       desiredVersion <- findDesiredOrNewestPossibleVersion(editableMappingId, version)
+      materialized <- getWithCache(editableMappingId, remoteFallbackLayer, userToken, desiredVersion)
+    } yield materialized
+
+  private def getWithCache(editableMappingId: String,
+                           remoteFallbackLayer: RemoteFallbackLayer,
+                           userToken: Option[String],
+                           desiredVersion: Long,
+  ): Fox[EditableMapping] = {
+    val key = f"$editableMappingId---$desiredVersion"
+    for {
+      materializedBox <- materializedEditableMappingCache.getOrLoad(
+        key,
+        _ => getVersioned(editableMappingId, remoteFallbackLayer, userToken, desiredVersion).futureBox)
+      materialized <- materializedBox.toFox
+    } yield materialized
+  }
+
+  private def getVersioned(editableMappingId: String,
+                           remoteFallbackLayer: RemoteFallbackLayer,
+                           userToken: Option[String],
+                           desiredVersion: Long,
+  ): Fox[EditableMapping] =
+    for {
+      closestMaterializedVersion: VersionedKeyValuePair[EditableMapping] <- tracingDataStore.editableMappings
+        .get(editableMappingId, Some(desiredVersion))(fromJson[EditableMapping])
       _ = logger.info(
         f"Loading mapping version $desiredVersion, closest materialized is version ${closestMaterializedVersion.version} (${closestMaterializedVersion.value})")
       materialized <- applyPendingUpdates(

From 7d5fbeec82c7b7bb09bce5f188bdbff9eb978ffb Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Thu, 19 May 2022 15:32:41 +0200
Subject: [PATCH 047/122] speed up ci

---
 .circleci/not-on-master.sh | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.circleci/not-on-master.sh b/.circleci/not-on-master.sh
index 581393ebead..c27783316d7 100755
--- a/.circleci/not-on-master.sh
+++ b/.circleci/not-on-master.sh
@@ -1,8 +1,4 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
-if [ "${CIRCLE_BRANCH}" == "master" ]; then
-  echo "Skipping this step on master..."
-else
-  exec "$@"
-fi
+echo "Skipping this step, this is a prototype!..."

From 370b6ad573495d0c6f20696b73f074cd0d96d45d Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Thu, 19 May 2022 15:48:35 +0200
Subject: [PATCH 048/122] [WIP] enable ad-hoc meshes for editable mapping
 annotations

---
 .../controllers/VolumeTracingController.scala |  18 +--
 .../EditableMappingsIsosurfaceService.scala   | 149 ++++++++++++++++++
 2 files changed, 157 insertions(+), 10 deletions(-)
 create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingsIsosurfaceService.scala

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index cf3c9a6d6c5..f7ff26d6187 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -13,15 +13,8 @@ import com.scalableminds.webknossos.datastore.models.{WebKnossosDataRequest, Web
 import com.scalableminds.webknossos.datastore.services.UserAccessRequest
 import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService
 import com.scalableminds.webknossos.tracingstore.tracings.UpdateActionGroup
-import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{
-  EditableMappingService,
-  EditableMappingUpdateActionGroup
-}
-import com.scalableminds.webknossos.tracingstore.tracings.volume.{
-  ResolutionRestrictions,
-  UpdateMappingNameAction,
-  VolumeTracingService
-}
+import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{EditableMappingService, EditableMappingUpdateActionGroup, EditableMappingsIsosurfaceService}
+import com.scalableminds.webknossos.tracingstore.tracings.volume.{ResolutionRestrictions, UpdateMappingNameAction, VolumeTracingService}
 import com.scalableminds.webknossos.tracingstore.{TSRemoteWebKnossosClient, TracingStoreAccessTokenService}
 import play.api.i18n.Messages
 import play.api.libs.Files.TemporaryFile
@@ -35,6 +28,7 @@ import scala.concurrent.ExecutionContext
 class VolumeTracingController @Inject()(val tracingService: VolumeTracingService,
                                         val remoteWebKnossosClient: TSRemoteWebKnossosClient,
                                         editableMappingService: EditableMappingService,
+                                        editableMappingIsosurfaceService: EditableMappingsIsosurfaceService,
                                         val accessTokenService: TracingStoreAccessTokenService,
                                         val slackNotificationService: TSSlackNotificationService)(
     implicit val ec: ExecutionContext,
@@ -206,7 +200,11 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
           // The client expects the isosurface as a flat float-array. Three consecutive floats form a 3D point, three
           // consecutive 3D points (i.e., nine floats) form a triangle.
           // There are no shared vertices between triangles.
-          (vertices, neighbors) <- tracingService.createIsosurface(tracingId, request.body)
+          tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound")
+          hasEditableMapping <- Fox.runOptional(tracing.mappingName)(editableMappingService.exists)
+          (vertices, neighbors) <- if (hasEditableMapping.getOrElse(false))
+            editableMappingIsosurfaceService.createIsosurface(tracing, request.body, token)
+          else tracingService.createIsosurface(tracingId, request.body)
         } yield {
           // We need four bytes for each float
           val responseBuffer = ByteBuffer.allocate(vertices.length * 4).order(ByteOrder.LITTLE_ENDIAN)
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingsIsosurfaceService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingsIsosurfaceService.scala
new file mode 100644
index 00000000000..93baacabd87
--- /dev/null
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingsIsosurfaceService.scala
@@ -0,0 +1,149 @@
+package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
+
+import java.nio.{Buffer, ByteBuffer, ByteOrder, IntBuffer, LongBuffer, ShortBuffer}
+
+import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int}
+import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing
+import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits
+import com.scalableminds.webknossos.datastore.models.WebKnossosIsosurfaceRequest
+import com.scalableminds.webknossos.datastore.models.datasource.{DataSource, ElementClass, SegmentationLayer}
+import com.scalableminds.webknossos.datastore.models.requests.{Cuboid, DataServiceDataRequest, DataServiceMappingRequest, DataServiceRequestSettings}
+import com.scalableminds.webknossos.datastore.services.mcubes.MarchingCubes
+import com.scalableminds.webknossos.datastore.services.{DataTypeFunctors, IsosurfaceRequest}
+import javax.inject.Inject
+
+import scala.collection.mutable
+import scala.reflect.ClassTag
+
+case class EditableMappingIsosurfaceRequest(
+  tracing: VolumeTracing,
+                              cuboid: Cuboid,
+                              segmentId: Long,
+                              subsamplingStrides: Vec3Int,
+                              scale: Vec3Double,
+                              mapping: Option[String] = None,
+                              mappingType: Option[String] = None
+                            )
+
+class EditableMappingsIsosurfaceService @Inject()(editableMappingService: EditableMappingService) extends ProtoGeometryImplicits {
+  def createIsosurface(tracing: VolumeTracing, request: WebKnossosIsosurfaceRequest): Fox[(Array[Float], List[Int])] = {
+    requestIsosurface(EditableMappingIsosurfaceRequest())
+  }
+
+  def requestIsosurface(request: EditableMappingIsosurfaceRequest): Fox[(Array[Float], List[Int])] =
+    elementClassFromProto(request.tracing.elementClass) match {
+      case ElementClass.uint8 =>
+        generateIsosurfaceImpl[Byte, ByteBuffer](request,
+          DataTypeFunctors[Byte, ByteBuffer](identity, _.get(_), _.toByte))
+      case ElementClass.uint16 =>
+        generateIsosurfaceImpl[Short, ShortBuffer](
+          request,
+          DataTypeFunctors[Short, ShortBuffer](_.asShortBuffer, _.get(_), _.toShort))
+      case ElementClass.uint32 =>
+        generateIsosurfaceImpl[Int, IntBuffer](request,
+          DataTypeFunctors[Int, IntBuffer](_.asIntBuffer, _.get(_), _.toInt))
+      case ElementClass.uint64 =>
+        generateIsosurfaceImpl[Long, LongBuffer](request,
+          DataTypeFunctors[Long, LongBuffer](_.asLongBuffer, _.get(_), identity))
+    }
+
+  private def generateIsosurfaceImpl[T: ClassTag, B <: Buffer](
+                                                                request: EditableMappingIsosurfaceRequest,
+                                                                dataTypeFunctors: DataTypeFunctors[T, B]): Fox[(Array[Float], List[Int])] = {
+
+    def convertData(data: Array[Byte]): Array[T] = {
+      val srcBuffer = dataTypeFunctors.getTypedBufferFn(ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN))
+      srcBuffer.rewind()
+      val dstArray = Array.ofDim[T](srcBuffer.remaining())
+      dataTypeFunctors.copyDataFn(srcBuffer, dstArray)
+      dstArray
+    }
+
+    def subVolumeContainsSegmentId[T](data: Array[T],
+                                      dataDimensions: Vec3Int,
+                                      boundingBox: BoundingBox,
+                                      segmentId: T): Boolean = {
+      for {
+        x <- boundingBox.topLeft.x until boundingBox.bottomRight.x
+        y <- boundingBox.topLeft.y until boundingBox.bottomRight.y
+        z <- boundingBox.topLeft.z until boundingBox.bottomRight.z
+      } {
+        val voxelOffset = x + y * dataDimensions.x + z * dataDimensions.x * dataDimensions.y
+        if (data(voxelOffset) == segmentId) return true
+      }
+      false
+    }
+
+    def findNeighbors[T](data: Array[T], dataDimensions: Vec3Int, segmentId: T): List[Int] = {
+      val x = dataDimensions.x - 1
+      val y = dataDimensions.y - 1
+      val z = dataDimensions.z - 1
+      val front_xy = BoundingBox(Vec3Int(0, 0, 0), x, y, 1)
+      val front_xz = BoundingBox(Vec3Int(0, 0, 0), x, 1, z)
+      val front_yz = BoundingBox(Vec3Int(0, 0, 0), 1, y, z)
+      val back_xy = BoundingBox(Vec3Int(0, 0, z), x, y, 1)
+      val back_xz = BoundingBox(Vec3Int(0, y, 0), x, 1, z)
+      val back_yz = BoundingBox(Vec3Int(x, 0, 0), 1, y, z)
+      val surfaceBoundingBoxes = List(front_xy, front_xz, front_yz, back_xy, back_xz, back_yz)
+      surfaceBoundingBoxes.zipWithIndex.filter {
+        case (surfaceBoundingBox, index) =>
+          subVolumeContainsSegmentId(data, dataDimensions, surfaceBoundingBox, segmentId)
+      }.map {
+        case (surfaceBoundingBox, index) => index
+      }
+    }
+
+    val cuboid = request.cuboid
+    val subsamplingStrides =
+      Vec3Double(request.subsamplingStrides.x, request.subsamplingStrides.y, request.subsamplingStrides.z)
+
+    val dataRequest = DataServiceDataRequest(request.dataSource.orNull,
+      request.dataLayer,
+      request.mapping,
+      cuboid,
+      DataServiceRequestSettings.default,
+      request.subsamplingStrides)
+
+    val dataDimensions = Vec3Int(
+      math.ceil(cuboid.width / subsamplingStrides.x).toInt,
+      math.ceil(cuboid.height / subsamplingStrides.y).toInt,
+      math.ceil(cuboid.depth / subsamplingStrides.z).toInt
+    )
+
+    val offset = Vec3Double(cuboid.topLeft.x, cuboid.topLeft.y, cuboid.topLeft.z)
+    val scale = Vec3Double(cuboid.topLeft.mag) * request.scale
+    val typedSegmentId = dataTypeFunctors.fromLong(request.segmentId)
+
+    val vertexBuffer = mutable.ArrayBuffer[Vec3Double]()
+
+    for {
+      data <- binaryDataService.handleDataRequest(dataRequest)
+      typedData = convertData(data)
+      neighbors = findNeighbors(typedData, dataDimensions, typedSegmentId)
+    } yield {
+      for {
+        x <- 0 until dataDimensions.x by 32
+        y <- 0 until dataDimensions.y by 32
+        z <- 0 until dataDimensions.z by 32
+      } {
+        val boundingBox = BoundingBox(Vec3Int(x, y, z),
+          math.min(dataDimensions.x - x, 33),
+          math.min(dataDimensions.y - y, 33),
+          math.min(dataDimensions.z - z, 33))
+        if (subVolumeContainsSegmentId(typedData, dataDimensions, boundingBox, typedSegmentId)) {
+          MarchingCubes.marchingCubes[T](typedData,
+            dataDimensions,
+            boundingBox,
+            typedSegmentId,
+            subsamplingStrides,
+            offset,
+            scale,
+            vertexBuffer)
+        }
+      }
+      (vertexBuffer.flatMap(_.toList.map(_.toFloat)).toArray, neighbors)
+    }
+  }
+
+}

From 387869b275ec018b1f8445ec26833b1a0f79238f Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Thu, 19 May 2022 16:09:33 +0200
Subject: [PATCH 049/122] depend only on the cache key

---
 .../EditableMappingService.scala               | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 4efcd737cbf..ed193bd2b23 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -33,6 +33,14 @@ import scala.concurrent.duration._
 import scala.collection.mutable
 import scala.concurrent.ExecutionContext
 
+
+case class EditableMappingKey(
+                               editableMappingId: String,
+                               remoteFallbackLayer: RemoteFallbackLayer,
+                               userToken: Option[String],
+                               desiredVersion: Long
+                             )
+
 class EditableMappingService @Inject()(
     val tracingDataStore: TracingDataStore,
     remoteDatastoreClient: TSRemoteDatastoreClient
@@ -44,7 +52,7 @@ class EditableMappingService @Inject()(
 
   private def generateId: String = UUID.randomUUID.toString
 
-  private lazy val materializedEditableMappingCache: Cache[String, Box[EditableMapping]] = {
+  private lazy val materializedEditableMappingCache: Cache[EditableMappingKey, Box[EditableMapping]] = {
     val maxEntries = 10
     val defaultCachingSettings = CachingSettings("")
     val lfuCacheSettings =
@@ -55,7 +63,7 @@ class EditableMappingService @Inject()(
         .withTimeToIdle(1 hour)
     val cachingSettings =
       defaultCachingSettings.withLfuCacheSettings(lfuCacheSettings)
-    val lfuCache: Cache[String, Box[EditableMapping]] = LfuCache(cachingSettings)
+    val lfuCache: Cache[EditableMappingKey, Box[EditableMapping]] = LfuCache(cachingSettings)
     lfuCache
   }
 
@@ -117,16 +125,18 @@ class EditableMappingService @Inject()(
       materialized <- getWithCache(editableMappingId, remoteFallbackLayer, userToken, desiredVersion)
     } yield materialized
 
+
   private def getWithCache(editableMappingId: String,
                            remoteFallbackLayer: RemoteFallbackLayer,
                            userToken: Option[String],
                            desiredVersion: Long,
   ): Fox[EditableMapping] = {
-    val key = f"$editableMappingId---$desiredVersion"
+    val key = EditableMappingKey(editableMappingId, remoteFallbackLayer, userToken, desiredVersion)
+    logger.info("getWithCache")
     for {
       materializedBox <- materializedEditableMappingCache.getOrLoad(
         key,
-        _ => getVersioned(editableMappingId, remoteFallbackLayer, userToken, desiredVersion).futureBox)
+        key => getVersioned(key.editableMappingId, key.remoteFallbackLayer, key.userToken, key.desiredVersion).futureBox)
       materialized <- materializedBox.toFox
     } yield materialized
   }

From d2def9c053e6d24c154444ade20932a807cba84e Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Thu, 19 May 2022 16:31:28 +0200
Subject: [PATCH 050/122] agglomerate graph resilience for agglomerate id 0

---
 .../datastore/controllers/DataSourceController.scala   |  2 ++
 .../datastore/services/AgglomerateService.scala        | 10 ++++++----
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
index 7e950d9ca4f..6d8214d884c 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
@@ -337,9 +337,11 @@ Expects:
     accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationName)),
                                       token) {
       for {
+        _ <- Fox.successful(logger.info(s"Generating agglomerate graph..."))
         agglomerateGraph <- binaryDataServiceHolder.binaryDataService.agglomerateService.generateAgglomerateGraph(
           AgglomerateFileKey(organizationName, dataSetName, dataLayerName, mappingName),
           agglomerateId) ?~> "agglomerateGraph.failed"
+        _ <- Fox.successful(logger.info(s"Serializing agglomerate graph ${agglomerateGraph}..."))
       } yield Ok(Json.toJson(agglomerateGraph))
     }
   }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 65b4efa8222..64e1399dcf2 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -228,6 +228,8 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
       val edgesRange: Array[Long] =
         reader.uint64().readArrayBlockWithOffset("/agglomerate_to_edges_offsets", 2, agglomerateId)
 
+      logger.info(s"positionsRange: ${positionsRange(0)} to ${positionsRange(1)}, agglomerateId: ${agglomerateId}")
+
       val nodeCount = positionsRange(1) - positionsRange(0)
       val edgeCount = edgesRange(1) - edgesRange(0)
       val edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
@@ -237,13 +239,13 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
       if (edgeCount > edgeLimit) {
         throw new Exception(s"Agglomerate has too many edges ($edgeCount > $edgeLimit)")
       }
-      val segmentIds: Array[Long] =
+      val segmentIds: Array[Long] = if (nodeCount == 0L) Array[Long]() else
         reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments", nodeCount.toInt, positionsRange(0))
-      val positions: Array[Array[Long]] =
+      val positions: Array[Array[Long]] = if (nodeCount == 0L) Array[Array[Long]]() else
         reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_positions", nodeCount.toInt, 3, positionsRange(0), 0)
-      val edges: Array[Array[Long]] =
+      val edges: Array[Array[Long]] = if (edgeCount == 0L) Array[Array[Long]]() else
         reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_edges", edgeCount.toInt, 2, edgesRange(0), 0)
-      val affinities: Array[Float] =
+      val affinities: Array[Float] = if (edgeCount == 0L) Array[Float]() else
         reader.float32().readArrayBlockWithOffset("/agglomerate_to_affinities", edgeCount.toInt, edgesRange(0))
 
       AgglomerateGraph(

From dc045a20806d3aab5aaa3b979eff94a51bad8885 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Thu, 19 May 2022 16:33:21 +0200
Subject: [PATCH 051/122] fix highest lowest resolution confusion

---
 frontend/javascripts/oxalis/model/sagas/proofread_saga.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index caf47262fd1..486dbe0cd87 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -87,8 +87,8 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
 
   const resolutionInfo = getResolutionInfo(volumeTracingLayer.resolutions);
   // The mag the agglomerate skeleton corresponds to should be the finest available mag of the volume tracing layer
-  const agglomerateFileMag = resolutionInfo.getHighestResolution();
-  const agglomerateFileZoomstep = resolutionInfo.getHighestResolutionIndex();
+  const agglomerateFileMag = resolutionInfo.getLowestResolution();
+  const agglomerateFileZoomstep = resolutionInfo.getLowestResolutionIndex();
   const { sourceNodeId, targetNodeId } = action;
 
   const skeletonTracing = yield* select((state) => enforceSkeletonTracing(state.tracing));

From b07fff8bafcffdb161d5dfe6c4856a776946d03b Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Thu, 19 May 2022 17:09:15 +0200
Subject: [PATCH 052/122] delete the edge even if one graph gets all nodes

---
 .../webknossos/tracingstore/TracingStoreModule.scala   |  2 ++
 .../editablemapping/EditableMappingService.scala       | 10 +++++-----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala
index 6c0db630f64..23814782f4f 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala
@@ -5,6 +5,7 @@ import com.google.inject.AbstractModule
 import com.google.inject.name.Names
 import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService
 import com.scalableminds.webknossos.tracingstore.tracings.TracingDataStore
+import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService
 import com.scalableminds.webknossos.tracingstore.tracings.skeleton.SkeletonTracingService
 import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingService
 
@@ -19,6 +20,7 @@ class TracingStoreModule extends AbstractModule {
     bind(classOf[VolumeTracingService]).asEagerSingleton()
     bind(classOf[TracingStoreAccessTokenService]).asEagerSingleton()
     bind(classOf[TSRemoteWebKnossosClient]).asEagerSingleton()
+    bind(classOf[EditableMappingService]).asEagerSingleton()
     bind(classOf[TSSlackNotificationService]).asEagerSingleton()
   }
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index ed193bd2b23..7dd8548fdb8 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -247,15 +247,15 @@ class EditableMappingService @Inject()(
   private def splitGraph(agglomerateGraph: AgglomerateGraph,
                          segmentId1: Long,
                          segmentId2: Long): (AgglomerateGraph, AgglomerateGraph) = {
-    val edgesMinusOne = agglomerateGraph.edges.filterNot {
-      case (from, to) =>
+    val edgesAndAffinitiesMinusOne: List[((Long, Long), Float)] = agglomerateGraph.edges.zip(agglomerateGraph.affinities).filterNot {
+      case ((from, to), _) =>
         (from == segmentId1 && to == segmentId2) || (from == segmentId2 && to == segmentId1)
     }
-    val graph1Nodes: Set[Long] = computeConnectedComponent(startNode = segmentId1, edgesMinusOne)
+    val graph1Nodes: Set[Long] = computeConnectedComponent(startNode = segmentId1, edgesAndAffinitiesMinusOne.map(_._1))
     val graph1NodesWithPositions = agglomerateGraph.segments.zip(agglomerateGraph.positions).filter {
       case (seg, _) => graph1Nodes.contains(seg)
     }
-    val graph1EdgesWithAffinities = agglomerateGraph.edges.zip(agglomerateGraph.affinities).filter {
+    val graph1EdgesWithAffinities = edgesAndAffinitiesMinusOne.filter {
       case (e, _) => graph1Nodes.contains(e._1) && graph1Nodes.contains(e._2)
     }
     val graph1 = AgglomerateGraph(
@@ -269,7 +269,7 @@ class EditableMappingService @Inject()(
     val graph2NodesWithPositions = agglomerateGraph.segments.zip(agglomerateGraph.positions).filter {
       case (seg, _) => graph2Nodes.contains(seg)
     }
-    val graph2EdgesWithAffinities = agglomerateGraph.edges.zip(agglomerateGraph.affinities).filter {
+    val graph2EdgesWithAffinities = edgesAndAffinitiesMinusOne.filter {
       case (e, _) => graph2Nodes.contains(e._1) && graph2Nodes.contains(e._2)
     }
     val graph2 = AgglomerateGraph(

From 0496ae4a78b476402eb9a1ec8821bcc5738b39be Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 23 May 2022 09:43:54 +0200
Subject: [PATCH 053/122] editable mapping meshes

---
 .../models/datasource/DataLayer.scala         |  2 +-
 .../services/AgglomerateService.scala         | 26 +++--
 .../TSRemoteDatastoreClient.scala             |  3 +
 .../controllers/VolumeTracingController.scala | 14 ++-
 .../EditableMappingLayer.scala                | 68 +++++++++++++
 .../EditableMappingService.scala              | 96 ++++++++++++-------
 .../EditableMappingsIsosurfaceService.scala   |  2 +
 7 files changed, 164 insertions(+), 47 deletions(-)
 create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala
index cc0f70dfcef..a7cd02e6517 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala
@@ -10,7 +10,7 @@ import com.scalableminds.webknossos.datastore.models.datasource.LayerViewConfigu
 import play.api.libs.json._
 
 object DataFormat extends ExtendedEnumeration {
-  val wkw, zarr, tracing = Value
+  val wkw, zarr, tracing, editableMapping = Value
 }
 
 object Category extends ExtendedEnumeration {
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index 64e1399dcf2..b2fd4d73cdb 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -239,14 +239,24 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
       if (edgeCount > edgeLimit) {
         throw new Exception(s"Agglomerate has too many edges ($edgeCount > $edgeLimit)")
       }
-      val segmentIds: Array[Long] = if (nodeCount == 0L) Array[Long]() else
-        reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments", nodeCount.toInt, positionsRange(0))
-      val positions: Array[Array[Long]] = if (nodeCount == 0L) Array[Array[Long]]() else
-        reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_positions", nodeCount.toInt, 3, positionsRange(0), 0)
-      val edges: Array[Array[Long]] = if (edgeCount == 0L) Array[Array[Long]]() else
-        reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_edges", edgeCount.toInt, 2, edgesRange(0), 0)
-      val affinities: Array[Float] = if (edgeCount == 0L) Array[Float]() else
-        reader.float32().readArrayBlockWithOffset("/agglomerate_to_affinities", edgeCount.toInt, edgesRange(0))
+      val segmentIds: Array[Long] =
+        if (nodeCount == 0L) Array[Long]()
+        else
+          reader.uint64().readArrayBlockWithOffset("/agglomerate_to_segments", nodeCount.toInt, positionsRange(0))
+      val positions: Array[Array[Long]] =
+        if (nodeCount == 0L) Array[Array[Long]]()
+        else
+          reader
+            .uint64()
+            .readMatrixBlockWithOffset("/agglomerate_to_positions", nodeCount.toInt, 3, positionsRange(0), 0)
+      val edges: Array[Array[Long]] =
+        if (edgeCount == 0L) Array[Array[Long]]()
+        else
+          reader.uint64().readMatrixBlockWithOffset("/agglomerate_to_edges", edgeCount.toInt, 2, edgesRange(0), 0)
+      val affinities: Array[Float] =
+        if (edgeCount == 0L) Array[Float]()
+        else
+          reader.float32().readArrayBlockWithOffset("/agglomerate_to_affinities", edgeCount.toInt, edgesRange(0))
 
       AgglomerateGraph(
         segments = segmentIds.toList,
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
index a41c3082c51..e5b13837f24 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
@@ -37,6 +37,7 @@ class TSRemoteDatastoreClient @Inject()(
     for {
       response <- rpc(s"${remoteLayerUri(remoteFallbackLayer)}/data")
         .addQueryStringOptional("token", userToken)
+        .silent
         .post(dataRequests)
       _ <- bool2Fox(Status.isSuccessful(response.status))
       bytes = response.bodyAsBytes.toArray
@@ -56,6 +57,7 @@ class TSRemoteDatastoreClient @Inject()(
       .addQueryString("height" -> "1")
       .addQueryString("depth" -> "1")
       .addQueryString("mag" -> mag.toMagLiteral())
+      .silent
       .getWithBytesResponse
 
   def getAgglomerateIdsForSegmentIds(remoteFallbackLayer: RemoteFallbackLayer,
@@ -64,6 +66,7 @@ class TSRemoteDatastoreClient @Inject()(
                                      userToken: Option[String]): Fox[List[Long]] =
     rpc(s"${remoteLayerUri(remoteFallbackLayer)}/agglomerates/$mappingName/agglomeratesForSegments")
       .addQueryStringOptional("token", userToken)
+      .silent
       .postJsonWithJsonResponse[List[Long], List[Long]](segmentIdsOrdered)
 
   private def remoteLayerUri(remoteLayer: RemoteFallbackLayer): String =
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index f7ff26d6187..a0039915baa 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -13,8 +13,15 @@ import com.scalableminds.webknossos.datastore.models.{WebKnossosDataRequest, Web
 import com.scalableminds.webknossos.datastore.services.UserAccessRequest
 import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService
 import com.scalableminds.webknossos.tracingstore.tracings.UpdateActionGroup
-import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{EditableMappingService, EditableMappingUpdateActionGroup, EditableMappingsIsosurfaceService}
-import com.scalableminds.webknossos.tracingstore.tracings.volume.{ResolutionRestrictions, UpdateMappingNameAction, VolumeTracingService}
+import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{
+  EditableMappingService,
+  EditableMappingUpdateActionGroup
+}
+import com.scalableminds.webknossos.tracingstore.tracings.volume.{
+  ResolutionRestrictions,
+  UpdateMappingNameAction,
+  VolumeTracingService
+}
 import com.scalableminds.webknossos.tracingstore.{TSRemoteWebKnossosClient, TracingStoreAccessTokenService}
 import play.api.i18n.Messages
 import play.api.libs.Files.TemporaryFile
@@ -28,7 +35,6 @@ import scala.concurrent.ExecutionContext
 class VolumeTracingController @Inject()(val tracingService: VolumeTracingService,
                                         val remoteWebKnossosClient: TSRemoteWebKnossosClient,
                                         editableMappingService: EditableMappingService,
-                                        editableMappingIsosurfaceService: EditableMappingsIsosurfaceService,
                                         val accessTokenService: TracingStoreAccessTokenService,
                                         val slackNotificationService: TSSlackNotificationService)(
     implicit val ec: ExecutionContext,
@@ -203,7 +209,7 @@ class VolumeTracingController @Inject()(val tracingService: VolumeTracingService
           tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound")
           hasEditableMapping <- Fox.runOptional(tracing.mappingName)(editableMappingService.exists)
           (vertices, neighbors) <- if (hasEditableMapping.getOrElse(false))
-            editableMappingIsosurfaceService.createIsosurface(tracing, request.body, token)
+            editableMappingService.createIsosurface(tracing, request.body, token)
           else tracingService.createIsosurface(tracingId, request.body)
         } yield {
           // We need four bytes for each float
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
new file mode 100644
index 00000000000..6914e05725b
--- /dev/null
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
@@ -0,0 +1,68 @@
+package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
+
+import com.scalableminds.util.geometry.{BoundingBox, Vec3Int}
+import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing
+import com.scalableminds.webknossos.datastore.dataformats.BucketProvider
+import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits
+import com.scalableminds.webknossos.datastore.models.{BucketPosition, WebKnossosDataRequest}
+import com.scalableminds.webknossos.datastore.models.datasource.LayerViewConfiguration.LayerViewConfiguration
+import com.scalableminds.webknossos.datastore.models.datasource.{DataFormat, DataLayer, ElementClass, SegmentationLayer}
+import com.scalableminds.webknossos.datastore.models.requests.DataReadInstruction
+import com.scalableminds.webknossos.datastore.storage.DataCubeCache
+
+import scala.concurrent.ExecutionContext
+
+class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketProvider with ProtoGeometryImplicits {
+  override def load(readInstruction: DataReadInstruction, cache: DataCubeCache)(
+      implicit ec: ExecutionContext): Fox[Array[Byte]] = {
+    val bucket: BucketPosition = readInstruction.bucket
+
+    for {
+      editableMappingId <- Fox.successful(layer.name)
+      remoteFallbackLayer <- layer.editableMappingService.remoteFallbackLayer(layer.tracing)
+      editableMapping <- layer.editableMappingService.get(editableMappingId, remoteFallbackLayer, layer.token)
+      dataRequest: WebKnossosDataRequest = WebKnossosDataRequest(
+        position = Vec3Int(bucket.globalX, bucket.globalY, bucket.globalZ),
+        mag = bucket.mag,
+        cubeSize = layer.lengthOfUnderlyingCubes(bucket.mag),
+        fourBit = None,
+        applyAgglomerate = None,
+        version = None
+      )
+      dataRequestCollection = List(dataRequest)
+      (unmappedData, indices) <- layer.editableMappingService.getUnmappedDataFromDatastore(remoteFallbackLayer,
+                                                                                           dataRequestCollection,
+                                                                                           layer.token)
+      _ <- bool2Fox(indices.isEmpty)
+      segmentIds <- layer.editableMappingService.collectSegmentIds(unmappedData, layer.tracing.elementClass)
+      relevantMapping <- layer.editableMappingService.generateCombinedMappingSubset(segmentIds,
+                                                                                    editableMapping,
+                                                                                    remoteFallbackLayer,
+                                                                                    layer.token)
+      mappedData <- layer.editableMappingService.mapData(unmappedData, relevantMapping, layer.elementClass)
+    } yield mappedData
+  }
+}
+
+case class EditableMappingLayer(name: String,
+                                boundingBox: BoundingBox,
+                                resolutions: List[Vec3Int],
+                                largestSegmentId: Long,
+                                elementClass: ElementClass.Value,
+                                token: Option[String],
+                                tracing: VolumeTracing,
+                                editableMappingService: EditableMappingService)
+    extends SegmentationLayer {
+  override def dataFormat: DataFormat.Value = DataFormat.editableMapping
+
+  override def lengthOfUnderlyingCubes(resolution: Vec3Int): Int = DataLayer.bucketLength
+
+  override def bucketProvider: BucketProvider = new EditableMappingBucketProvider(layer = this)
+
+  override def mappings: Option[Set[String]] = None
+
+  override def defaultViewConfiguration: Option[LayerViewConfiguration] = None
+
+  override def adminViewConfiguration: Option[LayerViewConfiguration] = None
+}
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 7dd8548fdb8..c0fce49ab88 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -12,12 +12,8 @@ import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing
 import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing.ElementClass
 import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, ProtoGeometryImplicits, SkeletonTracingDefaults}
 import com.scalableminds.webknossos.datastore.models.DataRequestCollection.DataRequestCollection
-import com.scalableminds.webknossos.datastore.models.{
-  AgglomerateGraph,
-  UnsignedInteger,
-  UnsignedIntegerArray,
-  WebKnossosDataRequest
-}
+import com.scalableminds.webknossos.datastore.models._
+import com.scalableminds.webknossos.datastore.services.{IsosurfaceRequest, IsosurfaceService, IsosurfaceServiceHolder}
 import com.scalableminds.webknossos.tracingstore.TSRemoteDatastoreClient
 import com.scalableminds.webknossos.tracingstore.tracings.{
   KeyValueStoreImplicits,
@@ -29,20 +25,20 @@ import net.liftweb.common.Box.tryo
 import net.liftweb.common.{Box, Empty, Full}
 import play.api.libs.json.{JsObject, JsValue, Json}
 
-import scala.concurrent.duration._
 import scala.collection.mutable
 import scala.concurrent.ExecutionContext
-
+import scala.concurrent.duration._
 
 case class EditableMappingKey(
-                               editableMappingId: String,
-                               remoteFallbackLayer: RemoteFallbackLayer,
-                               userToken: Option[String],
-                               desiredVersion: Long
-                             )
+    editableMappingId: String,
+    remoteFallbackLayer: RemoteFallbackLayer,
+    userToken: Option[String],
+    desiredVersion: Long
+)
 
 class EditableMappingService @Inject()(
     val tracingDataStore: TracingDataStore,
+    val isosurfaceServiceHolder: IsosurfaceServiceHolder,
     remoteDatastoreClient: TSRemoteDatastoreClient
 )(implicit ec: ExecutionContext)
     extends KeyValueStoreImplicits
@@ -52,6 +48,8 @@ class EditableMappingService @Inject()(
 
   private def generateId: String = UUID.randomUUID.toString
 
+  val isosurfaceService: IsosurfaceService = isosurfaceServiceHolder.tracingStoreIsosurfaceService
+
   private lazy val materializedEditableMappingCache: Cache[EditableMappingKey, Box[EditableMapping]] = {
     val maxEntries = 10
     val defaultCachingSettings = CachingSettings("")
@@ -116,16 +114,15 @@ class EditableMappingService @Inject()(
     } yield Json.toJson(updateActionGroupsJs)
   }
 
-  private def get(editableMappingId: String,
-                  remoteFallbackLayer: RemoteFallbackLayer,
-                  userToken: Option[String],
-                  version: Option[Long] = None): Fox[EditableMapping] =
+  def get(editableMappingId: String,
+          remoteFallbackLayer: RemoteFallbackLayer,
+          userToken: Option[String],
+          version: Option[Long] = None): Fox[EditableMapping] =
     for {
       desiredVersion <- findDesiredOrNewestPossibleVersion(editableMappingId, version)
       materialized <- getWithCache(editableMappingId, remoteFallbackLayer, userToken, desiredVersion)
     } yield materialized
 
-
   private def getWithCache(editableMappingId: String,
                            remoteFallbackLayer: RemoteFallbackLayer,
                            userToken: Option[String],
@@ -136,7 +133,8 @@ class EditableMappingService @Inject()(
     for {
       materializedBox <- materializedEditableMappingCache.getOrLoad(
         key,
-        key => getVersioned(key.editableMappingId, key.remoteFallbackLayer, key.userToken, key.desiredVersion).futureBox)
+        key =>
+          getVersioned(key.editableMappingId, key.remoteFallbackLayer, key.userToken, key.desiredVersion).futureBox)
       materialized <- materializedBox.toFox
     } yield materialized
   }
@@ -247,10 +245,11 @@ class EditableMappingService @Inject()(
   private def splitGraph(agglomerateGraph: AgglomerateGraph,
                          segmentId1: Long,
                          segmentId2: Long): (AgglomerateGraph, AgglomerateGraph) = {
-    val edgesAndAffinitiesMinusOne: List[((Long, Long), Float)] = agglomerateGraph.edges.zip(agglomerateGraph.affinities).filterNot {
-      case ((from, to), _) =>
-        (from == segmentId1 && to == segmentId2) || (from == segmentId2 && to == segmentId1)
-    }
+    val edgesAndAffinitiesMinusOne: List[((Long, Long), Float)] =
+      agglomerateGraph.edges.zip(agglomerateGraph.affinities).filterNot {
+        case ((from, to), _) =>
+          (from == segmentId1 && to == segmentId2) || (from == segmentId2 && to == segmentId1)
+      }
     val graph1Nodes: Set[Long] = computeConnectedComponent(startNode = segmentId1, edgesAndAffinitiesMinusOne.map(_._1))
     val graph1NodesWithPositions = agglomerateGraph.segments.zip(agglomerateGraph.positions).filter {
       case (seg, _) => graph1Nodes.contains(seg)
@@ -401,10 +400,10 @@ class EditableMappingService @Inject()(
       mappedData <- mapData(unmappedData, relevantMapping, tracing.elementClass)
     } yield (mappedData, indices)
 
-  private def generateCombinedMappingSubset(segmentIds: Set[Long],
-                                            editableMapping: EditableMapping,
-                                            remoteFallbackLayer: RemoteFallbackLayer,
-                                            userToken: Option[String]): Fox[Map[Long, Long]] = {
+  def generateCombinedMappingSubset(segmentIds: Set[Long],
+                                    editableMapping: EditableMapping,
+                                    remoteFallbackLayer: RemoteFallbackLayer,
+                                    userToken: Option[String]): Fox[Map[Long, Long]] = {
     val segmentIdsInEditableMapping: Set[Long] = segmentIds.intersect(editableMapping.segmentToAgglomerate.keySet)
     val segmentIdsInBaseMapping: Set[Long] = segmentIds.diff(segmentIdsInEditableMapping)
     val editableMappingSubset =
@@ -482,9 +481,9 @@ class EditableMappingService @Inject()(
     } yield segmentIdsOrdered.zip(agglomerateIdsOrdered).toMap
   }
 
-  private def getUnmappedDataFromDatastore(remoteFallbackLayer: RemoteFallbackLayer,
-                                           dataRequests: DataRequestCollection,
-                                           userToken: Option[String]): Fox[(Array[Byte], List[Int])] =
+  def getUnmappedDataFromDatastore(remoteFallbackLayer: RemoteFallbackLayer,
+                                   dataRequests: DataRequestCollection,
+                                   userToken: Option[String]): Fox[(Array[Byte], List[Int])] =
     for {
       dataRequestsTyped <- Fox.serialCombined(dataRequests) {
         case r: WebKnossosDataRequest => Fox.successful(r.copy(applyAgglomerate = None))
@@ -493,7 +492,7 @@ class EditableMappingService @Inject()(
       (data, indices) <- remoteDatastoreClient.getData(remoteFallbackLayer, dataRequestsTyped, userToken)
     } yield (data, indices)
 
-  private def collectSegmentIds(data: Array[Byte], elementClass: ElementClass): Fox[Set[Long]] =
+  def collectSegmentIds(data: Array[Byte], elementClass: ElementClass): Fox[Set[Long]] =
     for {
       dataAsLongs <- bytesToLongs(data, elementClass)
     } yield dataAsLongs.toSet
@@ -504,9 +503,9 @@ class EditableMappingService @Inject()(
       organizationName <- tracing.organizationName.toFox ?~> "This feature is only implemented for volume annotations with an explicit organization name tag, not for legacy volume annotations."
     } yield RemoteFallbackLayer(organizationName, tracing.dataSetName, layerName, tracing.elementClass)
 
-  private def mapData(unmappedData: Array[Byte],
-                      relevantMapping: Map[Long, Long],
-                      elementClass: ElementClass): Fox[Array[Byte]] =
+  def mapData(unmappedData: Array[Byte],
+              relevantMapping: Map[Long, Long],
+              elementClass: ElementClass): Fox[Array[Byte]] =
     for {
       unmappedDataLongs <- bytesToLongs(unmappedData, elementClass)
       mappedDataLongs = unmappedDataLongs.map(relevantMapping)
@@ -526,4 +525,33 @@ class EditableMappingService @Inject()(
       bytes = UnsignedIntegerArray.toByteArray(unsignedIntArray, elementClass)
     } yield bytes
 
+  def createIsosurface(tracing: VolumeTracing,
+                       request: WebKnossosIsosurfaceRequest,
+                       token: Option[String]): Fox[(Array[Float], List[Int])] =
+    for {
+      mappingName <- tracing.mappingName.toFox
+      segmentationLayer = EditableMappingLayer(
+        mappingName,
+        tracing.boundingBox,
+        resolutions = tracing.resolutions.map(vec3IntFromProto).toList,
+        largestSegmentId = 0L,
+        elementClass = tracing.elementClass,
+        token,
+        tracing = tracing,
+        editableMappingService = this
+      )
+
+      isosurfaceRequest = IsosurfaceRequest(
+        None,
+        segmentationLayer,
+        request.cuboid(segmentationLayer),
+        request.segmentId,
+        request.subsamplingStrides,
+        request.scale,
+        mapping = None,
+        mappingType = None
+      )
+      result <- isosurfaceService.requestIsosurfaceViaActor(isosurfaceRequest)
+    } yield result
+
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingsIsosurfaceService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingsIsosurfaceService.scala
index 93baacabd87..a75d1b8e699 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingsIsosurfaceService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingsIsosurfaceService.scala
@@ -1,3 +1,4 @@
+/*
 package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
 
 import java.nio.{Buffer, ByteBuffer, ByteOrder, IntBuffer, LongBuffer, ShortBuffer}
@@ -147,3 +148,4 @@ class EditableMappingsIsosurfaceService @Inject()(editableMappingService: Editab
   }
 
 }
+ */

From 9133312d4331cf4c7ff4d7af8b280ee6422eec5c Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 23 May 2022 11:02:02 +0200
Subject: [PATCH 054/122] make isosurface service not strictly depend on
 agglomerate service

---
 .../webknossos/datastore/services/IsosurfaceService.scala     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
index 1aa3b02f5a9..5bf6c6b259d 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
@@ -56,7 +56,7 @@ class IsosurfaceService(binaryDataService: BinaryDataService,
                         isosurfaceActorPoolSize: Int)(implicit ec: ExecutionContext)
     extends FoxImplicits {
 
-  private val agglomerateService: AgglomerateService = binaryDataService.agglomerateService
+  private val agglomerateService: Option[AgglomerateService] = Option(binaryDataService.agglomerateService)
 
   implicit val timeout: Timeout = Timeout(isosurfaceTimeout)
 
@@ -117,7 +117,7 @@ class IsosurfaceService(binaryDataService: BinaryDataService,
                 DataServiceRequestSettings(halfByte = false, request.mapping, None),
                 request.subsamplingStrides
               )
-              agglomerateService.applyAgglomerate(dataRequest)(data)
+              agglomerateService.get.applyAgglomerate(dataRequest)(data)
             case _ =>
               data
           }

From 7567a137d2274d361454b69f0c02f0031085115c Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 23 May 2022 11:33:48 +0200
Subject: [PATCH 055/122] guard for agglomerate service being null

---
 .../webknossos/datastore/services/IsosurfaceService.scala    | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
index 5bf6c6b259d..c7d83a59182 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
@@ -56,7 +56,10 @@ class IsosurfaceService(binaryDataService: BinaryDataService,
                         isosurfaceActorPoolSize: Int)(implicit ec: ExecutionContext)
     extends FoxImplicits {
 
-  private val agglomerateService: Option[AgglomerateService] = Option(binaryDataService.agglomerateService)
+  private val agglomerateService
+    : Option[AgglomerateService] = try { Some(binaryDataService.agglomerateService) } catch {
+    case _: NullPointerException => None
+  }
 
   implicit val timeout: Timeout = Timeout(isosurfaceTimeout)
 

From 182641e5afd6c1beec95e3970ca47c4b34c965ba Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Mon, 23 May 2022 13:16:40 +0200
Subject: [PATCH 056/122] use tracingstore for ad-hoc meshes

---
 frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
index a22438256a0..b4911487199 100644
--- a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
@@ -366,8 +366,8 @@ function* maybeLoadIsosurface(
   }`;
   const tracingStoreUrl = `${tracingStoreHost}/tracings/volume/${layer.name}`;
   const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
-  // Fetch from datastore if no volumetracing exists or if the tracing has a fallback layer.
-  const useDataStore = volumeTracing == null || volumeTracing.fallbackLayer != null;
+  // Fetch from datastore if no volumetracing exists
+  const useDataStore = volumeTracing == null;
   const mag = resolutionInfo.getResolutionByIndexOrThrow(zoomStep);
 
   if (isInitialRequest) {

From a123f415607edd12877557f8bdd3e2cacb12c744 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 23 May 2022 14:14:03 +0200
Subject: [PATCH 057/122] fix mesh positions

---
 .../editablemapping/EditableMappingLayer.scala  |  3 +--
 .../EditableMappingService.scala                | 17 +++++++++--------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
index 6914e05725b..6d633fc9871 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
@@ -17,13 +17,12 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP
   override def load(readInstruction: DataReadInstruction, cache: DataCubeCache)(
       implicit ec: ExecutionContext): Fox[Array[Byte]] = {
     val bucket: BucketPosition = readInstruction.bucket
-
     for {
       editableMappingId <- Fox.successful(layer.name)
       remoteFallbackLayer <- layer.editableMappingService.remoteFallbackLayer(layer.tracing)
       editableMapping <- layer.editableMappingService.get(editableMappingId, remoteFallbackLayer, layer.token)
       dataRequest: WebKnossosDataRequest = WebKnossosDataRequest(
-        position = Vec3Int(bucket.globalX, bucket.globalY, bucket.globalZ),
+        position = Vec3Int(bucket.topLeft.x, bucket.topLeft.y, bucket.topLeft.z),
         mag = bucket.mag,
         cubeSize = layer.lengthOfUnderlyingCubes(bucket.mag),
         fourBit = None,
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index c0fce49ab88..cd248df8a01 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -129,7 +129,6 @@ class EditableMappingService @Inject()(
                            desiredVersion: Long,
   ): Fox[EditableMapping] = {
     val key = EditableMappingKey(editableMappingId, remoteFallbackLayer, userToken, desiredVersion)
-    logger.info("getWithCache")
     for {
       materializedBox <- materializedEditableMappingCache.getOrLoad(
         key,
@@ -143,7 +142,8 @@ class EditableMappingService @Inject()(
                            remoteFallbackLayer: RemoteFallbackLayer,
                            userToken: Option[String],
                            desiredVersion: Long,
-  ): Fox[EditableMapping] =
+  ): Fox[EditableMapping] = {
+    logger.info("cache miss")
     for {
       closestMaterializedVersion: VersionedKeyValuePair[EditableMapping] <- tracingDataStore.editableMappings
         .get(editableMappingId, Some(desiredVersion))(fromJson[EditableMapping])
@@ -162,6 +162,7 @@ class EditableMappingService @Inject()(
         tracingDataStore.editableMappings.put(editableMappingId, desiredVersion, materialized)
       }
     } yield materialized
+  }
 
   private def shouldPersistMaterialized(previouslyMaterializedVersion: Long, newVersion: Long): Boolean =
     newVersion > previouslyMaterializedVersion && newVersion % 10 == 5
@@ -542,12 +543,12 @@ class EditableMappingService @Inject()(
       )
 
       isosurfaceRequest = IsosurfaceRequest(
-        None,
-        segmentationLayer,
-        request.cuboid(segmentationLayer),
-        request.segmentId,
-        request.subsamplingStrides,
-        request.scale,
+        dataSource = None,
+        dataLayer = segmentationLayer,
+        cuboid = request.cuboid(segmentationLayer),
+        segmentId = request.segmentId,
+        subsamplingStrides = request.subsamplingStrides,
+        scale = request.scale,
         mapping = None,
         mappingType = None
       )

From 3c9520f9bc50f984c9f52201c300749cc19c05b4 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Mon, 23 May 2022 14:21:35 +0200
Subject: [PATCH 058/122] fix editable mapping page reload

---
 frontend/javascripts/oxalis/model/sagas/update_actions.ts     | 4 +++-
 .../javascripts/oxalis/model/sagas/volumetracing_saga.tsx     | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/update_actions.ts b/frontend/javascripts/oxalis/model/sagas/update_actions.ts
index d6b4d738c27..f755dc05657 100644
--- a/frontend/javascripts/oxalis/model/sagas/update_actions.ts
+++ b/frontend/javascripts/oxalis/model/sagas/update_actions.ts
@@ -175,6 +175,7 @@ export type UpdateMappingNameUpdateAction = {
   name: "updateMappingName";
   value: {
     mappingName: string | null | undefined;
+    isEditable: boolean | undefined;
   };
 };
 export type SplitAgglomerateUpdateAction = {
@@ -544,10 +545,11 @@ export function serverCreateTracing(timestamp: number) {
 }
 export function updateMappingName(
   mappingName: string | null | undefined,
+  isEditable: boolean | undefined,
 ): UpdateMappingNameUpdateAction {
   return {
     name: "updateMappingName",
-    value: { mappingName },
+    value: { mappingName, isEditable },
   };
 }
 export function splitAgglomerate(
diff --git a/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx b/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx
index 23958283a1b..2aacda54637 100644
--- a/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx
+++ b/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx
@@ -713,7 +713,7 @@ export function* diffVolumeTracing(
   }
 
   if (prevVolumeTracing.mappingName !== volumeTracing.mappingName) {
-    yield updateMappingName(volumeTracing.mappingName);
+    yield updateMappingName(volumeTracing.mappingName, volumeTracing.mappingIsEditable);
   }
 }
 

From 52a104ae5aa374de1400cd5a06af9800f163ad01 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Mon, 23 May 2022 15:23:46 +0200
Subject: [PATCH 059/122] caching for unmapped data, skip buckets outside of
 bbox

---
 .../services/IsosurfaceService.scala          | 15 ++++++--
 .../EditableMappingLayer.scala                | 11 +++++-
 .../EditableMappingService.scala              | 38 +++++++++++++++++--
 3 files changed, 56 insertions(+), 8 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
index c7d83a59182..a8a5fd26f91 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
@@ -1,10 +1,12 @@
 package com.scalableminds.webknossos.datastore.services
 
+import java.nio._
+
 import akka.actor.{Actor, ActorRef, ActorSystem, Props}
 import akka.pattern.ask
 import akka.routing.RoundRobinPool
 import akka.util.Timeout
-import com.scalableminds.util.geometry.{BoundingBox, Vec3Int, Vec3Double}
+import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int}
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.models.datasource.{DataSource, ElementClass, SegmentationLayer}
 import com.scalableminds.webknossos.datastore.models.requests.{
@@ -14,8 +16,8 @@ import com.scalableminds.webknossos.datastore.models.requests.{
   DataServiceRequestSettings
 }
 import com.scalableminds.webknossos.datastore.services.mcubes.MarchingCubes
+import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common.{Box, Failure}
-import java.nio._
 
 import scala.collection.mutable
 import scala.concurrent.duration._
@@ -54,7 +56,8 @@ class IsosurfaceService(binaryDataService: BinaryDataService,
                         actorSystem: ActorSystem,
                         isosurfaceTimeout: FiniteDuration,
                         isosurfaceActorPoolSize: Int)(implicit ec: ExecutionContext)
-    extends FoxImplicits {
+    extends FoxImplicits
+    with LazyLogging {
 
   private val agglomerateService
     : Option[AgglomerateService] = try { Some(binaryDataService.agglomerateService) } catch {
@@ -194,12 +197,15 @@ class IsosurfaceService(binaryDataService: BinaryDataService,
     val vertexBuffer = mutable.ArrayBuffer[Vec3Double]()
 
     for {
+      before <- Fox.successful(System.currentTimeMillis())
       data <- binaryDataService.handleDataRequest(dataRequest)
+      afterLoading = System.currentTimeMillis()
       agglomerateMappedData = applyAgglomerate(data)
       typedData = convertData(agglomerateMappedData)
       mappedData <- applyMapping(typedData)
       mappedSegmentId <- applyMapping(Array(typedSegmentId)).map(_.head)
       neighbors = findNeighbors(mappedData, dataDimensions, mappedSegmentId)
+      afterPreprocessing = System.currentTimeMillis()
     } yield {
       for {
         x <- 0 until dataDimensions.x by 32
@@ -221,6 +227,9 @@ class IsosurfaceService(binaryDataService: BinaryDataService,
                                          vertexBuffer)
         }
       }
+      val afterMarchingCubes = System.currentTimeMillis()
+      logger.info(
+        s"Isosurface generation timing - loading: ${afterLoading - before} ms, preprocessing: ${afterPreprocessing - afterLoading}, marchingCubes: ${afterMarchingCubes - afterPreprocessing}")
       (vertexBuffer.flatMap(_.toList.map(_.toFloat)).toArray, neighbors)
     }
   }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
index 6d633fc9871..efbccd3097d 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
@@ -19,8 +19,11 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP
     val bucket: BucketPosition = readInstruction.bucket
     for {
       editableMappingId <- Fox.successful(layer.name)
+      _ <- bool2Fox(layer.doesContainBucket(bucket))
       remoteFallbackLayer <- layer.editableMappingService.remoteFallbackLayer(layer.tracing)
+      beforeGet = System.currentTimeMillis()
       editableMapping <- layer.editableMappingService.get(editableMappingId, remoteFallbackLayer, layer.token)
+      afterGet = System.currentTimeMillis()
       dataRequest: WebKnossosDataRequest = WebKnossosDataRequest(
         position = Vec3Int(bucket.topLeft.x, bucket.topLeft.y, bucket.topLeft.z),
         mag = bucket.mag,
@@ -33,13 +36,19 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP
       (unmappedData, indices) <- layer.editableMappingService.getUnmappedDataFromDatastore(remoteFallbackLayer,
                                                                                            dataRequestCollection,
                                                                                            layer.token)
+      afterGetUnmapped = System.currentTimeMillis()
       _ <- bool2Fox(indices.isEmpty)
       segmentIds <- layer.editableMappingService.collectSegmentIds(unmappedData, layer.tracing.elementClass)
+      afterCollectSegmentIds = System.currentTimeMillis()
       relevantMapping <- layer.editableMappingService.generateCombinedMappingSubset(segmentIds,
                                                                                     editableMapping,
                                                                                     remoteFallbackLayer,
                                                                                     layer.token)
-      mappedData <- layer.editableMappingService.mapData(unmappedData, relevantMapping, layer.elementClass)
+      afterCombineMapping = System.currentTimeMillis()
+      mappedData: Array[Byte] <- layer.editableMappingService.mapData(unmappedData, relevantMapping, layer.elementClass)
+      afterMapData = System.currentTimeMillis()
+      _ = logger.info(
+        s"load bucket timing: getMapping: ${afterGet - beforeGet} ms, getUnmapped: ${afterGetUnmapped - afterGet} ms, collectSegments: ${afterCollectSegmentIds - afterGet} ms, combine: ${afterCombineMapping - afterCollectSegmentIds}, mapData: ${afterMapData - afterCombineMapping}. Total ${afterMapData - beforeGet}. ${mappedData.length} bytes, ${segmentIds.size} segments")
     } yield mappedData
   }
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index cd248df8a01..335a5ea2ada 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -36,6 +36,12 @@ case class EditableMappingKey(
     desiredVersion: Long
 )
 
+case class UnmappedRemoteDataKey(
+    remoteFallbackLayer: RemoteFallbackLayer,
+    dataRequests: List[WebKnossosDataRequest],
+    userToken: Option[String]
+)
+
 class EditableMappingService @Inject()(
     val tracingDataStore: TracingDataStore,
     val isosurfaceServiceHolder: IsosurfaceServiceHolder,
@@ -51,7 +57,7 @@ class EditableMappingService @Inject()(
   val isosurfaceService: IsosurfaceService = isosurfaceServiceHolder.tracingStoreIsosurfaceService
 
   private lazy val materializedEditableMappingCache: Cache[EditableMappingKey, Box[EditableMapping]] = {
-    val maxEntries = 10
+    val maxEntries = 20
     val defaultCachingSettings = CachingSettings("")
     val lfuCacheSettings =
       defaultCachingSettings.lfuCacheSettings
@@ -65,6 +71,21 @@ class EditableMappingService @Inject()(
     lfuCache
   }
 
+  private lazy val unmappedRemoteDataCache: Cache[UnmappedRemoteDataKey, Box[(Array[Byte], List[Int])]] = {
+    val maxEntries = 3000
+    val defaultCachingSettings = CachingSettings("")
+    val lfuCacheSettings =
+      defaultCachingSettings.lfuCacheSettings
+        .withInitialCapacity(maxEntries)
+        .withMaxCapacity(maxEntries)
+        .withTimeToLive(2 hours)
+        .withTimeToIdle(1 hour)
+    val cachingSettings =
+      defaultCachingSettings.withLfuCacheSettings(lfuCacheSettings)
+    val lfuCache: Cache[UnmappedRemoteDataKey, Box[(Array[Byte], List[Int])]] = LfuCache(cachingSettings)
+    lfuCache
+  }
+
   def newestMaterializableVersion(editableMappingId: String): Fox[Long] =
     tracingDataStore.editableMappingUpdates.getVersion(editableMappingId,
                                                        mayBeEmpty = Some(true),
@@ -486,17 +507,26 @@ class EditableMappingService @Inject()(
                                    dataRequests: DataRequestCollection,
                                    userToken: Option[String]): Fox[(Array[Byte], List[Int])] =
     for {
-      dataRequestsTyped <- Fox.serialCombined(dataRequests) {
+      dataRequestsTyped: List[WebKnossosDataRequest] <- Fox.serialCombined(dataRequests) {
         case r: WebKnossosDataRequest => Fox.successful(r.copy(applyAgglomerate = None))
         case _                        => Fox.failure("Editable Mappings currently only work for webKnossos data requests")
       }
-      (data, indices) <- remoteDatastoreClient.getData(remoteFallbackLayer, dataRequestsTyped, userToken)
+      key = UnmappedRemoteDataKey(remoteFallbackLayer, dataRequestsTyped, userToken)
+      resultBox <- unmappedRemoteDataCache.getOrLoad(
+        key,
+        k => remoteDatastoreClient.getData(k.remoteFallbackLayer, k.dataRequests, k.userToken))
+      (data, indices) <- resultBox.toFox
     } yield (data, indices)
 
   def collectSegmentIds(data: Array[Byte], elementClass: ElementClass): Fox[Set[Long]] =
     for {
+      before <- Fox.successful(System.currentTimeMillis())
       dataAsLongs <- bytesToLongs(data, elementClass)
-    } yield dataAsLongs.toSet
+      afterToLong = System.currentTimeMillis()
+      result = dataAsLongs.toSet
+      afterToSet = System.currentTimeMillis()
+      //_ = logger.info(s"collect segment Ids timing - toLong: ${afterToLong - before} ms, toSet: ${afterToSet - afterToLong} ms")
+    } yield result
 
   def remoteFallbackLayer(tracing: VolumeTracing): Fox[RemoteFallbackLayer] =
     for {

From 9d461a28674f375d57209b6d28a175238b544c9c Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Mon, 23 May 2022 16:52:03 +0200
Subject: [PATCH 060/122] load meshes when proofreading

---
 frontend/javascripts/oxalis/api/api_latest.ts |   6 +-
 .../combinations/segmentation_handlers.ts     |  13 +-
 .../controller/combinations/tool_controls.ts  |   9 +-
 .../oxalis/model/actions/actions.ts           |   4 +-
 .../oxalis/model/actions/proofread_actions.ts |  13 ++
 .../model/actions/segmentation_actions.ts     |   1 +
 .../oxalis/model/sagas/isosurface_saga.ts     |   5 +-
 .../oxalis/model/sagas/proofread_saga.ts      | 111 +++++++++++++++++-
 .../model/sagas/skeletontracing_saga.ts       |   2 +-
 9 files changed, 149 insertions(+), 15 deletions(-)
 create mode 100644 frontend/javascripts/oxalis/model/actions/proofread_actions.ts

diff --git a/frontend/javascripts/oxalis/api/api_latest.ts b/frontend/javascripts/oxalis/api/api_latest.ts
index 4fbf73cb2ff..8e95164afbd 100644
--- a/frontend/javascripts/oxalis/api/api_latest.ts
+++ b/frontend/javascripts/oxalis/api/api_latest.ts
@@ -1524,7 +1524,11 @@ class DataApi {
     });
   }
 
-  getRawDataCuboid(layerName: string, topLeft: Vector3, bottomRight: Vector3): Promise<void> {
+  getRawDataCuboid(
+    layerName: string,
+    topLeft: Vector3,
+    bottomRight: Vector3,
+  ): Promise<ArrayBuffer> {
     return doWithToken((token) => {
       const downloadUrl = this._getDownloadUrlForRawDataCuboid(
         layerName,
diff --git a/frontend/javascripts/oxalis/controller/combinations/segmentation_handlers.ts b/frontend/javascripts/oxalis/controller/combinations/segmentation_handlers.ts
index 2f0152f70a1..ffc355dec6d 100644
--- a/frontend/javascripts/oxalis/controller/combinations/segmentation_handlers.ts
+++ b/frontend/javascripts/oxalis/controller/combinations/segmentation_handlers.ts
@@ -17,6 +17,7 @@ import {
   getSegmentIdForPositionAsync,
 } from "oxalis/controller/combinations/volume_handlers";
 import { setActiveConnectomeAgglomerateIdsAction } from "oxalis/model/actions/connectome_actions";
+import { getTreeNameForAgglomerateSkeleton } from "oxalis/model/accessors/skeletontracing_accessor";
 export function hasAgglomerateMapping(state: OxalisState) {
   const segmentation = Model.getVisibleSegmentationLayer();
 
@@ -81,22 +82,22 @@ export async function handleAgglomerateSkeletonAtClick(clickPosition: Point2) {
   const globalPosition = calculateGlobalPos(state, clickPosition);
   loadAgglomerateSkeletonAtPosition(globalPosition);
 }
-export async function loadAgglomerateSkeletonAtPosition(position: Vector3): Promise<void> {
+export async function loadAgglomerateSkeletonAtPosition(position: Vector3): Promise<string | null> {
   const segmentation = Model.getVisibleSegmentationLayer();
 
   if (!segmentation) {
-    return;
+    return null;
   }
 
   const segmentId = await getSegmentIdForPositionAsync(position);
-  loadAgglomerateSkeletonForSegmentId(segmentId);
+  return loadAgglomerateSkeletonForSegmentId(segmentId);
 }
-export function loadAgglomerateSkeletonForSegmentId(segmentId: number): void {
+export function loadAgglomerateSkeletonForSegmentId(segmentId: number): string | null {
   const state = Store.getState();
   const segmentation = Model.getVisibleSegmentationLayer();
 
   if (!segmentation) {
-    return;
+    return null;
   }
 
   const { mappingName } = getMappingInfo(
@@ -107,9 +108,11 @@ export function loadAgglomerateSkeletonForSegmentId(segmentId: number): void {
 
   if (mappingName && isAgglomerateMappingEnabled.value) {
     Store.dispatch(loadAgglomerateSkeletonAction(segmentation.name, mappingName, segmentId));
+    return getTreeNameForAgglomerateSkeleton(segmentId, mappingName);
   } else {
     Toast.error(isAgglomerateMappingEnabled.reason);
   }
+  return null;
 }
 export async function loadSynapsesOfAgglomerateAtPosition(position: Vector3) {
   const state = Store.getState();
diff --git a/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts b/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts
index 4e401b81ce7..820d568d8df 100644
--- a/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts
+++ b/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts
@@ -27,6 +27,8 @@ import * as Utils from "libs/utils";
 import * as VolumeHandlers from "oxalis/controller/combinations/volume_handlers";
 import { document } from "libs/window";
 import api from "oxalis/api/internal_api";
+import { proofreadAtPosition } from "oxalis/model/actions/proofread_actions";
+import { calculateGlobalPos } from "oxalis/model/accessors/view_mode_accessor";
 
 export type ActionDescriptor = {
   leftClick?: string;
@@ -621,11 +623,10 @@ export class ProofreadTool {
   static getPlaneMouseControls(_planeId: OrthoView, planeView: PlaneView): any {
     return {
       leftClick: (pos: Point2, plane: OrthoView, _event: MouseEvent, isTouch: boolean) => {
-        const didSelectNode = SkeletonHandlers.handleSelectNode(planeView, pos, plane, isTouch);
+        SkeletonHandlers.handleSelectNode(planeView, pos, plane, isTouch);
 
-        if (!didSelectNode) {
-          handleAgglomerateSkeletonAtClick(pos);
-        }
+        const globalPosition = calculateGlobalPos(Store.getState(), pos);
+        Store.dispatch(proofreadAtPosition(globalPosition));
       },
     };
   }
diff --git a/frontend/javascripts/oxalis/model/actions/actions.ts b/frontend/javascripts/oxalis/model/actions/actions.ts
index 38045f615c1..67073ef3161 100644
--- a/frontend/javascripts/oxalis/model/actions/actions.ts
+++ b/frontend/javascripts/oxalis/model/actions/actions.ts
@@ -11,6 +11,7 @@ import type { UserAction } from "oxalis/model/actions/user_actions";
 import type { ViewModeAction } from "oxalis/model/actions/view_mode_actions";
 import type { VolumeTracingAction } from "oxalis/model/actions/volumetracing_actions";
 import type { ConnectomeAction } from "oxalis/model/actions/connectome_actions";
+import { ProofreadAction } from "oxalis/model/actions//proofread_actions";
 export type Action =
   | SkeletonTracingAction
   | VolumeTracingAction
@@ -24,7 +25,8 @@ export type Action =
   | UserAction
   | UiAction
   | SegmentationAction
-  | ConnectomeAction;
+  | ConnectomeAction
+  | ProofreadAction;
 export const wkReadyAction = () => ({
   type: "WK_READY",
 });
diff --git a/frontend/javascripts/oxalis/model/actions/proofread_actions.ts b/frontend/javascripts/oxalis/model/actions/proofread_actions.ts
new file mode 100644
index 00000000000..8f05ab0d1fd
--- /dev/null
+++ b/frontend/javascripts/oxalis/model/actions/proofread_actions.ts
@@ -0,0 +1,13 @@
+import { Vector3 } from "oxalis/constants";
+
+export type ProofreadAtPositionAction = {
+  type: "PROOFREAD_AT_POSITION";
+  position: Vector3;
+};
+
+export type ProofreadAction = ProofreadAtPositionAction;
+
+export const proofreadAtPosition = (position: Vector3): ProofreadAtPositionAction => ({
+  type: "PROOFREAD_AT_POSITION",
+  position,
+});
diff --git a/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts b/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts
index 38fefaa4190..c4ec6330748 100644
--- a/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts
+++ b/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts
@@ -3,6 +3,7 @@ import type { MappingType } from "oxalis/store";
 export type IsosurfaceMappingInfo = {
   mappingName: string | null | undefined;
   mappingType: MappingType | null | undefined;
+  useDataStore?: boolean | null | undefined;
 };
 export type LoadAdHocMeshAction = {
   type: "LOAD_AD_HOC_MESH_ACTION";
diff --git a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
index b4911487199..30c287a1b5d 100644
--- a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
@@ -367,7 +367,10 @@ function* maybeLoadIsosurface(
   const tracingStoreUrl = `${tracingStoreHost}/tracings/volume/${layer.name}`;
   const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
   // Fetch from datastore if no volumetracing exists
-  const useDataStore = volumeTracing == null;
+  const useDataStore =
+    isosurfaceMappingInfo.useDataStore != null
+      ? isosurfaceMappingInfo.useDataStore
+      : volumeTracing == null;
   const mag = resolutionInfo.getResolutionByIndexOrThrow(zoomStep);
 
   if (isInitialRequest) {
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 486dbe0cd87..d96e2ef2df6 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -1,5 +1,5 @@
 import type { Saga } from "oxalis/model/sagas/effect-generators";
-import { takeEvery, put, call } from "typed-redux-saga";
+import { takeEvery, put, call, all } from "typed-redux-saga";
 import { select, take } from "oxalis/model/sagas/effect-generators";
 import { AnnotationToolEnum, MappingStatusEnum } from "oxalis/constants";
 import Toast from "libs/toast";
@@ -11,8 +11,10 @@ import {
   initializeEditableMappingAction,
   setMappingisEditableAction,
 } from "oxalis/model/actions/volumetracing_actions";
+import type { ProofreadAtPositionAction } from "oxalis/model/actions/proofread_actions";
 import {
   enforceSkeletonTracing,
+  findTreeByName,
   findTreeByNodeId,
 } from "oxalis/model/accessors/skeletontracing_accessor";
 import {
@@ -28,12 +30,117 @@ import {
 } from "oxalis/model/accessors/volumetracing_accessor";
 import { getMappingInfo, getResolutionInfo } from "oxalis/model/accessors/dataset_accessor";
 import { makeMappingEditable } from "admin/admin_rest_api";
-import { setMappingNameAction } from "oxalis/model/actions/settings_actions";
+import {
+  setMappingNameAction,
+  updateTemporarySettingAction,
+} from "oxalis/model/actions/settings_actions";
+import { loadAgglomerateSkeletonAtPosition } from "oxalis/controller/combinations/segmentation_handlers";
+import { getSegmentIdForPosition } from "oxalis/controller/combinations/volume_handlers";
+import { loadAdHocMeshAction } from "oxalis/model/actions/segmentation_actions";
+import { V3 } from "libs/mjs";
+import { removeIsosurfaceAction } from "oxalis/model/actions/annotation_actions";
 
 export default function* proofreadMapping(): Saga<any> {
   yield* take("INITIALIZE_SKELETONTRACING");
   yield* take("WK_READY");
   yield* takeEvery(["DELETE_EDGE", "MERGE_TREES"], splitOrMergeAgglomerate);
+  yield* takeEvery(["PROOFREAD_AT_POSITION"], proofreadAtPosition);
+}
+
+const COARSE_RESOLUTION_INDEX = 4;
+// @ts-ignore
+const PROOFREAD_SEGMENT_SURROUND_NM: number = window.__proofreadSurroundNm || 2000;
+let oldSegmentIdsInSurround: number[] | null = null;
+
+function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<any> {
+  const { position } = action;
+  const treeName = yield* call(loadAgglomerateSkeletonAtPosition, position);
+
+  const volumeTracingLayer = yield* select((state) => getActiveSegmentationTracingLayer(state));
+  if (volumeTracingLayer == null || volumeTracingLayer.tracingId == null) return;
+
+  const layerName = volumeTracingLayer.tracingId;
+
+  const oldPreferredQuality = yield* select(
+    (state) => state.temporaryConfiguration.preferredQualityForMeshAdHocComputation,
+  );
+  const resolutionInfo = getResolutionInfo(volumeTracingLayer.resolutions);
+  const resolutionIndices = resolutionInfo.getAllIndices();
+  const coarseResolutionIndex =
+    resolutionIndices[Math.min(COARSE_RESOLUTION_INDEX, resolutionIndices.length - 1)];
+  yield* put(
+    updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", coarseResolutionIndex),
+  );
+  const segmentId = getSegmentIdForPosition(position);
+  yield* put(loadAdHocMeshAction(segmentId, position));
+  yield* put(
+    updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", oldPreferredQuality),
+  );
+
+  if (treeName == null) return;
+
+  const skeletonTracing = yield* select((state) => enforceSkeletonTracing(state.tracing));
+  const { trees } = skeletonTracing;
+  const tree = findTreeByName(trees, treeName).getOrElse(null);
+
+  if (tree == null) return;
+
+  // Find all segments (nodes) that are within x µm to load meshes in oversegmentation
+  const nodePositions = tree.nodes.map((node) => node.position);
+  const distanceSquared = PROOFREAD_SEGMENT_SURROUND_NM ** 2;
+  const scale = yield* select((state) => state.dataset.dataSource.scale);
+
+  const nodePositionsInSurround = nodePositions.filter(
+    (nodePosition) => V3.scaledSquaredDist(nodePosition, position, scale) <= distanceSquared,
+  );
+  const mag = resolutionInfo.getLowestResolution();
+
+  const fallbackLayerName = volumeTracingLayer.fallbackLayer;
+  if (fallbackLayerName == null) return;
+
+  // Request unmapped segmentation ids
+  const segmentIdsArrayBuffers: ArrayBuffer[] = yield* all(
+    nodePositionsInSurround.map((nodePosition) =>
+      call(
+        [api.data, api.data.getRawDataCuboid],
+        fallbackLayerName,
+        nodePosition,
+        V3.add(nodePosition, mag),
+      ),
+    ),
+  );
+  // HACK: This only works for uint32 segmentations
+  const segmentIdsInSurround = segmentIdsArrayBuffers.map((buffer) => new Uint32Array(buffer)[0]);
+
+  if (oldSegmentIdsInSurround != null) {
+    // Remove old meshes in oversegmentation
+    yield* all(
+      oldSegmentIdsInSurround.map((nodeSegmentId) =>
+        put(removeIsosurfaceAction(layerName, nodeSegmentId)),
+      ),
+    );
+  }
+
+  // Load meshes in oversegmentation
+  const noMappingInfo = {
+    mappingName: null,
+    mappingType: null,
+    useDataStore: true,
+  };
+  yield* all(
+    segmentIdsInSurround.map((nodeSegmentId, index) =>
+      put(
+        loadAdHocMeshAction(
+          nodeSegmentId,
+          nodePositionsInSurround[index],
+          noMappingInfo,
+          layerName,
+        ),
+      ),
+    ),
+  );
+
+  oldSegmentIdsInSurround = segmentIdsInSurround;
 }
 
 function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
diff --git a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
index f0a183b8945..96654aa9266 100644
--- a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
@@ -330,7 +330,7 @@ function* loadAgglomerateSkeletonWithId(action: LoadAgglomerateSkeletonAction):
   const maybeTree = findTreeByName(trees, treeName).getOrElse(null);
 
   if (maybeTree != null) {
-    Toast.info(
+    console.warn(
       `Skeleton for agglomerate ${agglomerateId} with mapping ${mappingName} is already loaded. Its tree name is "${treeName}".`,
     );
     return;

From 1cd4551c11dab917d3d65801a4841ee4217827a7 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 24 May 2022 10:27:47 +0200
Subject: [PATCH 061/122] perf optimization for ad-hoc meshing

---
 .../services/BinaryDataService.scala          |  6 ++--
 .../EditableMappingLayer.scala                |  7 ++--
 .../EditableMappingService.scala              | 35 ++++++++++---------
 3 files changed, 27 insertions(+), 21 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
index 421bf55d9fb..aa0044d8a40 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
@@ -34,11 +34,11 @@ class BinaryDataService(val dataBaseDir: Path, maxCacheSize: Int, val agglomerat
         handleBucketRequest(request, bucket)
       }
     } else {
-      Fox
-        .serialSequence(bucketQueue.toList) { bucket =>
+      Fox.sequence {
+        bucketQueue.toList.map { bucket =>
           handleBucketRequest(request, bucket).map(r => bucket -> r)
         }
-        .map(buckets => cutOutCuboid(request, buckets.flatten))
+      }.map(buckets => cutOutCuboid(request, buckets.flatten))
     }
   }
 
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
index efbccd3097d..14c8195b9ed 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
@@ -38,14 +38,17 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP
                                                                                            layer.token)
       afterGetUnmapped = System.currentTimeMillis()
       _ <- bool2Fox(indices.isEmpty)
-      segmentIds <- layer.editableMappingService.collectSegmentIds(unmappedData, layer.tracing.elementClass)
+      unmappedDataTyped <- layer.editableMappingService.bytesToUnsignedInt(unmappedData, layer.tracing.elementClass)
+      segmentIds = layer.editableMappingService.collectSegmentIds(unmappedDataTyped)
       afterCollectSegmentIds = System.currentTimeMillis()
       relevantMapping <- layer.editableMappingService.generateCombinedMappingSubset(segmentIds,
                                                                                     editableMapping,
                                                                                     remoteFallbackLayer,
                                                                                     layer.token)
       afterCombineMapping = System.currentTimeMillis()
-      mappedData: Array[Byte] <- layer.editableMappingService.mapData(unmappedData, relevantMapping, layer.elementClass)
+      mappedData: Array[Byte] <- layer.editableMappingService.mapData(unmappedDataTyped,
+                                                                      relevantMapping,
+                                                                      layer.elementClass)
       afterMapData = System.currentTimeMillis()
       _ = logger.info(
         s"load bucket timing: getMapping: ${afterGet - beforeGet} ms, getUnmapped: ${afterGetUnmapped - afterGet} ms, collectSegments: ${afterCollectSegmentIds - afterGet} ms, combine: ${afterCombineMapping - afterCollectSegmentIds}, mapData: ${afterMapData - afterCombineMapping}. Total ${afterMapData - beforeGet}. ${mappedData.length} bytes, ${segmentIds.size} segments")
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 335a5ea2ada..c0f08a3851f 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -417,9 +417,10 @@ class EditableMappingService @Inject()(
       remoteFallbackLayer <- remoteFallbackLayer(tracing)
       editableMapping <- get(editableMappingId, remoteFallbackLayer, userToken)
       (unmappedData, indices) <- getUnmappedDataFromDatastore(remoteFallbackLayer, dataRequests, userToken)
-      segmentIds <- collectSegmentIds(unmappedData, tracing.elementClass)
+      unmappedDataTyped <- bytesToUnsignedInt(unmappedData, tracing.elementClass)
+      segmentIds = collectSegmentIds(unmappedDataTyped)
       relevantMapping <- generateCombinedMappingSubset(segmentIds, editableMapping, remoteFallbackLayer, userToken)
-      mappedData <- mapData(unmappedData, relevantMapping, tracing.elementClass)
+      mappedData <- mapData(unmappedDataTyped, relevantMapping, tracing.elementClass)
     } yield (mappedData, indices)
 
   def generateCombinedMappingSubset(segmentIds: Set[Long],
@@ -518,15 +519,10 @@ class EditableMappingService @Inject()(
       (data, indices) <- resultBox.toFox
     } yield (data, indices)
 
-  def collectSegmentIds(data: Array[Byte], elementClass: ElementClass): Fox[Set[Long]] =
-    for {
-      before <- Fox.successful(System.currentTimeMillis())
-      dataAsLongs <- bytesToLongs(data, elementClass)
-      afterToLong = System.currentTimeMillis()
-      result = dataAsLongs.toSet
-      afterToSet = System.currentTimeMillis()
-      //_ = logger.info(s"collect segment Ids timing - toLong: ${afterToLong - before} ms, toSet: ${afterToSet - afterToLong} ms")
-    } yield result
+  def collectSegmentIds(data: Array[UnsignedInteger]): Set[Long] =
+    data.toSet.map { u: UnsignedInteger =>
+      u.toPositiveLong
+    }
 
   def remoteFallbackLayer(tracing: VolumeTracing): Fox[RemoteFallbackLayer] =
     for {
@@ -534,14 +530,15 @@ class EditableMappingService @Inject()(
       organizationName <- tracing.organizationName.toFox ?~> "This feature is only implemented for volume annotations with an explicit organization name tag, not for legacy volume annotations."
     } yield RemoteFallbackLayer(organizationName, tracing.dataSetName, layerName, tracing.elementClass)
 
-  def mapData(unmappedData: Array[Byte],
+  def mapData(unmappedData: Array[UnsignedInteger],
               relevantMapping: Map[Long, Long],
-              elementClass: ElementClass): Fox[Array[Byte]] =
+              elementClass: ElementClass): Fox[Array[Byte]] = {
+    val unmappedDataLongs = unmappedData.map(_.toPositiveLong)
+    val mappedDataLongs = unmappedDataLongs.map(relevantMapping)
     for {
-      unmappedDataLongs <- bytesToLongs(unmappedData, elementClass)
-      mappedDataLongs = unmappedDataLongs.map(relevantMapping)
       bytes <- longsToBytes(mappedDataLongs, elementClass)
     } yield bytes
+  }
 
   private def bytesToLongs(bytes: Array[Byte], elementClass: ElementClass): Fox[Array[Long]] =
     for {
@@ -549,7 +546,13 @@ class EditableMappingService @Inject()(
       unsignedIntArray <- tryo(UnsignedIntegerArray.fromByteArray(bytes, elementClass)).toFox
     } yield unsignedIntArray.map(_.toPositiveLong)
 
-  private def longsToBytes(longs: Array[Long], elementClass: ElementClass): Fox[Array[Byte]] =
+  def bytesToUnsignedInt(bytes: Array[Byte], elementClass: ElementClass): Fox[Array[UnsignedInteger]] =
+    for {
+      _ <- bool2Fox(!elementClass.isuint64)
+      unsignedIntArray <- tryo(UnsignedIntegerArray.fromByteArray(bytes, elementClass)).toFox
+    } yield unsignedIntArray
+
+  def longsToBytes(longs: Array[Long], elementClass: ElementClass): Fox[Array[Byte]] =
     for {
       _ <- bool2Fox(!elementClass.isuint64)
       unsignedIntArray: Array[UnsignedInteger] = longs.map(UnsignedInteger.fromLongWithElementClass(_, elementClass))

From b53bc3ef539b27bf4b5f8a5058abfdd5fc55a119 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 24 May 2022 11:43:32 +0200
Subject: [PATCH 062/122] fix window configuration variables and allow to
 disable proofreading using meshes

---
 .../oxalis/model/sagas/proofread_saga.ts      | 40 ++++++++++++++++---
 1 file changed, 34 insertions(+), 6 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index d96e2ef2df6..eb4166b5609 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -47,27 +47,49 @@ export default function* proofreadMapping(): Saga<any> {
   yield* takeEvery(["PROOFREAD_AT_POSITION"], proofreadAtPosition);
 }
 
-const COARSE_RESOLUTION_INDEX = 4;
-// @ts-ignore
-const PROOFREAD_SEGMENT_SURROUND_NM: number = window.__proofreadSurroundNm || 2000;
+function proofreadCoarseResolutionIndex(): number {
+  // @ts-ignore
+  return window.__proofreadCoarseResolutionIndex != null
+    ? // @ts-ignore
+      window.__proofreadCoarseResolutionIndex
+    : 4;
+}
+function proofreadFineResolutionIndex(): number {
+  // @ts-ignore
+  return window.__proofreadFineResolutionIndex != null
+    ? // @ts-ignore
+      window.__proofreadFineResolutionIndex
+    : 0;
+}
+function proofreadUsingMeshes(): boolean {
+  // @ts-ignore
+  return window.__proofreadUsingMeshes != null ? window.__proofreadUsingMeshes : true;
+}
+function proofreadSegmentSurroundNm(): number {
+  // @ts-ignore
+  return window.__proofreadSurroundNm != null ? window.__proofreadSurroundNm : 2000;
+}
 let oldSegmentIdsInSurround: number[] | null = null;
 
 function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<any> {
   const { position } = action;
   const treeName = yield* call(loadAgglomerateSkeletonAtPosition, position);
 
+  if (!proofreadUsingMeshes()) return;
+
   const volumeTracingLayer = yield* select((state) => getActiveSegmentationTracingLayer(state));
   if (volumeTracingLayer == null || volumeTracingLayer.tracingId == null) return;
 
   const layerName = volumeTracingLayer.tracingId;
 
+  // Load the whole agglomerate mesh in a coarse resolution for performance reasons
   const oldPreferredQuality = yield* select(
     (state) => state.temporaryConfiguration.preferredQualityForMeshAdHocComputation,
   );
   const resolutionInfo = getResolutionInfo(volumeTracingLayer.resolutions);
   const resolutionIndices = resolutionInfo.getAllIndices();
   const coarseResolutionIndex =
-    resolutionIndices[Math.min(COARSE_RESOLUTION_INDEX, resolutionIndices.length - 1)];
+    resolutionIndices[Math.min(proofreadCoarseResolutionIndex(), resolutionIndices.length - 1)];
   yield* put(
     updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", coarseResolutionIndex),
   );
@@ -87,7 +109,7 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<any> {
 
   // Find all segments (nodes) that are within x µm to load meshes in oversegmentation
   const nodePositions = tree.nodes.map((node) => node.position);
-  const distanceSquared = PROOFREAD_SEGMENT_SURROUND_NM ** 2;
+  const distanceSquared = proofreadSegmentSurroundNm() ** 2;
   const scale = yield* select((state) => state.dataset.dataSource.scale);
 
   const nodePositionsInSurround = nodePositions.filter(
@@ -121,12 +143,18 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<any> {
     );
   }
 
-  // Load meshes in oversegmentation
+  // Load meshes in oversegmentation in fine resolution
   const noMappingInfo = {
     mappingName: null,
     mappingType: null,
     useDataStore: true,
   };
+  yield* put(
+    updateTemporarySettingAction(
+      "preferredQualityForMeshAdHocComputation",
+      proofreadFineResolutionIndex(),
+    ),
+  );
   yield* all(
     segmentIdsInSurround.map((nodeSegmentId, index) =>
       put(

From 89facdd4dd71dce0299ed1382697a87093fdc2b6 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 24 May 2022 12:41:39 +0200
Subject: [PATCH 063/122] make coarse agglomerate more transparent, exclude it
 from ray tracing and try to fix transparency issues

---
 .../oxalis/controller/scene_controller.ts     | 24 ++++++++++++++-----
 .../model/actions/segmentation_actions.ts     |  1 +
 .../oxalis/model/sagas/isosurface_saga.ts     |  6 ++++-
 .../oxalis/model/sagas/proofread_saga.ts      |  8 ++++++-
 .../javascripts/oxalis/view/plane_view.ts     |  4 +++-
 5 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/frontend/javascripts/oxalis/controller/scene_controller.ts b/frontend/javascripts/oxalis/controller/scene_controller.ts
index c30a9a320d5..195e671b1cd 100644
--- a/frontend/javascripts/oxalis/controller/scene_controller.ts
+++ b/frontend/javascripts/oxalis/controller/scene_controller.ts
@@ -191,7 +191,11 @@ class SceneController {
     return this.isosurfacesGroupsPerSegmentationId[cellId];
   }
 
-  constructSceneMesh(cellId: number, geometry: THREE.Geometry | THREE.BufferGeometry) {
+  constructSceneMesh(
+    cellId: number,
+    geometry: THREE.Geometry | THREE.BufferGeometry,
+    passive: boolean,
+  ) {
     const [hue] = jsConvertCellIdToHSLA(cellId);
     const color = new THREE.Color().setHSL(hue, 0.75, 0.05);
     const meshMaterial = new THREE.MeshLambertMaterial({
@@ -202,13 +206,14 @@ class SceneController {
     const mesh = new THREE.Mesh(geometry, meshMaterial);
     mesh.castShadow = true;
     mesh.receiveShadow = true;
+    mesh.renderOrder = passive ? 1 : 0;
     const tweenAnimation = new TWEEN.Tween({
       opacity: 0,
     });
     tweenAnimation
       .to(
         {
-          opacity: 0.95,
+          opacity: passive ? 0.2 : 0.7,
         },
         500,
       )
@@ -234,13 +239,17 @@ class SceneController {
 
     const meshNumber = _.size(this.stlMeshes);
 
-    const mesh = this.constructSceneMesh(meshNumber, geometry);
+    const mesh = this.constructSceneMesh(meshNumber, geometry, false);
     this.meshesRootGroup.add(mesh);
     this.stlMeshes[id] = mesh;
     this.updateMeshPostion(id, position);
   }
 
-  addIsosurfaceFromVertices(vertices: Float32Array, segmentationId: number): void {
+  addIsosurfaceFromVertices(
+    vertices: Float32Array,
+    segmentationId: number,
+    passive: boolean,
+  ): void {
     let bufferGeometry = new THREE.BufferGeometry();
     bufferGeometry.setAttribute("position", new THREE.BufferAttribute(vertices, 3));
     // convert to normal (unbuffered) geometry to merge vertices
@@ -255,14 +264,15 @@ class SceneController {
     geometry.computeFaceNormals();
     // and back to a BufferGeometry
     bufferGeometry = new THREE.BufferGeometry().fromGeometry(geometry);
-    this.addIsosurfaceFromGeometry(bufferGeometry, segmentationId);
+    this.addIsosurfaceFromGeometry(bufferGeometry, segmentationId, passive);
   }
 
   addIsosurfaceFromGeometry(
     geometry: THREE.Geometry | THREE.BufferGeometry,
     segmentationId: number,
+    passive: boolean = false,
   ): void {
-    const mesh = this.constructSceneMesh(segmentationId, geometry);
+    const mesh = this.constructSceneMesh(segmentationId, geometry, passive);
 
     if (this.isosurfacesGroupsPerSegmentationId[segmentationId] == null) {
       const newGroup = new THREE.Group();
@@ -270,6 +280,8 @@ class SceneController {
       this.isosurfacesRootGroup.add(newGroup);
       // @ts-ignore
       newGroup.cellId = segmentationId;
+      // @ts-ignore
+      newGroup.passive = passive;
     }
 
     this.isosurfacesGroupsPerSegmentationId[segmentationId].add(mesh);
diff --git a/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts b/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts
index c4ec6330748..3d6f16c0ab4 100644
--- a/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts
+++ b/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts
@@ -4,6 +4,7 @@ export type IsosurfaceMappingInfo = {
   mappingName: string | null | undefined;
   mappingType: MappingType | null | undefined;
   useDataStore?: boolean | null | undefined;
+  passive?: boolean | null | undefined;
 };
 export type LoadAdHocMeshAction = {
   type: "LOAD_AD_HOC_MESH_ACTION";
diff --git a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
index 30c287a1b5d..20d5efe0b73 100644
--- a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
@@ -405,7 +405,11 @@ function* maybeLoadIsosurface(
         getSceneController().removeIsosurfaceById(segmentId);
       }
 
-      getSceneController().addIsosurfaceFromVertices(vertices, segmentId);
+      getSceneController().addIsosurfaceFromVertices(
+        vertices,
+        segmentId,
+        isosurfaceMappingInfo.passive || false,
+      );
       return neighbors.map((neighbor) =>
         getNeighborPosition(clippedPosition, neighbor, zoomStep, resolutionInfo),
       );
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index eb4166b5609..d685bf8a88d 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -82,6 +82,12 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<any> {
 
   const layerName = volumeTracingLayer.tracingId;
 
+  const activeMappingByLayer = yield* select(
+    (state) => state.temporaryConfiguration.activeMappingByLayer,
+  );
+  const mappingInfo = getMappingInfo(activeMappingByLayer, layerName);
+  const { mappingName, mappingType } = mappingInfo;
+
   // Load the whole agglomerate mesh in a coarse resolution for performance reasons
   const oldPreferredQuality = yield* select(
     (state) => state.temporaryConfiguration.preferredQualityForMeshAdHocComputation,
@@ -94,7 +100,7 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<any> {
     updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", coarseResolutionIndex),
   );
   const segmentId = getSegmentIdForPosition(position);
-  yield* put(loadAdHocMeshAction(segmentId, position));
+  yield* put(loadAdHocMeshAction(segmentId, position, { mappingName, mappingType, passive: true }));
   yield* put(
     updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", oldPreferredQuality),
   );
diff --git a/frontend/javascripts/oxalis/view/plane_view.ts b/frontend/javascripts/oxalis/view/plane_view.ts
index c72a340ad17..daa9a002f3f 100644
--- a/frontend/javascripts/oxalis/view/plane_view.ts
+++ b/frontend/javascripts/oxalis/view/plane_view.ts
@@ -170,7 +170,9 @@ class PlaneView {
     raycaster.setFromCamera(mouse, this.cameras[OrthoViews.TDView]);
     // The second parameter of intersectObjects is set to true to ensure that
     // the groups which contain the actual meshes are traversed.
-    const intersections = raycaster.intersectObjects(isosurfacesRootGroup.children, true);
+    // @ts-ignore
+    const intersectableObjects = isosurfacesRootGroup.children.filter((obj) => !obj.passive);
+    const intersections = raycaster.intersectObjects(intersectableObjects, true);
     const hitObject = intersections.length > 0 ? intersections[0].object : null;
 
     // Check whether we are hitting the same object as before, since we can return early

From 78332c4eacbd036ac60213c4e5cc6abd3873b074 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 24 May 2022 14:27:17 +0200
Subject: [PATCH 064/122] use data store to request ad-hoc meshes if mapping is
 not yet editable

---
 .../oxalis/model/sagas/proofread_saga.ts            | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index d685bf8a88d..16ce37a33ad 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -79,6 +79,8 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<any> {
 
   const volumeTracingLayer = yield* select((state) => getActiveSegmentationTracingLayer(state));
   if (volumeTracingLayer == null || volumeTracingLayer.tracingId == null) return;
+  const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
+  if (volumeTracing == null) return;
 
   const layerName = volumeTracingLayer.tracingId;
 
@@ -100,7 +102,16 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<any> {
     updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", coarseResolutionIndex),
   );
   const segmentId = getSegmentIdForPosition(position);
-  yield* put(loadAdHocMeshAction(segmentId, position, { mappingName, mappingType, passive: true }));
+  // Use the data store if the mapping is not editable yet. If it, is request the mesh from the tracing store.
+  const useDataStore = !volumeTracing.mappingIsEditable;
+  yield* put(
+    loadAdHocMeshAction(segmentId, position, {
+      mappingName,
+      mappingType,
+      passive: true,
+      useDataStore,
+    }),
+  );
   yield* put(
     updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", oldPreferredQuality),
   );

From c6fe8047b74116d898b578aa2064f3a2c7d8caa2 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 24 May 2022 16:28:36 +0200
Subject: [PATCH 065/122] reload meshes after proofreading action

---
 .../controller/viewmodes/plane_controller.tsx |   1 +
 .../oxalis/model/sagas/proofread_saga.ts      | 104 +++++++++++++++---
 2 files changed, 89 insertions(+), 16 deletions(-)

diff --git a/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx b/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx
index 846210805ef..04f0826e7a3 100644
--- a/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx
+++ b/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx
@@ -312,6 +312,7 @@ class PlaneController extends React.PureComponent<Props> {
       Object.keys(fillCellControls),
       Object.keys(pickCellControls),
       Object.keys(boundingBoxControls),
+      Object.keys(proofreadControls),
     );
 
     const controls: Record<string, any> = {};
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 16ce37a33ad..87e50105a7d 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -1,7 +1,7 @@
 import type { Saga } from "oxalis/model/sagas/effect-generators";
 import { takeEvery, put, call, all } from "typed-redux-saga";
 import { select, take } from "oxalis/model/sagas/effect-generators";
-import { AnnotationToolEnum, MappingStatusEnum } from "oxalis/constants";
+import { AnnotationToolEnum, MappingStatusEnum, Vector3 } from "oxalis/constants";
 import Toast from "libs/toast";
 import type {
   DeleteEdgeAction,
@@ -28,7 +28,11 @@ import {
   getActiveSegmentationTracingLayer,
   getActiveSegmentationTracing,
 } from "oxalis/model/accessors/volumetracing_accessor";
-import { getMappingInfo, getResolutionInfo } from "oxalis/model/accessors/dataset_accessor";
+import {
+  getMappingInfo,
+  getResolutionInfo,
+  ResolutionInfo,
+} from "oxalis/model/accessors/dataset_accessor";
 import { makeMappingEditable } from "admin/admin_rest_api";
 import {
   setMappingNameAction,
@@ -71,19 +75,15 @@ function proofreadSegmentSurroundNm(): number {
 }
 let oldSegmentIdsInSurround: number[] | null = null;
 
-function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<any> {
-  const { position } = action;
-  const treeName = yield* call(loadAgglomerateSkeletonAtPosition, position);
-
-  if (!proofreadUsingMeshes()) return;
-
-  const volumeTracingLayer = yield* select((state) => getActiveSegmentationTracingLayer(state));
-  if (volumeTracingLayer == null || volumeTracingLayer.tracingId == null) return;
+function* loadCoarseAdHocMesh(
+  layerName: string,
+  resolutionInfo: ResolutionInfo,
+  segmentId: number,
+  position: Vector3,
+): Saga<void> {
   const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
   if (volumeTracing == null) return;
 
-  const layerName = volumeTracingLayer.tracingId;
-
   const activeMappingByLayer = yield* select(
     (state) => state.temporaryConfiguration.activeMappingByLayer,
   );
@@ -94,14 +94,14 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<any> {
   const oldPreferredQuality = yield* select(
     (state) => state.temporaryConfiguration.preferredQualityForMeshAdHocComputation,
   );
-  const resolutionInfo = getResolutionInfo(volumeTracingLayer.resolutions);
+
   const resolutionIndices = resolutionInfo.getAllIndices();
   const coarseResolutionIndex =
     resolutionIndices[Math.min(proofreadCoarseResolutionIndex(), resolutionIndices.length - 1)];
   yield* put(
     updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", coarseResolutionIndex),
   );
-  const segmentId = getSegmentIdForPosition(position);
+
   // Use the data store if the mapping is not editable yet. If it, is request the mesh from the tracing store.
   const useDataStore = !volumeTracing.mappingIsEditable;
   yield* put(
@@ -115,6 +115,23 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<any> {
   yield* put(
     updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", oldPreferredQuality),
   );
+}
+
+function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
+  const { position } = action;
+  const treeName = yield* call(loadAgglomerateSkeletonAtPosition, position);
+
+  if (!proofreadUsingMeshes()) return;
+
+  const volumeTracingLayer = yield* select((state) => getActiveSegmentationTracingLayer(state));
+  if (volumeTracingLayer == null || volumeTracingLayer.tracingId == null) return;
+
+  const resolutionInfo = getResolutionInfo(volumeTracingLayer.resolutions);
+
+  const layerName = volumeTracingLayer.tracingId;
+  const segmentId = getSegmentIdForPosition(position);
+
+  yield* call(loadCoarseAdHocMesh, layerName, resolutionInfo, segmentId, position);
 
   if (treeName == null) return;
 
@@ -160,12 +177,17 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<any> {
     );
   }
 
+  oldSegmentIdsInSurround = segmentIdsInSurround;
+
   // Load meshes in oversegmentation in fine resolution
   const noMappingInfo = {
     mappingName: null,
     mappingType: null,
     useDataStore: true,
   };
+  const oldPreferredQuality = yield* select(
+    (state) => state.temporaryConfiguration.preferredQualityForMeshAdHocComputation,
+  );
   yield* put(
     updateTemporarySettingAction(
       "preferredQualityForMeshAdHocComputation",
@@ -184,8 +206,9 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<any> {
       ),
     ),
   );
-
-  oldSegmentIdsInSurround = segmentIdsInSurround;
+  yield* put(
+    updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", oldPreferredQuality),
+  );
 }
 
 function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
@@ -304,4 +327,53 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
   yield* call([Model, Model.ensureSavedState]);
 
   yield* call([api.data, api.data.reloadBuckets], layerName);
+
+  if (proofreadUsingMeshes()) {
+    // Remove old over segmentation meshes
+    if (oldSegmentIdsInSurround != null) {
+      // Remove old meshes in oversegmentation
+      yield* all(
+        oldSegmentIdsInSurround.map((nodeSegmentId) =>
+          put(removeIsosurfaceAction(layerName, nodeSegmentId)),
+        ),
+      );
+      oldSegmentIdsInSurround = null;
+    }
+
+    // Remove old agglomerate mesh(es) and load new agglomerate mesh(es)
+    yield* put(removeIsosurfaceAction(layerName, sourceNodeAgglomerateId));
+    if (targetNodeAgglomerateId !== sourceNodeAgglomerateId) {
+      yield* put(removeIsosurfaceAction(layerName, targetNodeAgglomerateId));
+    } else {
+      const newTargetNodeAgglomerateId = yield* call(
+        [api.data, api.data.getDataValue],
+        layerName,
+        targetNodePosition,
+        agglomerateFileZoomstep,
+      );
+
+      yield* call(
+        loadCoarseAdHocMesh,
+        layerName,
+        resolutionInfo,
+        newTargetNodeAgglomerateId,
+        targetNodePosition,
+      );
+    }
+
+    const newSourceNodeAgglomerateId = yield* call(
+      [api.data, api.data.getDataValue],
+      layerName,
+      sourceNodePosition,
+      agglomerateFileZoomstep,
+    );
+
+    yield* call(
+      loadCoarseAdHocMesh,
+      layerName,
+      resolutionInfo,
+      newSourceNodeAgglomerateId,
+      sourceNodePosition,
+    );
+  }
 }

From 547c15d0fd532e2df5dfc9948aaaca907ef65200 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 24 May 2022 17:57:32 +0200
Subject: [PATCH 066/122] enable context menu in 3d view

---
 .../combinations/skeleton_handlers.ts         |  4 ----
 .../oxalis/controller/td_controller.tsx       | 22 ++++++++++++++++++-
 .../controller/viewmodes/plane_controller.tsx |  1 +
 .../javascripts/oxalis/view/context_menu.tsx  | 11 ++++------
 4 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/frontend/javascripts/oxalis/controller/combinations/skeleton_handlers.ts b/frontend/javascripts/oxalis/controller/combinations/skeleton_handlers.ts
index c91f111b223..048859c880a 100644
--- a/frontend/javascripts/oxalis/controller/combinations/skeleton_handlers.ts
+++ b/frontend/javascripts/oxalis/controller/combinations/skeleton_handlers.ts
@@ -132,10 +132,6 @@ export function handleOpenContextMenu(
 ) {
   const { activeViewport } = Store.getState().viewModeData.plane;
 
-  if (activeViewport === OrthoViews.TDView) {
-    return;
-  }
-
   const nodeId = maybeGetNodeIdFromPosition(planeView, position, plane, isTouch);
   const state = Store.getState();
   const globalPosition = calculateGlobalPos(state, position);
diff --git a/frontend/javascripts/oxalis/controller/td_controller.tsx b/frontend/javascripts/oxalis/controller/td_controller.tsx
index 252d559691f..46dc263d5ff 100644
--- a/frontend/javascripts/oxalis/controller/td_controller.tsx
+++ b/frontend/javascripts/oxalis/controller/td_controller.tsx
@@ -2,7 +2,13 @@ import { connect } from "react-redux";
 import * as React from "react";
 import * as THREE from "three";
 import { InputMouse } from "libs/input";
-import type { OrthoView, OrthoViewMap, Point2, Vector3 } from "oxalis/constants";
+import type {
+  OrthoView,
+  OrthoViewMap,
+  Point2,
+  ShowContextMenuFunction,
+  Vector3,
+} from "oxalis/constants";
 import { OrthoViews } from "oxalis/constants";
 import { V3 } from "libs/mjs";
 import { getPosition } from "oxalis/model/accessors/flycam_accessor";
@@ -28,6 +34,7 @@ import TrackballControls from "libs/trackball_controls";
 import * as Utils from "libs/utils";
 import { removeIsosurfaceAction } from "oxalis/model/actions/annotation_actions";
 import { SkeletonTool } from "oxalis/controller/combinations/tool_controls";
+import { handleOpenContextMenu } from "oxalis/controller/combinations/skeleton_handlers";
 
 export function threeCameraToCameraData(camera: THREE.OrthographicCamera): CameraData {
   const { position, up, near, far, lookAt, left, right, top, bottom } = camera;
@@ -68,6 +75,7 @@ type OwnProps = {
   cameras: OrthoViewMap<THREE.OrthographicCamera>;
   planeView?: PlaneView;
   tracing?: Tracing;
+  showContextMenuAt?: ShowContextMenuFunction;
 };
 type StateProps = {
   flycam: Flycam;
@@ -225,6 +233,18 @@ class TDController extends React.PureComponent<Props> {
           Store.dispatch(removeIsosurfaceAction(segmentationLayer.name, hoveredSegmentId));
         }
       },
+      rightClick: (pos: Point2, plane: OrthoView, event: MouseEvent, isTouch: boolean) => {
+        if (this.props.planeView == null || this.props.showContextMenuAt == null) return;
+
+        handleOpenContextMenu(
+          this.props.planeView,
+          pos,
+          plane,
+          isTouch,
+          event,
+          this.props.showContextMenuAt,
+        );
+      },
     };
     return controls;
   }
diff --git a/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx b/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx
index 04f0826e7a3..b028bed31ef 100644
--- a/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx
+++ b/frontend/javascripts/oxalis/controller/viewmodes/plane_controller.tsx
@@ -599,6 +599,7 @@ class PlaneController extends React.PureComponent<Props> {
         cameras={this.planeView.getCameras()}
         tracing={this.props.tracing}
         planeView={this.planeView}
+        showContextMenuAt={this.props.showContextMenuAt}
       />
     );
   }
diff --git a/frontend/javascripts/oxalis/view/context_menu.tsx b/frontend/javascripts/oxalis/view/context_menu.tsx
index 7cc2cf70c11..6dedd115f33 100644
--- a/frontend/javascripts/oxalis/view/context_menu.tsx
+++ b/frontend/javascripts/oxalis/view/context_menu.tsx
@@ -18,7 +18,7 @@ import type {
   UserBoundingBox,
   VolumeTracing,
 } from "oxalis/store";
-import type { AnnotationTool, Vector3, OrthoView } from "oxalis/constants";
+import { AnnotationTool, Vector3, OrthoView, OrthoViews } from "oxalis/constants";
 import { AnnotationToolEnum, VolumeTools } from "oxalis/constants";
 import { V3 } from "libs/mjs";
 import {
@@ -788,21 +788,18 @@ function NoNodeContextMenuOptions(props: NoNodeContextMenuProps) {
   }
 
   const isSkeletonToolActive = activeTool === AnnotationToolEnum.SKELETON;
-  let allActions = [];
+  const isTdViewport = viewport === OrthoViews.TDView;
+  let allActions: Array<JSX.Element | null> = [];
 
   if (isSkeletonToolActive) {
     // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
     allActions = skeletonActions.concat(nonSkeletonActions).concat(boundingBoxActions);
   } else if (isBoundingBoxToolActive) {
     allActions = boundingBoxActions.concat(nonSkeletonActions).concat(skeletonActions);
-  } else {
+  } else if (!isTdViewport) {
     allActions = nonSkeletonActions.concat(skeletonActions).concat(boundingBoxActions);
   }
 
-  if (allActions.length === 0) {
-    return null;
-  }
-
   return (
     <Menu
       onClick={hideContextMenu}

From d5c5577ff654f1a26835b0308f7521733ee800dd Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 24 May 2022 20:06:09 +0200
Subject: [PATCH 067/122] fix loading of segments on first click, properly
 reload agglomerate skeletons after proofreading action

---
 .../oxalis/model/sagas/proofread_saga.ts      | 92 ++++++++++++++-----
 .../model/sagas/skeletontracing_saga.ts       | 13 ++-
 2 files changed, 77 insertions(+), 28 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 87e50105a7d..1c47c42754e 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -3,8 +3,10 @@ import { takeEvery, put, call, all } from "typed-redux-saga";
 import { select, take } from "oxalis/model/sagas/effect-generators";
 import { AnnotationToolEnum, MappingStatusEnum, Vector3 } from "oxalis/constants";
 import Toast from "libs/toast";
-import type {
+import {
   DeleteEdgeAction,
+  deleteTreeAction,
+  loadAgglomerateSkeletonAction,
   MergeTreesAction,
 } from "oxalis/model/actions/skeletontracing_actions";
 import {
@@ -38,11 +40,11 @@ import {
   setMappingNameAction,
   updateTemporarySettingAction,
 } from "oxalis/model/actions/settings_actions";
-import { loadAgglomerateSkeletonAtPosition } from "oxalis/controller/combinations/segmentation_handlers";
 import { getSegmentIdForPosition } from "oxalis/controller/combinations/volume_handlers";
 import { loadAdHocMeshAction } from "oxalis/model/actions/segmentation_actions";
 import { V3 } from "libs/mjs";
 import { removeIsosurfaceAction } from "oxalis/model/actions/annotation_actions";
+import { loadAgglomerateSkeletonWithId } from "oxalis/model/sagas/skeletontracing_saga";
 
 export default function* proofreadMapping(): Saga<any> {
   yield* take("INITIALIZE_SKELETONTRACING");
@@ -119,17 +121,26 @@ function* loadCoarseAdHocMesh(
 
 function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
   const { position } = action;
-  const treeName = yield* call(loadAgglomerateSkeletonAtPosition, position);
-
-  if (!proofreadUsingMeshes()) return;
 
   const volumeTracingLayer = yield* select((state) => getActiveSegmentationTracingLayer(state));
   if (volumeTracingLayer == null || volumeTracingLayer.tracingId == null) return;
+  const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
+  if (volumeTracing == null) return;
 
   const resolutionInfo = getResolutionInfo(volumeTracingLayer.resolutions);
 
   const layerName = volumeTracingLayer.tracingId;
   const segmentId = getSegmentIdForPosition(position);
+  const { mappingName } = volumeTracing;
+
+  if (mappingName == null) return;
+
+  const treeName = yield* call(
+    loadAgglomerateSkeletonWithId,
+    loadAgglomerateSkeletonAction(layerName, mappingName, segmentId),
+  );
+
+  if (!proofreadUsingMeshes()) return;
 
   yield* call(loadCoarseAdHocMesh, layerName, resolutionInfo, segmentId, position);
 
@@ -329,6 +340,15 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
   yield* call([api.data, api.data.reloadBuckets], layerName);
 
   if (proofreadUsingMeshes()) {
+    const volumeTracingWithEditableMapping = yield* select((state) =>
+      getActiveSegmentationTracing(state),
+    );
+    if (
+      volumeTracingWithEditableMapping == null ||
+      volumeTracingWithEditableMapping.mappingName == null
+    )
+      return;
+
     // Remove old over segmentation meshes
     if (oldSegmentIdsInSurround != null) {
       // Remove old meshes in oversegmentation
@@ -340,18 +360,34 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
       oldSegmentIdsInSurround = null;
     }
 
+    const newSourceNodeAgglomerateId = yield* call(
+      [api.data, api.data.getDataValue],
+      layerName,
+      sourceNodePosition,
+      agglomerateFileZoomstep,
+    );
+
+    const newTargetNodeAgglomerateId = yield* call(
+      [api.data, api.data.getDataValue],
+      layerName,
+      targetNodePosition,
+      agglomerateFileZoomstep,
+    );
+
     // Remove old agglomerate mesh(es) and load new agglomerate mesh(es)
     yield* put(removeIsosurfaceAction(layerName, sourceNodeAgglomerateId));
     if (targetNodeAgglomerateId !== sourceNodeAgglomerateId) {
       yield* put(removeIsosurfaceAction(layerName, targetNodeAgglomerateId));
-    } else {
-      const newTargetNodeAgglomerateId = yield* call(
-        [api.data, api.data.getDataValue],
-        layerName,
-        targetNodePosition,
-        agglomerateFileZoomstep,
-      );
+    }
 
+    yield* call(
+      loadCoarseAdHocMesh,
+      layerName,
+      resolutionInfo,
+      newSourceNodeAgglomerateId,
+      sourceNodePosition,
+    );
+    if (newTargetNodeAgglomerateId !== newSourceNodeAgglomerateId) {
       yield* call(
         loadCoarseAdHocMesh,
         layerName,
@@ -361,19 +397,29 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
       );
     }
 
-    const newSourceNodeAgglomerateId = yield* call(
-      [api.data, api.data.getDataValue],
-      layerName,
-      sourceNodePosition,
-      agglomerateFileZoomstep,
-    );
+    // Remove old agglomerate skeleton(s) and load new agglomerate skeleton(s)
+    yield* put(deleteTreeAction(sourceTree.treeId));
+    if (sourceTree !== targetTree) {
+      yield* put(deleteTreeAction(targetTree.treeId));
+    }
 
     yield* call(
-      loadCoarseAdHocMesh,
-      layerName,
-      resolutionInfo,
-      newSourceNodeAgglomerateId,
-      sourceNodePosition,
+      loadAgglomerateSkeletonWithId,
+      loadAgglomerateSkeletonAction(
+        layerName,
+        volumeTracingWithEditableMapping.mappingName,
+        newSourceNodeAgglomerateId,
+      ),
     );
+    if (newTargetNodeAgglomerateId !== newSourceNodeAgglomerateId) {
+      yield* call(
+        loadAgglomerateSkeletonWithId,
+        loadAgglomerateSkeletonAction(
+          layerName,
+          volumeTracingWithEditableMapping.mappingName,
+          newTargetNodeAgglomerateId,
+        ),
+      );
+    }
   }
 }
diff --git a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
index 96654aa9266..7dae5802ff2 100644
--- a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
@@ -315,14 +315,16 @@ function handleAgglomerateLoadingError(
   ErrorHandling.notify(e);
 }
 
-function* loadAgglomerateSkeletonWithId(action: LoadAgglomerateSkeletonAction): Saga<void> {
+export function* loadAgglomerateSkeletonWithId(
+  action: LoadAgglomerateSkeletonAction,
+): Saga<string | null> {
   const allowUpdate = yield* select((state) => state.tracing.restrictions.allowUpdate);
-  if (!allowUpdate) return;
+  if (!allowUpdate) return null;
   const { layerName, mappingName, agglomerateId } = action;
 
   if (agglomerateId === 0) {
     Toast.error(messages["tracing.agglomerate_skeleton.no_cell"]);
-    return;
+    return null;
   }
 
   const treeName = getTreeNameForAgglomerateSkeleton(agglomerateId, mappingName);
@@ -333,7 +335,7 @@ function* loadAgglomerateSkeletonWithId(action: LoadAgglomerateSkeletonAction):
     console.warn(
       `Skeleton for agglomerate ${agglomerateId} with mapping ${mappingName} is already loaded. Its tree name is "${treeName}".`,
     );
-    return;
+    return treeName;
   }
 
   const progressCallback = createProgressCallback({
@@ -364,10 +366,11 @@ function* loadAgglomerateSkeletonWithId(action: LoadAgglomerateSkeletonAction):
     hideFn();
     // @ts-ignore
     handleAgglomerateLoadingError(e);
-    return;
+    return null;
   }
 
   yield* call(progressCallback, true, "Skeleton generation done.");
+  return treeName;
 }
 
 function* loadConnectomeAgglomerateSkeletonWithId(

From 38f8390990c996b4c51c3a470537c661d2f25d0a Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 25 May 2022 10:08:13 +0200
Subject: [PATCH 068/122] fix bucket position access

---
 .../controllers/BinaryDataController.scala    |  10 +-
 .../controllers/ZarrStreamingController.scala |   8 +-
 .../dataformats/wkw/WKWBucketProvider.scala   |   6 +-
 .../dataformats/wkw/WKWDataFormatHelper.scala |   2 +-
 .../dataformats/zarr/ZarrBucketProvider.scala |   2 +-
 .../datastore/models/DataRequests.scala       |   4 +-
 .../datastore/models/ImageThumbnail.scala     |   2 +-
 .../datastore/models/Positions.scala          | 113 ++++++++----------
 .../models/datasource/DataLayer.scala         |   2 +-
 .../datastore/models/requests/Cuboid.scala    |   6 +-
 .../services/BinaryDataService.scala          |  39 +++---
 .../datastore/services/FindDataService.scala  |   2 +-
 .../services/IsosurfaceService.scala          |   5 +-
 .../storage/AgglomerateFileCache.scala        |  22 ++--
 .../datastore/storage/DataCubeCache.scala     |   2 +-
 .../EditableMappingLayer.scala                |   5 +-
 .../volume/VolumeTracingBucketHelper.scala    |  10 +-
 .../volume/VolumeTracingDownsampling.scala    |  12 +-
 .../volume/VolumeTracingService.scala         |   2 +-
 19 files changed, 127 insertions(+), 127 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala
index fe1296f1926..ff8cd1d5f47 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala
@@ -119,7 +119,7 @@ class BinaryDataController @Inject()(
         magParsedOpt <- Fox.runOptional(mag)(Vec3Int.fromMagLiteral(_).toFox)
         magParsed <- magParsedOpt.orElse(magFromZoomStep).toFox ?~> "No mag supplied"
         request = DataRequest(
-          new VoxelPosition(x, y, z, magParsed),
+          VoxelPosition(x, y, z, magParsed),
           width,
           height,
           depth,
@@ -150,10 +150,10 @@ class BinaryDataController @Inject()(
                                                                                   dataSetName,
                                                                                   dataLayerName) ~> 404
         request = DataRequest(
-          new VoxelPosition(x * cubeSize * resolution,
-                            y * cubeSize * resolution,
-                            z * cubeSize * resolution,
-                            Vec3Int(resolution, resolution, resolution)),
+          VoxelPosition(x * cubeSize * resolution,
+                        y * cubeSize * resolution,
+                        z * cubeSize * resolution,
+                        Vec3Int(resolution, resolution, resolution)),
           cubeSize,
           cubeSize,
           cubeSize
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala
index 40b63d41ebf..fc44cf22a10 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala
@@ -167,10 +167,10 @@ class ZarrStreamingController @Inject()(
           dataLayer,
           None,
           Cuboid(
-            topLeft = new VoxelPosition(x * cubeSize * magParsed.x,
-                                        y * cubeSize * magParsed.y,
-                                        z * cubeSize * magParsed.z,
-                                        magParsed),
+            topLeft = VoxelPosition(x * cubeSize * magParsed.x,
+                                    y * cubeSize * magParsed.y,
+                                    z * cubeSize * magParsed.z,
+                                    magParsed),
             width = cubeSize,
             height = cubeSize,
             depth = cubeSize
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWBucketProvider.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWBucketProvider.scala
index 8e3e45c288d..a376880cc06 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWBucketProvider.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWBucketProvider.scala
@@ -14,9 +14,9 @@ class WKWCubeHandle(wkwFile: WKWFile) extends DataCubeHandle with FoxImplicits {
 
   def cutOutBucket(bucket: BucketPosition)(implicit ec: ExecutionContext): Fox[Array[Byte]] = {
     val numBlocksPerCubeDimension = wkwFile.header.numBlocksPerCubeDimension
-    val blockOffsetX = bucket.x % numBlocksPerCubeDimension
-    val blockOffsetY = bucket.y % numBlocksPerCubeDimension
-    val blockOffsetZ = bucket.z % numBlocksPerCubeDimension
+    val blockOffsetX = bucket.bucketX % numBlocksPerCubeDimension
+    val blockOffsetY = bucket.bucketY % numBlocksPerCubeDimension
+    val blockOffsetZ = bucket.bucketZ % numBlocksPerCubeDimension
     Fox(Future.successful(wkwFile.readBlock(blockOffsetX, blockOffsetY, blockOffsetZ)))
   }
 
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWDataFormatHelper.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWDataFormatHelper.scala
index 48dd7cdcb7c..7a44ca95df6 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWDataFormatHelper.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWDataFormatHelper.scala
@@ -23,7 +23,7 @@ trait WKWDataFormatHelper {
       .resolve(dataSourceId.map(_.team).getOrElse(""))
       .resolve(dataSourceId.map(_.name).getOrElse(""))
       .resolve(dataLayerName.getOrElse(""))
-      .resolve(formatResolution(cube.resolution, resolutionAsTriple))
+      .resolve(formatResolution(cube.mag, resolutionAsTriple))
       .resolve(s"z${cube.z}")
       .resolve(s"y${cube.y}")
       .resolve(s"x${cube.x}.$dataFileExtension")
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrBucketProvider.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrBucketProvider.scala
index 0a94c0de49c..0ea0bffceb2 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrBucketProvider.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrBucketProvider.scala
@@ -20,7 +20,7 @@ class ZarrCubeHandle(zarrArray: ZarrArray) extends DataCubeHandle with LazyLoggi
 
   def cutOutBucket(bucket: BucketPosition)(implicit ec: ExecutionContext): Fox[Array[Byte]] = {
     val shape = Vec3Int.full(bucket.bucketLength)
-    val offset = Vec3Int(bucket.globalXInMag, bucket.globalYInMag, bucket.globalZInMag)
+    val offset = Vec3Int(bucket.voxelXInMag, bucket.voxelYInMag, bucket.voxelZInMag)
     zarrArray.readBytesXYZ(shape, offset).recover {
       case t: Throwable => logError(t); return Failure(t.getMessage, Full(t), Empty)
     }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/DataRequests.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/DataRequests.scala
index 798d4891600..b96f55ed533 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/DataRequests.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/DataRequests.scala
@@ -33,7 +33,7 @@ case class WebKnossosDataRequest(
 ) extends AbstractDataRequest {
 
   def cuboid(dataLayer: DataLayer): Cuboid =
-    Cuboid(new VoxelPosition(position.x, position.y, position.z, mag), cubeSize, cubeSize, cubeSize)
+    Cuboid(VoxelPosition(position.x, position.y, position.z, mag), cubeSize, cubeSize, cubeSize)
 
   def settings: DataServiceRequestSettings =
     DataServiceRequestSettings(halfByte = fourBit.getOrElse(false), applyAgglomerate, version)
@@ -54,7 +54,7 @@ case class WebKnossosIsosurfaceRequest(
     mappingType: Option[String] = None
 ) {
   def cuboid(dataLayer: DataLayer): Cuboid =
-    Cuboid(new VoxelPosition(position.x, position.y, position.z, mag), cubeSize.x, cubeSize.y, cubeSize.z)
+    Cuboid(VoxelPosition(position.x, position.y, position.z, mag), cubeSize.x, cubeSize.y, cubeSize.z)
 }
 
 object WebKnossosIsosurfaceRequest {
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/ImageThumbnail.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/ImageThumbnail.scala
index a0c94eded6e..578d04c8a34 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/ImageThumbnail.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/ImageThumbnail.scala
@@ -27,6 +27,6 @@ object ImageThumbnail {
     val x = Math.max(0, center.x - width * mag.x / 2)
     val y = Math.max(0, center.y - height * mag.y / 2)
     val z = center.z
-    new VoxelPosition(x.toInt, y.toInt, z.toInt, mag)
+    VoxelPosition(x.toInt, y.toInt, z.toInt, mag)
   }
 }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/Positions.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/Positions.scala
index 24af607a4f9..03821bb2efb 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/Positions.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/Positions.scala
@@ -4,131 +4,122 @@ import com.scalableminds.webknossos.datastore.models.datasource.DataLayer
 import com.scalableminds.util.geometry.{BoundingBox, Vec3Int}
 import org.apache.commons.lang3.builder.HashCodeBuilder
 
-trait GenericPosition {
-  def x: Int
-  def y: Int
-  def z: Int
-}
-
-class VoxelPosition(
-    protected val globalX: Int,
-    protected val globalY: Int,
-    protected val globalZ: Int,
-    val mag: Vec3Int
-) extends GenericPosition {
+case class VoxelPosition(
+    mag1X: Int,
+    mag1Y: Int,
+    mag1Z: Int,
+    mag: Vec3Int
+) {
 
-  val x: Int = globalX / mag.x
+  val voxelXInMag: Int = mag1X / mag.x
 
-  val y: Int = globalY / mag.y
+  val voxelYInMag: Int = mag1Y / mag.y
 
-  val z: Int = globalZ / mag.z
+  val voxelZInMag: Int = mag1Z / mag.z
 
   def toBucket: BucketPosition =
-    BucketPosition(globalX, globalY, globalZ, mag)
+    BucketPosition(mag1X, mag1Y, mag1Z, mag)
 
   def move(dx: Int, dy: Int, dz: Int) =
-    new VoxelPosition(globalX + dx, globalY + dy, globalZ + dz, mag)
+    VoxelPosition(mag1X + dx, mag1Y + dy, mag1Z + dz, mag)
 
-  override def toString = s"($globalX, $globalY, $globalZ) / $mag"
+  override def toString = s"($mag1X, $mag1Y, $mag1Z) / $mag"
 
   override def equals(obj: scala.Any): Boolean =
     obj match {
       case other: VoxelPosition =>
-        other.globalX == globalX &&
-          other.globalY == globalY &&
-          other.globalZ == globalZ &&
+        other.mag1X == mag1X &&
+          other.mag1Y == mag1Y &&
+          other.mag1Z == mag1Z &&
           other.mag == mag
       case _ =>
         false
     }
 
   override def hashCode(): Int =
-    new HashCodeBuilder(17, 31).append(globalX).append(globalY).append(globalZ).append(mag).toHashCode
+    new HashCodeBuilder(17, 31).append(mag1X).append(mag1Y).append(mag1Z).append(mag).toHashCode
 }
 
 case class BucketPosition(
-    globalX: Int,
-    globalY: Int,
-    globalZ: Int,
+    voxelMag1X: Int,
+    voxelMag1Y: Int,
+    voxelMag1Z: Int,
     mag: Vec3Int
-) extends GenericPosition {
+) {
 
   val bucketLength: Int = DataLayer.bucketLength
 
-  val x: Int = globalX / bucketLength / mag.x
+  val bucketX: Int = voxelMag1X / bucketLength / mag.x
 
-  val y: Int = globalY / bucketLength / mag.y
+  val bucketY: Int = voxelMag1Y / bucketLength / mag.y
 
-  val z: Int = globalZ / bucketLength / mag.z
+  val bucketZ: Int = voxelMag1Z / bucketLength / mag.z
 
-  val globalXInMag: Int = globalX / mag.x
+  val voxelXInMag: Int = voxelMag1X / mag.x
 
-  val globalYInMag: Int = globalY / mag.y
+  val voxelYInMag: Int = voxelMag1Y / mag.y
 
-  val globalZInMag: Int = globalZ / mag.z
+  val voxelZInMag: Int = voxelMag1Z / mag.z
 
   def volume: Int = bucketLength * bucketLength * bucketLength
 
   def toCube(cubeLength: Int): CubePosition =
-    new CubePosition(globalX, globalY, globalZ, mag, cubeLength)
+    new CubePosition(voxelMag1X, voxelMag1Y, voxelMag1Z, mag, cubeLength)
 
   def topLeft: VoxelPosition = {
-    val tlx: Int = globalX - globalX % (bucketLength * mag.x)
-    val tly: Int = globalY - globalY % (bucketLength * mag.y)
-    val tlz: Int = globalZ - globalZ % (bucketLength * mag.z)
+    val tlx: Int = voxelMag1X - Math.floorMod(voxelMag1X, bucketLength * mag.x)
+    val tly: Int = voxelMag1Y - Math.floorMod(voxelMag1Y, bucketLength * mag.y)
+    val tlz: Int = voxelMag1Z - Math.floorMod(voxelMag1Z, bucketLength * mag.z)
 
-    new VoxelPosition(tlx, tly, tlz, mag)
+    VoxelPosition(tlx, tly, tlz, mag)
   }
 
   def nextBucketInX: BucketPosition =
-    BucketPosition(globalX + (bucketLength * mag.x), globalY, globalZ, mag)
+    BucketPosition(voxelMag1X + (bucketLength * mag.x), voxelMag1Y, voxelMag1Z, mag)
 
   def nextBucketInY: BucketPosition =
-    BucketPosition(globalX, globalY + (bucketLength * mag.y), globalZ, mag)
+    BucketPosition(voxelMag1X, voxelMag1Y + (bucketLength * mag.y), voxelMag1Z, mag)
 
   def nextBucketInZ: BucketPosition =
-    BucketPosition(globalX, globalY, globalZ + (bucketLength * mag.z), mag)
+    BucketPosition(voxelMag1X, voxelMag1Y, voxelMag1Z + (bucketLength * mag.z), mag)
 
-  def toHighestResBoundingBox: BoundingBox =
+  def toMag1BoundingBox: BoundingBox =
     new BoundingBox(
-      Vec3Int(topLeft.x * mag.x, topLeft.y * mag.y, topLeft.z * mag.z),
+      Vec3Int(topLeft.mag1X, topLeft.mag1Y, topLeft.mag1Z),
       bucketLength * mag.x,
       bucketLength * mag.y,
       bucketLength * mag.z
     )
 
   override def toString: String =
-    s"BucketPosition($globalX, $globalY, $globalZ, mag$mag)"
+    s"BucketPosition(voxelMag1 at ($voxelMag1X, $voxelMag1Y, $voxelMag1Z), bucket at ($bucketX,$bucketY,$bucketZ), mag$mag)"
 }
 
 class CubePosition(
-    protected val globalX: Int,
-    protected val globalY: Int,
-    protected val globalZ: Int,
-    val resolution: Vec3Int,
+    protected val mag1X: Int,
+    protected val mag1Y: Int,
+    protected val mag1Z: Int,
+    val mag: Vec3Int,
     val cubeLength: Int
-) extends GenericPosition {
+) {
 
-  val x: Int = globalX / cubeLength / resolution.x
+  val x: Int = mag1X / cubeLength / mag.x
 
-  val y: Int = globalY / cubeLength / resolution.y
+  val y: Int = mag1Y / cubeLength / mag.y
 
-  val z: Int = globalZ / cubeLength / resolution.z
+  val z: Int = mag1Z / cubeLength / mag.z
 
   def topLeft: VoxelPosition = {
-    val tlx: Int = globalX - globalX % (cubeLength * resolution.x)
-    val tly: Int = globalY - globalY % (cubeLength * resolution.y)
-    val tlz: Int = globalZ - globalZ % (cubeLength * resolution.z)
+    val tlx: Int = mag1X - mag1X % (cubeLength * mag.x)
+    val tly: Int = mag1Y - mag1Y % (cubeLength * mag.y)
+    val tlz: Int = mag1Z - mag1Z % (cubeLength * mag.z)
 
-    new VoxelPosition(tlx, tly, tlz, resolution)
+    VoxelPosition(tlx, tly, tlz, mag)
   }
 
-  def toHighestResBoundingBox: BoundingBox =
-    new BoundingBox(Vec3Int(globalX, globalY, globalZ),
-                    cubeLength * resolution.x,
-                    cubeLength * resolution.y,
-                    cubeLength * resolution.z)
+  def toMag1BoundingBox: BoundingBox =
+    new BoundingBox(Vec3Int(mag1X, mag1Y, mag1Z), cubeLength * mag.x, cubeLength * mag.y, cubeLength * mag.z)
 
   override def toString: String =
-    s"CPos($x,$y,$z,res=$resolution)"
+    s"CubePos($x,$y,$z,res=$mag)"
 }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala
index a7cd02e6517..a16ca0fc46a 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala
@@ -132,7 +132,7 @@ trait DataLayer extends DataLayerLike {
   def containsResolution(resolution: Vec3Int): Boolean = resolutions.contains(resolution)
 
   def doesContainBucket(bucket: BucketPosition): Boolean =
-    boundingBox.intersects(bucket.toHighestResBoundingBox)
+    boundingBox.intersects(bucket.toMag1BoundingBox)
 
   lazy val bytesPerElement: Int =
     ElementClass.bytesPerElement(elementClass)
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/requests/Cuboid.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/requests/Cuboid.scala
index 34992d5a58c..1b201fc9935 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/requests/Cuboid.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/requests/Cuboid.scala
@@ -28,11 +28,11 @@ case class Cuboid(topLeft: VoxelPosition, width: Int, height: Int, depth: Int) {
     val minBucket = topLeft.toBucket
     var bucketList: List[BucketPosition] = Nil
     var bucket = minBucket
-    while (bucket.topLeft.x < bottomRight.x) {
+    while (bucket.topLeft.voxelXInMag < bottomRight.voxelXInMag) {
       val prevX = bucket
-      while (bucket.topLeft.y < bottomRight.y) {
+      while (bucket.topLeft.voxelYInMag < bottomRight.voxelYInMag) {
         val prevY = bucket
-        while (bucket.topLeft.z < bottomRight.z) {
+        while (bucket.topLeft.voxelZInMag < bottomRight.voxelZInMag) {
           bucketList ::= bucket
           bucket = bucket.nextBucketInZ
         }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
index aa0044d8a40..9d106638b9d 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
@@ -34,11 +34,11 @@ class BinaryDataService(val dataBaseDir: Path, maxCacheSize: Int, val agglomerat
         handleBucketRequest(request, bucket)
       }
     } else {
-      Fox.sequence {
-        bucketQueue.toList.map { bucket =>
+      Fox
+        .serialSequence(bucketQueue.toList) { bucket =>
           handleBucketRequest(request, bucket).map(r => bucket -> r)
         }
-      }.map(buckets => cutOutCuboid(request, buckets.flatten))
+        .map(buckets => cutOutCuboid(request, buckets.flatten))
     }
   }
 
@@ -102,23 +102,32 @@ class BinaryDataService(val dataBaseDir: Path, maxCacheSize: Int, val agglomerat
 
     rs.reverse.foreach {
       case (bucket, data) =>
-        val xRemainder = cuboid.topLeft.x % subsamplingStrides.x
-        val yRemainder = cuboid.topLeft.y % subsamplingStrides.y
-        val zRemainder = cuboid.topLeft.z % subsamplingStrides.z
+        val xRemainder = cuboid.topLeft.voxelXInMag % subsamplingStrides.x
+        val yRemainder = cuboid.topLeft.voxelYInMag % subsamplingStrides.y
+        val zRemainder = cuboid.topLeft.voxelZInMag % subsamplingStrides.z
 
         val xMin = math
-          .ceil((math.max(cuboid.topLeft.x, bucket.topLeft.x).toDouble - xRemainder) / subsamplingStrides.x.toDouble)
+          .ceil(
+            (math
+              .max(cuboid.topLeft.voxelXInMag, bucket.topLeft.voxelXInMag)
+              .toDouble - xRemainder) / subsamplingStrides.x.toDouble)
           .toInt * subsamplingStrides.x + xRemainder
         val yMin = math
-          .ceil((math.max(cuboid.topLeft.y, bucket.topLeft.y).toDouble - yRemainder) / subsamplingStrides.y.toDouble)
+          .ceil(
+            (math
+              .max(cuboid.topLeft.voxelYInMag, bucket.topLeft.voxelYInMag)
+              .toDouble - yRemainder) / subsamplingStrides.y.toDouble)
           .toInt * subsamplingStrides.y + yRemainder
         val zMin = math
-          .ceil((math.max(cuboid.topLeft.z, bucket.topLeft.z).toDouble - zRemainder) / subsamplingStrides.z.toDouble)
+          .ceil(
+            (math
+              .max(cuboid.topLeft.voxelZInMag, bucket.topLeft.voxelZInMag)
+              .toDouble - zRemainder) / subsamplingStrides.z.toDouble)
           .toInt * subsamplingStrides.z + zRemainder
 
-        val xMax = math.min(cuboid.bottomRight.x, bucket.topLeft.x + bucketLength)
-        val yMax = math.min(cuboid.bottomRight.y, bucket.topLeft.y + bucketLength)
-        val zMax = math.min(cuboid.bottomRight.z, bucket.topLeft.z + bucketLength)
+        val xMax = math.min(cuboid.bottomRight.voxelXInMag, bucket.topLeft.voxelXInMag + bucketLength)
+        val yMax = math.min(cuboid.bottomRight.voxelYInMag, bucket.topLeft.voxelYInMag + bucketLength)
+        val zMax = math.min(cuboid.bottomRight.voxelZInMag, bucket.topLeft.voxelZInMag + bucketLength)
 
         for {
           z <- zMin until zMax by subsamplingStrides.z
@@ -131,9 +140,9 @@ class BinaryDataService(val dataBaseDir: Path, maxCacheSize: Int, val agglomerat
               y % bucketLength * bucketLength +
               z % bucketLength * bucketLength * bucketLength) * bytesPerElement
 
-          val rx = (x - cuboid.topLeft.x) / subsamplingStrides.x
-          val ry = (y - cuboid.topLeft.y) / subsamplingStrides.y
-          val rz = (z - cuboid.topLeft.z) / subsamplingStrides.z
+          val rx = (x - cuboid.topLeft.voxelXInMag) / subsamplingStrides.x
+          val ry = (y - cuboid.topLeft.voxelYInMag) / subsamplingStrides.y
+          val rz = (z - cuboid.topLeft.voxelZInMag) / subsamplingStrides.z
 
           val resultOffset = (rx + ry * resultVolume.x + rz * resultVolume.x * resultVolume.y) * bytesPerElement
           if (subsamplingStrides.x == 1) {
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/FindDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/FindDataService.scala
index d1761abd8fd..d4ee627f150 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/FindDataService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/FindDataService.scala
@@ -27,7 +27,7 @@ class FindDataService @Inject()(dataServicesHolder: BinaryDataServiceHolder)(imp
                          position: Vec3Int,
                          resolution: Vec3Int): Fox[Array[Byte]] = {
     val request = DataRequest(
-      new VoxelPosition(position.x, position.y, position.z, resolution),
+      VoxelPosition(position.x, position.y, position.z, resolution),
       DataLayer.bucketLength,
       DataLayer.bucketLength,
       DataLayer.bucketLength
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
index a8a5fd26f91..1b88f5fb1b2 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
@@ -190,7 +190,7 @@ class IsosurfaceService(binaryDataService: BinaryDataService,
       math.ceil(cuboid.depth / subsamplingStrides.z).toInt
     )
 
-    val offset = Vec3Double(cuboid.topLeft.x, cuboid.topLeft.y, cuboid.topLeft.z)
+    val offset = Vec3Double(cuboid.topLeft.voxelXInMag, cuboid.topLeft.voxelYInMag, cuboid.topLeft.voxelZInMag)
     val scale = Vec3Double(cuboid.topLeft.mag) * request.scale
     val typedSegmentId = dataTypeFunctors.fromLong(request.segmentId)
 
@@ -203,6 +203,7 @@ class IsosurfaceService(binaryDataService: BinaryDataService,
       agglomerateMappedData = applyAgglomerate(data)
       typedData = convertData(agglomerateMappedData)
       mappedData <- applyMapping(typedData)
+      agglomerateIds: Set[T] = mappedData.toSet
       mappedSegmentId <- applyMapping(Array(typedSegmentId)).map(_.head)
       neighbors = findNeighbors(mappedData, dataDimensions, mappedSegmentId)
       afterPreprocessing = System.currentTimeMillis()
@@ -229,7 +230,7 @@ class IsosurfaceService(binaryDataService: BinaryDataService,
       }
       val afterMarchingCubes = System.currentTimeMillis()
       logger.info(
-        s"Isosurface generation timing - loading: ${afterLoading - before} ms, preprocessing: ${afterPreprocessing - afterLoading}, marchingCubes: ${afterMarchingCubes - afterPreprocessing}")
+        s"Isosurface generation timing - loading: ${afterLoading - before} ms, preprocessing: ${afterPreprocessing - afterLoading}, marchingCubes: ${afterMarchingCubes - afterPreprocessing}. ${agglomerateIds.size} agglomerates in data, ${typedData.length} voxels.")
       (vertexBuffer.flatMap(_.toList.map(_.toFloat)).toArray, neighbors)
     }
   }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala
index b99468f3a4a..b4794d56c10 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala
@@ -101,9 +101,9 @@ case class BoundingBoxFinder(
     zCoordinates: util.TreeSet[Long],
     minBoundingBox: (Long, Long, Long)) {
   def findInitialBoundingBox(cuboid: Cuboid): (Long, Long, Long) = {
-    val x = Option(xCoordinates.floor(cuboid.topLeft.x))
-    val y = Option(yCoordinates.floor(cuboid.topLeft.y))
-    val z = Option(zCoordinates.floor(cuboid.topLeft.z))
+    val x = Option(xCoordinates.floor(cuboid.topLeft.voxelXInMag))
+    val y = Option(yCoordinates.floor(cuboid.topLeft.voxelYInMag))
+    val z = Option(zCoordinates.floor(cuboid.topLeft.voxelZInMag))
     (x.getOrElse(minBoundingBox._1), y.getOrElse(minBoundingBox._2), z.getOrElse(minBoundingBox._3)) // if the request is outside the layer box, use the minimal bb as start point
   }
 }
@@ -121,10 +121,12 @@ class BoundingBoxCache(
   private def getGlobalCuboid(cuboid: Cuboid): Cuboid = {
     val res = cuboid.resolution
     val tl = cuboid.topLeft
-    Cuboid(new VoxelPosition(tl.x * res.x, tl.y * res.y, tl.z * res.z, Vec3Int(1, 1, 1)),
-           cuboid.width * res.x,
-           cuboid.height * res.y,
-           cuboid.depth * res.z)
+    Cuboid(
+      VoxelPosition(tl.voxelXInMag * res.x, tl.voxelYInMag * res.y, tl.voxelZInMag * res.z, Vec3Int(1, 1, 1)),
+      cuboid.width * res.x,
+      cuboid.height * res.y,
+      cuboid.depth * res.z
+    )
   }
 
   // get the segment ID range for one cuboid
@@ -149,11 +151,11 @@ class BoundingBoxCache(
     var z = initialBoundingBox._3
 
     // step through each bb, but save starting coordinates to reset iteration once the outer bound is reached
-    while (x < requestedCuboid.x && x < dataLayerBox.x) {
+    while (x < requestedCuboid.voxelXInMag && x < dataLayerBox.x) {
       val nextBBinX = (x + currDimensions._1, y, z)
-      while (y < requestedCuboid.y && y < dataLayerBox.y) {
+      while (y < requestedCuboid.voxelYInMag && y < dataLayerBox.y) {
         val nextBBinY = (x, y + currDimensions._2, z)
-        while (z < requestedCuboid.z && z < dataLayerBox.z) {
+        while (z < requestedCuboid.voxelZInMag && z < dataLayerBox.z) {
           // get cached values for current bb and update the reader range by extending if necessary
           cache.get((x, y, z)).foreach { value =>
             range = (min(range._1, value.idRange._1), max(range._2, value.idRange._2))
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/DataCubeCache.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/DataCubeCache.scala
index 07d7c47bef0..3ad10d92edc 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/DataCubeCache.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/DataCubeCache.scala
@@ -27,7 +27,7 @@ object CachedCube {
       loadInstruction.dataSource.id.team,
       loadInstruction.dataSource.id.name,
       loadInstruction.dataLayer.name,
-      loadInstruction.cube.resolution,
+      loadInstruction.cube.mag,
       loadInstruction.cube.x,
       loadInstruction.cube.y,
       loadInstruction.cube.z
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
index 14c8195b9ed..76e66aeb3d6 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
@@ -25,7 +25,7 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP
       editableMapping <- layer.editableMappingService.get(editableMappingId, remoteFallbackLayer, layer.token)
       afterGet = System.currentTimeMillis()
       dataRequest: WebKnossosDataRequest = WebKnossosDataRequest(
-        position = Vec3Int(bucket.topLeft.x, bucket.topLeft.y, bucket.topLeft.z),
+        position = Vec3Int(bucket.topLeft.mag1X, bucket.topLeft.mag1Y, bucket.topLeft.mag1Z),
         mag = bucket.mag,
         cubeSize = layer.lengthOfUnderlyingCubes(bucket.mag),
         fourBit = None,
@@ -50,8 +50,11 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP
                                                                       relevantMapping,
                                                                       layer.elementClass)
       afterMapData = System.currentTimeMillis()
+      /*
+      _ = logger.info(s"load bucket $bucket")
       _ = logger.info(
         s"load bucket timing: getMapping: ${afterGet - beforeGet} ms, getUnmapped: ${afterGetUnmapped - afterGet} ms, collectSegments: ${afterCollectSegmentIds - afterGet} ms, combine: ${afterCombineMapping - afterCollectSegmentIds}, mapData: ${afterMapData - afterCombineMapping}. Total ${afterMapData - beforeGet}. ${mappedData.length} bytes, ${segmentIds.size} segments")
+     */
     } yield mappedData
   }
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala
index 0729f9d0834..da59d3cb813 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala
@@ -70,16 +70,10 @@ trait VolumeBucketCompression extends LazyLogging {
 
 trait BucketKeys extends WKWMortonHelper with WKWDataFormatHelper with LazyLogging {
   protected def buildBucketKey(dataLayerName: String, bucket: BucketPosition): String = {
-    val mortonIndex = mortonEncode(bucket.x, bucket.y, bucket.z)
-    s"$dataLayerName/${formatResolution(bucket.mag)}/$mortonIndex-[${bucket.x},${bucket.y},${bucket.z}]"
+    val mortonIndex = mortonEncode(bucket.bucketX, bucket.bucketY, bucket.bucketZ)
+    s"$dataLayerName/${bucket.mag.toMagLiteral(allowScalar = true)}/$mortonIndex-[${bucket.bucketX},${bucket.bucketY},${bucket.bucketZ}]"
   }
 
-  protected def formatResolution(resolution: Vec3Int): String =
-    if (resolution.x == resolution.y && resolution.x == resolution.z)
-      s"${resolution.maxDim}"
-    else
-      s"${resolution.x}-${resolution.y}-${resolution.z}"
-
   protected def buildKeyPrefix(dataLayerName: String): String =
     s"$dataLayerName/"
 
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala
index 1c1aedbb3d2..56edf4e4ea6 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala
@@ -131,9 +131,9 @@ trait VolumeTracingDownsampling
                                          requiredMag: Vec3Int): Set[BucketPosition] =
     originalBucketPositions.map { bucketPosition: BucketPosition =>
       BucketPosition(
-        (bucketPosition.globalX / requiredMag.x / 32) * requiredMag.x * 32,
-        (bucketPosition.globalY / requiredMag.y / 32) * requiredMag.y * 32,
-        (bucketPosition.globalZ / requiredMag.z / 32) * requiredMag.z * 32,
+        (bucketPosition.voxelMag1X / requiredMag.x / 32) * requiredMag.x * 32,
+        (bucketPosition.voxelMag1Y / requiredMag.y / 32) * requiredMag.y * 32,
+        (bucketPosition.voxelMag1Z / requiredMag.z / 32) * requiredMag.z * 32,
         requiredMag
       )
     }.toSet
@@ -147,9 +147,9 @@ trait VolumeTracingDownsampling
       x <- 0 until downScaleFactor.x
     } yield {
       BucketPosition(
-        bucketPosition.globalX + x * bucketPosition.bucketLength * previousMag.x,
-        bucketPosition.globalY + y * bucketPosition.bucketLength * previousMag.y,
-        bucketPosition.globalZ + z * bucketPosition.bucketLength * previousMag.z,
+        bucketPosition.voxelMag1X + x * bucketPosition.bucketLength * previousMag.x,
+        bucketPosition.voxelMag1Y + y * bucketPosition.bucketLength * previousMag.y,
+        bucketPosition.voxelMag1Z + z * bucketPosition.bucketLength * previousMag.z,
         previousMag
       )
     }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
index 09bd42e0153..5a75a49dd38 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
@@ -366,7 +366,7 @@ class VolumeTracingService @Inject()(
         val bucket = bucketStream.next()
         val bucketPos = bucket._1
         getPositionOfNonZeroData(bucket._2,
-                                 Vec3Int(bucketPos.globalX, bucketPos.globalY, bucketPos.globalZ),
+                                 Vec3Int(bucketPos.voxelMag1X, bucketPos.voxelMag1Y, bucketPos.voxelMag1Z),
                                  volumeLayer.bytesPerElement)
       } else None
     } yield bucketPosOpt

From fb2c63419f54e1194f83fd2ab4243a8cecedb606 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 25 May 2022 13:11:57 +0200
Subject: [PATCH 069/122] parallelize buckets again for larger cuboids

---
 .../webknossos/datastore/services/BinaryDataService.scala   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
index 9d106638b9d..8c8930a4478 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
@@ -34,11 +34,11 @@ class BinaryDataService(val dataBaseDir: Path, maxCacheSize: Int, val agglomerat
         handleBucketRequest(request, bucket)
       }
     } else {
-      Fox
-        .serialSequence(bucketQueue.toList) { bucket =>
+      Fox.sequence {
+        bucketQueue.toList.map { bucket =>
           handleBucketRequest(request, bucket).map(r => bucket -> r)
         }
-        .map(buckets => cutOutCuboid(request, buckets.flatten))
+      }.map(buckets => cutOutCuboid(request, buckets.flatten))
     }
   }
 

From bfb92396b03b9b35f0335be0b75d7866b6c6d6b5 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 25 May 2022 13:49:44 +0200
Subject: [PATCH 070/122] reuse binary data service code for loading
 editably-mapped data

---
 .../EditableMappingLayer.scala                |  3 +-
 .../EditableMappingService.scala              | 63 ++++++++++---------
 2 files changed, 36 insertions(+), 30 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
index 76e66aeb3d6..ab490319c4d 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
@@ -32,9 +32,8 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP
         applyAgglomerate = None,
         version = None
       )
-      dataRequestCollection = List(dataRequest)
       (unmappedData, indices) <- layer.editableMappingService.getUnmappedDataFromDatastore(remoteFallbackLayer,
-                                                                                           dataRequestCollection,
+                                                                                           List(dataRequest),
                                                                                            layer.token)
       afterGetUnmapped = System.currentTimeMillis()
       _ <- bool2Fox(indices.isEmpty)
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index c0f08a3851f..a771f643709 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -1,5 +1,6 @@
 package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
 
+import java.nio.file.Paths
 import java.util.UUID
 
 import akka.http.caching.LfuCache
@@ -13,7 +14,13 @@ import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing.Elemen
 import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, ProtoGeometryImplicits, SkeletonTracingDefaults}
 import com.scalableminds.webknossos.datastore.models.DataRequestCollection.DataRequestCollection
 import com.scalableminds.webknossos.datastore.models._
-import com.scalableminds.webknossos.datastore.services.{IsosurfaceRequest, IsosurfaceService, IsosurfaceServiceHolder}
+import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataRequest
+import com.scalableminds.webknossos.datastore.services.{
+  BinaryDataService,
+  IsosurfaceRequest,
+  IsosurfaceService,
+  IsosurfaceServiceHolder
+}
 import com.scalableminds.webknossos.tracingstore.TSRemoteDatastoreClient
 import com.scalableminds.webknossos.tracingstore.tracings.{
   KeyValueStoreImplicits,
@@ -56,6 +63,8 @@ class EditableMappingService @Inject()(
 
   val isosurfaceService: IsosurfaceService = isosurfaceServiceHolder.tracingStoreIsosurfaceService
 
+  val binaryDataService = new BinaryDataService(Paths.get(""), 100, null)
+
   private lazy val materializedEditableMappingCache: Cache[EditableMappingKey, Box[EditableMapping]] = {
     val maxEntries = 20
     val defaultCachingSettings = CachingSettings("")
@@ -414,14 +423,11 @@ class EditableMappingService @Inject()(
                  userToken: Option[String]): Fox[(Array[Byte], List[Int])] =
     for {
       editableMappingId <- tracing.mappingName.toFox
-      remoteFallbackLayer <- remoteFallbackLayer(tracing)
-      editableMapping <- get(editableMappingId, remoteFallbackLayer, userToken)
-      (unmappedData, indices) <- getUnmappedDataFromDatastore(remoteFallbackLayer, dataRequests, userToken)
-      unmappedDataTyped <- bytesToUnsignedInt(unmappedData, tracing.elementClass)
-      segmentIds = collectSegmentIds(unmappedDataTyped)
-      relevantMapping <- generateCombinedMappingSubset(segmentIds, editableMapping, remoteFallbackLayer, userToken)
-      mappedData <- mapData(unmappedDataTyped, relevantMapping, tracing.elementClass)
-    } yield (mappedData, indices)
+      dataLayer = editableMappingLayer(editableMappingId, tracing, userToken)
+      requests = dataRequests.map(r =>
+        DataServiceDataRequest(null, dataLayer, None, r.cuboid(dataLayer), r.settings.copy(appliedAgglomerate = None)))
+      data <- binaryDataService.handleDataRequests(requests)
+    } yield data
 
   def generateCombinedMappingSubset(segmentIds: Set[Long],
                                     editableMapping: EditableMapping,
@@ -505,19 +511,16 @@ class EditableMappingService @Inject()(
   }
 
   def getUnmappedDataFromDatastore(remoteFallbackLayer: RemoteFallbackLayer,
-                                   dataRequests: DataRequestCollection,
-                                   userToken: Option[String]): Fox[(Array[Byte], List[Int])] =
+                                   dataRequests: List[WebKnossosDataRequest],
+                                   userToken: Option[String]): Fox[(Array[Byte], List[Int])] = {
+    val key = UnmappedRemoteDataKey(remoteFallbackLayer, dataRequests, userToken)
     for {
-      dataRequestsTyped: List[WebKnossosDataRequest] <- Fox.serialCombined(dataRequests) {
-        case r: WebKnossosDataRequest => Fox.successful(r.copy(applyAgglomerate = None))
-        case _                        => Fox.failure("Editable Mappings currently only work for webKnossos data requests")
-      }
-      key = UnmappedRemoteDataKey(remoteFallbackLayer, dataRequestsTyped, userToken)
       resultBox <- unmappedRemoteDataCache.getOrLoad(
         key,
         k => remoteDatastoreClient.getData(k.remoteFallbackLayer, k.dataRequests, k.userToken))
       (data, indices) <- resultBox.toFox
     } yield (data, indices)
+  }
 
   def collectSegmentIds(data: Array[UnsignedInteger]): Set[Long] =
     data.toSet.map { u: UnsignedInteger =>
@@ -559,22 +562,26 @@ class EditableMappingService @Inject()(
       bytes = UnsignedIntegerArray.toByteArray(unsignedIntArray, elementClass)
     } yield bytes
 
+  private def editableMappingLayer(mappingName: String,
+                                   tracing: VolumeTracing,
+                                   userToken: Option[String]): EditableMappingLayer =
+    EditableMappingLayer(
+      mappingName,
+      tracing.boundingBox,
+      resolutions = tracing.resolutions.map(vec3IntFromProto).toList,
+      largestSegmentId = 0L,
+      elementClass = tracing.elementClass,
+      userToken,
+      tracing = tracing,
+      editableMappingService = this
+    )
+
   def createIsosurface(tracing: VolumeTracing,
                        request: WebKnossosIsosurfaceRequest,
-                       token: Option[String]): Fox[(Array[Float], List[Int])] =
+                       userToken: Option[String]): Fox[(Array[Float], List[Int])] =
     for {
       mappingName <- tracing.mappingName.toFox
-      segmentationLayer = EditableMappingLayer(
-        mappingName,
-        tracing.boundingBox,
-        resolutions = tracing.resolutions.map(vec3IntFromProto).toList,
-        largestSegmentId = 0L,
-        elementClass = tracing.elementClass,
-        token,
-        tracing = tracing,
-        editableMappingService = this
-      )
-
+      segmentationLayer = editableMappingLayer(mappingName, tracing, userToken)
       isosurfaceRequest = IsosurfaceRequest(
         dataSource = None,
         dataLayer = segmentationLayer,

From c7a8f612274ab2c459aad8857ddad51becf0936d Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Wed, 25 May 2022 18:31:55 +0200
Subject: [PATCH 071/122] reduce isosurface cube size according to mag, make
 cube size window configurable

---
 .../oxalis/model/sagas/isosurface_saga.ts     | 45 ++++++++-----------
 1 file changed, 19 insertions(+), 26 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
index 20d5efe0b73..21338e1a480 100644
--- a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
@@ -70,7 +70,13 @@ const PARALLEL_PRECOMPUTED_MESH_LOADING_COUNT = 6;
  *
  */
 const isosurfacesMapByLayer: Record<string, Map<number, ThreeDMap<boolean>>> = {};
-const cubeSize: Vector3 = [256, 256, 256];
+function cubeSizeInMag1(): Vector3 {
+  // @ts-ignore
+  return window.__cubeSizeInMag1 != null
+    ? // @ts-ignore
+      window.__cubeSizeInMag1
+    : [128, 128, 128];
+}
 const modifiedCells: Set<number> = new Set();
 export function isIsosurfaceStl(buffer: ArrayBuffer): boolean {
   const dataView = new DataView(buffer);
@@ -106,22 +112,17 @@ function removeMapForSegment(layerName: string, segmentId: number): void {
 
 function getZoomedCubeSize(zoomStep: number, resolutionInfo: ResolutionInfo): Vector3 {
   const [x, y, z] = zoomedAddressToAnotherZoomStepWithInfo(
-    [...cubeSize, zoomStep],
+    [...cubeSizeInMag1(), 0],
     resolutionInfo,
-    0,
+    zoomStep,
   );
   // Drop the last element of the Vector4;
   return [x, y, z];
 }
 
-function clipPositionToCubeBoundary(
-  position: Vector3,
-  zoomStep: number,
-  resolutionInfo: ResolutionInfo,
-): Vector3 {
-  const zoomedCubeSize = getZoomedCubeSize(zoomStep, resolutionInfo);
-  const currentCube = Utils.map3((el, idx) => Math.floor(el / zoomedCubeSize[idx]), position);
-  const clippedPosition = Utils.map3((el, idx) => el * zoomedCubeSize[idx], currentCube);
+function clipPositionToCubeBoundary(position: Vector3): Vector3 {
+  const currentCube = Utils.map3((el, idx) => Math.floor(el / cubeSizeInMag1()[idx]), position);
+  const clippedPosition = Utils.map3((el, idx) => el * cubeSizeInMag1()[idx], currentCube);
   return clippedPosition;
 }
 
@@ -135,18 +136,12 @@ const NEIGHBOR_LOOKUP = [
   [1, 0, 0],
 ];
 
-function getNeighborPosition(
-  clippedPosition: Vector3,
-  neighborId: number,
-  zoomStep: number,
-  resolutionInfo: ResolutionInfo,
-): Vector3 {
-  const zoomedCubeSize = getZoomedCubeSize(zoomStep, resolutionInfo);
+function getNeighborPosition(clippedPosition: Vector3, neighborId: number): Vector3 {
   const neighborMultiplier = NEIGHBOR_LOOKUP[neighborId];
   const neighboringPosition = [
-    clippedPosition[0] + neighborMultiplier[0] * zoomedCubeSize[0],
-    clippedPosition[1] + neighborMultiplier[1] * zoomedCubeSize[1],
-    clippedPosition[2] + neighborMultiplier[2] * zoomedCubeSize[2],
+    clippedPosition[0] + neighborMultiplier[0] * cubeSizeInMag1()[0],
+    clippedPosition[1] + neighborMultiplier[1] * cubeSizeInMag1()[1],
+    clippedPosition[2] + neighborMultiplier[2] * cubeSizeInMag1()[2],
   ];
   // @ts-expect-error ts-migrate(2322) FIXME: Type 'number[]' is not assignable to type 'Vector3... Remove this comment to see the full error message
   return neighboringPosition;
@@ -271,7 +266,7 @@ function* loadIsosurfaceWithNeighbors(
 ): Saga<void> {
   let isInitialRequest = true;
   const { mappingName, mappingType } = isosurfaceMappingInfo;
-  const clippedPosition = clipPositionToCubeBoundary(position, zoomStep, resolutionInfo);
+  const clippedPosition = clipPositionToCubeBoundary(position);
   let positionsToRequest = [clippedPosition];
   const hasIsosurface = yield* select(
     (state) =>
@@ -394,7 +389,7 @@ function* maybeLoadIsosurface(
           mag,
           segmentId,
           subsamplingStrides,
-          cubeSize,
+          cubeSize: getZoomedCubeSize(zoomStep, resolutionInfo),
           scale,
           ...isosurfaceMappingInfo,
         },
@@ -410,9 +405,7 @@ function* maybeLoadIsosurface(
         segmentId,
         isosurfaceMappingInfo.passive || false,
       );
-      return neighbors.map((neighbor) =>
-        getNeighborPosition(clippedPosition, neighbor, zoomStep, resolutionInfo),
-      );
+      return neighbors.map((neighbor) => getNeighborPosition(clippedPosition, neighbor));
     } catch (exception) {
       retryCount++;
       // @ts-ignore

From 97c689d6daef642b021abed17450b1177686ba39 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Wed, 25 May 2022 18:32:41 +0200
Subject: [PATCH 072/122] fix linting

---
 frontend/javascripts/oxalis/view/context_menu.tsx | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/frontend/javascripts/oxalis/view/context_menu.tsx b/frontend/javascripts/oxalis/view/context_menu.tsx
index 6dedd115f33..0cf5129f02c 100644
--- a/frontend/javascripts/oxalis/view/context_menu.tsx
+++ b/frontend/javascripts/oxalis/view/context_menu.tsx
@@ -18,8 +18,14 @@ import type {
   UserBoundingBox,
   VolumeTracing,
 } from "oxalis/store";
-import { AnnotationTool, Vector3, OrthoView, OrthoViews } from "oxalis/constants";
-import { AnnotationToolEnum, VolumeTools } from "oxalis/constants";
+import {
+  AnnotationTool,
+  Vector3,
+  OrthoView,
+  OrthoViews,
+  AnnotationToolEnum,
+  VolumeTools,
+} from "oxalis/constants";
 import { V3 } from "libs/mjs";
 import {
   loadAdHocMeshAction,

From 6b3f6f160dd622f60f16726a0ae3e1b84c679030 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Fri, 3 Jun 2022 09:55:19 +0200
Subject: [PATCH 073/122] cleanup

---
 .../controllers/VolumeTracingController.scala |   2 +
 .../EditableMappingsIsosurfaceService.scala   | 151 ------------------
 2 files changed, 2 insertions(+), 151 deletions(-)
 delete mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingsIsosurfaceService.scala

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index dfa8618e833..01725ccbf77 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -162,6 +162,8 @@ class VolumeTracingController @Inject()(
         accessTokenService.validateAccess(UserAccessRequest.webknossos, token) {
           for {
             tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound")
+            hasEditableMapping: Option[Boolean] <- Fox.runOptional(tracing.mappingName)(editableMappingService.exists)
+            _ <- bool2Fox(!hasEditableMapping.getOrElse(false)) ?~> "Duplicate is not yet implemented for editable mapping annotations"
             dataSetBoundingBox = request.body.asJson.flatMap(_.validateOpt[BoundingBox].asOpt.flatten)
             resolutionRestrictions = ResolutionRestrictions(minResolution, maxResolution)
             (newId, newTracing) <- tracingService.duplicate(tracingId,
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingsIsosurfaceService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingsIsosurfaceService.scala
deleted file mode 100644
index a75d1b8e699..00000000000
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingsIsosurfaceService.scala
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
-package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
-
-import java.nio.{Buffer, ByteBuffer, ByteOrder, IntBuffer, LongBuffer, ShortBuffer}
-
-import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int}
-import com.scalableminds.util.tools.Fox
-import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing
-import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits
-import com.scalableminds.webknossos.datastore.models.WebKnossosIsosurfaceRequest
-import com.scalableminds.webknossos.datastore.models.datasource.{DataSource, ElementClass, SegmentationLayer}
-import com.scalableminds.webknossos.datastore.models.requests.{Cuboid, DataServiceDataRequest, DataServiceMappingRequest, DataServiceRequestSettings}
-import com.scalableminds.webknossos.datastore.services.mcubes.MarchingCubes
-import com.scalableminds.webknossos.datastore.services.{DataTypeFunctors, IsosurfaceRequest}
-import javax.inject.Inject
-
-import scala.collection.mutable
-import scala.reflect.ClassTag
-
-case class EditableMappingIsosurfaceRequest(
-  tracing: VolumeTracing,
-                              cuboid: Cuboid,
-                              segmentId: Long,
-                              subsamplingStrides: Vec3Int,
-                              scale: Vec3Double,
-                              mapping: Option[String] = None,
-                              mappingType: Option[String] = None
-                            )
-
-class EditableMappingsIsosurfaceService @Inject()(editableMappingService: EditableMappingService) extends ProtoGeometryImplicits {
-  def createIsosurface(tracing: VolumeTracing, request: WebKnossosIsosurfaceRequest): Fox[(Array[Float], List[Int])] = {
-    requestIsosurface(EditableMappingIsosurfaceRequest())
-  }
-
-  def requestIsosurface(request: EditableMappingIsosurfaceRequest): Fox[(Array[Float], List[Int])] =
-    elementClassFromProto(request.tracing.elementClass) match {
-      case ElementClass.uint8 =>
-        generateIsosurfaceImpl[Byte, ByteBuffer](request,
-          DataTypeFunctors[Byte, ByteBuffer](identity, _.get(_), _.toByte))
-      case ElementClass.uint16 =>
-        generateIsosurfaceImpl[Short, ShortBuffer](
-          request,
-          DataTypeFunctors[Short, ShortBuffer](_.asShortBuffer, _.get(_), _.toShort))
-      case ElementClass.uint32 =>
-        generateIsosurfaceImpl[Int, IntBuffer](request,
-          DataTypeFunctors[Int, IntBuffer](_.asIntBuffer, _.get(_), _.toInt))
-      case ElementClass.uint64 =>
-        generateIsosurfaceImpl[Long, LongBuffer](request,
-          DataTypeFunctors[Long, LongBuffer](_.asLongBuffer, _.get(_), identity))
-    }
-
-  private def generateIsosurfaceImpl[T: ClassTag, B <: Buffer](
-                                                                request: EditableMappingIsosurfaceRequest,
-                                                                dataTypeFunctors: DataTypeFunctors[T, B]): Fox[(Array[Float], List[Int])] = {
-
-    def convertData(data: Array[Byte]): Array[T] = {
-      val srcBuffer = dataTypeFunctors.getTypedBufferFn(ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN))
-      srcBuffer.rewind()
-      val dstArray = Array.ofDim[T](srcBuffer.remaining())
-      dataTypeFunctors.copyDataFn(srcBuffer, dstArray)
-      dstArray
-    }
-
-    def subVolumeContainsSegmentId[T](data: Array[T],
-                                      dataDimensions: Vec3Int,
-                                      boundingBox: BoundingBox,
-                                      segmentId: T): Boolean = {
-      for {
-        x <- boundingBox.topLeft.x until boundingBox.bottomRight.x
-        y <- boundingBox.topLeft.y until boundingBox.bottomRight.y
-        z <- boundingBox.topLeft.z until boundingBox.bottomRight.z
-      } {
-        val voxelOffset = x + y * dataDimensions.x + z * dataDimensions.x * dataDimensions.y
-        if (data(voxelOffset) == segmentId) return true
-      }
-      false
-    }
-
-    def findNeighbors[T](data: Array[T], dataDimensions: Vec3Int, segmentId: T): List[Int] = {
-      val x = dataDimensions.x - 1
-      val y = dataDimensions.y - 1
-      val z = dataDimensions.z - 1
-      val front_xy = BoundingBox(Vec3Int(0, 0, 0), x, y, 1)
-      val front_xz = BoundingBox(Vec3Int(0, 0, 0), x, 1, z)
-      val front_yz = BoundingBox(Vec3Int(0, 0, 0), 1, y, z)
-      val back_xy = BoundingBox(Vec3Int(0, 0, z), x, y, 1)
-      val back_xz = BoundingBox(Vec3Int(0, y, 0), x, 1, z)
-      val back_yz = BoundingBox(Vec3Int(x, 0, 0), 1, y, z)
-      val surfaceBoundingBoxes = List(front_xy, front_xz, front_yz, back_xy, back_xz, back_yz)
-      surfaceBoundingBoxes.zipWithIndex.filter {
-        case (surfaceBoundingBox, index) =>
-          subVolumeContainsSegmentId(data, dataDimensions, surfaceBoundingBox, segmentId)
-      }.map {
-        case (surfaceBoundingBox, index) => index
-      }
-    }
-
-    val cuboid = request.cuboid
-    val subsamplingStrides =
-      Vec3Double(request.subsamplingStrides.x, request.subsamplingStrides.y, request.subsamplingStrides.z)
-
-    val dataRequest = DataServiceDataRequest(request.dataSource.orNull,
-      request.dataLayer,
-      request.mapping,
-      cuboid,
-      DataServiceRequestSettings.default,
-      request.subsamplingStrides)
-
-    val dataDimensions = Vec3Int(
-      math.ceil(cuboid.width / subsamplingStrides.x).toInt,
-      math.ceil(cuboid.height / subsamplingStrides.y).toInt,
-      math.ceil(cuboid.depth / subsamplingStrides.z).toInt
-    )
-
-    val offset = Vec3Double(cuboid.topLeft.x, cuboid.topLeft.y, cuboid.topLeft.z)
-    val scale = Vec3Double(cuboid.topLeft.mag) * request.scale
-    val typedSegmentId = dataTypeFunctors.fromLong(request.segmentId)
-
-    val vertexBuffer = mutable.ArrayBuffer[Vec3Double]()
-
-    for {
-      data <- binaryDataService.handleDataRequest(dataRequest)
-      typedData = convertData(data)
-      neighbors = findNeighbors(typedData, dataDimensions, typedSegmentId)
-    } yield {
-      for {
-        x <- 0 until dataDimensions.x by 32
-        y <- 0 until dataDimensions.y by 32
-        z <- 0 until dataDimensions.z by 32
-      } {
-        val boundingBox = BoundingBox(Vec3Int(x, y, z),
-          math.min(dataDimensions.x - x, 33),
-          math.min(dataDimensions.y - y, 33),
-          math.min(dataDimensions.z - z, 33))
-        if (subVolumeContainsSegmentId(typedData, dataDimensions, boundingBox, typedSegmentId)) {
-          MarchingCubes.marchingCubes[T](typedData,
-            dataDimensions,
-            boundingBox,
-            typedSegmentId,
-            subsamplingStrides,
-            offset,
-            scale,
-            vertexBuffer)
-        }
-      }
-      (vertexBuffer.flatMap(_.toList.map(_.toFloat)).toArray, neighbors)
-    }
-  }
-
-}
- */

From e397f05d01f8982e41b4ae7c3e8379f315b00053 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Fri, 3 Jun 2022 11:25:04 +0200
Subject: [PATCH 074/122] measure time for json conversions of editable mapping

---
 .../services/IsosurfaceService.scala          |  4 ++--
 .../controllers/VolumeTracingController.scala | 19 ++++++++++++++++---
 .../tracings/FossilDBClient.scala             | 18 +++++++++++++++++-
 .../EditableMappingService.scala              | 10 ++++------
 4 files changed, 39 insertions(+), 12 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
index 1b88f5fb1b2..bbf5aad5352 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
@@ -229,8 +229,8 @@ class IsosurfaceService(binaryDataService: BinaryDataService,
         }
       }
       val afterMarchingCubes = System.currentTimeMillis()
-      logger.info(
-        s"Isosurface generation timing - loading: ${afterLoading - before} ms, preprocessing: ${afterPreprocessing - afterLoading}, marchingCubes: ${afterMarchingCubes - afterPreprocessing}. ${agglomerateIds.size} agglomerates in data, ${typedData.length} voxels.")
+      /*logger.info(
+        s"Isosurface generation timing - loading: ${afterLoading - before} ms, preprocessing: ${afterPreprocessing - afterLoading}, marchingCubes: ${afterMarchingCubes - afterPreprocessing}. ${agglomerateIds.size} agglomerates in data, ${typedData.length} voxels.")*/
       (vertexBuffer.flatMap(_.toList.map(_.toFloat)).toArray, neighbors)
     }
   }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index 01725ccbf77..c8ded251b97 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -18,9 +18,22 @@ import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.scalableminds.webknossos.datastore.services.UserAccessRequest
 import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService
 import com.scalableminds.webknossos.tracingstore.tracings.UpdateActionGroup
-import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{EditableMappingService, EditableMappingUpdateActionGroup, RemoteFallbackLayer}
-import com.scalableminds.webknossos.tracingstore.tracings.volume.{ResolutionRestrictions, UpdateMappingNameAction, VolumeTracingService}
-import com.scalableminds.webknossos.tracingstore.{TSRemoteDatastoreClient, TSRemoteWebKnossosClient, TracingStoreAccessTokenService, TracingStoreConfig}
+import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{
+  EditableMappingService,
+  EditableMappingUpdateActionGroup,
+  RemoteFallbackLayer
+}
+import com.scalableminds.webknossos.tracingstore.tracings.volume.{
+  ResolutionRestrictions,
+  UpdateMappingNameAction,
+  VolumeTracingService
+}
+import com.scalableminds.webknossos.tracingstore.{
+  TSRemoteDatastoreClient,
+  TSRemoteWebKnossosClient,
+  TracingStoreAccessTokenService,
+  TracingStoreConfig
+}
 import play.api.i18n.Messages
 import play.api.libs.Files.TemporaryFile
 import play.api.libs.iteratee.Enumerator
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
index 6273937578c..72e34d8907d 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
@@ -16,7 +16,7 @@ import scalapb.{GeneratedMessage, GeneratedMessageCompanion}
 
 import scala.concurrent.ExecutionContext.Implicits.global
 
-trait KeyValueStoreImplicits extends BoxImplicits {
+trait KeyValueStoreImplicits extends BoxImplicits with LazyLogging {
 
   implicit def stringToByteArray(s: String): Array[Byte] = s.toCharArray.map(_.toByte)
 
@@ -24,8 +24,24 @@ trait KeyValueStoreImplicits extends BoxImplicits {
 
   implicit def asJson[T](o: T)(implicit w: Writes[T]): Array[Byte] = w.writes(o).toString.getBytes("UTF-8")
 
+  def asJsonWithTimeLogging[T](o: T)(implicit w: Writes[T]): Array[Byte] = {
+    val before = System.currentTimeMillis()
+    val res = w.writes(o).toString.getBytes("UTF-8")
+    val durationMs = System.currentTimeMillis() - before
+    logger.info(s"Editable Mapping Json writing took ${durationMs} ms")
+    res
+  }
+
   implicit def fromJson[T](a: Array[Byte])(implicit r: Reads[T]): Box[T] = jsResult2Box(Json.parse(a).validate)
 
+  def fromJsonWithTimeLogging[T](a: Array[Byte])(implicit r: Reads[T]): Box[T] = jsResult2Box {
+    val before = System.currentTimeMillis()
+    val res = Json.parse(a).validate
+    val durationMs = System.currentTimeMillis() - before
+    logger.info(s"Editable Mapping Json parsing took ${durationMs} ms")
+    res
+  }
+
   implicit def asProto[T <: GeneratedMessage](o: T): Array[Byte] = o.toByteArray
 
   implicit def fromProto[T <: GeneratedMessage](a: Array[Byte])(
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index a771f643709..13efb354424 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -118,7 +118,7 @@ class EditableMappingService @Inject()(
       agglomerateToGraph = Map()
     )
     for {
-      _ <- tracingDataStore.editableMappings.put(newId, 0L, newEditableMapping)
+      _ <- tracingDataStore.editableMappings.put(newId, 0L, asJsonWithTimeLogging(newEditableMapping))
     } yield newId
   }
 
@@ -172,11 +172,10 @@ class EditableMappingService @Inject()(
                            remoteFallbackLayer: RemoteFallbackLayer,
                            userToken: Option[String],
                            desiredVersion: Long,
-  ): Fox[EditableMapping] = {
-    logger.info("cache miss")
+  ): Fox[EditableMapping] =
     for {
       closestMaterializedVersion: VersionedKeyValuePair[EditableMapping] <- tracingDataStore.editableMappings
-        .get(editableMappingId, Some(desiredVersion))(fromJson[EditableMapping])
+        .get(editableMappingId, Some(desiredVersion))(fromJsonWithTimeLogging[EditableMapping])
       _ = logger.info(
         f"Loading mapping version $desiredVersion, closest materialized is version ${closestMaterializedVersion.version} (${closestMaterializedVersion.value})")
       materialized <- applyPendingUpdates(
@@ -189,10 +188,9 @@ class EditableMappingService @Inject()(
       )
       _ = logger.info(s"Materialized mapping: $materialized")
       _ <- Fox.runIf(shouldPersistMaterialized(closestMaterializedVersion.version, desiredVersion)) {
-        tracingDataStore.editableMappings.put(editableMappingId, desiredVersion, materialized)
+        tracingDataStore.editableMappings.put(editableMappingId, desiredVersion, asJsonWithTimeLogging(materialized))
       }
     } yield materialized
-  }
 
   private def shouldPersistMaterialized(previouslyMaterializedVersion: Long, newVersion: Long): Boolean =
     newVersion > previouslyMaterializedVersion && newVersion % 10 == 5

From d5def3e918c1470516123a628b526aa6347469ea Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Fri, 3 Jun 2022 11:53:30 +0200
Subject: [PATCH 075/122] add proto definition for AgglomerateGraph,
 EditableMapping

---
 .../proto/EditableMapping.proto               | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 webknossos-datastore/proto/EditableMapping.proto

diff --git a/webknossos-datastore/proto/EditableMapping.proto b/webknossos-datastore/proto/EditableMapping.proto
new file mode 100644
index 00000000000..21199319ba2
--- /dev/null
+++ b/webknossos-datastore/proto/EditableMapping.proto
@@ -0,0 +1,33 @@
+syntax = "proto2";
+
+package com.scalableminds.webknossos.datastore;
+
+import "geometry.proto";
+
+message AgglomerateEdge {
+  required int64 source = 1;
+  required int64 target = 2;
+}
+
+message AgglomerateGraph {
+  repeated int64 segments = 1;
+  repeated AgglomerateEdge edges = 2;
+  repeated Vec3IntProto positions = 3;
+  repeated double affinities = 4;
+}
+
+message AgglomerateToGraphPair {
+  required int64 agglomerateId = 1;
+  required AgglomerateGraph agglomerateGraph = 2;
+}
+
+message SegmentToAgglomeratePair {
+  required int64 segmentId = 1;
+  required int64 agglomerateId = 2;
+}
+
+message EditableMappingProto {
+  required string baseMappingName = 1;
+  repeated SegmentToAgglomeratePair segmentToAgglomerate = 2;
+  repeated AgglomerateToGraphPair agglomerateToGraph = 3;
+}

From 0536bd90f74a02985e046e99a935b28ca0d1ffd2 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Fri, 3 Jun 2022 13:42:15 +0200
Subject: [PATCH 076/122] add proto conversion methods, measure speed

---
 .../datastore/models/AgglomerateGraph.scala   | 24 ++++++++++-
 .../proto/EditableMapping.proto               |  2 +-
 .../controllers/VolumeTracingController.scala | 36 ++++++++++++++++-
 .../editablemapping/EditableMapping.scala     | 40 +++++++++++++++++++
 ...alableminds.webknossos.tracingstore.routes |  1 +
 5 files changed, 98 insertions(+), 5 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala
index c607f5a9762..c02df4ed06d 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala
@@ -1,18 +1,38 @@
 package com.scalableminds.webknossos.datastore.models
 
 import com.scalableminds.util.geometry.Vec3Int
+import com.scalableminds.webknossos.datastore.EditableMapping.AgglomerateEdge
+import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits
 import play.api.libs.json.{Json, OFormat}
 
 case class AgglomerateGraph(segments: List[Long],
                             edges: List[(Long, Long)],
                             positions: List[Vec3Int],
-                            affinities: List[Float]) {
+                            affinities: List[Float])
+    extends ProtoGeometryImplicits {
   override def toString: String =
     f"AgglomerateGraph(${segments.length} segments, ${edges.length} edges)"
+
+  def toProto: com.scalableminds.webknossos.datastore.EditableMapping.AgglomerateGraph =
+    com.scalableminds.webknossos.datastore.EditableMapping.AgglomerateGraph(
+      segments = segments,
+      edges = edges.map(e => AgglomerateEdge(e._1, e._2)),
+      positions = positions.map(vec3IntToProto),
+      affinities = affinities
+    )
 }
 
-object AgglomerateGraph {
+object AgglomerateGraph extends ProtoGeometryImplicits {
   implicit val jsonFormat: OFormat[AgglomerateGraph] = Json.format[AgglomerateGraph]
 
   def empty: AgglomerateGraph = AgglomerateGraph(List.empty, List.empty, List.empty, List.empty)
+
+  def fromProto(agglomerateGraphProto: com.scalableminds.webknossos.datastore.EditableMapping.AgglomerateGraph)
+    : AgglomerateGraph =
+    AgglomerateGraph(
+      segments = agglomerateGraphProto.segments.toList,
+      edges = agglomerateGraphProto.edges.map(e => (e.source, e.target)).toList,
+      positions = agglomerateGraphProto.positions.map(vec3IntFromProto).toList,
+      affinities = agglomerateGraphProto.affinities.toList
+    )
 }
diff --git a/webknossos-datastore/proto/EditableMapping.proto b/webknossos-datastore/proto/EditableMapping.proto
index 21199319ba2..50ee391f160 100644
--- a/webknossos-datastore/proto/EditableMapping.proto
+++ b/webknossos-datastore/proto/EditableMapping.proto
@@ -13,7 +13,7 @@ message AgglomerateGraph {
   repeated int64 segments = 1;
   repeated AgglomerateEdge edges = 2;
   repeated Vec3IntProto positions = 3;
-  repeated double affinities = 4;
+  repeated float affinities = 4;
 }
 
 message AgglomerateToGraphPair {
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index c8ded251b97..08679788005 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -8,6 +8,7 @@ import com.google.inject.Inject
 import com.scalableminds.util.geometry.{BoundingBox, Vec3Int}
 import com.scalableminds.util.tools.ExtendedTypes.ExtendedString
 import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.datastore.EditableMapping.EditableMappingProto
 import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings}
 import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser
 import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits
@@ -17,8 +18,9 @@ import com.scalableminds.webknossos.datastore.models.{WebKnossosDataRequest, Web
 import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.scalableminds.webknossos.datastore.services.UserAccessRequest
 import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService
-import com.scalableminds.webknossos.tracingstore.tracings.UpdateActionGroup
+import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, UpdateActionGroup}
 import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{
+  EditableMapping,
   EditableMappingService,
   EditableMappingUpdateActionGroup,
   RemoteFallbackLayer
@@ -53,7 +55,8 @@ class VolumeTracingController @Inject()(
     val slackNotificationService: TSSlackNotificationService,
     val rpc: RPC)(implicit val ec: ExecutionContext, val bodyParsers: PlayBodyParsers)
     extends TracingController[VolumeTracing, VolumeTracings]
-    with ProtoGeometryImplicits {
+    with ProtoGeometryImplicits
+    with KeyValueStoreImplicits {
 
   implicit val tracingsCompanion: VolumeTracings.type = VolumeTracings
 
@@ -519,4 +522,33 @@ class VolumeTracingController @Inject()(
         }
       }
   }
+
+  def test: Action[AnyContent] = Action {
+    val mapping = EditableMapping.createDummy(50000, 500)
+    for (_ <- 1 to 10) {
+      val before = System.currentTimeMillis()
+      val bytes = asJson(mapping)
+      val afterTo = System.currentTimeMillis()
+      val back = fromJson(bytes)
+      val afterBack = System.currentTimeMillis()
+
+      val durationTo = afterTo - before
+      val durationBack = afterBack - afterTo
+
+      println(f"to json: $durationTo ms, from json: $durationBack ms")
+    }
+    for (_ <- 1 to 10) {
+      val before = System.currentTimeMillis()
+      val bytes = asProto(mapping.toProto)
+      val afterTo = System.currentTimeMillis()
+      val back = EditableMapping.fromProto(fromProto[EditableMappingProto](bytes).openOrThrowException("yolo"))
+      val afterBack = System.currentTimeMillis()
+
+      val durationTo = afterTo - before
+      val durationBack = afterBack - afterTo
+
+      println(f"to proto: $durationTo ms, from proto: $durationBack ms")
+    }
+    Ok
+  }
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
index 75fb625e0e6..dd5ba7c0a8d 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
@@ -1,6 +1,12 @@
 package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
 
+import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.tools.AdditionalJsonFormats
+import com.scalableminds.webknossos.datastore.EditableMapping.{
+  AgglomerateToGraphPair,
+  EditableMappingProto,
+  SegmentToAgglomeratePair
+}
 import com.scalableminds.webknossos.datastore.models.AgglomerateGraph
 import play.api.libs.json.{Json, OFormat}
 
@@ -10,8 +16,42 @@ case class EditableMapping(
     agglomerateToGraph: Map[Long, AgglomerateGraph],
 ) {
   override def toString: String = f"EditableMapping(agglomerates:${agglomerateToGraph.keySet})"
+
+  def toProto: EditableMappingProto =
+    EditableMappingProto(
+      baseMappingName = baseMappingName,
+      segmentToAgglomerate = segmentToAgglomerate.map(tuple => SegmentToAgglomeratePair(tuple._1, tuple._2)).toSeq,
+      agglomerateToGraph = agglomerateToGraph.map(tuple => AgglomerateToGraphPair(tuple._1, tuple._2.toProto)).toSeq,
+    )
 }
 
 object EditableMapping extends AdditionalJsonFormats {
   implicit val jsonFormat: OFormat[EditableMapping] = Json.format[EditableMapping]
+
+  def fromProto(editableMappignProto: EditableMappingProto): EditableMapping =
+    EditableMapping(
+      baseMappingName = editableMappignProto.baseMappingName,
+      segmentToAgglomerate =
+        editableMappignProto.segmentToAgglomerate.map(pair => pair.segmentId -> pair.agglomerateId).toMap,
+      agglomerateToGraph = editableMappignProto.agglomerateToGraph
+        .map(pair => pair.agglomerateId -> AgglomerateGraph.fromProto(pair.agglomerateGraph))
+        .toMap
+    )
+
+  def createDummy(numSegments: Long, numAgglomerates: Long): EditableMapping =
+    EditableMapping(
+      baseMappingName = "dummyBaseMapping",
+      segmentToAgglomerate = 1L.to(numSegments).map(s => s -> s % numAgglomerates).toMap,
+      agglomerateToGraph = 1L
+        .to(numAgglomerates)
+        .map(a =>
+          a -> AgglomerateGraph(
+            segments = 1L.to(numSegments / numAgglomerates).toList,
+            edges = 1L.to(numSegments / numAgglomerates).map(s => s -> s).toList,
+            positions = 1L.to(numSegments / numAgglomerates).map(s => Vec3Int.full(s.toInt)).toList,
+            affinities = 1L.to(numSegments / numAgglomerates).map(s => s.toFloat).toList
+        ))
+        .toMap
+    )
+
 }
diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
index b6c10346072..fc8ef8ad7fa 100644
--- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
+++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
@@ -27,6 +27,7 @@ POST        /volume/mergedFromIds                               @com.scalablemin
 POST        /volume/mergedFromContents                          @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromContents(token: Option[String], persist: Boolean)
 
 # Editable Mappings
+GET         /mapping/test                                       @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.test
 POST        /mapping/:tracingId/update                          @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.updateEditableMapping(token: Option[String], tracingId: String)
 GET         /mapping/:tracingId/updateActionLog                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingUpdateActionLog(token: Option[String], tracingId: String)
 GET         /mapping/:tracingId                                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingInfo(token: Option[String], tracingId: String)

From ed8ef2c8fb21aca01be9b6a7cb6bf56a2a3c95fb Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Fri, 3 Jun 2022 13:46:13 +0200
Subject: [PATCH 077/122] cleanup, method naming

---
 .../controllers/VolumeTracingController.scala |  8 +++---
 .../tracings/FossilDBClient.scala             | 25 ++++---------------
 .../tracings/TracingService.scala             |  2 +-
 .../EditableMappingService.scala              | 10 ++++----
 .../skeleton/SkeletonTracingService.scala     |  6 ++---
 .../volume/VolumeTracingService.scala         |  2 +-
 6 files changed, 19 insertions(+), 34 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index 08679788005..a8e78427407 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -527,9 +527,9 @@ class VolumeTracingController @Inject()(
     val mapping = EditableMapping.createDummy(50000, 500)
     for (_ <- 1 to 10) {
       val before = System.currentTimeMillis()
-      val bytes = asJson(mapping)
+      val bytes = toJsonBytes(mapping)
       val afterTo = System.currentTimeMillis()
-      val back = fromJson(bytes)
+      val back = fromJsonBytes(bytes)
       val afterBack = System.currentTimeMillis()
 
       val durationTo = afterTo - before
@@ -539,9 +539,9 @@ class VolumeTracingController @Inject()(
     }
     for (_ <- 1 to 10) {
       val before = System.currentTimeMillis()
-      val bytes = asProto(mapping.toProto)
+      val bytes = toProtoBytes(mapping.toProto)
       val afterTo = System.currentTimeMillis()
-      val back = EditableMapping.fromProto(fromProto[EditableMappingProto](bytes).openOrThrowException("yolo"))
+      val back = EditableMapping.fromProto(fromProtoBytes[EditableMappingProto](bytes).openOrThrowException("yolo"))
       val afterBack = System.currentTimeMillis()
 
       val durationTo = afterTo - before
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
index 72e34d8907d..c2211a931ef 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
@@ -22,30 +22,15 @@ trait KeyValueStoreImplicits extends BoxImplicits with LazyLogging {
 
   implicit def toBox[T](x: T): Box[T] = Full(x)
 
-  implicit def asJson[T](o: T)(implicit w: Writes[T]): Array[Byte] = w.writes(o).toString.getBytes("UTF-8")
-
-  def asJsonWithTimeLogging[T](o: T)(implicit w: Writes[T]): Array[Byte] = {
-    val before = System.currentTimeMillis()
-    val res = w.writes(o).toString.getBytes("UTF-8")
-    val durationMs = System.currentTimeMillis() - before
-    logger.info(s"Editable Mapping Json writing took ${durationMs} ms")
-    res
-  }
-
-  implicit def fromJson[T](a: Array[Byte])(implicit r: Reads[T]): Box[T] = jsResult2Box(Json.parse(a).validate)
+  implicit def toJsonBytes[T](o: T)(implicit w: Writes[T]): Array[Byte] = w.writes(o).toString.getBytes("UTF-8")
 
-  def fromJsonWithTimeLogging[T](a: Array[Byte])(implicit r: Reads[T]): Box[T] = jsResult2Box {
-    val before = System.currentTimeMillis()
-    val res = Json.parse(a).validate
-    val durationMs = System.currentTimeMillis() - before
-    logger.info(s"Editable Mapping Json parsing took ${durationMs} ms")
-    res
-  }
+  implicit def fromJsonBytes[T](a: Array[Byte])(implicit r: Reads[T]): Box[T] = jsResult2Box(Json.parse(a).validate)
 
-  implicit def asProto[T <: GeneratedMessage](o: T): Array[Byte] = o.toByteArray
+  implicit def toProtoBytes[T <: GeneratedMessage](o: T): Array[Byte] = o.toByteArray
 
-  implicit def fromProto[T <: GeneratedMessage](a: Array[Byte])(
+  implicit def fromProtoBytes[T <: GeneratedMessage](a: Array[Byte])(
       implicit companion: GeneratedMessageCompanion[T]): Box[T] = tryo(companion.parseFrom(a))
+
 }
 
 case class KeyValuePair[T](key: String, value: T)
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala
index 6eb04e72f70..64a0dd33b80 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala
@@ -122,7 +122,7 @@ trait TracingService[T <: GeneratedMessage]
            useCache: Boolean = true,
            applyUpdates: Boolean = false): Fox[T] = {
     if (tracingId == TracingIds.dummyTracingId) return Fox.successful(dummyTracing)
-    val tracingFox = tracingStore.get(tracingId, version)(fromProto[T]).map(_.value)
+    val tracingFox = tracingStore.get(tracingId, version)(fromProtoBytes[T]).map(_.value)
     tracingFox.flatMap { tracing =>
       val updatedTracing = if (applyUpdates) {
         applyPendingUpdates(tracing, tracingId, version)
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 13efb354424..81c4a7ae462 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -118,7 +118,7 @@ class EditableMappingService @Inject()(
       agglomerateToGraph = Map()
     )
     for {
-      _ <- tracingDataStore.editableMappings.put(newId, 0L, asJsonWithTimeLogging(newEditableMapping))
+      _ <- tracingDataStore.editableMappings.put(newId, 0L, newEditableMapping)
     } yield newId
   }
 
@@ -139,7 +139,7 @@ class EditableMappingService @Inject()(
 
     for {
       updates <- tracingDataStore.editableMappingUpdates.getMultipleVersionsAsVersionValueTuple(editableMappingId)(
-        fromJson[List[EditableMappingUpdateAction]])
+        fromJsonBytes[List[EditableMappingUpdateAction]])
       updateActionGroupsJs = updates.map(versionedTupleToJson)
     } yield Json.toJson(updateActionGroupsJs)
   }
@@ -175,7 +175,7 @@ class EditableMappingService @Inject()(
   ): Fox[EditableMapping] =
     for {
       closestMaterializedVersion: VersionedKeyValuePair[EditableMapping] <- tracingDataStore.editableMappings
-        .get(editableMappingId, Some(desiredVersion))(fromJsonWithTimeLogging[EditableMapping])
+        .get(editableMappingId, Some(desiredVersion))(fromJsonBytes[EditableMapping])
       _ = logger.info(
         f"Loading mapping version $desiredVersion, closest materialized is version ${closestMaterializedVersion.version} (${closestMaterializedVersion.value})")
       materialized <- applyPendingUpdates(
@@ -188,7 +188,7 @@ class EditableMappingService @Inject()(
       )
       _ = logger.info(s"Materialized mapping: $materialized")
       _ <- Fox.runIf(shouldPersistMaterialized(closestMaterializedVersion.version, desiredVersion)) {
-        tracingDataStore.editableMappings.put(editableMappingId, desiredVersion, asJsonWithTimeLogging(materialized))
+        tracingDataStore.editableMappings.put(editableMappingId, desiredVersion, materialized)
       }
     } yield materialized
 
@@ -406,7 +406,7 @@ class EditableMappingService @Inject()(
           editableMappingId,
           Some(desiredVersion),
           Some(existingVersion + 1)
-        )(fromJson[List[EditableMappingUpdateAction]])
+        )(fromJsonBytes[List[EditableMappingUpdateAction]])
       } yield updates.reverse.flatten
     }
 
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala
index dd3e8a719e5..5df43dc6c70 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala
@@ -99,7 +99,7 @@ class SkeletonTracingService @Inject()(
         updateActionGroups <- tracingDataStore.skeletonUpdates.getMultipleVersions(
           tracingId,
           Some(desiredVersion),
-          Some(existingVersion + 1))(fromJson[List[SkeletonUpdateAction]])
+          Some(existingVersion + 1))(fromJsonBytes[List[SkeletonUpdateAction]])
       } yield updateActionGroups.reverse.flatten
     }
 
@@ -194,7 +194,7 @@ class SkeletonTracingService @Inject()(
       )
     for {
       updateActionGroups <- tracingDataStore.skeletonUpdates.getMultipleVersionsAsVersionValueTuple(tracingId)(
-        fromJson[List[SkeletonUpdateAction]])
+        fromJsonBytes[List[SkeletonUpdateAction]])
       updateActionGroupsJs = updateActionGroups.map(versionedTupleToJson)
     } yield Json.toJson(updateActionGroupsJs)
   }
@@ -202,7 +202,7 @@ class SkeletonTracingService @Inject()(
   def updateActionStatistics(tracingId: String): Fox[JsObject] =
     for {
       updateActionGroups <- tracingDataStore.skeletonUpdates.getMultipleVersions(tracingId)(
-        fromJson[List[SkeletonUpdateAction]])
+        fromJsonBytes[List[SkeletonUpdateAction]])
       updateActions = updateActionGroups.flatten
     } yield {
       Json.obj(
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
index 5a75a49dd38..17f62b3f26f 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
@@ -316,7 +316,7 @@ class VolumeTracingService @Inject()(
 
     for {
       volumeTracings <- tracingDataStore.volumeUpdates.getMultipleVersionsAsVersionValueTuple(tracingId)(
-        fromJson[List[CompactVolumeUpdateAction]])
+        fromJsonBytes[List[CompactVolumeUpdateAction]])
       updateActionGroupsJs = volumeTracings.map(versionedTupleToJson)
     } yield Json.toJson(updateActionGroupsJs)
   }

From 12bc4e449d7b6d5d2bc5ba416b63d1039558eb69 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 7 Jun 2022 09:53:07 +0200
Subject: [PATCH 078/122] fully change AgglomerateGraph to proto, clean up
 backend code

---
 app/controllers/AnnotationIOController.scala  |  4 +-
 app/controllers/DataSetController.scala       |  2 +-
 app/controllers/SitemapController.scala       |  2 +-
 app/models/annotation/AnnotationService.scala |  2 +-
 .../util/mvc/ExtendedController.scala         |  9 ++++
 .../controllers/BinaryDataController.scala    |  2 +-
 .../controllers/DataSourceController.scala    |  6 +--
 .../datastore/models/AgglomerateGraph.scala   | 38 --------------
 .../webknossos/datastore/rpc/RPCRequest.scala | 50 ++++++++-----------
 .../services/AgglomerateService.scala         | 13 +++--
 .../TSRemoteDatastoreClient.scala             |  5 +-
 .../controllers/TracingController.scala       |  4 +-
 .../controllers/VolumeTracingController.scala | 32 +-----------
 .../editablemapping/EditableMapping.scala     | 29 ++++-------
 .../EditableMappingService.scala              | 47 ++++++++---------
 ...alableminds.webknossos.tracingstore.routes |  1 -
 16 files changed, 83 insertions(+), 163 deletions(-)
 delete mode 100644 webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala

diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala
index 97a198c8ed8..cd46d39a1f1 100755
--- a/app/controllers/AnnotationIOController.scala
+++ b/app/controllers/AnnotationIOController.scala
@@ -403,9 +403,9 @@ Expects:
 
     def exportMimeTypeForAnnotation(annotation: Annotation): String =
       if (annotation.tracingType == TracingType.skeleton)
-        "application/xml"
+        xmlMimeType
       else
-        "application/zip"
+        zipMimeType
 
     for {
       annotation <- provider.provideAnnotation(typ, annotationId, issuingUser) ~> NOT_FOUND
diff --git a/app/controllers/DataSetController.scala b/app/controllers/DataSetController.scala
index f09c85f25e4..484c4e439ae 100755
--- a/app/controllers/DataSetController.scala
+++ b/app/controllers/DataSetController.scala
@@ -116,7 +116,7 @@ class DataSetController @Inject()(userService: UserService,
           dataLayerName) ~> NOT_FOUND
         image <- imageFromCacheIfPossible(dataSet)
       } yield {
-        addRemoteOriginHeaders(Ok(image)).as("image/jpeg").withHeaders(CACHE_CONTROL -> "public, max-age=86400")
+        addRemoteOriginHeaders(Ok(image)).as(jpegMimeType).withHeaders(CACHE_CONTROL -> "public, max-age=86400")
       }
     }
 
diff --git a/app/controllers/SitemapController.scala b/app/controllers/SitemapController.scala
index 438515e0ce5..eccb1e0fd8a 100644
--- a/app/controllers/SitemapController.scala
+++ b/app/controllers/SitemapController.scala
@@ -15,7 +15,7 @@ class SitemapController @Inject()(sitemapWriter: SitemapWriter, sil: Silhouette[
     val downloadStream = sitemapWriter.toSitemapStream(prefix)
 
     Ok.chunked(Source.fromPublisher(IterateeStreams.enumeratorToPublisher(downloadStream)))
-      .as("application/xml")
+      .as(xmlMimeType)
       .withHeaders(CONTENT_DISPOSITION ->
         """sitemap.xml""")
   }
diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala
index 31a8d1a6c09..87b073774d8 100755
--- a/app/models/annotation/AnnotationService.scala
+++ b/app/models/annotation/AnnotationService.scala
@@ -833,7 +833,7 @@ class AnnotationService @Inject()(
     }
 
   //for Explorative Annotations list
-  def compactWrites(annotation: Annotation)(implicit ctx: DBAccessContext): Fox[JsObject] =
+  def compactWrites(annotation: Annotation): Fox[JsObject] =
     for {
       dataSet <- dataSetDAO.findOne(annotation._dataSet)(GlobalAccessContext) ?~> "dataSet.notFoundForAnnotation"
       organization <- organizationDAO.findOne(dataSet._organization)(GlobalAccessContext) ?~> "organization.notFound"
diff --git a/util/src/main/scala/com/scalableminds/util/mvc/ExtendedController.scala b/util/src/main/scala/com/scalableminds/util/mvc/ExtendedController.scala
index 5bc3138ed78..e21a5fbdf53 100644
--- a/util/src/main/scala/com/scalableminds/util/mvc/ExtendedController.scala
+++ b/util/src/main/scala/com/scalableminds/util/mvc/ExtendedController.scala
@@ -170,6 +170,14 @@ class JsonResult(status: Int)
     Json.obj("messages" -> messages.map(m => Json.obj(m._1 -> m._2)))
 }
 
+trait MimeTypes {
+  val jpegMimeType: String = "image/jpeg"
+  val protobufMimeType: String = "application/x-protobuf"
+  val xmlMimeType: String = "application/xml"
+  val zipMimeType: String = "application/zip"
+  val jsonMimeType: String = "application/json"
+}
+
 trait JsonResults extends JsonResultAttribues {
   val JsonOk = new JsonResult(OK)
   val JsonBadRequest = new JsonResult(BAD_REQUEST)
@@ -217,6 +225,7 @@ trait ExtendedController
     with WithFilters
     with I18nSupport
     with InjectedController
+    with MimeTypes
     with ValidationHelpers
     with LazyLogging
     with RequestTokenHelper
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala
index ff8cd1d5f47..a366583833c 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala
@@ -200,7 +200,7 @@ class BinaryDataController @Inject()(
         firstSheet <- spriteSheet.pages.headOption ?~> "image.page.failed"
         outputStream = new ByteArrayOutputStream()
         _ = new JPEGWriter().writeToOutputStream(firstSheet.image)(outputStream)
-      } yield Ok(outputStream.toByteArray).as("image/jpeg")
+      } yield Ok(outputStream.toByteArray).as(jpegMimeType)
     }
   }
   @ApiOperation(hidden = true, value = "")
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
index 6d8214d884c..50c12ff33b1 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala
@@ -321,7 +321,7 @@ Expects:
           dataLayerName,
           mappingName,
           agglomerateId) ?~> "agglomerateSkeleton.failed"
-      } yield Ok(skeleton.toByteArray).as("application/x-protobuf")
+      } yield Ok(skeleton.toByteArray).as(protobufMimeType)
     }
   }
 
@@ -337,12 +337,10 @@ Expects:
     accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationName)),
                                       token) {
       for {
-        _ <- Fox.successful(logger.info(s"Generating agglomerate graph..."))
         agglomerateGraph <- binaryDataServiceHolder.binaryDataService.agglomerateService.generateAgglomerateGraph(
           AgglomerateFileKey(organizationName, dataSetName, dataLayerName, mappingName),
           agglomerateId) ?~> "agglomerateGraph.failed"
-        _ <- Fox.successful(logger.info(s"Serializing agglomerate graph ${agglomerateGraph}..."))
-      } yield Ok(Json.toJson(agglomerateGraph))
+      } yield Ok(agglomerateGraph.toByteArray).as(protobufMimeType)
     }
   }
 
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala
deleted file mode 100644
index c02df4ed06d..00000000000
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/AgglomerateGraph.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-package com.scalableminds.webknossos.datastore.models
-
-import com.scalableminds.util.geometry.Vec3Int
-import com.scalableminds.webknossos.datastore.EditableMapping.AgglomerateEdge
-import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits
-import play.api.libs.json.{Json, OFormat}
-
-case class AgglomerateGraph(segments: List[Long],
-                            edges: List[(Long, Long)],
-                            positions: List[Vec3Int],
-                            affinities: List[Float])
-    extends ProtoGeometryImplicits {
-  override def toString: String =
-    f"AgglomerateGraph(${segments.length} segments, ${edges.length} edges)"
-
-  def toProto: com.scalableminds.webknossos.datastore.EditableMapping.AgglomerateGraph =
-    com.scalableminds.webknossos.datastore.EditableMapping.AgglomerateGraph(
-      segments = segments,
-      edges = edges.map(e => AgglomerateEdge(e._1, e._2)),
-      positions = positions.map(vec3IntToProto),
-      affinities = affinities
-    )
-}
-
-object AgglomerateGraph extends ProtoGeometryImplicits {
-  implicit val jsonFormat: OFormat[AgglomerateGraph] = Json.format[AgglomerateGraph]
-
-  def empty: AgglomerateGraph = AgglomerateGraph(List.empty, List.empty, List.empty, List.empty)
-
-  def fromProto(agglomerateGraphProto: com.scalableminds.webknossos.datastore.EditableMapping.AgglomerateGraph)
-    : AgglomerateGraph =
-    AgglomerateGraph(
-      segments = agglomerateGraphProto.segments.toList,
-      edges = agglomerateGraphProto.edges.map(e => (e.source, e.target)).toList,
-      positions = agglomerateGraphProto.positions.map(vec3IntFromProto).toList,
-      affinities = agglomerateGraphProto.affinities.toList
-    )
-}
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala
index bb7686a09eb..5eee4f25d78 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala
@@ -4,6 +4,7 @@ import java.io.File
 
 import akka.stream.scaladsl.Source
 import akka.util.ByteString
+import com.scalableminds.util.mvc.MimeTypes
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common.{Failure, Full}
@@ -15,7 +16,10 @@ import scalapb.{GeneratedMessage, GeneratedMessageCompanion}
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration._
 
-class RPCRequest(val id: Int, val url: String, wsClient: WSClient) extends FoxImplicits with LazyLogging {
+class RPCRequest(val id: Int, val url: String, wsClient: WSClient)
+    extends FoxImplicits
+    with LazyLogging
+    with MimeTypes {
 
   var request: WSRequest = wsClient.url(url)
   private var verbose: Boolean = true
@@ -104,59 +108,45 @@ class RPCRequest(val id: Int, val url: String, wsClient: WSClient) extends FoxIm
   }
 
   def post[T: Writes](body: T = Json.obj()): Fox[WSResponse] = {
-    request = request
-      .addHttpHeaders(HeaderNames.CONTENT_TYPE -> "application/json")
-      .withBody(Json.toJson(body))
-      .withMethod("POST")
+    request =
+      request.addHttpHeaders(HeaderNames.CONTENT_TYPE -> jsonMimeType).withBody(Json.toJson(body)).withMethod("POST")
     performRequest
   }
 
   def put[T: Writes](body: T = Json.obj()): Fox[WSResponse] = {
-    request = request
-      .addHttpHeaders(HeaderNames.CONTENT_TYPE -> "application/json")
-      .withBody(Json.toJson(body))
-      .withMethod("PUT")
+    request =
+      request.addHttpHeaders(HeaderNames.CONTENT_TYPE -> jsonMimeType).withBody(Json.toJson(body)).withMethod("PUT")
     performRequest
   }
 
   def patch[T: Writes](body: T = Json.obj()): Fox[WSResponse] = {
-    request = request
-      .addHttpHeaders(HeaderNames.CONTENT_TYPE -> "application/json")
-      .withBody(Json.toJson(body))
-      .withMethod("PATCH")
+    request =
+      request.addHttpHeaders(HeaderNames.CONTENT_TYPE -> jsonMimeType).withBody(Json.toJson(body)).withMethod("PATCH")
     performRequest
   }
 
   def postJsonWithJsonResponse[T: Writes, U: Reads](body: T = Json.obj()): Fox[U] = {
-    request = request
-      .addHttpHeaders(HeaderNames.CONTENT_TYPE -> "application/json")
-      .withBody(Json.toJson(body))
-      .withMethod("POST")
+    request =
+      request.addHttpHeaders(HeaderNames.CONTENT_TYPE -> jsonMimeType).withBody(Json.toJson(body)).withMethod("POST")
     parseJsonResponse(performRequest)
   }
 
   def postJsonWithProtoResponse[J: Writes, T <: GeneratedMessage](body: J = Json.obj())(
       companion: GeneratedMessageCompanion[T]): Fox[T] = {
-    request = request
-      .addHttpHeaders(HeaderNames.CONTENT_TYPE -> "application/json")
-      .withBody(Json.toJson(body))
-      .withMethod("POST")
+    request =
+      request.addHttpHeaders(HeaderNames.CONTENT_TYPE -> jsonMimeType).withBody(Json.toJson(body)).withMethod("POST")
     parseProtoResponse(performRequest)(companion)
   }
 
   def postJson[J: Writes](body: J = Json.obj()): Unit = {
-    request = request
-      .addHttpHeaders(HeaderNames.CONTENT_TYPE -> "application/json")
-      .withBody(Json.toJson(body))
-      .withMethod("POST")
+    request =
+      request.addHttpHeaders(HeaderNames.CONTENT_TYPE -> jsonMimeType).withBody(Json.toJson(body)).withMethod("POST")
     performRequest
   }
 
   def postProtoWithJsonResponse[T <: GeneratedMessage, J: Reads](body: T): Fox[J] = {
-    request = request
-      .addHttpHeaders(HeaderNames.CONTENT_TYPE -> "application/x-protobuf")
-      .withBody(body.toByteArray)
-      .withMethod("POST")
+    request =
+      request.addHttpHeaders(HeaderNames.CONTENT_TYPE -> protobufMimeType).withBody(body.toByteArray).withMethod("POST")
     parseJsonResponse(performRequest)
   }
 
@@ -272,7 +262,7 @@ class RPCRequest(val id: Int, val url: String, wsClient: WSClient) extends FoxIm
   private def requestBodyPreview: String =
     request.body match {
       case body: InMemoryBody
-          if request.headers.getOrElse(HeaderNames.CONTENT_TYPE, List()).contains("application/x-protobuf") =>
+          if request.headers.getOrElse(HeaderNames.CONTENT_TYPE, List()).contains(protobufMimeType) =>
         s"<${body.bytes.length} bytes of protobuf data>"
       case body: InMemoryBody =>
         body.bytes.take(100).utf8String + (if (body.bytes.size > 100) s"... <omitted ${body.bytes.size - 100} bytes>"
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index b2fd4d73cdb..f92f7791179 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -4,13 +4,12 @@ import java.nio._
 import java.nio.file.{Files, Paths}
 
 import ch.systemsx.cisd.hdf5._
-import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.io.PathUtils
 import com.scalableminds.webknossos.datastore.DataStoreConfig
+import com.scalableminds.webknossos.datastore.EditableMapping.{AgglomerateEdge, AgglomerateGraph}
 import com.scalableminds.webknossos.datastore.SkeletonTracing.{Edge, SkeletonTracing, Tree}
 import com.scalableminds.webknossos.datastore.geometry.Vec3IntProto
 import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, SkeletonTracingDefaults}
-import com.scalableminds.webknossos.datastore.models.AgglomerateGraph
 import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataRequest
 import com.scalableminds.webknossos.datastore.storage._
 import com.typesafe.scalalogging.LazyLogging
@@ -228,7 +227,7 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
       val edgesRange: Array[Long] =
         reader.uint64().readArrayBlockWithOffset("/agglomerate_to_edges_offsets", 2, agglomerateId)
 
-      logger.info(s"positionsRange: ${positionsRange(0)} to ${positionsRange(1)}, agglomerateId: ${agglomerateId}")
+      logger.info(s"positionsRange: ${positionsRange(0)} to ${positionsRange(1)}, agglomerateId: $agglomerateId")
 
       val nodeCount = positionsRange(1) - positionsRange(0)
       val edgeCount = edgesRange(1) - edgesRange(0)
@@ -259,10 +258,10 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
           reader.float32().readArrayBlockWithOffset("/agglomerate_to_affinities", edgeCount.toInt, edgesRange(0))
 
       AgglomerateGraph(
-        segments = segmentIds.toList,
-        edges = edges.toList.map(e => (segmentIds(e(0).toInt), segmentIds(e(1).toInt))),
-        positions = positions.toList.map(pos => Vec3Int(pos(0).toInt, pos(1).toInt, pos(2).toInt)),
-        affinities = affinities.toList
+        segments = segmentIds,
+        edges = edges.map(e => AgglomerateEdge(source = segmentIds(e(0).toInt), target = segmentIds(e(1).toInt))),
+        positions = positions.map(pos => Vec3IntProto(pos(0).toInt, pos(1).toInt, pos(2).toInt)),
+        affinities = affinities
       )
     }
 
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
index 76454911183..307ebd1dc4d 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
@@ -5,8 +5,9 @@ import akka.http.caching.scaladsl.{Cache, CachingSettings}
 import com.google.inject.Inject
 import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.datastore.EditableMapping.AgglomerateGraph
 import com.scalableminds.webknossos.datastore.helpers.MissingBucketHeaders
-import com.scalableminds.webknossos.datastore.models.{AgglomerateGraph, WebKnossosDataRequest}
+import com.scalableminds.webknossos.datastore.models.WebKnossosDataRequest
 import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.RemoteFallbackLayer
 import com.typesafe.scalalogging.LazyLogging
@@ -110,7 +111,7 @@ class TSRemoteDatastoreClient @Inject()(
       remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer)
       result <- rpc(s"$remoteLayerUri/agglomerates/$baseMappingName/agglomerateGraph/$agglomerateId")
         .addQueryStringOptional("token", userToken)
-        .getWithJsonResponse[AgglomerateGraph]
+        .getWithProtoResponse[AgglomerateGraph](AgglomerateGraph)
     } yield result
 
   def getLargestAgglomerateId(remoteFallbackLayer: RemoteFallbackLayer,
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala
index f953084b72c..7b50e6b435d 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala
@@ -88,7 +88,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C
           for {
             tracing <- tracingService.find(tracingId, version, applyUpdates = true) ?~> Messages("tracing.notFound")
           } yield {
-            Ok(tracing.toByteArray).as("application/x-protobuf")
+            Ok(tracing.toByteArray).as(protobufMimeType)
           }
         }
       }
@@ -101,7 +101,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C
           for {
             tracings <- tracingService.findMultiple(request.body, applyUpdates = true)
           } yield {
-            Ok(tracings.toByteArray).as("application/x-protobuf")
+            Ok(tracings.toByteArray).as(protobufMimeType)
           }
         }
       }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index a8e78427407..0cfb4321cb9 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -8,7 +8,6 @@ import com.google.inject.Inject
 import com.scalableminds.util.geometry.{BoundingBox, Vec3Int}
 import com.scalableminds.util.tools.ExtendedTypes.ExtendedString
 import com.scalableminds.util.tools.Fox
-import com.scalableminds.webknossos.datastore.EditableMapping.EditableMappingProto
 import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings}
 import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser
 import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits
@@ -18,9 +17,7 @@ import com.scalableminds.webknossos.datastore.models.{WebKnossosDataRequest, Web
 import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.scalableminds.webknossos.datastore.services.UserAccessRequest
 import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService
-import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, UpdateActionGroup}
 import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{
-  EditableMapping,
   EditableMappingService,
   EditableMappingUpdateActionGroup,
   RemoteFallbackLayer
@@ -30,6 +27,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{
   UpdateMappingNameAction,
   VolumeTracingService
 }
+import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, UpdateActionGroup}
 import com.scalableminds.webknossos.tracingstore.{
   TSRemoteDatastoreClient,
   TSRemoteWebKnossosClient,
@@ -523,32 +521,4 @@ class VolumeTracingController @Inject()(
       }
   }
 
-  def test: Action[AnyContent] = Action {
-    val mapping = EditableMapping.createDummy(50000, 500)
-    for (_ <- 1 to 10) {
-      val before = System.currentTimeMillis()
-      val bytes = toJsonBytes(mapping)
-      val afterTo = System.currentTimeMillis()
-      val back = fromJsonBytes(bytes)
-      val afterBack = System.currentTimeMillis()
-
-      val durationTo = afterTo - before
-      val durationBack = afterBack - afterTo
-
-      println(f"to json: $durationTo ms, from json: $durationBack ms")
-    }
-    for (_ <- 1 to 10) {
-      val before = System.currentTimeMillis()
-      val bytes = toProtoBytes(mapping.toProto)
-      val afterTo = System.currentTimeMillis()
-      val back = EditableMapping.fromProto(fromProtoBytes[EditableMappingProto](bytes).openOrThrowException("yolo"))
-      val afterBack = System.currentTimeMillis()
-
-      val durationTo = afterTo - before
-      val durationBack = afterBack - afterTo
-
-      println(f"to proto: $durationTo ms, from proto: $durationBack ms")
-    }
-    Ok
-  }
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
index dd5ba7c0a8d..d725ba51c74 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
@@ -1,14 +1,7 @@
 package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
 
-import com.scalableminds.util.geometry.Vec3Int
-import com.scalableminds.util.tools.AdditionalJsonFormats
-import com.scalableminds.webknossos.datastore.EditableMapping.{
-  AgglomerateToGraphPair,
-  EditableMappingProto,
-  SegmentToAgglomeratePair
-}
-import com.scalableminds.webknossos.datastore.models.AgglomerateGraph
-import play.api.libs.json.{Json, OFormat}
+import com.scalableminds.webknossos.datastore.EditableMapping._
+import com.scalableminds.webknossos.datastore.geometry.Vec3IntProto
 
 case class EditableMapping(
     baseMappingName: String,
@@ -21,21 +14,19 @@ case class EditableMapping(
     EditableMappingProto(
       baseMappingName = baseMappingName,
       segmentToAgglomerate = segmentToAgglomerate.map(tuple => SegmentToAgglomeratePair(tuple._1, tuple._2)).toSeq,
-      agglomerateToGraph = agglomerateToGraph.map(tuple => AgglomerateToGraphPair(tuple._1, tuple._2.toProto)).toSeq,
+      agglomerateToGraph = agglomerateToGraph.map(tuple => AgglomerateToGraphPair(tuple._1, tuple._2)).toSeq,
     )
 }
 
-object EditableMapping extends AdditionalJsonFormats {
-  implicit val jsonFormat: OFormat[EditableMapping] = Json.format[EditableMapping]
+object EditableMapping {
 
   def fromProto(editableMappignProto: EditableMappingProto): EditableMapping =
     EditableMapping(
       baseMappingName = editableMappignProto.baseMappingName,
       segmentToAgglomerate =
         editableMappignProto.segmentToAgglomerate.map(pair => pair.segmentId -> pair.agglomerateId).toMap,
-      agglomerateToGraph = editableMappignProto.agglomerateToGraph
-        .map(pair => pair.agglomerateId -> AgglomerateGraph.fromProto(pair.agglomerateGraph))
-        .toMap
+      agglomerateToGraph =
+        editableMappignProto.agglomerateToGraph.map(pair => pair.agglomerateId -> pair.agglomerateGraph).toMap
     )
 
   def createDummy(numSegments: Long, numAgglomerates: Long): EditableMapping =
@@ -46,10 +37,10 @@ object EditableMapping extends AdditionalJsonFormats {
         .to(numAgglomerates)
         .map(a =>
           a -> AgglomerateGraph(
-            segments = 1L.to(numSegments / numAgglomerates).toList,
-            edges = 1L.to(numSegments / numAgglomerates).map(s => s -> s).toList,
-            positions = 1L.to(numSegments / numAgglomerates).map(s => Vec3Int.full(s.toInt)).toList,
-            affinities = 1L.to(numSegments / numAgglomerates).map(s => s.toFloat).toList
+            segments = 1L.to(numSegments / numAgglomerates),
+            edges = 1L.to(numSegments / numAgglomerates).map(s => AgglomerateEdge(s, s)),
+            positions = 1L.to(numSegments / numAgglomerates).map(s => Vec3IntProto(s.toInt, s.toInt, s.toInt)),
+            affinities = 1L.to(numSegments / numAgglomerates).map(s => s.toFloat)
         ))
         .toMap
     )
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 81c4a7ae462..c4dac2633f8 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -8,6 +8,7 @@ import akka.http.caching.scaladsl.{Cache, CachingSettings}
 import com.google.inject.Inject
 import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import com.scalableminds.webknossos.datastore.EditableMapping.{AgglomerateEdge, AgglomerateGraph, EditableMappingProto}
 import com.scalableminds.webknossos.datastore.SkeletonTracing.{Edge, Tree}
 import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing
 import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing.ElementClass
@@ -118,7 +119,7 @@ class EditableMappingService @Inject()(
       agglomerateToGraph = Map()
     )
     for {
-      _ <- tracingDataStore.editableMappings.put(newId, 0L, newEditableMapping)
+      _ <- tracingDataStore.editableMappings.put(newId, 0L, toProtoBytes(newEditableMapping.toProto))
     } yield newId
   }
 
@@ -175,7 +176,8 @@ class EditableMappingService @Inject()(
   ): Fox[EditableMapping] =
     for {
       closestMaterializedVersion: VersionedKeyValuePair[EditableMapping] <- tracingDataStore.editableMappings
-        .get(editableMappingId, Some(desiredVersion))(fromJsonBytes[EditableMapping])
+        .get(editableMappingId, Some(desiredVersion))(bytes =>
+          fromProtoBytes[EditableMappingProto](bytes).map(EditableMapping.fromProto))
       _ = logger.info(
         f"Loading mapping version $desiredVersion, closest materialized is version ${closestMaterializedVersion.version} (${closestMaterializedVersion.value})")
       materialized <- applyPendingUpdates(
@@ -188,7 +190,7 @@ class EditableMappingService @Inject()(
       )
       _ = logger.info(s"Materialized mapping: $materialized")
       _ <- Fox.runIf(shouldPersistMaterialized(closestMaterializedVersion.version, desiredVersion)) {
-        tracingDataStore.editableMappings.put(editableMappingId, desiredVersion, materialized)
+        tracingDataStore.editableMappings.put(editableMappingId, desiredVersion, materialized.toProto)
       }
     } yield materialized
 
@@ -254,15 +256,14 @@ class EditableMappingService @Inject()(
                                userToken: Option[String]): Fox[EditableMapping] =
     for {
       agglomerateGraph <- agglomerateGraphForId(mapping, update.agglomerateId, remoteFallbackLayer, userToken)
-      _ = logger.info(
-        s"Applying one split action on agglomerate ${update.agglomerateId} (previously $agglomerateGraph)...")
+      _ = logger.info(s"Applying one split action on agglomerate ${update.agglomerateId}...")
       segmentId1 <- findSegmentIdAtPosition(remoteFallbackLayer, update.segmentPosition1, update.mag, userToken)
       segmentId2 <- findSegmentIdAtPosition(remoteFallbackLayer, update.segmentPosition2, update.mag, userToken)
       largestExistingAgglomerateId <- largestAgglomerateId(mapping, remoteFallbackLayer, userToken)
       agglomerateId2 = largestExistingAgglomerateId + 1L
       (graph1, graph2) = splitGraph(agglomerateGraph, segmentId1, segmentId2)
       _ = logger.info(
-        s"Graphs after split: Agglomerate ${update.agglomerateId}: $graph1, Aggloemrate $agglomerateId2: $graph2")
+        s"Graphs after split: Agglomerate ${update.agglomerateId}: AgglomerateGraph(${graph1.segments.length} segments, ${graph1.edges.length} edges), Aggloemrate $agglomerateId2: AgglomerateGraph(${graph2.segments.length} segments, ${graph2.edges.length} edges)")
       splitSegmentToAgglomerate = graph2.segments.map(_ -> agglomerateId2).toMap
     } yield
       EditableMapping(
@@ -274,9 +275,9 @@ class EditableMappingService @Inject()(
   private def splitGraph(agglomerateGraph: AgglomerateGraph,
                          segmentId1: Long,
                          segmentId2: Long): (AgglomerateGraph, AgglomerateGraph) = {
-    val edgesAndAffinitiesMinusOne: List[((Long, Long), Float)] =
+    val edgesAndAffinitiesMinusOne: Seq[(AgglomerateEdge, Float)] =
       agglomerateGraph.edges.zip(agglomerateGraph.affinities).filterNot {
-        case ((from, to), _) =>
+        case (AgglomerateEdge(from, to, _), _) =>
           (from == segmentId1 && to == segmentId2) || (from == segmentId2 && to == segmentId1)
       }
     val graph1Nodes: Set[Long] = computeConnectedComponent(startNode = segmentId1, edgesAndAffinitiesMinusOne.map(_._1))
@@ -284,7 +285,7 @@ class EditableMappingService @Inject()(
       case (seg, _) => graph1Nodes.contains(seg)
     }
     val graph1EdgesWithAffinities = edgesAndAffinitiesMinusOne.filter {
-      case (e, _) => graph1Nodes.contains(e._1) && graph1Nodes.contains(e._2)
+      case (e, _) => graph1Nodes.contains(e.source) && graph1Nodes.contains(e.target)
     }
     val graph1 = AgglomerateGraph(
       segments = graph1NodesWithPositions.map(_._1),
@@ -298,7 +299,7 @@ class EditableMappingService @Inject()(
       case (seg, _) => graph2Nodes.contains(seg)
     }
     val graph2EdgesWithAffinities = edgesAndAffinitiesMinusOne.filter {
-      case (e, _) => graph2Nodes.contains(e._1) && graph2Nodes.contains(e._2)
+      case (e, _) => graph2Nodes.contains(e.source) && graph2Nodes.contains(e.target)
     }
     val graph2 = AgglomerateGraph(
       segments = graph2NodesWithPositions.map(_._1),
@@ -309,13 +310,12 @@ class EditableMappingService @Inject()(
     (graph1, graph2)
   }
 
-  private def computeConnectedComponent(startNode: Long, edges: List[(Long, Long)]): Set[Long] = {
+  private def computeConnectedComponent(startNode: Long, edges: Seq[AgglomerateEdge]): Set[Long] = {
     val neighborsByNode =
       mutable.HashMap[Long, List[Long]]().withDefaultValue(List[Long]())
-    edges.foreach {
-      case (from, to) =>
-        neighborsByNode(from) = to :: neighborsByNode(from)
-        neighborsByNode(to) = from :: neighborsByNode(to)
+    edges.foreach { e =>
+      neighborsByNode(e.source) = e.target :: neighborsByNode(e.source)
+      neighborsByNode(e.target) = e.source :: neighborsByNode(e.target)
     }
     val nodesToVisit = mutable.HashSet[Long](startNode)
     val visitedNodes = mutable.HashSet[Long]()
@@ -358,20 +358,21 @@ class EditableMappingService @Inject()(
       EditableMapping(
         baseMappingName = mapping.baseMappingName,
         segmentToAgglomerate = mapping.segmentToAgglomerate ++ mergedSegmentToAgglomerate,
-        agglomerateToGraph = mapping.agglomerateToGraph ++ Map(update.agglomerateId1 -> mergedGraph,
-                                                               update.agglomerateId2 -> AgglomerateGraph.empty)
+        agglomerateToGraph = mapping.agglomerateToGraph ++ Map(
+          update.agglomerateId1 -> mergedGraph,
+          update.agglomerateId2 -> AgglomerateGraph(List.empty, List.empty, List.empty, List.empty))
       )
 
   private def mergeGraph(agglomerateGraph1: AgglomerateGraph,
                          agglomerateGraph2: AgglomerateGraph,
                          segmentId1: Long,
                          segmentId2: Long): AgglomerateGraph = {
-    val newEdge = (segmentId1, segmentId2)
-    val newEdgeAffinity = 255L
+    val newEdge = AgglomerateEdge(segmentId1, segmentId2)
+    val newEdgeAffinity = 255.0f
     AgglomerateGraph(
       segments = agglomerateGraph1.segments ++ agglomerateGraph2.segments,
-      edges = newEdge :: (agglomerateGraph1.edges ++ agglomerateGraph2.edges),
-      affinities = newEdgeAffinity :: (agglomerateGraph1.affinities ++ agglomerateGraph2.affinities),
+      edges = newEdge +: (agglomerateGraph1.edges ++ agglomerateGraph2.edges),
+      affinities = newEdgeAffinity +: (agglomerateGraph1.affinities ++ agglomerateGraph2.affinities),
       positions = agglomerateGraph1.positions ++ agglomerateGraph2.positions
     )
   }
@@ -475,8 +476,8 @@ class EditableMappingService @Inject()(
       }
       segmentIdToNodeIdMinusOne: Map[Long, Int] = graph.segments.zipWithIndex.toMap
       skeletonEdges = graph.edges.map { e =>
-        Edge(source = segmentIdToNodeIdMinusOne(e._1) + nodeIdStartAtOneOffset,
-             target = segmentIdToNodeIdMinusOne(e._2) + nodeIdStartAtOneOffset)
+        Edge(source = segmentIdToNodeIdMinusOne(e.source) + nodeIdStartAtOneOffset,
+             target = segmentIdToNodeIdMinusOne(e.target) + nodeIdStartAtOneOffset)
       }
 
       trees = Seq(
diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
index fc8ef8ad7fa..b6c10346072 100644
--- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
+++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes
@@ -27,7 +27,6 @@ POST        /volume/mergedFromIds                               @com.scalablemin
 POST        /volume/mergedFromContents                          @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromContents(token: Option[String], persist: Boolean)
 
 # Editable Mappings
-GET         /mapping/test                                       @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.test
 POST        /mapping/:tracingId/update                          @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.updateEditableMapping(token: Option[String], tracingId: String)
 GET         /mapping/:tracingId/updateActionLog                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingUpdateActionLog(token: Option[String], tracingId: String)
 GET         /mapping/:tracingId                                 @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingInfo(token: Option[String], tracingId: String)

From 458c8a0c4de06aa320aa9569d6542a7ac8c590d0 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 7 Jun 2022 11:24:20 +0200
Subject: [PATCH 079/122] extract AlfuFoxCache, add cache for largest
 agglomerate id query

---
 project/Dependencies.scala                    |  4 +-
 .../scalableminds/util/cache/AlfuCache.scala  | 55 +++++++++++++++++
 .../datastore/jzarr/ZarrArray.scala           | 18 +-----
 .../TSRemoteDatastoreClient.scala             | 56 ++++++++---------
 .../TSRemoteWebKnossosClient.scala            |  6 +-
 .../EditableMappingService.scala              | 61 ++++---------------
 6 files changed, 101 insertions(+), 99 deletions(-)
 create mode 100644 util/src/main/scala/com/scalableminds/util/cache/AlfuCache.scala

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index f47cf46e2cf..e2c23f2a80b 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -69,7 +69,8 @@ object Dependencies {
     playFramework,
     reactiveBson,
     scalapbRuntime,
-    scalaLogging
+    scalaLogging,
+    akkaCaching
   )
 
   val webknossosDatastoreDependencies: Seq[ModuleID] = Seq(
@@ -77,7 +78,6 @@ object Dependencies {
     grpcServices,
     scalapbRuntimeGrpc,
     akkaLogging,
-    akkaCaching,
     ehcache,
     gson,
     webknossosWrap,
diff --git a/util/src/main/scala/com/scalableminds/util/cache/AlfuCache.scala b/util/src/main/scala/com/scalableminds/util/cache/AlfuCache.scala
new file mode 100644
index 00000000000..a0c30f8ec9c
--- /dev/null
+++ b/util/src/main/scala/com/scalableminds/util/cache/AlfuCache.scala
@@ -0,0 +1,55 @@
+package com.scalableminds.util.cache
+
+import akka.http.caching.LfuCache
+import akka.http.caching.scaladsl.{Cache, CachingSettings}
+import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import net.liftweb.common.Box
+
+import scala.concurrent.ExecutionContext
+import scala.concurrent.duration.{DurationInt, FiniteDuration}
+
+object AlfuCache {
+  def apply[K, V](maxEntries: Int = 1000,
+                  timeToLive: FiniteDuration = 2 hours,
+                  timeToIdle: FiniteDuration = 1 hour): Cache[K, V] = {
+    val defaultCachingSettings = CachingSettings("")
+    val lfuCacheSettings =
+      defaultCachingSettings.lfuCacheSettings
+        .withInitialCapacity(maxEntries)
+        .withMaxCapacity(maxEntries)
+        .withTimeToLive(timeToLive)
+        .withTimeToIdle(timeToIdle)
+    val cachingSettings =
+      defaultCachingSettings.withLfuCacheSettings(lfuCacheSettings)
+    val lfuCache: Cache[K, V] = LfuCache(cachingSettings)
+    lfuCache
+  }
+}
+
+class AlfuFoxCache[K, V](underlyingAkkaCache: Cache[K, Box[V]]) extends FoxImplicits {
+  def getOrLoad(key: K, loadFn: K => Fox[V])(implicit ec: ExecutionContext): Fox[V] =
+    for {
+      box <- underlyingAkkaCache.getOrLoad(key, key => loadFn(key).futureBox)
+      result <- box.toFox
+    } yield result
+
+  def clear(): Unit = underlyingAkkaCache.clear()
+}
+
+object AlfuFoxCache {
+  def apply[K, V](maxEntries: Int = 1000,
+                  timeToLive: FiniteDuration = 2 hours,
+                  timeToIdle: FiniteDuration = 1 hour): AlfuFoxCache[K, V] = {
+    val defaultCachingSettings = CachingSettings("")
+    val lfuCacheSettings =
+      defaultCachingSettings.lfuCacheSettings
+        .withInitialCapacity(maxEntries)
+        .withMaxCapacity(maxEntries)
+        .withTimeToLive(timeToLive)
+        .withTimeToIdle(timeToIdle)
+    val cachingSettings =
+      defaultCachingSettings.withLfuCacheSettings(lfuCacheSettings)
+    val lfuCache: Cache[K, Box[V]] = LfuCache(cachingSettings)
+    new AlfuFoxCache(lfuCache)
+  }
+}
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/jzarr/ZarrArray.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/jzarr/ZarrArray.scala
index 6df97a0c693..68d1bf91a54 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/jzarr/ZarrArray.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/jzarr/ZarrArray.scala
@@ -5,15 +5,14 @@ import java.nio.ByteOrder
 import java.nio.file.Path
 import java.util
 
-import akka.http.caching.LfuCache
-import akka.http.caching.scaladsl.{Cache, CachingSettings}
+import akka.http.caching.scaladsl.Cache
+import com.scalableminds.util.cache.AlfuCache
 import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.tools.Fox
 import com.typesafe.scalalogging.LazyLogging
 import play.api.libs.json.{JsError, JsSuccess, Json}
 import ucar.ma2.{InvalidRangeException, Array => MultiArray}
 
-import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}
 import scala.io.Source
 
@@ -56,19 +55,8 @@ class ZarrArray(relativePath: ZarrPath, store: Store, header: ZarrHeader) extend
   // cache currently limited to 100 MB per array
   private lazy val chunkContentsCache: Cache[String, MultiArray] = {
     val maxSizeBytes = 1000 * 1000 * 100
-
     val maxEntries = maxSizeBytes / header.bytesPerChunk
-    val defaultCachingSettings = CachingSettings("")
-    val lfuCacheSettings =
-      defaultCachingSettings.lfuCacheSettings
-        .withInitialCapacity(maxEntries)
-        .withMaxCapacity(maxEntries)
-        .withTimeToLive(2.hours)
-        .withTimeToIdle(1.hour)
-    val cachingSettings =
-      defaultCachingSettings.withLfuCacheSettings(lfuCacheSettings)
-    val lfuCache: Cache[String, MultiArray] = LfuCache(cachingSettings)
-    lfuCache
+    AlfuCache(maxEntries)
   }
 
   // @return Byte array in fortran-order with little-endian values
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
index 307ebd1dc4d..f5b8d1259fb 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala
@@ -1,8 +1,7 @@
 package com.scalableminds.webknossos.tracingstore
 
-import akka.http.caching.LfuCache
-import akka.http.caching.scaladsl.{Cache, CachingSettings}
 import com.google.inject.Inject
+import com.scalableminds.util.cache.AlfuFoxCache
 import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.EditableMapping.AgglomerateGraph
@@ -11,12 +10,10 @@ import com.scalableminds.webknossos.datastore.models.WebKnossosDataRequest
 import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.RemoteFallbackLayer
 import com.typesafe.scalalogging.LazyLogging
-import net.liftweb.common.Box
 import play.api.http.Status
 import play.api.inject.ApplicationLifecycle
 
-import scala.concurrent.duration.DurationInt
-import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.ExecutionContext
 
 class TSRemoteDatastoreClient @Inject()(
     rpc: RPC,
@@ -26,20 +23,9 @@ class TSRemoteDatastoreClient @Inject()(
     extends LazyLogging
     with MissingBucketHeaders {
 
-  private lazy val dataStoreUriCache: Cache[(Option[String], String), Box[String]] = {
-    val defaultCachingSettings = CachingSettings("")
-    val maxEntries = 1000
-    val lfuCacheSettings =
-      defaultCachingSettings.lfuCacheSettings
-        .withInitialCapacity(maxEntries)
-        .withMaxCapacity(maxEntries)
-        .withTimeToLive(2 hours)
-        .withTimeToIdle(1 hour)
-    val cachingSettings =
-      defaultCachingSettings.withLfuCacheSettings(lfuCacheSettings)
-    val lfuCache: Cache[(Option[String], String), Box[String]] = LfuCache(cachingSettings)
-    lfuCache
-  }
+  private lazy val dataStoreUriCache: AlfuFoxCache[(String, String), String] = AlfuFoxCache()
+  private lazy val largestAgglomerateIdCache: AlfuFoxCache[(RemoteFallbackLayer, String, Option[String]), Long] =
+    AlfuFoxCache()
 
   def fallbackLayerBucket(remoteFallbackLayer: RemoteFallbackLayer,
                           mag: String,
@@ -116,29 +102,35 @@ class TSRemoteDatastoreClient @Inject()(
 
   def getLargestAgglomerateId(remoteFallbackLayer: RemoteFallbackLayer,
                               mappingName: String,
-                              userToken: Option[String]): Fox[Long] =
-    for {
-      remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer)
-      result <- rpc(s"$remoteLayerUri/agglomerates/$mappingName/largestAgglomerateId")
-        .addQueryStringOptional("token", userToken)
-        .getWithJsonResponse[Long]
-    } yield result
+                              userToken: Option[String]): Fox[Long] = {
+    val cacheKey = (remoteFallbackLayer, mappingName, userToken)
+    largestAgglomerateIdCache.getOrLoad(
+      cacheKey,
+      k =>
+        for {
+          remoteLayerUri <- getRemoteLayerUri(k._1)
+          result <- rpc(s"$remoteLayerUri/agglomerates/${k._2}/largestAgglomerateId")
+            .addQueryStringOptional("token", k._3)
+            .getWithJsonResponse[Long]
+        } yield result
+    )
+  }
 
   private def getRemoteLayerUri(remoteLayer: RemoteFallbackLayer): Fox[String] =
     for {
-      datastoreUri <- dataStoreUriFromCache(remoteLayer.organizationName, remoteLayer.dataSetName).toFox
+      datastoreUri <- dataStoreUriWithCache(remoteLayer.organizationName, remoteLayer.dataSetName)
     } yield
       s"$datastoreUri/data/datasets/${remoteLayer.organizationName}/${remoteLayer.dataSetName}/layers/${remoteLayer.layerName}"
 
   private def getRemoteLayerUriZarr(remoteLayer: RemoteFallbackLayer): Fox[String] =
     for {
-      datastoreUri <- dataStoreUriFromCache(remoteLayer.organizationName, remoteLayer.dataSetName).toFox
+      datastoreUri <- dataStoreUriWithCache(remoteLayer.organizationName, remoteLayer.dataSetName)
     } yield
       s"$datastoreUri/data/datasets/${remoteLayer.organizationName}/${remoteLayer.dataSetName}/layers/${remoteLayer.layerName}"
 
-  private def dataStoreUriFromCache(organizationName: String, dataSetName: String): Future[Box[String]] =
+  private def dataStoreUriWithCache(organizationName: String, dataSetName: String): Fox[String] =
     dataStoreUriCache.getOrLoad(
-      (Some(organizationName), dataSetName),
-      keyTuple => remoteWebKnossosClient.getDataStoreUriForDataSource(keyTuple._1, keyTuple._2)
-    )
+      (organizationName, dataSetName),
+      keyTuple => remoteWebKnossosClient.getDataStoreUriForDataSource(keyTuple._1, keyTuple._2))
+
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebKnossosClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebKnossosClient.scala
index 1b64f047b50..33ff1e1cacc 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebKnossosClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebKnossosClient.scala
@@ -55,11 +55,13 @@ class TSRemoteWebKnossosClient @Inject()(
       .addQueryString("key" -> tracingStoreKey)
       .getWithJsonResponse[DataSourceLike]
 
-  def getDataStoreUriForDataSource(organizationNameOpt: Option[String], dataSetName: String): Fox[String] =
+  def getDataStoreUriForDataSource(organizationName: String, dataSetName: String): Fox[String] = {
+    logger.error(s"get uri for data source $organizationName, $dataSetName")
     rpc(s"$webKnossosUrl/api/tracingstores/$tracingStoreName/dataStoreURI/$dataSetName")
-      .addQueryStringOptional("organizationName", organizationNameOpt)
+      .addQueryString("organizationName" -> organizationName)
       .addQueryString("key" -> tracingStoreKey)
       .getWithJsonResponse[String]
+  }
 
   override def requestUserAccess(token: Option[String], accessRequest: UserAccessRequest): Fox[UserAccessAnswer] =
     rpc(s"$webKnossosUrl/api/tracingstores/$tracingStoreName/validateUserAccess")
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index c4dac2633f8..d0332351db0 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -3,9 +3,8 @@ package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
 import java.nio.file.Paths
 import java.util.UUID
 
-import akka.http.caching.LfuCache
-import akka.http.caching.scaladsl.{Cache, CachingSettings}
 import com.google.inject.Inject
+import com.scalableminds.util.cache.AlfuFoxCache
 import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.EditableMapping.{AgglomerateEdge, AgglomerateGraph, EditableMappingProto}
@@ -30,12 +29,11 @@ import com.scalableminds.webknossos.tracingstore.tracings.{
 }
 import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common.Box.tryo
-import net.liftweb.common.{Box, Empty, Full}
+import net.liftweb.common.{Empty, Full}
 import play.api.libs.json.{JsObject, JsValue, Json}
 
 import scala.collection.mutable
 import scala.concurrent.ExecutionContext
-import scala.concurrent.duration._
 
 case class EditableMappingKey(
     editableMappingId: String,
@@ -66,35 +64,11 @@ class EditableMappingService @Inject()(
 
   val binaryDataService = new BinaryDataService(Paths.get(""), 100, null)
 
-  private lazy val materializedEditableMappingCache: Cache[EditableMappingKey, Box[EditableMapping]] = {
-    val maxEntries = 20
-    val defaultCachingSettings = CachingSettings("")
-    val lfuCacheSettings =
-      defaultCachingSettings.lfuCacheSettings
-        .withInitialCapacity(maxEntries)
-        .withMaxCapacity(maxEntries)
-        .withTimeToLive(2 hours)
-        .withTimeToIdle(1 hour)
-    val cachingSettings =
-      defaultCachingSettings.withLfuCacheSettings(lfuCacheSettings)
-    val lfuCache: Cache[EditableMappingKey, Box[EditableMapping]] = LfuCache(cachingSettings)
-    lfuCache
-  }
+  private lazy val materializedEditableMappingCache: AlfuFoxCache[EditableMappingKey, EditableMapping] = AlfuFoxCache(
+    maxEntries = 50)
 
-  private lazy val unmappedRemoteDataCache: Cache[UnmappedRemoteDataKey, Box[(Array[Byte], List[Int])]] = {
-    val maxEntries = 3000
-    val defaultCachingSettings = CachingSettings("")
-    val lfuCacheSettings =
-      defaultCachingSettings.lfuCacheSettings
-        .withInitialCapacity(maxEntries)
-        .withMaxCapacity(maxEntries)
-        .withTimeToLive(2 hours)
-        .withTimeToIdle(1 hour)
-    val cachingSettings =
-      defaultCachingSettings.withLfuCacheSettings(lfuCacheSettings)
-    val lfuCache: Cache[UnmappedRemoteDataKey, Box[(Array[Byte], List[Int])]] = LfuCache(cachingSettings)
-    lfuCache
-  }
+  private lazy val unmappedRemoteDataCache: AlfuFoxCache[UnmappedRemoteDataKey, (Array[Byte], List[Int])] =
+    AlfuFoxCache(maxEntries = 3000)
 
   def newestMaterializableVersion(editableMappingId: String): Fox[Long] =
     tracingDataStore.editableMappingUpdates.getVersion(editableMappingId,
@@ -160,13 +134,9 @@ class EditableMappingService @Inject()(
                            desiredVersion: Long,
   ): Fox[EditableMapping] = {
     val key = EditableMappingKey(editableMappingId, remoteFallbackLayer, userToken, desiredVersion)
-    for {
-      materializedBox <- materializedEditableMappingCache.getOrLoad(
-        key,
-        key =>
-          getVersioned(key.editableMappingId, key.remoteFallbackLayer, key.userToken, key.desiredVersion).futureBox)
-      materialized <- materializedBox.toFox
-    } yield materialized
+    materializedEditableMappingCache.getOrLoad(
+      key,
+      key => getVersioned(key.editableMappingId, key.remoteFallbackLayer, key.userToken, key.desiredVersion))
   }
 
   private def getVersioned(editableMappingId: String,
@@ -511,15 +481,10 @@ class EditableMappingService @Inject()(
 
   def getUnmappedDataFromDatastore(remoteFallbackLayer: RemoteFallbackLayer,
                                    dataRequests: List[WebKnossosDataRequest],
-                                   userToken: Option[String]): Fox[(Array[Byte], List[Int])] = {
-    val key = UnmappedRemoteDataKey(remoteFallbackLayer, dataRequests, userToken)
-    for {
-      resultBox <- unmappedRemoteDataCache.getOrLoad(
-        key,
-        k => remoteDatastoreClient.getData(k.remoteFallbackLayer, k.dataRequests, k.userToken))
-      (data, indices) <- resultBox.toFox
-    } yield (data, indices)
-  }
+                                   userToken: Option[String]): Fox[(Array[Byte], List[Int])] =
+    unmappedRemoteDataCache.getOrLoad(
+      UnmappedRemoteDataKey(remoteFallbackLayer, dataRequests, userToken),
+      k => remoteDatastoreClient.getData(k.remoteFallbackLayer, k.dataRequests, k.userToken))
 
   def collectSegmentIds(data: Array[UnsignedInteger]): Set[Long] =
     data.toSet.map { u: UnsignedInteger =>

From eb55f2eb9451f48eb7fe9e1b734b6ead61c15142 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 7 Jun 2022 11:42:50 +0200
Subject: [PATCH 080/122] minor backend code cleanup

---
 .../scalableminds/webknossos/datastore/models/Positions.scala | 2 +-
 .../webknossos/datastore/models/datasource/DataLayer.scala    | 2 +-
 .../webknossos/datastore/models/requests/Cuboid.scala         | 4 ++--
 .../webknossos/datastore/services/BinaryDataService.scala     | 2 +-
 .../webknossos/datastore/storage/AgglomerateFileCache.scala   | 2 +-
 .../tracings/editablemapping/EditableMappingLayer.scala       | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/Positions.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/Positions.scala
index 03821bb2efb..488b2377775 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/Positions.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/Positions.scala
@@ -121,5 +121,5 @@ class CubePosition(
     new BoundingBox(Vec3Int(mag1X, mag1Y, mag1Z), cubeLength * mag.x, cubeLength * mag.y, cubeLength * mag.z)
 
   override def toString: String =
-    s"CubePos($x,$y,$z,res=$mag)"
+    s"CubePos($x,$y,$z,mag=$mag)"
 }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala
index bc8ef8ac920..f615bedc735 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala
@@ -10,7 +10,7 @@ import com.scalableminds.webknossos.datastore.models.datasource.LayerViewConfigu
 import play.api.libs.json._
 
 object DataFormat extends ExtendedEnumeration {
-  val wkw, zarr, tracing, editableMapping = Value
+  val wkw, zarr, tracing = Value
 }
 
 object Category extends ExtendedEnumeration {
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/requests/Cuboid.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/requests/Cuboid.scala
index 1b201fc9935..b476a251fc5 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/requests/Cuboid.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/requests/Cuboid.scala
@@ -4,7 +4,7 @@ import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.webknossos.datastore.models.{BucketPosition, VoxelPosition}
 
 /**
-  * A cuboid represents a generic cuboid at a specified position.
+  * Mag-aware BoundingBox
   */
 case class Cuboid(topLeft: VoxelPosition, width: Int, height: Int, depth: Int) {
 
@@ -43,5 +43,5 @@ case class Cuboid(topLeft: VoxelPosition, width: Int, height: Int, depth: Int) {
     bucketList
   }
 
-  def resolution: Vec3Int = topLeft.mag
+  def mag: Vec3Int = topLeft.mag
 }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
index 8c8930a4478..00073e5c379 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
@@ -54,7 +54,7 @@ class BinaryDataService(val dataBaseDir: Path, maxCacheSize: Int, val agglomerat
         for {
           data <- handleDataRequest(request)
           mappedData = convertIfNecessary(
-            request.settings.appliedAgglomerate.isDefined && request.dataLayer.category == Category.segmentation && request.cuboid.resolution.maxDim <= MaxMagForAgglomerateMapping,
+            request.settings.appliedAgglomerate.isDefined && request.dataLayer.category == Category.segmentation && request.cuboid.mag.maxDim <= MaxMagForAgglomerateMapping,
             data,
             agglomerateService.applyAgglomerate(request)
           )
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala
index b4794d56c10..625c2b70d0a 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/AgglomerateFileCache.scala
@@ -119,7 +119,7 @@ class BoundingBoxCache(
     val maxReaderRange: ULong) // config value for maximum amount of elements that are allowed to be read as once
     extends LazyLogging {
   private def getGlobalCuboid(cuboid: Cuboid): Cuboid = {
-    val res = cuboid.resolution
+    val res = cuboid.mag
     val tl = cuboid.topLeft
     Cuboid(
       VoxelPosition(tl.voxelXInMag * res.x, tl.voxelYInMag * res.y, tl.voxelZInMag * res.z, Vec3Int(1, 1, 1)),
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
index ab490319c4d..7d2fc111d47 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
@@ -67,7 +67,7 @@ case class EditableMappingLayer(name: String,
                                 tracing: VolumeTracing,
                                 editableMappingService: EditableMappingService)
     extends SegmentationLayer {
-  override def dataFormat: DataFormat.Value = DataFormat.editableMapping
+  override def dataFormat: DataFormat.Value = DataFormat.wkw
 
   override def lengthOfUnderlyingCubes(resolution: Vec3Int): Int = DataLayer.bucketLength
 

From 9e698ea8fdd7c9e69224dccec3d6727196ba74a3 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 7 Jun 2022 11:46:33 +0200
Subject: [PATCH 081/122] re-enable CI checks

---
 .circleci/not-on-master.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.circleci/not-on-master.sh b/.circleci/not-on-master.sh
index c27783316d7..581393ebead 100755
--- a/.circleci/not-on-master.sh
+++ b/.circleci/not-on-master.sh
@@ -1,4 +1,8 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
-echo "Skipping this step, this is a prototype!..."
+if [ "${CIRCLE_BRANCH}" == "master" ]; then
+  echo "Skipping this step on master..."
+else
+  exec "$@"
+fi

From 04a325bedf5297cdde2e6c880fa72b919347bbe1 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 7 Jun 2022 12:02:58 +0200
Subject: [PATCH 082/122] remove unused code

---
 .../scalableminds/util/cache/AlfuCache.scala  |  11 +-
 .../scalableminds/util/tools/JsonHelper.scala | 109 +++++++-----------
 .../controllers/ZarrStreamingController.scala |   6 +-
 .../helpers/MissingBucketHeaders.scala        |   2 +-
 .../webknossos/datastore/rpc/RPCRequest.scala |   5 -
 .../tracings/FossilDBClient.scala             |   2 +-
 .../editablemapping/EditableMapping.scala     |   3 +-
 7 files changed, 48 insertions(+), 90 deletions(-)

diff --git a/util/src/main/scala/com/scalableminds/util/cache/AlfuCache.scala b/util/src/main/scala/com/scalableminds/util/cache/AlfuCache.scala
index a0c30f8ec9c..a6b8c2aea2c 100644
--- a/util/src/main/scala/com/scalableminds/util/cache/AlfuCache.scala
+++ b/util/src/main/scala/com/scalableminds/util/cache/AlfuCache.scala
@@ -40,16 +40,7 @@ object AlfuFoxCache {
   def apply[K, V](maxEntries: Int = 1000,
                   timeToLive: FiniteDuration = 2 hours,
                   timeToIdle: FiniteDuration = 1 hour): AlfuFoxCache[K, V] = {
-    val defaultCachingSettings = CachingSettings("")
-    val lfuCacheSettings =
-      defaultCachingSettings.lfuCacheSettings
-        .withInitialCapacity(maxEntries)
-        .withMaxCapacity(maxEntries)
-        .withTimeToLive(timeToLive)
-        .withTimeToIdle(timeToIdle)
-    val cachingSettings =
-      defaultCachingSettings.withLfuCacheSettings(lfuCacheSettings)
-    val lfuCache: Cache[K, Box[V]] = LfuCache(cachingSettings)
+    val lfuCache: Cache[K, Box[V]] = AlfuCache(maxEntries, timeToLive, timeToIdle)
     new AlfuFoxCache(lfuCache)
   }
 }
diff --git a/util/src/main/scala/com/scalableminds/util/tools/JsonHelper.scala b/util/src/main/scala/com/scalableminds/util/tools/JsonHelper.scala
index 5b512ef6b02..b703b108137 100644
--- a/util/src/main/scala/com/scalableminds/util/tools/JsonHelper.scala
+++ b/util/src/main/scala/com/scalableminds/util/tools/JsonHelper.scala
@@ -7,80 +7,15 @@ import com.scalableminds.util.io.FileIO
 import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common._
 import play.api.i18n.Messages
+import play.api.libs.json._
 import play.api.libs.json.Reads._
 import play.api.libs.json.Writes._
-import play.api.libs.json._
-
 import scala.concurrent.ExecutionContext.Implicits._
+
 import scala.concurrent.duration._
 import scala.io.{BufferedSource, Source}
 
-trait AdditionalJsonFormats {
-  implicit def boxFormat[T: Format]: Format[Box[T]] = new Format[Box[T]] {
-    override def reads(json: JsValue): JsResult[Box[T]] =
-      (json \ "status").validate[String].flatMap {
-        case "Full"    => (json \ "value").validate[T].map(Full(_))
-        case "Empty"   => JsSuccess(Empty)
-        case "Failure" => (json \ "value").validate[String].map(Failure(_))
-        case _         => JsError("invalid status")
-      }
-
-    override def writes(o: Box[T]): JsValue = o match {
-      case Full(t)    => Json.obj("status" -> "Full", "value" -> Json.toJson(t))
-      case Empty      => Json.obj("status" -> "Empty")
-      case f: Failure => Json.obj("status" -> "Failure", "value" -> f.msg)
-    }
-  }
-
-  def oFormat[T](format: Format[T]): OFormat[T] = {
-    val oFormat: OFormat[T] = new OFormat[T]() {
-      override def writes(o: T): JsObject = format.writes(o).as[JsObject]
-      override def reads(json: JsValue): JsResult[T] = format.reads(json)
-    }
-    oFormat
-  }
-
-  implicit object FiniteDurationFormat extends Format[FiniteDuration] {
-    def reads(json: JsValue): JsResult[FiniteDuration] = LongReads.reads(json).map(_.seconds)
-    def writes(o: FiniteDuration): JsValue = LongWrites.writes(o.toSeconds)
-  }
-
-  implicit def optionFormat[T: Format]: Format[Option[T]] = new Format[Option[T]] {
-    override def reads(json: JsValue): JsResult[Option[T]] = json.validateOpt[T]
-
-    override def writes(o: Option[T]): JsValue = o match {
-      case Some(t) ⇒ implicitly[Writes[T]].writes(t)
-      case None ⇒ JsNull
-    }
-  }
-
-  implicit def longMapFormat[T: Format]: Format[Map[Long, T]] = new Format[Map[Long, T]] {
-    override def reads(jsValue: JsValue): JsResult[Map[Long, T]] =
-      jsValue match {
-        case JsObject(map) =>
-          val mapProcessed = map.map {
-            case (k, v: JsValue) => k.toLong -> v.validate[T]
-          }.toMap
-          if (mapProcessed.forall {
-                case (_, _: JsSuccess[T]) => true
-                case _                    => false
-              }) {
-            JsSuccess(mapProcessed.flatMap {
-              case (k, v: JsSuccess[T]) => Some(k -> v.value)
-              case _                    => None
-            })
-          } else JsError()
-        case _ => JsError()
-      }
-
-    override def writes(m: Map[Long, T]): JsValue =
-      JsObject(m.map {
-        case (k, v) => k.toString -> Json.toJson(v)
-      })
-  }
-}
-
-object JsonHelper extends BoxImplicits with LazyLogging with AdditionalJsonFormats {
+object JsonHelper extends BoxImplicits with LazyLogging {
 
   def jsonToFile[A: Writes](path: Path, value: A) =
     FileIO.printToFile(path.toFile) { printer =>
@@ -129,6 +64,44 @@ object JsonHelper extends BoxImplicits with LazyLogging with AdditionalJsonForma
         s"Error at json path '$path': $errorStr."
     }.mkString("\n")
 
+  implicit def boxFormat[T: Format]: Format[Box[T]] = new Format[Box[T]] {
+    override def reads(json: JsValue): JsResult[Box[T]] =
+      (json \ "status").validate[String].flatMap {
+        case "Full"    => (json \ "value").validate[T].map(Full(_))
+        case "Empty"   => JsSuccess(Empty)
+        case "Failure" => (json \ "value").validate[String].map(Failure(_))
+        case _         => JsError("invalid status")
+      }
+
+    override def writes(o: Box[T]): JsValue = o match {
+      case Full(t)    => Json.obj("status" -> "Full", "value" -> Json.toJson(t))
+      case Empty      => Json.obj("status" -> "Empty")
+      case f: Failure => Json.obj("status" -> "Failure", "value" -> f.msg)
+    }
+  }
+
+  def oFormat[T](format: Format[T]): OFormat[T] = {
+    val oFormat: OFormat[T] = new OFormat[T]() {
+      override def writes(o: T): JsObject = format.writes(o).as[JsObject]
+      override def reads(json: JsValue): JsResult[T] = format.reads(json)
+    }
+    oFormat
+  }
+
+  implicit object FiniteDurationFormat extends Format[FiniteDuration] {
+    def reads(json: JsValue): JsResult[FiniteDuration] = LongReads.reads(json).map(_.seconds)
+    def writes(o: FiniteDuration): JsValue = LongWrites.writes(o.toSeconds)
+  }
+
+  implicit def optionFormat[T: Format]: Format[Option[T]] = new Format[Option[T]] {
+    override def reads(json: JsValue): JsResult[Option[T]] = json.validateOpt[T]
+
+    override def writes(o: Option[T]): JsValue = o match {
+      case Some(t) ⇒ implicitly[Writes[T]].writes(t)
+      case None ⇒ JsNull
+    }
+  }
+
   def parseJsonToFox[T: Reads](s: String): Box[T] =
     Json.parse(s).validate[T] match {
       case JsSuccess(parsed, _) =>
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala
index 3f879a57b64..7c7c3bf6946 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala
@@ -1,8 +1,7 @@
 package com.scalableminds.webknossos.datastore.controllers
 
 import com.google.inject.Inject
-import com.scalableminds.util.geometry.{Vec3Double, Vec3Int}
-import com.scalableminds.util.tools.Fox
+import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.webknossos.datastore.DataStoreConfig
 import com.scalableminds.webknossos.datastore.dataformats.wkw.{WKWDataLayer, WKWSegmentationLayer}
 import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser.parseDotCoordinates
@@ -18,8 +17,7 @@ import com.scalableminds.webknossos.datastore.models.VoxelPosition
 import com.scalableminds.webknossos.datastore.services._
 import io.swagger.annotations._
 import play.api.i18n.Messages
-import play.api.libs.json.JsonConfiguration.Aux
-import play.api.libs.json.{Json, JsonConfiguration, OptionHandlers}
+import play.api.libs.json.Json
 import play.api.mvc._
 
 import scala.concurrent.{ExecutionContext, Future}
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/MissingBucketHeaders.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/MissingBucketHeaders.scala
index d4cddd89f9a..0e1198fb755 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/MissingBucketHeaders.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/MissingBucketHeaders.scala
@@ -13,7 +13,7 @@ trait MissingBucketHeaders extends FoxImplicits {
     List(missingBucketsHeader -> formatMissingBucketList(indices),
          "Access-Control-Expose-Headers" -> missingBucketsHeader)
 
-  protected def formatMissingBucketList(indices: List[Int]): String =
+  private def formatMissingBucketList(indices: List[Int]): String =
     "[" + indices.mkString(", ") + "]"
 
   protected def parseMissingBucketHeader(headerLiteralOpt: Option[String])(
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala
index 5eee4f25d78..ad665fb0307 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala
@@ -87,11 +87,6 @@ class RPCRequest(val id: Int, val url: String, wsClient: WSClient)
     performRequest
   }
 
-  def postWithBytesResponse: Fox[Array[Byte]] = {
-    request = request.withMethod("POST")
-    extractBytesResponse(performRequest)
-  }
-
   def postWithJsonResponse[T: Reads]: Fox[T] = {
     request = request.withMethod("POST")
     parseJsonResponse(performRequest)
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
index c2211a931ef..80ed16fce48 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
@@ -16,7 +16,7 @@ import scalapb.{GeneratedMessage, GeneratedMessageCompanion}
 
 import scala.concurrent.ExecutionContext.Implicits.global
 
-trait KeyValueStoreImplicits extends BoxImplicits with LazyLogging {
+trait KeyValueStoreImplicits extends BoxImplicits {
 
   implicit def stringToByteArray(s: String): Array[Byte] = s.toCharArray.map(_.toByte)
 
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
index d725ba51c74..b6152d11de9 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
@@ -1,7 +1,6 @@
 package com.scalableminds.webknossos.tracingstore.tracings.editablemapping
 
 import com.scalableminds.webknossos.datastore.EditableMapping._
-import com.scalableminds.webknossos.datastore.geometry.Vec3IntProto
 
 case class EditableMapping(
     baseMappingName: String,
@@ -29,6 +28,7 @@ object EditableMapping {
         editableMappignProto.agglomerateToGraph.map(pair => pair.agglomerateId -> pair.agglomerateGraph).toMap
     )
 
+  /*
   def createDummy(numSegments: Long, numAgglomerates: Long): EditableMapping =
     EditableMapping(
       baseMappingName = "dummyBaseMapping",
@@ -44,5 +44,6 @@ object EditableMapping {
         ))
         .toMap
     )
+ */
 
 }

From 91e471b5efb27328d362f8ad324a27737c4966da Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 7 Jun 2022 13:33:00 +0200
Subject: [PATCH 083/122] migration guide

---
 MIGRATIONS.unreleased.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/MIGRATIONS.unreleased.md b/MIGRATIONS.unreleased.md
index cdd42d7dc54..2726a336dca 100644
--- a/MIGRATIONS.unreleased.md
+++ b/MIGRATIONS.unreleased.md
@@ -8,4 +8,6 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 ## Unreleased
 [Commits](https://github.com/scalableminds/webknossos/compare/22.06.0...HEAD)
 
+ - Note that this upgrade can not be trivially rolled back, since new rocksDB column families are added and it is not easy to remove them again from an existing database. [#6195](https://github.com/scalableminds/webknossos/pull/6195)
+
 ### Postgres Evolutions:

From 74ee6e068704b13d8699abffb0f551cbff466578 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 7 Jun 2022 15:15:44 +0200
Subject: [PATCH 084/122] [WIP] prevent making editable mapping after volume
 brushing

---
 conf/messages                                 |  1 +
 .../services/BinaryDataService.scala          | 12 +++--
 .../controllers/VolumeTracingController.scala | 47 ++++++++++---------
 .../tracings/FossilDBClient.scala             | 10 ++++
 .../volume/VolumeTracingService.scala         | 18 +++++--
 5 files changed, 58 insertions(+), 30 deletions(-)

diff --git a/conf/messages b/conf/messages
index da462164569..4dbe5020fbb 100644
--- a/conf/messages
+++ b/conf/messages
@@ -278,6 +278,7 @@ annotation.sandbox.skeletonOnly=Sandbox annotations are currently available as s
 annotation.multiLayers.skeleton.notImplemented=This feature is not implemented for annotations with more than one skeleton layer
 annotation.multiLayers.volume.notImplemented=This feature is not implemented for annotations with more than one volume layer
 annotation.noMappingSet=No mapping is pinned for this annotation, cannot generate agglomerate skeleton.
+annotation.volumeBucketsNotEmpty=Cannot make mapping editable in an annotation with mutated volume data
 
 mesh.notFound=Mesh couldn’t be found
 mesh.write.failed=Failed to convert mesh info to json
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
index 00073e5c379..d6444056fac 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
@@ -53,11 +53,13 @@ class BinaryDataService(val dataBaseDir: Path, maxCacheSize: Int, val agglomerat
       case (request, index) =>
         for {
           data <- handleDataRequest(request)
-          mappedData = convertIfNecessary(
-            request.settings.appliedAgglomerate.isDefined && request.dataLayer.category == Category.segmentation && request.cuboid.mag.maxDim <= MaxMagForAgglomerateMapping,
-            data,
-            agglomerateService.applyAgglomerate(request)
-          )
+          mappedData = if (agglomerateService == null) data
+          else
+            convertIfNecessary(
+              request.settings.appliedAgglomerate.isDefined && request.dataLayer.category == Category.segmentation && request.cuboid.mag.maxDim <= MaxMagForAgglomerateMapping,
+              data,
+              agglomerateService.applyAgglomerate(request)
+            )
           convertedData = convertIfNecessary(
             request.dataLayer.elementClass == ElementClass.uint64 && request.dataLayer.category == Category.segmentation,
             mappedData,
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index 0cfb4321cb9..67d2d4428e4 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -452,28 +452,31 @@ class VolumeTracingController @Inject()(
 
   def makeMappingEditable(token: Option[String], tracingId: String): Action[AnyContent] =
     Action.async { implicit request =>
-      accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) {
-        for {
-          tracing <- tracingService.find(tracingId)
-          tracingMappingName <- tracing.mappingName ?~> "annotation.noMappingSet"
-          editableMappingId <- editableMappingService.create(baseMappingName = tracingMappingName)
-          volumeUpdate = UpdateMappingNameAction(Some(editableMappingId),
-                                                 isEditable = Some(true),
-                                                 actionTimestamp = Some(System.currentTimeMillis()))
-          _ <- tracingService.handleUpdateGroup(
-            tracingId,
-            UpdateActionGroup[VolumeTracing](tracing.version + 1,
-                                             System.currentTimeMillis(),
-                                             List(volumeUpdate),
-                                             None,
-                                             None,
-                                             None,
-                                             None,
-                                             None),
-            tracing.version
-          )
-          infoJson <- editableMappingService.infoJson(tracingId = tracingId, editableMappingId = editableMappingId)
-        } yield Ok(infoJson)
+      log() {
+        accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) {
+          for {
+            tracing <- tracingService.find(tracingId)
+            tracingMappingName <- tracing.mappingName ?~> "annotation.noMappingSet"
+            _ <- Fox.assertTrue(tracingService.volumeBucketsAreEmpty(tracingId)) ?~> "annotation.volumeBucketsNotEmpty"
+            editableMappingId <- editableMappingService.create(baseMappingName = tracingMappingName)
+            volumeUpdate = UpdateMappingNameAction(Some(editableMappingId),
+                                                   isEditable = Some(true),
+                                                   actionTimestamp = Some(System.currentTimeMillis()))
+            _ <- tracingService.handleUpdateGroup(
+              tracingId,
+              UpdateActionGroup[VolumeTracing](tracing.version + 1,
+                                               System.currentTimeMillis(),
+                                               List(volumeUpdate),
+                                               None,
+                                               None,
+                                               None,
+                                               None,
+                                               None),
+              tracing.version
+            )
+            infoJson <- editableMappingService.infoJson(tracingId = tracingId, editableMappingId = editableMappingId)
+          } yield Ok(infoJson)
+        }
       }
     }
 
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
index 80ed16fce48..387f421da02 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
@@ -163,6 +163,16 @@ class FossilDBClient(collection: String,
         Fox.failure("could not save to FossilDB: " + e.getMessage)
     }
 
+  def listKeys(limit: Option[Int], startAfterKey: Option[String]): Fox[Seq[String]] =
+    try {
+      val reply = blockingStub.listKeys(ListKeysRequest(collection, limit, startAfterKey))
+      if (!reply.success) throw new Exception(reply.errorMessage.getOrElse(""))
+      Fox.successful(reply.keys)
+    } catch {
+      case e: Exception =>
+        Fox.failure("could not list keys from FossilDB: " + e.getMessage)
+    }
+
   def shutdown(): Boolean = {
     channel.shutdownNow()
     channel.awaitTermination(10, java.util.concurrent.TimeUnit.SECONDS)
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
index 17f62b3f26f..88b84d660e9 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
@@ -19,6 +19,7 @@ import com.scalableminds.webknossos.datastore.models.{BucketPosition, WebKnossos
 import com.scalableminds.webknossos.datastore.services._
 import com.scalableminds.webknossos.tracingstore.tracings.TracingType.TracingType
 import com.scalableminds.webknossos.tracingstore.tracings._
+import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService
 import com.scalableminds.webknossos.tracingstore.{TSRemoteWebKnossosClient, TracingStoreRedisStore}
 import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.common.{Empty, Failure, Full}
@@ -41,7 +42,8 @@ class VolumeTracingService @Inject()(
     val handledGroupIdStore: TracingStoreRedisStore,
     val uncommittedUpdatesStore: TracingStoreRedisStore,
     val temporaryTracingIdStore: TracingStoreRedisStore,
-    val temporaryFileCreator: TemporaryFileCreator
+    val temporaryFileCreator: TemporaryFileCreator,
+    editableMappingService: EditableMappingService
 ) extends TracingService[VolumeTracing]
     with VolumeTracingBucketHelper
     with VolumeTracingDownsampling
@@ -81,12 +83,16 @@ class VolumeTracingService @Inject()(
                         updateGroup: UpdateActionGroup[VolumeTracing],
                         previousVersion: Long): Fox[Unit] =
     for {
-      updatedTracing: VolumeTracing <- updateGroup.actions.foldLeft(find(tracingId)) { (tracingFox, action) =>
+      tracing <- find(tracingId)
+      hasEditableMapping <- Fox.runOptional(tracing.mappingName)(editableMappingService.exists)
+      updatedTracing: VolumeTracing <- updateGroup.actions.foldLeft(Fox.successful(tracing)) { (tracingFox, action) =>
         tracingFox.futureBox.flatMap {
           case Full(tracing) =>
             action match {
               case a: UpdateBucketVolumeAction =>
-                updateBucket(tracingId, tracing, a, updateGroup.version)
+                if (hasEditableMapping.getOrElse(false)) {
+                  Fox.failure("Cannot mutate buckets in annotation with editable mapping.")
+                } else updateBucket(tracingId, tracing, a, updateGroup.version)
               case a: UpdateTracingVolumeAction =>
                 Fox.successful(
                   tracing.copy(
@@ -340,6 +346,12 @@ class VolumeTracingService @Inject()(
       _ <- updateResolutionList(tracingId, tracing, resultingResolutions.toSet)
     } yield ()
 
+  def volumeBucketsAreEmpty(tracingId: String): Fox[Boolean] =
+    for {
+      keyList <- volumeDataStore.listKeys(limit = Some(1), startAfterKey = Some(tracingId))
+      filtered = keyList.filter(_.startsWith(tracingId))
+    } yield filtered.isEmpty
+
   def createIsosurface(tracingId: String, request: WebKnossosIsosurfaceRequest): Fox[(Array[Float], List[Int])] =
     for {
       tracing <- find(tracingId) ?~> "tracing.notFound"

From 26c33ba2fc8fca633960632225e8a2db150b4e59 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 7 Jun 2022 16:48:06 +0200
Subject: [PATCH 085/122] increase agglomerate skeleton max edge limit by one
 order of magnitude since the larger value has been tested thoroughly in
 production

---
 conf/application.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/conf/application.conf b/conf/application.conf
index dd8005e5c40..3457b26a175 100644
--- a/conf/application.conf
+++ b/conf/application.conf
@@ -145,7 +145,7 @@ datastore {
     address = "localhost"
     port = 6379
   }
-  agglomerateSkeleton.maxEdges = 10000
+  agglomerateSkeleton.maxEdges = 100000
 }
 
 # Proxy some routes to prefix + route (only if features.isDemoInstance, route "/" only if logged out)

From 824135675bf918db582b5d5661fc56dcd8bdd67e Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 7 Jun 2022 16:50:27 +0200
Subject: [PATCH 086/122] make proofreading click in 3d view work, adapt
 proofreading defaults, reload agglomerate skeletons even if mesh proofreading
 is disabled

---
 .../controller/combinations/tool_controls.ts  | 40 ++++++--
 .../oxalis/controller/td_controller.tsx       | 40 +++++---
 .../accessors/skeletontracing_accessor.ts     |  6 +-
 .../oxalis/model/sagas/isosurface_saga.ts     | 27 +++--
 .../oxalis/model/sagas/proofread_saga.ts      | 98 +++++++++----------
 5 files changed, 128 insertions(+), 83 deletions(-)

diff --git a/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts b/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts
index 820d568d8df..246f8e116c6 100644
--- a/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts
+++ b/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts
@@ -22,13 +22,17 @@ import {
   handleResizingBoundingBox,
   highlightAndSetCursorOnHoveredBoundingBox,
 } from "oxalis/controller/combinations/bounding_box_handlers";
-import Store from "oxalis/store";
+import Store, { SkeletonTracing } from "oxalis/store";
 import * as Utils from "libs/utils";
 import * as VolumeHandlers from "oxalis/controller/combinations/volume_handlers";
 import { document } from "libs/window";
 import api from "oxalis/api/internal_api";
 import { proofreadAtPosition } from "oxalis/model/actions/proofread_actions";
 import { calculateGlobalPos } from "oxalis/model/accessors/view_mode_accessor";
+import {
+  getNodeAndTree,
+  getSkeletonTracing,
+} from "oxalis/model/accessors/skeletontracing_accessor";
 
 export type ActionDescriptor = {
   leftClick?: string;
@@ -622,15 +626,39 @@ export class BoundingBoxTool {
 export class ProofreadTool {
   static getPlaneMouseControls(_planeId: OrthoView, planeView: PlaneView): any {
     return {
-      leftClick: (pos: Point2, plane: OrthoView, _event: MouseEvent, isTouch: boolean) => {
-        SkeletonHandlers.handleSelectNode(planeView, pos, plane, isTouch);
-
-        const globalPosition = calculateGlobalPos(Store.getState(), pos);
-        Store.dispatch(proofreadAtPosition(globalPosition));
+      leftClick: (pos: Point2, plane: OrthoView, event: MouseEvent, isTouch: boolean) => {
+        this.onLeftClick(planeView, pos, plane, event, isTouch);
       },
     };
   }
 
+  static onLeftClick(
+    planeView: PlaneView,
+    pos: Point2,
+    plane: OrthoView,
+    _event: MouseEvent,
+    isTouch: boolean,
+  ) {
+    const didSelectNode = SkeletonHandlers.handleSelectNode(planeView, pos, plane, isTouch);
+
+    let globalPosition;
+    if (plane === OrthoViews.TDView) {
+      if (didSelectNode) {
+        getSkeletonTracing(Store.getState().tracing).map((skeletonTracing: SkeletonTracing) =>
+          getNodeAndTree(skeletonTracing).map(([_activeTree, activeNode]) => {
+            globalPosition = activeNode.position;
+          }),
+        );
+      }
+    } else {
+      globalPosition = calculateGlobalPos(Store.getState(), pos);
+    }
+
+    if (globalPosition == null) return;
+
+    Store.dispatch(proofreadAtPosition(globalPosition));
+  }
+
   static getActionDescriptors(
     _activeTool: AnnotationTool,
     _useLegacyBindings: boolean,
diff --git a/frontend/javascripts/oxalis/controller/td_controller.tsx b/frontend/javascripts/oxalis/controller/td_controller.tsx
index 46dc263d5ff..89d150a3352 100644
--- a/frontend/javascripts/oxalis/controller/td_controller.tsx
+++ b/frontend/javascripts/oxalis/controller/td_controller.tsx
@@ -2,14 +2,16 @@ import { connect } from "react-redux";
 import * as React from "react";
 import * as THREE from "three";
 import { InputMouse } from "libs/input";
-import type {
+import {
+  AnnotationTool,
+  AnnotationToolEnum,
   OrthoView,
+  OrthoViews,
   OrthoViewMap,
   Point2,
   ShowContextMenuFunction,
   Vector3,
 } from "oxalis/constants";
-import { OrthoViews } from "oxalis/constants";
 import { V3 } from "libs/mjs";
 import { getPosition } from "oxalis/model/accessors/flycam_accessor";
 import { getViewportScale, getInputCatcherRect } from "oxalis/model/accessors/view_mode_accessor";
@@ -33,7 +35,7 @@ import Store from "oxalis/store";
 import TrackballControls from "libs/trackball_controls";
 import * as Utils from "libs/utils";
 import { removeIsosurfaceAction } from "oxalis/model/actions/annotation_actions";
-import { SkeletonTool } from "oxalis/controller/combinations/tool_controls";
+import { ProofreadTool, SkeletonTool } from "oxalis/controller/combinations/tool_controls";
 import { handleOpenContextMenu } from "oxalis/controller/combinations/skeleton_handlers";
 
 export function threeCameraToCameraData(camera: THREE.OrthographicCamera): CameraData {
@@ -57,16 +59,24 @@ export function threeCameraToCameraData(camera: THREE.OrthographicCamera): Camer
 
 function getTDViewMouseControlsSkeleton(planeView: PlaneView): Record<string, any> {
   return {
-    leftClick: (pos: Point2, plane: OrthoView, event: MouseEvent, isTouch: boolean) =>
-      SkeletonTool.onLegacyLeftClick(
-        planeView,
-        pos,
-        event.shiftKey,
-        event.altKey,
-        event.ctrlKey,
-        OrthoViews.TDView,
-        isTouch,
-      ),
+    leftClick: (
+      pos: Point2,
+      plane: OrthoView,
+      event: MouseEvent,
+      isTouch: boolean,
+      activeTool: AnnotationTool,
+    ) =>
+      activeTool === AnnotationToolEnum.PROOFREAD
+        ? ProofreadTool.onLeftClick(planeView, pos, plane, event, isTouch)
+        : SkeletonTool.onLegacyLeftClick(
+            planeView,
+            pos,
+            event.shiftKey,
+            event.altKey,
+            event.ctrlKey,
+            OrthoViews.TDView,
+            isTouch,
+          ),
   };
 }
 
@@ -80,6 +90,7 @@ type OwnProps = {
 type StateProps = {
   flycam: Flycam;
   scale: Vector3;
+  activeTool: AnnotationTool;
 };
 type Props = OwnProps & StateProps;
 
@@ -199,7 +210,7 @@ class TDController extends React.PureComponent<Props> {
       },
       leftClick: (pos: Point2, plane: OrthoView, event: MouseEvent, isTouch: boolean) => {
         if (skeletonControls != null) {
-          skeletonControls.leftClick(pos, plane, event, isTouch);
+          skeletonControls.leftClick(pos, plane, event, isTouch, this.props.activeTool);
         }
 
         if (this.props.planeView == null) {
@@ -327,6 +338,7 @@ export function mapStateToProps(state: OxalisState): StateProps {
   return {
     flycam: state.flycam,
     scale: state.dataset.dataSource.scale,
+    activeTool: state.uiInformation.activeTool,
   };
 }
 const connector = connect(mapStateToProps);
diff --git a/frontend/javascripts/oxalis/model/accessors/skeletontracing_accessor.ts b/frontend/javascripts/oxalis/model/accessors/skeletontracing_accessor.ts
index 76d4be45aac..53ac2dad359 100644
--- a/frontend/javascripts/oxalis/model/accessors/skeletontracing_accessor.ts
+++ b/frontend/javascripts/oxalis/model/accessors/skeletontracing_accessor.ts
@@ -121,7 +121,7 @@ export function getTree(
 }
 export function getNodeAndTree(
   skeletonTracing: SkeletonTracing,
-  nodeId: number | null | undefined,
+  nodeId?: number | null | undefined,
   treeId?: number | null | undefined,
 ): Maybe<[Tree, Node]> {
   let tree;
@@ -129,9 +129,7 @@ export function getNodeAndTree(
   if (treeId != null) {
     tree = skeletonTracing.trees[treeId];
   } else if (nodeId != null) {
-    // Flow doesn't understand that nodeId is not null, otherwise ¯\_(ツ)_/¯
-    const nonNullNodeId = nodeId;
-    tree = _.values(skeletonTracing.trees).find((__) => __.nodes.has(nonNullNodeId));
+    tree = _.values(skeletonTracing.trees).find((__) => __.nodes.has(nodeId));
   } else {
     const { activeTreeId } = skeletonTracing;
 
diff --git a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
index 21338e1a480..152fe0f9e28 100644
--- a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
@@ -70,11 +70,11 @@ const PARALLEL_PRECOMPUTED_MESH_LOADING_COUNT = 6;
  *
  */
 const isosurfacesMapByLayer: Record<string, Map<number, ThreeDMap<boolean>>> = {};
-function cubeSizeInMag1(): Vector3 {
+function marchingCubeSizeInMag1(): Vector3 {
   // @ts-ignore
-  return window.__cubeSizeInMag1 != null
+  return window.__marchingCubeSizeInMag1 != null
     ? // @ts-ignore
-      window.__cubeSizeInMag1
+      window.__marchingCubeSizeInMag1
     : [128, 128, 128];
 }
 const modifiedCells: Set<number> = new Set();
@@ -112,7 +112,7 @@ function removeMapForSegment(layerName: string, segmentId: number): void {
 
 function getZoomedCubeSize(zoomStep: number, resolutionInfo: ResolutionInfo): Vector3 {
   const [x, y, z] = zoomedAddressToAnotherZoomStepWithInfo(
-    [...cubeSizeInMag1(), 0],
+    [...marchingCubeSizeInMag1(), 0],
     resolutionInfo,
     zoomStep,
   );
@@ -121,8 +121,11 @@ function getZoomedCubeSize(zoomStep: number, resolutionInfo: ResolutionInfo): Ve
 }
 
 function clipPositionToCubeBoundary(position: Vector3): Vector3 {
-  const currentCube = Utils.map3((el, idx) => Math.floor(el / cubeSizeInMag1()[idx]), position);
-  const clippedPosition = Utils.map3((el, idx) => el * cubeSizeInMag1()[idx], currentCube);
+  const currentCube = Utils.map3(
+    (el, idx) => Math.floor(el / marchingCubeSizeInMag1()[idx]),
+    position,
+  );
+  const clippedPosition = Utils.map3((el, idx) => el * marchingCubeSizeInMag1()[idx], currentCube);
   return clippedPosition;
 }
 
@@ -139,9 +142,9 @@ const NEIGHBOR_LOOKUP = [
 function getNeighborPosition(clippedPosition: Vector3, neighborId: number): Vector3 {
   const neighborMultiplier = NEIGHBOR_LOOKUP[neighborId];
   const neighboringPosition = [
-    clippedPosition[0] + neighborMultiplier[0] * cubeSizeInMag1()[0],
-    clippedPosition[1] + neighborMultiplier[1] * cubeSizeInMag1()[1],
-    clippedPosition[2] + neighborMultiplier[2] * cubeSizeInMag1()[2],
+    clippedPosition[0] + neighborMultiplier[0] * marchingCubeSizeInMag1()[0],
+    clippedPosition[1] + neighborMultiplier[1] * marchingCubeSizeInMag1()[1],
+    clippedPosition[2] + neighborMultiplier[2] * marchingCubeSizeInMag1()[2],
   ];
   // @ts-expect-error ts-migrate(2322) FIXME: Type 'number[]' is not assignable to type 'Vector3... Remove this comment to see the full error message
   return neighboringPosition;
@@ -349,8 +352,12 @@ function* maybeLoadIsosurface(
 
   batchCounterPerSegment[segmentId]++;
   threeDMap.set(clippedPosition, true);
+  // In general, it is more performant to compute meshes in a more coarse resolution instead of using subsampling strides
+  // since in the coarse resolution less data needs to be loaded. Another possibility to increase performance is
+  // window.__marchingCubeSizeInMag1 which affects the cube size the marching cube algorithm will work on. If the cube is significantly larger than the
+  // segments, computations are wasted.
   // @ts-expect-error ts-migrate(2339) FIXME: Property '__isosurfaceSubsamplingStrides' does not... Remove this comment to see the full error message
-  const subsamplingStrides = window.__isosurfaceSubsamplingStrides || [4, 4, 4];
+  const subsamplingStrides = window.__isosurfaceSubsamplingStrides || [1, 1, 1];
   const scale = yield* select((state) => state.dataset.dataSource.scale);
   const dataStoreHost = yield* select((state) => state.dataset.dataStore.url);
   const owningOrganization = yield* select((state) => state.dataset.owningOrganization);
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 1c47c42754e..50ca6cd5cbe 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -58,14 +58,14 @@ function proofreadCoarseResolutionIndex(): number {
   return window.__proofreadCoarseResolutionIndex != null
     ? // @ts-ignore
       window.__proofreadCoarseResolutionIndex
-    : 4;
+    : 3;
 }
 function proofreadFineResolutionIndex(): number {
   // @ts-ignore
   return window.__proofreadFineResolutionIndex != null
     ? // @ts-ignore
       window.__proofreadFineResolutionIndex
-    : 0;
+    : 2;
 }
 function proofreadUsingMeshes(): boolean {
   // @ts-ignore
@@ -339,16 +339,55 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
 
   yield* call([api.data, api.data.reloadBuckets], layerName);
 
-  if (proofreadUsingMeshes()) {
-    const volumeTracingWithEditableMapping = yield* select((state) =>
-      getActiveSegmentationTracing(state),
+  const volumeTracingWithEditableMapping = yield* select((state) =>
+    getActiveSegmentationTracing(state),
+  );
+  if (
+    volumeTracingWithEditableMapping == null ||
+    volumeTracingWithEditableMapping.mappingName == null
+  )
+    return;
+
+  const newSourceNodeAgglomerateId = yield* call(
+    [api.data, api.data.getDataValue],
+    layerName,
+    sourceNodePosition,
+    agglomerateFileZoomstep,
+  );
+
+  const newTargetNodeAgglomerateId = yield* call(
+    [api.data, api.data.getDataValue],
+    layerName,
+    targetNodePosition,
+    agglomerateFileZoomstep,
+  );
+
+  // Remove old agglomerate skeleton(s) and load new agglomerate skeleton(s)
+  yield* put(deleteTreeAction(sourceTree.treeId));
+  if (sourceTree !== targetTree) {
+    yield* put(deleteTreeAction(targetTree.treeId));
+  }
+
+  yield* call(
+    loadAgglomerateSkeletonWithId,
+    loadAgglomerateSkeletonAction(
+      layerName,
+      volumeTracingWithEditableMapping.mappingName,
+      newSourceNodeAgglomerateId,
+    ),
+  );
+  if (newTargetNodeAgglomerateId !== newSourceNodeAgglomerateId) {
+    yield* call(
+      loadAgglomerateSkeletonWithId,
+      loadAgglomerateSkeletonAction(
+        layerName,
+        volumeTracingWithEditableMapping.mappingName,
+        newTargetNodeAgglomerateId,
+      ),
     );
-    if (
-      volumeTracingWithEditableMapping == null ||
-      volumeTracingWithEditableMapping.mappingName == null
-    )
-      return;
+  }
 
+  if (proofreadUsingMeshes()) {
     // Remove old over segmentation meshes
     if (oldSegmentIdsInSurround != null) {
       // Remove old meshes in oversegmentation
@@ -360,20 +399,6 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
       oldSegmentIdsInSurround = null;
     }
 
-    const newSourceNodeAgglomerateId = yield* call(
-      [api.data, api.data.getDataValue],
-      layerName,
-      sourceNodePosition,
-      agglomerateFileZoomstep,
-    );
-
-    const newTargetNodeAgglomerateId = yield* call(
-      [api.data, api.data.getDataValue],
-      layerName,
-      targetNodePosition,
-      agglomerateFileZoomstep,
-    );
-
     // Remove old agglomerate mesh(es) and load new agglomerate mesh(es)
     yield* put(removeIsosurfaceAction(layerName, sourceNodeAgglomerateId));
     if (targetNodeAgglomerateId !== sourceNodeAgglomerateId) {
@@ -396,30 +421,5 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
         targetNodePosition,
       );
     }
-
-    // Remove old agglomerate skeleton(s) and load new agglomerate skeleton(s)
-    yield* put(deleteTreeAction(sourceTree.treeId));
-    if (sourceTree !== targetTree) {
-      yield* put(deleteTreeAction(targetTree.treeId));
-    }
-
-    yield* call(
-      loadAgglomerateSkeletonWithId,
-      loadAgglomerateSkeletonAction(
-        layerName,
-        volumeTracingWithEditableMapping.mappingName,
-        newSourceNodeAgglomerateId,
-      ),
-    );
-    if (newTargetNodeAgglomerateId !== newSourceNodeAgglomerateId) {
-      yield* call(
-        loadAgglomerateSkeletonWithId,
-        loadAgglomerateSkeletonAction(
-          layerName,
-          volumeTracingWithEditableMapping.mappingName,
-          newTargetNodeAgglomerateId,
-        ),
-      );
-    }
   }
 }

From c3716d241554fc47450b6c6dcc23bdb43c9b4d74 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 7 Jun 2022 17:15:47 +0200
Subject: [PATCH 087/122] improve version typing, show warning when trying to
 restore older editable mapping version

---
 .../oxalis/model/sagas/update_actions.ts      |  2 +-
 .../javascripts/oxalis/view/version_list.tsx  | 32 +++++++++++--------
 .../javascripts/oxalis/view/version_view.tsx  |  5 ++-
 3 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/update_actions.ts b/frontend/javascripts/oxalis/model/sagas/update_actions.ts
index f755dc05657..c2bf134d65c 100644
--- a/frontend/javascripts/oxalis/model/sagas/update_actions.ts
+++ b/frontend/javascripts/oxalis/model/sagas/update_actions.ts
@@ -535,7 +535,7 @@ export function updateTdCamera(): UpdateTdCameraUpdateAction {
     value: {},
   };
 }
-export function serverCreateTracing(timestamp: number) {
+export function serverCreateTracing(timestamp: number): AsServerAction<CreateTracingUpdateAction> {
   return {
     name: "createTracing",
     value: {
diff --git a/frontend/javascripts/oxalis/view/version_list.tsx b/frontend/javascripts/oxalis/view/version_list.tsx
index 76c6913d996..1cde1b7b751 100644
--- a/frontend/javascripts/oxalis/view/version_list.tsx
+++ b/frontend/javascripts/oxalis/view/version_list.tsx
@@ -12,7 +12,11 @@ import {
   SaveQueueType,
   setVersionNumberAction,
 } from "oxalis/model/actions/save_actions";
-import { revertToVersion, serverCreateTracing } from "oxalis/model/sagas/update_actions";
+import {
+  revertToVersion,
+  serverCreateTracing,
+  type ServerUpdateAction,
+} from "oxalis/model/sagas/update_actions";
 import { setAnnotationAllowUpdateAction } from "oxalis/model/actions/annotation_actions";
 import { setVersionRestoreVisibilityAction } from "oxalis/model/actions/ui_actions";
 import Model from "oxalis/model";
@@ -20,6 +24,7 @@ import type { EditableMapping, SkeletonTracing, VolumeTracing } from "oxalis/sto
 import Store from "oxalis/store";
 import VersionEntryGroup from "oxalis/view/version_entry_group";
 import api from "oxalis/api/internal_api";
+import Toast from "libs/toast";
 type Props = {
   versionedObjectType: SaveQueueType;
   tracing: SkeletonTracing | VolumeTracing | EditableMapping;
@@ -92,12 +97,13 @@ class VersionList extends React.Component<Props, State> {
         tracingId,
         this.props.versionedObjectType,
       );
-      // Insert version 0
-      updateActionLog.push({
-        version: 0,
-        // @ts-expect-error ts-migrate(2322) FIXME: Type '{ name: string; value: { actionTimestamp: nu... Remove this comment to see the full error message
-        value: [serverCreateTracing(this.props.tracing.createdTimestamp)],
-      });
+      if (this.props.tracing.type !== "mapping") {
+        // Insert version 0
+        updateActionLog.push({
+          version: 0,
+          value: [serverCreateTracing(this.props.tracing.createdTimestamp)],
+        });
+      }
       this.setState({
         versions: updateActionLog,
       });
@@ -147,12 +153,15 @@ class VersionList extends React.Component<Props, State> {
       return previewVersion({
         skeleton: version,
       });
-    } else {
+    } else if (this.props.versionedObjectType === "volume") {
       return previewVersion({
         volumes: {
           [this.props.tracing.tracingId]: version,
         },
       });
+    } else {
+      Toast.warning(`Version preview for ${this.props.versionedObjectType}s is not supported yet.`);
+      return Promise.resolve();
     }
   };
 
@@ -168,13 +177,10 @@ class VersionList extends React.Component<Props, State> {
           .calendar(null, MOMENT_CALENDAR_FORMAT),
       );
 
-      // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'batch' implicitly has an 'any' type.
-      const getBatchTime = (batch) =>
-        // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'action' implicitly has an 'any' type.
-        _.max(batch.value.map((action) => action.value.actionTimestamp));
+      const getBatchTime = (batch: APIUpdateActionBatch): number =>
+        _.max(batch.value.map((action: ServerUpdateAction) => action.value.actionTimestamp)) || 0;
 
       return _.mapValues(groupedVersions, (versionsOfOneDay) =>
-        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type '(batch: any) => unknown' is not ... Remove this comment to see the full error message
         chunkIntoTimeWindows(versionsOfOneDay, getBatchTime, 5),
       );
     },
diff --git a/frontend/javascripts/oxalis/view/version_view.tsx b/frontend/javascripts/oxalis/view/version_view.tsx
index 2942fb3d920..92e63aaa8a9 100644
--- a/frontend/javascripts/oxalis/view/version_view.tsx
+++ b/frontend/javascripts/oxalis/view/version_view.tsx
@@ -133,7 +133,10 @@ class VersionView extends React.Component<Props, State> {
             ))}
             {this.props.tracing.mappings.map((mapping) => (
               <TabPane
-                tab={getReadableNameByVolumeTracingId(this.props.tracing, mapping.tracingId)}
+                tab={`${getReadableNameByVolumeTracingId(
+                  this.props.tracing,
+                  mapping.tracingId,
+                )} (Editable Mapping)`}
                 key={`${mapping.tracingId}-${mapping.mappingName}`}
               >
                 <VersionList

From 8cb2bda948f4e355b965f4d7a1b2a70ea5a652e8 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 7 Jun 2022 17:31:48 +0200
Subject: [PATCH 088/122] disable proofreading tool for non-hybrid annotations

---
 .../oxalis/model/accessors/tool_accessor.ts   |  5 +-
 .../oxalis/view/action-bar/toolbar_view.tsx   | 67 +++++++++----------
 2 files changed, 34 insertions(+), 38 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
index f1c175d9bd4..7237510df4c 100644
--- a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
+++ b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
@@ -82,10 +82,7 @@ function _getDisabledInfoWhenVolumeIsDisabled(
     [AnnotationToolEnum.FILL_CELL]: disabledInfo,
     [AnnotationToolEnum.PICK_CELL]: disabledInfo,
     [AnnotationToolEnum.BOUNDING_BOX]: notDisabledInfo,
-    [AnnotationToolEnum.PROOFREAD]: {
-      isDisabled: !hasSkeleton,
-      explanation: disabledSkeletonExplanation,
-    },
+    [AnnotationToolEnum.PROOFREAD]: disabledInfo,
   };
 }
 
diff --git a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
index 1d166f7ee97..0f7fba63bca 100644
--- a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
+++ b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
@@ -578,49 +578,48 @@ export default function ToolbarView() {
         </RadioButtonWithTooltip>
 
         {hasSkeleton ? (
-          <>
-            <RadioButtonWithTooltip
-              title={skeletonToolDescription}
-              disabledTitle=""
-              disabled={disabledInfosForTools[AnnotationToolEnum.SKELETON].isDisabled}
-              style={narrowButtonStyle}
-              value={AnnotationToolEnum.SKELETON}
-            >
-              {/*
+          <RadioButtonWithTooltip
+            title={skeletonToolDescription}
+            disabledTitle=""
+            disabled={disabledInfosForTools[AnnotationToolEnum.SKELETON].isDisabled}
+            style={narrowButtonStyle}
+            value={AnnotationToolEnum.SKELETON}
+          >
+            {/*
            When visible changes to false, the tooltip fades out in an animation. However, skeletonToolHint
            will be null, too, which means the tooltip text would immediately change to an empty string.
            To avoid this, we fallback to previousSkeletonToolHint.
           */}
-              <Tooltip
-                title={skeletonToolHint || previousSkeletonToolHint}
-                visible={skeletonToolHint != null}
-              >
-                <i
-                  style={{
-                    paddingLeft: 4,
-                    opacity: disabledInfosForTools[AnnotationToolEnum.SKELETON].isDisabled
-                      ? 0.5
-                      : 1,
-                  }}
-                  className="fas fa-project-diagram"
-                />
-              </Tooltip>
-            </RadioButtonWithTooltip>
-            <RadioButtonWithTooltip
-              title="Proofreading Tool - Modify an agglomerated segmentation."
-              disabledTitle={disabledInfosForTools[AnnotationToolEnum.PROOFREAD].explanation}
-              disabled={disabledInfosForTools[AnnotationToolEnum.PROOFREAD].isDisabled}
-              style={narrowButtonStyle}
-              value={AnnotationToolEnum.PROOFREAD}
+            <Tooltip
+              title={skeletonToolHint || previousSkeletonToolHint}
+              visible={skeletonToolHint != null}
             >
               <i
-                className="fas fa-clipboard-check"
                 style={{
-                  opacity: disabledInfosForTools[AnnotationToolEnum.PROOFREAD].isDisabled ? 0.5 : 1,
+                  paddingLeft: 4,
+                  opacity: disabledInfosForTools[AnnotationToolEnum.SKELETON].isDisabled ? 0.5 : 1,
                 }}
+                className="fas fa-project-diagram"
               />
-            </RadioButtonWithTooltip>
-          </>
+            </Tooltip>
+          </RadioButtonWithTooltip>
+        ) : null}
+
+        {hasSkeleton && isVolumeSupported ? (
+          <RadioButtonWithTooltip
+            title="Proofreading Tool - Modify an agglomerated segmentation."
+            disabledTitle={disabledInfosForTools[AnnotationToolEnum.PROOFREAD].explanation}
+            disabled={disabledInfosForTools[AnnotationToolEnum.PROOFREAD].isDisabled}
+            style={narrowButtonStyle}
+            value={AnnotationToolEnum.PROOFREAD}
+          >
+            <i
+              className="fas fa-clipboard-check"
+              style={{
+                opacity: disabledInfosForTools[AnnotationToolEnum.PROOFREAD].isDisabled ? 0.5 : 1,
+              }}
+            />
+          </RadioButtonWithTooltip>
         ) : null}
 
         {isVolumeSupported ? (

From 0433cfd66eb2ca109967841690c076a093346a44 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 7 Jun 2022 18:30:44 +0200
Subject: [PATCH 089/122] disallow to change mapping after editable mapping was
 activated

---
 .../model/accessors/volumetracing_accessor.ts | 28 +++++++++++++++++++
 .../oxalis/model/reducers/settings_reducer.ts | 16 +++++++++++
 .../model/reducers/volumetracing_reducer.ts   |  6 ++++
 .../reducers/volumetracing_reducer_helpers.ts |  2 ++
 .../oxalis/model/sagas/mapping_saga.ts        | 11 ++++++--
 .../mapping_settings_view.tsx                 |  5 ++++
 6 files changed, 65 insertions(+), 3 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts b/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts
index d5524778912..5bcb24e77ea 100644
--- a/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts
+++ b/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts
@@ -416,6 +416,34 @@ export function getMappingInfoForVolumeTracing(
   return getMappingInfo(state.temporaryConfiguration.activeMappingByLayer, tracingId);
 }
 
+export function hasEditableMapping(state: OxalisState): boolean {
+  const volumeTracing = getActiveSegmentationTracing(state);
+
+  if (volumeTracing == null) return false;
+
+  return !!volumeTracing.mappingIsEditable;
+}
+
+export function isMappingActivationAllowed(
+  state: OxalisState,
+  mappingName: string | null | undefined,
+): boolean {
+  const isEditableMappingActive = hasEditableMapping(state);
+
+  if (isEditableMappingActive) {
+    const volumeTracing = getActiveSegmentationTracing(state);
+
+    // This should never be the case, since editable mappings can only be active for volume tracings
+    if (volumeTracing == null) return false;
+
+    // Only allow mapping activations of the editable mapping itself if an editable mapping is saved
+    // in the volume tracing. Editable mappings cannot be disabled or switched for now.
+    return mappingName === volumeTracing.mappingName;
+  }
+
+  return true;
+}
+
 export function getLastLabelAction(volumeTracing: VolumeTracing): LabelAction | undefined {
   return volumeTracing.lastLabelActions[0];
 }
diff --git a/frontend/javascripts/oxalis/model/reducers/settings_reducer.ts b/frontend/javascripts/oxalis/model/reducers/settings_reducer.ts
index 9523668666b..bf091eacf1b 100644
--- a/frontend/javascripts/oxalis/model/reducers/settings_reducer.ts
+++ b/frontend/javascripts/oxalis/model/reducers/settings_reducer.ts
@@ -12,6 +12,10 @@ import {
 import type { StateShape1 } from "oxalis/model/helpers/deep_update";
 import { updateKey, updateKey3 } from "oxalis/model/helpers/deep_update";
 import { userSettings } from "types/schemas/user_settings.schema";
+import {
+  hasEditableMapping,
+  isMappingActivationAllowed,
+} from "oxalis/model/accessors/volumetracing_accessor";
 
 //
 // Update helpers
@@ -207,6 +211,10 @@ function SettingsReducer(state: OxalisState, action: Action): OxalisState {
     }
 
     case "SET_MAPPING_ENABLED": {
+      // Editable mappings cannot be disabled or switched for now
+      const isEditableMappingActive = hasEditableMapping(state);
+      if (isEditableMappingActive && !action.isMappingEnabled) return state;
+
       const { isMappingEnabled, layerName } = action;
       return updateActiveMapping(
         state,
@@ -230,6 +238,10 @@ function SettingsReducer(state: OxalisState, action: Action): OxalisState {
 
     case "SET_MAPPING": {
       const { mappingName, mapping, mappingKeys, mappingColors, mappingType, layerName } = action;
+
+      // Editable mappings cannot be disabled or switched for now
+      if (!isMappingActivationAllowed(state, mappingName)) return state;
+
       const hideUnmappedIds =
         action.hideUnmappedIds != null
           ? action.hideUnmappedIds
@@ -254,6 +266,10 @@ function SettingsReducer(state: OxalisState, action: Action): OxalisState {
 
     case "SET_MAPPING_NAME": {
       const { mappingName, layerName } = action;
+
+      // Editable mappings cannot be disabled or switched for now
+      if (!isMappingActivationAllowed(state, mappingName)) return state;
+
       return updateActiveMapping(state, { mappingName }, layerName);
     }
 
diff --git a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
index ba0b7bf078c..3fdc36d7bbf 100644
--- a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
+++ b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
@@ -307,6 +307,9 @@ function VolumeTracingReducer(
     }
 
     case "SET_MAPPING_NAME": {
+      // Editable mappings cannot be disabled or switched for now
+      if (volumeTracing.mappingIsEditable) return state;
+
       const { mappingName } = action;
       return updateVolumeTracing(state, volumeTracing.tracingId, {
         mappingName,
@@ -314,6 +317,9 @@ function VolumeTracingReducer(
     }
 
     case "SET_MAPPING_IS_EDITABLE": {
+      // Editable mappings cannot be disabled or switched for now
+      if (volumeTracing.mappingIsEditable) return state;
+
       return updateVolumeTracing(state, volumeTracing.tracingId, {
         mappingIsEditable: true,
       });
diff --git a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts
index 95617935fd1..930a89e6d49 100644
--- a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts
+++ b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts
@@ -147,6 +147,8 @@ export function setMappingNameReducer(
   mappingType: MappingType,
   isMappingEnabled: boolean = true,
 ) {
+  // Editable mappings cannot be disabled or switched for now
+  if (volumeTracing.mappingIsEditable) return state;
   // Only HDF5 mappings are persisted in volume annotations for now
   if (mappingType !== "HDF5" || !isMappingEnabled) {
     mappingName = null;
diff --git a/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts b/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts
index 98abb3b79c8..4b693b11dd5 100644
--- a/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts
@@ -20,6 +20,7 @@ import ErrorHandling from "libs/error_handling";
 import { MAPPING_MESSAGE_KEY } from "oxalis/model/bucket_data_handling/mappings";
 import api from "oxalis/api/internal_api";
 import { MappingStatusEnum } from "oxalis/constants";
+import { isMappingActivationAllowed } from "oxalis/model/accessors/volumetracing_accessor";
 type APIMappings = Record<string, APIMapping>;
 
 const isAgglomerate = (mapping: ActiveMappingInfo) => {
@@ -83,9 +84,13 @@ function* maybeFetchMapping(
     showLoadingIndicator,
   } = action;
 
-  if (mappingName == null || existingMapping != null) {
-    return;
-  }
+  // Editable mappings cannot be disabled or switched for now
+  const isEditableMappingActivationAllowed = yield* select((state) =>
+    isMappingActivationAllowed(state, mappingName),
+  );
+  if (!isEditableMappingActivationAllowed) return;
+
+  if (mappingName == null || existingMapping != null) return;
 
   if (showLoadingIndicator) {
     message.loading({
diff --git a/frontend/javascripts/oxalis/view/left-border-tabs/mapping_settings_view.tsx b/frontend/javascripts/oxalis/view/left-border-tabs/mapping_settings_view.tsx
index 113a5c7dbf1..1060f12b779 100644
--- a/frontend/javascripts/oxalis/view/left-border-tabs/mapping_settings_view.tsx
+++ b/frontend/javascripts/oxalis/view/left-border-tabs/mapping_settings_view.tsx
@@ -22,6 +22,7 @@ import {
 import { SwitchSetting } from "oxalis/view/components/setting_input_views";
 import * as Utils from "libs/utils";
 import { jsConvertCellIdToHSLA } from "oxalis/shaders/segmentation.glsl";
+import { hasEditableMapping } from "oxalis/model/accessors/volumetracing_accessor";
 const { Option, OptGroup } = Select;
 
 type OwnProps = {
@@ -49,6 +50,7 @@ type StateProps = {
   activeViewport: OrthoView;
   isMergerModeEnabled: boolean;
   allowUpdate: boolean;
+  isEditableMappingActive: boolean;
 };
 type Props = OwnProps & StateProps;
 type State = {
@@ -225,6 +227,7 @@ class MappingSettingsView extends React.Component<Props, State> {
                   value={shouldMappingBeEnabled}
                   label="ID Mapping"
                   loading={this.state.isRefreshingMappingList}
+                  disabled={this.props.isEditableMappingActive}
                 />
               </div>
 
@@ -243,6 +246,7 @@ class MappingSettingsView extends React.Component<Props, State> {
                   {...selectValueProp}
                   onChange={this.handleChangeMapping}
                   notFoundContent="No mappings found."
+                  disabled={this.props.isEditableMappingActive}
                 >
                   {renderCategoryOptions(availableMappings, "JSON")}
                   {renderCategoryOptions(availableAgglomerates, "HDF5")}
@@ -289,6 +293,7 @@ function mapStateToProps(state: OxalisState, ownProps: OwnProps) {
     segmentationLayer: getSegmentationLayerByName(state.dataset, ownProps.layerName),
     isMergerModeEnabled: state.temporaryConfiguration.isMergerModeEnabled,
     allowUpdate: state.tracing.restrictions.allowUpdate,
+    isEditableMappingActive: hasEditableMapping(state),
   };
 }
 

From 81c21338e7138d47443fc097e0356955ef1073b7 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 7 Jun 2022 18:39:18 +0200
Subject: [PATCH 090/122] hide volume modification tools after editable mapping
 was activated

---
 .../oxalis/view/action-bar/toolbar_view.tsx    | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
index 0f7fba63bca..fd7c2931351 100644
--- a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
+++ b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
@@ -20,6 +20,7 @@ import {
   getMappingInfoForVolumeTracing,
   getMaximumBrushSize,
   getRenderableResolutionForActiveSegmentationTracing,
+  hasEditableMapping,
 } from "oxalis/model/accessors/volumetracing_accessor";
 import { getActiveTree } from "oxalis/model/accessors/skeletontracing_accessor";
 import {
@@ -503,8 +504,9 @@ function ChangeBrushSizeButton() {
 export default function ToolbarView() {
   const hasVolume = useSelector((state: OxalisState) => state.tracing.volumes.length > 0);
   const hasSkeleton = useSelector((state: OxalisState) => state.tracing.skeleton != null);
-  const viewMode = useSelector((state: OxalisState) => state.temporaryConfiguration.viewMode);
-  const isVolumeSupported = hasVolume && !Constants.MODES_ARBITRARY.includes(viewMode);
+  const isVolumeModificationAllowed = useSelector(
+    (state: OxalisState) => !hasEditableMapping(state),
+  );
   const useLegacyBindings = useSelector(
     (state: OxalisState) => state.userConfiguration.useLegacyBindings,
   );
@@ -605,7 +607,7 @@ export default function ToolbarView() {
           </RadioButtonWithTooltip>
         ) : null}
 
-        {hasSkeleton && isVolumeSupported ? (
+        {hasSkeleton && hasVolume ? (
           <RadioButtonWithTooltip
             title="Proofreading Tool - Modify an agglomerated segmentation."
             disabledTitle={disabledInfosForTools[AnnotationToolEnum.PROOFREAD].explanation}
@@ -622,7 +624,7 @@ export default function ToolbarView() {
           </RadioButtonWithTooltip>
         ) : null}
 
-        {isVolumeSupported ? (
+        {hasVolume && isVolumeModificationAllowed ? (
           <React.Fragment>
             <RadioButtonWithTooltip
               title="Brush – Draw over the voxels you would like to label. Adjust the brush size with Shift + Mousewheel."
@@ -762,7 +764,7 @@ export default function ToolbarView() {
       <ToolSpecificSettings
         hasSkeleton={hasSkeleton}
         adaptedActiveTool={adaptedActiveTool}
-        isVolumeSupported={isVolumeSupported}
+        hasVolume={hasVolume}
         isControlPressed={isControlPressed}
         isShiftPressed={isShiftPressed}
       />
@@ -773,19 +775,19 @@ export default function ToolbarView() {
 function ToolSpecificSettings({
   hasSkeleton,
   adaptedActiveTool,
-  isVolumeSupported,
+  hasVolume,
   isControlPressed,
   isShiftPressed,
 }: {
   hasSkeleton: boolean;
   adaptedActiveTool: AnnotationTool;
-  isVolumeSupported: boolean;
+  hasVolume: boolean;
   isControlPressed: boolean;
   isShiftPressed: boolean;
 }) {
   const showCreateTreeButton = hasSkeleton && adaptedActiveTool === AnnotationToolEnum.SKELETON;
   const showNewBoundingBoxButton = adaptedActiveTool === AnnotationToolEnum.BOUNDING_BOX;
-  const showCreateCellButton = isVolumeSupported && VolumeTools.includes(adaptedActiveTool);
+  const showCreateCellButton = hasVolume && VolumeTools.includes(adaptedActiveTool);
   const showChangeBrushSizeButton =
     showCreateCellButton &&
     (adaptedActiveTool === AnnotationToolEnum.BRUSH ||

From efe5ce7c05ab8683d01669c43e72f3a39660adb8 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 7 Jun 2022 19:05:07 +0200
Subject: [PATCH 091/122] fix linting, only remove meshes that are not
 immediately loaded again

---
 frontend/javascripts/oxalis/model/sagas/proofread_saga.ts | 8 ++++++--
 .../javascripts/oxalis/view/action-bar/toolbar_view.tsx   | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 50ca6cd5cbe..f7037f0fffc 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -180,15 +180,19 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
   const segmentIdsInSurround = segmentIdsArrayBuffers.map((buffer) => new Uint32Array(buffer)[0]);
 
   if (oldSegmentIdsInSurround != null) {
+    const segmentIdsInSurroundSet = new Set(segmentIdsInSurround);
     // Remove old meshes in oversegmentation
     yield* all(
       oldSegmentIdsInSurround.map((nodeSegmentId) =>
-        put(removeIsosurfaceAction(layerName, nodeSegmentId)),
+        // Only remove meshes that are not part of the new segment surround
+        segmentIdsInSurroundSet.has(nodeSegmentId)
+          ? null
+          : put(removeIsosurfaceAction(layerName, nodeSegmentId)),
       ),
     );
   }
 
-  oldSegmentIdsInSurround = segmentIdsInSurround;
+  oldSegmentIdsInSurround = [...segmentIdsInSurround];
 
   // Load meshes in oversegmentation in fine resolution
   const noMappingInfo = {
diff --git a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
index fd7c2931351..e392eca7d6f 100644
--- a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
+++ b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
@@ -34,7 +34,7 @@ import { usePrevious, useKeyPress } from "libs/react_hooks";
 import { userSettings } from "types/schemas/user_settings.schema";
 import ButtonComponent from "oxalis/view/components/button_component";
 import { MaterializeVolumeAnnotationModal } from "oxalis/view/right-border-tabs/starting_job_modals";
-import Constants, {
+import {
   ToolsWithOverwriteCapabilities,
   AnnotationToolEnum,
   OverwriteModeEnum,

From ccd03707f07363adcce4c0d901fb755dcbcbbeae Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Wed, 8 Jun 2022 14:54:36 +0200
Subject: [PATCH 092/122] show empty placeholder instead of misleading info
 rows in td view context menu

---
 frontend/javascripts/oxalis/view/context_menu.tsx | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/frontend/javascripts/oxalis/view/context_menu.tsx b/frontend/javascripts/oxalis/view/context_menu.tsx
index 0cf5129f02c..9275d1dff1f 100644
--- a/frontend/javascripts/oxalis/view/context_menu.tsx
+++ b/frontend/javascripts/oxalis/view/context_menu.tsx
@@ -1,6 +1,6 @@
 import { CopyOutlined } from "@ant-design/icons";
 import type { Dispatch } from "redux";
-import { Dropdown, Menu, notification, Tooltip, Popover, Input } from "antd";
+import { Dropdown, Empty, Menu, notification, Tooltip, Popover, Input } from "antd";
 import { connect, useDispatch, useSelector } from "react-redux";
 import React, { useEffect } from "react";
 import type {
@@ -806,6 +806,10 @@ function NoNodeContextMenuOptions(props: NoNodeContextMenuProps) {
     allActions = nonSkeletonActions.concat(skeletonActions).concat(boundingBoxActions);
   }
 
+  const empty = (
+    <Empty description="&nbsp;No actions available&nbsp;" image={Empty.PRESENTED_IMAGE_SIMPLE} />
+  );
+
   return (
     <Menu
       onClick={hideContextMenu}
@@ -815,7 +819,8 @@ function NoNodeContextMenuOptions(props: NoNodeContextMenuProps) {
       mode="vertical"
     >
       {allActions}
-      {infoRows}
+      {/* In the TD viewport the info rows are not usable since the click position cannot be computed */}
+      {isTdViewport ? empty : infoRows}
     </Menu>
   );
 }

From b71fa7900553dd42320cce1707e1a1c856519844 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Wed, 8 Jun 2022 15:19:45 +0200
Subject: [PATCH 093/122] add created timestamp to editable mappings

---
 webknossos-datastore/proto/EditableMapping.proto       |  1 +
 .../tracingstore/TSRemoteWebKnossosClient.scala        |  4 +---
 .../tracings/editablemapping/EditableMapping.scala     |  5 ++++-
 .../editablemapping/EditableMappingService.scala       | 10 +++++++---
 .../tracings/volume/VolumeTracingService.scala         |  1 +
 5 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/webknossos-datastore/proto/EditableMapping.proto b/webknossos-datastore/proto/EditableMapping.proto
index 50ee391f160..a30ed209881 100644
--- a/webknossos-datastore/proto/EditableMapping.proto
+++ b/webknossos-datastore/proto/EditableMapping.proto
@@ -30,4 +30,5 @@ message EditableMappingProto {
   required string baseMappingName = 1;
   repeated SegmentToAgglomeratePair segmentToAgglomerate = 2;
   repeated AgglomerateToGraphPair agglomerateToGraph = 3;
+  required int64 createdTimestamp = 4;
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebKnossosClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebKnossosClient.scala
index 33ff1e1cacc..5e35ccf1ec2 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebKnossosClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebKnossosClient.scala
@@ -55,13 +55,11 @@ class TSRemoteWebKnossosClient @Inject()(
       .addQueryString("key" -> tracingStoreKey)
       .getWithJsonResponse[DataSourceLike]
 
-  def getDataStoreUriForDataSource(organizationName: String, dataSetName: String): Fox[String] = {
-    logger.error(s"get uri for data source $organizationName, $dataSetName")
+  def getDataStoreUriForDataSource(organizationName: String, dataSetName: String): Fox[String] =
     rpc(s"$webKnossosUrl/api/tracingstores/$tracingStoreName/dataStoreURI/$dataSetName")
       .addQueryString("organizationName" -> organizationName)
       .addQueryString("key" -> tracingStoreKey)
       .getWithJsonResponse[String]
-  }
 
   override def requestUserAccess(token: Option[String], accessRequest: UserAccessRequest): Fox[UserAccessAnswer] =
     rpc(s"$webKnossosUrl/api/tracingstores/$tracingStoreName/validateUserAccess")
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
index b6152d11de9..eba87312b04 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
@@ -6,6 +6,7 @@ case class EditableMapping(
     baseMappingName: String,
     segmentToAgglomerate: Map[Long, Long],
     agglomerateToGraph: Map[Long, AgglomerateGraph],
+    createdTimestamp: Long,
 ) {
   override def toString: String = f"EditableMapping(agglomerates:${agglomerateToGraph.keySet})"
 
@@ -14,6 +15,7 @@ case class EditableMapping(
       baseMappingName = baseMappingName,
       segmentToAgglomerate = segmentToAgglomerate.map(tuple => SegmentToAgglomeratePair(tuple._1, tuple._2)).toSeq,
       agglomerateToGraph = agglomerateToGraph.map(tuple => AgglomerateToGraphPair(tuple._1, tuple._2)).toSeq,
+      createdTimestamp = createdTimestamp
     )
 }
 
@@ -25,7 +27,8 @@ object EditableMapping {
       segmentToAgglomerate =
         editableMappignProto.segmentToAgglomerate.map(pair => pair.segmentId -> pair.agglomerateId).toMap,
       agglomerateToGraph =
-        editableMappignProto.agglomerateToGraph.map(pair => pair.agglomerateId -> pair.agglomerateGraph).toMap
+        editableMappignProto.agglomerateToGraph.map(pair => pair.agglomerateId -> pair.agglomerateGraph).toMap,
+      createdTimestamp = editableMappignProto.createdTimestamp
     )
 
   /*
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index d0332351db0..f9e9fe05173 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -90,7 +90,8 @@ class EditableMappingService @Inject()(
     val newEditableMapping = EditableMapping(
       baseMappingName = baseMappingName,
       segmentToAgglomerate = Map(),
-      agglomerateToGraph = Map()
+      agglomerateToGraph = Map(),
+      createdTimestamp = System.currentTimeMillis()
     )
     for {
       _ <- tracingDataStore.editableMappings.put(newId, 0L, toProtoBytes(newEditableMapping.toProto))
@@ -239,7 +240,9 @@ class EditableMappingService @Inject()(
       EditableMapping(
         mapping.baseMappingName,
         segmentToAgglomerate = mapping.segmentToAgglomerate ++ splitSegmentToAgglomerate,
-        agglomerateToGraph = mapping.agglomerateToGraph ++ Map(update.agglomerateId -> graph1, agglomerateId2 -> graph2)
+        agglomerateToGraph = mapping.agglomerateToGraph ++ Map(update.agglomerateId -> graph1,
+                                                               agglomerateId2 -> graph2),
+        createdTimestamp = mapping.createdTimestamp
       )
 
   private def splitGraph(agglomerateGraph: AgglomerateGraph,
@@ -330,7 +333,8 @@ class EditableMappingService @Inject()(
         segmentToAgglomerate = mapping.segmentToAgglomerate ++ mergedSegmentToAgglomerate,
         agglomerateToGraph = mapping.agglomerateToGraph ++ Map(
           update.agglomerateId1 -> mergedGraph,
-          update.agglomerateId2 -> AgglomerateGraph(List.empty, List.empty, List.empty, List.empty))
+          update.agglomerateId2 -> AgglomerateGraph(List.empty, List.empty, List.empty, List.empty)),
+        createdTimestamp = mapping.createdTimestamp
       )
 
   private def mergeGraph(agglomerateGraph1: AgglomerateGraph,
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
index 88b84d660e9..75e3ed2d185 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
@@ -348,6 +348,7 @@ class VolumeTracingService @Inject()(
 
   def volumeBucketsAreEmpty(tracingId: String): Fox[Boolean] =
     for {
+      keyListAll <- volumeDataStore.listKeys(None, None)
       keyList <- volumeDataStore.listKeys(limit = Some(1), startAfterKey = Some(tracingId))
       filtered = keyList.filter(_.startsWith(tracingId))
     } yield filtered.isEmpty

From 077884d9ee5c4c5cbb567db737e56de29d7c5f22 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Wed, 8 Jun 2022 16:36:43 +0200
Subject: [PATCH 094/122] first round of refactoring, commenting, tweaking

---
 .../oxalis/controller/scene_controller.ts     |  5 ++-
 .../model/actions/segmentation_actions.ts     |  8 ++--
 .../oxalis/model/sagas/isosurface_saga.ts     | 38 ++++++++--------
 .../oxalis/model/sagas/proofread_saga.ts      | 44 ++++++++++---------
 frontend/javascripts/oxalis/store.ts          |  1 +
 .../javascripts/oxalis/view/version_list.tsx  | 12 +++--
 frontend/javascripts/types/api_flow_types.ts  |  1 +
 7 files changed, 57 insertions(+), 52 deletions(-)

diff --git a/frontend/javascripts/oxalis/controller/scene_controller.ts b/frontend/javascripts/oxalis/controller/scene_controller.ts
index 195e671b1cd..e500c37d7ca 100644
--- a/frontend/javascripts/oxalis/controller/scene_controller.ts
+++ b/frontend/javascripts/oxalis/controller/scene_controller.ts
@@ -213,7 +213,7 @@ class SceneController {
     tweenAnimation
       .to(
         {
-          opacity: passive ? 0.2 : 0.7,
+          opacity: passive ? 0.2 : 0.6,
         },
         500,
       )
@@ -248,6 +248,9 @@ class SceneController {
   addIsosurfaceFromVertices(
     vertices: Float32Array,
     segmentationId: number,
+    // Passive isosurfaces are ignored during picking, are shown more transparently, and are rendered
+    // last so that all non-passive isosurfaces are rendered before them. This makes sure that non-passive
+    // isosurfaces are not skipped during rendering if they are overlapped by passive ones.
     passive: boolean,
   ): void {
     let bufferGeometry = new THREE.BufferGeometry();
diff --git a/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts b/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts
index 3d6f16c0ab4..617c4265d6d 100644
--- a/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts
+++ b/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts
@@ -1,6 +1,6 @@
 import type { Vector3 } from "oxalis/constants";
 import type { MappingType } from "oxalis/store";
-export type IsosurfaceMappingInfo = {
+export type AdHocIsosurfaceInfo = {
   mappingName: string | null | undefined;
   mappingType: MappingType | null | undefined;
   useDataStore?: boolean | null | undefined;
@@ -10,7 +10,7 @@ export type LoadAdHocMeshAction = {
   type: "LOAD_AD_HOC_MESH_ACTION";
   cellId: number;
   seedPosition: Vector3;
-  mappingInfo?: IsosurfaceMappingInfo;
+  extraInfo?: AdHocIsosurfaceInfo;
   layerName?: string;
 };
 export type LoadPrecomputedMeshAction = {
@@ -24,13 +24,13 @@ export type SegmentationAction = LoadAdHocMeshAction | LoadPrecomputedMeshAction
 export const loadAdHocMeshAction = (
   cellId: number,
   seedPosition: Vector3,
-  mappingInfo?: IsosurfaceMappingInfo,
+  extraInfo?: AdHocIsosurfaceInfo,
   layerName?: string,
 ): LoadAdHocMeshAction => ({
   type: "LOAD_AD_HOC_MESH_ACTION",
   cellId,
   seedPosition,
-  mappingInfo,
+  extraInfo,
   layerName,
 });
 export const loadPrecomputedMeshAction = (
diff --git a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
index 152fe0f9e28..d9e168ec3a9 100644
--- a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
@@ -14,7 +14,7 @@ import {
 import type {
   LoadAdHocMeshAction,
   LoadPrecomputedMeshAction,
-  IsosurfaceMappingInfo,
+  AdHocIsosurfaceInfo,
 } from "oxalis/model/actions/segmentation_actions";
 import type { Action } from "oxalis/model/actions/actions";
 import type { Vector3 } from "oxalis/constants";
@@ -163,7 +163,7 @@ function* loadAdHocIsosurfaceFromAction(action: LoadAdHocMeshAction): Saga<void>
     action.cellId,
     false,
     action.layerName,
-    action.mappingInfo,
+    action.extraInfo,
   );
 }
 
@@ -172,7 +172,7 @@ function* loadAdHocIsosurface(
   cellId: number,
   removeExistingIsosurface: boolean = false,
   layerName?: string | null | undefined,
-  maybeMappingInfo?: IsosurfaceMappingInfo,
+  maybeExtraInfo?: AdHocIsosurfaceInfo,
 ): Saga<void> {
   const layer =
     layerName != null ? Model.getLayerByName(layerName) : Model.getVisibleSegmentationLayer();
@@ -181,25 +181,25 @@ function* loadAdHocIsosurface(
     return;
   }
 
-  const isosurfaceMappingInfo = yield* call(getIsosurfaceMappingInfo, layer.name, maybeMappingInfo);
+  const isosurfaceExtraInfo = yield* call(getIsosurfaceExtraInfo, layer.name, maybeExtraInfo);
   yield* call(
     loadIsosurfaceForSegmentId,
     cellId,
     seedPosition,
-    isosurfaceMappingInfo,
+    isosurfaceExtraInfo,
     removeExistingIsosurface,
     layer,
   );
 }
 
-function* getIsosurfaceMappingInfo(
+function* getIsosurfaceExtraInfo(
   layerName: string,
-  maybeMappingInfo: IsosurfaceMappingInfo | null | undefined,
-): Saga<IsosurfaceMappingInfo> {
+  maybeExtraInfo: AdHocIsosurfaceInfo | null | undefined,
+): Saga<AdHocIsosurfaceInfo> {
   const activeMappingByLayer = yield* select(
     (state) => state.temporaryConfiguration.activeMappingByLayer,
   );
-  if (maybeMappingInfo != null) return maybeMappingInfo;
+  if (maybeExtraInfo != null) return maybeExtraInfo;
   const mappingInfo = getMappingInfo(activeMappingByLayer, layerName);
   const isMappingActive = mappingInfo.mappingStatus === MappingStatusEnum.ENABLED;
   const mappingName = isMappingActive ? mappingInfo.mappingName : null;
@@ -228,7 +228,7 @@ function* getInfoForIsosurfaceLoading(layer: DataLayer): Saga<{
 function* loadIsosurfaceForSegmentId(
   segmentId: number,
   seedPosition: Vector3,
-  isosurfaceMappingInfo: IsosurfaceMappingInfo,
+  isosurfaceExtraInfo: AdHocIsosurfaceInfo,
   removeExistingIsosurface: boolean,
   layer: DataLayer,
 ): Saga<void> {
@@ -245,7 +245,7 @@ function* loadIsosurfaceForSegmentId(
       segmentId,
       seedPosition,
       zoomStep,
-      isosurfaceMappingInfo,
+      isosurfaceExtraInfo,
       resolutionInfo,
       removeExistingIsosurface,
     ),
@@ -263,12 +263,12 @@ function* loadIsosurfaceWithNeighbors(
   segmentId: number,
   position: Vector3,
   zoomStep: number,
-  isosurfaceMappingInfo: IsosurfaceMappingInfo,
+  isosurfaceExtraInfo: AdHocIsosurfaceInfo,
   resolutionInfo: ResolutionInfo,
   removeExistingIsosurface: boolean,
 ): Saga<void> {
   let isInitialRequest = true;
-  const { mappingName, mappingType } = isosurfaceMappingInfo;
+  const { mappingName, mappingType } = isosurfaceExtraInfo;
   const clippedPosition = clipPositionToCubeBoundary(position);
   let positionsToRequest = [clippedPosition];
   const hasIsosurface = yield* select(
@@ -295,7 +295,7 @@ function* loadIsosurfaceWithNeighbors(
       segmentId,
       currentPosition,
       zoomStep,
-      isosurfaceMappingInfo,
+      isosurfaceExtraInfo,
       resolutionInfo,
       isInitialRequest,
       removeExistingIsosurface && isInitialRequest,
@@ -334,7 +334,7 @@ function* maybeLoadIsosurface(
   segmentId: number,
   clippedPosition: Vector3,
   zoomStep: number,
-  isosurfaceMappingInfo: IsosurfaceMappingInfo,
+  isosurfaceExtraInfo: AdHocIsosurfaceInfo,
   resolutionInfo: ResolutionInfo,
   isInitialRequest: boolean,
   removeExistingIsosurface: boolean,
@@ -370,8 +370,8 @@ function* maybeLoadIsosurface(
   const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
   // Fetch from datastore if no volumetracing exists
   const useDataStore =
-    isosurfaceMappingInfo.useDataStore != null
-      ? isosurfaceMappingInfo.useDataStore
+    isosurfaceExtraInfo.useDataStore != null
+      ? isosurfaceExtraInfo.useDataStore
       : volumeTracing == null;
   const mag = resolutionInfo.getResolutionByIndexOrThrow(zoomStep);
 
@@ -398,7 +398,7 @@ function* maybeLoadIsosurface(
           subsamplingStrides,
           cubeSize: getZoomedCubeSize(zoomStep, resolutionInfo),
           scale,
-          ...isosurfaceMappingInfo,
+          ...isosurfaceExtraInfo,
         },
       );
       const vertices = new Float32Array(responseBuffer);
@@ -410,7 +410,7 @@ function* maybeLoadIsosurface(
       getSceneController().addIsosurfaceFromVertices(
         vertices,
         segmentId,
-        isosurfaceMappingInfo.passive || false,
+        isosurfaceExtraInfo.passive || false,
       );
       return neighbors.map((neighbor) => getNeighborPosition(clippedPosition, neighbor));
     } catch (exception) {
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index f7037f0fffc..066e95e1de0 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -226,6 +226,24 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
   );
 }
 
+function* createEditableMapping(): Saga<void> {
+  const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url);
+  // Save before making the mapping editable to make sure the correct mapping is activated in the backend
+  yield* call([Model, Model.ensureSavedState]);
+  // Get volume tracing again to make sure the version is up to date
+  const upToDateVolumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
+  if (upToDateVolumeTracing == null) return;
+
+  const volumeTracingId = upToDateVolumeTracing.tracingId;
+  const layerName = volumeTracingId;
+  const serverEditableMapping = yield* call(makeMappingEditable, tracingStoreUrl, volumeTracingId);
+  // The server increments the volume tracing's version by 1 when switching the mapping to an editable one
+  yield* put(setVersionNumberAction(upToDateVolumeTracing.version + 1, "volume", volumeTracingId));
+  yield* put(setMappingNameAction(layerName, serverEditableMapping.mappingName));
+  yield* put(setMappingisEditableAction());
+  yield* put(initializeEditableMappingAction(serverEditableMapping));
+}
+
 function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
   const allowUpdate = yield* select((state) => state.tracing.restrictions.allowUpdate);
   if (!allowUpdate) return;
@@ -239,7 +257,7 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
   if (volumeTracing == null) return;
   const { tracingId: volumeTracingId } = volumeTracing;
 
-  const layerName = volumeTracingLayer.name;
+  const layerName = volumeTracingId;
   const activeMappingByLayer = yield* select(
     (state) => state.temporaryConfiguration.activeMappingByLayer,
   );
@@ -251,28 +269,11 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
     mappingStatus === MappingStatusEnum.DISABLED
   ) {
     Toast.error("An HDF5 mapping needs to be enabled to use the proofreading tool.");
+    return;
   }
 
   if (!volumeTracing.mappingIsEditable) {
-    const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url);
-    // Save before making the mapping editable to make sure the correct mapping is activated in the backend
-    yield* call([Model, Model.ensureSavedState]);
-    // Get volume tracing again to make sure the version is up to date
-    const upToDateVolumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
-    if (upToDateVolumeTracing == null) return;
-
-    const serverEditableMapping = yield* call(
-      makeMappingEditable,
-      tracingStoreUrl,
-      volumeTracingId,
-    );
-    // The server increments the volume tracing's version by 1 when switching the mapping to an editable one
-    yield* put(
-      setVersionNumberAction(upToDateVolumeTracing.version + 1, "volume", volumeTracingId),
-    );
-    yield* put(setMappingNameAction(layerName, serverEditableMapping.mappingName));
-    yield* put(setMappingisEditableAction());
-    yield* put(initializeEditableMappingAction(serverEditableMapping));
+    yield* call(createEditableMapping);
   }
 
   const resolutionInfo = getResolutionInfo(volumeTracingLayer.resolutions);
@@ -349,8 +350,9 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
   if (
     volumeTracingWithEditableMapping == null ||
     volumeTracingWithEditableMapping.mappingName == null
-  )
+  ) {
     return;
+  }
 
   const newSourceNodeAgglomerateId = yield* call(
     [api.data, api.data.getDataValue],
diff --git a/frontend/javascripts/oxalis/store.ts b/frontend/javascripts/oxalis/store.ts
index cc4dee4cd77..db6535202e4 100644
--- a/frontend/javascripts/oxalis/store.ts
+++ b/frontend/javascripts/oxalis/store.ts
@@ -239,6 +239,7 @@ export type ReadOnlyTracing = TracingBase & {
 };
 export type EditableMapping = {
   readonly type: "mapping";
+  readonly createdTimestamp: number;
   readonly version: number;
   readonly mappingName: string;
   // The id of the volume tracing the editable mapping belongs to
diff --git a/frontend/javascripts/oxalis/view/version_list.tsx b/frontend/javascripts/oxalis/view/version_list.tsx
index 1cde1b7b751..8b9ba2d44d7 100644
--- a/frontend/javascripts/oxalis/view/version_list.tsx
+++ b/frontend/javascripts/oxalis/view/version_list.tsx
@@ -97,13 +97,11 @@ class VersionList extends React.Component<Props, State> {
         tracingId,
         this.props.versionedObjectType,
       );
-      if (this.props.tracing.type !== "mapping") {
-        // Insert version 0
-        updateActionLog.push({
-          version: 0,
-          value: [serverCreateTracing(this.props.tracing.createdTimestamp)],
-        });
-      }
+      // Insert version 0
+      updateActionLog.push({
+        version: 0,
+        value: [serverCreateTracing(this.props.tracing.createdTimestamp)],
+      });
       this.setState({
         versions: updateActionLog,
       });
diff --git a/frontend/javascripts/types/api_flow_types.ts b/frontend/javascripts/types/api_flow_types.ts
index ab5f13adfcd..239b743269f 100644
--- a/frontend/javascripts/types/api_flow_types.ts
+++ b/frontend/javascripts/types/api_flow_types.ts
@@ -615,6 +615,7 @@ export type ServerVolumeTracing = ServerTracingBase & {
 };
 export type ServerTracing = ServerSkeletonTracing | ServerVolumeTracing;
 export type ServerEditableMapping = {
+  createdTimestamp: number;
   version: number;
   mappingName: string;
   // The id of the volume tracing the editable mapping belongs to

From 6eb866dd4984bd9e8e1566417d949fd969444b28 Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Thu, 9 Jun 2022 09:19:29 +0200
Subject: [PATCH 095/122] fix null reference in isosurface service

---
 .../webknossos/tracingstore/TracingStoreModule.scala        | 2 ++
 .../tracings/editablemapping/EditableMappingService.scala   | 6 ++++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala
index 552fc8070cc..0ff349ab5fc 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala
@@ -3,6 +3,7 @@ package com.scalableminds.webknossos.tracingstore
 import akka.actor.ActorSystem
 import com.google.inject.AbstractModule
 import com.google.inject.name.Names
+import com.scalableminds.webknossos.datastore.services.IsosurfaceServiceHolder
 import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService
 import com.scalableminds.webknossos.tracingstore.tracings.TracingDataStore
 import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService
@@ -23,5 +24,6 @@ class TracingStoreModule extends AbstractModule {
     bind(classOf[TSRemoteDatastoreClient]).asEagerSingleton()
     bind(classOf[EditableMappingService]).asEagerSingleton()
     bind(classOf[TSSlackNotificationService]).asEagerSingleton()
+    bind(classOf[IsosurfaceServiceHolder]).asEagerSingleton()
   }
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index f9e9fe05173..8ef3c89bd72 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -14,6 +14,8 @@ import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing.Elemen
 import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, ProtoGeometryImplicits, SkeletonTracingDefaults}
 import com.scalableminds.webknossos.datastore.models.DataRequestCollection.DataRequestCollection
 import com.scalableminds.webknossos.datastore.models._
+
+import scala.concurrent.duration._
 import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataRequest
 import com.scalableminds.webknossos.datastore.services.{
   BinaryDataService,
@@ -60,9 +62,9 @@ class EditableMappingService @Inject()(
 
   private def generateId: String = UUID.randomUUID.toString
 
-  val isosurfaceService: IsosurfaceService = isosurfaceServiceHolder.tracingStoreIsosurfaceService
-
   val binaryDataService = new BinaryDataService(Paths.get(""), 100, null)
+  isosurfaceServiceHolder.tracingStoreIsosurfaceConfig = (binaryDataService, 30 seconds, 1)
+  val isosurfaceService: IsosurfaceService = isosurfaceServiceHolder.tracingStoreIsosurfaceService
 
   private lazy val materializedEditableMappingCache: AlfuFoxCache[EditableMappingKey, EditableMapping] = AlfuFoxCache(
     maxEntries = 50)

From ba3a7f31b30aedfc8c9bfd570b7fda46782fb9ae Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Thu, 9 Jun 2022 09:55:38 +0200
Subject: [PATCH 096/122] assert volume buckets empty

---
 .../scala/com/scalableminds/util/cache/AlfuCache.scala    | 4 ++--
 .../controllers/VolumeTracingController.scala             | 2 +-
 .../webknossos/tracingstore/tracings/FossilDBClient.scala | 1 +
 .../tracings/volume/VolumeTracingService.scala            | 8 ++------
 4 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/util/src/main/scala/com/scalableminds/util/cache/AlfuCache.scala b/util/src/main/scala/com/scalableminds/util/cache/AlfuCache.scala
index a6b8c2aea2c..bb4eb00569f 100644
--- a/util/src/main/scala/com/scalableminds/util/cache/AlfuCache.scala
+++ b/util/src/main/scala/com/scalableminds/util/cache/AlfuCache.scala
@@ -40,7 +40,7 @@ object AlfuFoxCache {
   def apply[K, V](maxEntries: Int = 1000,
                   timeToLive: FiniteDuration = 2 hours,
                   timeToIdle: FiniteDuration = 1 hour): AlfuFoxCache[K, V] = {
-    val lfuCache: Cache[K, Box[V]] = AlfuCache(maxEntries, timeToLive, timeToIdle)
-    new AlfuFoxCache(lfuCache)
+    val alfuCache: Cache[K, Box[V]] = AlfuCache(maxEntries, timeToLive, timeToIdle)
+    new AlfuFoxCache(alfuCache)
   }
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index 67d2d4428e4..c19e624c7c9 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -457,7 +457,7 @@ class VolumeTracingController @Inject()(
           for {
             tracing <- tracingService.find(tracingId)
             tracingMappingName <- tracing.mappingName ?~> "annotation.noMappingSet"
-            _ <- Fox.assertTrue(tracingService.volumeBucketsAreEmpty(tracingId)) ?~> "annotation.volumeBucketsNotEmpty"
+            _ <- bool2Fox(tracingService.volumeBucketsAreEmpty(tracingId)) ?~> "annotation.volumeBucketsNotEmpty"
             editableMappingId <- editableMappingService.create(baseMappingName = tracingMappingName)
             volumeUpdate = UpdateMappingNameAction(Some(editableMappingId),
                                                    isEditable = Some(true),
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
index 387f421da02..3ef31e57950 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
@@ -31,6 +31,7 @@ trait KeyValueStoreImplicits extends BoxImplicits {
   implicit def fromProtoBytes[T <: GeneratedMessage](a: Array[Byte])(
       implicit companion: GeneratedMessageCompanion[T]): Box[T] = tryo(companion.parseFrom(a))
 
+  implicit def bytesIdentity(a: Array[Byte]): Box[Array[Byte]] = Full(a)
 }
 
 case class KeyValuePair[T](key: String, value: T)
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
index 75e3ed2d185..60eee5253e2 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
@@ -346,12 +346,8 @@ class VolumeTracingService @Inject()(
       _ <- updateResolutionList(tracingId, tracing, resultingResolutions.toSet)
     } yield ()
 
-  def volumeBucketsAreEmpty(tracingId: String): Fox[Boolean] =
-    for {
-      keyListAll <- volumeDataStore.listKeys(None, None)
-      keyList <- volumeDataStore.listKeys(limit = Some(1), startAfterKey = Some(tracingId))
-      filtered = keyList.filter(_.startsWith(tracingId))
-    } yield filtered.isEmpty
+  def volumeBucketsAreEmpty(tracingId: String): Boolean =
+    volumeDataStore.getMultipleKeys(tracingId, Some(tracingId), limit = Some(1))(bytesIdentity).isEmpty
 
   def createIsosurface(tracingId: String, request: WebKnossosIsosurfaceRequest): Fox[(Array[Float], List[Int])] =
     for {

From 34d97c34563f2b5e68d82fbde25fe8a6d5b3b6ea Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Thu, 9 Jun 2022 17:15:40 +0200
Subject: [PATCH 097/122] backend pr feedback (part 1)

---
 .../proto/EditableMapping.proto                |  2 +-
 .../controllers/VolumeTracingController.scala  | 12 +++++++++---
 .../tracingstore/tracings/FossilDBClient.scala | 12 ------------
 .../editablemapping/EditableMapping.scala      | 18 ------------------
 .../editablemapping/EditableMappingLayer.scala | 11 -----------
 .../EditableMappingService.scala               | 14 +++++++-------
 .../tracings/volume/VolumeTracingService.scala |  2 +-
 7 files changed, 18 insertions(+), 53 deletions(-)

diff --git a/webknossos-datastore/proto/EditableMapping.proto b/webknossos-datastore/proto/EditableMapping.proto
index a30ed209881..0d56ab0e090 100644
--- a/webknossos-datastore/proto/EditableMapping.proto
+++ b/webknossos-datastore/proto/EditableMapping.proto
@@ -4,7 +4,7 @@ package com.scalableminds.webknossos.datastore;
 
 import "geometry.proto";
 
-message AgglomerateEdge {
+message AgglomerateEdge { // Note that the edges are stored directed but semantically undirected. When testing for an edge, check both directions.
   required int64 source = 1;
   required int64 target = 2;
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index c19e624c7c9..f5ff3439b78 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -458,7 +458,7 @@ class VolumeTracingController @Inject()(
             tracing <- tracingService.find(tracingId)
             tracingMappingName <- tracing.mappingName ?~> "annotation.noMappingSet"
             _ <- bool2Fox(tracingService.volumeBucketsAreEmpty(tracingId)) ?~> "annotation.volumeBucketsNotEmpty"
-            editableMappingId <- editableMappingService.create(baseMappingName = tracingMappingName)
+            (editableMappingId, editableMapping) <- editableMappingService.create(baseMappingName = tracingMappingName)
             volumeUpdate = UpdateMappingNameAction(Some(editableMappingId),
                                                    isEditable = Some(true),
                                                    actionTimestamp = Some(System.currentTimeMillis()))
@@ -474,7 +474,9 @@ class VolumeTracingController @Inject()(
                                                None),
               tracing.version
             )
-            infoJson <- editableMappingService.infoJson(tracingId = tracingId, editableMappingId = editableMappingId)
+            infoJson <- editableMappingService.infoJson(tracingId = tracingId,
+                                                        editableMappingId = editableMappingId,
+                                                        editableMapping = editableMapping)
           } yield Ok(infoJson)
         }
       }
@@ -518,7 +520,11 @@ class VolumeTracingController @Inject()(
           for {
             tracing <- tracingService.find(tracingId)
             mappingName <- tracing.mappingName.toFox
-            infoJson <- editableMappingService.infoJson(tracingId = tracingId, editableMappingId = mappingName)
+            remoteFallbackLayer <- editableMappingService.remoteFallbackLayer(tracing)
+            editableMapping <- editableMappingService.get(mappingName, remoteFallbackLayer, token)
+            infoJson <- editableMappingService.infoJson(tracingId = tracingId,
+                                                        editableMappingId = mappingName,
+                                                        editableMapping = editableMapping)
           } yield Ok(infoJson)
         }
       }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
index 3ef31e57950..4bc2124f240 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/FossilDBClient.scala
@@ -30,8 +30,6 @@ trait KeyValueStoreImplicits extends BoxImplicits {
 
   implicit def fromProtoBytes[T <: GeneratedMessage](a: Array[Byte])(
       implicit companion: GeneratedMessageCompanion[T]): Box[T] = tryo(companion.parseFrom(a))
-
-  implicit def bytesIdentity(a: Array[Byte]): Box[Array[Byte]] = Full(a)
 }
 
 case class KeyValuePair[T](key: String, value: T)
@@ -164,16 +162,6 @@ class FossilDBClient(collection: String,
         Fox.failure("could not save to FossilDB: " + e.getMessage)
     }
 
-  def listKeys(limit: Option[Int], startAfterKey: Option[String]): Fox[Seq[String]] =
-    try {
-      val reply = blockingStub.listKeys(ListKeysRequest(collection, limit, startAfterKey))
-      if (!reply.success) throw new Exception(reply.errorMessage.getOrElse(""))
-      Fox.successful(reply.keys)
-    } catch {
-      case e: Exception =>
-        Fox.failure("could not list keys from FossilDB: " + e.getMessage)
-    }
-
   def shutdown(): Boolean = {
     channel.shutdownNow()
     channel.awaitTermination(10, java.util.concurrent.TimeUnit.SECONDS)
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
index eba87312b04..129bb1e218f 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMapping.scala
@@ -31,22 +31,4 @@ object EditableMapping {
       createdTimestamp = editableMappignProto.createdTimestamp
     )
 
-  /*
-  def createDummy(numSegments: Long, numAgglomerates: Long): EditableMapping =
-    EditableMapping(
-      baseMappingName = "dummyBaseMapping",
-      segmentToAgglomerate = 1L.to(numSegments).map(s => s -> s % numAgglomerates).toMap,
-      agglomerateToGraph = 1L
-        .to(numAgglomerates)
-        .map(a =>
-          a -> AgglomerateGraph(
-            segments = 1L.to(numSegments / numAgglomerates),
-            edges = 1L.to(numSegments / numAgglomerates).map(s => AgglomerateEdge(s, s)),
-            positions = 1L.to(numSegments / numAgglomerates).map(s => Vec3IntProto(s.toInt, s.toInt, s.toInt)),
-            affinities = 1L.to(numSegments / numAgglomerates).map(s => s.toFloat)
-        ))
-        .toMap
-    )
- */
-
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
index 7d2fc111d47..f016c8e7bfd 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala
@@ -21,9 +21,7 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP
       editableMappingId <- Fox.successful(layer.name)
       _ <- bool2Fox(layer.doesContainBucket(bucket))
       remoteFallbackLayer <- layer.editableMappingService.remoteFallbackLayer(layer.tracing)
-      beforeGet = System.currentTimeMillis()
       editableMapping <- layer.editableMappingService.get(editableMappingId, remoteFallbackLayer, layer.token)
-      afterGet = System.currentTimeMillis()
       dataRequest: WebKnossosDataRequest = WebKnossosDataRequest(
         position = Vec3Int(bucket.topLeft.mag1X, bucket.topLeft.mag1Y, bucket.topLeft.mag1Z),
         mag = bucket.mag,
@@ -35,25 +33,16 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP
       (unmappedData, indices) <- layer.editableMappingService.getUnmappedDataFromDatastore(remoteFallbackLayer,
                                                                                            List(dataRequest),
                                                                                            layer.token)
-      afterGetUnmapped = System.currentTimeMillis()
       _ <- bool2Fox(indices.isEmpty)
       unmappedDataTyped <- layer.editableMappingService.bytesToUnsignedInt(unmappedData, layer.tracing.elementClass)
       segmentIds = layer.editableMappingService.collectSegmentIds(unmappedDataTyped)
-      afterCollectSegmentIds = System.currentTimeMillis()
       relevantMapping <- layer.editableMappingService.generateCombinedMappingSubset(segmentIds,
                                                                                     editableMapping,
                                                                                     remoteFallbackLayer,
                                                                                     layer.token)
-      afterCombineMapping = System.currentTimeMillis()
       mappedData: Array[Byte] <- layer.editableMappingService.mapData(unmappedDataTyped,
                                                                       relevantMapping,
                                                                       layer.elementClass)
-      afterMapData = System.currentTimeMillis()
-      /*
-      _ = logger.info(s"load bucket $bucket")
-      _ = logger.info(
-        s"load bucket timing: getMapping: ${afterGet - beforeGet} ms, getUnmapped: ${afterGetUnmapped - afterGet} ms, collectSegments: ${afterCollectSegmentIds - afterGet} ms, combine: ${afterCombineMapping - afterCollectSegmentIds}, mapData: ${afterMapData - afterCombineMapping}. Total ${afterMapData - beforeGet}. ${mappedData.length} bytes, ${segmentIds.size} segments")
-     */
     } yield mappedData
   }
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
index 8ef3c89bd72..42ee85b038a 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala
@@ -77,17 +77,18 @@ class EditableMappingService @Inject()(
                                                        mayBeEmpty = Some(true),
                                                        emptyFallback = Some(0L))
 
-  def infoJson(tracingId: String, editableMappingId: String): Fox[JsObject] =
+  def infoJson(tracingId: String, editableMapping: EditableMapping, editableMappingId: String): Fox[JsObject] =
     for {
       version <- newestMaterializableVersion(editableMappingId)
     } yield
       Json.obj(
         "mappingName" -> editableMappingId,
         "version" -> version,
-        "tracingId" -> tracingId
+        "tracingId" -> tracingId,
+        "createdTimestamp" -> editableMapping.createdTimestamp
       )
 
-  def create(baseMappingName: String): Fox[String] = {
+  def create(baseMappingName: String): Fox[(String, EditableMapping)] = {
     val newId = generateId
     val newEditableMapping = EditableMapping(
       baseMappingName = baseMappingName,
@@ -97,7 +98,7 @@ class EditableMappingService @Inject()(
     )
     for {
       _ <- tracingDataStore.editableMappings.put(newId, 0L, toProtoBytes(newEditableMapping.toProto))
-    } yield newId
+    } yield (newId, newEditableMapping)
   }
 
   def exists(editableMappingId: String): Fox[Boolean] =
@@ -411,7 +412,7 @@ class EditableMappingService @Inject()(
     val segmentIdsInEditableMapping: Set[Long] = segmentIds.intersect(editableMapping.segmentToAgglomerate.keySet)
     val segmentIdsInBaseMapping: Set[Long] = segmentIds.diff(segmentIdsInEditableMapping)
     val editableMappingSubset =
-      editableMapping.segmentToAgglomerate.filterKeys(key => segmentIdsInEditableMapping.contains(key))
+      segmentIdsInEditableMapping.map(segmentId => segmentId -> editableMapping.segmentToAgglomerate(segmentId)).toMap
     for {
       baseMappingSubset <- getBaseSegmentToAgglomeate(editableMapping.baseMappingName,
                                                       segmentIdsInBaseMapping,
@@ -506,8 +507,7 @@ class EditableMappingService @Inject()(
   def mapData(unmappedData: Array[UnsignedInteger],
               relevantMapping: Map[Long, Long],
               elementClass: ElementClass): Fox[Array[Byte]] = {
-    val unmappedDataLongs = unmappedData.map(_.toPositiveLong)
-    val mappedDataLongs = unmappedDataLongs.map(relevantMapping)
+    val mappedDataLongs = unmappedData.map(element => relevantMapping(element.toPositiveLong))
     for {
       bytes <- longsToBytes(mappedDataLongs, elementClass)
     } yield bytes
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
index 60eee5253e2..e104042205f 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
@@ -347,7 +347,7 @@ class VolumeTracingService @Inject()(
     } yield ()
 
   def volumeBucketsAreEmpty(tracingId: String): Boolean =
-    volumeDataStore.getMultipleKeys(tracingId, Some(tracingId), limit = Some(1))(bytesIdentity).isEmpty
+    volumeDataStore.getMultipleKeys(tracingId, Some(tracingId), limit = Some(1))(toBox).isEmpty
 
   def createIsosurface(tracingId: String, request: WebKnossosIsosurfaceRequest): Fox[(Array[Float], List[Int])] =
     for {

From d9009f6a3ea101fbcd54300d94b7b3c2f1cf978e Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Thu, 9 Jun 2022 18:58:11 +0200
Subject: [PATCH 098/122] add some comments to proofreading saga

---
 frontend/javascripts/admin/admin_rest_api.ts  |  2 ++
 .../oxalis/model/sagas/isosurface_saga.ts     |  1 +
 .../oxalis/model/sagas/proofread_saga.ts      | 30 +++++++++++++++----
 3 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts
index da880eb8582..fe2de72d98e 100644
--- a/frontend/javascripts/admin/admin_rest_api.ts
+++ b/frontend/javascripts/admin/admin_rest_api.ts
@@ -1880,10 +1880,12 @@ export function getMeshData(id: string): Promise<ArrayBuffer> {
 // These parameters are bundled into an object to avoid that the computeIsosurface function
 // receives too many parameters, since this doesn't play well with the saga typings.
 type IsosurfaceRequest = {
+  // The position is in voxels in mag 1
   position: Vector3;
   mag: Vector3;
   segmentId: number;
   subsamplingStrides: Vector3;
+  // The cubeSize is in voxels in mag <mag>
   cubeSize: Vector3;
   scale: Vector3;
   mappingName: string | null | undefined;
diff --git a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
index d9e168ec3a9..2d9977e234c 100644
--- a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
@@ -111,6 +111,7 @@ function removeMapForSegment(layerName: string, segmentId: number): void {
 }
 
 function getZoomedCubeSize(zoomStep: number, resolutionInfo: ResolutionInfo): Vector3 {
+  // Convert marchingCubeSizeInMag1 to another resolution (zoomStep)
   const [x, y, z] = zoomedAddressToAnotherZoomStepWithInfo(
     [...marchingCubeSizeInMag1(), 0],
     resolutionInfo,
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 066e95e1de0..9680640e680 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -83,6 +83,9 @@ function* loadCoarseAdHocMesh(
   segmentId: number,
   position: Vector3,
 ): Saga<void> {
+  // Since it is currently not possible to specify the quality in the loadAdHocMeshAction
+  // this method sets the preferredQualityForMeshAdHocComputation, then requests the mesh
+  // and then resets preferredQualityForMeshAdHocComputation to the previous value.
   const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
   if (volumeTracing == null) return;
 
@@ -135,6 +138,8 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
 
   if (mappingName == null) return;
 
+  /* Load agglomerate skeleton of the agglomerate at the click position */
+
   const treeName = yield* call(
     loadAgglomerateSkeletonWithId,
     loadAgglomerateSkeletonAction(layerName, mappingName, segmentId),
@@ -142,6 +147,8 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
 
   if (!proofreadUsingMeshes()) return;
 
+  /* Load a coarse ad hoc mesh of the agglomerate at the click position */
+
   yield* call(loadCoarseAdHocMesh, layerName, resolutionInfo, segmentId, position);
 
   if (treeName == null) return;
@@ -152,7 +159,9 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
 
   if (tree == null) return;
 
-  // Find all segments (nodes) that are within x µm to load meshes in oversegmentation
+  /* Find all segments (nodes) of the agglomerate skeleton within proofreadSegmentSurroundNm (graph distance)
+     and request the segment IDs in the oversegmentation at the node positions */
+
   const nodePositions = tree.nodes.map((node) => node.position);
   const distanceSquared = proofreadSegmentSurroundNm() ** 2;
   const scale = yield* select((state) => state.dataset.dataSource.scale);
@@ -176,7 +185,7 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
       ),
     ),
   );
-  // HACK: This only works for uint32 segmentations
+  // TODO HACK: This only works for uint32 segmentations
   const segmentIdsInSurround = segmentIdsArrayBuffers.map((buffer) => new Uint32Array(buffer)[0]);
 
   if (oldSegmentIdsInSurround != null) {
@@ -194,7 +203,8 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
 
   oldSegmentIdsInSurround = [...segmentIdsInSurround];
 
-  // Load meshes in oversegmentation in fine resolution
+  /* Load fine ad hoc meshes of the segments in the oversegmentation in the click position surround */
+
   const noMappingInfo = {
     mappingName: null,
     mappingType: null,
@@ -276,6 +286,8 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
     yield* call(createEditableMapping);
   }
 
+  /* Find out the agglomerate IDs at the two node positions */
+
   const resolutionInfo = getResolutionInfo(volumeTracingLayer.resolutions);
   // The mag the agglomerate skeleton corresponds to should be the finest available mag of the volume tracing layer
   const agglomerateFileMag = resolutionInfo.getLowestResolution();
@@ -307,6 +319,9 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
     agglomerateFileZoomstep,
   );
 
+  /* Send the respective split/merge update action to the backend (by pushing to the save queue
+     and saving immediately) */
+
   const items = [];
   if (action.type === "MERGE_TREES") {
     if (sourceNodeAgglomerateId === targetNodeAgglomerateId) {
@@ -342,6 +357,8 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
   yield* put(pushSaveQueueTransaction(items, "mapping", volumeTracingId));
   yield* call([Model, Model.ensureSavedState]);
 
+  /* Reload the segmentation */
+
   yield* call([api.data, api.data.reloadBuckets], layerName);
 
   const volumeTracingWithEditableMapping = yield* select((state) =>
@@ -368,7 +385,8 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
     agglomerateFileZoomstep,
   );
 
-  // Remove old agglomerate skeleton(s) and load new agglomerate skeleton(s)
+  /* Remove old agglomerate skeleton(s) and load updated agglomerate skeleton(s) */
+
   yield* put(deleteTreeAction(sourceTree.treeId));
   if (sourceTree !== targetTree) {
     yield* put(deleteTreeAction(targetTree.treeId));
@@ -393,6 +411,8 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
     );
   }
 
+  /* Remove old meshes and load updated meshes */
+
   if (proofreadUsingMeshes()) {
     // Remove old over segmentation meshes
     if (oldSegmentIdsInSurround != null) {
@@ -405,7 +425,7 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
       oldSegmentIdsInSurround = null;
     }
 
-    // Remove old agglomerate mesh(es) and load new agglomerate mesh(es)
+    // Remove old agglomerate mesh(es) and load updated agglomerate mesh(es)
     yield* put(removeIsosurfaceAction(layerName, sourceNodeAgglomerateId));
     if (targetNodeAgglomerateId !== sourceNodeAgglomerateId) {
       yield* put(removeIsosurfaceAction(layerName, targetNodeAgglomerateId));

From e4c1261633938f4225563586af191b831b4df7fe Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Fri, 10 Jun 2022 09:32:04 +0200
Subject: [PATCH 099/122] pr feedback (part 2)

---
 MIGRATIONS.unreleased.md                          |  2 +-
 .../datastore/services/AgglomerateService.scala   |  2 --
 .../datastore/services/BinaryDataService.scala    | 12 +++++-------
 .../datastore/services/IsosurfaceService.scala    |  7 -------
 .../controllers/VolumeTracingController.scala     | 15 ++++++---------
 .../tracings/volume/VolumeTracingService.scala    |  6 ++----
 6 files changed, 14 insertions(+), 30 deletions(-)

diff --git a/MIGRATIONS.unreleased.md b/MIGRATIONS.unreleased.md
index 2726a336dca..ea9528d33ca 100644
--- a/MIGRATIONS.unreleased.md
+++ b/MIGRATIONS.unreleased.md
@@ -8,6 +8,6 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 ## Unreleased
 [Commits](https://github.com/scalableminds/webknossos/compare/22.06.0...HEAD)
 
- - Note that this upgrade can not be trivially rolled back, since new rocksDB column families are added and it is not easy to remove them again from an existing database. [#6195](https://github.com/scalableminds/webknossos/pull/6195)
+ - FossilDB now has to be started with two new additional column families: editableMappings,editableMappingUpdates. Note that this upgrade can not be trivially rolled back, since new rocksDB column families are added and it is not easy to remove them again from an existing database. In case webKnossos needs to be rolled back, it is recommended to still keep the new column families in FossilDB. [#6195](https://github.com/scalableminds/webknossos/pull/6195)
 
 ### Postgres Evolutions:
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
index f92f7791179..b5ea04904a2 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala
@@ -227,8 +227,6 @@ class AgglomerateService @Inject()(config: DataStoreConfig) extends DataConverte
       val edgesRange: Array[Long] =
         reader.uint64().readArrayBlockWithOffset("/agglomerate_to_edges_offsets", 2, agglomerateId)
 
-      logger.info(s"positionsRange: ${positionsRange(0)} to ${positionsRange(1)}, agglomerateId: $agglomerateId")
-
       val nodeCount = positionsRange(1) - positionsRange(0)
       val edgeCount = edgesRange(1) - edgesRange(0)
       val edgeLimit = config.Datastore.AgglomerateSkeleton.maxEdges
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
index d6444056fac..00073e5c379 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
@@ -53,13 +53,11 @@ class BinaryDataService(val dataBaseDir: Path, maxCacheSize: Int, val agglomerat
       case (request, index) =>
         for {
           data <- handleDataRequest(request)
-          mappedData = if (agglomerateService == null) data
-          else
-            convertIfNecessary(
-              request.settings.appliedAgglomerate.isDefined && request.dataLayer.category == Category.segmentation && request.cuboid.mag.maxDim <= MaxMagForAgglomerateMapping,
-              data,
-              agglomerateService.applyAgglomerate(request)
-            )
+          mappedData = convertIfNecessary(
+            request.settings.appliedAgglomerate.isDefined && request.dataLayer.category == Category.segmentation && request.cuboid.mag.maxDim <= MaxMagForAgglomerateMapping,
+            data,
+            agglomerateService.applyAgglomerate(request)
+          )
           convertedData = convertIfNecessary(
             request.dataLayer.elementClass == ElementClass.uint64 && request.dataLayer.category == Category.segmentation,
             mappedData,
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
index bbf5aad5352..2ebda08e2a2 100644
--- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
+++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/IsosurfaceService.scala
@@ -197,16 +197,12 @@ class IsosurfaceService(binaryDataService: BinaryDataService,
     val vertexBuffer = mutable.ArrayBuffer[Vec3Double]()
 
     for {
-      before <- Fox.successful(System.currentTimeMillis())
       data <- binaryDataService.handleDataRequest(dataRequest)
-      afterLoading = System.currentTimeMillis()
       agglomerateMappedData = applyAgglomerate(data)
       typedData = convertData(agglomerateMappedData)
       mappedData <- applyMapping(typedData)
-      agglomerateIds: Set[T] = mappedData.toSet
       mappedSegmentId <- applyMapping(Array(typedSegmentId)).map(_.head)
       neighbors = findNeighbors(mappedData, dataDimensions, mappedSegmentId)
-      afterPreprocessing = System.currentTimeMillis()
     } yield {
       for {
         x <- 0 until dataDimensions.x by 32
@@ -228,9 +224,6 @@ class IsosurfaceService(binaryDataService: BinaryDataService,
                                          vertexBuffer)
         }
       }
-      val afterMarchingCubes = System.currentTimeMillis()
-      /*logger.info(
-        s"Isosurface generation timing - loading: ${afterLoading - before} ms, preprocessing: ${afterPreprocessing - afterLoading}, marchingCubes: ${afterMarchingCubes - afterPreprocessing}. ${agglomerateIds.size} agglomerates in data, ${typedData.length} voxels.")*/
       (vertexBuffer.flatMap(_.toList.map(_.toFloat)).toArray, neighbors)
     }
   }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index f5ff3439b78..82ed2c00244 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -150,8 +150,7 @@ class VolumeTracingController @Inject()(
         accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) {
           for {
             tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound")
-            hasEditableMapping <- Fox.runOptional(tracing.mappingName)(editableMappingService.exists)
-            (data, indices) <- if (hasEditableMapping.getOrElse(false))
+            (data, indices) <- if (tracing.mappingIsEditable.getOrElse(false))
               editableMappingService.volumeData(tracing, request.body, token)
             else tracingService.data(tracingId, tracing, request.body)
           } yield Ok(data).withHeaders(getMissingBucketsHeaders(indices): _*)
@@ -176,8 +175,7 @@ class VolumeTracingController @Inject()(
         accessTokenService.validateAccess(UserAccessRequest.webknossos, token) {
           for {
             tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound")
-            hasEditableMapping: Option[Boolean] <- Fox.runOptional(tracing.mappingName)(editableMappingService.exists)
-            _ <- bool2Fox(!hasEditableMapping.getOrElse(false)) ?~> "Duplicate is not yet implemented for editable mapping annotations"
+            _ <- bool2Fox(!tracing.mappingIsEditable.getOrElse(false)) ?~> "Duplicate is not yet implemented for editable mapping annotations"
             dataSetBoundingBox = request.body.asJson.flatMap(_.validateOpt[BoundingBox].asOpt.flatten)
             resolutionRestrictions = ResolutionRestrictions(minResolution, maxResolution)
             (newId, newTracing) <- tracingService.duplicate(tracingId,
@@ -224,8 +222,7 @@ class VolumeTracingController @Inject()(
           // consecutive 3D points (i.e., nine floats) form a triangle.
           // There are no shared vertices between triangles.
           tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound")
-          hasEditableMapping <- Fox.runOptional(tracing.mappingName)(editableMappingService.exists)
-          (vertices, neighbors) <- if (hasEditableMapping.getOrElse(false))
+          (vertices, neighbors) <- if (tracing.mappingIsEditable.getOrElse(false))
             editableMappingService.createIsosurface(tracing, request.body, token)
           else tracingService.createIsosurface(tracingId, request.body)
         } yield {
@@ -439,9 +436,9 @@ class VolumeTracingController @Inject()(
       accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) {
         for {
           tracing <- tracingService.find(tracingId)
+          _ <- bool2Fox(tracing.mappingIsEditable.getOrElse(false)) ?~> "Cannot query agglomerate skeleton for volume annotation"
           mappingName <- tracing.mappingName ?~> "annotation.agglomerateSkeleton.noMappingSet"
           remoteFallbackLayer <- editableMappingService.remoteFallbackLayer(tracing)
-          _ <- Fox.assertTrue(editableMappingService.exists(mappingName)) ?~> "Cannot query agglomerate skeleton for volume annotation"
           agglomerateSkeletonBytes <- editableMappingService.getAgglomerateSkeletonWithFallback(mappingName,
                                                                                                 remoteFallbackLayer,
                                                                                                 agglomerateId,
@@ -488,7 +485,7 @@ class VolumeTracingController @Inject()(
         for {
           tracing <- tracingService.find(tracingId)
           mappingName <- tracing.mappingName.toFox
-          _ <- Fox.assertTrue(editableMappingService.exists(mappingName))
+          _ <- bool2Fox(tracing.mappingIsEditable.getOrElse(false)) ?~> "Mapping is not editable"
           currentVersion <- editableMappingService.newestMaterializableVersion(mappingName)
           _ <- bool2Fox(request.body.length == 1) ?~> "Editable mapping update group must contain exactly one update group"
           updateGroup <- request.body.headOption.toFox
@@ -506,7 +503,7 @@ class VolumeTracingController @Inject()(
           for {
             tracing <- tracingService.find(tracingId)
             mappingName <- tracing.mappingName.toFox
-            _ <- Fox.assertTrue(editableMappingService.exists(mappingName))
+            _ <- bool2Fox(tracing.mappingIsEditable.getOrElse(false)) ?~> "Mapping is not editable"
             updateLog <- editableMappingService.updateActionLog(mappingName)
           } yield Ok(updateLog)
         }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
index e104042205f..373fcc9fd88 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
@@ -83,14 +83,12 @@ class VolumeTracingService @Inject()(
                         updateGroup: UpdateActionGroup[VolumeTracing],
                         previousVersion: Long): Fox[Unit] =
     for {
-      tracing <- find(tracingId)
-      hasEditableMapping <- Fox.runOptional(tracing.mappingName)(editableMappingService.exists)
-      updatedTracing: VolumeTracing <- updateGroup.actions.foldLeft(Fox.successful(tracing)) { (tracingFox, action) =>
+      updatedTracing: VolumeTracing <- updateGroup.actions.foldLeft(find(tracingId)) { (tracingFox, action) =>
         tracingFox.futureBox.flatMap {
           case Full(tracing) =>
             action match {
               case a: UpdateBucketVolumeAction =>
-                if (hasEditableMapping.getOrElse(false)) {
+                if (tracing.mappingIsEditable.getOrElse(false)) {
                   Fox.failure("Cannot mutate buckets in annotation with editable mapping.")
                 } else updateBucket(tracingId, tracing, a, updateGroup.version)
               case a: UpdateTracingVolumeAction =>

From bd7a56072d8ad59aa5db41e19c234e91f82fe537 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Mon, 13 Jun 2022 19:18:32 +0200
Subject: [PATCH 100/122] PR feedback part 1/x

---
 frontend/javascripts/admin/admin_rest_api.ts  |  5 +-
 .../controller/combinations/tool_controls.ts  |  3 +
 .../model/accessors/volumetracing_accessor.ts | 26 ++++---
 .../oxalis/model/actions/settings_actions.ts  |  3 +
 .../model/actions/volumetracing_actions.ts    |  2 +-
 .../oxalis/model/reducers/save_reducer.ts     |  2 +-
 .../oxalis/model/reducers/settings_reducer.ts |  9 ++-
 .../model/reducers/volumetracing_reducer.ts   |  6 +-
 .../oxalis/model/sagas/isosurface_saga.ts     |  8 +-
 .../oxalis/model/sagas/mapping_saga.ts        |  2 +-
 .../oxalis/model/sagas/proofread_saga.ts      | 74 +++++++++----------
 .../oxalis/model/sagas/save_saga.ts           | 10 +--
 .../oxalis/model/sagas/update_actions.ts      | 37 ++--------
 frontend/javascripts/oxalis/store.ts          |  8 +-
 .../mapping_settings_view.tsx                 |  2 +-
 .../test/sagas/skeletontracing_saga.spec.ts   |  6 +-
 .../volumetracing/volumetracing_saga.spec.ts  |  6 +-
 17 files changed, 93 insertions(+), 116 deletions(-)

diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts
index fe2de72d98e..11d9bafb6b3 100644
--- a/frontend/javascripts/admin/admin_rest_api.ts
+++ b/frontend/javascripts/admin/admin_rest_api.ts
@@ -1949,7 +1949,8 @@ export function getAgglomerateSkeleton(
   return doWithToken((token) =>
     Request.receiveArraybuffer(
       `${dataStoreUrl}/data/datasets/${datasetId.owningOrganization}/${datasetId.name}/layers/${layerName}/agglomerates/${mappingId}/skeleton/${agglomerateId}?token=${token}`, // The webworker code cannot do proper error handling and always expects an array buffer from the server.
-      // In this case, the server sends an error json instead of an array buffer sometimes. Therefore, don't use the webworker code.
+      // The webworker code cannot do proper error handling and always expects an array buffer from the server.
+      // However, the server might send an error json instead of an array buffer. Therefore, don't use the webworker code.
       {
         useWebworkerForArrayBuffer: false,
         showErrorToast: false,
@@ -1967,7 +1968,7 @@ export function getEditableAgglomerateSkeleton(
     Request.receiveArraybuffer(
       `${tracingStoreUrl}/tracings/volume/${tracingId}/agglomerateSkeleton/${agglomerateId}?token=${token}`,
       // The webworker code cannot do proper error handling and always expects an array buffer from the server.
-      // In this case, the server sends an error json instead of an array buffer sometimes. Therefore, don't use the webworker code.
+      // However, the server might send an error json instead of an array buffer. Therefore, don't use the webworker code.
       {
         useWebworkerForArrayBuffer: false,
         showErrorToast: false,
diff --git a/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts b/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts
index 246f8e116c6..c2c64dc3a63 100644
--- a/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts
+++ b/frontend/javascripts/oxalis/controller/combinations/tool_controls.ts
@@ -643,6 +643,9 @@ export class ProofreadTool {
 
     let globalPosition;
     if (plane === OrthoViews.TDView) {
+      // In the 3D viewport the click position cannot be uniquely determined, because the position on the
+      // third axis is ambiguous. However, if the user clicked on a node, we can determine the position
+      // by looking up the position of the selected node.
       if (didSelectNode) {
         getSkeletonTracing(Store.getState().tracing).map((skeletonTracing: SkeletonTracing) =>
           getNodeAndTree(skeletonTracing).map(([_activeTree, activeNode]) => {
diff --git a/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts b/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts
index 5bcb24e77ea..54a42bb942b 100644
--- a/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts
+++ b/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts
@@ -416,8 +416,11 @@ export function getMappingInfoForVolumeTracing(
   return getMappingInfo(state.temporaryConfiguration.activeMappingByLayer, tracingId);
 }
 
-export function hasEditableMapping(state: OxalisState): boolean {
-  const volumeTracing = getActiveSegmentationTracing(state);
+export function hasEditableMapping(
+  state: OxalisState,
+  layerName?: string | null | undefined,
+): boolean {
+  const volumeTracing = getRequestedOrDefaultSegmentationTracingLayer(state, layerName);
 
   if (volumeTracing == null) return false;
 
@@ -427,21 +430,20 @@ export function hasEditableMapping(state: OxalisState): boolean {
 export function isMappingActivationAllowed(
   state: OxalisState,
   mappingName: string | null | undefined,
+  layerName?: string | null | undefined,
 ): boolean {
-  const isEditableMappingActive = hasEditableMapping(state);
+  const isEditableMappingActive = hasEditableMapping(state, layerName);
 
-  if (isEditableMappingActive) {
-    const volumeTracing = getActiveSegmentationTracing(state);
+  if (!isEditableMappingActive) return true;
 
-    // This should never be the case, since editable mappings can only be active for volume tracings
-    if (volumeTracing == null) return false;
+  const volumeTracing = getRequestedOrDefaultSegmentationTracingLayer(state, layerName);
 
-    // Only allow mapping activations of the editable mapping itself if an editable mapping is saved
-    // in the volume tracing. Editable mappings cannot be disabled or switched for now.
-    return mappingName === volumeTracing.mappingName;
-  }
+  // This should never be the case, since editable mappings can only be active for volume tracings
+  if (volumeTracing == null) return false;
 
-  return true;
+  // Only allow mapping activations of the editable mapping itself if an editable mapping is saved
+  // in the volume tracing. Editable mappings cannot be disabled or switched for now.
+  return mappingName === volumeTracing.mappingName;
 }
 
 export function getLastLabelAction(volumeTracing: VolumeTracing): LabelAction | undefined {
diff --git a/frontend/javascripts/oxalis/model/actions/settings_actions.ts b/frontend/javascripts/oxalis/model/actions/settings_actions.ts
index 841ce299234..aea16a375d6 100644
--- a/frontend/javascripts/oxalis/model/actions/settings_actions.ts
+++ b/frontend/javascripts/oxalis/model/actions/settings_actions.ts
@@ -88,6 +88,7 @@ export type SetMappingAction = {
 export type SetMappingNameAction = {
   type: "SET_MAPPING_NAME";
   mappingName: string;
+  mappingType: MappingType;
   layerName: string;
 };
 type SetHideUnmappedIdsAction = {
@@ -242,10 +243,12 @@ export const setMappingAction = (
 export const setMappingNameAction = (
   layerName: string,
   mappingName: string,
+  mappingType: MappingType,
 ): SetMappingNameAction => ({
   type: "SET_MAPPING_NAME",
   layerName,
   mappingName,
+  mappingType,
 });
 export const setHideUnmappedIdsAction = (
   layerName: string,
diff --git a/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts b/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts
index 1adca1a2007..2ff4b5b3c98 100644
--- a/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts
+++ b/frontend/javascripts/oxalis/model/actions/volumetracing_actions.ts
@@ -279,6 +279,6 @@ export const dispatchFloodfillAsync = async (
   dispatch(action);
   await readyDeferred.promise();
 };
-export const setMappingisEditableAction = (): SetMappingIsEditableAction => ({
+export const setMappingIsEditableAction = (): SetMappingIsEditableAction => ({
   type: "SET_MAPPING_IS_EDITABLE",
 });
diff --git a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts
index 67c3e70fecc..ff8dd8bc38a 100644
--- a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts
+++ b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts
@@ -104,7 +104,7 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState {
 
     case "INITIALIZE_EDITABLE_MAPPING": {
       // Set up empty save queue array for editable mapping
-      const newMappingsQueue = { ...state.save.queue.volumes, [action.mapping.tracingId]: [] };
+      const newMappingsQueue = { ...state.save.queue.mappings, [action.mapping.tracingId]: [] };
       return updateKey2(state, "save", "queue", {
         mappings: newMappingsQueue,
       });
diff --git a/frontend/javascripts/oxalis/model/reducers/settings_reducer.ts b/frontend/javascripts/oxalis/model/reducers/settings_reducer.ts
index bf091eacf1b..c2756d2556d 100644
--- a/frontend/javascripts/oxalis/model/reducers/settings_reducer.ts
+++ b/frontend/javascripts/oxalis/model/reducers/settings_reducer.ts
@@ -211,11 +211,12 @@ function SettingsReducer(state: OxalisState, action: Action): OxalisState {
     }
 
     case "SET_MAPPING_ENABLED": {
+      const { isMappingEnabled, layerName } = action;
+
       // Editable mappings cannot be disabled or switched for now
-      const isEditableMappingActive = hasEditableMapping(state);
+      const isEditableMappingActive = hasEditableMapping(state, layerName);
       if (isEditableMappingActive && !action.isMappingEnabled) return state;
 
-      const { isMappingEnabled, layerName } = action;
       return updateActiveMapping(
         state,
         {
@@ -240,7 +241,7 @@ function SettingsReducer(state: OxalisState, action: Action): OxalisState {
       const { mappingName, mapping, mappingKeys, mappingColors, mappingType, layerName } = action;
 
       // Editable mappings cannot be disabled or switched for now
-      if (!isMappingActivationAllowed(state, mappingName)) return state;
+      if (!isMappingActivationAllowed(state, mappingName, layerName)) return state;
 
       const hideUnmappedIds =
         action.hideUnmappedIds != null
@@ -268,7 +269,7 @@ function SettingsReducer(state: OxalisState, action: Action): OxalisState {
       const { mappingName, layerName } = action;
 
       // Editable mappings cannot be disabled or switched for now
-      if (!isMappingActivationAllowed(state, mappingName)) return state;
+      if (!isMappingActivationAllowed(state, mappingName, layerName)) return state;
 
       return updateActiveMapping(state, { mappingName }, layerName);
     }
diff --git a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
index 3fdc36d7bbf..20569967832 100644
--- a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
+++ b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts
@@ -310,10 +310,8 @@ function VolumeTracingReducer(
       // Editable mappings cannot be disabled or switched for now
       if (volumeTracing.mappingIsEditable) return state;
 
-      const { mappingName } = action;
-      return updateVolumeTracing(state, volumeTracing.tracingId, {
-        mappingName,
-      });
+      const { mappingName, mappingType } = action;
+      return setMappingNameReducer(state, volumeTracing, mappingName, mappingType);
     }
 
     case "SET_MAPPING_IS_EDITABLE": {
diff --git a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
index 2d9977e234c..ce0297173b5 100644
--- a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
@@ -49,7 +49,6 @@ import { zoomedAddressToAnotherZoomStepWithInfo } from "oxalis/model/helpers/pos
 import DataLayer from "oxalis/model/data_layer";
 import Model from "oxalis/model";
 import ThreeDMap from "libs/ThreeDMap";
-import * as Utils from "libs/utils";
 import exportToStl from "libs/stl_exporter";
 import getSceneController from "oxalis/controller/scene_controller_provider";
 import parseStlBuffer from "libs/parse_stl_buffer";
@@ -122,11 +121,8 @@ function getZoomedCubeSize(zoomStep: number, resolutionInfo: ResolutionInfo): Ve
 }
 
 function clipPositionToCubeBoundary(position: Vector3): Vector3 {
-  const currentCube = Utils.map3(
-    (el, idx) => Math.floor(el / marchingCubeSizeInMag1()[idx]),
-    position,
-  );
-  const clippedPosition = Utils.map3((el, idx) => el * marchingCubeSizeInMag1()[idx], currentCube);
+  const currentCube = V3.floor(V3.divide3(position, marchingCubeSizeInMag1()));
+  const clippedPosition = V3.scale3(currentCube, marchingCubeSizeInMag1());
   return clippedPosition;
 }
 
diff --git a/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts b/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts
index 4b693b11dd5..98a0f70c98d 100644
--- a/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts
@@ -86,7 +86,7 @@ function* maybeFetchMapping(
 
   // Editable mappings cannot be disabled or switched for now
   const isEditableMappingActivationAllowed = yield* select((state) =>
-    isMappingActivationAllowed(state, mappingName),
+    isMappingActivationAllowed(state, mappingName, layerName),
   );
   if (!isEditableMappingActivationAllowed) return;
 
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 9680640e680..3bedbd9f651 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -11,7 +11,7 @@ import {
 } from "oxalis/model/actions/skeletontracing_actions";
 import {
   initializeEditableMappingAction,
-  setMappingisEditableAction,
+  setMappingIsEditableAction,
 } from "oxalis/model/actions/volumetracing_actions";
 import type { ProofreadAtPositionAction } from "oxalis/model/actions/proofread_actions";
 import {
@@ -71,11 +71,11 @@ function proofreadUsingMeshes(): boolean {
   // @ts-ignore
   return window.__proofreadUsingMeshes != null ? window.__proofreadUsingMeshes : true;
 }
-function proofreadSegmentSurroundNm(): number {
+function proofreadSegmentProximityNm(): number {
   // @ts-ignore
-  return window.__proofreadSurroundNm != null ? window.__proofreadSurroundNm : 2000;
+  return window.__proofreadProximityNm != null ? window.__proofreadProximityNm : 2000;
 }
-let oldSegmentIdsInSurround: number[] | null = null;
+let oldSegmentIdsInProximity: number[] | null = null;
 
 function* loadCoarseAdHocMesh(
   layerName: string,
@@ -89,10 +89,9 @@ function* loadCoarseAdHocMesh(
   const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
   if (volumeTracing == null) return;
 
-  const activeMappingByLayer = yield* select(
-    (state) => state.temporaryConfiguration.activeMappingByLayer,
+  const mappingInfo = yield* select((state) =>
+    getMappingInfo(state.temporaryConfiguration.activeMappingByLayer, layerName),
   );
-  const mappingInfo = getMappingInfo(activeMappingByLayer, layerName);
   const { mappingName, mappingType } = mappingInfo;
 
   // Load the whole agglomerate mesh in a coarse resolution for performance reasons
@@ -100,9 +99,9 @@ function* loadCoarseAdHocMesh(
     (state) => state.temporaryConfiguration.preferredQualityForMeshAdHocComputation,
   );
 
-  const resolutionIndices = resolutionInfo.getAllIndices();
-  const coarseResolutionIndex =
-    resolutionIndices[Math.min(proofreadCoarseResolutionIndex(), resolutionIndices.length - 1)];
+  const coarseResolutionIndex = resolutionInfo.getClosestExistingIndex(
+    proofreadCoarseResolutionIndex(),
+  );
   yield* put(
     updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", coarseResolutionIndex),
   );
@@ -159,15 +158,16 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
 
   if (tree == null) return;
 
-  /* Find all segments (nodes) of the agglomerate skeleton within proofreadSegmentSurroundNm (graph distance)
+  /* Find all segments (nodes) of the agglomerate skeleton within proofreadSegmentProximityNm (graph distance)
      and request the segment IDs in the oversegmentation at the node positions */
 
   const nodePositions = tree.nodes.map((node) => node.position);
-  const distanceSquared = proofreadSegmentSurroundNm() ** 2;
+  const proximityDistanceSquared = proofreadSegmentProximityNm() ** 2;
   const scale = yield* select((state) => state.dataset.dataSource.scale);
 
-  const nodePositionsInSurround = nodePositions.filter(
-    (nodePosition) => V3.scaledSquaredDist(nodePosition, position, scale) <= distanceSquared,
+  const nodePositionsInProximity = nodePositions.filter(
+    (nodePosition) =>
+      V3.scaledSquaredDist(nodePosition, position, scale) <= proximityDistanceSquared,
   );
   const mag = resolutionInfo.getLowestResolution();
 
@@ -176,7 +176,7 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
 
   // Request unmapped segmentation ids
   const segmentIdsArrayBuffers: ArrayBuffer[] = yield* all(
-    nodePositionsInSurround.map((nodePosition) =>
+    nodePositionsInProximity.map((nodePosition) =>
       call(
         [api.data, api.data.getRawDataCuboid],
         fallbackLayerName,
@@ -186,24 +186,24 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
     ),
   );
   // TODO HACK: This only works for uint32 segmentations
-  const segmentIdsInSurround = segmentIdsArrayBuffers.map((buffer) => new Uint32Array(buffer)[0]);
+  const segmentIdsInProximity = segmentIdsArrayBuffers.map((buffer) => new Uint32Array(buffer)[0]);
 
-  if (oldSegmentIdsInSurround != null) {
-    const segmentIdsInSurroundSet = new Set(segmentIdsInSurround);
+  if (oldSegmentIdsInProximity != null) {
+    const segmentIdsInProximitySet = new Set(segmentIdsInProximity);
     // Remove old meshes in oversegmentation
     yield* all(
-      oldSegmentIdsInSurround.map((nodeSegmentId) =>
-        // Only remove meshes that are not part of the new segment surround
-        segmentIdsInSurroundSet.has(nodeSegmentId)
+      oldSegmentIdsInProximity.map((nodeSegmentId) =>
+        // Only remove meshes that are not part of the new proximity set
+        segmentIdsInProximitySet.has(nodeSegmentId)
           ? null
           : put(removeIsosurfaceAction(layerName, nodeSegmentId)),
       ),
     );
   }
 
-  oldSegmentIdsInSurround = [...segmentIdsInSurround];
+  oldSegmentIdsInProximity = [...segmentIdsInProximity];
 
-  /* Load fine ad hoc meshes of the segments in the oversegmentation in the click position surround */
+  /* Load fine ad hoc meshes of the segments in the oversegmentation in the proximity of the click position */
 
   const noMappingInfo = {
     mappingName: null,
@@ -213,18 +213,19 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
   const oldPreferredQuality = yield* select(
     (state) => state.temporaryConfiguration.preferredQualityForMeshAdHocComputation,
   );
+
+  const fineResolutionIndex = resolutionInfo.getClosestExistingIndex(
+    proofreadFineResolutionIndex(),
+  );
   yield* put(
-    updateTemporarySettingAction(
-      "preferredQualityForMeshAdHocComputation",
-      proofreadFineResolutionIndex(),
-    ),
+    updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", fineResolutionIndex),
   );
   yield* all(
-    segmentIdsInSurround.map((nodeSegmentId, index) =>
+    segmentIdsInProximity.map((nodeSegmentId, index) =>
       put(
         loadAdHocMeshAction(
           nodeSegmentId,
-          nodePositionsInSurround[index],
+          nodePositionsInProximity[index],
           noMappingInfo,
           layerName,
         ),
@@ -249,8 +250,8 @@ function* createEditableMapping(): Saga<void> {
   const serverEditableMapping = yield* call(makeMappingEditable, tracingStoreUrl, volumeTracingId);
   // The server increments the volume tracing's version by 1 when switching the mapping to an editable one
   yield* put(setVersionNumberAction(upToDateVolumeTracing.version + 1, "volume", volumeTracingId));
-  yield* put(setMappingNameAction(layerName, serverEditableMapping.mappingName));
-  yield* put(setMappingisEditableAction());
+  yield* put(setMappingNameAction(layerName, serverEditableMapping.mappingName, "HDF5"));
+  yield* put(setMappingIsEditableAction());
   yield* put(initializeEditableMappingAction(serverEditableMapping));
 }
 
@@ -268,10 +269,9 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
   const { tracingId: volumeTracingId } = volumeTracing;
 
   const layerName = volumeTracingId;
-  const activeMappingByLayer = yield* select(
-    (state) => state.temporaryConfiguration.activeMappingByLayer,
+  const mappingInfo = yield* select((state) =>
+    getMappingInfo(state.temporaryConfiguration.activeMappingByLayer, layerName),
   );
-  const mappingInfo = getMappingInfo(activeMappingByLayer, layerName);
   const { mappingName, mappingType, mappingStatus } = mappingInfo;
   if (
     mappingName == null ||
@@ -415,14 +415,14 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
 
   if (proofreadUsingMeshes()) {
     // Remove old over segmentation meshes
-    if (oldSegmentIdsInSurround != null) {
+    if (oldSegmentIdsInProximity != null) {
       // Remove old meshes in oversegmentation
       yield* all(
-        oldSegmentIdsInSurround.map((nodeSegmentId) =>
+        oldSegmentIdsInProximity.map((nodeSegmentId) =>
           put(removeIsosurfaceAction(layerName, nodeSegmentId)),
         ),
       );
-      oldSegmentIdsInSurround = null;
+      oldSegmentIdsInProximity = null;
     }
 
     // Remove old agglomerate mesh(es) and load updated agglomerate mesh(es)
diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
index 786c8ef07b1..5a4956e4eb1 100644
--- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
@@ -998,12 +998,12 @@ function setWkReady() {
 }
 export function* saveTracingAsync(): Saga<void> {
   yield* takeEvery("WK_READY", setWkReady);
-  yield* takeEvery("INITIALIZE_SKELETONTRACING", saveTracingTypeAsync);
-  yield* takeEvery("INITIALIZE_VOLUMETRACING", saveTracingTypeAsync);
-  yield* takeEvery("INITIALIZE_EDITABLE_MAPPING", saveEditableMappingAsync);
+  yield* takeEvery("INITIALIZE_SKELETONTRACING", setupSavingForTracingType);
+  yield* takeEvery("INITIALIZE_VOLUMETRACING", setupSavingForTracingType);
+  yield* takeEvery("INITIALIZE_EDITABLE_MAPPING", setupSavingForEditableMapping);
 }
 
-export function* saveEditableMappingAsync(
+export function* setupSavingForEditableMapping(
   initializeAction: InitializeEditableMappingAction,
 ): Saga<void> {
   // No diffing needs to be done for editable mappings as the saga pushes update actions
@@ -1011,7 +1011,7 @@ export function* saveEditableMappingAsync(
   const volumeTracingId = initializeAction.mapping.tracingId;
   yield* fork(pushSaveQueueAsync, "mapping", volumeTracingId, isWkReady);
 }
-export function* saveTracingTypeAsync(
+export function* setupSavingForTracingType(
   initializeAction: InitializeSkeletonTracingAction | InitializeVolumeTracingAction,
 ): Saga<void> {
   /*
diff --git a/frontend/javascripts/oxalis/model/sagas/update_actions.ts b/frontend/javascripts/oxalis/model/sagas/update_actions.ts
index c2bf134d65c..a1ff4ea4bce 100644
--- a/frontend/javascripts/oxalis/model/sagas/update_actions.ts
+++ b/frontend/javascripts/oxalis/model/sagas/update_actions.ts
@@ -242,37 +242,14 @@ type AddServerValuesFn<T extends { value: any }> = (arg0: T) => T & {
 };
 
 type AsServerAction<A extends { value: any }> = ReturnType<AddServerValuesFn<A>>;
-// Since typescript does not provide ways to perform type transformations on the
-// single parts of a union, we need to write this out manually.
-export type ServerUpdateAction =
-  | AsServerAction<UpdateTreeUpdateAction>
-  | AsServerAction<DeleteTreeUpdateAction>
-  | AsServerAction<MergeTreeUpdateAction>
-  | AsServerAction<MoveTreeComponentUpdateAction>
-  | AsServerAction<CreateNodeUpdateAction>
-  | AsServerAction<UpdateNodeUpdateAction>
-  | AsServerAction<DeleteNodeUpdateAction>
-  | AsServerAction<CreateEdgeUpdateAction>
-  | AsServerAction<DeleteEdgeUpdateAction>
-  | AsServerAction<UpdateSkeletonTracingUpdateAction>
-  | AsServerAction<UpdateVolumeTracingUpdateAction>
-  | AsServerAction<UpdateUserBoundingBoxesUpdateAction>
-  | AsServerAction<CreateSegmentUpdateAction>
-  | AsServerAction<UpdateSegmentUpdateAction>
-  | AsServerAction<DeleteSegmentUpdateAction>
-  | AsServerAction<UpdateBucketUpdateAction>
-  | AsServerAction<UpdateTreeVisibilityUpdateAction>
-  | AsServerAction<UpdateTreeGroupVisibilityUpdateAction>
-  | AsServerAction<RevertToVersionUpdateAction>
-  | AsServerAction<UpdateTreeGroupsUpdateAction>
-  | AsServerAction<RemoveFallbackLayerUpdateAction>
-  | AsServerAction<UpdateTdCameraUpdateAction>
-  | AsServerAction<UpdateMappingNameUpdateAction>
-  | AsServerAction<SplitAgglomerateUpdateAction>
-  | AsServerAction<MergeAgglomerateUpdateAction>
+
+export type ServerUpdateAction = AsServerAction<
+  | UpdateAction
   // These two actions are never sent by the frontend and, therefore, don't exist in the UpdateAction type
-  | AsServerAction<ImportVolumeTracingUpdateAction>
-  | AsServerAction<CreateTracingUpdateAction>;
+  | ImportVolumeTracingUpdateAction
+  | CreateTracingUpdateAction
+>;
+
 export function createTree(tree: Tree): UpdateTreeUpdateAction {
   return {
     name: "createTree",
diff --git a/frontend/javascripts/oxalis/store.ts b/frontend/javascripts/oxalis/store.ts
index db6535202e4..d3a5a96d93f 100644
--- a/frontend/javascripts/oxalis/store.ts
+++ b/frontend/javascripts/oxalis/store.ts
@@ -23,6 +23,7 @@ import type {
   MeshMetaData,
   TracingType,
   APIMeshFile,
+  ServerEditableMapping,
 } from "types/api_flow_types";
 import type { Action } from "oxalis/model/actions/actions";
 import type {
@@ -237,13 +238,8 @@ export type VolumeTracing = TracingBase & {
 export type ReadOnlyTracing = TracingBase & {
   readonly type: "readonly";
 };
-export type EditableMapping = {
+export type EditableMapping = Readonly<ServerEditableMapping> & {
   readonly type: "mapping";
-  readonly createdTimestamp: number;
-  readonly version: number;
-  readonly mappingName: string;
-  // The id of the volume tracing the editable mapping belongs to
-  readonly tracingId: string;
 };
 export type HybridTracing = Annotation & {
   readonly skeleton: SkeletonTracing | null | undefined;
diff --git a/frontend/javascripts/oxalis/view/left-border-tabs/mapping_settings_view.tsx b/frontend/javascripts/oxalis/view/left-border-tabs/mapping_settings_view.tsx
index 1060f12b779..93333b109f8 100644
--- a/frontend/javascripts/oxalis/view/left-border-tabs/mapping_settings_view.tsx
+++ b/frontend/javascripts/oxalis/view/left-border-tabs/mapping_settings_view.tsx
@@ -293,7 +293,7 @@ function mapStateToProps(state: OxalisState, ownProps: OwnProps) {
     segmentationLayer: getSegmentationLayerByName(state.dataset, ownProps.layerName),
     isMergerModeEnabled: state.temporaryConfiguration.isMergerModeEnabled,
     allowUpdate: state.tracing.restrictions.allowUpdate,
-    isEditableMappingActive: hasEditableMapping(state),
+    isEditableMappingActive: hasEditableMapping(state, ownProps.layerName),
   };
 }
 
diff --git a/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts b/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts
index 5cba4a0f111..5a977ba4995 100644
--- a/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts
+++ b/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts
@@ -27,7 +27,7 @@ mockRequire("oxalis/model/sagas/root_saga", function* () {
   yield;
 });
 const { diffSkeletonTracing } = mockRequire.reRequire("oxalis/model/sagas/skeletontracing_saga");
-const { saveTracingTypeAsync } = mockRequire.reRequire("oxalis/model/sagas/save_saga");
+const { setupSavingForTracingType } = mockRequire.reRequire("oxalis/model/sagas/save_saga");
 const SkeletonTracingActions = mockRequire.reRequire(
   "oxalis/model/actions/skeletontracing_actions",
 );
@@ -109,7 +109,7 @@ const createBranchPointAction = SkeletonTracingActions.createBranchPointAction(
   12345678,
 );
 test("SkeletonTracingSaga shouldn't do anything if unchanged (saga test)", (t) => {
-  const saga = saveTracingTypeAsync(
+  const saga = setupSavingForTracingType(
     SkeletonTracingActions.initializeSkeletonTracingAction(skeletonTracing),
   );
   saga.next(); // forking pushSaveQueueAsync
@@ -129,7 +129,7 @@ test("SkeletonTracingSaga shouldn't do anything if unchanged (saga test)", (t) =
 });
 test("SkeletonTracingSaga should do something if changed (saga test)", (t) => {
   const newState = SkeletonTracingReducer(initialState, createNodeAction);
-  const saga = saveTracingTypeAsync(
+  const saga = setupSavingForTracingType(
     SkeletonTracingActions.initializeSkeletonTracingAction(skeletonTracing),
   );
   saga.next(); // forking pushSaveQueueAsync
diff --git a/frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.spec.ts b/frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.spec.ts
index c39e8bad6d7..bbc3f03db87 100644
--- a/frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.spec.ts
+++ b/frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.spec.ts
@@ -29,7 +29,7 @@ mockRequire("oxalis/model/sagas/root_saga", function* () {
   yield;
 });
 
-const { saveTracingTypeAsync } = require("oxalis/model/sagas/save_saga");
+const { setupSavingForTracingType } = require("oxalis/model/sagas/save_saga");
 
 const { editVolumeLayerAsync, finishLayer } = require("oxalis/model/sagas/volumetracing_saga");
 
@@ -116,7 +116,7 @@ test.before("Mock Date.now", async () => {
   sinon.stub(Date, "now").returns(TIMESTAMP);
 });
 test("VolumeTracingSaga shouldn't do anything if unchanged (saga test)", (t) => {
-  const saga = saveTracingTypeAsync(
+  const saga = setupSavingForTracingType(
     VolumeTracingActions.initializeVolumeTracingAction(serverVolumeTracing),
   );
   saga.next(); // forking pushSaveQueueAsync
@@ -136,7 +136,7 @@ test("VolumeTracingSaga shouldn't do anything if unchanged (saga test)", (t) =>
 });
 test("VolumeTracingSaga should do something if changed (saga test)", (t) => {
   const newState = VolumeTracingReducer(initialState, setActiveCellAction);
-  const saga = saveTracingTypeAsync(
+  const saga = setupSavingForTracingType(
     VolumeTracingActions.initializeVolumeTracingAction(serverVolumeTracing),
   );
   saga.next(); // forking pushSaveQueueAsync

From 5944b599db17bc6aebda9982f9eec1a11984e7d9 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 14 Jun 2022 14:46:22 +0200
Subject: [PATCH 101/122] PR feedback 2/x

---
 frontend/javascripts/oxalis/constants.ts      |   2 +-
 .../combinations/bounding_box_handlers.ts     |  10 +-
 .../combinations/skeleton_handlers.ts         |  10 +-
 .../model/accessors/view_mode_accessor.ts     |   3 +-
 .../model/actions/segmentation_actions.ts     |   1 +
 .../oxalis/model/sagas/isosurface_saga.ts     |  20 ++-
 .../oxalis/model/sagas/proofread_saga.ts      |  50 ++-----
 .../oxalis/model/sagas/root_saga.ts           |   3 +
 .../oxalis/model/sagas/save_saga.ts           |  19 +--
 .../oxalis/model/sagas/wk_ready_saga.ts       |  20 +++
 .../javascripts/oxalis/view/context_menu.tsx  | 125 +++++++-----------
 .../view/layouting/tracing_layout_view.tsx    |   8 +-
 12 files changed, 129 insertions(+), 142 deletions(-)
 create mode 100644 frontend/javascripts/oxalis/model/sagas/wk_ready_saga.ts

diff --git a/frontend/javascripts/oxalis/constants.ts b/frontend/javascripts/oxalis/constants.ts
index 913475c6ef6..196b8cdab80 100644
--- a/frontend/javascripts/oxalis/constants.ts
+++ b/frontend/javascripts/oxalis/constants.ts
@@ -258,7 +258,7 @@ export type ShowContextMenuFunction = (
   arg1: number,
   arg2: number | null | undefined,
   arg3: number | null | undefined,
-  arg4: Vector3,
+  arg4: Vector3 | null | undefined,
   arg5: OrthoView,
 ) => void;
 const Constants = {
diff --git a/frontend/javascripts/oxalis/controller/combinations/bounding_box_handlers.ts b/frontend/javascripts/oxalis/controller/combinations/bounding_box_handlers.ts
index 03e9d033fee..3f8257aabdc 100644
--- a/frontend/javascripts/oxalis/controller/combinations/bounding_box_handlers.ts
+++ b/frontend/javascripts/oxalis/controller/combinations/bounding_box_handlers.ts
@@ -1,4 +1,7 @@
-import { calculateGlobalPos } from "oxalis/model/accessors/view_mode_accessor";
+import {
+  calculateGlobalPos,
+  calculateMaybeGlobalPos,
+} from "oxalis/model/accessors/view_mode_accessor";
 import _ from "lodash";
 import type { OrthoView, Point2, Vector3, BoundingBoxType } from "oxalis/constants";
 import Store from "oxalis/store";
@@ -140,7 +143,10 @@ export function getClosestHoveredBoundingBox(
   plane: OrthoView,
 ): [SelectedEdge, SelectedEdge | null | undefined] | null {
   const state = Store.getState();
-  const globalPosition = calculateGlobalPos(state, pos, plane);
+  const globalPosition = calculateMaybeGlobalPos(state, pos, plane);
+
+  if (globalPosition == null) return null;
+
   const { userBoundingBoxes } = getSomeTracing(state.tracing);
   const indices = Dimension.getIndices(plane);
   const planeRatio = getBaseVoxelFactors(state.dataset.dataSource.scale);
diff --git a/frontend/javascripts/oxalis/controller/combinations/skeleton_handlers.ts b/frontend/javascripts/oxalis/controller/combinations/skeleton_handlers.ts
index 048859c880a..2e4e81901d6 100644
--- a/frontend/javascripts/oxalis/controller/combinations/skeleton_handlers.ts
+++ b/frontend/javascripts/oxalis/controller/combinations/skeleton_handlers.ts
@@ -17,7 +17,11 @@ import {
   getNodeAndTree,
   getNodeAndTreeOrNull,
 } from "oxalis/model/accessors/skeletontracing_accessor";
-import { getInputCatcherRect, calculateGlobalPos } from "oxalis/model/accessors/view_mode_accessor";
+import {
+  getInputCatcherRect,
+  calculateGlobalPos,
+  calculateMaybeGlobalPos,
+} from "oxalis/model/accessors/view_mode_accessor";
 import {
   getPosition,
   getRotationOrtho,
@@ -134,7 +138,9 @@ export function handleOpenContextMenu(
 
   const nodeId = maybeGetNodeIdFromPosition(planeView, position, plane, isTouch);
   const state = Store.getState();
-  const globalPosition = calculateGlobalPos(state, position);
+  // Use calculateMaybeGlobalPos instead of calculateGlobalPos, since calculateGlobalPos
+  // only works for the data viewports, but this function is also called for the 3d viewport.
+  const globalPosition = calculateMaybeGlobalPos(state, position);
   const hoveredEdgesInfo = getClosestHoveredBoundingBox(position, plane);
   const clickedBoundingBoxId = hoveredEdgesInfo != null ? hoveredEdgesInfo[0].boxId : null;
   showNodeContextMenuAt(
diff --git a/frontend/javascripts/oxalis/model/accessors/view_mode_accessor.ts b/frontend/javascripts/oxalis/model/accessors/view_mode_accessor.ts
index 6ad8977cb9b..2e78f264cff 100644
--- a/frontend/javascripts/oxalis/model/accessors/view_mode_accessor.ts
+++ b/frontend/javascripts/oxalis/model/accessors/view_mode_accessor.ts
@@ -103,7 +103,7 @@ function _calculateMaybeGlobalPos(
   clickPos: Point2,
   planeId?: OrthoView | null | undefined,
 ): Vector3 | null | undefined {
-  let position;
+  let position: Vector3;
   planeId = planeId || state.viewModeData.plane.activeViewport;
   const curGlobalPos = getPosition(state.flycam);
   const planeRatio = getBaseVoxelFactors(state.dataset.dataSource.scale);
@@ -142,7 +142,6 @@ function _calculateMaybeGlobalPos(
       return null;
   }
 
-  // @ts-expect-error ts-migrate(2322) FIXME: Type 'number[]' is not assignable to type 'Vector3... Remove this comment to see the full error message
   return position;
 }
 
diff --git a/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts b/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts
index 617c4265d6d..48870a12442 100644
--- a/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts
+++ b/frontend/javascripts/oxalis/model/actions/segmentation_actions.ts
@@ -5,6 +5,7 @@ export type AdHocIsosurfaceInfo = {
   mappingType: MappingType | null | undefined;
   useDataStore?: boolean | null | undefined;
   passive?: boolean | null | undefined;
+  preferredQuality?: number | null | undefined;
 };
 export type LoadAdHocMeshAction = {
   type: "LOAD_AD_HOC_MESH_ACTION";
diff --git a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
index ce0297173b5..771b3266705 100644
--- a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
@@ -207,14 +207,20 @@ function* getIsosurfaceExtraInfo(
   };
 }
 
-function* getInfoForIsosurfaceLoading(layer: DataLayer): Saga<{
+function* getInfoForIsosurfaceLoading(
+  layer: DataLayer,
+  isosurfaceExtraInfo: AdHocIsosurfaceInfo,
+): Saga<{
   zoomStep: number;
   resolutionInfo: ResolutionInfo;
 }> {
   const resolutionInfo = getResolutionInfo(layer.resolutions);
-  const preferredZoomStep = yield* select(
-    (state) => state.temporaryConfiguration.preferredQualityForMeshAdHocComputation,
-  );
+  const preferredZoomStep =
+    isosurfaceExtraInfo.preferredQuality != null
+      ? isosurfaceExtraInfo.preferredQuality
+      : yield* select(
+          (state) => state.temporaryConfiguration.preferredQualityForMeshAdHocComputation,
+        );
   const zoomStep = resolutionInfo.getClosestExistingIndex(preferredZoomStep);
   return {
     zoomStep,
@@ -229,7 +235,11 @@ function* loadIsosurfaceForSegmentId(
   removeExistingIsosurface: boolean,
   layer: DataLayer,
 ): Saga<void> {
-  const { zoomStep, resolutionInfo } = yield* call(getInfoForIsosurfaceLoading, layer);
+  const { zoomStep, resolutionInfo } = yield* call(
+    getInfoForIsosurfaceLoading,
+    layer,
+    isosurfaceExtraInfo,
+  );
   batchCounterPerSegment[segmentId] = 0;
   // If a REMOVE_ISOSURFACE action is dispatched and consumed
   // here before loadIsosurfaceWithNeighbors is finished, the latter saga
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 3bedbd9f651..0b6c65a3b81 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -31,20 +31,19 @@ import {
   getActiveSegmentationTracing,
 } from "oxalis/model/accessors/volumetracing_accessor";
 import {
+  getLayerByName,
   getMappingInfo,
   getResolutionInfo,
   ResolutionInfo,
 } from "oxalis/model/accessors/dataset_accessor";
 import { makeMappingEditable } from "admin/admin_rest_api";
-import {
-  setMappingNameAction,
-  updateTemporarySettingAction,
-} from "oxalis/model/actions/settings_actions";
+import { setMappingNameAction } from "oxalis/model/actions/settings_actions";
 import { getSegmentIdForPosition } from "oxalis/controller/combinations/volume_handlers";
 import { loadAdHocMeshAction } from "oxalis/model/actions/segmentation_actions";
 import { V3 } from "libs/mjs";
 import { removeIsosurfaceAction } from "oxalis/model/actions/annotation_actions";
 import { loadAgglomerateSkeletonWithId } from "oxalis/model/sagas/skeletontracing_saga";
+import { getConstructorForElementClass } from "oxalis/model/bucket_data_handling/bucket";
 
 export default function* proofreadMapping(): Saga<any> {
   yield* take("INITIALIZE_SKELETONTRACING");
@@ -83,9 +82,6 @@ function* loadCoarseAdHocMesh(
   segmentId: number,
   position: Vector3,
 ): Saga<void> {
-  // Since it is currently not possible to specify the quality in the loadAdHocMeshAction
-  // this method sets the preferredQualityForMeshAdHocComputation, then requests the mesh
-  // and then resets preferredQualityForMeshAdHocComputation to the previous value.
   const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
   if (volumeTracing == null) return;
 
@@ -94,31 +90,19 @@ function* loadCoarseAdHocMesh(
   );
   const { mappingName, mappingType } = mappingInfo;
 
-  // Load the whole agglomerate mesh in a coarse resolution for performance reasons
-  const oldPreferredQuality = yield* select(
-    (state) => state.temporaryConfiguration.preferredQualityForMeshAdHocComputation,
-  );
-
-  const coarseResolutionIndex = resolutionInfo.getClosestExistingIndex(
-    proofreadCoarseResolutionIndex(),
-  );
-  yield* put(
-    updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", coarseResolutionIndex),
-  );
-
   // Use the data store if the mapping is not editable yet. If it, is request the mesh from the tracing store.
   const useDataStore = !volumeTracing.mappingIsEditable;
+  // Load the whole agglomerate mesh in a coarse resolution for performance reasons
+  const preferredQuality = proofreadCoarseResolutionIndex();
   yield* put(
     loadAdHocMeshAction(segmentId, position, {
       mappingName,
       mappingType,
       passive: true,
       useDataStore,
+      preferredQuality,
     }),
   );
-  yield* put(
-    updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", oldPreferredQuality),
-  );
 }
 
 function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
@@ -185,8 +169,12 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
       ),
     ),
   );
-  // TODO HACK: This only works for uint32 segmentations
-  const segmentIdsInProximity = segmentIdsArrayBuffers.map((buffer) => new Uint32Array(buffer)[0]);
+
+  const { elementClass } = yield* select((state) => getLayerByName(state.dataset, layerName));
+  const [TypedArrayClass] = getConstructorForElementClass(elementClass);
+  const segmentIdsInProximity = segmentIdsArrayBuffers.map(
+    (buffer) => new TypedArrayClass(buffer)[0],
+  );
 
   if (oldSegmentIdsInProximity != null) {
     const segmentIdsInProximitySet = new Set(segmentIdsInProximity);
@@ -209,17 +197,8 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
     mappingName: null,
     mappingType: null,
     useDataStore: true,
+    preferredQuality: proofreadFineResolutionIndex(),
   };
-  const oldPreferredQuality = yield* select(
-    (state) => state.temporaryConfiguration.preferredQualityForMeshAdHocComputation,
-  );
-
-  const fineResolutionIndex = resolutionInfo.getClosestExistingIndex(
-    proofreadFineResolutionIndex(),
-  );
-  yield* put(
-    updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", fineResolutionIndex),
-  );
   yield* all(
     segmentIdsInProximity.map((nodeSegmentId, index) =>
       put(
@@ -232,9 +211,6 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
       ),
     ),
   );
-  yield* put(
-    updateTemporarySettingAction("preferredQualityForMeshAdHocComputation", oldPreferredQuality),
-  );
 }
 
 function* createEditableMapping(): Saga<void> {
diff --git a/frontend/javascripts/oxalis/model/sagas/root_saga.ts b/frontend/javascripts/oxalis/model/sagas/root_saga.ts
index 5747590bfd1..ae1d19b1a6a 100644
--- a/frontend/javascripts/oxalis/model/sagas/root_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/root_saga.ts
@@ -17,6 +17,8 @@ import loadHistogramDataSaga from "oxalis/model/sagas/load_histogram_data_saga";
 import listenToClipHistogramSaga from "oxalis/model/sagas/clip_histogram_saga";
 import MappingSaga from "oxalis/model/sagas/mapping_saga";
 import ProofreadSaga from "oxalis/model/sagas/proofread_saga";
+import { listenForWkReady } from "oxalis/model/sagas/wk_ready_saga";
+
 let rootSagaCrashed = false;
 export default function* rootSaga(): Saga<void> {
   while (true) {
@@ -33,6 +35,7 @@ export function hasRootSagaCrashed() {
 function* restartableSaga(): Saga<void> {
   try {
     yield* all([
+      call(listenForWkReady),
       call(warnAboutMagRestriction),
       call(SettingsSaga),
       ...SkeletontracingSagas.map((saga) => call(saga)),
diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
index 5a4956e4eb1..80bebb42167 100644
--- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
@@ -103,7 +103,8 @@ import compactUpdateActions from "oxalis/model/helpers/compaction/compact_update
 import createProgressCallback from "libs/progress_callback";
 import messages from "messages";
 import window, { alert, document, location } from "libs/window";
-import { enforceSkeletonTracing } from "../accessors/skeletontracing_accessor";
+import { enforceSkeletonTracing } from "oxalis/model/accessors/skeletontracing_accessor";
+import { ensureWkReady } from "oxalis/model/sagas/wk_ready_saga";
 
 // This function is needed so that Flow is satisfied
 // with how a mere promise is awaited within a saga.
@@ -714,12 +715,8 @@ function* applyAndGetRevertingVolumeBatch(
   };
 }
 
-export function* pushSaveQueueAsync(
-  saveQueueType: SaveQueueType,
-  tracingId: string,
-  isWkReady: boolean = false,
-): Saga<void> {
-  if (!isWkReady) yield* take("WK_READY");
+export function* pushSaveQueueAsync(saveQueueType: SaveQueueType, tracingId: string): Saga<void> {
+  yield* call(ensureWkReady);
 
   yield* put(setLastSaveTimestampAction(saveQueueType, tracingId));
   let loopCounter = 0;
@@ -991,13 +988,7 @@ export function performDiffTracing(
   return actions;
 }
 
-let isWkReady = false;
-
-function setWkReady() {
-  isWkReady = true;
-}
 export function* saveTracingAsync(): Saga<void> {
-  yield* takeEvery("WK_READY", setWkReady);
   yield* takeEvery("INITIALIZE_SKELETONTRACING", setupSavingForTracingType);
   yield* takeEvery("INITIALIZE_VOLUMETRACING", setupSavingForTracingType);
   yield* takeEvery("INITIALIZE_EDITABLE_MAPPING", setupSavingForEditableMapping);
@@ -1009,7 +1000,7 @@ export function* setupSavingForEditableMapping(
   // No diffing needs to be done for editable mappings as the saga pushes update actions
   // to the respective save queues, itself
   const volumeTracingId = initializeAction.mapping.tracingId;
-  yield* fork(pushSaveQueueAsync, "mapping", volumeTracingId, isWkReady);
+  yield* fork(pushSaveQueueAsync, "mapping", volumeTracingId);
 }
 export function* setupSavingForTracingType(
   initializeAction: InitializeSkeletonTracingAction | InitializeVolumeTracingAction,
diff --git a/frontend/javascripts/oxalis/model/sagas/wk_ready_saga.ts b/frontend/javascripts/oxalis/model/sagas/wk_ready_saga.ts
new file mode 100644
index 00000000000..b38ddfc02ac
--- /dev/null
+++ b/frontend/javascripts/oxalis/model/sagas/wk_ready_saga.ts
@@ -0,0 +1,20 @@
+import type { Saga } from "oxalis/model/sagas/effect-generators";
+import { take, takeEvery } from "typed-redux-saga";
+
+let isWkReady = false;
+
+function setWkReady() {
+  isWkReady = true;
+}
+export function* listenForWkReady(): Saga<void> {
+  yield* takeEvery("WK_READY", setWkReady);
+}
+
+export function* ensureWkReady(): Saga<void> {
+  // This saga is useful for sagas that might be instantiated before or after
+  // the WK_READY action was dispatched. If the action was dispatched
+  // before, this saga immediately returns, otherwise it waits
+  // until the action is dispatched.
+  if (isWkReady) return;
+  yield* take("WK_READY");
+}
diff --git a/frontend/javascripts/oxalis/view/context_menu.tsx b/frontend/javascripts/oxalis/view/context_menu.tsx
index 9275d1dff1f..da6810f738f 100644
--- a/frontend/javascripts/oxalis/view/context_menu.tsx
+++ b/frontend/javascripts/oxalis/view/context_menu.tsx
@@ -22,7 +22,6 @@ import {
   AnnotationTool,
   Vector3,
   OrthoView,
-  OrthoViews,
   AnnotationToolEnum,
   VolumeTools,
 } from "oxalis/constants";
@@ -76,7 +75,6 @@ import {
 } from "oxalis/model/actions/volumetracing_actions";
 import { roundTo, hexToRgb, rgbToHex } from "libs/utils";
 import { setWaypoint } from "oxalis/controller/combinations/skeleton_handlers";
-import Model from "oxalis/model";
 import Shortcut from "libs/shortcut_component";
 import Toast from "libs/toast";
 import api from "oxalis/api/internal_api";
@@ -89,7 +87,7 @@ type OwnProps = {
   contextMenuPosition: [number, number] | null | undefined;
   maybeClickedNodeId: number | null | undefined;
   clickedBoundingBoxId: number | null | undefined;
-  globalPosition: Vector3;
+  globalPosition: Vector3 | null | undefined;
   maybeViewport: OrthoView | null | undefined;
   hideContextMenu: () => void;
 };
@@ -184,27 +182,6 @@ function measureAndShowFullTreeLength(treeId: number, treeName: string) {
   });
 }
 
-function getMaybeHoveredCellMenuItem(globalPosition: Vector3) {
-  const hoveredCellInfo = Model.getHoveredCellId(globalPosition);
-
-  if (!hoveredCellInfo) {
-    return null;
-  }
-
-  const cellIdAsString = hoveredCellInfo.isMapped
-    ? `${hoveredCellInfo.id} (mapped)`
-    : hoveredCellInfo.id;
-  return (
-    <div key="hovered-info" className="node-context-menu-item">
-      <i className="fas fa-ruler" /> Hovered Segment: {cellIdAsString}
-      {copyIconWithTooltip(
-        hoveredCellInfo.id,
-        `Copy ${hoveredCellInfo.isMapped ? "mapped" : ""} segment id`,
-      )}
-    </div>
-  );
-}
-
 function positionToString(pos: Vector3): string {
   return pos.map((value) => roundTo(value, 2)).join(", ");
 }
@@ -485,6 +462,8 @@ function getBoundingBoxMenuOptions({
   deleteBoundingBox,
   allowUpdate,
 }: NoNodeContextMenuProps) {
+  if (globalPosition == null) return [];
+
   const isBoundingBoxToolActive = activeTool === AnnotationToolEnum.BOUNDING_BOX;
   const newBoundingBoxMenuItem = (
     <Menu.Item
@@ -648,7 +627,7 @@ function NoNodeContextMenuOptions(props: NoNodeContextMenuProps) {
   }, [visibleSegmentationLayer, dataset]);
 
   const loadPrecomputedMesh = async () => {
-    if (!currentMeshFile || !visibleSegmentationLayer) return;
+    if (!currentMeshFile || !visibleSegmentationLayer || globalPosition == null) return;
     // Ensure that the segment ID is loaded, since a mapping might have been activated
     // shortly before
     const segmentId = await getSegmentIdForPositionAsync(globalPosition);
@@ -662,7 +641,7 @@ function NoNodeContextMenuOptions(props: NoNodeContextMenuProps) {
   };
 
   const computeMeshAdHoc = () => {
-    if (!visibleSegmentationLayer) {
+    if (!visibleSegmentationLayer || globalPosition == null) {
       return;
     }
 
@@ -679,7 +658,7 @@ function NoNodeContextMenuOptions(props: NoNodeContextMenuProps) {
   const isVolumeBasedToolActive = VolumeTools.includes(activeTool);
   const isBoundingBoxToolActive = activeTool === AnnotationToolEnum.BOUNDING_BOX;
   const skeletonActions =
-    skeletonTracing != null && allowUpdate
+    skeletonTracing != null && globalPosition != null && allowUpdate
       ? [
           <Menu.Item key="create-node" onClick={() => setWaypoint(globalPosition, viewport, false)}>
             Create Node here
@@ -711,29 +690,31 @@ function NoNodeContextMenuOptions(props: NoNodeContextMenuProps) {
       : [];
   const segmentationLayerName =
     visibleSegmentationLayer != null ? visibleSegmentationLayer.name : null;
-  const connectomeFileMappingName =
-    currentConnectomeFile != null ? currentConnectomeFile.mappingName : undefined;
-  const loadSynapsesItem = (
-    <MenuItemWithMappingActivationConfirmation
-      // @ts-expect-error ts-migrate(2322) FIXME: Type '{ children: string | Element; className: str... Remove this comment to see the full error message
-      className="node-context-menu-item"
-      key="load-synapses"
-      disabled={!isConnectomeMappingEnabled.value}
-      onClick={() => loadSynapsesOfAgglomerateAtPosition(globalPosition)}
-      mappingName={connectomeFileMappingName}
-      descriptor="connectome file"
-      layerName={segmentationLayerName}
-      mappingInfo={mappingInfo}
-    >
-      {isConnectomeMappingEnabled.value ? (
-        "Import Agglomerate and Synapses"
-      ) : (
-        <Tooltip title={isConnectomeMappingEnabled.reason}>Import Agglomerate and Synapses</Tooltip>
-      )}
-    </MenuItemWithMappingActivationConfirmation>
-  );
 
-  if (visibleSegmentationLayer != null) {
+  if (visibleSegmentationLayer != null && globalPosition != null) {
+    const connectomeFileMappingName =
+      currentConnectomeFile != null ? currentConnectomeFile.mappingName : undefined;
+    const loadSynapsesItem = (
+      <MenuItemWithMappingActivationConfirmation
+        // @ts-expect-error ts-migrate(2322) FIXME: Type '{ children: string | Element; className: str... Remove this comment to see the full error message
+        className="node-context-menu-item"
+        key="load-synapses"
+        disabled={!isConnectomeMappingEnabled.value}
+        onClick={() => loadSynapsesOfAgglomerateAtPosition(globalPosition)}
+        mappingName={connectomeFileMappingName}
+        descriptor="connectome file"
+        layerName={segmentationLayerName}
+        mappingInfo={mappingInfo}
+      >
+        {isConnectomeMappingEnabled.value ? (
+          "Import Agglomerate and Synapses"
+        ) : (
+          <Tooltip title={isConnectomeMappingEnabled.reason}>
+            Import Agglomerate and Synapses
+          </Tooltip>
+        )}
+      </MenuItemWithMappingActivationConfirmation>
+    );
     // This action doesn't need a skeleton tracing but is conceptually related to the "Import Agglomerate Skeleton" action
     skeletonActions.push(loadSynapsesItem);
   }
@@ -759,7 +740,7 @@ function NoNodeContextMenuOptions(props: NoNodeContextMenuProps) {
     </Menu.Item>
   );
   const nonSkeletonActions =
-    volumeTracing != null
+    volumeTracing != null && globalPosition != null
       ? [
           // Segment 0 cannot/shouldn't be made active (as this
           // would be an eraser effectively).
@@ -794,7 +775,6 @@ function NoNodeContextMenuOptions(props: NoNodeContextMenuProps) {
   }
 
   const isSkeletonToolActive = activeTool === AnnotationToolEnum.SKELETON;
-  const isTdViewport = viewport === OrthoViews.TDView;
   let allActions: Array<JSX.Element | null> = [];
 
   if (isSkeletonToolActive) {
@@ -802,12 +782,14 @@ function NoNodeContextMenuOptions(props: NoNodeContextMenuProps) {
     allActions = skeletonActions.concat(nonSkeletonActions).concat(boundingBoxActions);
   } else if (isBoundingBoxToolActive) {
     allActions = boundingBoxActions.concat(nonSkeletonActions).concat(skeletonActions);
-  } else if (!isTdViewport) {
+  } else {
     allActions = nonSkeletonActions.concat(skeletonActions).concat(boundingBoxActions);
   }
 
   const empty = (
-    <Empty description="&nbsp;No actions available&nbsp;" image={Empty.PRESENTED_IMAGE_SIMPLE} />
+    <Menu.Item key="empty">
+      <Empty description="No actions available" image={Empty.PRESENTED_IMAGE_SIMPLE} />
+    </Menu.Item>
   );
 
   return (
@@ -818,9 +800,7 @@ function NoNodeContextMenuOptions(props: NoNodeContextMenuProps) {
       }}
       mode="vertical"
     >
-      {allActions}
-      {/* In the TD viewport the info rows are not usable since the click position cannot be computed */}
-      {isTdViewport ? empty : infoRows}
+      {allActions.length + infoRows.length > 0 ? [...allActions, ...infoRows] : [empty]}
     </Menu>
   );
 }
@@ -896,7 +876,6 @@ function ContextMenuInner(propsWithInputRef: PropsWithRef) {
   const { inputRef, ...props } = propsWithInputRef;
   const {
     skeletonTracing,
-    activeTool,
     maybeClickedNodeId,
     contextMenuPosition,
     hideContextMenu,
@@ -928,7 +907,7 @@ function ContextMenuInner(propsWithInputRef: PropsWithRef) {
         ? getNodeAndTree(skeletonTracing, activeNodeId, activeTreeId).get()[1]
         : null;
     const distanceToSelection =
-      activeNode != null
+      activeNode != null && positionToMeasureDistanceTo != null
         ? [
             formatNumberToLength(
               V3.scaledDist(activeNode.position, positionToMeasureDistanceTo, datasetScale),
@@ -939,7 +918,8 @@ function ContextMenuInner(propsWithInputRef: PropsWithRef) {
     const nodePositionAsString =
       // @ts-expect-error ts-migrate(2339) FIXME: Property 'position' does not exist on type 'never'... Remove this comment to see the full error message
       nodeContextMenuNode != null ? positionToString(nodeContextMenuNode.position) : "";
-    const segmentIdAtPosition = getSegmentIdForPosition(globalPosition);
+    const segmentIdAtPosition =
+      globalPosition != null ? getSegmentIdForPosition(globalPosition) : 0;
     const infoRows = [];
 
     if (maybeClickedNodeId != null && nodeContextMenuTree != null) {
@@ -958,7 +938,7 @@ function ContextMenuInner(propsWithInputRef: PropsWithRef) {
           {copyIconWithTooltip(nodePositionAsString, "Copy node position")}
         </div>,
       );
-    } else {
+    } else if (globalPosition != null) {
       const positionAsString = positionToString(globalPosition);
       infoRows.push(
         <div key="positionInfo" className="node-context-menu-item">
@@ -988,21 +968,18 @@ function ContextMenuInner(propsWithInputRef: PropsWithRef) {
       );
     }
 
-    const maybeHoveredCellMenuItem = getMaybeHoveredCellMenuItem(globalPosition);
-
-    if (!maybeHoveredCellMenuItem) {
-      infoRows.push(maybeHoveredCellMenuItem);
+    if (infoRows.length > 0) {
+      infoRows.unshift(
+        <Menu.Divider
+          key="divider"
+          className="hide-if-first hide-if-last"
+          style={{
+            margin: "4px 0px",
+          }}
+        />,
+      );
     }
 
-    infoRows.unshift(
-      <Menu.Divider
-        key="divider"
-        className="hide-if-first hide-if-last"
-        style={{
-          margin: "4px 0px",
-        }}
-      />,
-    );
     // It's important to not use <NodeContextMenuOptions ...>
     // or <NoNodeContextMenuOptions ... />
     // for the following two expressions, since this breaks
@@ -1018,8 +995,6 @@ function ContextMenuInner(propsWithInputRef: PropsWithRef) {
             ...props,
           })
         : NoNodeContextMenuOptions({
-            // @ts-expect-error ts-migrate(2783) FIXME: 'activeTool' is specified more than once, so this ... Remove this comment to see the full error message
-            activeTool,
             segmentIdAtPosition,
             infoRows,
             viewport: maybeViewport,
diff --git a/frontend/javascripts/oxalis/view/layouting/tracing_layout_view.tsx b/frontend/javascripts/oxalis/view/layouting/tracing_layout_view.tsx
index ccce87a2b61..50121cb25d7 100644
--- a/frontend/javascripts/oxalis/view/layouting/tracing_layout_view.tsx
+++ b/frontend/javascripts/oxalis/view/layouting/tracing_layout_view.tsx
@@ -76,7 +76,7 @@ type State = {
   contextMenuPosition: [number, number] | null | undefined;
   clickedNodeId: number | null | undefined;
   clickedBoundingBoxId: number | null | undefined;
-  contextMenuGlobalPosition: Vector3;
+  contextMenuGlobalPosition: Vector3 | null | undefined;
   contextMenuViewport: OrthoView | null | undefined;
   model: Record<string, any>;
 };
@@ -107,7 +107,7 @@ class TracingLayoutView extends React.PureComponent<PropsWithRouter, State> {
       contextMenuPosition: null,
       clickedNodeId: null,
       clickedBoundingBoxId: null,
-      contextMenuGlobalPosition: [0, 0, 0],
+      contextMenuGlobalPosition: null,
       contextMenuViewport: null,
       model: layout,
     };
@@ -160,7 +160,7 @@ class TracingLayoutView extends React.PureComponent<PropsWithRouter, State> {
     yPos: number,
     nodeId: number | null | undefined,
     boundingBoxId: number | null | undefined,
-    globalPosition: Vector3,
+    globalPosition: Vector3 | null | undefined,
     viewport: OrthoView,
   ) => {
     // On Windows the right click to open the context menu is also triggered for the overlay
@@ -185,7 +185,7 @@ class TracingLayoutView extends React.PureComponent<PropsWithRouter, State> {
       contextMenuPosition: null,
       clickedNodeId: null,
       clickedBoundingBoxId: null,
-      contextMenuGlobalPosition: [0, 0, 0],
+      contextMenuGlobalPosition: null,
       contextMenuViewport: null,
     });
   };

From ac0770fd5b344dcf946c6e822866f033f28157dc Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 14 Jun 2022 15:37:19 +0200
Subject: [PATCH 102/122] disable loading of fine over segmentation meshes in
 proximity by default

---
 .../oxalis/model/sagas/proofread_saga.ts      | 54 +++++++++----------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 0b6c65a3b81..066594a3e8b 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -34,7 +34,6 @@ import {
   getLayerByName,
   getMappingInfo,
   getResolutionInfo,
-  ResolutionInfo,
 } from "oxalis/model/accessors/dataset_accessor";
 import { makeMappingEditable } from "admin/admin_rest_api";
 import { setMappingNameAction } from "oxalis/model/actions/settings_actions";
@@ -44,6 +43,8 @@ import { V3 } from "libs/mjs";
 import { removeIsosurfaceAction } from "oxalis/model/actions/annotation_actions";
 import { loadAgglomerateSkeletonWithId } from "oxalis/model/sagas/skeletontracing_saga";
 import { getConstructorForElementClass } from "oxalis/model/bucket_data_handling/bucket";
+import { Tree } from "oxalis/store";
+import { APISegmentationLayer } from "types/api_flow_types";
 
 export default function* proofreadMapping(): Saga<any> {
   yield* take("INITIALIZE_SKELETONTRACING");
@@ -70,18 +71,15 @@ function proofreadUsingMeshes(): boolean {
   // @ts-ignore
   return window.__proofreadUsingMeshes != null ? window.__proofreadUsingMeshes : true;
 }
+// The default of 0 effectively disables the loading of the high-quality meshes of
+// the oversegmentation by default
 function proofreadSegmentProximityNm(): number {
   // @ts-ignore
-  return window.__proofreadProximityNm != null ? window.__proofreadProximityNm : 2000;
+  return window.__proofreadProximityNm != null ? window.__proofreadProximityNm : 0;
 }
 let oldSegmentIdsInProximity: number[] | null = null;
 
-function* loadCoarseAdHocMesh(
-  layerName: string,
-  resolutionInfo: ResolutionInfo,
-  segmentId: number,
-  position: Vector3,
-): Saga<void> {
+function* loadCoarseAdHocMesh(layerName: string, segmentId: number, position: Vector3): Saga<void> {
   const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
   if (volumeTracing == null) return;
 
@@ -113,8 +111,6 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
   const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
   if (volumeTracing == null) return;
 
-  const resolutionInfo = getResolutionInfo(volumeTracingLayer.resolutions);
-
   const layerName = volumeTracingLayer.tracingId;
   const segmentId = getSegmentIdForPosition(position);
   const { mappingName } = volumeTracing;
@@ -132,19 +128,29 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
 
   /* Load a coarse ad hoc mesh of the agglomerate at the click position */
 
-  yield* call(loadCoarseAdHocMesh, layerName, resolutionInfo, segmentId, position);
+  yield* call(loadCoarseAdHocMesh, layerName, segmentId, position);
 
   if (treeName == null) return;
 
   const skeletonTracing = yield* select((state) => enforceSkeletonTracing(state.tracing));
   const { trees } = skeletonTracing;
   const tree = findTreeByName(trees, treeName).getOrElse(null);
-
   if (tree == null) return;
 
+  yield* call(loadFineAdHocMeshesInProximity, layerName, volumeTracingLayer, tree, position);
+}
+
+function* loadFineAdHocMeshesInProximity(
+  layerName: string,
+  volumeTracingLayer: APISegmentationLayer,
+  tree: Tree,
+  position: Vector3,
+): Saga<void> {
   /* Find all segments (nodes) of the agglomerate skeleton within proofreadSegmentProximityNm (graph distance)
      and request the segment IDs in the oversegmentation at the node positions */
 
+  if (proofreadSegmentProximityNm() <= 0) return;
+
   const nodePositions = tree.nodes.map((node) => node.position);
   const proximityDistanceSquared = proofreadSegmentProximityNm() ** 2;
   const scale = yield* select((state) => state.dataset.dataSource.scale);
@@ -153,6 +159,7 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
     (nodePosition) =>
       V3.scaledSquaredDist(nodePosition, position, scale) <= proximityDistanceSquared,
   );
+  const resolutionInfo = getResolutionInfo(volumeTracingLayer.resolutions);
   const mag = resolutionInfo.getLowestResolution();
 
   const fallbackLayerName = volumeTracingLayer.fallbackLayer;
@@ -259,7 +266,12 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
   }
 
   if (!volumeTracing.mappingIsEditable) {
-    yield* call(createEditableMapping);
+    try {
+      yield* call(createEditableMapping);
+    } catch (e) {
+      console.error(e);
+      return;
+    }
   }
 
   /* Find out the agglomerate IDs at the two node positions */
@@ -407,21 +419,9 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
       yield* put(removeIsosurfaceAction(layerName, targetNodeAgglomerateId));
     }
 
-    yield* call(
-      loadCoarseAdHocMesh,
-      layerName,
-      resolutionInfo,
-      newSourceNodeAgglomerateId,
-      sourceNodePosition,
-    );
+    yield* call(loadCoarseAdHocMesh, layerName, newSourceNodeAgglomerateId, sourceNodePosition);
     if (newTargetNodeAgglomerateId !== newSourceNodeAgglomerateId) {
-      yield* call(
-        loadCoarseAdHocMesh,
-        layerName,
-        resolutionInfo,
-        newTargetNodeAgglomerateId,
-        targetNodePosition,
-      );
+      yield* call(loadCoarseAdHocMesh, layerName, newTargetNodeAgglomerateId, targetNodePosition);
     }
   }
 }

From d6d29f683b891599a9898f6ac41cb3fa318f99df Mon Sep 17 00:00:00 2001
From: Florian M <florian@scm.io>
Date: Tue, 14 Jun 2022 16:12:39 +0200
Subject: [PATCH 103/122] avoid requesting mapping when loading brushed volume
 data

---
 .../tracingstore/tracings/volume/VolumeTracingService.scala    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
index 373fcc9fd88..95a2f32c4ac 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala
@@ -247,7 +247,8 @@ class VolumeTracingService @Inject()(
     for {
       isTemporaryTracing <- isTemporaryTracing(tracingId)
       dataLayer = volumeTracingLayer(tracingId, tracing, isTemporaryTracing)
-      requests = dataRequests.map(r => DataServiceDataRequest(null, dataLayer, None, r.cuboid(dataLayer), r.settings))
+      requests = dataRequests.map(r =>
+        DataServiceDataRequest(null, dataLayer, None, r.cuboid(dataLayer), r.settings.copy(appliedAgglomerate = None)))
       data <- binaryDataService.handleDataRequests(requests)
     } yield data
 

From 06392dca2b139393f8d03d6055f95b3d921bf8b9 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 14 Jun 2022 16:53:45 +0200
Subject: [PATCH 104/122] PR feedback 3/x, ensure agglomerate mapping is active
 when proofreading, fix tests

---
 .../oxalis/controller/scene_controller.ts     |  2 +-
 .../model/accessors/volumetracing_accessor.ts |  9 +++++
 .../oxalis/model/sagas/isosurface_saga.ts     |  4 +-
 .../oxalis/model/sagas/proofread_saga.ts      | 39 +++++++++++--------
 4 files changed, 35 insertions(+), 19 deletions(-)

diff --git a/frontend/javascripts/oxalis/controller/scene_controller.ts b/frontend/javascripts/oxalis/controller/scene_controller.ts
index e500c37d7ca..1d25100815f 100644
--- a/frontend/javascripts/oxalis/controller/scene_controller.ts
+++ b/frontend/javascripts/oxalis/controller/scene_controller.ts
@@ -213,7 +213,7 @@ class SceneController {
     tweenAnimation
       .to(
         {
-          opacity: passive ? 0.2 : 0.6,
+          opacity: passive ? 0.4 : 0.9,
         },
         500,
       )
diff --git a/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts b/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts
index 54a42bb942b..abf813d1d3b 100644
--- a/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts
+++ b/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts
@@ -420,6 +420,15 @@ export function hasEditableMapping(
   state: OxalisState,
   layerName?: string | null | undefined,
 ): boolean {
+  if (layerName != null) {
+    // This needs to be checked before calling getRequestedOrDefaultSegmentationTracingLayer,
+    // as the function will throw an error if layerName is given but not a tracing layer
+    const layer = getSegmentationLayerByName(state.dataset, layerName);
+    const tracing = getTracingForSegmentationLayer(state, layer);
+
+    if (tracing == null) return false;
+  }
+
   const volumeTracing = getRequestedOrDefaultSegmentationTracingLayer(state, layerName);
 
   if (volumeTracing == null) return false;
diff --git a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
index 771b3266705..cca0ba7b38c 100644
--- a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
@@ -375,11 +375,11 @@ function* maybeLoadIsosurface(
   }`;
   const tracingStoreUrl = `${tracingStoreHost}/tracings/volume/${layer.name}`;
   const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
-  // Fetch from datastore if no volumetracing exists
+  // Fetch from datastore if no volumetracing exists or if the tracing has a fallback layer.
   const useDataStore =
     isosurfaceExtraInfo.useDataStore != null
       ? isosurfaceExtraInfo.useDataStore
-      : volumeTracing == null;
+      : volumeTracing == null || volumeTracing.fallbackLayer != null;
   const mag = resolutionInfo.getResolutionByIndexOrThrow(zoomStep);
 
   if (isInitialRequest) {
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 066594a3e8b..7835e4a45d8 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -112,16 +112,16 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
   if (volumeTracing == null) return;
 
   const layerName = volumeTracingLayer.tracingId;
-  const segmentId = getSegmentIdForPosition(position);
-  const { mappingName } = volumeTracing;
+  const isHdf5MappingEnabled = yield* call(ensureHdf5MappingIsEnabled, layerName);
+  if (!isHdf5MappingEnabled || volumeTracing.mappingName == null) return;
 
-  if (mappingName == null) return;
+  const segmentId = getSegmentIdForPosition(position);
 
   /* Load agglomerate skeleton of the agglomerate at the click position */
 
   const treeName = yield* call(
     loadAgglomerateSkeletonWithId,
-    loadAgglomerateSkeletonAction(layerName, mappingName, segmentId),
+    loadAgglomerateSkeletonAction(layerName, volumeTracing.mappingName, segmentId),
   );
 
   if (!proofreadUsingMeshes()) return;
@@ -238,6 +238,23 @@ function* createEditableMapping(): Saga<void> {
   yield* put(initializeEditableMappingAction(serverEditableMapping));
 }
 
+function* ensureHdf5MappingIsEnabled(layerName: string): Saga<boolean> {
+  const mappingInfo = yield* select((state) =>
+    getMappingInfo(state.temporaryConfiguration.activeMappingByLayer, layerName),
+  );
+  const { mappingName, mappingType, mappingStatus } = mappingInfo;
+  if (
+    mappingName == null ||
+    mappingType !== "HDF5" ||
+    mappingStatus === MappingStatusEnum.DISABLED
+  ) {
+    Toast.error("An HDF5 mapping needs to be enabled to use the proofreading tool.");
+    return false;
+  }
+
+  return true;
+}
+
 function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
   const allowUpdate = yield* select((state) => state.tracing.restrictions.allowUpdate);
   if (!allowUpdate) return;
@@ -252,18 +269,8 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
   const { tracingId: volumeTracingId } = volumeTracing;
 
   const layerName = volumeTracingId;
-  const mappingInfo = yield* select((state) =>
-    getMappingInfo(state.temporaryConfiguration.activeMappingByLayer, layerName),
-  );
-  const { mappingName, mappingType, mappingStatus } = mappingInfo;
-  if (
-    mappingName == null ||
-    mappingType !== "HDF5" ||
-    mappingStatus === MappingStatusEnum.DISABLED
-  ) {
-    Toast.error("An HDF5 mapping needs to be enabled to use the proofreading tool.");
-    return;
-  }
+  const isHdf5MappingEnabled = yield* call(ensureHdf5MappingIsEnabled, layerName);
+  if (!isHdf5MappingEnabled) return;
 
   if (!volumeTracing.mappingIsEditable) {
     try {

From f11cb7f659696eea8852d34a9a4696ac141b5c52 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 14 Jun 2022 17:16:12 +0200
Subject: [PATCH 105/122] fix tests for real git co l4-spine-heads elf

---
 frontend/javascripts/test/sagas/save_saga.spec.ts | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/frontend/javascripts/test/sagas/save_saga.spec.ts b/frontend/javascripts/test/sagas/save_saga.spec.ts
index 1f5e593a687..92a09d61c58 100644
--- a/frontend/javascripts/test/sagas/save_saga.spec.ts
+++ b/frontend/javascripts/test/sagas/save_saga.spec.ts
@@ -2,6 +2,7 @@ import { alert } from "libs/window";
 import { setSaveBusyAction } from "oxalis/model/actions/save_actions";
 import DiffableMap from "libs/diffable_map";
 import compactSaveQueue from "oxalis/model/helpers/compaction/compact_save_queue";
+import { ensureWkReady } from "oxalis/model/sagas/wk_ready_saga";
 import mockRequire from "mock-require";
 import test from "ava";
 import { createSaveQueueFromUpdateActions } from "../helpers/saveHelpers";
@@ -80,7 +81,7 @@ test("SaveSaga should send update actions", (t) => {
   const updateActions = [UpdateActions.createEdge(1, 0, 1), UpdateActions.createEdge(1, 1, 2)];
   const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP);
   const saga = pushSaveQueueAsync(TRACING_TYPE, tracingId);
-  expectValueDeepEqual(t, saga.next(), take(INIT_ACTION));
+  expectValueDeepEqual(t, saga.next(), call(ensureWkReady));
   saga.next(); // setLastSaveTimestampAction
 
   saga.next(); // select state
@@ -195,7 +196,7 @@ test("SaveSaga should send update actions right away and try to reach a state wh
   const updateActions = [UpdateActions.createEdge(1, 0, 1), UpdateActions.createEdge(1, 1, 2)];
   const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP);
   const saga = pushSaveQueueAsync(TRACING_TYPE, tracingId);
-  expectValueDeepEqual(t, saga.next(), take(INIT_ACTION));
+  expectValueDeepEqual(t, saga.next(), call(ensureWkReady));
   saga.next();
   saga.next(); // select state
 
@@ -218,7 +219,7 @@ test("SaveSaga should not try to reach state with all actions being saved when s
   const updateActions = [UpdateActions.createEdge(1, 0, 1), UpdateActions.createEdge(1, 1, 2)];
   const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP);
   const saga = pushSaveQueueAsync(TRACING_TYPE, tracingId);
-  expectValueDeepEqual(t, saga.next(), take(INIT_ACTION));
+  expectValueDeepEqual(t, saga.next(), call(ensureWkReady));
   saga.next();
   saga.next(); // select state
 

From 937f22fb10ef10eef5e1b495889754b697887a07 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 14 Jun 2022 17:31:17 +0200
Subject: [PATCH 106/122] fix linting

---
 frontend/javascripts/test/sagas/save_saga.spec.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/frontend/javascripts/test/sagas/save_saga.spec.ts b/frontend/javascripts/test/sagas/save_saga.spec.ts
index 92a09d61c58..4e8d5af9260 100644
--- a/frontend/javascripts/test/sagas/save_saga.spec.ts
+++ b/frontend/javascripts/test/sagas/save_saga.spec.ts
@@ -63,7 +63,6 @@ const initialState = {
     },
   },
 };
-const INIT_ACTION = "WK_READY";
 const LAST_VERSION = 2;
 const TRACINGSTORE_URL = "test.webknossos.xyz";
 const TRACING_TYPE = "skeleton";

From 37fdc4314725ef5f5d9e246891788e1401d3119d Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 14 Jun 2022 17:56:33 +0200
Subject: [PATCH 107/122] fix tool cycling and move proofread tool to the right

---
 .../oxalis/model/accessors/tool_accessor.ts   | 27 ++++++++++++---
 .../oxalis/view/action-bar/toolbar_view.tsx   | 34 +++++++++----------
 .../test/sagas/annotation_tool_saga.spec.ts   |  5 ++-
 3 files changed, 41 insertions(+), 25 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
index 7237510df4c..7b1a039a87b 100644
--- a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
+++ b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
@@ -20,6 +20,7 @@ const getExplanationForDisabledVolume = (
   isInMergerMode: boolean,
   isSegmentationTracingVisibleForMag: boolean,
   isZoomInvalidForTracing: boolean,
+  isEditableMappingActive: boolean,
 ) => {
   if (!isSegmentationTracingVisible) {
     return "Volume annotation is disabled since no segmentation tracing layer is enabled. Enable it in the left settings sidebar.";
@@ -37,6 +38,10 @@ const getExplanationForDisabledVolume = (
     return "Volume annotation is disabled since no segmentation data can be shown at the current magnification. Please adjust the zoom level.";
   }
 
+  if (isEditableMappingActive) {
+    return "Volume annotation is disabled while an editable mapping is active.";
+  }
+
   return "Volume annotation is currently disabled.";
 };
 
@@ -60,6 +65,7 @@ const disabledSkeletonExplanation =
 function _getDisabledInfoWhenVolumeIsDisabled(
   genericDisabledExplanation: string,
   hasSkeleton: boolean,
+  isVolumeDisabled: boolean,
 ) {
   const disabledInfo = {
     isDisabled: true,
@@ -82,7 +88,10 @@ function _getDisabledInfoWhenVolumeIsDisabled(
     [AnnotationToolEnum.FILL_CELL]: disabledInfo,
     [AnnotationToolEnum.PICK_CELL]: disabledInfo,
     [AnnotationToolEnum.BOUNDING_BOX]: notDisabledInfo,
-    [AnnotationToolEnum.PROOFREAD]: disabledInfo,
+    [AnnotationToolEnum.PROOFREAD]: {
+      isDisabled: isVolumeDisabled,
+      explanation: genericDisabledExplanation,
+    },
   };
 }
 
@@ -164,21 +173,29 @@ export function getDisabledInfoForTools(state: OxalisState): Record<
     segmentationTracingLayer != null &&
     visibleSegmentationLayer != null &&
     visibleSegmentationLayer.name === segmentationTracingLayer.tracingId;
+  const isEditableMappingActive =
+    segmentationTracingLayer != null && !!segmentationTracingLayer.mappingIsEditable;
   const genericDisabledExplanation = getExplanationForDisabledVolume(
     isSegmentationTracingVisible,
     isInMergerMode,
     isSegmentationTracingVisibleForMag,
     isZoomInvalidForTracing,
+    isEditableMappingActive,
   );
 
-  if (
+  const isVolumeDisabled =
     !hasVolume ||
     !isSegmentationTracingVisible ||
     !isSegmentationTracingVisibleForMag ||
-    isInMergerMode
-  ) {
+    isInMergerMode;
+
+  if (isVolumeDisabled || isEditableMappingActive) {
     // All segmentation-related tools are disabled.
-    return getDisabledInfoWhenVolumeIsDisabled(genericDisabledExplanation, hasSkeleton);
+    return getDisabledInfoWhenVolumeIsDisabled(
+      genericDisabledExplanation,
+      hasSkeleton,
+      isVolumeDisabled,
+    );
   }
 
   const isZoomStepTooHighForBrushing = isZoomStepTooHighFor(state, AnnotationToolEnum.BRUSH);
diff --git a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
index e392eca7d6f..a0d5e7be851 100644
--- a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
+++ b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
@@ -607,23 +607,6 @@ export default function ToolbarView() {
           </RadioButtonWithTooltip>
         ) : null}
 
-        {hasSkeleton && hasVolume ? (
-          <RadioButtonWithTooltip
-            title="Proofreading Tool - Modify an agglomerated segmentation."
-            disabledTitle={disabledInfosForTools[AnnotationToolEnum.PROOFREAD].explanation}
-            disabled={disabledInfosForTools[AnnotationToolEnum.PROOFREAD].isDisabled}
-            style={narrowButtonStyle}
-            value={AnnotationToolEnum.PROOFREAD}
-          >
-            <i
-              className="fas fa-clipboard-check"
-              style={{
-                opacity: disabledInfosForTools[AnnotationToolEnum.PROOFREAD].isDisabled ? 0.5 : 1,
-              }}
-            />
-          </RadioButtonWithTooltip>
-        ) : null}
-
         {hasVolume && isVolumeModificationAllowed ? (
           <React.Fragment>
             <RadioButtonWithTooltip
@@ -759,6 +742,23 @@ export default function ToolbarView() {
             }}
           />
         </RadioButtonWithTooltip>
+
+        {hasSkeleton && hasVolume ? (
+          <RadioButtonWithTooltip
+            title="Proofreading Tool - Modify an agglomerated segmentation."
+            disabledTitle={disabledInfosForTools[AnnotationToolEnum.PROOFREAD].explanation}
+            disabled={disabledInfosForTools[AnnotationToolEnum.PROOFREAD].isDisabled}
+            style={narrowButtonStyle}
+            value={AnnotationToolEnum.PROOFREAD}
+          >
+            <i
+              className="fas fa-clipboard-check"
+              style={{
+                opacity: disabledInfosForTools[AnnotationToolEnum.PROOFREAD].isDisabled ? 0.5 : 1,
+              }}
+            />
+          </RadioButtonWithTooltip>
+        ) : null}
       </Radio.Group>
 
       <ToolSpecificSettings
diff --git a/frontend/javascripts/test/sagas/annotation_tool_saga.spec.ts b/frontend/javascripts/test/sagas/annotation_tool_saga.spec.ts
index cc80fe97b2c..61e19c9ac5c 100644
--- a/frontend/javascripts/test/sagas/annotation_tool_saga.spec.ts
+++ b/frontend/javascripts/test/sagas/annotation_tool_saga.spec.ts
@@ -1,5 +1,5 @@
 import test from "ava";
-import { AnnotationToolEnum } from "oxalis/constants";
+import { AnnotationToolEnum, AnnotationTool } from "oxalis/constants";
 import mockRequire from "mock-require";
 import { initialState } from "test/fixtures/volumetracing_object";
 import sinon from "sinon";
@@ -88,8 +88,7 @@ test.serial("Selecting another tool should trigger a deselection of the previous
   saga.next(wkReadyAction());
   saga.next(newState.uiInformation.activeTool);
 
-  // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'nextTool' implicitly has an 'any' type.
-  const cycleTool = (nextTool) => {
+  const cycleTool = (nextTool: AnnotationTool) => {
     const action = setToolAction(nextTool);
     newState = UiReducer(newState, action);
     saga.next(action);

From 69cda104815d85d79ef9e83cc57a618de00a4fc9 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 14 Jun 2022 19:46:08 +0200
Subject: [PATCH 108/122] fix tooltips for navbar buttons, disable merge mode
 if editable mapping is active, PR feedback

---
 .../oxalis/model/accessors/tool_accessor.ts   |   2 +
 .../oxalis/view/action-bar/toolbar_view.tsx   | 166 +++++++++---------
 .../javascripts/oxalis/view/version_list.tsx  |   4 +-
 3 files changed, 91 insertions(+), 81 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
index 7b1a039a87b..a97cf6244a5 100644
--- a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
+++ b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
@@ -186,6 +186,8 @@ export function getDisabledInfoForTools(state: OxalisState): Record<
   const isVolumeDisabled =
     !hasVolume ||
     !isSegmentationTracingVisible ||
+    // isSegmentationTracingVisibleForMag is false if isZoomInvalidForTracing is true which is why
+    // this condition doesn't need to be checked here
     !isSegmentationTracingVisibleForMag ||
     isInMergerMode;
 
diff --git a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
index a0d5e7be851..33d010a89a1 100644
--- a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
+++ b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
@@ -265,6 +265,15 @@ function AdditionalSkeletonModesButtons() {
     (state: OxalisState) => state.userConfiguration.newNodeNewTree,
   );
 
+  const segmentationTracingLayer = useSelector((state: OxalisState) =>
+    getActiveSegmentationTracing(state),
+  );
+  const isEditableMappingActive =
+    segmentationTracingLayer != null && !!segmentationTracingLayer.mappingIsEditable;
+  const mergerModeTooltipText = isEditableMappingActive
+    ? "Merger mode cannot be enabled while an editable mapping is active."
+    : "Toggle Merger Mode - When enabled, skeletons that connect multiple segments will merge those segments.";
+
   const toggleNewNodeNewTreeMode = () =>
     dispatch(updateUserSettingAction("newNodeNewTree", !isNewNodeNewTreeModeOn));
 
@@ -278,37 +287,40 @@ function AdditionalSkeletonModesButtons() {
   const mergerModeButtonStyle = isMergerModeEnabled ? activeButtonStyle : narrowButtonStyle;
   return (
     <React.Fragment>
-      <Tooltip title="Toggle the Single node Tree (soma clicking) mode - If enabled, each node creation will create a new tree.">
+      <ButtonComponent
+        style={newNodeNewTreeModeButtonStyle}
+        onClick={toggleNewNodeNewTreeMode}
+        title="Toggle the Single node Tree (soma clicking) mode - If enabled, each node creation will create a new tree."
+      >
+        <img
+          style={imgStyleForSpaceyIcons}
+          src="/assets/images/soma-clicking-icon.svg"
+          alt="Single Node Tree Mode"
+        />
+      </ButtonComponent>
+      <ButtonComponent
+        style={{
+          ...mergerModeButtonStyle,
+          opacity: isEditableMappingActive ? 0.5 : 1,
+        }}
+        onClick={toggleMergerMode}
+        disabled={isEditableMappingActive}
+        title={mergerModeTooltipText}
+      >
+        <img
+          style={imgStyleForSpaceyIcons}
+          src="/assets/images/merger-mode-icon.svg"
+          alt="Merger Mode"
+        />
+      </ButtonComponent>
+      {features().jobsEnabled && isMergerModeEnabled && (
         <ButtonComponent
-          style={newNodeNewTreeModeButtonStyle}
-          value="active"
-          onClick={toggleNewNodeNewTreeMode}
+          style={narrowButtonStyle}
+          onClick={() => setShowMaterializeVolumeAnnotationModal(true)}
+          title="Materialize this merger mode annotation into a new dataset."
         >
-          <img
-            style={imgStyleForSpaceyIcons}
-            src="/assets/images/soma-clicking-icon.svg"
-            alt="Single Node Tree Mode"
-          />
-        </ButtonComponent>
-      </Tooltip>
-      <Tooltip title="Toggle Merger Mode - When enabled, skeletons that connect multiple segments will merge those segments.">
-        <ButtonComponent style={mergerModeButtonStyle} value="active" onClick={toggleMergerMode}>
-          <img
-            style={imgStyleForSpaceyIcons}
-            src="/assets/images/merger-mode-icon.svg"
-            alt="Merger Mode"
-          />
+          <ExportOutlined />
         </ButtonComponent>
-      </Tooltip>
-      {features().jobsEnabled && isMergerModeEnabled && (
-        <Tooltip title="Materialize this merger mode annotation into a new dataset.">
-          <ButtonComponent
-            style={narrowButtonStyle}
-            onClick={() => setShowMaterializeVolumeAnnotationModal(true)}
-          >
-            <ExportOutlined />
-          </ButtonComponent>
-        </Tooltip>
       )}
       {features().jobsEnabled && showMaterializeVolumeAnnotationModal && (
         <MaterializeVolumeAnnotationModal
@@ -351,40 +363,36 @@ function CreateCellButton() {
         zIndex: 1000,
       }}
     >
-      <Tooltip
+      <ButtonComponent
+        onClick={handleCreateCell}
+        style={{
+          width: 36,
+          paddingLeft: 10,
+        }}
         title={`Create a new segment id (C) – The active segment id is ${unmappedActiveCellId}${mappedIdInfo}.`}
       >
-        <ButtonComponent
-          onClick={handleCreateCell}
-          style={{
-            width: 36,
-            paddingLeft: 10,
-          }}
-        >
-          <img src="/assets/images/new-cell.svg" alt="New Segment Icon" />
-        </ButtonComponent>
-      </Tooltip>
+        <img src="/assets/images/new-cell.svg" alt="New Segment Icon" />
+      </ButtonComponent>
     </Badge>
   );
 }
 
 function CreateNewBoundingBoxButton() {
   return (
-    <Tooltip title="Create a new bounding box centered around the current position.">
-      <ButtonComponent
-        onClick={handleAddNewUserBoundingBox}
-        style={{
-          paddingLeft: 9,
-          paddingRight: 9,
-        }}
-      >
-        <img
-          src="/assets/images/new-bounding-box.svg"
-          alt="New Bounding Box Icon"
-          style={imgStyleForSpaceyIcons}
-        />
-      </ButtonComponent>
-    </Tooltip>
+    <ButtonComponent
+      onClick={handleAddNewUserBoundingBox}
+      style={{
+        paddingLeft: 9,
+        paddingRight: 9,
+      }}
+      title="Create a new bounding box centered around the current position."
+    >
+      <img
+        src="/assets/images/new-bounding-box.svg"
+        alt="New Bounding Box Icon"
+        style={imgStyleForSpaceyIcons}
+      />
+    </ButtonComponent>
   );
 }
 
@@ -414,30 +422,29 @@ function CreateTreeButton() {
         zIndex: 1000,
       }}
     >
-      <Tooltip title={`Create a new Tree (C) – ${activeTreeHint}`}>
-        <ButtonComponent
-          onClick={handleCreateTree}
-          style={{ ...narrowButtonStyle, paddingRight: 5 }}
-        >
-          <i
-            style={{
-              opacity: 0.9,
-              transform: "scale(0.9) translate(-2px, -1px)",
-              marginRight: 3,
-            }}
-            className="fas fa-project-diagram"
-          />
-          <i
-            className="fas fa-plus"
-            style={{
-              position: "absolute",
-              top: 13,
-              left: 21,
-              fontSize: 11,
-            }}
-          />
-        </ButtonComponent>
-      </Tooltip>
+      <ButtonComponent
+        onClick={handleCreateTree}
+        style={{ ...narrowButtonStyle, paddingRight: 5 }}
+        title={`Create a new Tree (C) – ${activeTreeHint}`}
+      >
+        <i
+          style={{
+            opacity: 0.9,
+            transform: "scale(0.9) translate(-2px, -1px)",
+            marginRight: 3,
+          }}
+          className="fas fa-project-diagram"
+        />
+        <i
+          className="fas fa-plus"
+          style={{
+            position: "absolute",
+            top: 13,
+            left: 21,
+            fontSize: 11,
+          }}
+        />
+      </ButtonComponent>
     </Badge>
   );
 }
@@ -485,11 +492,10 @@ function ChangeBrushSizeButton() {
             width: 36,
             padding: 0,
           }}
-          value="active"
         >
           <img
             src="/assets/images/brush-size-icon.svg"
-            alt="Merger Mode"
+            alt="Brush Size"
             style={{
               width: 20,
               height: 20,
diff --git a/frontend/javascripts/oxalis/view/version_list.tsx b/frontend/javascripts/oxalis/view/version_list.tsx
index 8b9ba2d44d7..0609a36c48e 100644
--- a/frontend/javascripts/oxalis/view/version_list.tsx
+++ b/frontend/javascripts/oxalis/view/version_list.tsx
@@ -158,7 +158,9 @@ class VersionList extends React.Component<Props, State> {
         },
       });
     } else {
-      Toast.warning(`Version preview for ${this.props.versionedObjectType}s is not supported yet.`);
+      Toast.warning(
+        `Version preview and restoring for ${this.props.versionedObjectType}s is not supported yet.`,
+      );
       return Promise.resolve();
     }
   };

From c850a35471707c7148b2004f629134ab219129f4 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 14 Jun 2022 20:14:36 +0200
Subject: [PATCH 109/122] undo last action if editable mapping creation fails,
 add more explanation to proofreading tool

---
 frontend/javascripts/oxalis/model/sagas/proofread_saga.ts    | 3 +++
 frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 7835e4a45d8..4bb0fa2a59a 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -22,6 +22,7 @@ import {
 import {
   pushSaveQueueTransaction,
   setVersionNumberAction,
+  undoAction,
 } from "oxalis/model/actions/save_actions";
 import { splitAgglomerate, mergeAgglomerate } from "oxalis/model/sagas/update_actions";
 import Model from "oxalis/model";
@@ -277,6 +278,8 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
       yield* call(createEditableMapping);
     } catch (e) {
       console.error(e);
+      // Undo the last splitting/merging action since the proofreading action did not succeed
+      yield* put(undoAction());
       return;
     }
   }
diff --git a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
index 33d010a89a1..13464c9ed64 100644
--- a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
+++ b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
@@ -751,7 +751,7 @@ export default function ToolbarView() {
 
         {hasSkeleton && hasVolume ? (
           <RadioButtonWithTooltip
-            title="Proofreading Tool - Modify an agglomerated segmentation."
+            title="Proofreading Tool - Modify an agglomerated segmentation. Other segmentation modifications, like brushing, are not allowed if this tool is used."
             disabledTitle={disabledInfosForTools[AnnotationToolEnum.PROOFREAD].explanation}
             disabled={disabledInfosForTools[AnnotationToolEnum.PROOFREAD].isDisabled}
             style={narrowButtonStyle}

From 9682f8e17ebaf4140a86ef205d88bb96d2647858 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 14 Jun 2022 20:22:40 +0200
Subject: [PATCH 110/122] instead of reloading agglomerate skeletons after
 proofreading action, simply rename them, fixing multiple bugs

---
 .../oxalis/model/sagas/proofread_saga.ts      | 38 +++++++++----------
 1 file changed, 18 insertions(+), 20 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 4bb0fa2a59a..e30f5457a17 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -5,9 +5,9 @@ import { AnnotationToolEnum, MappingStatusEnum, Vector3 } from "oxalis/constants
 import Toast from "libs/toast";
 import {
   DeleteEdgeAction,
-  deleteTreeAction,
   loadAgglomerateSkeletonAction,
   MergeTreesAction,
+  setTreeNameAction,
 } from "oxalis/model/actions/skeletontracing_actions";
 import {
   initializeEditableMappingAction,
@@ -18,6 +18,7 @@ import {
   enforceSkeletonTracing,
   findTreeByName,
   findTreeByNodeId,
+  getTreeNameForAgglomerateSkeleton,
 } from "oxalis/model/accessors/skeletontracing_accessor";
 import {
   pushSaveQueueTransaction,
@@ -383,28 +384,25 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
     agglomerateFileZoomstep,
   );
 
-  /* Remove old agglomerate skeleton(s) and load updated agglomerate skeleton(s) */
+  /* Rename agglomerate skeleton(s) according to their new id and mapping name */
 
-  yield* put(deleteTreeAction(sourceTree.treeId));
-  if (sourceTree !== targetTree) {
-    yield* put(deleteTreeAction(targetTree.treeId));
-  }
-
-  yield* call(
-    loadAgglomerateSkeletonWithId,
-    loadAgglomerateSkeletonAction(
-      layerName,
-      volumeTracingWithEditableMapping.mappingName,
-      newSourceNodeAgglomerateId,
+  yield* put(
+    setTreeNameAction(
+      getTreeNameForAgglomerateSkeleton(
+        newSourceNodeAgglomerateId,
+        volumeTracingWithEditableMapping.mappingName,
+      ),
+      sourceTree.treeId,
     ),
   );
-  if (newTargetNodeAgglomerateId !== newSourceNodeAgglomerateId) {
-    yield* call(
-      loadAgglomerateSkeletonWithId,
-      loadAgglomerateSkeletonAction(
-        layerName,
-        volumeTracingWithEditableMapping.mappingName,
-        newTargetNodeAgglomerateId,
+  if (sourceTree !== targetTree) {
+    yield* put(
+      setTreeNameAction(
+        getTreeNameForAgglomerateSkeleton(
+          newTargetNodeAgglomerateId,
+          volumeTracingWithEditableMapping.mappingName,
+        ),
+        targetTree.treeId,
       ),
     );
   }

From 208b04ae437a7c0d1921e722f1cf965234a92eb1 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Tue, 14 Jun 2022 20:52:00 +0200
Subject: [PATCH 111/122] fix ad-hoc mesh loading for editable mappings

---
 .../oxalis/model/sagas/isosurface_saga.ts          | 14 +++++++++-----
 .../oxalis/model/sagas/proofread_saga.ts           |  6 ------
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
index cca0ba7b38c..fc9618f3080 100644
--- a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
@@ -375,11 +375,15 @@ function* maybeLoadIsosurface(
   }`;
   const tracingStoreUrl = `${tracingStoreHost}/tracings/volume/${layer.name}`;
   const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
-  // Fetch from datastore if no volumetracing exists or if the tracing has a fallback layer.
-  const useDataStore =
-    isosurfaceExtraInfo.useDataStore != null
-      ? isosurfaceExtraInfo.useDataStore
-      : volumeTracing == null || volumeTracing.fallbackLayer != null;
+  // Fetch from datastore if no volumetracing exists or if the tracing has a fallback layer ...
+  let useDataStore = volumeTracing == null || volumeTracing.fallbackLayer != null;
+  if (isosurfaceExtraInfo.useDataStore != null) {
+    // ... except if the caller specified whether to use the data store ...
+    useDataStore = isosurfaceExtraInfo.useDataStore;
+  } else if (volumeTracing != null && volumeTracing.mappingIsEditable) {
+    // ... or if an editable mapping is active.
+    useDataStore = false;
+  }
   const mag = resolutionInfo.getResolutionByIndexOrThrow(zoomStep);
 
   if (isInitialRequest) {
diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index e30f5457a17..0c0a120902d 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -82,16 +82,11 @@ function proofreadSegmentProximityNm(): number {
 let oldSegmentIdsInProximity: number[] | null = null;
 
 function* loadCoarseAdHocMesh(layerName: string, segmentId: number, position: Vector3): Saga<void> {
-  const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
-  if (volumeTracing == null) return;
-
   const mappingInfo = yield* select((state) =>
     getMappingInfo(state.temporaryConfiguration.activeMappingByLayer, layerName),
   );
   const { mappingName, mappingType } = mappingInfo;
 
-  // Use the data store if the mapping is not editable yet. If it, is request the mesh from the tracing store.
-  const useDataStore = !volumeTracing.mappingIsEditable;
   // Load the whole agglomerate mesh in a coarse resolution for performance reasons
   const preferredQuality = proofreadCoarseResolutionIndex();
   yield* put(
@@ -99,7 +94,6 @@ function* loadCoarseAdHocMesh(layerName: string, segmentId: number, position: Ve
       mappingName,
       mappingType,
       passive: true,
-      useDataStore,
       preferredQuality,
     }),
   );

From 5ce4c66a3e65488b11c3534321e46fc0248e9acc Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Wed, 15 Jun 2022 15:26:06 +0200
Subject: [PATCH 112/122] increase ad-hoc batch limit 8-fold since cube size
 edge length per dimension was halved

---
 frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
index fc9618f3080..957f48b0bc1 100644
--- a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
@@ -151,7 +151,7 @@ function getNeighborPosition(clippedPosition: Vector3, neighborId: number): Vect
 // In order to avoid, that too many chunks are computed for one user interaction,
 // we store the amount of requests in a batch per segment.
 const batchCounterPerSegment: Record<number, number> = {};
-const MAXIMUM_BATCH_SIZE = 50;
+const MAXIMUM_BATCH_SIZE = 400;
 
 function* loadAdHocIsosurfaceFromAction(action: LoadAdHocMeshAction): Saga<void> {
   yield* call(

From 5f39dcf24c6ffab8611166120b8d1d6f23bdd6f8 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Wed, 15 Jun 2022 17:04:14 +0200
Subject: [PATCH 113/122] hide and disable proofreading tool if no agglomerate
 views are available

---
 .../oxalis/model/accessors/tool_accessor.ts          | 12 ++++++++++--
 .../oxalis/model/sagas/skeletontracing_saga.ts       | 11 ++++++++++-
 .../oxalis/view/action-bar/toolbar_view.tsx          | 11 ++++++++++-
 3 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
index a97cf6244a5..de3234ec8c5 100644
--- a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
+++ b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
@@ -61,6 +61,7 @@ export function isTraceTool(activeTool: AnnotationTool): boolean {
 }
 const disabledSkeletonExplanation =
   "This annotation does not have a skeleton. Please convert it to a hybrid annotation.";
+const disabledAgglomerateMappingsExplanation = "This dataset does not have agglomerate mappings.";
 
 function _getDisabledInfoWhenVolumeIsDisabled(
   genericDisabledExplanation: string,
@@ -102,6 +103,7 @@ function _getDisabledInfoFromArgs(
   isZoomStepTooHighForBrushing: boolean,
   isZoomStepTooHighForTracing: boolean,
   isZoomStepTooHighForFilling: boolean,
+  hasAgglomerateMappings: boolean,
   genericDisabledExplanation: string,
 ) {
   return {
@@ -142,8 +144,10 @@ function _getDisabledInfoFromArgs(
       explanation: disabledSkeletonExplanation,
     },
     [AnnotationToolEnum.PROOFREAD]: {
-      isDisabled: !hasSkeleton,
-      explanation: disabledSkeletonExplanation,
+      isDisabled: !hasSkeleton || !hasAgglomerateMappings,
+      explanation: !hasSkeleton
+        ? disabledSkeletonExplanation
+        : disabledAgglomerateMappingsExplanation,
     },
   };
 }
@@ -203,11 +207,15 @@ export function getDisabledInfoForTools(state: OxalisState): Record<
   const isZoomStepTooHighForBrushing = isZoomStepTooHighFor(state, AnnotationToolEnum.BRUSH);
   const isZoomStepTooHighForTracing = isZoomStepTooHighFor(state, AnnotationToolEnum.TRACE);
   const isZoomStepTooHighForFilling = isZoomStepTooHighFor(state, AnnotationToolEnum.FILL_CELL);
+  const hasAgglomerateMappings =
+    visibleSegmentationLayer.agglomerates != null &&
+    visibleSegmentationLayer.agglomerates.length > 0;
   return getDisabledInfoFromArgs(
     hasSkeleton,
     isZoomStepTooHighForBrushing,
     isZoomStepTooHighForTracing,
     isZoomStepTooHighForFilling,
+    hasAgglomerateMappings,
     genericDisabledExplanation,
   );
 }
diff --git a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
index 7dae5802ff2..39a78d6717a 100644
--- a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
@@ -265,11 +265,20 @@ function* getAgglomerateSkeletonTracing(
     const parsedTracing = parseProtoTracing(nmlProtoBuffer, "skeleton");
 
     if (!("trees" in parsedTracing)) {
-      // This check is only for flow to realize that we have a skeleton tracing
+      // This check is only for typescript to realize that we have a skeleton tracing
       // on our hands.
       throw new Error("Skeleton tracing doesn't contain trees");
     }
 
+    if (parsedTracing.trees.length !== 1) {
+      throw new Error(
+        `Agglomerate skeleton response does not contain exactly one tree, but ${parsedTracing.trees.length} instead.`,
+      );
+    }
+
+    // Make sure the tree is named as expected
+    parsedTracing.trees[0].name = getTreeNameForAgglomerateSkeleton(agglomerateId, mappingName);
+
     return parsedTracing;
   } catch (e) {
     // @ts-ignore
diff --git a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
index 13464c9ed64..33f94076e9b 100644
--- a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
+++ b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
@@ -50,6 +50,7 @@ import Store, { OxalisState, VolumeTracing } from "oxalis/store";
 
 import features from "features";
 import { getInterpolationInfo } from "oxalis/model/sagas/volume/volume_interpolation_saga";
+import { getVisibleSegmentationLayer } from "oxalis/model/accessors/dataset_accessor";
 
 const narrowButtonStyle = {
   paddingLeft: 10,
@@ -510,6 +511,14 @@ function ChangeBrushSizeButton() {
 export default function ToolbarView() {
   const hasVolume = useSelector((state: OxalisState) => state.tracing.volumes.length > 0);
   const hasSkeleton = useSelector((state: OxalisState) => state.tracing.skeleton != null);
+  const hasAgglomerateMappings = useSelector((state: OxalisState) => {
+    const visibleSegmentationLayer = getVisibleSegmentationLayer(state);
+    return (
+      visibleSegmentationLayer != null &&
+      visibleSegmentationLayer.agglomerates != null &&
+      visibleSegmentationLayer.agglomerates.length > 0
+    );
+  });
   const isVolumeModificationAllowed = useSelector(
     (state: OxalisState) => !hasEditableMapping(state),
   );
@@ -749,7 +758,7 @@ export default function ToolbarView() {
           />
         </RadioButtonWithTooltip>
 
-        {hasSkeleton && hasVolume ? (
+        {hasSkeleton && hasVolume && hasAgglomerateMappings ? (
           <RadioButtonWithTooltip
             title="Proofreading Tool - Modify an agglomerated segmentation. Other segmentation modifications, like brushing, are not allowed if this tool is used."
             disabledTitle={disabledInfosForTools[AnnotationToolEnum.PROOFREAD].explanation}

From 18d74b92e841ea46037493a5691fabf57207abc6 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Wed, 15 Jun 2022 17:10:40 +0200
Subject: [PATCH 114/122] fix merge

---
 .../tracingstore/controllers/VolumeTracingController.scala      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
index 28552955406..12ac20fabf2 100644
--- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
+++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
@@ -5,7 +5,7 @@ import java.nio.{ByteBuffer, ByteOrder}
 
 import akka.stream.scaladsl.Source
 import com.google.inject.Inject
-import com.scalableminds.util.geometry.{BoundingBox, Vec3Int}
+import com.scalableminds.util.geometry.{BoundingBox, Vec3Int, Vec3Double}
 import com.scalableminds.util.tools.ExtendedTypes.ExtendedString
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings}

From d5ead75cb079e6809613f72140f1a80bc15d15d9 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Wed, 15 Jun 2022 19:22:21 +0200
Subject: [PATCH 115/122] fix agglomerate too large error message for editable
 mappings

---
 .../oxalis/model/sagas/skeletontracing_saga.ts     | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
index 39a78d6717a..ade09dd9225 100644
--- a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
@@ -284,10 +284,16 @@ function* getAgglomerateSkeletonTracing(
     // @ts-ignore
     if (e.messages != null) {
       // Enhance the error message for agglomerates that are too large
-      // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'message' implicitly has an 'any' type.
-      const agglomerateTooLargeMessages = e.messages.filter((message) =>
-        message.chain != null ? message.chain.includes("too many") : false,
-      );
+      // @ts-ignore
+      const agglomerateTooLargeMessages = e.messages
+        .filter(
+          (message: Message) =>
+            (message.chain != null && message.chain.includes("too many")) ||
+            (message.error != null && message.error.includes("too many")),
+        )
+        // Demote error message to chain message so that it is shown in conjunction with the newly
+        // introduced error (as the chain). Otherwise there would be two toasts.
+        .map((message: Message) => (message.error != null ? { chain: message.error } : message));
 
       if (agglomerateTooLargeMessages.length > 0) {
         throw {

From 0f037b5898c34894ce48c09597cecd8688680fab Mon Sep 17 00:00:00 2001
From: Daniel <daniel.werner@scalableminds.com>
Date: Thu, 16 Jun 2022 13:51:45 +0200
Subject: [PATCH 116/122] Apply suggestions from code review

Co-authored-by: Philipp Otto <philippotto@users.noreply.github.com>
---
 frontend/javascripts/oxalis/model/accessors/tool_accessor.ts  | 3 +--
 frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts    | 2 +-
 .../javascripts/oxalis/model/sagas/skeletontracing_saga.ts    | 4 ++--
 frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx  | 4 +---
 4 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
index de3234ec8c5..d4d9d8c8fee 100644
--- a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
+++ b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
@@ -208,8 +208,7 @@ export function getDisabledInfoForTools(state: OxalisState): Record<
   const isZoomStepTooHighForTracing = isZoomStepTooHighFor(state, AnnotationToolEnum.TRACE);
   const isZoomStepTooHighForFilling = isZoomStepTooHighFor(state, AnnotationToolEnum.FILL_CELL);
   const hasAgglomerateMappings =
-    visibleSegmentationLayer.agglomerates != null &&
-    visibleSegmentationLayer.agglomerates.length > 0;
+    visibleSegmentationLayer.agglomerates?.length > 0;
   return getDisabledInfoFromArgs(
     hasSkeleton,
     isZoomStepTooHighForBrushing,
diff --git a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
index 957f48b0bc1..4976e38d203 100644
--- a/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/isosurface_saga.ts
@@ -380,7 +380,7 @@ function* maybeLoadIsosurface(
   if (isosurfaceExtraInfo.useDataStore != null) {
     // ... except if the caller specified whether to use the data store ...
     useDataStore = isosurfaceExtraInfo.useDataStore;
-  } else if (volumeTracing != null && volumeTracing.mappingIsEditable) {
+  } else if (volumeTracing?.mappingIsEditable) {
     // ... or if an editable mapping is active.
     useDataStore = false;
   }
diff --git a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
index ade09dd9225..78ff3eb54d8 100644
--- a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
@@ -288,8 +288,8 @@ function* getAgglomerateSkeletonTracing(
       const agglomerateTooLargeMessages = e.messages
         .filter(
           (message: Message) =>
-            (message.chain != null && message.chain.includes("too many")) ||
-            (message.error != null && message.error.includes("too many")),
+            (message.chain?.includes("too many")) ||
+            (message.error?.includes("too many")),
         )
         // Demote error message to chain message so that it is shown in conjunction with the newly
         // introduced error (as the chain). Otherwise there would be two toasts.
diff --git a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
index 33f94076e9b..841a961974e 100644
--- a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
+++ b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
@@ -514,9 +514,7 @@ export default function ToolbarView() {
   const hasAgglomerateMappings = useSelector((state: OxalisState) => {
     const visibleSegmentationLayer = getVisibleSegmentationLayer(state);
     return (
-      visibleSegmentationLayer != null &&
-      visibleSegmentationLayer.agglomerates != null &&
-      visibleSegmentationLayer.agglomerates.length > 0
+      visibleSegmentationLayer?.agglomerates?.length > 0
     );
   });
   const isVolumeModificationAllowed = useSelector(

From 03d132ac22f86d8e4967360ee015a3358531f254 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Thu, 16 Jun 2022 14:26:36 +0200
Subject: [PATCH 117/122] update changelog and pretty

---
 CHANGELOG.unreleased.md                                       | 1 +
 frontend/javascripts/oxalis/model/accessors/tool_accessor.ts  | 3 +--
 .../javascripts/oxalis/model/sagas/skeletontracing_saga.ts    | 3 +--
 frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx  | 4 +---
 4 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
index 50a58aa1390..762d0d85cff 100644
--- a/CHANGELOG.unreleased.md
+++ b/CHANGELOG.unreleased.md
@@ -13,6 +13,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 ### Added
 
 - Added a warning for when the resolution in the XY viewport on z=1-downsampled datasets becomes too low, explaining the problem and how to mitigate it. [#6255](https://github.com/scalableminds/webknossos/pull/6255)
+- Added a proofreading tool which can be used to edit agglomerate mappings. After activating an agglomerate mapping the proofreading tool can be selected. While the tool is active, agglomerates can be clicked to load their agglomerate skeletons. Use the context menu to delete or create edges for those agglomerate skeletons to split or merge agglomerates. The changes will immediately reflect in the segmentation and meshes. [#6195](https://github.com/scalableminds/webknossos/pull/6195)
 
 ### Changed
 
diff --git a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
index d4d9d8c8fee..1d756c8bbff 100644
--- a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
+++ b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
@@ -207,8 +207,7 @@ export function getDisabledInfoForTools(state: OxalisState): Record<
   const isZoomStepTooHighForBrushing = isZoomStepTooHighFor(state, AnnotationToolEnum.BRUSH);
   const isZoomStepTooHighForTracing = isZoomStepTooHighFor(state, AnnotationToolEnum.TRACE);
   const isZoomStepTooHighForFilling = isZoomStepTooHighFor(state, AnnotationToolEnum.FILL_CELL);
-  const hasAgglomerateMappings =
-    visibleSegmentationLayer.agglomerates?.length > 0;
+  const hasAgglomerateMappings = visibleSegmentationLayer.agglomerates?.length > 0;
   return getDisabledInfoFromArgs(
     hasSkeleton,
     isZoomStepTooHighForBrushing,
diff --git a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
index 78ff3eb54d8..1175d42dc93 100644
--- a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts
@@ -288,8 +288,7 @@ function* getAgglomerateSkeletonTracing(
       const agglomerateTooLargeMessages = e.messages
         .filter(
           (message: Message) =>
-            (message.chain?.includes("too many")) ||
-            (message.error?.includes("too many")),
+            message.chain?.includes("too many") || message.error?.includes("too many"),
         )
         // Demote error message to chain message so that it is shown in conjunction with the newly
         // introduced error (as the chain). Otherwise there would be two toasts.
diff --git a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
index 841a961974e..42af291b193 100644
--- a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
+++ b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
@@ -513,9 +513,7 @@ export default function ToolbarView() {
   const hasSkeleton = useSelector((state: OxalisState) => state.tracing.skeleton != null);
   const hasAgglomerateMappings = useSelector((state: OxalisState) => {
     const visibleSegmentationLayer = getVisibleSegmentationLayer(state);
-    return (
-      visibleSegmentationLayer?.agglomerates?.length > 0
-    );
+    return visibleSegmentationLayer?.agglomerates?.length > 0;
   });
   const isVolumeModificationAllowed = useSelector(
     (state: OxalisState) => !hasEditableMapping(state),

From 5e8bf846a80feb34d8a8ed18cee45e8875cae50f Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Thu, 16 Jun 2022 15:06:35 +0200
Subject: [PATCH 118/122] fix typing

---
 frontend/javascripts/oxalis/model/accessors/tool_accessor.ts | 2 +-
 frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
index 1d756c8bbff..d190e2a74a9 100644
--- a/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
+++ b/frontend/javascripts/oxalis/model/accessors/tool_accessor.ts
@@ -207,7 +207,7 @@ export function getDisabledInfoForTools(state: OxalisState): Record<
   const isZoomStepTooHighForBrushing = isZoomStepTooHighFor(state, AnnotationToolEnum.BRUSH);
   const isZoomStepTooHighForTracing = isZoomStepTooHighFor(state, AnnotationToolEnum.TRACE);
   const isZoomStepTooHighForFilling = isZoomStepTooHighFor(state, AnnotationToolEnum.FILL_CELL);
-  const hasAgglomerateMappings = visibleSegmentationLayer.agglomerates?.length > 0;
+  const hasAgglomerateMappings = (visibleSegmentationLayer.agglomerates?.length ?? 0) > 0;
   return getDisabledInfoFromArgs(
     hasSkeleton,
     isZoomStepTooHighForBrushing,
diff --git a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
index 42af291b193..429f7e9fe40 100644
--- a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
+++ b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx
@@ -513,7 +513,7 @@ export default function ToolbarView() {
   const hasSkeleton = useSelector((state: OxalisState) => state.tracing.skeleton != null);
   const hasAgglomerateMappings = useSelector((state: OxalisState) => {
     const visibleSegmentationLayer = getVisibleSegmentationLayer(state);
-    return visibleSegmentationLayer?.agglomerates?.length > 0;
+    return (visibleSegmentationLayer?.agglomerates?.length ?? 0) > 0;
   });
   const isVolumeModificationAllowed = useSelector(
     (state: OxalisState) => !hasEditableMapping(state),

From ff5f39f7357d41825c59de73a6881d977f7cf196 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Mon, 20 Jun 2022 16:00:26 +0200
Subject: [PATCH 119/122] fix proofreading when selecting far away node in 3d
 view

---
 frontend/javascripts/oxalis/model/sagas/proofread_saga.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 0c0a120902d..82c2eae893f 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -39,7 +39,7 @@ import {
 } from "oxalis/model/accessors/dataset_accessor";
 import { makeMappingEditable } from "admin/admin_rest_api";
 import { setMappingNameAction } from "oxalis/model/actions/settings_actions";
-import { getSegmentIdForPosition } from "oxalis/controller/combinations/volume_handlers";
+import { getSegmentIdForPositionAsync } from "oxalis/controller/combinations/volume_handlers";
 import { loadAdHocMeshAction } from "oxalis/model/actions/segmentation_actions";
 import { V3 } from "libs/mjs";
 import { removeIsosurfaceAction } from "oxalis/model/actions/annotation_actions";
@@ -111,7 +111,7 @@ function* proofreadAtPosition(action: ProofreadAtPositionAction): Saga<void> {
   const isHdf5MappingEnabled = yield* call(ensureHdf5MappingIsEnabled, layerName);
   if (!isHdf5MappingEnabled || volumeTracing.mappingName == null) return;
 
-  const segmentId = getSegmentIdForPosition(position);
+  const segmentId = yield* call(getSegmentIdForPositionAsync, position);
 
   /* Load agglomerate skeleton of the agglomerate at the click position */
 

From 1bbcb0eb91d4e8aaa21181fe6a96e7d0b6815ed0 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Mon, 20 Jun 2022 16:01:05 +0200
Subject: [PATCH 120/122] try one-sided mesh rendering to avoid transparency
 artifacts

---
 frontend/javascripts/oxalis/controller/scene_controller.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/javascripts/oxalis/controller/scene_controller.ts b/frontend/javascripts/oxalis/controller/scene_controller.ts
index 1d25100815f..c92394a05d8 100644
--- a/frontend/javascripts/oxalis/controller/scene_controller.ts
+++ b/frontend/javascripts/oxalis/controller/scene_controller.ts
@@ -201,7 +201,7 @@ class SceneController {
     const meshMaterial = new THREE.MeshLambertMaterial({
       color,
     });
-    meshMaterial.side = THREE.DoubleSide;
+    meshMaterial.side = THREE.FrontSide;
     meshMaterial.transparent = true;
     const mesh = new THREE.Mesh(geometry, meshMaterial);
     mesh.castShadow = true;

From 8d8da9011c237510bc20bef28cd5e34a4ef0cc86 Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Mon, 20 Jun 2022 16:25:25 +0200
Subject: [PATCH 121/122] disable undo/redo during proofreading and show
 warning toast

---
 frontend/javascripts/messages.ts                    |  3 +++
 .../oxalis/model/actions/save_actions.ts            |  7 +++----
 .../javascripts/oxalis/model/sagas/save_saga.ts     | 13 +++++++++++--
 3 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/frontend/javascripts/messages.ts b/frontend/javascripts/messages.ts
index 8ad80aa6a96..ac3cd06807c 100644
--- a/frontend/javascripts/messages.ts
+++ b/frontend/javascripts/messages.ts
@@ -113,6 +113,9 @@ In order to restore the current window, a reload is necessary.`,
   "undo.no_undo":
     "There is no action that could be undone. However, if you want to restore an earlier version of this annotation, use the 'Restore Older Version' functionality in the dropdown next to the 'Save' button.",
   "undo.no_redo": "There is no action that could be redone.",
+  "undo.no_undo_during_proofread":
+    "Undo is not supported during proofreading yet. Please manually revert the last action you took.",
+  "undo.no_redo_during_proofread": "Redo is not supported during proofreading yet.",
   "undo.import_volume_tracing":
     "Importing a volume annotation cannot be undone. However, if you want to restore an earlier version of this annotation, use the 'Restore Older Version' functionality in the dropdown next to the 'Save' button.",
   "download.wait": "Please wait...",
diff --git a/frontend/javascripts/oxalis/model/actions/save_actions.ts b/frontend/javascripts/oxalis/model/actions/save_actions.ts
index b7f15c18b57..4810c2b4a65 100644
--- a/frontend/javascripts/oxalis/model/actions/save_actions.ts
+++ b/frontend/javascripts/oxalis/model/actions/save_actions.ts
@@ -1,3 +1,4 @@
+import type { Dispatch } from "redux";
 import type { UpdateAction } from "oxalis/model/sagas/update_actions";
 import { getUid } from "libs/uid_generator";
 import Date from "libs/date";
@@ -127,15 +128,13 @@ export const redoAction = (callback?: () => void): RedoAction => ({
 export const disableSavingAction = (): DisableSavingAction => ({
   type: "DISABLE_SAVING",
 });
-// Unfortunately, using type Dispatch produces countless Flow errors.
-export const dispatchUndoAsync = async (dispatch: (arg0: any) => any): Promise<void> => {
+export const dispatchUndoAsync = async (dispatch: Dispatch<any>): Promise<void> => {
   const readyDeferred = new Deferred();
   const action = undoAction(() => readyDeferred.resolve(null));
   dispatch(action);
   await readyDeferred.promise();
 };
-// Unfortunately, using type Dispatch produces countless Flow errors.
-export const dispatchRedoAsync = async (dispatch: (arg0: any) => any): Promise<void> => {
+export const dispatchRedoAsync = async (dispatch: Dispatch<any>): Promise<void> => {
   const readyDeferred = new Deferred();
   const action = redoAction(() => readyDeferred.resolve(null));
   dispatch(action);
diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
index 80bebb42167..2c15cc396f5 100644
--- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts
@@ -56,8 +56,7 @@ import {
 } from "oxalis/model/actions/save_actions";
 import type { UpdateAction } from "oxalis/model/sagas/update_actions";
 import { updateTdCamera } from "oxalis/model/sagas/update_actions";
-import type { Vector4 } from "oxalis/constants";
-import { ControlModeEnum } from "oxalis/constants";
+import { AnnotationToolEnum, type Vector4, ControlModeEnum } from "oxalis/constants";
 import { ViewModeSaveRelevantActions } from "oxalis/model/actions/view_mode_actions";
 import {
   actionChannel,
@@ -529,6 +528,16 @@ function* applyStateOfStack(
     return;
   }
 
+  const activeTool = yield* select((state) => state.uiInformation.activeTool);
+  if (activeTool === AnnotationToolEnum.PROOFREAD) {
+    const warningMessage =
+      direction === "undo"
+        ? messages["undo.no_undo_during_proofread"]
+        : messages["undo.no_redo_during_proofread"];
+    Toast.warning(warningMessage);
+    return;
+  }
+
   const busyBlockingInfo = yield* select((state) => state.uiInformation.busyBlockingInfo);
 
   if (busyBlockingInfo.isBusy) {

From d664624c2acef591cf15dfb6a2008d083111745b Mon Sep 17 00:00:00 2001
From: Daniel Werner <daniel.werner@scalableminds.com>
Date: Mon, 20 Jun 2022 16:55:51 +0200
Subject: [PATCH 122/122] warn if it looks like the user wanted to proofread
 but the proofread tool was not active

---
 .../oxalis/model/sagas/proofread_saga.ts          | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
index 82c2eae893f..f17f9220e79 100644
--- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
+++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts
@@ -256,14 +256,25 @@ function* splitOrMergeAgglomerate(action: MergeTreesAction | DeleteEdgeAction) {
   if (!allowUpdate) return;
 
   const activeTool = yield* select((state) => state.uiInformation.activeTool);
-  if (activeTool !== AnnotationToolEnum.PROOFREAD) return;
-
   const volumeTracingLayer = yield* select((state) => getActiveSegmentationTracingLayer(state));
   if (volumeTracingLayer == null) return;
   const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state));
   if (volumeTracing == null) return;
   const { tracingId: volumeTracingId } = volumeTracing;
 
+  if (activeTool !== AnnotationToolEnum.PROOFREAD) {
+    // Warn the user if an editable mapping is active and an agglomerate skeleton edge was added/deleted,
+    // but the proofreading mode was not active
+    if (volumeTracing.mappingIsEditable) {
+      Toast.warning(
+        "In order to edit the active mapping by deleting or adding edges, the proofreading tool needs to be active." +
+          " If you want your last action to edit the active mapping, undo it (Ctrl + Z), activate the proofreading tool and then manually redo the action.",
+        { timeout: 12000 },
+      );
+    }
+    return;
+  }
+
   const layerName = volumeTracingId;
   const isHdf5MappingEnabled = yield* call(ensureHdf5MappingIsEnabled, layerName);
   if (!isHdf5MappingEnabled) return;