Merge pull request #88 from input-output-hk/slicedavltree-rework

New features and improvements in sliced authenticated AVL+ trees

Author: kushti

README.md

@@ -18,7 +18,7 @@ resolvers += "Sonatype Releases" at "https://oss.sonatype.org/content/repositori
 
 You can use Scrypto in your sbt project by simply adding the following dependency to your build file:
 ```scala
-libraryDependencies += "org.scorexfoundation" %% "scrypto" % "2.1.6"
+libraryDependencies += "org.scorexfoundation" %% "scrypto" % "2.2.0"
 ```
 
 ### Hash functions

build.sbt

@@ -35,7 +35,7 @@ libraryDependencies ++= Seq(
   "com.google.guava" % "guava" % "23.0",
   "com.typesafe.scala-logging" %% "scala-logging" % "3.9.2",
   "org.whispersystems" % "curve25519-java" % "0.5.0",
-  "org.bouncycastle" % "bcprov-jdk15on" % "1.66",
+  "org.bouncycastle" % "bcprov-jdk15to18" % "1.66",
   "org.scorexfoundation" %% "scorex-util" % "0.1.8"
 )
 

release-notes.md

@@ -1,3 +1,27 @@
+**2.2.0**
+---------
+
+* Rework of sliced AVL+ trees (BatchAVLProverManifest / BatchAVLProverSubtree)
+* Batch Merkle proof implementation
+* AuthenticationTreeOps.logError 
+* Scala, scorex-util, Guava, BouncyCastle dependencies updated
+* example app in SparseMarkleTree reworked into tests (#38)
+* switch from sbt-git to sbt-dynver
+* migration from Travis to GA
+
+
+**2.1.8**
+---------
+
+* Guava's comparator for byte arrays is used instead of custom old one ( #74 )
+* Scala 2.13 support ( #75 )
+
+**2.1.7**
+---------
+
+* remove logback dependency, add sbt-dependency-graph plugin
+* add git versioning, cross-build with scala 2.11, auto sonatype publishing(WIP); remove sbt-lock
+
 **2.1.6**
 ---------
 

src/main/scala/scorex/crypto/authds/avltree/batch/BatchNode.scala

@@ -11,6 +11,9 @@ sealed trait Node[D <: Digest] extends ToStringHelper {
 
   protected var labelOpt: Option[D] = None
 
+  /**
+    * Get digest of the node. If it was computed previously, read the digest from hash, otherwise, compute it.
+    */
   def label: D = labelOpt match {
     case None =>
       val l = computeLabel
@@ -34,11 +37,15 @@ class LabelOnlyNode[D <: Digest](l: D) extends VerifierNodes[D] {
 }
 
 sealed trait InternalNode[D <: Digest] extends Node[D] {
+  import InternalNode.InternalNodePrefix
+
   protected var b: Balance
 
   protected val hf: CryptographicHash[D]
 
-  protected def computeLabel: D = hf.prefixedHash(1: Byte, Array(b), left.label, right.label)
+  override protected def computeLabel: D = {
+    hf.hash(Array(InternalNodePrefix, b), left.label, right.label)
+  }
 
   def balance: Balance = b
 
@@ -52,6 +59,10 @@ sealed trait InternalNode[D <: Digest] extends Node[D] {
   def getNewKey(newKey: ADKey): InternalNode[D]
 }
 
+object InternalNode {
+  val InternalNodePrefix: Byte = 1: Byte
+}
+
 class InternalProverNode[D <: Digest](protected var k: ADKey,
                                       protected var l: ProverNodes[D],
                                       protected var r: ProverNodes[D],
@@ -120,23 +131,31 @@ class InternalVerifierNode[D <: Digest](protected var l: Node[D], protected var
 }
 
 sealed trait Leaf[D <: Digest] extends Node[D] with KeyInVar {
+  import Leaf.LeafPrefix
+
   protected var nk: ADKey
   protected var v: ADValue
 
-
   def nextLeafKey: ADKey = nk
 
   def value: ADValue = v
 
   protected val hf: CryptographicHash[D] // TODO: Seems very wasteful to store hf in every node of the tree, when they are all the same. Is there a better way? Pass them in to label method from above? Same for InternalNode and for other, non-batch, trees
 
-  protected def computeLabel: D = hf.prefixedHash(0: Byte, k, v, nk)
+  protected def computeLabel: D = {
+    hf.prefixedHash(LeafPrefix, k, v, nk)
+  }
 
   def getNew(newKey: ADKey = k, newValue: ADValue = v, newNextLeafKey: ADKey = nk): Leaf[D]
 
   override def toString: String = {
     s"${arrayToString(label)}: Leaf(${arrayToString(key)}, ${arrayToString(value)}, ${arrayToString(nextLeafKey)})"
   }
+
+}
+
+object Leaf {
+  val LeafPrefix = 0: Byte
 }
 
 class VerifierLeaf[D <: Digest](protected var k: ADKey, protected var v: ADValue, protected var nk: ADKey)

src/main/scala/scorex/crypto/authds/avltree/batch/serialization/BatchAVLProverManifest.scala

@@ -1,12 +1,46 @@
 package scorex.crypto.authds.avltree.batch.serialization
 
-import scorex.crypto.authds.avltree.batch.ProverNodes
+import scorex.crypto.authds.avltree.batch.{InternalProverNode, ProverLeaf, ProverNodes}
 import scorex.crypto.hash.Digest
 
+import scala.collection.mutable
+
 
 /**
-  * Top subtree of AVL tree, starting from root node and ending with ProxyInternalNode
+  * A subtree of AVL tree, which is starting from root node and ending at certain depth with nodes
+  * having no children (ProxyInternalNode). The manifest commits to subtrees below the depth.
   */
-case class BatchAVLProverManifest[D <: Digest](keyLength: Int,
-                                               valueLengthOpt: Option[Int],
-                                               rootAndHeight: (ProverNodes[D], Int))
+case class BatchAVLProverManifest[D <: Digest](root: ProverNodes[D], rootHeight: Int) {
+
+  /**
+    * Unique (and cryptographically strong) identifier of the manifest (digest of the root node)
+    */
+  def id: D = root.label
+
+  /**
+    * Verify that manifest corresponds to expected digest and height provided by a trusted party
+    * (for blockchain protocols, it can be digest and height included by a miner)
+    */
+  def verify(expectedDigest: D, expectedHeight: Int): Boolean = {
+    id.sameElements(expectedDigest) && expectedHeight == rootHeight
+  }
+
+  /**
+    * Identifiers (digests) of subtrees below the manifest
+    */
+  def subtreesIds: mutable.Buffer[D] = {
+    def idCollector(node: ProverNodes[D], acc: mutable.Buffer[D]): mutable.Buffer[D] = {
+      node match {
+        case n: ProxyInternalNode[D] if n.isEmpty =>
+          (acc += n.leftLabel) += n.rightLabel
+        case i : InternalProverNode[D] =>
+          idCollector(i.right, idCollector(i.left, acc))
+        case _: ProverLeaf[D] =>
+          acc
+      }
+    }
+
+    idCollector(root, mutable.Buffer.empty)
+  }
+
+}

src/main/scala/scorex/crypto/authds/avltree/batch/serialization/BatchAVLProverSerializer.scala

@@ -13,8 +13,6 @@ class BatchAVLProverSerializer[D <: Digest, HF <: CryptographicHash[D]](implicit
 
   type SlicedTree = (BatchAVLProverManifest[D], Seq[BatchAVLProverSubtree[D]])
 
-  def slice(tree: BatchAVLProver[D, HF]): SlicedTree = slice(tree, tree.rootNodeHeight / 2)
-
   /**
     * Slice AVL tree to top subtree tree (BatchAVLProverManifest) and
     * bottom subtrees (BatchAVLProverSubtree) with height `subtreeDepth`
@@ -31,87 +29,88 @@ class BatchAVLProverSerializer[D <: Digest, HF <: CryptographicHash[D]](implicit
         currentNode match {
           case n: InternalProverNode[D] if currentHeight > subtreeDepth =>
             val nextParent = ProxyInternalNode(n)
-            parent.mutate(nextParent)
+            parent.setChild(nextParent)
             val leftSubtrees = getSubtrees(n.left, currentHeight - 1, nextParent)
             val rightSubtrees = getSubtrees(n.right, currentHeight - 1, nextParent)
             leftSubtrees ++ rightSubtrees
           case n: InternalProverNode[D] =>
-            parent.mutate(ProxyInternalNode(n))
+            parent.setChild(ProxyInternalNode(n))
             Seq(BatchAVLProverSubtree(n.left), BatchAVLProverSubtree(n.right))
           case l: ProverLeaf[D] =>
-            parent.mutate(l)
+            parent.setChild(l)
             Seq(BatchAVLProverSubtree(l))
         }
       }
 
       val subtrees = getSubtrees(tn.left, height - 1, rootProxyNode) ++ getSubtrees(tn.right, height - 1, rootProxyNode)
-      val manifest = BatchAVLProverManifest[D](tree.keyLength, tree.valueLengthOpt, (rootProxyNode, height))
+      val manifest = BatchAVLProverManifest[D](rootProxyNode, height)
       (manifest, subtrees)
     case l: ProverLeaf[D] =>
-      (BatchAVLProverManifest[D](tree.keyLength, tree.valueLengthOpt, (l, tree.rootNodeHeight)), Seq.empty)
+      (BatchAVLProverManifest[D](l, tree.rootNodeHeight), Seq.empty)
   }
 
   /**
     * Combine tree pieces into one big tree
     */
-  def combine(sliced: SlicedTree): Try[BatchAVLProver[D, HF]] = Try {
-    sliced._1.rootAndHeight._1 match {
+  def combine(sliced: SlicedTree,
+              keyLength: Int,
+              valueLengthOpt: Option[Int]): Try[BatchAVLProver[D, HF]] = Try {
+    val manifest = sliced._1
+    manifest.root match {
       case tn: InternalProverNode[D] =>
+
+        // manifest being mutated here
         def mutateLoop(n: ProverNodes[D]): Unit = n match {
           case n: ProxyInternalNode[D] if n.isEmpty =>
-            val left = sliced._2.find(_.subtreeTop.label sameElements n.leftLabel).get.subtreeTop
-            val right = sliced._2.find(_.subtreeTop.label sameElements n.rightLabel).get.subtreeTop
-            n.mutate(left)
-            n.mutate(right)
+            val left = sliced._2.find(_.id sameElements n.leftLabel).get.subtreeTop
+            val right = sliced._2.find(_.id sameElements n.rightLabel).get.subtreeTop
+            n.setChild(left)
+            n.setChild(right)
           case n: InternalProverNode[D] =>
             mutateLoop(n.left)
             mutateLoop(n.right)
           case _ =>
         }
 
         mutateLoop(tn)
-        new BatchAVLProver[D, HF](sliced._1.keyLength, sliced._1.valueLengthOpt, Some(sliced._1.rootAndHeight))
       case _: ProverLeaf[D] =>
-        new BatchAVLProver[D, HF](sliced._1.keyLength, sliced._1.valueLengthOpt, Some(sliced._1.rootAndHeight))
     }
+
+    new BatchAVLProver[D, HF](keyLength, valueLengthOpt, Some(manifest.root -> manifest.rootHeight))
   }
 
   def manifestToBytes(manifest: BatchAVLProverManifest[D]): Array[Byte] = {
-    Bytes.concat(Ints.toByteArray(manifest.keyLength),
-      Ints.toByteArray(manifest.valueLengthOpt.getOrElse(-1)),
-      Ints.toByteArray(manifest.rootAndHeight._2),
-      nodesToBytes(manifest.rootAndHeight._1)
+    Bytes.concat(
+      Ints.toByteArray(manifest.rootHeight),
+      nodesToBytes(manifest.root)
     )
   }
 
-  def manifestFromBytes(bytes: Array[Byte]): Try[BatchAVLProverManifest[D]] = Try {
-    val keyLength = Ints.fromByteArray(bytes.slice(0, 4))
-    val valueLength = Ints.fromByteArray(bytes.slice(4, 8))
-    if (valueLength < -1) throw new Error(s"Wrong valueLength: $valueLength")
-    val valueLengthOpt = if (valueLength == -1) None else Some(valueLength)
-    val oldHeight = Ints.fromByteArray(bytes.slice(8, 12))
-    val oldTop = nodesFromBytes(bytes.slice(12, bytes.length), keyLength).get
-    BatchAVLProverManifest[D](keyLength, valueLengthOpt, (oldTop, oldHeight))
+  def manifestFromBytes(bytes: Array[Byte],
+                        keyLength: Int): Try[BatchAVLProverManifest[D]] = Try {
+    val oldHeight = Ints.fromByteArray(bytes.slice(0, 4))
+    val oldTop = nodesFromBytes(bytes.slice(4, bytes.length), keyLength).get
+    BatchAVLProverManifest[D](oldTop, oldHeight)
   }
 
   def subtreeToBytes(t: BatchAVLProverSubtree[D]): Array[Byte] = nodesToBytes(t.subtreeTop)
 
   def subtreeFromBytes(b: Array[Byte], kl: Int): Try[BatchAVLProverSubtree[D]] = nodesFromBytes(b, kl).
     map(topNode => BatchAVLProverSubtree[D](topNode))
 
-  def nodesToBytes(obj: ProverNodes[D]): Array[Byte] = {
+  def nodesToBytes(rootNode: ProverNodes[D]): Array[Byte] = {
     def loop(currentNode: ProverNodes[D]): Array[Byte] = currentNode match {
       case l: ProverLeaf[D] =>
         Bytes.concat(Array(0.toByte), l.key, l.nextLeafKey, l.value)
       case n: ProxyInternalNode[D] if n.isEmpty =>
-        Bytes.concat(Array(2.toByte, n.balance), n.key, n.leftLabel, n.rightLabel, n.label)
+        Bytes.concat(Array(2.toByte, n.balance), n.key, n.leftLabel, n.rightLabel)
       case n: InternalProverNode[D] =>
         val leftBytes = loop(n.left)
         val rightBytes = loop(n.right)
         Bytes.concat(Array(1.toByte, n.balance), n.key, Ints.toByteArray(leftBytes.length), leftBytes, rightBytes)
     }
 
-    loop(obj)
+    loop(rootNode)
   }
 
   def nodesFromBytes(bytesIn: Array[Byte], keyLength: Int): Try[ProverNodes[D]] = Try {
@@ -135,8 +134,7 @@ class BatchAVLProverSerializer[D <: Digest, HF <: CryptographicHash[D]](implicit
         val key = ADKey @@ bytes.slice(2, keyLength + 2)
         val leftLabel = hf.byteArrayToDigest(bytes.slice(keyLength + 2, keyLength + 2 + labelLength)).get
         val rightLabel = hf.byteArrayToDigest(bytes.slice(keyLength + 2 + labelLength, keyLength + 2 + 2 * labelLength)).get
-        val selfLabel = hf.byteArrayToDigest(bytes.slice(keyLength + 2 + 2 * labelLength, keyLength + 2 + 3 * labelLength)).get
-        new ProxyInternalNode[D](key, Some(selfLabel), leftLabel, rightLabel, balance)
+        new ProxyInternalNode[D](key, leftLabel, rightLabel, balance)
       case _ =>
         ???
     }

src/main/scala/scorex/crypto/authds/avltree/batch/serialization/BatchAVLProverSubtree.scala

@@ -1,9 +1,41 @@
 package scorex.crypto.authds.avltree.batch.serialization
 
-import scorex.crypto.authds.avltree.batch.ProverNodes
+import scorex.crypto.authds.ADValue
+import scorex.crypto.authds.avltree.batch.{InternalProverNode, ProverLeaf, ProverNodes}
 import scorex.crypto.hash.Digest
 
+import scala.collection.mutable
+
 /**
-  * AVL subtree, starting from Manifests FinalInternalNode and ending with Leafs
+  * AVL subtree, starting from a manifest's terminal internal node and ending with Leafs
   */
-case class BatchAVLProverSubtree[D <: Digest](subtreeTop: ProverNodes[D])
+case class BatchAVLProverSubtree[D <: Digest](subtreeTop: ProverNodes[D]) {
+  /**
+    * Unique (and cryptographically strong) identifier of the sub-tree
+    */
+  def id: D = subtreeTop.label
+
+  /**
+    * Verify that manifest corresponds to expected digest (e.g. got from a manifest)
+    */
+  def verify(expectedDigest: D): Boolean = {
+    subtreeTop.label.sameElements(expectedDigest)
+  }
+
+  /**
+    * @return leafs of the subtree
+    */
+  def leafValues: mutable.Buffer[ADValue] = {
+    def idCollector(node: ProverNodes[D], acc: mutable.Buffer[ADValue]): mutable.Buffer[ADValue] = {
+      node match {
+        case i : InternalProverNode[D] =>
+          idCollector(i.right, idCollector(i.left, acc))
+        case l: ProverLeaf[D] =>
+          acc += l.value
+      }
+    }
+
+    idCollector(subtreeTop, mutable.Buffer.empty)
+  }
+
+}