feat: store blocks under multihash key

BREAKING CHANGE: repo.blocks.query() now returns multihashes as a key
instead of CID. If you want to have CID returned call it as query({},
true), which will constructs CIDv1 using IPLD's RAW codec. This means
that this constructed CID might not equal to the one that the block was originally
saved. Related to ipfs/js-ipfs#2415

Author: AuHau

README.md

@@ -197,6 +197,13 @@ Get block.
 
 * `cid` is the content id of [type CID](https://github.com/ipld/js-cid#readme).
 
+#### `Promise<Array<Object>> repo.blocks.query (query, reconstructsCids)`
+
+Query what blocks are available in blockstore.
+
+* `query` is a object as specified in [interface-datastore](https://github.com/ipfs/interface-datastore#query).
+* `reconstructsCids` a flag defining if the block's key is a reconstructed CID (eq. CIDv1 with RAW IPLD codec) or multihash
+
 Datastore:
 
 #### `repo.datastore`

src/blockstore-utils.js

@@ -12,18 +12,53 @@ const CID = require('cids')
  */
 exports.cidToKey = cid => {
   const enc = new base32.Encoder()
-  return new Key('/' + enc.write(cid.buffer).finalize(), false)
+  return new Key('/' + enc.write(cid.multihash).finalize(), false)
 }
 
 /**
  * Transform a datastore Key instance to a CID
+ * As Key is a multihash of the CID, it is reconstructed using IPLD's RAW codec.
+ * Hence it is highly probable that stored CID will differ from a CID retrieved from blockstore.
  *
  * @param {Key} key
  * @returns {CID}
  */
-exports.keyToCid = key => {
+
+function keyToCid (key) {
+  // Block key is of the form /<base32 encoded string>
+  const decoder = new base32.Decoder()
+  const buff = decoder.write(key.toString().slice(1)).finalize()
+  return new CID(1, 'raw', Buffer.from(buff))
+}
+
+exports.keyToCid = keyToCid
+
+/**
+ * Transform a datastore Key instance to a multihash instance.
+ *
+ * @param {Key} key
+ * @returns {Buffer}
+ */
+
+function keyToMultihash (key) {
   // Block key is of the form /<base32 encoded string>
   const decoder = new base32.Decoder()
   const buff = decoder.write(key.toString().slice(1)).finalize()
-  return new CID(Buffer.from(buff))
+  return Buffer.from(buff)
+}
+
+exports.keyToMultihash = keyToMultihash
+
+/**
+ * Transforms a datastore Key containing multihash to a Key that contains reconstructed CID
+ *
+ * @param {Key} key
+ * @returns {CID}
+ */
+function keyToCidKey (key) {
+  const cid = keyToCid(key)
+  const enc = new base32.Encoder()
+  return new Key('/' + enc.write(cid.buffer).finalize(), false)
 }
+
+exports.keyToCidKey = keyToCidKey

src/blockstore.js

@@ -5,7 +5,7 @@ const ShardingStore = core.ShardingDatastore
 const Block = require('ipfs-block')
 const CID = require('cids')
 const errcode = require('err-code')
-const { cidToKey } = require('./blockstore-utils')
+const { cidToKey, keyToCidKey } = require('./blockstore-utils')
 
 module.exports = async (filestore, options) => {
   const store = await maybeWithSharding(filestore, options)
@@ -26,10 +26,15 @@ function createBaseStore (store) {
      * Query the store.
      *
      * @param {object} query
+     * @param {boolean} reconstructsCids - Defines if Keys are converted to a reconstructed CID using IPLD_RAW codec
      * @return {Iterable}
      */
-    async * query (query) {
+    async * query (query, reconstructsCids = false) {
       for await (const block of store.query(query)) {
+        if (reconstructsCids) {
+          block.key = keyToCidKey(block.key)
+        }
+
         yield block
       }
     },
@@ -44,26 +49,8 @@ function createBaseStore (store) {
         throw errcode(new Error('Not a valid cid'), 'ERR_INVALID_CID')
       }
       const key = cidToKey(cid)
-      let blockData
-      try {
-        blockData = await store.get(key)
-        return new Block(blockData, cid)
-      } catch (err) {
-        if (err.code === 'ERR_NOT_FOUND') {
-          const otherCid = cidToOtherVersion(cid)
-
-          if (!otherCid) {
-            throw err
-          }
-
-          const otherKey = cidToKey(otherCid)
-          const blockData = await store.get(otherKey)
-          await store.put(key, blockData)
-          return new Block(blockData, cid)
-        }
-
-        throw err
-      }
+      const blockData = await store.get(key)
+      return new Block(blockData, cid)
     },
     /**
      * Write a single block to the store.
@@ -112,18 +99,14 @@ function createBaseStore (store) {
      * Does the store contain block with this cid?
      *
      * @param {CID} cid
-     * @returns {Promise<bool>}
+     * @returns {Promise<boolean>}
      */
-    async has (cid) {
+    has (cid) {
       if (!CID.isCID(cid)) {
         throw errcode(new Error('Not a valid cid'), 'ERR_INVALID_CID')
       }
 
-      const exists = await store.has(cidToKey(cid))
-      if (exists) return exists
-      const otherCid = cidToOtherVersion(cid)
-      if (!otherCid) return false
-      return store.has(cidToKey(otherCid))
+      return store.has(cidToKey(cid))
     },
     /**
      * Delete a block from the store
@@ -147,11 +130,3 @@ function createBaseStore (store) {
     }
   }
 }
-
-function cidToOtherVersion (cid) {
-  try {
-    return cid.version === 0 ? cid.toV1() : cid.toV0()
-  } catch (err) {
-    return null
-  }
-}

src/index.js

@@ -277,11 +277,11 @@ class IpfsRepo {
     let count = new Big(0)
     let size = new Big(0)
 
-    for await (const block of this.blocks.query({})) {
+    for await (const block of this.blocks.query({}, false)) {
       count = count.plus(1)
       size = size
         .plus(block.value.byteLength)
-        .plus(block.key._buf.byteLength)
+        .plus(block.key.toBuffer().byteLength)
     }
 
     return { count, size }
@@ -292,7 +292,7 @@ async function getSize (queryFn) {
   let sum = new Big(0)
   for await (const block of queryFn.query({})) {
     sum.plus(block.value.byteLength)
-      .plus(block.key._buf.byteLength)
+      .plus(block.key.toBuffer().byteLength)
   }
   return sum
 }

test/blockstore-test.js

@@ -85,9 +85,11 @@ module.exports = (repo) => {
               close () {
 
               }
+
               has () {
                 return true
               }
+
               batch () {
                 return {
                   put () {
@@ -217,6 +219,7 @@ module.exports = (repo) => {
               close () {
 
               }
+
               get (c) {
                 if (c.toString() === key.toString()) {
                   throw err

test/blockstore-utils-test.js

@@ -11,7 +11,8 @@ const Repo = require('../src')
 module.exports = () => {
   describe('blockstore utils', () => {
     it('converts a CID to a datastore Key and back', () => {
-      const originalCid = new CID('Qme6KJdKcp85TYbLxuLV7oQzMiLremD7HMoXLZEmgo6Rnh')
+      // CIDv1 in base32 with IPLD raw codec
+      const originalCid = new CID('bafkreihkb3vrxxex5zvzkr3s3a6noe223r7jka4ofjy2nkzu27kueg76ii')
       const key = Repo.utils.blockstore.cidToKey(originalCid)
       expect(key instanceof Key).to.be.true()
       const cid = Repo.utils.blockstore.keyToCid(key)