Migration 8 (#4)

Migrates blockstore keys to store blocks by multihash instead of CID

Co-authored-by: achingbrain <[email protected]>

Author: AuHau

migrations/index.js

@@ -11,11 +11,12 @@ const emptyMigration = {
 }
 
 module.exports = [
-  Object.assign({}, emptyMigration, { version: 7, revert: undefined }),
-  Object.assign({}, emptyMigration, { version: 6, revert: undefined }),
-  Object.assign({}, emptyMigration, { version: 5, revert: undefined }),
-  Object.assign({}, emptyMigration, { version: 4, revert: undefined }),
-  Object.assign({}, emptyMigration, { version: 3, revert: undefined }),
-  Object.assign({}, emptyMigration, { version: 2, revert: undefined }),
-  Object.assign({}, emptyMigration, { version: 1, revert: undefined })
+  Object.assign({version: 1}, emptyMigration),
+  Object.assign({version: 2}, emptyMigration),
+  Object.assign({version: 3}, emptyMigration),
+  Object.assign({version: 4}, emptyMigration),
+  Object.assign({version: 5}, emptyMigration),
+  Object.assign({version: 6}, emptyMigration),
+  Object.assign({version: 7}, emptyMigration),
+  require('./migration-8')
 ]

migrations/migration-8/index.js

@@ -0,0 +1,88 @@
+'use strict'
+
+const path = require('path')
+const CID = require('cids')
+const Key = require('interface-datastore').Key
+const core = require('datastore-core')
+const ShardingStore = core.ShardingDatastore
+const mb = require('multibase')
+const utils = require('../../src/utils')
+const log = require('debug')('ipfs-repo-migrations:migration-8')
+
+// This function in js-ipfs-repo defaults to not using sharding
+// but the default value of the options.sharding is true hence this
+// function defaults to use sharding.
+async function maybeWithSharding (filestore, options) {
+  if (options.sharding === false) {
+    return filestore
+  }
+
+  const shard = new core.shard.NextToLast(2)
+
+  return ShardingStore.createOrOpen(filestore, shard)
+}
+
+function keyToMultihash (key) {
+  const buf = mb.decode(`b${key.toString().slice(1)}`)
+
+  // Extract multihash from CID
+  let multihash = new CID(buf).multihash
+
+  // Encode and slice off multibase codec
+  multihash = mb.encode('base32', multihash).slice(1)
+
+  // Should be uppercase for interop with go
+  multihash = multihash.toString().toUpperCase()
+
+  return new Key(`/${multihash}`, false)
+}
+
+function keyToCid (key) {
+  const buf = mb.decode(`b${key.toString().slice(1)}`)
+
+  // CID to Key
+  const multihash = mb.encode('base32', new CID(1, 'raw', buf).buffer).slice(1)
+
+  return new Key(`/${multihash}`.toUpperCase(), false)
+}
+
+async function process (repoPath, options, keyFunction){
+  const { StorageBackend, storageOptions } = utils.getDatastoreAndOptions(options, 'blocks')
+
+  const baseStore = new StorageBackend(path.join(repoPath, 'blocks'), storageOptions)
+  await baseStore.open()
+  const store = await maybeWithSharding(baseStore, storageOptions)
+  await store.open()
+
+  try {
+    let counter = 0
+
+    for await (const block of store.query({})) {
+      const newKey = keyFunction(block.key)
+
+      // If the Key is base32 CIDv0 then there's nothing to do
+      if(newKey.toString() !== block.key.toString()) {
+        counter += 1
+
+        log(`Migrating Block from ${block.key.toString()} to ${newKey.toString()}`)
+        await store.delete(block.key)
+        await store.put(newKey, block.value)
+      }
+    }
+
+    log(`Changed ${ counter } blocks`)
+  } finally {
+    await store.close()
+  }
+}
+
+module.exports = {
+  version: 8,
+  description: 'Transforms key names into base32 encoding and converts Block store to use bare multihashes encoded as base32',
+  migrate: (repoPath, options = {}) => {
+    return process(repoPath, options, keyToMultihash)
+  },
+  revert: (repoPath, options = {}) => {
+    return process(repoPath, options, keyToCid)
+  }
+}

package.json

@@ -20,7 +20,7 @@
   "main": "src/index.js",
   "browser": {
     "./src/repo/lock.js": "./src/repo/lock-memory.js",
-    "datastore-fs": "datastore-idb"
+    "datastore-fs": "datastore-level"
   },
   "bin": {
     "jsipfs-migrations": "./src/cli.js"
@@ -46,16 +46,19 @@
   "dependencies": {
     "buffer": "^5.6.0",
     "chalk": "^4.0.0",
+    "cids": "^0.8.3",
+    "datastore-core": "^1.1.0",
     "datastore-fs": "^1.0.0",
-    "datastore-idb": "^1.0.2",
+    "datastore-level": "^1.1.0",
     "debug": "^4.1.0",
-    "interface-datastore": "^1.0.4",
+    "interface-datastore": "^1.0.2",
+    "multibase": "^1.0.1",
     "proper-lockfile": "^4.1.1",
     "yargs": "^15.3.1",
     "yargs-promise": "^1.1.0"
   },
   "devDependencies": {
-    "aegir": "^25.0.0",
+    "aegir": "^23.0.0",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
     "dirty-chai": "^2.0.1",

test/browser.js

@@ -3,7 +3,7 @@
 
 const { Buffer } = require('buffer')
 const loadFixture = require('aegir/fixtures')
-const Datastore = require('datastore-idb')
+const Datastore = require('datastore-level')
 
 const Key = require('interface-datastore').Key
 const CONFIG_KEY = new Key('config')
@@ -44,6 +44,10 @@ describe('Browser specific tests', () => {
     require('./version-test')(createRepo, repoCleanup)
   })
 
+  describe('migrations tests', () => {
+    require('./migrations/migration-8-test')(createRepo, repoCleanup)
+  })
+
   describe('init tests', () => {
     require('./init-test')(createRepo, repoCleanup)
   })

test/migrations/migration-8-test.js

@@ -0,0 +1,77 @@
+/* eslint-env mocha */
+'use strict'
+
+const chai = require('chai')
+chai.use(require('dirty-chai'))
+const chaiAsPromised = require('chai-as-promised')
+chai.use(chaiAsPromised)
+const expect = chai.expect
+
+const path = require('path')
+const migration = require('../../migrations/migration-8')
+const Key = require('interface-datastore').Key
+const Datastore = require('datastore-fs')
+const core = require('datastore-core')
+const ShardingStore = core.ShardingDatastore
+
+const blocksFixtures = [
+  ['AFKREIBFG77IKIKDMBDUFDCSPK7H5TE5LNPMCSXYLPML27WSTT5YA5IUNU',
+    'CIQCKN76QUQUGYCHIKGFE6V6P3GJ2W26YFFPQW6YXV7NFHH3QB2RI3I']
+]
+
+async function bootstrapBlocks (dir, encoded) {
+  const baseStore = new Datastore(path.join(dir, 'blocks'), { extension: '.data', createIfMissing: true })
+  const shard = new core.shard.NextToLast(2)
+
+  await baseStore.open()
+  const store = await ShardingStore.createOrOpen(baseStore, shard)
+
+  let name
+  for (const blocksNames of blocksFixtures) {
+    name = encoded ? blocksNames[1] : blocksNames[0]
+    await store.put(new Key(name), '')
+  }
+
+  await store.close()
+}
+
+async function validateBlocks (dir, shouldBeEncoded) {
+  const baseStore = new Datastore(path.join(dir, 'blocks'), { extension: '.data', createIfMissing: false })
+  const shard = new core.shard.NextToLast(2)
+
+  await baseStore.open()
+  const store = await ShardingStore.createOrOpen(baseStore, shard)
+
+  let newName, oldName
+  for (const blockNames of blocksFixtures) {
+    newName = shouldBeEncoded ? blockNames[1] : blockNames[0]
+    oldName = shouldBeEncoded ? blockNames[0] : blockNames[1]
+    expect(await store.has(new Key(`/${oldName}`))).to.be.false(`${oldName} was not migrated to ${newName}`)
+    expect(await store.has(new Key(`/${newName}`))).to.be.true(`${newName} was not removed`)
+  }
+
+  await store.close()
+}
+
+module.exports = (setup, cleanup) => {
+  describe('migration 8', () => {
+    let dir
+
+    beforeEach(async () => {
+      dir = await setup()
+    })
+    afterEach(() => cleanup(dir))
+
+    it('should migrate blocks forward', async () => {
+      await bootstrapBlocks(dir, false)
+      await migration.migrate(dir)
+      await validateBlocks(dir, true)
+    })
+
+    it('should migrate blocks backward', async () => {
+      await bootstrapBlocks(dir, true)
+      await migration.revert(dir)
+      await validateBlocks(dir, false)
+    })
+  })
+}

test/node.js

@@ -43,6 +43,10 @@ describe('Node specific tests', () => {
     require('./version-test')(createRepo, repoCleanup)
   })
 
+  describe('migrations tests', () => {
+    require('./migrations/migration-8-test')(createRepo, repoCleanup)
+  })
+
   describe('init tests', () => {
     require('./init-test')(createRepo, repoCleanup)
   })