feat(gateway): range and conditional requests

- Switched from deprecated `hapi` and `joi` to `@hapi/hapi` and `@hapi/joi`
- Added support for Conditional Requests (RFC7232)
  - Returning `304 Not Modified` if `If-None-Match` is a CID matching `Etag`
  - Added `Last-Modified` to `/ipfs/` responses (improves client-side caching)
  - Always returning `304 Not Modified`
    when `If-Modified-Since` is present for immutable `/ipfs/`
- Added support for Byte Range requests (RFC7233, Section-2.1)
- Added support for `?filename=` parameter (improves downloads of raw cids)

License: MIT
Signed-off-by: Marcin Rataj <[email protected]>

Author: lidel

Diff for: package.json

@@ -82,6 +82,9 @@
     "stream-to-promise": "^2.2.0"
   },
   "dependencies": {
+    "@hapi/ammo": "^3.1.0",
+    "@hapi/hapi": "^18.3.1",
+    "@hapi/joi": "^15.0.0",
     "async": "^2.6.1",
     "bignumber.js": "^8.0.2",
     "binary-querystring": "~0.1.2",
@@ -103,7 +106,6 @@
     "fsm-event": "^2.1.0",
     "get-folder-size": "^2.0.0",
     "glob": "^7.1.3",
-    "hapi": "^18.0.0",
     "hapi-pino": "^5.2.0",
     "human-to-milliseconds": "^1.0.0",
     "interface-datastore": "~0.6.0",
@@ -131,7 +133,6 @@
     "is-pull-stream": "~0.0.0",
     "is-stream": "^1.1.0",
     "iso-url": "~0.4.6",
-    "joi": "^14.3.0",
     "just-flatten-it": "^2.1.0",
     "just-safe-set": "^2.1.0",
     "libp2p": "~0.25.0",

Diff for: src/http/api/resources/bitswap.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const Joi = require('joi')
+const Joi = require('@hapi/joi')
 const multibase = require('multibase')
 const { cidToString } = require('../../../utils/cid')
 const { parseKey } = require('./block')

Diff for: src/http/api/resources/block.js

@@ -2,7 +2,7 @@
 
 const CID = require('cids')
 const multipart = require('ipfs-multipart')
-const Joi = require('joi')
+const Joi = require('@hapi/joi')
 const multibase = require('multibase')
 const Boom = require('boom')
 const { cidToString } = require('../../../utils/cid')

Diff for: src/http/api/resources/dag.js

@@ -4,7 +4,7 @@ const promisify = require('promisify-es6')
 const CID = require('cids')
 const multipart = require('ipfs-multipart')
 const mh = require('multihashes')
-const Joi = require('joi')
+const Joi = require('@hapi/joi')
 const multibase = require('multibase')
 const Boom = require('boom')
 const debug = require('debug')

Diff for: src/http/api/resources/dht.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const Joi = require('joi')
+const Joi = require('@hapi/joi')
 const Boom = require('boom')
 
 const CID = require('cids')

Diff for: src/http/api/resources/files-regular.js

@@ -10,7 +10,7 @@ const toPull = require('stream-to-pull-stream')
 const pushable = require('pull-pushable')
 const toStream = require('pull-stream-to-stream')
 const abortable = require('pull-abortable')
-const Joi = require('joi')
+const Joi = require('@hapi/joi')
 const Boom = require('boom')
 const ndjson = require('pull-ndjson')
 const { PassThrough } = require('readable-stream')

Diff for: src/http/api/resources/name.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const Joi = require('joi')
+const Joi = require('@hapi/joi')
 
 exports.resolve = {
   validate: {

Diff for: src/http/api/resources/object.js

@@ -8,7 +8,7 @@ const { DAGNode, DAGLink } = dagPB
 const calculateCid = promisify(dagPB.util.cid)
 const deserialize = promisify(dagPB.util.deserialize)
 const createDagNode = promisify(DAGNode.create)
-const Joi = require('joi')
+const Joi = require('@hapi/joi')
 const multibase = require('multibase')
 const Boom = require('boom')
 const { cidToString } = require('../../../utils/cid')

Diff for: src/http/api/resources/pin.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const multibase = require('multibase')
-const Joi = require('joi')
+const Joi = require('@hapi/joi')
 const Boom = require('boom')
 const isIpfs = require('is-ipfs')
 const { cidToString } = require('../../../utils/cid')

Diff for: src/http/api/resources/ping.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const Joi = require('joi')
+const Joi = require('@hapi/joi')
 const pull = require('pull-stream')
 const ndjson = require('pull-ndjson')
 const { PassThrough } = require('readable-stream')

Diff for: src/http/api/resources/resolve.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const Joi = require('joi')
+const Joi = require('@hapi/joi')
 const debug = require('debug')
 const multibase = require('multibase')
 

Diff for: src/http/gateway/resources/gateway.js

@@ -8,6 +8,7 @@ const fileType = require('file-type')
 const mime = require('mime-types')
 const { PassThrough } = require('readable-stream')
 const Boom = require('boom')
+const Ammo = require('@hapi/ammo') // HTTP Range processing utilities
 const peek = require('buffer-peek-stream')
 
 const { resolver } = require('ipfs-http-response')
@@ -98,7 +99,47 @@ module.exports = {
       return h.redirect(PathUtils.removeTrailingSlash(ref)).permanent(true)
     }
 
-    const rawStream = ipfs.catReadableStream(data.cid)
+    // Support If-None-Match & Etag (Conditional Requests from RFC7232)
+    // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
+    const etag = `"${data.cid}"`
+    const cachedEtag = request.headers['if-none-match']
+    if (cachedEtag === etag || cachedEtag === `W/${etag}`) {
+      return h.response().code(304) // Not Modified
+    }
+
+    // Immutable content produces 304 Not Modified for all values of If-Modified-Since
+    if (ref.startsWith('/ipfs/') && request.headers['if-modified-since']) {
+      return h.response().code(304) // Not Modified
+    }
+
+    // This necessary to set correct Content-Length and validate Range requests
+    // Note: we need `size` (raw data), not `cumulativeSize` (data + DAGNodes)
+    const { size } = await ipfs.files.stat(`/ipfs/${data.cid}`)
+
+    // Handle Byte Range requests (https://tools.ietf.org/html/rfc7233#section-2.1)
+    const catOptions = {}
+    let rangeResponse = false
+    if (request.headers.range) {
+      // If-Range is respected (when present), but we compare it only against Etag
+      // (Last-Modified date is too weak for IPFS use cases)
+      if (!request.headers['if-range'] || request.headers['if-range'] === etag) {
+        const ranges = Ammo.header(request.headers.range, size)
+        if (!ranges) {
+          const error = Boom.rangeNotSatisfiable()
+          error.output.headers['content-range'] = `bytes */${size}`
+          throw error
+        }
+
+        if (ranges.length === 1) { // Ignore requests for multiple ranges (hard to map to ipfs.cat and not used in practice)
+          rangeResponse = true
+          const range = ranges[0]
+          catOptions.offset = range.from
+          catOptions.length = (range.to - range.from + 1)
+        }
+      }
+    }
+
+    const rawStream = ipfs.catReadableStream(data.cid, catOptions)
     const responseStream = new ResponseStream()
 
     // Pass-through Content-Type sniffing over initial bytes
@@ -119,10 +160,11 @@ module.exports = {
       }
     })
 
-    const res = h.response(responseStream)
+    const res = h.response(responseStream).code(rangeResponse ? 206 : 200)
 
     // Etag maps directly to an identifier for a specific version of a resource
-    res.header('Etag', `"${data.cid}"`)
+    // and enables smart client-side caching thanks to If-None-Match
+    res.header('etag', etag)
 
     // Set headers specific to the immutable namespace
     if (ref.startsWith('/ipfs/')) {
@@ -137,15 +179,38 @@ module.exports = {
       res.header('Content-Type', contentType)
     }
 
+    if (rangeResponse) {
+      const from = catOptions.offset
+      const to = catOptions.offset + catOptions.length - 1
+      res.header('Content-Range', `bytes ${from}-${to}/${size}`)
+      res.header('Content-Length', catOptions.length)
+    } else {
+      // Announce support for Range requests
+      res.header('Accept-Ranges', 'bytes')
+      res.header('Content-Length', size)
+    }
+
+    // Support Content-Disposition via ?filename=foo parameter
+    // (useful for browser vendor to download raw CID into custom filename)
+    // Source: https://github.com/ipfs/go-ipfs/blob/v0.4.20/core/corehttp/gateway_handler.go#L232-L236
+    if (request.query.filename) {
+      res.header('Content-Disposition', `inline; filename*=UTF-8''${encodeURIComponent(request.query.filename)}`)
+    }
+
     return res
   },
 
   afterHandler (request, h) {
     const { response } = request
-    if (response.statusCode === 200) {
+    // Add headers to successfult responses (regular or range)
+    if (response.statusCode === 200 || response.statusCode === 206) {
       const { ref } = request.pre.args
       response.header('X-Ipfs-Path', ref)
       if (ref.startsWith('/ipfs/')) {
+        // "set modtime to a really long time ago, since files are immutable and should stay cached"
+        // Source: https://github.com/ipfs/go-ipfs/blob/v0.4.20/core/corehttp/gateway_handler.go#L228-L229
+        response.header('Last-Modified', 'Thu, 01 Jan 1970 00:00:01 GMT')
+        // Suborigins: https://github.com/ipfs/in-web-browsers/issues/66
         const rootCid = ref.split('/')[2]
         const ipfsOrigin = cidToString(rootCid, { base: 'base32' })
         response.header('Suborigin', 'ipfs000' + ipfsOrigin)

Diff for: src/http/gateway/routes/gateway.js

@@ -10,6 +10,9 @@ module.exports = {
     pre: [
       { method: resources.gateway.checkCID, assign: 'args' }
     ],
+    response: {
+      ranges: false // disable built-in support, we do it manually
+    },
     ext: {
       onPostHandler: { method: resources.gateway.afterHandler }
     }

Diff for: src/http/index.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const Hapi = require('hapi')
+const Hapi = require('@hapi/hapi')
 const Pino = require('hapi-pino')
 const debug = require('debug')
 const multiaddr = require('multiaddr')