docs: add mfs stream ls methods

The only mfs ls method at the moment buffers the output into an array
before returning it to the user. This PR adds two new methods,
lsPullStream and lsReadableStream to allow the user to either buffer
the output themseleves (in case they need sorting, etc) or just
stream it on to an output of some sort.

N.b the http API will not actually do any streaming until ipfs/kubo#5611
is released.

Author: achingbrain

SPEC/FILES.md

@@ -22,6 +22,8 @@
   - [files.cp](#filescp)
   - [files.flush](#filesflush)
   - [files.ls](#filesls)
+  - [files.lsReadableStream](#fileslsreadablestream)
+  - [files.lsPullStream](#fileslspullstream)
   - [files.mkdir](#filesmkdir)
   - [files.mv](#filesmv)
   - [files.read](#filesread)
@@ -1090,6 +1092,7 @@ Where:
 - `options` is an optional Object that might contain the following keys:
   - `long` is a Boolean value to decide whether or not to populate `type`, `size` and `hash` (default: false)
   - `cidBase` is which number base to use to format hashes - e.g. `base32`, `base64` etc (default: `base58btc`)
+  - `sort` is a Boolean value, if true entries will be sorted by filename (default: false)
 - `callback` is an optional function with the signature `function (error, files) {}`, where `error` may be an Error that occured if the operation was not successful and `files` is an array containing Objects that contain the following keys:
 
   - `name` which is the file's name
@@ -1112,6 +1115,75 @@ ipfs.files.ls('/screenshots', function (err, files) {
 // 2018-01-22T18:08:49.184Z.png
 ```
 
+#### `files.lsReadableStream`
+
+> Lists a directory from the local mutable namespace that is addressed by a valid IPFS Path. The list will be yielded as Readable Streams.
+
+##### `Go` **WIP**
+
+##### `JavaScript` - ipfs.files.lsReadableStream([path], [options]) -> [Readable Stream][rs]
+
+Where:
+
+- `path` is an optional string to show listing for (default: `/`)
+- `options` is an optional Object that might contain the following keys:
+  - `long` is a Boolean value to decide whether or not to populate `type`, `size` and `hash` (default: false)
+  - `cidBase` is which number base to use to format hashes - e.g. `base32`, `base64` etc (default: `base58btc`)
+
+It returns a [Readable Stream][rs] in [Object mode](https://nodejs.org/api/stream.html#stream_object_mode) that will yield objects containing the following keys:
+
+  - `name` which is the file's name
+  - `type` which is the object's type (`directory` or `file`)
+  - `size` the size of the file in bytes
+  - `hash` the hash of the file
+
+**Example:**
+
+```JavaScript
+const stream = ipfs.lsReadableStream('/some-dir')
+
+stream.on('data', (file) => {
+  // write the file's path and contents to standard out
+  console.log(file.name)
+})
+```
+
+#### `files.lsPullStream`
+
+> Fetch a file or an entire directory tree from IPFS that is addressed by a valid IPFS Path. The files will be yielded through a Pull Stream.
+
+##### `Go` **WIP**
+
+##### `JavaScript` - ipfs.lsPullStream([path], [options]) -> [Pull Stream][ps]
+
+Where:
+
+- `path` is an optional string to show listing for (default: `/`)
+- `options` is an optional Object that might contain the following keys:
+  - `long` is a Boolean value to decide whether or not to populate `type`, `size` and `hash` (default: false)
+  - `cidBase` is which number base to use to format hashes - e.g. `base32`, `base64` etc (default: `base58btc`)
+
+It returns a [Pull Stream][os] that will yield objects containing the following keys:
+
+  - `name` which is the file's name
+  - `type` which is the object's type (`directory` or `file`)
+  - `size` the size of the file in bytes
+  - `hash` the hash of the file
+
+**Example:**
+
+```JavaScript
+pull(
+  ipfs.lsPullStream('/some-dir'),
+  pull.through(file => {
+    console.log(file.name)
+  })
+  pull.onEnd(...)
+)
+```
+
+A great source of [examples][] can be found in the tests for this API.
+
 [examples]: https://github.com/ipfs/interface-ipfs-core/blob/master/js/src/files
 [b]: https://www.npmjs.com/package/buffer
 [rs]: https://www.npmjs.com/package/readable-stream

js/src/files-mfs/index.js

@@ -13,6 +13,8 @@ const tests = {
   readReadableStream: require('./read-readable-stream'),
   readPullStream: require('./read-pull-stream'),
   ls: require('./ls'),
+  lsReadableStream: require('./ls-readable-stream'),
+  lsPullStream: require('./ls-pull-stream'),
   flush: require('./flush')
 }
 

js/src/files-mfs/ls-pull-stream.js

@@ -0,0 +1,107 @@
+/* eslint-env mocha */
+'use strict'
+
+const series = require('async/series')
+const hat = require('hat')
+const { getDescribe, getIt, expect } = require('../utils/mocha')
+const pull = require('pull-stream/pull')
+const onEnd = require('pull-stream/sinks/on-end')
+const collect = require('pull-stream/sinks/collect')
+
+module.exports = (createCommon, options) => {
+  const describe = getDescribe(options)
+  const it = getIt(options)
+  const common = createCommon()
+
+  describe('.files.lsPullStream', function () {
+    this.timeout(40 * 1000)
+
+    let ipfs
+
+    before(function (done) {
+      // CI takes longer to instantiate the daemon, so we need to increase the
+      // timeout for the before step
+      this.timeout(60 * 1000)
+
+      common.setup((err, factory) => {
+        expect(err).to.not.exist()
+        factory.spawnNode((err, node) => {
+          expect(err).to.not.exist()
+          ipfs = node
+          done()
+        })
+      })
+    })
+
+    after((done) => common.teardown(done))
+
+    it('should not ls not found file/dir, expect error', (done) => {
+      const testDir = `/test-${hat()}`
+
+      pull(
+        ipfs.files.lsPullStream(`${testDir}/404`),
+        onEnd((err) => {
+          expect(err).to.exist()
+          expect(err.message).to.include('does not exist')
+          done()
+        })
+      )
+    })
+
+    it('should ls directory', (done) => {
+      const testDir = `/test-${hat()}`
+
+      series([
+        (cb) => ipfs.files.mkdir(`${testDir}/lv1`, { p: true }, cb),
+        (cb) => ipfs.files.write(`${testDir}/b`, Buffer.from('Hello, world!'), { create: true }, cb)
+      ], (err) => {
+        expect(err).to.not.exist()
+
+        pull(
+          ipfs.files.lsPullStream(testDir),
+          collect((err, entries) => {
+            expect(err).to.not.exist()
+            expect(entries.sort((a, b) => a.name.localeCompare(b.name))).to.eql([
+              { name: 'b', type: 0, size: 0, hash: '' },
+              { name: 'lv1', type: 0, size: 0, hash: '' }
+            ])
+            done()
+          })
+        )
+      })
+    })
+
+    it('should ls -l directory', (done) => {
+      const testDir = `/test-${hat()}`
+
+      series([
+        (cb) => ipfs.files.mkdir(`${testDir}/lv1`, { p: true }, cb),
+        (cb) => ipfs.files.write(`${testDir}/b`, Buffer.from('Hello, world!'), { create: true }, cb)
+      ], (err) => {
+        expect(err).to.not.exist()
+
+        pull(
+          ipfs.files.lsPullStream(testDir, { l: true }),
+          collect((err, entries) => {
+            expect(err).to.not.exist()
+            expect(entries.sort((a, b) => a.name.localeCompare(b.name))).to.eql([
+              {
+                name: 'b',
+                type: 0,
+                size: 13,
+                hash: 'QmcZojhwragQr5qhTeFAmELik623Z21e3jBTpJXoQ9si1T'
+              },
+              {
+                name: 'lv1',
+                type: 1,
+                size: 0,
+                hash: 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'
+              }
+            ])
+            done()
+          })
+        )
+      })
+    })
+  })
+}

js/src/files-mfs/ls-readable-stream.js

@@ -0,0 +1,107 @@
+/* eslint-env mocha */
+'use strict'
+
+const series = require('async/series')
+const hat = require('hat')
+const { getDescribe, getIt, expect } = require('../utils/mocha')
+
+module.exports = (createCommon, options) => {
+  const describe = getDescribe(options)
+  const it = getIt(options)
+  const common = createCommon()
+
+  describe('.files.lsReadableStream', function () {
+    this.timeout(40 * 1000)
+
+    let ipfs
+
+    before(function (done) {
+      // CI takes longer to instantiate the daemon, so we need to increase the
+      // timeout for the before step
+      this.timeout(60 * 1000)
+
+      common.setup((err, factory) => {
+        expect(err).to.not.exist()
+        factory.spawnNode((err, node) => {
+          expect(err).to.not.exist()
+          ipfs = node
+          done()
+        })
+      })
+    })
+
+    after((done) => common.teardown(done))
+
+    it('should not ls not found file/dir, expect error', (done) => {
+      const testDir = `/test-${hat()}`
+
+      const stream = ipfs.files.lsReadableStream(`${testDir}/404`)
+
+      stream.once('error', (err) => {
+        expect(err).to.exist()
+        expect(err.message).to.include('does not exist')
+        done()
+      })
+    })
+
+    it('should ls directory', (done) => {
+      const testDir = `/test-${hat()}`
+
+      series([
+        (cb) => ipfs.files.mkdir(`${testDir}/lv1`, { p: true }, cb),
+        (cb) => ipfs.files.write(`${testDir}/b`, Buffer.from('Hello, world!'), { create: true }, cb)
+      ], (err) => {
+        expect(err).to.not.exist()
+
+        const stream = ipfs.files.lsReadableStream(testDir)
+
+        let entries = []
+
+        stream.on('data', entry => entries.push(entry))
+
+        stream.once('end', () => {
+          expect(entries.sort((a, b) => a.name.localeCompare(b.name))).to.eql([
+            { name: 'b', type: 0, size: 0, hash: '' },
+            { name: 'lv1', type: 0, size: 0, hash: '' }
+          ])
+          done()
+        })
+      })
+    })
+
+    it('should ls -l directory', (done) => {
+      const testDir = `/test-${hat()}`
+
+      series([
+        (cb) => ipfs.files.mkdir(`${testDir}/lv1`, { p: true }, cb),
+        (cb) => ipfs.files.write(`${testDir}/b`, Buffer.from('Hello, world!'), { create: true }, cb)
+      ], (err) => {
+        expect(err).to.not.exist()
+
+        const stream = ipfs.files.lsReadableStream(testDir, { l: true })
+
+        let entries = []
+
+        stream.on('data', entry => entries.push(entry))
+
+        stream.once('end', () => {
+          expect(entries.sort((a, b) => a.name.localeCompare(b.name))).to.eql([
+            {
+              name: 'b',
+              type: 0,
+              size: 13,
+              hash: 'QmcZojhwragQr5qhTeFAmELik623Z21e3jBTpJXoQ9si1T'
+            },
+            {
+              name: 'lv1',
+              type: 1,
+              size: 0,
+              hash: 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'
+            }
+          ])
+          done()
+        })
+      })
+    })
+  })
+}

js/src/files-mfs/ls.js

@@ -53,9 +53,9 @@ module.exports = (createCommon, options) => {
 
         ipfs.files.ls(testDir, (err, info) => {
           expect(err).to.not.exist()
-          expect(info).to.eql([
-            { name: 'lv1', type: 0, size: 0, hash: '' },
-            { name: 'b', type: 0, size: 0, hash: '' }
+          expect(info.sort((a, b) => a.name.localeCompare(b.name))).to.eql([
+            { name: 'b', type: 0, size: 0, hash: '' },
+            { name: 'lv1', type: 0, size: 0, hash: '' }
           ])
           done()
         })
@@ -73,18 +73,18 @@ module.exports = (createCommon, options) => {
 
         ipfs.files.ls(testDir, { l: true }, (err, info) => {
           expect(err).to.not.exist()
-          expect(info).to.eql([
-            {
-              name: 'lv1',
-              type: 1,
-              size: 0,
-              hash: 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'
-            },
+          expect(info.sort((a, b) => a.name.localeCompare(b.name))).to.eql([
             {
               name: 'b',
               type: 0,
               size: 13,
               hash: 'QmcZojhwragQr5qhTeFAmELik623Z21e3jBTpJXoQ9si1T'
+            },
+            {
+              name: 'lv1',
+              type: 1,
+              size: 0,
+              hash: 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'
             }
           ])
           done()