feat: option to produce TAR archives (#15)

adds opt-in "archive mode" which produces TAR archive, matching ipfs
get.

--archive produces .tar file instead of an unpacked directory tree
--archive --compress produces a .tar.gz file

License: MIT
Signed-off-by: Marcin Rataj <[email protected]>

Author: lidel

README.md

@@ -8,9 +8,13 @@
 ## Usage
 
 ```
-npx ipfs-or-gateway -c cid -p path [--clean -a apiUrl]
+npx ipfs-or-gateway -c cid -p path [--clean --archive --compress -a apiUrl]
 ```
 
+- `--clean` – remove destination if it already exists
+- `--archive` – produce `.tar` archive instead of unpacked directory tree
+- `--compress` – compress produced archive with Gzipi, produce `.tar.gz` (requires `--archive`)
+
 ## Contributing
 
 PRs accepted.

bin/index.js

@@ -18,35 +18,38 @@ const argv = yargs
     demandOption: true
   }).option('clean', {
     describe: 'clean path first',
-    type: 'boolean'
+    type: 'boolean',
+    default: false
+  }).option('archive', {
+    describe: 'output a TAR archive',
+    type: 'boolean',
+    default: false
+  }).option('compress', {
+    describe: 'compress the archive with GZIP compression',
+    type: 'boolean',
+    default: false
   }).option('api', {
     alias: 'a',
     describe: 'api url',
-    type: 'string'
+    type: 'string',
+    default: 'https://ipfs.io/api'
   }).option('retries', {
     alias: 'r',
     describe: 'number of retries for each gateway',
-    type: 'number'
+    type: 'number',
+    default: 3
   }).option('timeout', {
     alias: 't',
     describe: 'timeout of request without data from the server',
-    type: 'number'
+    type: 'number',
+    default: 60000
   })
   .help()
   .argv
 
 async function run () {
   try {
-    const opts = {
-      cid: argv.cid,
-      path: argv.path,
-      clean: argv.clean,
-      api: argv.api,
-      retries: argv.retries,
-      timeout: argv.timeout
-    }
-
-    await download(opts)
+    await download(argv)
   } catch (error) {
     console.error(error.toString())
     process.exit(1)

lib/index.js

@@ -1,22 +1,25 @@
 const fs = require('fs-extra')
 const fetch = require('node-fetch')
 const tar = require('tar')
+const { createGunzip } = require('zlib')
 const { exec } = require('child_process')
 const Progress = require('node-fetch-progress')
 const AbortController = require('abort-controller')
 const ora = require('ora')
 const prettyBytes = require('pretty-bytes')
 
-function fetchIPFS ({ cid, path }) {
+function fetchIPFS ({ cid, path, archive, compress }) {
   return new Promise((resolve, reject) => {
-    exec(`ipfs get ${cid} -o ${path}`, err => {
+    archive = archive ? '-a' : ''
+    compress = compress ? '-C' : ''
+    exec(`ipfs get ${cid} -o ${path} ${archive} ${compress}`, err => {
       if (err) return reject(err)
       resolve()
     })
   })
 }
 
-async function fetchHTTP ({ api, cid, timeout: timeoutMs, path, spinner }) {
+async function fetchHTTP ({ api, cid, timeout: timeoutMs, path, archive, compress, spinner }) {
   const url = `${api}/v0/get?arg=${cid}&archive=true&compress=true`
   const controller = new AbortController()
   const fetchPromise = fetch(url, { signal: controller.signal })
@@ -40,14 +43,18 @@ async function fetchHTTP ({ api, cid, timeout: timeoutMs, path, spinner }) {
       }
     })
 
-    const extractor = tar.extract({
-      strip: 1,
-      C: path,
-      strict: true
-    })
+    const writer = archive
+      ? fs.createWriteStream(path)
+      : tar.extract({
+        strip: 1,
+        C: path,
+        strict: true
+      })
 
     await new Promise((resolve, reject) => {
-      res.body.pipe(extractor)
+      (compress
+        ? res.body.pipe(writer)
+        : res.body.pipe(createGunzip()).pipe(writer))
         .on('error', reject)
         .on('finish', () => {
           if (progress) progress.removeAllListeners('progress')
@@ -59,31 +66,35 @@ async function fetchHTTP ({ api, cid, timeout: timeoutMs, path, spinner }) {
   }
 }
 
-module.exports = async (opts) => {
-  opts.timeout = opts.timeout || 60000
-  opts.retries = opts.retries || 3
-  opts.api = opts.api || 'https://ipfs.io/api'
-
-  const { cid, path, clean, verbose, timeout, api, retries } = opts
-
+module.exports = async ({ cid, path, clean, archive, compress, verbose, timeout, api, retries }) => {
   if (!cid || !path) {
     throw new Error('cid and path must be defined')
   }
+  if (compress && !archive) {
+    throw new Error('compress requires archive mode')
+  }
+
+  // match go-ipfs behaviour: 'ipfs get' adds .tar and .tar.gz if missing
+  if (compress && !path.endsWith('.tar.gz')) { path += '.tar.gz' }
+  if (archive && !path.includes('.tar')) { path += '.tar' }
 
   if (await fs.pathExists(path)) {
     if (clean) {
-      await fs.emptyDir(path)
+      fs.lstatSync(path).isDirectory()
+        ? fs.emptyDirSync(path)
+        : fs.unlinkSync(path) // --archive produces a file
     } else {
+      // no-op if destination already exists
       return
     }
   }
 
-  await fs.ensureDir(path)
+  if (!archive) await fs.ensureDir(path)
   let spinner = ora()
 
   try {
     spinner.start('Fetching via IPFS…')
-    await fetchIPFS({ cid, path })
+    await fetchIPFS({ cid, path, archive, compress })
     spinner.succeed(`Fetched ${cid} to ${path}!`)
     return
   } catch (_error) {
@@ -97,7 +108,7 @@ module.exports = async (opts) => {
     spinner.start(`Fetching via IPFS HTTP gateway (attempt ${i})…`)
 
     try {
-      await fetchHTTP({ cid, path, timeout, api, verbose, spinner })
+      await fetchHTTP({ cid, path, archive, compress, timeout, api, verbose, spinner })
       spinner.succeed(`Fetched ${cid} to ${path}!`)
       return
     } catch (e) {