feat: 优化上传功能

Author: PRC-Cc

README.md

@@ -34,6 +34,7 @@ $ cshell upload -a accessKey -s secretKey -b bucket -p prefix/ -o originDirector
   -f, --force                        Whether to overwrite upload.
   -z --zone [value]                  Online zone. one of "huad"、"huab"、"huan"、"beim"、"z0"、"z1"、"z2"、"na0"
   -o, --originDirectoryPath [value]  Directory's path of the files to upload, default is "dist"
+  -n, --concurrencyNumber [value]    Concurrency number of action. default is 40
   -h, --help                         output usage information
 ```
 

bin/cshell-upload.js

@@ -18,6 +18,10 @@ program
     "-o, --originDirectoryPath [value]",
     'Directory\'s path of the files to upload, default is "dist"'
   )
+  .option(
+    "-n, --concurrencyNumber [value]",
+    'Concurrency number of action. default is 40'
+  )
   .parse(process.argv);
 
 if (isEmpty(program.accessKey)) {

lib/scripts/upload.js

@@ -10,6 +10,8 @@ let mac;
 let originDirectoryPath; // 源目录路径
 let zone;
 let force;
+let concurrencyNumber = 40; // 并发数
+const chain = [];
 
 const zoneMap = {
   huad: "Zone_z0",
@@ -28,15 +30,14 @@ const zoneMap = {
  * @param {String} fileName 文件名
  * @param {String} prefix 前缀
  */
-function uploadFile(filedir, fileNameIn, prefix = "") {
+function uploadFile(filedir, fileNameIn, prefix = "", callback) {
   let fileName = prefix + fileNameIn;
   const options = {
     scope: force ? `${bucket}:${fileName}` : bucket
   };
   const putPolicy = new qiniu.rs.PutPolicy(options);
   const uploadToken = putPolicy.uploadToken(mac);
   const putExtra = new qiniu.form_up.PutExtra();
-
   const config = new qiniu.conf.Config();
   if (!isEmpty(zone)) {
     config.zone = qiniu.zone[zoneMap[zone]];
@@ -50,14 +51,17 @@ function uploadFile(filedir, fileNameIn, prefix = "") {
     respInfo
   ) {
     if (respErr) {
+      log.fail(`Upload ${bucket} ${fileName} fail.`);
       log.error(respErr);
     }
     if (respInfo.statusCode == 200) {
-      log.success(`Upload ${bucket} ${fileName} success`);
+      log.success(`Upload ${bucket} ${fileName} success.`);
+      callback();
     } else {
       log.fail(
         `Upload ${bucket} ${fileName} fail. ${respInfo.statusCode} ${respBody.error}`
       );
+      callback();
     }
   });
 }
@@ -67,36 +71,38 @@ function uploadFile(filedir, fileNameIn, prefix = "") {
  * @param {String} filePath 初始目录
  */
 function fileDisplay(filePath) {
-  fs.readdir(filePath, function(err, files) {
-    if (err) {
-      log.error(err);
-    } else {
-      files.forEach(function(filename) {
-        const filedir = path.join(filePath, filename);
-        fs.stat(filedir, function(eror, stats) {
-          if (eror) {
-            log.error(error);
-          } else {
-            const isFile = stats.isFile();
-            const isDir = stats.isDirectory();
-            if (isFile) {
-              const fileName = filedir
-                .replace(`${originDirectoryPath}${path.sep}`, "")
-                .replace(/\\/g, "/");
-              uploadFile(filedir, fileName, prefix);
-            } else if (isDir) {
-              fileDisplay(filedir);
-            }
-          }
-        });
-      });
-    }
-  });
+  try {
+    const files = fs.readdirSync(filePath);
+    files.forEach(function(filename) {
+      const filedir = path.join(filePath, filename);
+      try {
+        const stats = fs.statSync(filedir);
+        const isFile = stats.isFile();
+        const isDir = stats.isDirectory();
+        if (isFile) {
+          const fileName = filedir
+            .replace(`${originDirectoryPath}${path.sep}`, "")
+            .replace(/\\/g, "/");
+          chain.push((...args) =>
+            uploadFile(filedir, fileName, prefix, ...args)
+          );
+        } else if (isDir) {
+          fileDisplay(filedir);
+        }
+      } catch (error) {
+        log.error(error);
+      }
+    });
+  } catch (error) {
+    log.error(error);
+  }
 }
 
 function upload(program) {
   const accessKey = program.accessKey;
   const secretKey = program.secretKey;
+  const pConcurrencyNumber = Number(program.concurrencyNumber);
+
   bucket = program.bucket;
   prefix = isEmpty(program.prefix) ? "" : program.prefix;
   force = !!program.force;
@@ -109,12 +115,59 @@ function upload(program) {
     );
   }
 
+  if (Number.isInteger(pConcurrencyNumber)) {
+    concurrencyNumber = pConcurrencyNumber;
+  }
+
   originDirectoryPath = path.resolve(
     !isEmpty(program.originDirectoryPath)
       ? program.originDirectoryPath.replace(/\\/g, "/")
       : "dist"
   );
   fileDisplay(originDirectoryPath);
+  batchExec(chain, successCallback);
+}
+
+/**
+ * 成功回调
+ * @param {Number} length
+ */
+function successCallback(length) {
+  log.success(`Done, all ${length} files upload success.`);
+}
+
+/**
+ *
+ * @param {Array} chain 执行队列
+ */
+async function batchExec(chain, callback) {
+  const chainLen = chain.length;
+  const promises = {};
+  let diff;
+  let promisesLen;
+  let promiseKey;
+  let startNum = 0;
+  let timer = setInterval(() => {
+    if (chain.length === 0) {
+      clearInterval(timer);
+      timer = null;
+      return;
+    }
+    diff = concurrencyNumber - Object.keys(promises).length;
+    if (diff <= 0) return;
+    chain.splice(0, diff).forEach(fn => {
+      promiseKey = startNum++;
+      promises[promiseKey] = true;
+      (function(promiseKey) {
+        fn(() => {
+          delete promises[promiseKey];
+          if (Object.keys(promises).length === 0) {
+            callback(chainLen);
+          }
+        });
+      })(promiseKey);
+    });
+  }, 0);
 }
 
 module.exports = upload;