Feat/concurrency

.gitignore

@@ -1,4 +1,5 @@
 node_modules
+node_modules.nosync
 dist
 .webpack
 .serverless

README.md

@@ -527,6 +527,18 @@ if you are trying to override the entry in webpack.config.js with other unsuppor
 The individual packaging needs more time at the packaging phase, but you'll
 get that paid back twice at runtime.
 
+#### Individual packaging concurrency/serializdCompile
+
+```yaml
+# serverless.yml
+custom:
+  webpack:
+    concurrency: 5
+    serializedCompile: true # for backward compatibility, same as concurrency: 1
+```
+
+This runs webpack builds in parallel, throttled to `concurrency`. It reduces memory usage and in some cases impoves overall build performance.
+
 ## Usage
 
 ### Automatic bundling

index.test.js

@@ -85,7 +85,7 @@ describe('ServerlessWebpack', () => {
 
       _.set(serverless, 'service.custom.webpack.webpackConfig', 'webpack.config.ts');
 
-      const badDeps = function() {
+      const badDeps = function () {
         new ServerlessWebpack(serverless, {});
       };
 

lib/Configuration.js

@@ -14,7 +14,8 @@ const DefaultConfig = {
   packager: 'npm',
   packagerOptions: {},
   keepOutputDirectory: false,
-  config: null
+  config: null,
+  concurrency: Infinity
 };
 
 class Configuration {
@@ -37,6 +38,21 @@ class Configuration {
       }
     }
 
+    // Concurrency may be passed via CLI, e.g.
+    // custom:
+    //   webpack:
+    //     concurrency: ${opt:compile-concurrency, 7}
+    // In this case it is typed as a string and we have to validate it
+    if (this._config.concurrency !== undefined) {
+      this._config.concurrency = Number(this._config.concurrency);
+      if (isNaN(this._config.concurrency) || this._config.concurrency < 1) {
+        throw new Error('concurrency option must be a positive number');
+      }
+    } else if (this._config.serializedCompile === true) {
+      // Backwards compatibility with serializedCompile setting
+      this._config.concurrency = 1;
+    }
+
     // Set defaults for all missing properties
     _.defaults(this._config, DefaultConfig);
   }
@@ -73,6 +89,10 @@ class Configuration {
     return this._config.keepOutputDirectory;
   }
 
+  get concurrency() {
+    return this._config.concurrency;
+  }
+
   toJSON() {
     return _.omitBy(this._config, _.isNil);
   }

lib/Configuration.test.js

@@ -19,7 +19,8 @@ describe('Configuration', () => {
         packager: 'npm',
         packagerOptions: {},
         keepOutputDirectory: false,
-        config: null
+        config: null,
+        concurrency: Infinity
       };
     });
 
@@ -68,7 +69,8 @@ describe('Configuration', () => {
         packager: 'npm',
         packagerOptions: {},
         keepOutputDirectory: false,
-        config: null
+        config: null,
+        concurrency: Infinity
       });
     });
   });
@@ -88,7 +90,8 @@ describe('Configuration', () => {
         packager: 'npm',
         packagerOptions: {},
         keepOutputDirectory: false,
-        config: null
+        config: null,
+        concurrency: Infinity
       });
     });
 
@@ -107,8 +110,43 @@ describe('Configuration', () => {
         packager: 'npm',
         packagerOptions: {},
         keepOutputDirectory: false,
-        config: null
+        config: null,
+        concurrency: Infinity
       });
     });
+
+    it('should accept a numeric string as concurrency value', () => {
+      const testCustom = {
+        webpack: {
+          includeModules: { forceInclude: ['mod1'] },
+          webpackConfig: 'myWebpackFile.js',
+          concurrency: '3'
+        }
+      };
+      const config = new Configuration(testCustom);
+      expect(config._config.concurrency).to.equal(3);
+    });
+
+    it('should not accept an invalid string as concurrency value', () => {
+      const testCustom = {
+        webpack: {
+          includeModules: { forceInclude: ['mod1'] },
+          webpackConfig: 'myWebpackFile.js',
+          concurrency: '3abc'
+        }
+      };
+      expect(() => new Configuration(testCustom)).throws();
+    });
+
+    it('should not accept a non-positive number as concurrency value', () => {
+      const testCustom = {
+        webpack: {
+          includeModules: { forceInclude: ['mod1'] },
+          webpackConfig: 'myWebpackFile.js',
+          concurrency: 0
+        }
+      };
+      expect(() => new Configuration(testCustom)).throws();
+    });
   });
 });

lib/compile.js

@@ -5,44 +5,62 @@ const BbPromise = require('bluebird');
 const webpack = require('webpack');
 const tty = require('tty');
 
+const defaultStatsConfig = {
+  colors: tty.isatty(process.stdout.fd),
+  hash: false,
+  version: false,
+  chunks: false,
+  children: false
+};
+
+function ensureArray(obj) {
+  return _.isArray(obj) ? obj : [obj];
+}
+
+function getStatsLogger(statsConfig, consoleLog) {
+  return stats => consoleLog(stats.toString(statsConfig || defaultStatsConfig));
+}
+
+function webpackCompile(config, logStats) {
+  return BbPromise
+    .fromCallback(cb => webpack(config).run(cb))
+    .then(stats => {
+      // ensure stats in any array in the case of multiCompile
+      stats = stats.stats ? stats.stats : [stats];
+
+      _.forEach(stats, compileStats => {
+        logStats(compileStats);
+        if (compileStats.hasErrors()) {
+          throw new Error('Webpack compilation error, see stats above');
+        }
+      });
+
+      return stats;
+    });
+}
+
+function webpackConcurrentCompile(configs, logStats, concurrency) {
+  return BbPromise
+    .map(configs, config => webpackCompile(config, logStats), { concurrency })
+    .then(stats => _.flatten(stats));
+}
+
 module.exports = {
   compile() {
     this.serverless.cli.log('Bundling with Webpack...');
 
-    const compiler = webpack(this.webpackConfig);
-
-    return BbPromise.fromCallback(cb => compiler.run(cb)).then(stats => {
-      if (!this.multiCompile) {
-        stats = { stats: [stats] };
-      }
-
-      const compileOutputPaths = [];
-      const consoleStats = this.webpackConfig.stats ||
-        _.get(this, 'webpackConfig[0].stats') || {
-        colors: tty.isatty(process.stdout.fd),
-        hash: false,
-        version: false,
-        chunks: false,
-        children: false
-      };
-
-      _.forEach(stats.stats, compileStats => {
-        const statsOutput = compileStats.toString(consoleStats);
-        if (statsOutput) {
-          this.serverless.cli.consoleLog(statsOutput);
-        }
+    const configs = ensureArray(this.webpackConfig);
+    const logStats = getStatsLogger(configs[0].stats, this.serverless.cli.consoleLog);
 
-        if (compileStats.compilation.errors.length) {
-          throw new Error('Webpack compilation error, see above');
-        }
+    if (!this.configuration) {
+      return BbPromise.reject('Missing plugin configuration');
+    }
+    const concurrency = this.configuration.concurrency;
 
-        compileOutputPaths.push(compileStats.compilation.compiler.outputPath);
+    return webpackConcurrentCompile(configs, logStats, concurrency)
+      .then(stats => {
+        this.compileStats = { stats };
+        return BbPromise.resolve();
       });
-
-      this.compileOutputPaths = compileOutputPaths;
-      this.compileStats = stats;
-
-      return BbPromise.resolve();
-    });
   }
 };

lib/packagers/yarn.test.js

@@ -134,11 +134,11 @@ describe('yarn', () => {
       acorn@^2.1.0, acorn@^2.4.0:
         version "2.7.0"
         resolved "https://registry.yarnpkg.com/acorn/-/acorn-2.7.0.tgz#ab6e7d9d886aaca8b085bc3312b79a198433f0e7"
-    
+
       acorn@^3.0.4:
         version "3.3.0"
         resolved "https://registry.yarnpkg.com/acorn/-/acorn-3.3.0.tgz#45e37fb39e8da3f25baee3ff5369e2bb5f22017a"
-      
+
       otherModule@file:../../otherModule/the-new-version:
         version "1.2.0"
 
@@ -162,7 +162,7 @@ describe('yarn', () => {
           request "^2.83.0"
           ulid "^0.1.0"
           uuid "^3.1.0"
-    
+
         acorn@^5.0.0, acorn@^5.5.0:
           version "5.5.3"
           resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.5.3.tgz#f473dd47e0277a08e28e9bec5aeeb04751f0b8c9"
@@ -172,11 +172,11 @@ describe('yarn', () => {
       acorn@^2.1.0, acorn@^2.4.0:
         version "2.7.0"
         resolved "https://registry.yarnpkg.com/acorn/-/acorn-2.7.0.tgz#ab6e7d9d886aaca8b085bc3312b79a198433f0e7"
-    
+
       acorn@^3.0.4:
         version "3.3.0"
         resolved "https://registry.yarnpkg.com/acorn/-/acorn-3.3.0.tgz#45e37fb39e8da3f25baee3ff5369e2bb5f22017a"
-      
+
       otherModule@file:../../project/../../otherModule/the-new-version:
         version "1.2.0"
 
@@ -200,7 +200,7 @@ describe('yarn', () => {
           request "^2.83.0"
           ulid "^0.1.0"
           uuid "^3.1.0"
-    
+
         acorn@^5.0.0, acorn@^5.5.0:
           version "5.5.3"
           resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.5.3.tgz#f473dd47e0277a08e28e9bec5aeeb04751f0b8c9"

lib/validate.js

@@ -171,6 +171,7 @@ module.exports = {
         this.webpackConfig.output.path = path.join(this.serverless.config.servicePath, this.options.out);
       }
 
+      // Skip compilation with --no-build
       if (this.skipCompile) {
         this.serverless.cli.log('Skipping build and using existing compiled output');
         if (!fse.pathExistsSync(this.webpackConfig.output.path)) {
@@ -187,8 +188,13 @@ module.exports = {
 
       // In case of individual packaging we have to create a separate config for each function
       if (_.has(this.serverless, 'service.package') && this.serverless.service.package.individually) {
-        this.options.verbose && this.serverless.cli.log('Using multi-compile (individual packaging)');
         this.multiCompile = true;
+        this.options.verbose &&
+          this.serverless.cli.log(
+            `Using ${
+              this.configuration.concurrency !== Infinity ? 'concurrent' : 'multi'
+            }-compile (individual packaging)`
+          );
 
         if (this.webpackConfig.entry && !_.isEqual(this.webpackConfig.entry, entries)) {
           return BbPromise.reject(