task loading improvements

CHANGELOG.md

@@ -1,5 +1,8 @@
 ## {{ UNRELEASED_VERSION }} - [{{ UNRELEASED_DATE }}]({{ UNRELEASED_LINK }})
 
+* Improved efficiency of `task` loading
+* Improved handling around corrupted `task` caches
+
 ## v3.24.0-beta.4 - [December 9, 2024](https://github.com/lando/core/releases/tag/v3.24.0-beta.4)
 
 * Added `getLodash()` to `lando.utils`

hooks/app-override-tooling-defaults.js

@@ -5,6 +5,8 @@ const merge = require('lodash/merge');
 module.exports = async (app, lando) => {
   for (const task of lando.tasks.filter(task => task.override)) {
     app.log.debug('overriding task %s with dynamic app options', task.command);
-    app._coreToolingOverrides = merge({}, app._coreToolingOverrides, {[task.command]: require(task.file)(lando, app)});
+    app._coreToolingOverrides = merge({}, app._coreToolingOverrides, {
+      [task.command]: {...require(task.file)(lando, app), file: task.file},
+    });
   }
 };

hooks/lando-generate-tasks-cache.js

@@ -0,0 +1,31 @@
+'use strict';
+
+const _ = require('lodash');
+const fs = require('fs');
+const path = require('path');
+
+module.exports = async lando => {
+  // load in legacy inits
+  await require('./lando-load-legacy-inits')(lando);
+
+  // build the cache
+  return lando.Promise.resolve(lando.config.plugins)
+    // Make sure the tasks dir exists
+    .filter(plugin => fs.existsSync(plugin.tasks))
+    // Get a list off full js files that exist in that dir
+    .map(plugin => _(fs.readdirSync(plugin.tasks))
+      .map(file => path.join(plugin.tasks, file))
+      .filter(path => _.endsWith(path, '.js'))
+      .value(),
+    )
+    // Loadem and loggem
+    .then(tasks => _.flatten(tasks))
+    .each(file => {
+      lando.tasks.push({...require(file)(lando, {}), file});
+      lando.log.debug('autoloaded global task %s', path.basename(file, '.js'));
+    })
+    // Reset the task cache
+    .then(() => {
+      lando.cache.set('_.tasks.cache', JSON.stringify(lando.tasks), {persist: true});
+    });
+};

hooks/lando-load-legacy-inits.js

@@ -0,0 +1,51 @@
+'use strict';
+
+const _ = require('lodash');
+const fs = require('fs');
+const glob = require('glob');
+const path = require('path');
+
+// Helper to get init config
+const getLegacyInitConfig = dirs => _(dirs)
+  .filter(dir => fs.existsSync(dir))
+  .flatMap(dir => glob.sync(path.join(dir, '*', 'init.js')))
+  .map(file => require(file))
+  .value();
+
+// Helper to get init config
+const getInitConfig = dirs => _(dirs)
+  .filter(dir => fs.existsSync(dir))
+  .flatMap(dir => fs.readdirSync(dir).map(file => path.join(dir, file)))
+  .map(file => require(file))
+  .value();
+
+// Helper to get init source config
+const getInitSourceConfig = dirs => _(dirs)
+  .filter(dir => fs.existsSync(dir))
+  .flatMap(dir => glob.sync(path.join(dir, '*.js')))
+  .map(file => require(file))
+  .flatMap(source => source.sources)
+  .value();
+
+module.exports = async lando => {
+  const legacyInits = getLegacyInitConfig(_.map(lando.config.plugins, 'recipes'));
+  const inits = getInitConfig(_.map(lando.config.plugins, 'inits'));
+
+  lando.config.inits = _.sortBy(_.map(_.merge(
+    {},
+    _.fromPairs(_.map(legacyInits, init => ([init.name, init]))),
+    _.fromPairs(_.map(inits, init => ([init.name, init]))),
+  ), init => init), 'name');
+
+  // Load in config frmo sources
+  const sources = getInitSourceConfig(_.map(lando.config.plugins, 'sources'));
+  const initSources = _(lando.config.inits)
+    .filter(init => _.has(init, 'sources'))
+    .flatMap(init => init.sources)
+    .value();
+
+  lando.config.sources = _.sortBy(sources.concat(initSources), 'label');
+
+  // And finally the recipes
+  lando.config.recipes = _.sortBy(_.map(lando.config.inits, init => init.name), 'name');
+};

index.js

@@ -115,6 +115,9 @@ module.exports = async lando => {
   // flush update cache if it needs to be
   lando.events.on('ready', async () => await require('./hooks/lando-flush-updates-cache')(lando));
 
+  // merge in needed legacy init stuff
+  lando.events.on('cli-init-answers', async () => await require('./hooks/lando-load-legacy-inits')(lando));
+
   // this is a gross hack we need to do to reset the engine because the lando 3 runtime has no idea
   lando.events.on('almost-ready', 1, async () => await require('./hooks/lando-reset-orchestrator')(lando));
   lando.events.on('post-setup', 1, async () => await require('./hooks/lando-reset-orchestrator')(lando));
@@ -134,6 +137,9 @@ module.exports = async lando => {
   // clean networks
   lando.events.on('pre-engine-start', 1, async () => await require('./hooks/lando-clean-networks')(lando));
 
+  // regen task cache
+  lando.events.on('before-end', 9999, async () => await require('./hooks/lando-generate-tasks-cache')(lando));
+
   // return some default things
   return _.merge({}, defaults, uc(), {config: {
     appEnv: {

lib/cli.js

@@ -261,6 +261,9 @@ module.exports = class Cli {
     // Set the verbosity
     error.verbose = verbose;
 
+    // clear tasks caches on all errors
+    this.clearTaskCaches();
+
     // if report_errors is not set but -y was passed in then set it here to avoid the prompt below
     if (_.isNil(lando.cache.get('report_errors')) && yes) {
       lando.cache.set('report_errors', yes, {persist: true});
@@ -385,6 +388,7 @@ module.exports = class Cli {
     command,
     describe,
     examples = [],
+    file = undefined,
     options = {},
     positionals = {},
     run = {},
@@ -428,7 +432,7 @@ module.exports = class Cli {
 
         // Attempt to filter out questions that have already been answered
         // Prompt the use for feedback if needed and sort by weight
-        .then(() => formatters.handleInteractive(data.inquiry, data.options, command, lando))
+        .then(() => formatters.handleInteractive(data.inquiry, data.options, command, lando, file))
 
         /**
          * Event that allows final altering of answers before the task runs
@@ -445,8 +449,12 @@ module.exports = class Cli {
         // Find and run the task, unless we already have one
         // @TODO: somehow handle when commands break eg change task name, malformed tasks
         .then(() => {
-          if (_.isFunction(run)) return run(data.options, lando);
-          else return _.find(lando.tasks, {command}).run(data.options);
+          // if run is already a function
+          if (_.isFunction(run)) return run(data.options, lando, config);
+          // if we have a task then do that
+          else if (file && fs.existsSync(file)) return require(file)(lando, config).run(data.options);
+          // error?
+          throw new Error(`Could not locate a runner for ${command}!`);
         })
 
         // Add a final event for other stuff

lib/formatters.js

@@ -2,6 +2,7 @@
 
 // Modules
 const _ = require('lodash');
+const fs = require('fs');
 const os = require('os');
 const util = require('util');
 
@@ -108,11 +109,20 @@ exports.getInteractive = (options, argv) => _(options)
 /*
  * Helper to prompt the user if needed
  */
-exports.handleInteractive = (inquiry, argv, command, lando) => lando.Promise.try(() => {
+exports.handleInteractive = (inquiry, argv, command, lando, file) => lando.Promise.try(() => {
   if (_.isEmpty(inquiry)) return {};
   else {
     const inquirer = require('inquirer');
     inquirer.registerPrompt('autocomplete', require('inquirer-autocomplete-prompt'));
+
+    // mix in the full task object so we can load in functions and that sort of thing
+    if (file && fs.existsSync(file)) {
+      // find index of task
+      const tid = lando.tasks.findIndex(task => task.command === command);
+      // replace
+      lando.tasks[tid] = require(file)(lando, lando.appConfig);
+    }
+
     // Try to rebuild the inquiry if this is app level bootstrap and we have an app
     if (!_.isEmpty(argv._app) && lando._bootstrap === 'engine') {
       // get id

lib/lando.js

@@ -26,28 +26,6 @@ const resolveSetupTasks = (tasks = []) => {
   return tasks;
 };
 
-// Helper to get init config
-const getLegacyInitConfig = dirs => _(dirs)
-  .filter(dir => fs.existsSync(dir))
-  .flatMap(dir => glob.sync(path.join(dir, '*', 'init.js')))
-  .map(file => require(file))
-  .value();
-
-// Helper to get init config
-const getInitConfig = dirs => _(dirs)
-  .filter(dir => fs.existsSync(dir))
-  .flatMap(dir => fs.readdirSync(dir).map(file => path.join(dir, file)))
-  .map(file => require(file))
-  .value();
-
-// Helper to get init source config
-const getInitSourceConfig = dirs => _(dirs)
-  .filter(dir => fs.existsSync(dir))
-  .flatMap(dir => glob.sync(path.join(dir, '*.js')))
-  .map(file => require(file))
-  .flatMap(source => source.sources)
-  .value();
-
 /*
  * Helper to bootstrap plugins
  */
@@ -113,47 +91,11 @@ const bootstrapConfig = async lando => {
 /*
  * Helper to bootstrap tasks
  */
-const bootstrapTasks = lando => {
-  // Load in config from inits
-  const legacyInits = getLegacyInitConfig(_.map(lando.config.plugins, 'recipes'));
-  const inits = getInitConfig(_.map(lando.config.plugins, 'inits'));
-  lando.config.inits = _.sortBy(_.map(_.merge(
-    {},
-    _.fromPairs(_.map(legacyInits, init => ([init.name, init]))),
-    _.fromPairs(_.map(inits, init => ([init.name, init]))),
-  ), init => init), 'name');
-
-  // Load in config frmo sources
-  const sources = getInitSourceConfig(_.map(lando.config.plugins, 'sources'));
-  const initSources = _(lando.config.inits)
-    .filter(init => _.has(init, 'sources'))
-    .flatMap(init => init.sources)
-    .value();
-  lando.config.sources = _.sortBy(sources.concat(initSources), 'label');
-
-  // And finally the recipes
-  lando.config.recipes = _.sortBy(_.map(lando.config.inits, init => init.name), 'name');
-
-  // Load in all our tasks
-  return lando.Promise.resolve(lando.config.plugins)
-    // Make sure the tasks dir exists
-    .filter(plugin => fs.existsSync(plugin.tasks))
-    // Get a list off full js files that exist in that dir
-    .map(plugin => _(fs.readdirSync(plugin.tasks))
-      .map(file => path.join(plugin.tasks, file))
-      .filter(path => _.endsWith(path, '.js'))
-      .value(),
-    )
-    // Loadem and loggem
-    .then(tasks => _.flatten(tasks))
-    .each(file => {
-      lando.tasks.push({...require(file)(lando, {}), file});
-      lando.log.debug('autoloaded global task %s', path.basename(file, '.js'));
-    })
-    // Reset the task cache
-    .then(() => {
-      lando.cache.set('_.tasks.cache', JSON.stringify(lando.tasks), {persist: true});
-    });
+const bootstrapTasks = async lando => {
+  // if we already have cached tasks tehn load that
+  if (!lando.cache.get('_.tasks.cache')) await require('../hooks/lando-generate-tasks-cache')(lando);
+  // push it
+  lando.tasks.push(...JSON.parse(lando.cache.get('_.tasks.cache')));
 };
 
 /*

utils/get-tasks.js

@@ -31,7 +31,8 @@ const loadCacheFile = file => {
   try {
     return JSON.parse(JSON.parse(fs.readFileSync(file, {encoding: 'utf-8'})));
   } catch (e) {
-    throw new Error(`There was a problem with parsing ${file}. Ensure it is valid JSON! ${e}`);
+    fs.rmSync(file, {force: true, retryDelay: 201, maxRetries: 16, recursive: true});
+    return {};
   }
 };