From ba244546ad381c8c755a77f52f7b384addeb9a86 Mon Sep 17 00:00:00 2001 From: Mike Pirog Date: Tue, 10 Dec 2024 11:10:47 -0500 Subject: [PATCH] task loading improvements --- CHANGELOG.md | 3 ++ hooks/app-override-tooling-defaults.js | 4 +- hooks/lando-generate-tasks-cache.js | 31 ++++++++++++ hooks/lando-load-legacy-inits.js | 51 +++++++++++++++++++ index.js | 6 +++ lib/cli.js | 14 ++++-- lib/formatters.js | 12 ++++- lib/lando.js | 68 ++------------------------ package-lock.json | 2 +- utils/get-tasks.js | 3 +- 10 files changed, 124 insertions(+), 70 deletions(-) create mode 100644 hooks/lando-generate-tasks-cache.js create mode 100644 hooks/lando-load-legacy-inits.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 554353503..b1186d7aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## {{ UNRELEASED_VERSION }} - [{{ UNRELEASED_DATE }}]({{ UNRELEASED_LINK }}) +* Improved efficiency of `task` loading +* Improved handling around corrupted `task` caches + ## v3.24.0-beta.4 - [December 9, 2024](https://github.com/lando/core/releases/tag/v3.24.0-beta.4) * Added `getLodash()` to `lando.utils` diff --git a/hooks/app-override-tooling-defaults.js b/hooks/app-override-tooling-defaults.js index e2847b6e6..2818fef47 100644 --- a/hooks/app-override-tooling-defaults.js +++ b/hooks/app-override-tooling-defaults.js @@ -5,6 +5,8 @@ const merge = require('lodash/merge'); module.exports = async (app, lando) => { for (const task of lando.tasks.filter(task => task.override)) { app.log.debug('overriding task %s with dynamic app options', task.command); - app._coreToolingOverrides = merge({}, app._coreToolingOverrides, {[task.command]: require(task.file)(lando, app)}); + app._coreToolingOverrides = merge({}, app._coreToolingOverrides, { + [task.command]: {...require(task.file)(lando, app), file: task.file}, + }); } }; diff --git a/hooks/lando-generate-tasks-cache.js b/hooks/lando-generate-tasks-cache.js new file mode 100644 index 000000000..728553d5b --- /dev/null +++ b/hooks/lando-generate-tasks-cache.js @@ -0,0 +1,31 @@ +'use strict'; + +const _ = require('lodash'); +const fs = require('fs'); +const path = require('path'); + +module.exports = async lando => { + // load in legacy inits + await require('./lando-load-legacy-inits')(lando); + + // build the cache + return lando.Promise.resolve(lando.config.plugins) + // Make sure the tasks dir exists + .filter(plugin => fs.existsSync(plugin.tasks)) + // Get a list off full js files that exist in that dir + .map(plugin => _(fs.readdirSync(plugin.tasks)) + .map(file => path.join(plugin.tasks, file)) + .filter(path => _.endsWith(path, '.js')) + .value(), + ) + // Loadem and loggem + .then(tasks => _.flatten(tasks)) + .each(file => { + lando.tasks.push({...require(file)(lando, {}), file}); + lando.log.debug('autoloaded global task %s', path.basename(file, '.js')); + }) + // Reset the task cache + .then(() => { + lando.cache.set('_.tasks.cache', JSON.stringify(lando.tasks), {persist: true}); + }); +}; diff --git a/hooks/lando-load-legacy-inits.js b/hooks/lando-load-legacy-inits.js new file mode 100644 index 000000000..14eab002e --- /dev/null +++ b/hooks/lando-load-legacy-inits.js @@ -0,0 +1,51 @@ +'use strict'; + +const _ = require('lodash'); +const fs = require('fs'); +const glob = require('glob'); +const path = require('path'); + +// Helper to get init config +const getLegacyInitConfig = dirs => _(dirs) + .filter(dir => fs.existsSync(dir)) + .flatMap(dir => glob.sync(path.join(dir, '*', 'init.js'))) + .map(file => require(file)) + .value(); + +// Helper to get init config +const getInitConfig = dirs => _(dirs) + .filter(dir => fs.existsSync(dir)) + .flatMap(dir => fs.readdirSync(dir).map(file => path.join(dir, file))) + .map(file => require(file)) + .value(); + +// Helper to get init source config +const getInitSourceConfig = dirs => _(dirs) + .filter(dir => fs.existsSync(dir)) + .flatMap(dir => glob.sync(path.join(dir, '*.js'))) + .map(file => require(file)) + .flatMap(source => source.sources) + .value(); + +module.exports = async lando => { + const legacyInits = getLegacyInitConfig(_.map(lando.config.plugins, 'recipes')); + const inits = getInitConfig(_.map(lando.config.plugins, 'inits')); + + lando.config.inits = _.sortBy(_.map(_.merge( + {}, + _.fromPairs(_.map(legacyInits, init => ([init.name, init]))), + _.fromPairs(_.map(inits, init => ([init.name, init]))), + ), init => init), 'name'); + + // Load in config frmo sources + const sources = getInitSourceConfig(_.map(lando.config.plugins, 'sources')); + const initSources = _(lando.config.inits) + .filter(init => _.has(init, 'sources')) + .flatMap(init => init.sources) + .value(); + + lando.config.sources = _.sortBy(sources.concat(initSources), 'label'); + + // And finally the recipes + lando.config.recipes = _.sortBy(_.map(lando.config.inits, init => init.name), 'name'); +}; diff --git a/index.js b/index.js index e7fa0733b..11cc01e7b 100644 --- a/index.js +++ b/index.js @@ -115,6 +115,9 @@ module.exports = async lando => { // flush update cache if it needs to be lando.events.on('ready', async () => await require('./hooks/lando-flush-updates-cache')(lando)); + // merge in needed legacy init stuff + lando.events.on('cli-init-answers', async () => await require('./hooks/lando-load-legacy-inits')(lando)); + // this is a gross hack we need to do to reset the engine because the lando 3 runtime has no idea lando.events.on('almost-ready', 1, async () => await require('./hooks/lando-reset-orchestrator')(lando)); lando.events.on('post-setup', 1, async () => await require('./hooks/lando-reset-orchestrator')(lando)); @@ -134,6 +137,9 @@ module.exports = async lando => { // clean networks lando.events.on('pre-engine-start', 1, async () => await require('./hooks/lando-clean-networks')(lando)); + // regen task cache + lando.events.on('before-end', 9999, async () => await require('./hooks/lando-generate-tasks-cache')(lando)); + // return some default things return _.merge({}, defaults, uc(), {config: { appEnv: { diff --git a/lib/cli.js b/lib/cli.js index 60098cc6a..91af82162 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -261,6 +261,9 @@ module.exports = class Cli { // Set the verbosity error.verbose = verbose; + // clear tasks caches on all errors + this.clearTaskCaches(); + // if report_errors is not set but -y was passed in then set it here to avoid the prompt below if (_.isNil(lando.cache.get('report_errors')) && yes) { lando.cache.set('report_errors', yes, {persist: true}); @@ -385,6 +388,7 @@ module.exports = class Cli { command, describe, examples = [], + file = undefined, options = {}, positionals = {}, run = {}, @@ -428,7 +432,7 @@ module.exports = class Cli { // Attempt to filter out questions that have already been answered // Prompt the use for feedback if needed and sort by weight - .then(() => formatters.handleInteractive(data.inquiry, data.options, command, lando)) + .then(() => formatters.handleInteractive(data.inquiry, data.options, command, lando, file)) /** * Event that allows final altering of answers before the task runs @@ -445,8 +449,12 @@ module.exports = class Cli { // Find and run the task, unless we already have one // @TODO: somehow handle when commands break eg change task name, malformed tasks .then(() => { - if (_.isFunction(run)) return run(data.options, lando); - else return _.find(lando.tasks, {command}).run(data.options); + // if run is already a function + if (_.isFunction(run)) return run(data.options, lando, config); + // if we have a task then do that + else if (file && fs.existsSync(file)) return require(file)(lando, config).run(data.options); + // error? + throw new Error(`Could not locate a runner for ${command}!`); }) // Add a final event for other stuff diff --git a/lib/formatters.js b/lib/formatters.js index 9407e4ac5..b9ae15c0e 100644 --- a/lib/formatters.js +++ b/lib/formatters.js @@ -2,6 +2,7 @@ // Modules const _ = require('lodash'); +const fs = require('fs'); const os = require('os'); const util = require('util'); @@ -108,11 +109,20 @@ exports.getInteractive = (options, argv) => _(options) /* * Helper to prompt the user if needed */ -exports.handleInteractive = (inquiry, argv, command, lando) => lando.Promise.try(() => { +exports.handleInteractive = (inquiry, argv, command, lando, file) => lando.Promise.try(() => { if (_.isEmpty(inquiry)) return {}; else { const inquirer = require('inquirer'); inquirer.registerPrompt('autocomplete', require('inquirer-autocomplete-prompt')); + + // mix in the full task object so we can load in functions and that sort of thing + if (file && fs.existsSync(file)) { + // find index of task + const tid = lando.tasks.findIndex(task => task.command === command); + // replace + lando.tasks[tid] = require(file)(lando, lando.appConfig); + } + // Try to rebuild the inquiry if this is app level bootstrap and we have an app if (!_.isEmpty(argv._app) && lando._bootstrap === 'engine') { // get id diff --git a/lib/lando.js b/lib/lando.js index f04c28d4d..5bfa1a8dc 100644 --- a/lib/lando.js +++ b/lib/lando.js @@ -26,28 +26,6 @@ const resolveSetupTasks = (tasks = []) => { return tasks; }; -// Helper to get init config -const getLegacyInitConfig = dirs => _(dirs) - .filter(dir => fs.existsSync(dir)) - .flatMap(dir => glob.sync(path.join(dir, '*', 'init.js'))) - .map(file => require(file)) - .value(); - -// Helper to get init config -const getInitConfig = dirs => _(dirs) - .filter(dir => fs.existsSync(dir)) - .flatMap(dir => fs.readdirSync(dir).map(file => path.join(dir, file))) - .map(file => require(file)) - .value(); - -// Helper to get init source config -const getInitSourceConfig = dirs => _(dirs) - .filter(dir => fs.existsSync(dir)) - .flatMap(dir => glob.sync(path.join(dir, '*.js'))) - .map(file => require(file)) - .flatMap(source => source.sources) - .value(); - /* * Helper to bootstrap plugins */ @@ -113,47 +91,11 @@ const bootstrapConfig = async lando => { /* * Helper to bootstrap tasks */ -const bootstrapTasks = lando => { - // Load in config from inits - const legacyInits = getLegacyInitConfig(_.map(lando.config.plugins, 'recipes')); - const inits = getInitConfig(_.map(lando.config.plugins, 'inits')); - lando.config.inits = _.sortBy(_.map(_.merge( - {}, - _.fromPairs(_.map(legacyInits, init => ([init.name, init]))), - _.fromPairs(_.map(inits, init => ([init.name, init]))), - ), init => init), 'name'); - - // Load in config frmo sources - const sources = getInitSourceConfig(_.map(lando.config.plugins, 'sources')); - const initSources = _(lando.config.inits) - .filter(init => _.has(init, 'sources')) - .flatMap(init => init.sources) - .value(); - lando.config.sources = _.sortBy(sources.concat(initSources), 'label'); - - // And finally the recipes - lando.config.recipes = _.sortBy(_.map(lando.config.inits, init => init.name), 'name'); - - // Load in all our tasks - return lando.Promise.resolve(lando.config.plugins) - // Make sure the tasks dir exists - .filter(plugin => fs.existsSync(plugin.tasks)) - // Get a list off full js files that exist in that dir - .map(plugin => _(fs.readdirSync(plugin.tasks)) - .map(file => path.join(plugin.tasks, file)) - .filter(path => _.endsWith(path, '.js')) - .value(), - ) - // Loadem and loggem - .then(tasks => _.flatten(tasks)) - .each(file => { - lando.tasks.push({...require(file)(lando, {}), file}); - lando.log.debug('autoloaded global task %s', path.basename(file, '.js')); - }) - // Reset the task cache - .then(() => { - lando.cache.set('_.tasks.cache', JSON.stringify(lando.tasks), {persist: true}); - }); +const bootstrapTasks = async lando => { + // if we already have cached tasks tehn load that + if (!lando.cache.get('_.tasks.cache')) await require('../hooks/lando-generate-tasks-cache')(lando); + // push it + lando.tasks.push(...JSON.parse(lando.cache.get('_.tasks.cache'))); }; /* diff --git a/package-lock.json b/package-lock.json index 45d424751..0f64b1899 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "@lando/core", - "version": "3.23.15", + "version": "3.24.0-beta.4", "license": "GPL-3.0", "dependencies": { "@lando/argv": "^1.1.2", diff --git a/utils/get-tasks.js b/utils/get-tasks.js index a7668c804..9a8a25ada 100644 --- a/utils/get-tasks.js +++ b/utils/get-tasks.js @@ -31,7 +31,8 @@ const loadCacheFile = file => { try { return JSON.parse(JSON.parse(fs.readFileSync(file, {encoding: 'utf-8'}))); } catch (e) { - throw new Error(`There was a problem with parsing ${file}. Ensure it is valid JSON! ${e}`); + fs.rmSync(file, {force: true, retryDelay: 201, maxRetries: 16, recursive: true}); + return {}; } };