diff --git a/packages/zpm-config/schema.json b/packages/zpm-config/schema.json index 64992a3f..9522050e 100644 --- a/packages/zpm-config/schema.json +++ b/packages/zpm-config/schema.json @@ -56,6 +56,11 @@ "description": "Folder where the versioning files are stored.", "default": ".yarn/versions" }, + "enableAutoDedupe": { + "type": "boolean", + "description": "Whether to automatically deduplicate dependencies using the highest strategy during full-project installs", + "default": false + }, "enableAutoTypes": { "type": "boolean", "description": "Whether to automatically add @types/ packages to the dependencies", diff --git a/packages/zpm/src/commands/dedupe.rs b/packages/zpm/src/commands/dedupe.rs index 9cb0892c..3cd07a3d 100644 --- a/packages/zpm/src/commands/dedupe.rs +++ b/packages/zpm/src/commands/dedupe.rs @@ -2,11 +2,10 @@ use std::{collections::BTreeMap, process::ExitCode, str::FromStr}; use clipanion::cli; use indexmap::IndexMap; -use itertools::Itertools; -use zpm_primitives::{Descriptor, Ident, IdentGlob, Locator, Range, Reference, RegistryReference, RegistrySemverRange, ShorthandReference}; +use zpm_primitives::{Descriptor, IdentGlob, Locator}; use zpm_utils::{tree, AbstractValue, ToFileString}; -use crate::{error::Error, project::{InstallMode, Project, RunInstallOptions}}; +use crate::{dedupe::prepare_highest_dedupe, error::Error, project::{InstallMode, Project, RunInstallOptions}}; #[derive(Debug, Default)] enum DedupeStrategy { @@ -87,10 +86,10 @@ impl Dedupe { = Project::new(None).await?; project - .lazy_install().await?; + .lazy_install_with_auto_dedupe(false).await?; let enforced_resolutions - = prepare_highest_dedupe(&project, &self.patterns)?; + = prepare_highest_dedupe(project.install_state.as_ref().ok_or(Error::InstallStateNotFound)?, &self.patterns); if self.check { if enforced_resolutions.is_empty() { @@ -105,6 +104,7 @@ impl Dedupe { .map(|(d, l)| (d, Some(l))) .collect(), mode: self.mode, + skip_auto_dedupe: true, ..Default::default() }).await?; @@ -181,94 +181,3 @@ impl Dedupe { Ok(()) } } - -fn extract_semver_version(locator: &Locator) -> Option<(&Ident, &zpm_semver::Version)> { - match &locator.reference { - Reference::Shorthand(params) - => Some((&locator.ident, ¶ms.version)), - - Reference::Registry(params) - => Some((¶ms.ident, ¶ms.version)), - - _ => None, - } -} - -fn prepare_highest_dedupe(project: &Project, patterns: &Vec) -> Result, Error> { - let install_state - = project.install_state.as_ref() - .ok_or(Error::InstallStateNotFound)?; - - let locators_by_ident - = install_state.normalized_resolutions.keys() - .filter_map(extract_semver_version) - .into_group_map_by(|(ident, _)| *ident); - - let best_version_for = |ident: &Ident, range: &zpm_semver::Range| { - locators_by_ident.get(&ident).unwrap().iter() - .map(|(_, version)| *version) - .filter(|version| range.check(version)) - .max() - }; - - let attach_highest_version = |descriptor: &Descriptor| { - let suggested_locator = match &descriptor.range { - Range::AnonymousSemver(params) => { - let best_version - = best_version_for(&descriptor.ident, ¶ms.range); - - best_version.map(|version| { - Locator::new(descriptor.ident.clone(), ShorthandReference { - version: version.clone(), - }.into()) - }) - }, - - Range::RegistrySemver(RegistrySemverRange {ident: None, range}) => { - let best_version - = best_version_for(&descriptor.ident, range); - - best_version.map(|version| { - Locator::new(descriptor.ident.clone(), ShorthandReference { - version: version.clone(), - }.into()) - }) - }, - - Range::RegistrySemver(RegistrySemverRange {ident: Some(ident), range}) => { - let best_version - = best_version_for(ident, range); - - best_version.map(|version| { - Locator::new(descriptor.ident.clone(), RegistryReference { - ident: ident.clone(), - version: version.clone(), - url: None, - }.into()) - }) - }, - - _ => return None, - }; - - suggested_locator.and_then(|locator| { - let current_resolution - = install_state.descriptor_to_locator - .get(descriptor); - - if current_resolution != Some(&locator) { - Some((descriptor.clone(), locator)) - } else { - None - } - }) - }; - - let upgradable_candidates - = install_state.descriptor_to_locator.keys() - .filter(|descriptor| patterns.is_empty() || patterns.iter().any(|matcher| matcher.check(&descriptor.ident))) - .filter_map(attach_highest_version) - .collect::>(); - - Ok(upgradable_candidates) -} diff --git a/packages/zpm/src/commands/dlx.rs b/packages/zpm/src/commands/dlx.rs index f40008b0..616eedf1 100644 --- a/packages/zpm/src/commands/dlx.rs +++ b/packages/zpm/src/commands/dlx.rs @@ -159,6 +159,7 @@ pub async fn install_dependencies(workspace_path: &Path, loose_resolutions: Vec< .run_install(RunInstallOptions { silent_or_error: quiet, skip_constraints_check: true, + skip_auto_dedupe: true, enforced_resolutions, ..Default::default() }).await?; diff --git a/packages/zpm/src/dedupe.rs b/packages/zpm/src/dedupe.rs new file mode 100644 index 00000000..dc9e3599 --- /dev/null +++ b/packages/zpm/src/dedupe.rs @@ -0,0 +1,94 @@ +use std::collections::BTreeMap; + +use itertools::Itertools; +use zpm_primitives::{Descriptor, Ident, IdentGlob, Locator, Range, Reference, RegistryReference, RegistrySemverRange, ShorthandReference}; + +use crate::install::InstallState; + +fn extract_semver_version(locator: &Locator) -> Option<(&Ident, &zpm_semver::Version)> { + match &locator.reference { + Reference::Shorthand(params) + => Some((&locator.ident, ¶ms.version)), + + Reference::Registry(params) + => Some((¶ms.ident, ¶ms.version)), + + _ => None, + } +} + +pub fn prepare_highest_dedupe(install_state: &InstallState, patterns: &[IdentGlob]) -> BTreeMap { + + let locators_by_ident + = install_state.normalized_resolutions.keys() + .filter_map(extract_semver_version) + .into_group_map_by(|(ident, _)| *ident); + + let best_version_for = |ident: &Ident, range: &zpm_semver::Range| { + locators_by_ident.get(&ident).into_iter().flatten() + .map(|(_, version)| *version) + .filter(|version| range.check(version)) + .max() + }; + + let attach_highest_version = |descriptor: &Descriptor| { + let suggested_locator = match &descriptor.range { + Range::AnonymousSemver(params) => { + let best_version + = best_version_for(&descriptor.ident, ¶ms.range); + + best_version.map(|version| { + Locator::new(descriptor.ident.clone(), ShorthandReference { + version: version.clone(), + }.into()) + }) + }, + + Range::RegistrySemver(RegistrySemverRange {ident: None, range}) => { + let best_version + = best_version_for(&descriptor.ident, range); + + best_version.map(|version| { + Locator::new(descriptor.ident.clone(), ShorthandReference { + version: version.clone(), + }.into()) + }) + }, + + Range::RegistrySemver(RegistrySemverRange {ident: Some(ident), range}) => { + let best_version + = best_version_for(ident, range); + + best_version.map(|version| { + Locator::new(descriptor.ident.clone(), RegistryReference { + ident: ident.clone(), + version: version.clone(), + url: None, + }.into()) + }) + }, + + _ => return None, + }; + + suggested_locator.and_then(|locator| { + let current_resolution + = install_state.descriptor_to_locator + .get(descriptor); + + if current_resolution != Some(&locator) { + Some((descriptor.clone(), locator)) + } else { + None + } + }) + }; + + let upgradable_candidates + = install_state.descriptor_to_locator.keys() + .filter(|descriptor| patterns.is_empty() || patterns.iter().any(|matcher| matcher.check(&descriptor.ident))) + .filter_map(attach_highest_version) + .collect::>(); + + upgradable_candidates +} diff --git a/packages/zpm/src/lib.rs b/packages/zpm/src/lib.rs index c1b5168c..b7cee362 100644 --- a/packages/zpm/src/lib.rs +++ b/packages/zpm/src/lib.rs @@ -7,6 +7,7 @@ pub mod commands; pub mod constraints; pub mod content_flags; pub mod daemon; +pub mod dedupe; pub mod descriptor_loose; pub mod diff_finder; pub mod manifest_finder; diff --git a/packages/zpm/src/project.rs b/packages/zpm/src/project.rs index 35dc9d23..8f8b0d2b 100644 --- a/packages/zpm/src/project.rs +++ b/packages/zpm/src/project.rs @@ -127,6 +127,7 @@ pub struct RunInstallOptions { pub silent_or_error: bool, /// Temporary installs have no constraints configuration to check. pub skip_constraints_check: bool, + pub skip_auto_dedupe: bool, pub json: bool, pub inline_builds: bool, pub force: bool, @@ -1286,6 +1287,15 @@ impl Project { return Ok(false); } + // Explicit dedupe commands may have checked without changing anything, + // or only deduplicated selected patterns. Don't treat such a graph as + // current when a full install would still automatically deduplicate it. + if self.config.settings.enable_auto_dedupe.value + && !crate::dedupe::prepare_highest_dedupe(install_state, &[]).is_empty() + { + return Ok(false); + } + if !self.config.settings.unstable_islands.is_empty() { return Ok(false); } @@ -1460,6 +1470,10 @@ impl Project { } pub async fn lazy_install(&mut self) -> Result<(), Error> { + self.lazy_install_with_auto_dedupe(true).await + } + + pub(crate) async fn lazy_install_with_auto_dedupe(&mut self, auto_dedupe: bool) -> Result<(), Error> { match self.import_install_state() { Ok(_) => {}, @@ -1538,6 +1552,7 @@ impl Project { prune_dev_dependencies: false, refresh_lockfile: false, silent_or_error: true, + skip_auto_dedupe: !auto_dedupe, mode: None, roots: install_roots, ..Default::default() @@ -1650,7 +1665,7 @@ impl Project { } } - let install_context + let mut install_context = InstallContext::default() .with_package_cache(Some(&package_cache)) .with_project(Some(self)) @@ -1664,25 +1679,45 @@ impl Project { .with_systems(Some(&systems)) .with_background_writes(Some(background_writes.clone())); - let roots + let roots: BTreeSet<_> = self.workspaces.iter() .filter(|w| options.roots.as_ref().map_or(true, |r| r.contains(&w.name))) .map(|w| w.descriptor()) .collect(); - let install_result - = InstallManager::new() - .with_context(install_context) - .with_lockfile(lockfile?) + let make_manager = |context, lockfile| { + InstallManager::new() + .with_context(context) + .with_lockfile(lockfile) .with_previous_state(self.install_state.as_ref()) - .with_roots(roots) + .with_roots(roots.clone()) .with_installed_workspaces(options.roots.clone()) .with_constraints_check(!options.silent_or_error && !options.skip_constraints_check && self.config.settings.enable_constraints_checks.value && options.roots.is_none()) .with_skip_link_step(options.mode == Some(InstallMode::UpdateLockfile)) .with_skip_lockfile_update(options.roots.is_some()) .with_force(options.force) - .resolve_and_fetch().await? - .link_and_build(self).await?; + }; + + let mut install = make_manager(install_context.clone(), lockfile?) + .resolve_and_fetch().await?; + + // Use the freshly resolved graph so versions introduced by this + // install can be reused. Focused installs preserve the lockfile's + // resolutions. + if self.config.settings.enable_auto_dedupe.value && !options.skip_auto_dedupe && options.roots.is_none() { + let dedupe_resolutions = crate::dedupe::prepare_highest_dedupe(&install.install_state, &[]); + + if !dedupe_resolutions.is_empty() { + install_context.enforced_resolutions = dedupe_resolutions.into_iter() + .map(|(descriptor, locator)| (descriptor, Some(locator))) + .collect(); + + install = make_manager(install_context, install.lockfile) + .resolve_and_fetch().await?; + } + } + + let install_result = install.link_and_build(self).await?; Ok(install_result) }).await; diff --git a/tests/acceptance-tests/pkg-tests-specs/sources/commands/dlx.test.js b/tests/acceptance-tests/pkg-tests-specs/sources/commands/dlx.test.js index 1ac5d539..81aa35a8 100644 --- a/tests/acceptance-tests/pkg-tests-specs/sources/commands/dlx.test.js +++ b/tests/acceptance-tests/pkg-tests-specs/sources/commands/dlx.test.js @@ -97,10 +97,11 @@ describe(`Commands`, () => { ); test( - `it should skip the caller's constraints for temporary installs while checking normal installs`, + `it should skip the caller's constraints for temporary installs with auto-dedupe enabled while checking normal installs`, makeTemporaryEnv({}, async ({path, run}) => { await yarn.writeConfiguration(path, { enableConstraintsChecks: true, + enableAutoDedupe: true, }); await writeFile(`${path}/yarn.config.cjs`, [ diff --git a/tests/acceptance-tests/pkg-tests-specs/sources/features/autoDedupe.test.ts b/tests/acceptance-tests/pkg-tests-specs/sources/features/autoDedupe.test.ts new file mode 100644 index 00000000..380813e7 --- /dev/null +++ b/tests/acceptance-tests/pkg-tests-specs/sources/features/autoDedupe.test.ts @@ -0,0 +1,142 @@ +import {Filename, PortablePath, ppath, xfs} from '@yarnpkg/fslib'; +import {tests} from 'pkg-tests-core'; + +const {setPackageWhitelist} = tests; + +describe(`Features`, () => { + describe(`enableAutoDedupe`, () => { + for (const command of [[`install`], [`install`, `--silent`], [`install`, `--mode=update-lockfile`], [`run`, `check`], [`node`, `-e`, ``], [`add`, `no-deps@1.1.0`], [`up`, `no-deps@1.1.0`], [`remove`, `one-range-dep-too`]]) { + it( + `should dedupe after ${command.join(` `)}`, + makeTemporaryEnv({scripts: {check: `node -e ""`}}, async ({path, run, source}) => { + await setPackageWhitelist(new Map([[`no-deps`, new Set([`1.0.0`])]]), async () => { + await run(`add`, `one-range-dep`, `one-range-dep-too`); + }); + await run(`add`, `no-deps@1.1.0`); + + await expect(run(`dedupe`, `--check`)).rejects.toMatchObject({code: 1}); + await run(...command, {enableAutoDedupe: true}); + + if (command.includes(`--mode=update-lockfile`)) { + const lockfile = await xfs.readJsonPromise(ppath.join(path, Filename.lockfile)); + expect(lockfile.entries[`no-deps@npm:1.1.0, no-deps@npm:^1.0.0`].resolution.version).toEqual(`1.1.0`); + // Lockfile-only installs intentionally leave the linked tree unchanged. + await expect(source(`require('one-range-dep').dependencies['no-deps'].version`)).resolves.toEqual(`1.0.0`); + await run(`install`, `--force`); + } + + await expect(run(`dedupe`, `--check`)).resolves.toMatchObject({code: 0}); + await expect(source(`require('one-range-dep').dependencies['no-deps'].version`)).resolves.toEqual(`1.1.0`); + }), + ); + } + + for (const command of [[`install`], [`run`, `check`]]) { + it( + `should skip a repeated install after ${command.join(` `)} deduplicates the project`, + makeTemporaryEnv({scripts: {check: `node -e ""`}}, async ({path, run}) => { + await setPackageWhitelist(new Map([[`no-deps`, new Set([`1.0.0`])]]), async () => { + await run(`add`, `one-range-dep`); + }); + await run(`add`, `no-deps@1.1.0`); + await run(...command, {enableAutoDedupe: true}); + + const lockfilePath = ppath.join(path, Filename.lockfile); + const before = await xfs.readFilePromise(lockfilePath, `utf8`); + const {stdout} = await run(`install`, {enableAutoDedupe: true}); + expect(stdout).toContain(`All dependencies are up-to-date, nothing to do.`); + expect(await xfs.readFilePromise(lockfilePath, `utf8`)).toEqual(before); + await expect(run(`dedupe`, `--check`, {enableAutoDedupe: true})).resolves.toMatchObject({code: 0}); + }), + ); + } + + it( + `should preserve explicit dedupe checks and patterns before a full install`, + makeTemporaryEnv({}, {enableAutoDedupe: true}, async ({run, source}) => { + await setPackageWhitelist(new Map([[`no-deps`, new Set([`1.0.0`])]]), async () => { + await run(`add`, `one-range-dep`, {enableAutoDedupe: false}); + }); + await run(`add`, `no-deps@1.1.0`, {enableAutoDedupe: false}); + + await expect(run(`dedupe`, `--check`)).rejects.toMatchObject({code: 1}); + await run(`dedupe`, `unrelated-package`); + await expect(run(`dedupe`, `--check`)).rejects.toMatchObject({code: 1}); + + await run(`install`); + await expect(source(`require('one-range-dep').dependencies['no-deps'].version`)).resolves.toEqual(`1.1.0`); + }), + ); + + it( + `should preserve focused resolutions and dedupe when installing the full project`, + makeTemporaryEnv({private: true, workspaces: [`workspace`]}, async ({path, run, source}) => { + const workspacePath = ppath.join(path, `workspace` as PortablePath); + await xfs.mkdirpPromise(workspacePath); + await xfs.writeJsonPromise(ppath.join(workspacePath, Filename.manifest), {name: `workspace`}); + await setPackageWhitelist(new Map([[`no-deps`, new Set([`1.0.0`])]]), async () => { + await run(`add`, `one-range-dep`, {cwd: workspacePath}); + }); + await run(`add`, `no-deps@1.1.0`, {cwd: workspacePath}); + + const lockfilePath = ppath.join(path, Filename.lockfile); + const before = await xfs.readFilePromise(lockfilePath, `utf8`); + await run(`workspaces`, `focus`, `workspace`, {enableAutoDedupe: true}); + expect(await xfs.readFilePromise(lockfilePath, `utf8`)).toEqual(before); + + await run(`install`, {enableAutoDedupe: true}); + await expect(source(`require('one-range-dep').dependencies['no-deps'].version`, {cwd: workspacePath})).resolves.toEqual(`1.1.0`); + }), + ); + + it( + `should reuse versions introduced by add`, + makeTemporaryEnv({}, {enableAutoDedupe: true}, async ({run, source}) => { + await setPackageWhitelist(new Map([[`no-deps`, new Set([`1.0.0`])]]), async () => { + await run(`add`, `one-range-dep`, `one-range-dep-too`); + }); + await run(`add`, `no-deps@1.1.0`); + await expect(source(`require('one-range-dep').dependencies['no-deps'].version`)).resolves.toEqual(`1.1.0`); + await expect(run(`dedupe`, `--check`)).resolves.toMatchObject({code: 0}); + }), + ); + + it( + `should allow disabling automatic deduplication`, + makeTemporaryEnv({}, {enableAutoDedupe: true}, async ({run}) => { + await setPackageWhitelist(new Map([[`no-deps`, new Set([`1.0.0`])]]), async () => { + await run(`add`, `one-range-dep`); + }); + await run(`add`, `no-deps@1.1.0`, {enableAutoDedupe: false}); + await expect(run(`dedupe`, `--check`)).rejects.toMatchObject({code: 1}); + }), + ); + + it( + `should preserve incompatible dependency ranges`, + makeTemporaryEnv({}, {enableAutoDedupe: true}, async ({run, source}) => { + await setPackageWhitelist(new Map([[`no-deps`, new Set([`1.0.0`])]]), async () => { + await run(`add`, `one-range-dep`); + }); + await run(`add`, `no-deps@2.0.0`); + await expect(source(`require('one-range-dep').dependencies['no-deps'].version`)).resolves.toEqual(`1.0.0`); + await expect(source(`require('no-deps').version`)).resolves.toEqual(`2.0.0`); + }), + ); + + it( + `should respect immutable installs`, + makeTemporaryEnv({}, async ({path, run}) => { + await setPackageWhitelist(new Map([[`no-deps`, new Set([`1.0.0`])]]), async () => { + await run(`add`, `one-range-dep`, `one-range-dep-too`); + }); + await run(`add`, `no-deps@1.1.0`); + const lockfilePath = ppath.join(path, Filename.lockfile); + const before = await xfs.readFilePromise(lockfilePath, `utf8`); + + await expect(run(`install`, `--immutable`, {enableAutoDedupe: true})).rejects.toMatchObject({code: 1}); + expect(await xfs.readFilePromise(lockfilePath, `utf8`)).toEqual(before); + }), + ); + }); +}); diff --git a/website/config/yarnrc.json b/website/config/yarnrc.json index 769998c1..73c674ce 100644 --- a/website/config/yarnrc.json +++ b/website/config/yarnrc.json @@ -76,6 +76,14 @@ "format": "uri-reference", "default": ".yarn/versions" }, + "enableAutoDedupe": { + "_package": "@yarnpkg/core", + "title": "Define whether Yarn should automatically deduplicate dependencies.", + "description": "When enabled, Yarn automatically deduplicates dependencies during full-project installs using the `highest` strategy, reusing the highest compatible versions already present in the dependency graph. This also applies to installs triggered by commands such as `yarn add` and automatic installs before running scripts. Focused installs preserve the lockfile's resolutions, and explicit dedupe commands retain control over their patterns and checks. Immutable installs fail if deduplication would change the lockfile.", + "type": "boolean", + "default": false, + "examples": [true] + }, "enableConstraintsChecks": { "_package": "@yarnpkg/plugin-constraints", "title": "Define whether constraints should run on every install.",