Add enableAutoDedupe option

packages/zpm-config/schema.json

@@ -56,6 +56,11 @@
       "description": "Folder where the versioning files are stored.",
       "default": ".yarn/versions"
     },
+    "enableAutoDedupe": {
+      "type": "boolean",
+      "description": "Whether to automatically deduplicate dependencies using the highest strategy during full-project installs",
+      "default": false
+    },
     "enableAutoTypes": {
       "type": "boolean",
       "description": "Whether to automatically add @types/ packages to the dependencies",

packages/zpm/src/commands/dedupe.rs

@@ -2,11 +2,10 @@ use std::{collections::BTreeMap, process::ExitCode, str::FromStr};
 
 use clipanion::cli;
 use indexmap::IndexMap;
-use itertools::Itertools;
-use zpm_primitives::{Descriptor, Ident, IdentGlob, Locator, Range, Reference, RegistryReference, RegistrySemverRange, ShorthandReference};
+use zpm_primitives::{Descriptor, IdentGlob, Locator};
 use zpm_utils::{tree, AbstractValue, ToFileString};
 
-use crate::{error::Error, project::{InstallMode, Project, RunInstallOptions}};
+use crate::{dedupe::prepare_highest_dedupe, error::Error, project::{InstallMode, Project, RunInstallOptions}};
 
 #[derive(Debug, Default)]
 enum DedupeStrategy {
@@ -87,10 +86,10 @@ impl Dedupe {
             = Project::new(None).await?;
 
         project
-            .lazy_install().await?;
+            .lazy_install_with_auto_dedupe(false).await?;
 
         let enforced_resolutions
-            = prepare_highest_dedupe(&project, &self.patterns)?;
+            = prepare_highest_dedupe(project.install_state.as_ref().ok_or(Error::InstallStateNotFound)?, &self.patterns);
 
         if self.check {
             if enforced_resolutions.is_empty() {
@@ -105,6 +104,7 @@ impl Dedupe {
                     .map(|(d, l)| (d, Some(l)))
                     .collect(),
                 mode: self.mode,
+                skip_auto_dedupe: true,
                 ..Default::default()
             }).await?;
 
@@ -181,94 +181,3 @@ impl Dedupe {
         Ok(())
     }
 }
-
-fn extract_semver_version(locator: &Locator) -> Option<(&Ident, &zpm_semver::Version)> {
-    match &locator.reference {
-        Reference::Shorthand(params)
-            => Some((&locator.ident, &params.version)),
-
-        Reference::Registry(params)
-            => Some((&params.ident, &params.version)),
-
-        _ => None,
-    }
-}
-
-fn prepare_highest_dedupe(project: &Project, patterns: &Vec<IdentGlob>) -> Result<BTreeMap<Descriptor, Locator>, Error> {
-    let install_state
-        = project.install_state.as_ref()
-            .ok_or(Error::InstallStateNotFound)?;
-
-    let locators_by_ident
-        = install_state.normalized_resolutions.keys()
-            .filter_map(extract_semver_version)
-            .into_group_map_by(|(ident, _)| *ident);
-
-    let best_version_for = |ident: &Ident, range: &zpm_semver::Range| {
-        locators_by_ident.get(&ident).unwrap().iter()
-            .map(|(_, version)| *version)
-            .filter(|version| range.check(version))
-            .max()
-    };
-
-    let attach_highest_version = |descriptor: &Descriptor| {
-        let suggested_locator = match &descriptor.range {
-            Range::AnonymousSemver(params) => {
-                let best_version
-                    = best_version_for(&descriptor.ident, &params.range);
-
-                best_version.map(|version| {
-                    Locator::new(descriptor.ident.clone(), ShorthandReference {
-                        version: version.clone(),
-                    }.into())
-                })
-            },
-
-            Range::RegistrySemver(RegistrySemverRange {ident: None, range}) => {
-                let best_version
-                    = best_version_for(&descriptor.ident, range);
-
-                best_version.map(|version| {
-                    Locator::new(descriptor.ident.clone(), ShorthandReference {
-                        version: version.clone(),
-                    }.into())
-                })
-            },
-
-            Range::RegistrySemver(RegistrySemverRange {ident: Some(ident), range}) => {
-                let best_version
-                    = best_version_for(ident, range);
-
-                best_version.map(|version| {
-                    Locator::new(descriptor.ident.clone(), RegistryReference {
-                        ident: ident.clone(),
-                        version: version.clone(),
-                        url: None,
-                    }.into())
-                })
-            },
-
-            _ => return None,
-        };
-
-        suggested_locator.and_then(|locator| {
-            let current_resolution
-                = install_state.descriptor_to_locator
-                    .get(descriptor);
-
-            if current_resolution != Some(&locator) {
-                Some((descriptor.clone(), locator))
-            } else {
-                None
-            }
-        })
-    };
-
-    let upgradable_candidates
-        = install_state.descriptor_to_locator.keys()
-            .filter(|descriptor| patterns.is_empty() || patterns.iter().any(|matcher| matcher.check(&descriptor.ident)))
-            .filter_map(attach_highest_version)
-            .collect::<BTreeMap<_, _>>();
-
-    Ok(upgradable_candidates)
-}

packages/zpm/src/commands/dlx.rs

@@ -159,6 +159,7 @@ pub async fn install_dependencies(workspace_path: &Path, loose_resolutions: Vec<
         .run_install(RunInstallOptions {
             silent_or_error: quiet,
             skip_constraints_check: true,
+            skip_auto_dedupe: true,
             enforced_resolutions,
             ..Default::default()
         }).await?;

packages/zpm/src/dedupe.rs

@@ -0,0 +1,94 @@
+use std::collections::BTreeMap;
+
+use itertools::Itertools;
+use zpm_primitives::{Descriptor, Ident, IdentGlob, Locator, Range, Reference, RegistryReference, RegistrySemverRange, ShorthandReference};
+
+use crate::install::InstallState;
+
+fn extract_semver_version(locator: &Locator) -> Option<(&Ident, &zpm_semver::Version)> {
+    match &locator.reference {
+        Reference::Shorthand(params)
+            => Some((&locator.ident, &params.version)),
+
+        Reference::Registry(params)
+            => Some((&params.ident, &params.version)),
+
+        _ => None,
+    }
+}
+
+pub fn prepare_highest_dedupe(install_state: &InstallState, patterns: &[IdentGlob]) -> BTreeMap<Descriptor, Locator> {
+
+    let locators_by_ident
+        = install_state.normalized_resolutions.keys()
+            .filter_map(extract_semver_version)
+            .into_group_map_by(|(ident, _)| *ident);
+
+    let best_version_for = |ident: &Ident, range: &zpm_semver::Range| {
+        locators_by_ident.get(&ident).into_iter().flatten()
+            .map(|(_, version)| *version)
+            .filter(|version| range.check(version))
+            .max()
+    };
+
+    let attach_highest_version = |descriptor: &Descriptor| {
+        let suggested_locator = match &descriptor.range {
+            Range::AnonymousSemver(params) => {
+                let best_version
+                    = best_version_for(&descriptor.ident, &params.range);
+
+                best_version.map(|version| {
+                    Locator::new(descriptor.ident.clone(), ShorthandReference {
+                        version: version.clone(),
+                    }.into())
+                })
+            },
+
+            Range::RegistrySemver(RegistrySemverRange {ident: None, range}) => {
+                let best_version
+                    = best_version_for(&descriptor.ident, range);
+
+                best_version.map(|version| {
+                    Locator::new(descriptor.ident.clone(), ShorthandReference {
+                        version: version.clone(),
+                    }.into())
+                })
+            },
+
+            Range::RegistrySemver(RegistrySemverRange {ident: Some(ident), range}) => {
+                let best_version
+                    = best_version_for(ident, range);
+
+                best_version.map(|version| {
+                    Locator::new(descriptor.ident.clone(), RegistryReference {
+                        ident: ident.clone(),
+                        version: version.clone(),
+                        url: None,
+                    }.into())
+                })
+            },
+
+            _ => return None,
+        };
+
+        suggested_locator.and_then(|locator| {
+            let current_resolution
+                = install_state.descriptor_to_locator
+                    .get(descriptor);
+
+            if current_resolution != Some(&locator) {
+                Some((descriptor.clone(), locator))
+            } else {
+                None
+            }
+        })
+    };
+
+    let upgradable_candidates
+        = install_state.descriptor_to_locator.keys()
+            .filter(|descriptor| patterns.is_empty() || patterns.iter().any(|matcher| matcher.check(&descriptor.ident)))
+            .filter_map(attach_highest_version)
+            .collect::<BTreeMap<_, _>>();
+
+    upgradable_candidates
+}

packages/zpm/src/lib.rs

@@ -7,6 +7,7 @@ pub mod commands;
 pub mod constraints;
 pub mod content_flags;
 pub mod daemon;
+pub mod dedupe;
 pub mod descriptor_loose;
 pub mod diff_finder;
 pub mod manifest_finder;

packages/zpm/src/project.rs

@@ -127,6 +127,7 @@ pub struct RunInstallOptions {
     pub silent_or_error: bool,
     /// Temporary installs have no constraints configuration to check.
     pub skip_constraints_check: bool,
+    pub skip_auto_dedupe: bool,
     pub json: bool,
     pub inline_builds: bool,
     pub force: bool,
@@ -1286,6 +1287,15 @@ impl Project {
             return Ok(false);
         }
 
+        // Explicit dedupe commands may have checked without changing anything,
+        // or only deduplicated selected patterns. Don't treat such a graph as
+        // current when a full install would still automatically deduplicate it.
+        if self.config.settings.enable_auto_dedupe.value
+            && !crate::dedupe::prepare_highest_dedupe(install_state, &[]).is_empty()
+        {
+            return Ok(false);
+        }
+
         if !self.config.settings.unstable_islands.is_empty() {
             return Ok(false);
         }
@@ -1460,6 +1470,10 @@ impl Project {
     }
 
     pub async fn lazy_install(&mut self) -> Result<(), Error> {
+        self.lazy_install_with_auto_dedupe(true).await
+    }
+
+    pub(crate) async fn lazy_install_with_auto_dedupe(&mut self, auto_dedupe: bool) -> Result<(), Error> {
         match self.import_install_state() {
             Ok(_) => {},
 
@@ -1538,6 +1552,7 @@ impl Project {
             prune_dev_dependencies: false,
             refresh_lockfile: false,
             silent_or_error: true,
+            skip_auto_dedupe: !auto_dedupe,
             mode: None,
             roots: install_roots,
             ..Default::default()
@@ -1650,7 +1665,7 @@ impl Project {
                 }
             }
 
-            let install_context
+            let mut install_context
                 = InstallContext::default()
                     .with_package_cache(Some(&package_cache))
                     .with_project(Some(self))
@@ -1664,25 +1679,45 @@ impl Project {
                     .with_systems(Some(&systems))
                     .with_background_writes(Some(background_writes.clone()));
 
-            let roots
+            let roots: BTreeSet<_>
                 = self.workspaces.iter()
                     .filter(|w| options.roots.as_ref().map_or(true, |r| r.contains(&w.name)))
                     .map(|w| w.descriptor())
                     .collect();
 
-            let install_result
-                = InstallManager::new()
-                    .with_context(install_context)
-                    .with_lockfile(lockfile?)
+            let make_manager = |context, lockfile| {
+                InstallManager::new()
+                    .with_context(context)
+                    .with_lockfile(lockfile)
                     .with_previous_state(self.install_state.as_ref())
-                    .with_roots(roots)
+                    .with_roots(roots.clone())
                     .with_installed_workspaces(options.roots.clone())
                     .with_constraints_check(!options.silent_or_error && !options.skip_constraints_check && self.config.settings.enable_constraints_checks.value && options.roots.is_none())
                     .with_skip_link_step(options.mode == Some(InstallMode::UpdateLockfile))
                     .with_skip_lockfile_update(options.roots.is_some())
                     .with_force(options.force)
-                    .resolve_and_fetch().await?
-                    .link_and_build(self).await?;
+            };
+
+            let mut install = make_manager(install_context.clone(), lockfile?)
+                .resolve_and_fetch().await?;
+
+            // Use the freshly resolved graph so versions introduced by this
+            // install can be reused. Focused installs preserve the lockfile's
+            // resolutions.
+            if self.config.settings.enable_auto_dedupe.value && !options.skip_auto_dedupe && options.roots.is_none() {
+                let dedupe_resolutions = crate::dedupe::prepare_highest_dedupe(&install.install_state, &[]);
+
+                if !dedupe_resolutions.is_empty() {
+                    install_context.enforced_resolutions = dedupe_resolutions.into_iter()
+                        .map(|(descriptor, locator)| (descriptor, Some(locator)))
+                        .collect();
+
+                    install = make_manager(install_context, install.lockfile)
+                        .resolve_and_fetch().await?;
+                }
+            }
+
+            let install_result = install.link_and_build(self).await?;
 
             Ok(install_result)
         }).await;

tests/acceptance-tests/pkg-tests-specs/sources/commands/dlx.test.js

@@ -97,10 +97,11 @@ describe(`Commands`, () => {
     );
 
     test(
-      `it should skip the caller's constraints for temporary installs while checking normal installs`,
+      `it should skip the caller's constraints for temporary installs with auto-dedupe enabled while checking normal installs`,
       makeTemporaryEnv({}, async ({path, run}) => {
         await yarn.writeConfiguration(path, {
           enableConstraintsChecks: true,
+          enableAutoDedupe: true,
         });
 
         await writeFile(`${path}/yarn.config.cjs`, [