new: Expand hasher and debugging. (#615)

* Add var. * Add new hash setting. * Debug windows. * Use paths. * Update to use paths. * Fix tests. * Polish. * Update changelog.
moonrepo · Feb 13, 2023 · f574541 · f574541
1 parent 3d2ff06
commit f574541
Show file tree

Hide file tree

Showing 20 changed files with 166 additions and 79 deletions.
diff --git a/crates/cli/tests/run_test.rs b/crates/cli/tests/run_test.rs
@@ -1,5 +1,5 @@
 use moon_cache::CacheEngine;
-use moon_config::WorkspaceConfig;
+use moon_config::{HasherWalkStrategy, WorkspaceConfig};
 use moon_test_utils::{
     assert_debug_snapshot, assert_snapshot, create_sandbox_with_config, get_cases_fixture_configs,
     predicates::{self, prelude::*},
@@ -423,6 +423,32 @@ mod hashing {
         assert_ne!(hash1, hash3);
         assert_ne!(hash2, hash3);
     }
+
+    #[test]
+    fn supports_diff_walking_strategies() {
+        let sandbox = cases_sandbox();
+        sandbox.enable_git();
+
+        sandbox.run_moon(|cmd| {
+            cmd.arg("run").arg("outputs:noOutput");
+        });
+
+        let hash_vcs = extract_hash_from_run(sandbox.path(), "outputs:noOutput");
+
+        // Run again with a different strategy
+        let sandbox = cases_sandbox_with_config(|workspace_config| {
+            workspace_config.hasher.walk_strategy = HasherWalkStrategy::Glob;
+        });
+        sandbox.enable_git();
+
+        sandbox.run_moon(|cmd| {
+            cmd.arg("run").arg("outputs:noOutput");
+        });
+
+        let hash_glob = extract_hash_from_run(sandbox.path(), "outputs:noOutput");
+
+        assert_eq!(hash_vcs, hash_glob);
+    }
 }
 
 mod outputs {

diff --git a/...s/cli/tests/snapshots/run_test__outputs__hydration__reuses_cache_from_previous_run-2.snap b/...s/cli/tests/snapshots/run_test__outputs__hydration__reuses_cache_from_previous_run-2.snap
@@ -1,6 +1,5 @@
 ---
 source: crates/cli/tests/run_test.rs
-assertion_line: 658
 expression: assert2.output()
 ---
 ▪▪▪▪ outputs:generateFileAndFolder (cached from previous run)

diff --git a/crates/cli/tests/snapshots/run_test__outputs__hydration__reuses_cache_from_previous_run.snap b/crates/cli/tests/snapshots/run_test__outputs__hydration__reuses_cache_from_previous_run.snap
@@ -1,7 +1,6 @@
 ---
 source: crates/cli/tests/run_test.rs
-assertion_line: 571
-expression: get_assert_output(&assert1)
+expression: assert1.output()
 ---
 ▪▪▪▪ npm install
 ▪▪▪▪ outputs:generateFileAndFolder

diff --git a/crates/core/config/src/workspace/hasher.rs b/crates/core/config/src/workspace/hasher.rs
@@ -11,10 +11,21 @@ pub enum HasherOptimization {
     Performance,
 }
 
+#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub enum HasherWalkStrategy {
+    Glob,
+    #[default]
+    Vcs,
+}
+
 #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize, Validate)]
 #[schemars(default)]
 #[serde(default, rename_all = "camelCase")]
 pub struct HasherConfig {
     #[serde(skip_serializing_if = "is_default")]
     pub optimization: HasherOptimization,
+
+    #[serde(skip_serializing_if = "is_default")]
+    pub walk_strategy: HasherWalkStrategy,
 }
diff --git a/...graph/tests/snapshots/dep_graph_test__run_target_if_touched__skips_if_untouched_task.snap b/...graph/tests/snapshots/dep_graph_test__run_target_if_touched__skips_if_untouched_task.snap
@@ -1,6 +1,5 @@
 ---
 source: crates/core/dep-graph/tests/dep_graph_test.rs
-assertion_line: 411
 expression: graph.to_dot()
 ---
 digraph {

diff --git a/crates/core/hasher/src/helpers.rs b/crates/core/hasher/src/helpers.rs
@@ -35,7 +35,7 @@ pub fn convert_paths_to_strings(
 
     for path in paths {
         // Inputs may not exist and `git hash-object` will fail if you pass an unknown file
-        if path.exists() {
+        if path.exists() && path.is_file() {
             // We also need to use relative paths from the workspace root,
             // so that it works across machines
             let rel_path = if path.starts_with(workspace_root) {

diff --git a/crates/core/project-graph/src/project_builder.rs b/crates/core/project-graph/src/project_builder.rs
@@ -228,7 +228,7 @@ impl<'ws> ProjectGraphBuilder<'ws> {
                 }
 
                 for glob in globs {
-                    args.push(handle_path(PathBuf::from(glob), true)?);
+                    args.push(handle_path(glob, true)?);
                 }
             } else if token_resolver.has_token_var(arg) {
                 args.push(token_resolver.resolve_vars(arg, task)?);
@@ -383,7 +383,7 @@ impl<'ws> ProjectGraphBuilder<'ws> {
         let (paths, globs) = token_resolver.resolve(&inputs_without_vars, task)?;
 
         task.input_paths.extend(paths);
-        task.input_globs.extend(globs);
+        task.input_globs.extend(self.normalize_glob_list(globs)?);
 
         Ok(())
     }
@@ -403,7 +403,7 @@ impl<'ws> ProjectGraphBuilder<'ws> {
         let (paths, globs) = token_resolver.resolve(&task.outputs, task)?;
 
         task.output_paths.extend(paths);
-        task.output_globs.extend(globs);
+        task.output_globs.extend(self.normalize_glob_list(globs)?);
 
         Ok(())
     }
@@ -596,4 +596,17 @@ impl<'ws> ProjectGraphBuilder<'ws> {
 
         Ok(hash)
     }
+
+    fn normalize_glob_list(&self, globs: Vec<PathBuf>) -> Result<Vec<String>, ProjectError> {
+        let mut normalized_globs = vec![];
+
+        for glob in globs {
+            normalized_globs.push(glob::normalize(
+                //  glob.strip_prefix(&self.workspace.root).unwrap(),
+                glob,
+            )?);
+        }
+
+        Ok(normalized_globs)
+    }
 }
diff --git a/crates/core/project-graph/src/token_resolver.rs b/crates/core/project-graph/src/token_resolver.rs
@@ -1,5 +1,4 @@
 use crate::errors::TokenError;
-use moon_config::FileGlob;
 use moon_logger::{color, warn};
 use moon_project::Project;
 use moon_task::Task;
@@ -10,7 +9,7 @@ use moon_utils::regex::{
 use moon_utils::{glob, path};
 use std::path::{Path, PathBuf};
 
-type PathsGlobsNormalized = (Vec<PathBuf>, Vec<FileGlob>);
+type PathsGlobsResolved = (Vec<PathBuf>, Vec<PathBuf>);
 
 #[derive(Debug, Eq, PartialEq)]
 pub enum TokenContext {
@@ -141,9 +140,9 @@ impl<'task> TokenResolver<'task> {
         &self,
         values: &[String],
         task: &Task,
-    ) -> Result<PathsGlobsNormalized, TokenError> {
+    ) -> Result<PathsGlobsResolved, TokenError> {
         let mut paths: Vec<PathBuf> = vec![];
-        let mut globs: Vec<String> = vec![];
+        let mut globs: Vec<PathBuf> = vec![];
 
         for value in values {
             if self.has_token_func(value) {
@@ -177,7 +176,7 @@ impl<'task> TokenResolver<'task> {
                 }
 
                 if is_glob {
-                    globs.push(glob::normalize(resolved)?);
+                    globs.push(resolved);
                 } else {
                     paths.push(resolved);
                 }
@@ -187,11 +186,7 @@ impl<'task> TokenResolver<'task> {
         Ok((paths, globs))
     }
 
-    pub fn resolve_func(
-        &self,
-        value: &str,
-        task: &Task,
-    ) -> Result<PathsGlobsNormalized, TokenError> {
+    pub fn resolve_func(&self, value: &str, task: &Task) -> Result<PathsGlobsResolved, TokenError> {
         let matches = TOKEN_FUNC_PATTERN.captures(value).unwrap();
         let token = matches.get(0).unwrap().as_str(); // @name(arg)
         let func = matches.get(1).unwrap().as_str(); // name
@@ -280,11 +275,11 @@ impl<'task> TokenResolver<'task> {
     fn replace_file_group_tokens(
         &self,
         token_type: TokenType,
-    ) -> Result<PathsGlobsNormalized, TokenError> {
+    ) -> Result<PathsGlobsResolved, TokenError> {
         token_type.check_context(&self.context)?;
 
         let mut paths: Vec<PathBuf> = vec![];
-        let mut globs: Vec<String> = vec![];
+        let mut globs: Vec<PathBuf> = vec![];
         let file_groups = &self.project.file_groups;
 
         let get_file_group = |token: &str, id: &str| {
@@ -326,7 +321,7 @@ impl<'task> TokenResolver<'task> {
         &self,
         token_type: TokenType,
         task: &Task,
-    ) -> Result<PathsGlobsNormalized, TokenError> {
+    ) -> Result<PathsGlobsResolved, TokenError> {
         token_type.check_context(&self.context)?;
 
         let mut paths = vec![];
@@ -341,7 +336,7 @@ impl<'task> TokenResolver<'task> {
             if glob::is_glob(input) {
                 match task.input_globs.iter().find(|g| g.ends_with(input)) {
                     Some(g) => {
-                        globs.push(g.clone());
+                        globs.push(PathBuf::from(g));
                     }
                     None => {
                         return Err(error);
@@ -370,11 +365,11 @@ impl<'task> TokenResolver<'task> {
         &self,
         token_type: TokenType,
         task: &Task,
-    ) -> Result<PathsGlobsNormalized, TokenError> {
+    ) -> Result<PathsGlobsResolved, TokenError> {
         token_type.check_context(&self.context)?;
 
         let mut paths: Vec<PathBuf> = vec![];
-        let mut globs: Vec<String> = vec![];
+        let mut globs: Vec<PathBuf> = vec![];
 
         if let TokenType::Out(token, index) = token_type {
             let error = TokenError::InvalidOutIndex(token.clone(), index);
@@ -389,7 +384,7 @@ impl<'task> TokenResolver<'task> {
             if glob::is_glob(output) {
                 match task.output_globs.iter().find(|g| g.ends_with(output)) {
                     Some(g) => {
-                        globs.push(g.clone());
+                        globs.push(PathBuf::from(g));
                     }
                     None => {
                         return Err(error);

diff --git a/crates/core/project-graph/tests/token_resolver_test.rs b/crates/core/project-graph/tests/token_resolver_test.rs
@@ -314,8 +314,8 @@ mod resolve_args {
             (
                 vec![],
                 vec![
-                    glob::normalize(project.root.join("**/*.{ts,tsx}")).unwrap(),
-                    glob::normalize(project.root.join("*.js")).unwrap()
+                    project.root.join("**/*.{ts,tsx}"),
+                    project.root.join("*.js")
                 ]
             )
         );
@@ -353,10 +353,7 @@ mod resolve_args {
 
         assert_eq!(
             resolver.resolve(&string_vec!["@in(0)"], &task).unwrap(),
-            (
-                vec![],
-                vec![glob::normalize(project.root.join("src/**/*")).unwrap()]
-            )
+            (vec![], vec![project.root.join("src/**/*")])
         );
     }
 
@@ -564,8 +561,8 @@ mod resolve_inputs {
             (
                 vec![],
                 vec![
-                    glob::normalize(project.root.join("**/*.{ts,tsx}")).unwrap(),
-                    glob::normalize(project.root.join("*.js")).unwrap()
+                    project.root.join("**/*.{ts,tsx}"),
+                    project.root.join("*.js")
                 ]
             ),
         );
@@ -617,14 +614,7 @@ mod resolve_inputs {
 
         assert_eq!(
             resolver.resolve(&string_vec!["dir"], &task).unwrap(),
-            (
-                vec![],
-                vec![if cfg!(windows) {
-                    glob::normalize(project.root.join("dir/**/*")).unwrap()
-                } else {
-                    project.root.join("dir/**/*").to_string_lossy().to_string()
-                }]
-            ),
+            (vec![], vec![project.root.join("dir/**/*")]),
         );
     }
 }
@@ -732,8 +722,8 @@ mod resolve_outputs {
             (
                 vec![],
                 vec![
-                    glob::normalize(project.root.join("**/*.{ts,tsx}")).unwrap(),
-                    glob::normalize(project.root.join("*.js")).unwrap()
+                    project.root.join("**/*.{ts,tsx}"),
+                    project.root.join("*.js")
                 ]
             ),
         );

diff --git a/crates/core/runner/src/errors.rs b/crates/core/runner/src/errors.rs
@@ -3,11 +3,15 @@ use moon_project::ProjectError;
 use moon_target::TargetError;
 use moon_task::TaskError;
 use moon_tool::ToolError;
+use moon_utils::glob::GlobError;
 use moon_workspace::{VcsError, WorkspaceError};
 use thiserror::Error;
 
 #[derive(Error, Debug)]
 pub enum RunnerError {
+    #[error(transparent)]
+    Glob(#[from] GlobError),
+
     #[error(transparent)]
     Moon(#[from] MoonError),