Add option for shallow clones

To speed up cloning big repositories, add an option for a shallow clone.

Author: PatKamin

devops/scripts/benchmarks/git_project.py

@@ -20,14 +20,16 @@ def __init__(
         name: str,
         force_rebuild: bool = False,
         no_suffix_src: bool = False,
+        shallow_clone: bool = False,
     ) -> None:
         self._url = url
         self._ref = ref
         self._directory = directory
         self._name = name
         self._force_rebuild = force_rebuild
         self._no_suffix_src = no_suffix_src
-        self._rebuild_needed = self._git_clone()
+        self._shallow_clone = shallow_clone
+        self._rebuild_needed = self._setup_repo()
 
     @property
     def src_dir(self) -> Path:
@@ -128,42 +130,80 @@ def install(self) -> None:
         """Installs the project."""
         run(f"cmake --install {self.build_dir}")
 
-    def _git_clone(self) -> bool:
+    def _git_clone(self) -> None:
+        """Clone the git repository."""
+        try:
+            log.debug(f"Cloning {self._url} into {self.src_dir} at commit {self._ref}")
+            git_clone_cmd = "git clone --recursive"
+            if self._shallow_clone:
+                git_clone_cmd += f" --depth 1 --branch {self._ref}"
+            git_clone_cmd += f" {self._url} {self.src_dir}"
+            run(git_clone_cmd)
+            run(f"git checkout {self._ref}", cwd=self.src_dir)
+            log.debug(f"Cloned {self._url} into {self.src_dir} at commit {self._ref}")
+        except Exception as e:
+            log.error(f"Failed to clone repository {self._url}: {e}")
+            raise
+
+    def _git_fetch(self) -> None:
+        """Fetch the latest changes from the remote repository."""
+        try:
+            log.debug(f"Fetching latest changes for {self._url} in {self.src_dir}")
+            run("git fetch", cwd=self.src_dir)
+            run("git reset --hard", cwd=self.src_dir)
+            run(f"git checkout {self._ref}", cwd=self.src_dir)
+            log.debug(f"Fetched latest changes for {self._url} in {self.src_dir}")
+        except Exception as e:
+            log.error(f"Failed to fetch updates for repository {self._url}: {e}")
+            raise
+
+    def _setup_repo(self) -> bool:
         """Clone a git repository into a specified directory at a specific commit.
         Returns:
             bool: True if the repository was cloned or updated, False if it was already up-to-date.
         """
-        log.debug(f"Cloning {self._url} into {self.src_dir} at commit {self._ref}")
-        if self.src_dir.exists() and Path(self.src_dir, ".git").exists():
+        if not self.src_dir.exists():
+            self._git_clone()
+            return True
+        elif Path(self.src_dir, ".git").exists():
             log.debug(
                 f"Repository {self._url} already exists at {self.src_dir}, checking for updates."
             )
-            run("git fetch", cwd=self.src_dir)
-            target_commit = (
-                run(f"git rev-parse {self._ref}", cwd=self.src_dir)
+            current_commit = (
+                run("git rev-parse HEAD^{commit}", cwd=self.src_dir)
                 .stdout.decode()
                 .strip()
             )
-            current_commit = (
-                run("git rev-parse HEAD", cwd=self.src_dir).stdout.decode().strip()
-            )
-            if current_commit != target_commit:
-                log.debug(
-                    f"Current commit {current_commit} does not match target {target_commit}, checking out {self._ref}."
+            try:
+                target_commit = (
+                    run(f"git rev-parse {self._ref}^{{commit}}", cwd=self.src_dir)
+                    .stdout.decode()
+                    .strip()
+                )
+                if current_commit != target_commit:
+                    log.debug(
+                        f"Current commit {current_commit} does not match target {target_commit}, checking out {self._ref}."
+                    )
+                    run("git reset --hard", cwd=self.src_dir)
+                    run(f"git checkout {self._ref}", cwd=self.src_dir)
+                    return True
+            except Exception:
+                log.error(
+                    f"Failed to resolve target commit {self._ref}. Fetching updates."
                 )
-                run("git reset --hard", cwd=self.src_dir)
-                run(f"git checkout {self._ref}", cwd=self.src_dir)
+                if self._shallow_clone:
+                    log.debug(f"Cloning a clean shallow copy.")
+                    shutil.rmtree(self.src_dir)
+                    self._git_clone()
+                    return True
+                else:
+                    self._git_fetch()
             else:
                 log.debug(
                     f"Current commit {current_commit} matches target {target_commit}, no update needed."
                 )
                 return False
-        elif not self.src_dir.exists():
-            run(f"git clone --recursive {self._url} {self.src_dir}")
-            run(f"git checkout {self._ref}", cwd=self.src_dir)
         else:
             raise Exception(
                 f"The directory {self.src_dir} exists but is not a git repository."
             )
-        log.debug(f"Cloned {self._url} into {self.src_dir} at commit {self._ref}")
-        return True

devops/scripts/benchmarks/utils/compute_runtime.py

@@ -103,6 +103,7 @@ def build_igc(self, repo, commit) -> tuple[Path, bool]:
                 Path(options.workdir),
                 "llvm-project",
                 no_suffix_src=True,
+                shallow_clone=True,
             )
             llvm_projects = llvm_project.src_dir / "llvm" / "projects"
             GitProject(