Refactored my Cargo to work with the newly updated common functions

Signed-off-by: ziad hany <[email protected]>

Author: ziadhany

minecode_pipelines/miners/cargo.py

@@ -6,40 +6,58 @@
 # See https://github.com/aboutcode-org/purldb for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-from minecode_pipelines.pipes import get_last_commit
+from datetime import datetime
+
+from minecode_pipelines.pipes import fetch_checkpoint_from_github
+from minecode_pipelines.pipes import update_checkpoints_in_github
+from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO
 from minecode_pipelines.pipes import get_changed_files
-from minecode_pipelines.pipes import update_last_commit
 from minecode_pipelines.pipes.cargo import store_cargo_packages
+from scanpipe.pipes.federatedcode import commit_changes
+from scanpipe.pipes.federatedcode import push_changes
+from minecode_pipelines import VERSION
+
 import json
 from pathlib import Path
 
 from minecode_pipelines.utils import get_next_x_commit
 
+PACKAGE_BATCH_SIZE = 500
+CARGO_CHECKPOINT_PATH = "cargo/checkpoints.json"
 
-def process_cargo_packages(cargo_repo, fed_repo, fed_conf_repo, logger):
+
+def process_cargo_packages(cargo_index_repo, cloned_data_repo, config_repo, logger):
     """
     Process Cargo index files commit by commit.
     Push changes to fed_repo after:
-    - every `commit_batch` commits, OR
-    - when reaching HEAD.
+    - every `commit_batch` commits, OR when reaching HEAD.
     """
 
-    base_path = Path(cargo_repo.working_tree_dir)
+    base_path = Path(cargo_index_repo.working_tree_dir)
 
     while True:
-        setting_last_commit = get_last_commit(fed_conf_repo, "cargo")
-        next_commit = get_next_x_commit(cargo_repo, setting_last_commit, x=10, branch="master")
+        cargo_checkpoints = (
+            fetch_checkpoint_from_github(MINECODE_PIPELINES_CONFIG_REPO, CARGO_CHECKPOINT_PATH)
+            or {}
+        )
+        checkpoints_last_commit = cargo_checkpoints.get("last_commit")
 
-        if next_commit == setting_last_commit:
+        next_commit = get_next_x_commit(
+            cargo_index_repo, checkpoints_last_commit, x=10, branch="master"
+        )
+
+        if next_commit == checkpoints_last_commit:
             logger("No new commits to mine")
             break
 
         changed_files = get_changed_files(
-            cargo_repo, commit_x=setting_last_commit, commit_y=next_commit
+            cargo_index_repo, commit_x=checkpoints_last_commit, commit_y=next_commit
         )
         logger(f"Found {len(changed_files)} changed files in Cargo index.")
 
         file_counter = 0
+        purl_files = []
+        purls = []
         for idx, rel_path in enumerate(changed_files):
             file_path = base_path / rel_path
             logger(f"Found {file_path}.")
@@ -57,8 +75,45 @@ def process_cargo_packages(cargo_repo, fed_repo, fed_conf_repo, logger):
                         packages.append(json.loads(line))
 
             file_counter += 1
-            push_commit = (file_counter % 1000 == 0) or (idx == len(changed_files))
-            store_cargo_packages(packages, fed_repo, push_commit)
+            commit_and_push = (file_counter % PACKAGE_BATCH_SIZE == 0) or (
+                idx == len(changed_files)
+            )
+            purl_file, base_purl = store_cargo_packages(packages, cloned_data_repo)
+            logger(f"writing packageURLs for package: {base_purl} at: {purl_file}")
+
+            purl_files.append(purl_file)
+            purls.append(str(base_purl))
+            if not commit_and_push:
+                continue
+
+            commit_changes(
+                repo=cloned_data_repo,
+                files_to_commit=purl_files,
+                purls=purls,
+                mine_type="packageURL",
+                tool_name="pkg:cargo/minecode-pipelines",
+                tool_version=VERSION,
+            )
+
+            # Push changes to remote repository
+            push_changes(repo=cloned_data_repo)
+            purl_files = []
+            purls = []
+
+            if logger:
+                logger(
+                    f"Updating checkpoint at: {CARGO_CHECKPOINT_PATH} with last commit: {checkpoints_last_commit}"
+                )
+
+            settings_data = {
+                "date": str(datetime.now()),
+                "last_commit": next_commit,
+            }
+
+            update_checkpoints_in_github(
+                checkpoint=settings_data,
+                cloned_repo=config_repo,
+                path=CARGO_CHECKPOINT_PATH,
+            )
 
-        update_last_commit(next_commit, fed_conf_repo, "cargo")
-        logger(f"Pushed batch for commit range {setting_last_commit}:{next_commit}.")
+        logger(f"Pushed batch for commit range {checkpoints_last_commit}:{next_commit}.")

minecode_pipelines/pipelines/mine_cargo.py

@@ -19,22 +19,18 @@
 #
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
-import os
 
-from git.repo.base import Repo
-from scanpipe.pipes.federatedcode import delete_local_clone
-from minecode_pipelines.utils import get_temp_file
+import os
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import federatedcode
 from minecode_pipelines.miners import cargo
+from minecode_pipelines import pipes
+from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO
 
-FEDERATEDCODE_CARGO_GIT_URL = os.environ.get(
-    "FEDERATEDCODE_CARGO_GIT_URL", "https://github.com/ziadhany/cargo-test"
-)
-
-FEDERATEDCODE_CONFIG_GIT_URL = os.environ.get(
-    "FEDERATEDCODE_CONFIG_GIT_URL", "https://github.com/ziadhany/federatedcode-config"
+MINECODE_DATA_CARGO_REPO = os.environ.get(
+    "MINECODE_DATA_CARGO_REPO", "https://github.com/aboutcode-data/minecode-data-cargo-test"
 )
+MINECODE_CARGO_INDEX_REPO = "https://github.com/rust-lang/crates.io-index"
 
 
 class MineandPublishCargoPURLs(Pipeline):
@@ -45,39 +41,43 @@ def steps(cls):
         return (
             cls.check_federatedcode_eligibility,
             cls.clone_cargo_repo,
-            cls.collect_packages_from_cargo,
-            cls.clean_cargo_repo,
+            cls.mine_and_publish_cargo_packageurls,
+            cls.delete_cloned_repos,
         )
 
     def check_federatedcode_eligibility(self):
         """
         Check if the project fulfills the following criteria for
         pushing the project result to FederatedCode.
         """
-        federatedcode.check_federatedcode_configured_and_available(project=self.project)
+        federatedcode.check_federatedcode_configured_and_available(logger=self.log)
 
     def clone_cargo_repo(self):
         """
         Clone the repo at repo_url and return the Repo object
         """
-        conan_repo_url = "https://github.com/rust-lang/crates.io-index"
-
-        self.fed_repo = federatedcode.clone_repository(FEDERATEDCODE_CARGO_GIT_URL)
-        self.fed_conf_repo = federatedcode.clone_repository(FEDERATEDCODE_CONFIG_GIT_URL)
-        self.cargo_repo = Repo.clone_from(conan_repo_url, get_temp_file())
+        self.cargo_index_repo = federatedcode.clone_repository(MINECODE_CARGO_INDEX_REPO)
+        self.cloned_data_repo = federatedcode.clone_repository(MINECODE_DATA_CARGO_REPO)
+        self.cloned_config_repo = federatedcode.clone_repository(MINECODE_PIPELINES_CONFIG_REPO)
 
-    def collect_packages_from_cargo(self):
-        cargo.process_cargo_packages(self.cargo_repo, self.fed_repo, self.fed_conf_repo, self.log)
+        if self.log:
+            self.log(
+                f"{MINECODE_CARGO_INDEX_REPO} repo cloned at: {self.cargo_index_repo.working_dir}"
+            )
+            self.log(
+                f"{MINECODE_DATA_CARGO_REPO} repo cloned at: {self.cloned_data_repo.working_dir}"
+            )
+            self.log(
+                f"{MINECODE_PIPELINES_CONFIG_REPO} repo cloned at: {self.cloned_config_repo.working_dir}"
+            )
 
-    def clean_cargo_repo(self):
-        """
-        Delete the federatedcode repository if it exists, and also delete the Cargo repository if it exists.
-        """
-        if self.cargo_repo:
-            delete_local_clone(self.cargo_repo)
-
-        if self.fed_repo:
-            delete_local_clone(self.fed_repo)
+    def mine_and_publish_cargo_packageurls(self):
+        cargo.process_cargo_packages(
+            self.cargo_index_repo, self.cloned_data_repo, self.cloned_config_repo, self.log
+        )
 
-        if self.fed_conf_repo:
-            delete_local_clone(self.fed_repo)
+    def delete_cloned_repos(self):
+        pipes.delete_cloned_repos(
+            repos=[self.cargo_index_repo, self.cloned_data_repo, self.cloned_config_repo],
+            logger=self.log,
+        )

minecode_pipelines/pipes/__init__.py

@@ -15,6 +15,7 @@
 from pathlib import Path
 
 from aboutcode.hashid import PURLS_FILENAME
+from git import Repo
 
 from scanpipe.pipes.federatedcode import delete_local_clone
 from scanpipe.pipes.federatedcode import commit_and_push_changes
@@ -106,3 +107,60 @@ def delete_cloned_repos(repos, logger=None):
         if logger:
             logger(f"Deleting local clone at: {repo.working_dir}")
         delete_local_clone(repo)
+
+
+def get_changed_files(repo: Repo, commit_x: str = None, commit_y: str = None):
+    """
+    Return a list of files changed between two commits using GitPython.
+    Includes added, modified, deleted, and renamed files.
+    - commit_x: base commit (or the empty tree hash for the first commit)
+    - commit_y: target commit (defaults to HEAD if not provided)
+    """
+    EMPTY_TREE_HASH = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
+
+    if commit_y is None:
+        commit_y = repo.head.commit.hexsha
+    commit_y_obj = repo.commit(commit_y)
+
+    if commit_x is None or commit_x == EMPTY_TREE_HASH:
+        # First commit case: diff against empty tree
+        diff_index = commit_y_obj.diff(EMPTY_TREE_HASH, R=True)
+    else:
+        commit_x_obj = repo.commit(commit_x)
+        diff_index = commit_x_obj.diff(commit_y_obj, R=True)
+
+    changed_files = {item.a_path or item.b_path for item in diff_index}
+    return list(changed_files)
+
+
+def get_last_commit(repo, ecosystem):
+    """
+    Retrieve the last mined commit for a given ecosystem.
+    This function reads a JSON checkpoint file from the repository, which stores
+    mining progress. Each checkpoint contains the "last_commit" from the package
+    index (e.g., PyPI) that was previously mined.
+    https://github.com/AyanSinhaMahapatra/minecode-test/blob/main/minecode_checkpoints/pypi.json
+    https://github.com/ziadhany/cargo-test/blob/main/minecode_checkpoints/cargo.json
+    """
+
+    last_commit_file_path = (
+        Path(repo.working_tree_dir) / "minecode_checkpoints" / f"{ecosystem}.json"
+    )
+    try:
+        with open(last_commit_file_path) as f:
+            settings_data = json.load(f)
+    except FileNotFoundError:
+        return
+    return settings_data.get("last_commit")
+
+
+def get_next_x_commit(repo: Repo, current_commit: str, x: int = 10, branch: str = "master") -> str:
+    """
+    Get the x-th next commit after the current commit in the specified branch.
+    """
+    if not current_commit:
+        current_commit = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
+    revs = repo.git.rev_list(f"^{current_commit}", branch).splitlines()
+    if len(revs) < x:
+        raise ValueError(f"Not enough commits ahead; only {len(revs)} available.")
+    return revs[-x]

minecode_pipelines/pipes/cargo.py

@@ -1,13 +1,15 @@
+from aboutcode import hashid
 from packageurl import PackageURL
 from aboutcode.hashid import get_core_purl
-from minecode_pipelines.pipes import write_purls_to_repo
 
+from minecode_pipelines.miners import write_packageurls_to_file
 
-def store_cargo_packages(packages, fed_repo, push_commit=False):
+
+def store_cargo_packages(packages, repo):
     """Collect Cargo package versions into purls and write them to the repo."""
 
     if not packages:
-        raise ValueError("No packages found")
+        return
 
     first_pkg = packages[0]
     name = first_pkg.get("name")
@@ -22,4 +24,5 @@ def store_cargo_packages(packages, fed_repo, push_commit=False):
         purl = PackageURL(type="cargo", name=name, version=version).to_string()
         updated_purls.append(purl)
 
-    write_purls_to_repo(fed_repo, base_purl, updated_purls, push_commit)
+    ppath = hashid.get_package_purls_yml_file_path(base_purl)
+    return write_packageurls_to_file(repo, ppath, updated_purls), base_purl

minecode_pipelines/tests/pipes/test_cargo_pipes.py

@@ -5,21 +5,15 @@
 from unittest.mock import Mock, patch
 import saneyaml
 from django.test import TestCase
-from packageurl import PackageURL
 
-from minecode_pipelines.pipes import git_stage_purls
+from minecode_pipelines.miners import write_packageurls_to_file
 from minecode_pipelines.pipes.cargo import store_cargo_packages
 
 DATA_DIR = Path(__file__).parent.parent / "test_data" / "cargo"
 
 
 class CargoPipelineTests(TestCase):
-    def _get_temp_dir(self):
-        import tempfile
-
-        return tempfile.mkdtemp()
-
-    @patch("minecode_pipelines.pipes.cargo.write_purls_to_repo")
+    @patch("minecode_pipelines.pipes.cargo.write_packageurls_to_file")
     def test_collect_packages_from_cargo_calls_write(self, mock_write):
         packages_file = DATA_DIR / "c5store"
         expected_file = DATA_DIR / "c5store-expected.yaml"
@@ -34,22 +28,16 @@ def test_collect_packages_from_cargo_calls_write(self, mock_write):
             expected = saneyaml.load(f)
 
         repo = Mock()
-        result = store_cargo_packages(packages, repo)
-        self.assertIsNone(result)
+        store_cargo_packages(packages, repo)
 
         mock_write.assert_called_once()
         args, kwargs = mock_write.call_args
-        called_repo, base_purl, written_packages, push_commit = args
+        called_repo, base_purl, written_packages = args
 
         self.assertEqual(called_repo, repo)
 
-        first_pkg = packages[0]
-        expected_base_purl = PackageURL(
-            type="cargo",
-            name=first_pkg["name"],
-        )
+        expected_base_purl = 'aboutcode-packages-cargo-0/cargo/c5store/purls.yml'
         self.assertEqual(str(base_purl), str(expected_base_purl))
-
         self.assertEqual(written_packages, expected)
 
     def test_add_purl_result_with_mock_repo(self):
@@ -64,13 +52,11 @@ def test_add_purl_result_with_mock_repo(self):
 
             purls_file = repo_dir / "purls.yaml"
 
-            relative_path = git_stage_purls(purls, mock_repo, purls_file)
+            relative_path = write_packageurls_to_file(mock_repo, purls_file, purls)
 
             written_file = repo_dir / relative_path
             self.assertTrue(written_file.exists())
 
             with open(written_file, encoding="utf-8") as f:
                 content = saneyaml.load(f)
-            self.assertEqual(content, purls)
-
-            mock_repo.index.add.assert_called_once_with([relative_path])
+            self.assertEqual(content, purls)