Add a test for mine cargo

Signed-off-by: ziad hany <[email protected]>

Author: ziadhany

minecode_pipeline/pipelines/mine_cargo.py

@@ -23,11 +23,13 @@
 from pathlib import Path
 
 from minecode_pipeline.pipes import cargo
-from scanpipe.pipelines.publish_to_federatedcode import PublishToFederatedCode
+
+from scanpipe.pipelines import Pipeline
 from fetchcode.vcs import fetch_via_vcs
+from scanpipe.pipes import federatedcode
 
 
-class MineCargo(PublishToFederatedCode):
+class MineCargo(Pipeline):
     """Pipeline to mine Cargo (crates.io) packages and publish them to FederatedCode."""
 
     repo_url = "git+https://github.com/rust-lang/crates.io-index"
@@ -36,13 +38,18 @@ class MineCargo(PublishToFederatedCode):
     def steps(cls):
         return (
             cls.check_federatedcode_eligibility,
-            cls.clone_cargo_index,
-            cls.clone_repository,
+            cls.clone_cargo_repo,
             cls.collect_packages_from_cargo,
-            cls.delete_local_clone,
         )
 
-    def clone_cargo_index(self, repo_url):
+    def check_federatedcode_eligibility(self):
+        """
+        Check if the project fulfills the following criteria for
+        pushing the project result to FederatedCode.
+        """
+        federatedcode.check_federatedcode_eligibility(project=self.project)
+
+    def clone_cargo_repo(self, repo_url):
         """
         Clone the repo at repo_url and return the VCSResponse object
         """
@@ -62,10 +69,21 @@ def collect_packages_from_cargo(self):
         for idx, file_path in enumerate(json_files, start=1):
             try:
                 with open(file_path, encoding="utf-8") as f:
-                    packages = json.load(f)
+                    packages = []
+                    for line in f:
+                        if line.strip():
+                            packages.append(json.loads(line))
+
             except (json.JSONDecodeError, UnicodeDecodeError):
                 continue
 
             if packages:
                 push_commit = idx == len(json_files)  # only True on last
                 cargo.collect_packages_from_cargo(packages, self.vcs_response, push_commit)
+
+    def clean_cargo_repo(self):
+        """
+        Delete the VCS response repository if it exists.
+        """
+        if self.vcs_response:
+            self.vcs_response.delete()

minecode_pipeline/pipes/__init__.py

@@ -18,11 +18,11 @@
 remote_name = "origin"
 
 
-def write_purls_to_repo(repo, package, packages_yaml, push_commit=False):
+def write_purls_to_repo(repo, package, updated_purls, push_commit=False):
     """Write or update package purls in the repo and optionally commit/push changes."""
 
     ppath = hashid.get_package_purls_yml_file_path(package)
-    add_purl_result(packages_yaml, repo, ppath)
+    add_purl_result(updated_purls, repo, ppath)
 
     if push_commit:
         change_type = "Add" if ppath in repo.untracked_files else "Update"
@@ -40,7 +40,7 @@ def write_purls_to_repo(repo, package, packages_yaml, push_commit=False):
 
 def add_purl_result(purls, repo, purls_file):
     """Add package urls result to the local Git repository."""
-    relative_purl_file_path = Path(*purls_file.parts[1:])
+    relative_purl_file_path = Path(purls_file)
 
     write_to = Path(repo.working_dir) / relative_purl_file_path
     write_to.parent.mkdir(parents=True, exist_ok=True)

minecode_pipeline/pipes/cargo.py

@@ -6,21 +6,20 @@
 def collect_packages_from_cargo(packages, repo, push_commit=False):
     """Collect Cargo package versions into purls and write them to the repo."""
 
-    if not packages and len(packages) > 0:
+    if not packages:
         raise ValueError("No packages found")
 
-    updated_purls = []
     first_pkg = packages[0]
-    version = first_pkg.get("vers")
     name = first_pkg.get("name")
+    version = first_pkg.get("vers")
     purl = PackageURL(type="cargo", name=name, version=version)
     base_purl = get_core_purl(purl)
 
+    updated_purls = []
     for package in packages:
-        version = package.get("vers")
         name = package.get("name")
-
+        version = package.get("vers")
         purl = PackageURL(type="cargo", name=name, version=version).to_string()
         updated_purls.append(purl)
 
-    write_purls_to_repo(repo, base_purl, packages, push_commit)
+    write_purls_to_repo(repo, base_purl, updated_purls, push_commit)

minecode_pipeline/tests/test_cargo_pipes.py

@@ -0,0 +1,77 @@
+import json
+import tempfile
+from pathlib import Path
+from unittest import mock
+from unittest.mock import Mock, patch
+import saneyaml
+
+from django.test import TestCase
+from packageurl import PackageURL
+
+from minecode_pipeline.pipes import add_purl_result
+from minecode_pipeline.pipes.cargo import collect_packages_from_cargo
+
+DATA_DIR = Path(__file__).parent / "test_data"
+
+
+class CargoPipelineTests(TestCase):
+    def _get_temp_dir(self):
+        import tempfile
+
+        return tempfile.mkdtemp()
+
+    @patch("minecode_pipeline.pipes.cargo.write_purls_to_repo")
+    def test_collect_packages_from_cargo_calls_write(self, mock_write):
+        packages_file = DATA_DIR / "c5store"
+        expected_file = DATA_DIR / "c5store-expected.yaml"
+
+        packages = []
+        with open(packages_file, encoding="utf-8") as f:
+            for line in f:
+                if line.strip():
+                    packages.append(json.loads(line))
+
+        with open(expected_file, encoding="utf-8") as f:
+            expected = saneyaml.load(f)
+
+        repo = Mock()
+        result = collect_packages_from_cargo(packages, repo)
+        self.assertIsNone(result)
+
+        mock_write.assert_called_once()
+        args, kwargs = mock_write.call_args
+        called_repo, base_purl, written_packages, push_commit = args
+
+        self.assertEqual(called_repo, repo)
+
+        first_pkg = packages[0]
+        expected_base_purl = PackageURL(
+            type="cargo",
+            name=first_pkg["name"],
+        )
+        self.assertEqual(str(base_purl), str(expected_base_purl))
+
+        self.assertEqual(written_packages, expected)
+
+    def test_add_purl_result_with_mock_repo(self):
+        purls = [{"purl": "pkg:pypi/[email protected]"}, {"purl": "pkg:pypi/[email protected]"}]
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            repo_dir = Path(tmpdir)
+
+            mock_repo = mock.MagicMock()
+            mock_repo.working_dir = str(repo_dir)
+            mock_repo.index.add = mock.MagicMock()
+
+            purls_file = repo_dir / "purls.yaml"
+
+            relative_path = add_purl_result(purls, mock_repo, purls_file)
+
+            written_file = repo_dir / relative_path
+            self.assertTrue(written_file.exists())
+
+            with open(written_file, encoding="utf-8") as f:
+                content = saneyaml.load(f)
+            self.assertEqual(content, purls)
+
+            mock_repo.index.add.assert_called_once_with([relative_path])