diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c95f794..7829d5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 diff --git a/test/__init__.py b/test/__init__.py index fb1d337..22b61d0 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,6 +1,7 @@ """Fixtures and utilities for testing vcs2l.""" import os +import re import shutil import subprocess import tarfile @@ -12,120 +13,142 @@ import yaml +REPO_LINK = 'https://github.com/ros-infrastructure/vcs2l.git' + def to_file_url(path): return Path(path).as_uri() +def extract_commit(url): + """Extract commit hash from version for zip/tar clients.""" + m = re.search(r'vcs2l-([a-fA-F0-9]{40})', url) + return m.group(1) if m else None + + +def setup_git_repository(temp_dir): + """Create a git repository for testing.""" + repo_root = os.path.dirname(os.path.dirname(__file__)) + gitrepo_path = os.path.join(temp_dir.name, 'gitrepo') + + commits_count = int( + subprocess.check_output( + ['git', 'rev-list', '--count', 'HEAD'], cwd=repo_root + ).strip() + ) + + if commits_count == 1: + repo_root = REPO_LINK # codecov sparsely clones the repo + + subprocess.check_call(['git', 'clone', repo_root, gitrepo_path]) + try: + subprocess.check_call(['git', 'checkout', 'main'], cwd=gitrepo_path) + except subprocess.CalledProcessError: + # Create branch named 'main' as CI checks out with no branch + subprocess.check_call(['git', 'checkout', '-b', 'main'], cwd=gitrepo_path) + + return gitrepo_path + + +def setup_tar_archive(temp_dir, target_commit): + """Create a tar archive for testing by checking out the target commit.""" + gitrepo_path = os.path.join(temp_dir.name, 'gitrepo') + tar_path = os.path.join(temp_dir.name, 'tar') + + subprocess.check_call(['git', 'clone', gitrepo_path, tar_path]) + subprocess.check_call(['git', 'checkout', target_commit], cwd=tar_path) + shutil.rmtree(os.path.join(tar_path, '.git')) # .git not present in tar archive + + archive_dir = os.path.join(temp_dir.name, 'archive') + os.makedirs(archive_dir, exist_ok=True) + tarball_path = os.path.join(archive_dir, f'{target_commit}.tar.gz') + + with tarfile.TarFile.open(tarball_path, 'w:gz') as tar: + tar.add(tar_path, arcname=f'vcs2l-{target_commit}') + + return tarball_path + + +def setup_zip_archive(temp_dir, target_commit): + """Create a zip archive for testing by checking out the target commit.""" + gitrepo_path = os.path.join(temp_dir.name, 'gitrepo') + zip_path = os.path.join(temp_dir.name, 'zip') + + subprocess.check_call(['git', 'clone', gitrepo_path, zip_path]) + subprocess.check_call(['git', 'checkout', target_commit], cwd=zip_path) + shutil.rmtree(os.path.join(zip_path, '.git')) # .git not present in zip archive + + archive_dir = os.path.join(temp_dir.name, 'archive') + os.makedirs(archive_dir, exist_ok=True) + zipfile_path = os.path.join(archive_dir, f'{target_commit}.zip') + + with zipfile.ZipFile(zipfile_path, 'w') as zipf: + for root, dirs, files in os.walk(zip_path): + # Add directory entries + for dir_name in dirs: + dir_path = os.path.join(root, dir_name) + arcname = ( + os.path.join( + f'vcs2l-{target_commit}', os.path.relpath(dir_path, zip_path) + ) + + '/' + ) + zipf.write(dir_path, arcname) + + # Add files + for file in files: + file_path = os.path.join(root, file) + arcname = os.path.join( + f'vcs2l-{target_commit}', os.path.relpath(file_path, zip_path) + ) + zipf.write(file_path, arcname) + + return zipfile_path + + class StagedReposFile(unittest.TestCase): """Fixture for testing git, tar, and zip clients.""" _git = which('git') - _git_env = { - **os.environ, - 'GIT_AUTHOR_NAME': 'vcs2l', - 'GIT_AUTHOR_DATE': '2005-01-01T00:00:00-06:00', - 'GIT_AUTHOR_EMAIL': 'vcs2l@example.com', - 'GIT_COMMITTER_NAME': 'vcs2l', - 'GIT_COMMITTER_DATE': '2005-01-01T00:00:00-06:00', - 'GIT_COMMITTER_EMAIL': 'vcs2l@example.com', - 'GIT_CONFIG_GLOBAL': os.path.join(os.path.dirname(__file__), '.gitconfig'), - 'LANG': 'en_US.UTF-8', - } - _commit_date = '2005-01-01T00:00:00' - temp_dir = None repos_file_path = None @classmethod - def setUpClass(cls): + def setUpClass(cls, repos_file=None): if not cls._git: raise unittest.SkipTest('`git` was not found') cls.temp_dir = TemporaryDirectory(suffix='.vcstmp') - # Create the staged git repository - gitrepo_path = os.path.join(cls.temp_dir.name, 'gitrepo') - os.mkdir(gitrepo_path) - shutil.copy( - os.path.join(os.path.dirname(os.path.dirname(__file__)), 'LICENSE'), - gitrepo_path, - ) - for command in ( - ('init', '--quiet', '.'), - ('commit', '--quiet', '--allow-empty', '-m', '0.1.26'), - ('tag', '0.1.26'), - ('checkout', '--quiet', '-b', 'license'), - ('add', 'LICENSE'), - ('commit', '--quiet', '-m', 'Add LICENSE'), - ('checkout', '--quiet', 'main'), - ('merge', '--no-ff', '--quiet', 'license', '-m', "Merge branch 'license'"), - ('branch', '--quiet', '-D', 'license'), - ('commit', '--quiet', '--allow-empty', '-m', 'update changelog'), - ('commit', '--quiet', '--allow-empty', '-m', '0.1.27'), - ('tag', '0.1.27'), - ('commit', '--quiet', '--allow-empty', '-m', "codin' codin' codin'"), - ): - subprocess.check_call( - [ - cls._git, - *command, - ], - cwd=gitrepo_path, - env=cls._git_env, - ) + if repos_file is None: + raise ValueError('A repos file must be provided') - # Create the archive stage - archive_path = os.path.join(cls.temp_dir.name, 'archive_dir') - os.mkdir(archive_path) - with open(os.path.join(archive_path, 'file_name.txt'), 'wb') as f: - f.write(b'Lorem Ipsum\n') - - # Create a tar file - tarball_path = os.path.join(cls.temp_dir.name, 'archive.tar.gz') - with tarfile.TarFile.open(tarball_path, 'w:gz') as f: - f.add(archive_path, 'archive_dir') - - # Create a zip file - zip_path = os.path.join(cls.temp_dir.name, 'archive.zip') - with zipfile.ZipFile(zip_path, mode='w') as f: - f.write( - os.path.join(archive_path, 'file_name.txt'), - os.path.join('archive_dir', 'file_name.txt'), - ) + with open(repos_file, 'r', encoding='utf-8') as f: + repos_data = yaml.safe_load(f) - # Populate the staged.repos file - repos = { - 'immutable/hash': { - 'type': 'git', - 'url': to_file_url(gitrepo_path), - 'version': '5b3504594f7354121cf024dc734bf79e270cffd3', - }, - 'immutable/hash_tar': { - 'type': 'tar', - 'url': to_file_url(tarball_path), - 'version': 'archive_dir', - }, - 'immutable/hash_zip': { - 'type': 'zip', - 'url': to_file_url(zip_path), - 'version': 'archive_dir', - }, - 'immutable/tag': { - 'type': 'git', - 'url': to_file_url(gitrepo_path), - 'version': 'tags/0.1.27', - }, - 'vcs2l': { - 'type': 'git', - 'url': to_file_url(gitrepo_path), - 'version': 'heads/main', - }, - 'without_version': { - 'type': 'git', - 'url': to_file_url(gitrepo_path), - }, - } + repos = repos_data.get('repositories') + + # Create the temp git repository + gitrepo_path = setup_git_repository(cls.temp_dir) + + # Use the existing file as a baseline + repos = repos_data['repositories'].copy() + + for repo_name, repo_config in repos.items(): + if repo_config['type'] == 'git': + repos[repo_name]['url'] = to_file_url(gitrepo_path) + + elif repo_config['type'] == 'tar': + extracted_commit = extract_commit(repo_config['version']) + repos[repo_name]['url'] = to_file_url( + setup_tar_archive(cls.temp_dir, extracted_commit) + ) + + elif repo_config['type'] == 'zip': + extracted_commit = extract_commit(repo_config['version']) + repos[repo_name]['url'] = to_file_url( + setup_zip_archive(cls.temp_dir, extracted_commit) + ) cls.repos_file_path = os.path.join(cls.temp_dir.name, 'staged.repos') with open(cls.repos_file_path, 'wb') as f: diff --git a/test/branch.txt b/test/branch.txt index 61818b8..c0a2e41 100644 --- a/test/branch.txt +++ b/test/branch.txt @@ -1,6 +1,6 @@ .... === ./immutable/hash (git) === -(HEAD detached at 5b35045) +(HEAD detached at 377d5b3) === ./immutable/tag (git) === (HEAD detached at 0.1.27) === ./vcs2l (git) === diff --git a/test/export_exact.txt b/test/export_exact.txt index c466690..288ce77 100644 --- a/test/export_exact.txt +++ b/test/export_exact.txt @@ -2,8 +2,8 @@ repositories: hash: type: git url: file:///vcstmp/gitrepo - version: 5b3504594f7354121cf024dc734bf79e270cffd3 + version: 377d5b3d03c212f015cc832fdb368f4534d0d583 tag: type: git url: file:///vcstmp/gitrepo - version: 8087b72504968800cdf54759b11e0b753ec90736 + version: bf9ca56de693a02b93ed423bcef589259d75eb0f diff --git a/test/export_exact_with_tags.txt b/test/export_exact_with_tags.txt index 59cd53b..2c241bc 100644 --- a/test/export_exact_with_tags.txt +++ b/test/export_exact_with_tags.txt @@ -2,7 +2,7 @@ repositories: hash: type: git url: file:///vcstmp/gitrepo - version: 5b3504594f7354121cf024dc734bf79e270cffd3 + version: 377d5b3d03c212f015cc832fdb368f4534d0d583 tag: type: git url: file:///vcstmp/gitrepo diff --git a/test/import.txt b/test/import.txt index 95523e2..7c9d783 100644 --- a/test/import.txt +++ b/test/import.txt @@ -1,7 +1,7 @@ ...... === ./immutable/hash (git) === Cloning into '.'... -Note: switching to '5b3504594f7354121cf024dc734bf79e270cffd3'. +Note: switching to '377d5b3d03c212f015cc832fdb368f4534d0d583'. You are in 'detached HEAD' state. You can look around, make experimental changes and commit them, and you can discard any commits you make in this @@ -18,11 +18,11 @@ Or undo this operation with: Turn off this advice by setting config variable advice.detachedHead to false -HEAD is now at 5b35045... update changelog +HEAD is now at 377d5b3... update changelog === ./immutable/hash_tar (tar) === -Downloaded tarball from 'file:///vcstmp/archive.tar.gz' and unpacked it +Downloaded tarball from 'file:///vcstmp/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.tar.gz' and unpacked it === ./immutable/hash_zip (zip) === -Downloaded zipfile from 'file:///vcstmp/archive.zip' and unpacked it +Downloaded zipfile from 'file:///vcstmp/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.zip' and unpacked it === ./immutable/tag (git) === Cloning into '.'... Note: switching to 'tags/0.1.27'. @@ -42,7 +42,7 @@ Or undo this operation with: Turn off this advice by setting config variable advice.detachedHead to false -HEAD is now at 8087b72... 0.1.27 +HEAD is now at bf9ca56... 0.1.27 === ./vcs2l (git) === Cloning into '.'... === ./without_version (git) === diff --git a/test/import_shallow.txt b/test/import_shallow.txt index e664894..cb6e08d 100644 --- a/test/import_shallow.txt +++ b/test/import_shallow.txt @@ -3,8 +3,8 @@ Initialized empty Git repository in ./immutable/hash/.git/ From file:///vcstmp/gitrepo - * branch 5b3504594f7354121cf024dc734bf79e270cffd3 -> FETCH_HEAD -Note: switching to '5b3504594f7354121cf024dc734bf79e270cffd3'. + * branch 377d5b3d03c212f015cc832fdb368f4534d0d583 -> FETCH_HEAD +Note: switching to '377d5b3d03c212f015cc832fdb368f4534d0d583'. You are in 'detached HEAD' state. You can look around, make experimental changes and commit them, and you can discard any commits you make in this @@ -21,11 +21,11 @@ Or undo this operation with: Turn off this advice by setting config variable advice.detachedHead to false -HEAD is now at 5b35045... update changelog +HEAD is now at 377d5b3... update changelog === ./immutable/hash_tar (tar) === -Downloaded tarball from 'file:///vcstmp/archive.tar.gz' and unpacked it +Downloaded tarball from 'file:///vcstmp/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.tar.gz' and unpacked it === ./immutable/hash_zip (zip) === -Downloaded zipfile from 'file:///vcstmp/archive.zip' and unpacked it +Downloaded zipfile from 'file:///vcstmp/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.zip' and unpacked it === ./immutable/tag (git) === Initialized empty Git repository in ./immutable/tag/.git/ @@ -48,7 +48,7 @@ Or undo this operation with: Turn off this advice by setting config variable advice.detachedHead to false -HEAD is now at 8087b72... 0.1.27 +HEAD is now at bf9ca56... 0.1.27 === ./vcs2l (git) === Cloning into '.'... === ./without_version (git) === diff --git a/test/list_repos.yaml b/test/list_repos.yaml new file mode 100644 index 0000000..ad7aedd --- /dev/null +++ b/test/list_repos.yaml @@ -0,0 +1,25 @@ +--- +repositories: + immutable/hash: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: 377d5b3d03c212f015cc832fdb368f4534d0d583 + immutable/hash_tar: + type: tar + url: https://github.com/ros-infrastructure/vcs2l/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.tar.gz + version: vcs2l-377d5b3d03c212f015cc832fdb368f4534d0d583 + immutable/hash_zip: + type: zip + url: https://github.com/ros-infrastructure/vcs2l/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.zip + version: vcs2l-377d5b3d03c212f015cc832fdb368f4534d0d583 + immutable/tag: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: tags/0.1.27 + vcs2l: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git + version: heads/main + without_version: + type: git + url: https://github.com/ros-infrastructure/vcs2l.git diff --git a/test/log_limit.txt b/test/log_limit.txt index a60a59a..eee49cd 100644 --- a/test/log_limit.txt +++ b/test/log_limit.txt @@ -1,22 +1,22 @@ .. === ./hash (git) === -commit 5b3504594f7354121cf024dc734bf79e270cffd3 (HEAD) -Author: vcs2l +commit 377d5b3d03c212f015cc832fdb368f4534d0d583 (HEAD) +Author: Dirk Thomas update changelog -commit e1ac7fa53634c69bdb8e181a20d3717e52dea234 -Merge: 89e3430 ec135e9 -Author: vcs2l +commit f01ce3845fa0783bef9f97545e518e0f02cd509a +Merge: 14f9968 e7770d3 +Author: Dirk Thomas - Merge branch 'license' + Merge pull request #44 from dirk-thomas/fix_loop_exit === ./tag (git) === -commit 8087b72504968800cdf54759b11e0b753ec90736 (HEAD, tag: 0.1.27) -Author: vcs2l +commit bf9ca56de693a02b93ed423bcef589259d75eb0f (HEAD, tag: 0.1.27) +Author: Dirk Thomas 0.1.27 -commit 5b3504594f7354121cf024dc734bf79e270cffd3 -Author: vcs2l +commit 377d5b3d03c212f015cc832fdb368f4534d0d583 +Author: Dirk Thomas update changelog diff --git a/test/log_merges_only.txt b/test/log_merges_only.txt index ce49b9c..2001edc 100644 --- a/test/log_merges_only.txt +++ b/test/log_merges_only.txt @@ -1,6 +1,18 @@ === . (git) === -commit e1ac7fa53634c69bdb8e181a20d3717e52dea234 -Merge: 89e3430 ec135e9 -Author: vcs2l +commit f01ce3845fa0783bef9f97545e518e0f02cd509a +Merge: 14f9968 e7770d3 +Author: Dirk Thomas - Merge branch 'license' + Merge pull request #44 from dirk-thomas/fix_loop_exit + +commit 418cd63cc242aeb19bff17696adf1617b9c994b7 +Merge: 6fdb8e8 89448bd +Author: Dirk Thomas + + Merge pull request #42 from dirk-thomas/fix_depends_regression + +commit a3c4c33d9e958a5d297e2d1d777fe2d850b2566f +Merge: a5dfaac 8a7101c +Author: Dirk Thomas + + Merge pull request #41 from dirk-thomas/parent_path_dependencies diff --git a/test/reimport_force.txt b/test/reimport_force.txt index 791a028..e88393b 100644 --- a/test/reimport_force.txt +++ b/test/reimport_force.txt @@ -1,14 +1,14 @@ ...... === ./immutable/hash (git) === -HEAD is now at 5b35045... update changelog +HEAD is now at 377d5b3... update changelog === ./immutable/hash_tar (tar) === -Downloaded tarball from 'file:///vcstmp/archive.tar.gz' and unpacked it +Downloaded tarball from 'file:///vcstmp/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.tar.gz' and unpacked it === ./immutable/hash_zip (zip) === -Downloaded zipfile from 'file:///vcstmp/archive.zip' and unpacked it +Downloaded zipfile from 'file:///vcstmp/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.zip' and unpacked it === ./immutable/tag (git) === -HEAD is now at 8087b72... 0.1.27 +HEAD is now at bf9ca56... 0.1.27 === ./vcs2l (git) === Already on 'main' diff --git a/test/reimport_skip.txt b/test/reimport_skip.txt index c3f5a5a..be8c2a0 100644 --- a/test/reimport_skip.txt +++ b/test/reimport_skip.txt @@ -1,9 +1,9 @@ ...... === ./immutable/hash (git) === === ./immutable/hash_tar (tar) === -Downloaded tarball from 'file:///vcstmp/archive.tar.gz' and unpacked it +Downloaded tarball from 'file:///vcstmp/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.tar.gz' and unpacked it === ./immutable/hash_zip (zip) === -Downloaded zipfile from 'file:///vcstmp/archive.zip' and unpacked it +Downloaded zipfile from 'file:///vcstmp/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.zip' and unpacked it === ./immutable/tag (git) === === ./vcs2l (git) === === ./without_version (git) === diff --git a/test/status.txt b/test/status.txt index 0c48ff2..3eb0863 100644 --- a/test/status.txt +++ b/test/status.txt @@ -1,6 +1,6 @@ .... === ./immutable/hash (git) === -HEAD detached at 5b35045 +HEAD detached at 377d5b3 nothing to commit, working tree clean === ./immutable/tag (git) === HEAD detached at 0.1.27 diff --git a/test/test_commands.py b/test/test_commands.py index 151deef..78d2d04 100644 --- a/test/test_commands.py +++ b/test/test_commands.py @@ -15,6 +15,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) BAD_REPOS_FILE = os.path.join(os.path.dirname(__file__), 'bad.repos') +LIST_REPOS_FILE = os.path.join(os.path.dirname(__file__), 'list_repos.yaml') TEST_WORKSPACE = os.path.join( os.path.dirname(os.path.dirname(__file__)), 'test_workspace' ) @@ -22,9 +23,9 @@ class TestCommands(StagedReposFile): @classmethod - def setUpClass(cls): - super().setUpClass() - cls.repos_file_url = to_file_url(cls.repos_file_path) + def setUpClass(cls, repos_file=None): + super().setUpClass(repos_file if repos_file is not None else LIST_REPOS_FILE) + assert not os.path.exists(TEST_WORKSPACE) os.makedirs(TEST_WORKSPACE) @@ -275,7 +276,7 @@ def test_import_url(self): os.makedirs(workdir) try: output = run_command( - 'import', ['--input', self.repos_file_url, '.'], subfolder='import-url' + 'import', ['--input', self.repos_file_path, '.'], subfolder='import-url' ) # the actual output contains absolute paths output = output.replace( @@ -293,11 +294,11 @@ def test_deletion(self): os.makedirs(workdir) try: run_command( - 'import', ['--input', self.repos_file_url, '.'], subfolder='deletion' + 'import', ['--input', self.repos_file_path, '.'], subfolder='deletion' ) output = run_command( 'delete', - ['--force', '--input', self.repos_file_url, '.'], + ['--force', '--input', self.repos_file_path, '.'], subfolder='deletion', ) expected = get_expected_output('delete') diff --git a/test/validate.txt b/test/validate.txt index f2fed2f..caff27e 100644 --- a/test/validate.txt +++ b/test/validate.txt @@ -1,10 +1,10 @@ ...... === immutable/hash (git) === -Found git repository 'file:///vcstmp/gitrepo' but unable to verify non-branch / non-tag ref '5b3504594f7354121cf024dc734bf79e270cffd3' without cloning the repo +Found git repository 'file:///vcstmp/gitrepo' but unable to verify non-branch / non-tag ref '377d5b3d03c212f015cc832fdb368f4534d0d583' without cloning the repo === immutable/hash_tar (tar) === -Tarball url 'file:///vcstmp/archive.tar.gz' exists +Tarball url 'file:///vcstmp/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.tar.gz' exists === immutable/hash_zip (zip) === -Zip url 'file:///vcstmp/archive.zip' exists +Zip url 'file:///vcstmp/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.zip' exists === immutable/tag (git) === Found git repository 'file:///vcstmp/gitrepo' with tag '0.1.27' === vcs2l (git) ===