diff --git a/.github/workflows/unittests.yaml b/.github/workflows/unittests.yaml index 3140088..b7cd893 100644 --- a/.github/workflows/unittests.yaml +++ b/.github/workflows/unittests.yaml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9"] + python-version: ["3.8", "3.9"] steps: - uses: actions/checkout@v3 diff --git a/charon/pkgs/npm.py b/charon/pkgs/npm.py index 7c16067..61ed549 100644 --- a/charon/pkgs/npm.py +++ b/charon/pkgs/npm.py @@ -16,7 +16,7 @@ import logging import os import sys -from json import load, loads, dump, JSONDecodeError +from json import load, loads, dump, JSONDecodeError, JSONEncoder import tarfile from tempfile import mkdtemp from typing import List, Set, Tuple @@ -32,7 +32,7 @@ from charon.pkgs.pkg_utils import upload_post_process, rollback_post_process from charon.utils.strings import remove_prefix from charon.utils.files import write_manifest -from charon.utils.map import del_none +from charon.utils.map import del_none, replace_field logger = logging.getLogger(__name__) @@ -61,10 +61,15 @@ def __init__(self, metadata, is_version): self.dist_tags = {'latest': metadata.get('version')} self.versions = {metadata.get('version'): metadata} else: - self.dist_tags = metadata.get('dist_tags', None) + self.dist_tags = metadata.get('dist-tags', None) self.versions = metadata.get('versions', None) +class NPMPackageMetadataEncoder(JSONEncoder): + def default(self, o): + return replace_field(del_none(o.__dict__.copy()), "dist_tags", "dist-tags") + + def handle_npm_uploading( tarball_path: str, product: str, @@ -549,7 +554,7 @@ def _write_package_metadata_to_file(package_metadata: NPMPackageMetadata, root=' final_package_metadata_path = os.path.join(root, package_metadata.name, PACKAGE_JSON) try: with open(final_package_metadata_path, mode='w', encoding='utf-8') as f: - dump(del_none(package_metadata.__dict__.copy()), f) + dump(obj=package_metadata, cls=NPMPackageMetadataEncoder, fp=f) return final_package_metadata_path except FileNotFoundError: logger.error( diff --git a/charon/storage.py b/charon/storage.py index e040b5a..45963c4 100644 --- a/charon/storage.py +++ b/charon/storage.py @@ -698,19 +698,22 @@ def list_folder_content(self, bucket_name: str, folder: str) -> List[str]: not in its subfolders. """ bucket = self.__get_bucket(bucket_name) + try: + paginator = bucket.meta.client.get_paginator('list_objects_v2') if not folder or folder.strip() == "/" or folder.strip() == "": - result = bucket.meta.client.list_objects( + pages = paginator.paginate( Bucket=bucket.name, Delimiter='/' ) else: prefix = folder if folder.endswith("/") else folder+"/" - result = bucket.meta.client.list_objects( + pages = paginator.paginate( Bucket=bucket.name, Prefix=prefix, Delimiter='/' ) + except (ClientError, HTTPClientError) as e: logger.error("ERROR: Can not get contents of %s from bucket" " %s due to error: %s ", folder, @@ -718,12 +721,13 @@ def list_folder_content(self, bucket_name: str, folder: str) -> List[str]: return [] contents = [] - folders = result.get("CommonPrefixes") - if folders: - contents.extend([f.get("Prefix") for f in folders]) - files = result.get("Contents") - if files: - contents.extend([f.get("Key") for f in files]) + for page in pages: + folders = page.get("CommonPrefixes") + if folders: + contents.extend([f.get("Prefix") for f in folders]) + files = page.get("Contents") + if files: + contents.extend([f.get("Key") for f in files]) return contents def file_exists_in_bucket( diff --git a/charon/utils/map.py b/charon/utils/map.py index 606b0d9..fe602ee 100644 --- a/charon/utils/map.py +++ b/charon/utils/map.py @@ -5,3 +5,10 @@ def del_none(d): elif isinstance(value, dict): del_none(value) return d + + +def replace_field(d, field, new_field): + if d[field]: + d[new_field] = d[field] + del d[field] + return d diff --git a/requirements.txt b/requirements.txt index 043083c..563abc2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -setuptools-rust==1.1.2 -jinja2==3.0.3 -boto3==1.20.45 -botocore==1.23.45 -click==8.0.3 -requests==2.27.1 -PyYAML==6.0 +setuptools-rust==1.7.0 +Jinja2==3.1.2 +boto3==1.28.46 +botocore==1.31.46 +click==8.1.7 +requests==2.31.0 +PyYAML==6.0.1 defusedxml==0.7.1 subresource-integrity==0.2 -jsonschema==3.2.0 +jsonschema==4.19.0 diff --git a/setup.py b/setup.py index c930138..011b7ce 100755 --- a/setup.py +++ b/setup.py @@ -13,14 +13,9 @@ See the License for the specific language governing permissions and limitations under the License. """ - -# !/usr/bin/env python - -import re - from setuptools import setup, find_packages -version = "1.1.2" +version = "1.2.1" # f = open('README.md') # long_description = f.read().strip() @@ -33,18 +28,6 @@ in future. And Ronda service will be hosted in AWS S3. """ - -def _get_requirements(path): - try: - with open(path, encoding="utf-8") as f: - packages = f.read().splitlines() - except (IOError, OSError) as ex: - raise RuntimeError(f"Can't open file with requirements: {ex}") from ex - packages = (p.strip() for p in packages if not re.match(r'^\s*#', p)) - packages = list(filter(None, packages)) - return packages - - setup( zip_safe=True, name="charon", @@ -62,7 +45,6 @@ def _get_requirements(path): author="RedHat EXD SPMM", license="APLv2", packages=find_packages(exclude=["ez_setup", "examples", "tests"]), - install_requires=_get_requirements('requirements.txt'), package_data={'charon': ['schemas/*.json']}, test_suite="tests", entry_points={ diff --git a/tests/test_npm_del.py b/tests/test_npm_del.py index ad2b1f8..8d6a6df 100644 --- a/tests/test_npm_del.py +++ b/tests/test_npm_del.py @@ -83,7 +83,7 @@ def __test_prefix(self, prefix: str = None): self.assertIn("\"versions\": {\"7.15.8\":", meta_content_client) self.assertNotIn("\"7.14.5\": {\"name\":", meta_content_client) self.assertIn("\"license\": \"MIT\"", meta_content_client) - self.assertIn("\"dist_tags\": {\"latest\": \"7.15.8\"}", meta_content_client) + self.assertIn("\"dist-tags\": {\"latest\": \"7.15.8\"}", meta_content_client) test_tgz = os.path.join(INPUTS, "code-frame-7.15.8.tgz") handle_npm_del( diff --git a/tests/test_npm_del_multi_tgts.py b/tests/test_npm_del_multi_tgts.py index 1746fba..ac1e2c3 100644 --- a/tests/test_npm_del_multi_tgts.py +++ b/tests/test_npm_del_multi_tgts.py @@ -117,7 +117,7 @@ def __test_prefix(self, prefix: str = None): "\"license\": \"MIT\"", meta_content_client, msg=f'{bucket_name}' ) self.assertIn( - "\"dist_tags\": {\"latest\": \"7.15.8\"}", + "\"dist-tags\": {\"latest\": \"7.15.8\"}", meta_content_client, msg=f'{bucket_name}' ) diff --git a/tests/test_npm_meta.py b/tests/test_npm_meta.py index df66049..b9cae1a 100644 --- a/tests/test_npm_meta.py +++ b/tests/test_npm_meta.py @@ -48,7 +48,7 @@ def test_handle_npm_uploading_for_old_version(self): bucket = self.mock_s3.Bucket(MY_BUCKET) original_version_0_5_8_package_json = """ {"name": "@redhat/kogito-tooling-workspace", - "dist_tags": {"latest": "0.5.8"}, "versions": {"0.5.8": {"name": + "dist-tags": {"latest": "0.5.8"}, "versions": {"0.5.8": {"name": "@redhat/kogito-tooling-workspace", "version": "0.5.8", "title": "0.5.8title", "description": "0.5.8description", "keywords": ["0.5.8"], "maintainers": [ "0.5.8maintainer"], "repository": {"type": "git", "url": "https://github.com/0.5.8.git"}, @@ -98,7 +98,7 @@ def test_handle_npm_uploading_for_old_version(self): def test_handle_npm_uploading_for_new_version(self): bucket = self.mock_s3.Bucket(MY_BUCKET) original_version_1_0_1_package_json = """ - {"name": "@redhat/kogito-tooling-workspace", "dist_tags": {"latest": "1.0.1"}, + {"name": "@redhat/kogito-tooling-workspace", "dist-tags": {"latest": "1.0.1"}, "versions": {"1.0.1": {"name": "@redhat/kogito-tooling-workspace", "version": "1.0.1", "title": "1.0.1title", "description": "1.0.1description", "keywords": ["1.0.1"], "maintainers": ["1.0.1maintainer"], "repository": {"type": "git", diff --git a/tests/test_npm_upload.py b/tests/test_npm_upload.py index 1130b4d..941f98c 100644 --- a/tests/test_npm_upload.py +++ b/tests/test_npm_upload.py @@ -86,7 +86,7 @@ def test_double_uploads(self): self.assertIn("\"7.15.8\": {\"name\":", meta_content_client) self.assertIn("\"7.14.5\": {\"name\":", meta_content_client) self.assertIn("\"license\": \"MIT\"", meta_content_client) - self.assertIn("\"dist_tags\": {\"latest\": \"7.15.8\"}", meta_content_client) + self.assertIn("\"dist-tags\": {\"latest\": \"7.15.8\"}", meta_content_client) def __test_prefix(self, prefix: str = None): test_tgz = os.path.join(INPUTS, "code-frame-7.14.5.tgz") @@ -131,4 +131,4 @@ def __test_prefix(self, prefix: str = None): self.assertIn("\"version\": \"7.14.5\"", meta_content_client) self.assertIn("\"versions\": {\"7.14.5\":", meta_content_client) self.assertIn("\"license\": \"MIT\"", meta_content_client) - self.assertIn("\"dist_tags\": {\"latest\": \"7.14.5\"}", meta_content_client) + self.assertIn("\"dist-tags\": {\"latest\": \"7.14.5\"}", meta_content_client) diff --git a/tests/test_npm_upload_multi_tgts.py b/tests/test_npm_upload_multi_tgts.py index d95868b..4827e29 100644 --- a/tests/test_npm_upload_multi_tgts.py +++ b/tests/test_npm_upload_multi_tgts.py @@ -121,7 +121,7 @@ def test_double_uploads(self): "\"license\": \"MIT\"", meta_content_client, msg=f'{bucket_name}' ) self.assertIn( - "\"dist_tags\": {\"latest\": \"7.15.8\"}", meta_content_client, msg=f'{bucket_name}' + "\"dist-tags\": {\"latest\": \"7.15.8\"}", meta_content_client, msg=f'{bucket_name}' ) def __test_prefix(self, prefix: str = None): @@ -192,6 +192,6 @@ def __test_prefix(self, prefix: str = None): msg=f'{bucket_name}' ) self.assertIn( - "\"dist_tags\": {\"latest\": \"7.14.5\"}", + "\"dist-tags\": {\"latest\": \"7.14.5\"}", meta_content_client, msg=f'{bucket_name}' )