From 07d700a54ee89ad9cc5bce2c851803ab475f0437 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Sun, 29 Sep 2024 12:16:23 +0800 Subject: [PATCH] Enable mypy types check and fix reported issues Signed-off-by: Gang Li --- .github/workflows/linters.yaml | 49 +++++------ Makefile | 1 + charon/cache.py | 15 ++-- charon/cmd/cmd_cache.py | 31 +++---- charon/cmd/cmd_checksum.py | 6 +- charon/cmd/cmd_delete.py | 14 ++-- charon/cmd/cmd_index.py | 3 +- charon/cmd/cmd_upload.py | 14 ++-- charon/cmd/internal.py | 21 ++--- charon/config.py | 8 +- charon/pkgs/checksum_http.py | 12 +-- charon/pkgs/indexing.py | 36 ++++----- charon/pkgs/maven.py | 40 +++++---- charon/pkgs/npm.py | 112 ++++++++++++++------------ charon/pkgs/pkg_utils.py | 19 ++--- charon/pkgs/signature.py | 12 +-- charon/storage.py | 14 ++-- charon/types.py | 20 +++++ charon/utils/archive.py | 1 + charon/utils/logs.py | 2 +- tests/test_cf_maven_ops.py | 6 +- tests/test_cf_npm_ops.py | 6 +- tests/test_cf_reindex.py | 4 +- tests/test_manifest_del.py | 8 +- tests/test_manifest_upload.py | 4 +- tests/test_maven_del.py | 10 +-- tests/test_maven_del_multi_tgts.py | 10 +-- tests/test_maven_index.py | 22 ++--- tests/test_maven_index_multi_tgts.py | 20 ++--- tests/test_maven_sign.py | 6 +- tests/test_maven_upload.py | 8 +- tests/test_maven_upload_multi_tgts.py | 8 +- tests/test_npm_del.py | 8 +- tests/test_npm_del_multi_tgts.py | 8 +- tests/test_npm_dist_gen.py | 8 +- tests/test_npm_index.py | 12 +-- tests/test_npm_index_multi_tgts.py | 10 +-- tests/test_npm_meta.py | 6 +- tests/test_npm_upload.py | 6 +- tests/test_npm_upload_diff_pkgs.py | 4 +- tests/test_npm_upload_multi_tgts.py | 6 +- tests/test_pkgs_dryrun.py | 16 ++-- 42 files changed, 328 insertions(+), 298 deletions(-) create mode 100644 charon/types.py diff --git a/.github/workflows/linters.yaml b/.github/workflows/linters.yaml index f69c8959..224781a6 100644 --- a/.github/workflows/linters.yaml +++ b/.github/workflows/linters.yaml @@ -29,17 +29,6 @@ jobs: - name: Run flake8 on python${{ matrix.python-version }} run: python -m tox -e flake8 - # markdownlint: - # name: Markdownlint - # runs-on: ubuntu-latest - - # steps: - # - name: Check out repo - # uses: actions/checkout@v2 - - # - name: Run markdownlint - # uses: containerbuildsystem/actions/markdownlint@master - pylint: name: Pylint analyzer for Python ${{ matrix.python-version }} runs-on: ubuntu-latest @@ -71,25 +60,25 @@ jobs: - name: Run ShellCheck uses: containerbuildsystem/actions/shellcheck@master -# mypy: -# name: mypy type checker for Python ${{ matrix.python-version }} -# runs-on: ubuntu-latest -# -# strategy: -# matrix: -# python-version: [ "3.8" ] -# -# steps: -# - uses: actions/checkout@v3 -# - uses: actions/setup-python@v4 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Install dependencies -# run: | -# python -m pip install --upgrade pip setuptools tox -# -# - name: Run mypy on python${{ matrix.python-version }} -# run: python -m tox -e mypy + mypy: + name: mypy type checker for Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: [ "3.9" ] + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools tox + + - name: Run mypy on python${{ matrix.python-version }} + run: python -m tox -e mypy # bandit: # name: Bandit analyzer for Python ${{ matrix.python-version }} diff --git a/Makefile b/Makefile index 8d6a00cd..5c58beae 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ init-venv: lint: @python -m tox -e flake8 @python -m tox -e pylint + @python -m tox -e mypy .PHONY: lint test-only: diff --git a/charon/cache.py b/charon/cache.py index 45a57751..5b8d1227 100644 --- a/charon/cache.py +++ b/charon/cache.py @@ -1,6 +1,6 @@ from boto3 import session from botocore.exceptions import ClientError -from typing import Dict, List +from typing import Dict, List, Optional import os import logging import uuid @@ -57,7 +57,7 @@ def __init_aws_client( endpoint_url=endpoint_url ) - def __get_endpoint(self, extra_conf) -> str: + def __get_endpoint(self, extra_conf) -> Optional[str]: endpoint_url = os.getenv(ENDPOINT_ENV) if not endpoint_url or not endpoint_url.strip(): if isinstance(extra_conf, Dict): @@ -97,14 +97,14 @@ def invalidate_paths( " will take more than %d seconds", len(real_paths), total_time_approx) results = [] - current_invalidation = {} + current_invalidation: Dict[str, str] = {} processed_count = 0 for batch_paths in real_paths: while (current_invalidation and INVALIDATION_STATUS_INPROGRESS == current_invalidation.get('Status', '')): time.sleep(INPRO_W_SECS) try: - result = self.check_invalidation(distr_id, current_invalidation.get('Id')) + result = self.check_invalidation(distr_id, current_invalidation.get('Id', '')) if result: current_invalidation = { 'Id': result.get('Id', None), @@ -159,7 +159,7 @@ def invalidate_paths( results.append(current_invalidation) return results - def check_invalidation(self, distr_id: str, invalidation_id: str) -> dict: + def check_invalidation(self, distr_id: str, invalidation_id: str) -> Optional[dict]: try: response = self.__client.get_invalidation( DistributionId=distr_id, @@ -177,8 +177,9 @@ def check_invalidation(self, distr_id: str, invalidation_id: str) -> dict: "[CloudFront] Error occurred while check invalidation of id %s, " "error: %s", invalidation_id, err ) + return None - def get_dist_id_by_domain(self, domain: str) -> str: + def get_dist_id_by_domain(self, domain: str) -> Optional[str]: """Get distribution id by a domain name. The id can be used to send invalidating request through #invalidate_paths function * Domain are Ronda domains, like "maven.repository.redhat.com" @@ -200,5 +201,5 @@ def get_dist_id_by_domain(self, domain: str) -> str: ) return None - def get_domain_by_bucket(self, bucket: str) -> str: + def get_domain_by_bucket(self, bucket: str) -> Optional[str]: return DEFAULT_BUCKET_TO_DOMAIN.get(bucket, None) diff --git a/charon/cmd/cmd_cache.py b/charon/cmd/cmd_cache.py index e7f55282..6a5a0113 100644 --- a/charon/cmd/cmd_cache.py +++ b/charon/cmd/cmd_cache.py @@ -15,11 +15,12 @@ """ from charon.config import get_config -from charon.cmd.internal import _decide_mode, _get_buckets +from charon.cmd.internal import _decide_mode, _get_targets from charon.cache import CFClient from charon.pkgs.pkg_utils import invalidate_cf_paths +from charon.types import TARGET_TYPE from click import command, option, argument, group -from typing import List, Tuple +from typing import List, Tuple, Optional import traceback import logging @@ -87,7 +88,7 @@ def invalidate( target: str, paths: List[str], path_file: str, - config: str = None, + config: str = "", quiet: bool = False, debug: bool = False ): @@ -119,20 +120,20 @@ def invalidate( break try: - (buckets, aws_profile) = _init_cmd(target, config) + (targets, aws_profile) = _init_cmd(target, config) - for b in buckets: + for t in targets: cf_client = CFClient(aws_profile=aws_profile) # Per aws official doc, if the paths contains wildcard, it is # limited to 15 as max items in one request. Otherwise it could # be 3000 if use_wildcard: invalidate_cf_paths( - cf_client, b, work_paths + cf_client, t, work_paths ) else: invalidate_cf_paths( - cf_client, b, work_paths, batch_size=3000 + cf_client, t, work_paths, batch_size=3000 ) except Exception: print(traceback.format_exc()) @@ -181,7 +182,7 @@ def invalidate( def check( invalidation_id: str, target: str, - config: str = None, + config: str = "", quiet: bool = False, debug: bool = False ): @@ -193,14 +194,14 @@ def check( is_quiet=quiet, is_debug=debug, use_log_file=False ) try: - (buckets, aws_profile) = _init_cmd(target, config) - if not buckets: + (targets, aws_profile) = _init_cmd(target, config) + if not targets: sys.exit(1) - for b in buckets: + for t in targets: cf_client = CFClient(aws_profile=aws_profile) - bucket_name = b[1] - domain = b[4] + bucket_name = t[1] + domain: Optional[str] = t[4] if not domain: domain = cf_client.get_domain_by_bucket(bucket_name) if domain: @@ -221,7 +222,7 @@ def check( sys.exit(2) -def _init_cmd(target: str, config: str) -> Tuple[List[Tuple[str, str, str, str, str]], str]: +def _init_cmd(target: str, config: str) -> Tuple[List[TARGET_TYPE], str]: conf = get_config(config) if not conf: sys.exit(1) @@ -231,7 +232,7 @@ def _init_cmd(target: str, config: str) -> Tuple[List[Tuple[str, str, str, str, logger.error("No AWS profile specified!") sys.exit(1) - return (_get_buckets([target], conf), aws_profile) + return (_get_targets([target], conf), aws_profile) @group() diff --git a/charon/cmd/cmd_checksum.py b/charon/cmd/cmd_checksum.py index 6940798f..a0aea5cc 100644 --- a/charon/cmd/cmd_checksum.py +++ b/charon/cmd/cmd_checksum.py @@ -104,7 +104,7 @@ def validate( path: str, target: str, - includes: List[str], + includes: str, report_file_path: str, skips: List[str], recursive: bool = False, @@ -259,12 +259,12 @@ def _init_cmd(target: str) -> Tuple[str, str]: conf = get_config() if not conf: sys.exit(1) - aws_bucket = "" t = conf.get_target(target) if not t: sys.exit(1) + aws_bucket = '' for b in t: - aws_bucket = b.get('bucket') + aws_bucket = b.get('bucket', '') prefix = b.get('prefix', '') return (aws_bucket, prefix) diff --git a/charon/cmd/cmd_delete.py b/charon/cmd/cmd_delete.py index 7d9ae5f0..4cb982bd 100644 --- a/charon/cmd/cmd_delete.py +++ b/charon/cmd/cmd_delete.py @@ -21,7 +21,7 @@ from charon.pkgs.npm import handle_npm_del from charon.cmd.internal import ( _decide_mode, _validate_prod_key, - _get_local_repo, _get_buckets, + _get_local_repo, _get_targets, _get_ignore_patterns, _safe_delete ) from click import command, option, argument @@ -158,19 +158,19 @@ def delete( npm_archive_type = detect_npm_archive(archive_path) product_key = f"{product}-{version}" manifest_bucket_name = conf.get_manifest_bucket() - buckets = _get_buckets(targets, conf) - if not buckets: + targets_ = _get_targets(targets, conf) + if not targets_: logger.error( "The targets %s can not be found! Please check" " your charon configuration to confirm the targets" - " are set correctly.", targets + " are set correctly.", targets_ ) if npm_archive_type != NpmArchiveType.NOT_NPM: logger.info("This is a npm archive") tmp_dir, succeeded = handle_npm_del( archive_path, product_key, - buckets=buckets, + targets=targets_, aws_profile=aws_profile, dir_=work_dir, cf_enable=conf.is_aws_cf_enable(), @@ -191,7 +191,7 @@ def delete( product_key, ignore_patterns_list, root=root_path, - buckets=buckets, + targets=targets_, aws_profile=aws_profile, dir_=work_dir, cf_enable=conf.is_aws_cf_enable(), @@ -204,5 +204,5 @@ def delete( print(traceback.format_exc()) sys.exit(2) # distinguish between exception and bad config or bad state finally: - if not debug: + if not debug and tmp_dir: _safe_delete(tmp_dir) diff --git a/charon/cmd/cmd_index.py b/charon/cmd/cmd_index.py index 14e0deaa..e5dd11a5 100644 --- a/charon/cmd/cmd_index.py +++ b/charon/cmd/cmd_index.py @@ -97,7 +97,7 @@ def index( sys.exit(1) for b in tgt: - aws_bucket = b.get('bucket') + aws_bucket = b.get('bucket', '') package_type = None if "maven" in aws_bucket: @@ -115,6 +115,7 @@ def index( "The target %s is not supported. Only maven or npm target is supported.", target ) + continue if not aws_bucket: logger.error("No bucket specified for target %s!", target) diff --git a/charon/cmd/cmd_upload.py b/charon/cmd/cmd_upload.py index a55c5e0f..a867df01 100644 --- a/charon/cmd/cmd_upload.py +++ b/charon/cmd/cmd_upload.py @@ -21,7 +21,7 @@ from charon.pkgs.npm import handle_npm_uploading from charon.cmd.internal import ( _decide_mode, _validate_prod_key, - _get_local_repo, _get_buckets, + _get_local_repo, _get_targets, _get_ignore_patterns, _safe_delete ) from click import command, option, argument @@ -177,12 +177,12 @@ def upload( npm_archive_type = detect_npm_archive(archive_path) product_key = f"{product}-{version}" manifest_bucket_name = conf.get_manifest_bucket() - buckets = _get_buckets(targets, conf) - if not buckets: + targets_ = _get_targets(targets, conf) + if not targets_: logger.error( "The targets %s can not be found! Please check" " your charon configuration to confirm the targets" - " are set correctly.", targets + " are set correctly.", targets_ ) sys.exit(1) if npm_archive_type != NpmArchiveType.NOT_NPM: @@ -190,7 +190,7 @@ def upload( tmp_dir, succeeded = handle_npm_uploading( archive_path, product_key, - buckets=buckets, + targets=targets_, aws_profile=aws_profile, dir_=work_dir, gen_sign=contain_signature, @@ -213,7 +213,7 @@ def upload( product_key, ignore_patterns_list, root=root_path, - buckets=buckets, + targets=targets_, aws_profile=aws_profile, dir_=work_dir, gen_sign=contain_signature, @@ -229,5 +229,5 @@ def upload( print(traceback.format_exc()) sys.exit(2) # distinguish between exception and bad config or bad state finally: - if not debug: + if not debug and tmp_dir: _safe_delete(tmp_dir) diff --git a/charon/cmd/internal.py b/charon/cmd/internal.py index 11c92a0c..e7e7d14a 100644 --- a/charon/cmd/internal.py +++ b/charon/cmd/internal.py @@ -13,10 +13,11 @@ See the License for the specific language governing permissions and limitations under the License. """ -from typing import List, Tuple +from typing import List, Optional from charon.config import CharonConfig from charon.constants import DEFAULT_REGISTRY +from charon.types import TARGET_TYPE from charon.utils.logs import set_logging from charon.utils.archive import download_archive from json import loads @@ -28,18 +29,18 @@ logger = logging.getLogger(__name__) -def _get_buckets( - targets: List[str], conf: CharonConfig -) -> List[Tuple[str, str, str, str, str]]: - buckets = [] - for target in targets: +def _get_targets( + target_names: List[str], conf: CharonConfig +) -> List[TARGET_TYPE]: + targets: List[TARGET_TYPE] = [] + for target in target_names: for bucket in conf.get_target(target): - aws_bucket = bucket.get('bucket') + aws_bucket = bucket.get('bucket', '') prefix = bucket.get('prefix', '') registry = bucket.get('registry', DEFAULT_REGISTRY) cf_domain = bucket.get('domain', None) - buckets.append((target, aws_bucket, prefix, registry, cf_domain)) - return buckets + targets.append((target, aws_bucket, prefix, registry, cf_domain)) + return targets def _safe_delete(tmp_dir: str): @@ -51,7 +52,7 @@ def _safe_delete(tmp_dir: str): logger.error("Failed to clear work directory. %s", e) -def _get_ignore_patterns(conf: CharonConfig) -> List[str]: +def _get_ignore_patterns(conf: CharonConfig) -> Optional[List[str]]: ignore_patterns = os.getenv("CHARON_IGNORE_PATTERNS") if ignore_patterns: try: diff --git a/charon/config.py b/charon/config.py index 84b4aec4..86f826ea 100644 --- a/charon/config.py +++ b/charon/config.py @@ -55,8 +55,8 @@ def get_aws_profile(self) -> str: def get_manifest_bucket(self) -> str: return self.__manifest_bucket - def get_ignore_signature_suffix(self, package_type: str) -> List[str]: - xartifact_list: List = self.__ignore_signature_suffix.get(package_type) + def get_ignore_signature_suffix(self, package_type: str) -> Optional[List[str]]: + xartifact_list = self.__ignore_signature_suffix.get(package_type) if not xartifact_list: logger.error("package type %s does not have ignore artifact config.", package_type) return xartifact_list @@ -68,10 +68,10 @@ def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable -def get_config(cfgPath=None) -> Optional[CharonConfig]: +def get_config(cfgPath=None) -> CharonConfig: config_file_path = cfgPath if not config_file_path or not os.path.isfile(config_file_path): - config_file_path = os.path.join(os.getenv("HOME"), ".charon", CONFIG_FILE) + config_file_path = os.path.join(os.getenv("HOME", ""), ".charon", CONFIG_FILE) data = read_yaml_from_file_path(config_file_path, 'schemas/charon.json') return CharonConfig(data) diff --git a/charon/pkgs/checksum_http.py b/charon/pkgs/checksum_http.py index 4654559b..e57dab34 100644 --- a/charon/pkgs/checksum_http.py +++ b/charon/pkgs/checksum_http.py @@ -15,7 +15,7 @@ """ from charon.utils.files import digest, HashType from charon.storage import S3Client -from typing import Tuple, List, Dict +from typing import Tuple, List, Dict, Optional from html.parser import HTMLParser import tempfile import os @@ -51,7 +51,7 @@ def handle_checksum_validation_http( checksum files. Will use sha1 to do the validation. """ local_dir = tempfile.mkdtemp() - results = ([], [], []) + results: Tuple[List[str], List[str], List[Dict[str, str]]] = ([], [], []) try: if not os.path.exists(local_dir): os.makedirs(local_dir) @@ -76,7 +76,7 @@ def _collect_invalid_files( includes: str, work_dir: str, recursive: bool, - skips: List[str], + skips: Optional[List[str]], results: Tuple[List[str], List[str], List[Dict[str, str]]] ): if skips and path in skips: @@ -250,14 +250,14 @@ def get_content(self, parent): return [os.path.join(parent, i) for i in self.__content] -def _read_remote_file_content(remote_file_url: str) -> str: +def _read_remote_file_content(remote_file_url: str) -> Optional[str]: try: with requests.get(remote_file_url) as r: if r.status_code == 200: return r.text.strip() if r.text else "" except Exception as e: logger.error("Can not read file %s. The error is %s", remote_file_url, e) - return None + return None def _decide_root_url(bucket: str) -> str: @@ -265,7 +265,7 @@ def _decide_root_url(bucket: str) -> str: return "https://maven.repository.redhat.com" if bucket.strip().startswith("stage-maven"): return "https://maven.stage.repository.redhat.com" - return None + return "" def refresh_checksum( diff --git a/charon/pkgs/indexing.py b/charon/pkgs/indexing.py index 6d9c8d42..4d50e036 100644 --- a/charon/pkgs/indexing.py +++ b/charon/pkgs/indexing.py @@ -23,7 +23,7 @@ from jinja2 import Template import os import logging -from typing import List, Set, Tuple +from typing import List, Set, Dict from charon.utils.strings import remove_prefix @@ -37,10 +37,9 @@ def __get_index_template(package_type: str) -> str: except FileNotFoundError: logger.info("index template file not defined," " will use default template.") - if package_type == PACKAGE_TYPE_MAVEN: - return INDEX_HTML_TEMPLATE - elif package_type == PACKAGE_TYPE_NPM: + if package_type == PACKAGE_TYPE_NPM: return NPM_INDEX_HTML_TEMPLATE + return INDEX_HTML_TEMPLATE MAVEN_INDEX_TEMPLATE = __get_index_template(PACKAGE_TYPE_MAVEN) @@ -55,13 +54,10 @@ def __init__(self, title: str, header: str, items: Set[str]): self.items = items def generate_index_file_content(self, package_type: str) -> str: - template = None - if package_type == PACKAGE_TYPE_MAVEN: - template = Template(MAVEN_INDEX_TEMPLATE) - elif package_type == PACKAGE_TYPE_NPM: + template = Template(MAVEN_INDEX_TEMPLATE) + if package_type == PACKAGE_TYPE_NPM: template = Template(NPM_INDEX_TEMPLATE) - if template: - return template.render(index=self) + return template.render(index=self) def generate_indexes( @@ -70,7 +66,7 @@ def generate_indexes( changed_dirs: List[str], s3_client: S3Client, bucket: str, - prefix: str = None + prefix: str = "" ) -> List[str]: if top_level[-1] != '/': top_level += '/' @@ -87,8 +83,8 @@ def generate_indexes( s3_folders.add(path) generated_htmls = [] - s3_folders = sorted(s3_folders, key=FolderLenCompareKey) - for folder_ in s3_folders: + s3_folders_list = sorted(list(s3_folders), key=FolderLenCompareKey) + for folder_ in s3_folders_list: index_html = __generate_index_html( package_type, s3_client, bucket, folder_, top_level, prefix ) @@ -110,7 +106,7 @@ def __generate_index_html( bucket: str, folder_: str, top_level: str, - prefix: str = None + prefix: str = "" ) -> str: if folder_ != "/": search_folder = os.path.join(prefix, folder_) if prefix else folder_ @@ -122,7 +118,7 @@ def __generate_index_html( ) # Should filter out the .prodinfo files contents = [c for c in contents if not c.endswith(PROD_INFO_SUFFIX)] - index = None + index = "" if len(contents) == 1 and contents[0].endswith("index.html"): logger.info("The folder %s only contains index.html, " "will remove it.", folder_) @@ -178,8 +174,8 @@ def __to_html_content(package_type: str, contents: List[str], folder: str) -> st items = temp_items else: items.extend(contents) - items = __sort_index_items(items) - index = IndexedHTML(title=folder, header=folder, items=items) + items_set = set(__sort_index_items(items)) + index = IndexedHTML(title=folder, header=folder, items=items_set) return index.generate_index_file_content(package_type) @@ -267,7 +263,7 @@ def __compare(self, other) -> int: def re_index( - bucket: Tuple[str, str, str, str, str], + target: Dict[str, str], path: str, package_type: str, aws_profile: str = None, @@ -276,8 +272,8 @@ def re_index( ): """Refresh the index.html for the specified folder in the bucket. """ - bucket_name = bucket.get("bucket") - prefix = bucket.get("prefix") + bucket_name = target.get("bucket", "") + prefix = target.get("prefix", "") s3_client = S3Client(aws_profile=aws_profile, dry_run=dry_run) real_prefix = prefix if prefix.strip() != "/" else "" s3_folder = os.path.join(real_prefix, path) diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 3a73f528..9f50f35b 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -21,6 +21,7 @@ from charon.utils.strings import remove_prefix from charon.storage import S3Client from charon.cache import CFClient +from charon.types import TARGET_TYPE from charon.pkgs.pkg_utils import ( upload_post_process, rollback_post_process, @@ -204,7 +205,7 @@ def parse_gavs(pom_paths: List[str], root="/") -> Dict[str, Dict[str, List[str]] from them. The result will be a dict like {groupId: {artifactId: [versions list]}}. Root is like a prefix of the path which is not part of the maven GAV """ - gavs = dict() + gavs: Dict[str, Dict] = dict() for pom in pom_paths: (g, a, v) = __parse_gav(pom, root) avs = gavs.get(g, dict()) @@ -264,7 +265,7 @@ def handle_maven_uploading( prod_key: str, ignore_patterns=None, root="maven-repository", - buckets: List[Tuple[str, str, str, str, str]] = None, + targets: List[TARGET_TYPE] = None, aws_profile=None, dir_=None, do_index=True, @@ -291,6 +292,8 @@ def handle_maven_uploading( Returns the directory used for archive processing and if the uploading is successful """ + if targets is None: + targets = [] # 1. extract tarball tmp_root = _extract_tarball(repo, prod_key, dir__=dir_) @@ -316,10 +319,10 @@ def handle_maven_uploading( # 4. Do uploading s3_client = S3Client(aws_profile=aws_profile, dry_run=dry_run) - targets_ = [(bucket[1], remove_prefix(bucket[2], "/")) for bucket in buckets] + targets_ = [(target[1], remove_prefix(target[2], "/")) for target in targets] logger.info( "Start uploading files to s3 buckets: %s", - [bucket[1] for bucket in buckets] + [target[1] for target in targets] ) failed_files = s3_client.upload_files( file_paths=valid_mvn_paths, @@ -330,7 +333,7 @@ def handle_maven_uploading( logger.info("Files uploading done\n") succeeded = True generated_signs = [] - for bucket in buckets: + for bucket in targets: # prepare cf invalidate files cf_invalidate_paths = [] @@ -477,7 +480,7 @@ def handle_maven_del( prod_key: str, ignore_patterns=None, root="maven-repository", - buckets: List[Tuple[str, str, str, str, str]] = None, + targets: List[TARGET_TYPE] = None, aws_profile=None, dir_=None, do_index=True, @@ -493,7 +496,7 @@ def handle_maven_del( need to upload in the tarball * root is a prefix in the tarball to identify which path is the beginning of the maven GAV path - * buckets contains the target name with its bucket name and prefix + * targets contains the target name with its bucket name and prefix for the bucket, which will be used to store artifacts with the prefix. See target definition in Charon configuration for details * dir is base dir for extracting the tarball, will use system @@ -501,6 +504,9 @@ def handle_maven_del( Returns the directory used for archive processing and if the rollback is successful """ + if targets is None: + targets = [] + # 1. extract tarball tmp_root = _extract_tarball(repo, prod_key, dir__=dir_) @@ -514,13 +520,13 @@ def handle_maven_del( # 3. Delete all valid_paths from s3 logger.debug("Valid poms: %s", valid_poms) succeeded = True - for bucket in buckets: + for target in targets: # prepare cf invalidation paths cf_invalidate_paths = [] - prefix = remove_prefix(bucket[2], "/") + prefix = remove_prefix(target[2], "/") s3_client = S3Client(aws_profile=aws_profile, dry_run=dry_run) - bucket_name = bucket[1] + bucket_name = target[1] logger.info("Start deleting files from s3 bucket %s", bucket_name) failed_files = s3_client.delete_files( valid_mvn_paths, @@ -531,7 +537,7 @@ def handle_maven_del( logger.info("Files deletion done\n") # 4. Delete related manifest from s3 - manifest_folder = bucket[1] + manifest_folder = target[1] logger.info( "Start deleting manifest from s3 bucket %s in folder %s", manifest_bucket_name, manifest_folder @@ -644,7 +650,7 @@ def handle_maven_del( if cf_enable and len(cf_invalidate_paths): cf_client = CFClient(aws_profile=aws_profile) cf_invalidate_paths = __wildcard_metadata_paths(cf_invalidate_paths) - invalidate_cf_paths(cf_client, bucket, cf_invalidate_paths, top_level) + invalidate_cf_paths(cf_client, target, cf_invalidate_paths, top_level) rollback_post_process(failed_files, failed_metas, prod_key, bucket_name) succeeded = succeeded and len(failed_files) == 0 and len(failed_metas) == 0 @@ -1008,8 +1014,8 @@ def _generate_metadatas( ga_dict[os.path.join(g_path, a)] = True # Note: here we don't need to add original poms, because # they have already been uploaded to s3. - all_poms = [] - meta_files = {} + all_poms: List[str] = [] + meta_files: Dict[str, List[str]] = {} for path, _ in ga_dict.items(): # avoid some wrong prefix, like searching org/apache # but got org/apache-commons @@ -1084,7 +1090,7 @@ def _is_ignored(filename: str, ignore_patterns: List[str]) -> bool: def _validate_maven(paths: List[str]) -> Tuple[List[str], bool]: # Reminder: need to implement later - return (list, True) + return (list(), True) def _handle_error(err_msgs: List[str]): @@ -1094,7 +1100,9 @@ def _handle_error(err_msgs: List[str]): def __get_suffix(package_type: str, conf: CharonConfig) -> List[str]: if package_type: - return conf.get_ignore_signature_suffix(package_type) + suffix = conf.get_ignore_signature_suffix(package_type) + if suffix: + return suffix return [] diff --git a/charon/pkgs/npm.py b/charon/pkgs/npm.py index 33d0c961..0bbfad01 100644 --- a/charon/pkgs/npm.py +++ b/charon/pkgs/npm.py @@ -19,7 +19,7 @@ from json import load, loads, dump, JSONDecodeError, JSONEncoder import tarfile from tempfile import mkdtemp -from typing import List, Set, Tuple +from typing import List, Set, Tuple, Dict, Optional from semantic_version import compare @@ -29,6 +29,7 @@ from charon.constants import META_FILE_GEN_KEY, META_FILE_DEL_KEY, PACKAGE_TYPE_NPM from charon.storage import S3Client from charon.cache import CFClient +from charon.types import TARGET_TYPE from charon.utils.archive import extract_npm_tarball from charon.pkgs.pkg_utils import ( upload_post_process, @@ -78,7 +79,7 @@ def default(self, o): def handle_npm_uploading( tarball_path: str, product: str, - buckets: List[Tuple[str, str, str, str]], + targets: List[TARGET_TYPE], aws_profile=None, dir_=None, root_path="package", @@ -96,7 +97,7 @@ def handle_npm_uploading( * tarball_path is the location of the tarball in filesystem * product is used to identify which product this repo tar belongs to - * buckets contains the target name with its bucket name and prefix + * targets contains the target name with its bucket name and prefix for the bucket, which will be used to store artifacts with the prefix. See target definition in Charon configuration for details * dir_ is base dir for extracting the tarball, will use system @@ -109,13 +110,13 @@ def handle_npm_uploading( generated_signs = [] succeeded = True root_dir = mkdtemp(prefix=f"npm-charon-{product}-", dir=dir_) - for bucket in buckets: + for target in targets: # prepare cf invalidate files cf_invalidate_paths = [] - bucket_name = bucket[1] - prefix = remove_prefix(bucket[2], "/") - registry = bucket[3] + bucket_name = target[1] + prefix = remove_prefix(target[2], "/") + registry = target[3] target_dir, valid_paths, package_metadata = _scan_metadata_paths_from_archive( tarball_path, registry, prod=product, dir__=dir_, pkg_root=root_path ) @@ -148,10 +149,11 @@ def handle_npm_uploading( ) logger.info("Manifest uploading is done\n") - logger.info( - "Start generating version-level package.json for package: %s in s3 bucket %s", - package_metadata.name, bucket_name - ) + if package_metadata: + logger.info( + "Start generating version-level package.json for package: %s in s3 bucket %s", + package_metadata.name, bucket_name + ) failed_metas = [] _version_metadata_path = valid_paths[1] _failed_metas = client.upload_metadatas( @@ -163,10 +165,11 @@ def handle_npm_uploading( failed_metas.extend(_failed_metas) logger.info("version-level package.json uploading done") - logger.info( - "Start generating package.json for package: %s in s3 bucket %s", - package_metadata.name, bucket_name - ) + if package_metadata: + logger.info( + "Start generating package.json for package: %s in s3 bucket %s", + package_metadata.name, bucket_name + ) meta_files = _gen_npm_package_metadata_for_upload( client, bucket_name, target_dir, package_metadata, prefix ) @@ -224,7 +227,7 @@ def handle_npm_uploading( if do_index: logger.info("Start generating index files to s3 bucket %s", bucket_name) created_indexes = indexing.generate_indexes( - PACKAGE_TYPE_NPM, target_dir, valid_dirs, client, bucket_name, prefix + PACKAGE_TYPE_NPM, target_dir, list(valid_dirs), client, bucket_name, prefix ) logger.info("Index files generation done.\n") @@ -246,7 +249,7 @@ def handle_npm_uploading( # Do CloudFront invalidating for generated metadata if cf_enable and len(cf_invalidate_paths): cf_client = CFClient(aws_profile=aws_profile) - invalidate_cf_paths(cf_client, bucket, cf_invalidate_paths, target_dir) + invalidate_cf_paths(cf_client, target, cf_invalidate_paths, target_dir) upload_post_process(failed_files, failed_metas, product, bucket_name) succeeded = succeeded and len(failed_files) == 0 and len(failed_metas) == 0 @@ -257,7 +260,7 @@ def handle_npm_uploading( def handle_npm_del( tarball_path: str, product: str, - buckets: List[Tuple[str, str, str, str]], + targets: List[TARGET_TYPE], aws_profile=None, dir_=None, root_path="package", @@ -265,12 +268,12 @@ def handle_npm_del( cf_enable=False, dry_run=False, manifest_bucket_name=None -) -> Tuple[str, str]: +) -> Tuple[str, bool]: """ Handle the npm product release tarball deletion process. * tarball_path is the location of the tarball in filesystem * product is used to identify which product this repo tar belongs to - * buckets contains the target name with its bucket name and prefix + * targets contains the target name with its bucket name and prefix for the bucket, which will be used to store artifacts with the prefix. See target definition in Charon configuration for details * dir is base dir for extracting the tarball, will use system @@ -286,12 +289,12 @@ def handle_npm_del( client = S3Client(aws_profile=aws_profile, dry_run=dry_run) succeeded = True - for bucket in buckets: + for target in targets: # prepare cf invalidate files cf_invalidate_paths = [] - bucket_name = bucket[1] - prefix = remove_prefix(bucket[2], "/") + bucket_name = target[1] + prefix = remove_prefix(target[2], "/") logger.info("Start deleting files from s3 bucket %s", bucket_name) failed_files = client.delete_files( file_paths=valid_paths, @@ -301,7 +304,7 @@ def handle_npm_del( logger.info("Files deletion done\n") if manifest_bucket_name: - manifest_folder = bucket[1] + manifest_folder = target[1] logger.info( "Start deleting manifest from s3 bucket %s in folder %s", manifest_bucket_name, manifest_folder @@ -351,7 +354,7 @@ def handle_npm_del( bucket_name ) created_indexes = indexing.generate_indexes( - PACKAGE_TYPE_NPM, target_dir, valid_dirs, client, bucket_name, prefix + PACKAGE_TYPE_NPM, target_dir, list(valid_dirs), client, bucket_name, prefix ) logger.info("Index files generation done.\n") @@ -374,7 +377,7 @@ def handle_npm_del( # Do CloudFront invalidating for generated metadata if cf_enable and len(cf_invalidate_paths): cf_client = CFClient(aws_profile=aws_profile) - invalidate_cf_paths(cf_client, bucket, cf_invalidate_paths, target_dir) + invalidate_cf_paths(cf_client, target, cf_invalidate_paths, target_dir) rollback_post_process(failed_files, failed_metas, product, bucket_name) succeeded = succeeded and len(failed_files) <= 0 and len(failed_metas) <= 0 @@ -382,19 +385,20 @@ def handle_npm_del( return (target_dir, succeeded) -def read_package_metadata_from_content(content: str, is_version) -> NPMPackageMetadata: +def read_package_metadata_from_content(content: str, is_version) -> Optional[NPMPackageMetadata]: try: package_metadata = loads(content) return NPMPackageMetadata(package_metadata, is_version) except JSONDecodeError: logger.error('Error: Failed to parse json!') + return None def _gen_npm_package_metadata_for_upload( client: S3Client, bucket: str, - target_dir: str, source_package: NPMPackageMetadata, + target_dir: str, source_package: Optional[NPMPackageMetadata], prefix: str = None -) -> dict: +) -> Dict: """Collect NPM versions package.json and generate the package package.json. For uploading mode, package.json will merge the original in S3 with the local source. What we should do here is: @@ -403,23 +407,24 @@ def _gen_npm_package_metadata_for_upload( * Use converted package.json to generate the package.json then update in S3 """ meta_files = {} - package_metadata_key = os.path.join(source_package.name, PACKAGE_JSON) - if prefix and prefix != "/": - package_metadata_key = os.path.join(prefix, package_metadata_key) - (package_json_files, success) = client.get_files( - bucket_name=bucket, - prefix=package_metadata_key - ) - if not success: - logger.warning("Error to get remote metadata files for %s", package_metadata_key) - result = source_package - if len(package_json_files) > 0: - result = _merge_package_metadata( - source_package, client, bucket, package_json_files[0] + if source_package: + package_metadata_key = os.path.join(source_package.name, PACKAGE_JSON) + if prefix and prefix != "/": + package_metadata_key = os.path.join(prefix, package_metadata_key) + (package_json_files, success) = client.get_files( + bucket_name=bucket, + prefix=package_metadata_key ) - logger.debug("Merge the S3 %s with local source", package_json_files[0]) - meta_file = _write_package_metadata_to_file(result, target_dir) - meta_files[META_FILE_GEN_KEY] = meta_file + if not success: + logger.warning("Error to get remote metadata files for %s", package_metadata_key) + result = source_package + if len(package_json_files) > 0: + result = _merge_package_metadata( + source_package, client, bucket, package_json_files[0] + ) + logger.debug("Merge the S3 %s with local source", package_json_files[0]) + meta_file = _write_package_metadata_to_file(result, target_dir) + meta_files[META_FILE_GEN_KEY] = meta_file return meta_files @@ -427,7 +432,7 @@ def _gen_npm_package_metadata_for_del( client: S3Client, bucket: str, target_dir: str, package_path_prefix: str, prefix: str = None -) -> dict: +) -> Dict: """Collect NPM versions package.json and generate the package package.json. For del mode, all the version package.json contents to be merged will be read from S3. What we should do here is: @@ -462,7 +467,7 @@ def _gen_npm_package_metadata_for_del( continue meta_contents.append(meta) if len(meta_contents) == 0: - return + return {} original = meta_contents[0] for source in meta_contents: source_version = list(source.versions.keys())[0] @@ -479,7 +484,7 @@ def _gen_npm_package_metadata_for_del( def _scan_metadata_paths_from_archive( path: str, registry: str, prod="", dir__=None, pkg_root="pakage" -) -> Tuple[str, list, NPMPackageMetadata]: +) -> Tuple[str, list, Optional[NPMPackageMetadata]]: tmp_root = mkdtemp(prefix=f"npm-charon-{prod}-", dir=dir__) try: _, valid_paths = extract_npm_tarball( @@ -507,14 +512,14 @@ def _scan_paths_from_archive( def _merge_package_metadata( - package_metadata: NPMPackageMetadata, + package_metadata: Optional[NPMPackageMetadata], client: S3Client, bucket: str, key: str ): content = client.read_file_content(bucket, key) original = read_package_metadata_from_content(content, False) - if original: + if original and package_metadata: source_version = list(package_metadata.versions.keys())[0] is_latest = _is_latest_version(source_version, list(original.versions.keys())) _do_merge(original, package_metadata, is_latest) @@ -617,9 +622,10 @@ def _write_package_metadata_to_file(package_metadata: NPMPackageMetadata, root=' logger.error( 'Can not create file %s because of some missing folders', final_package_metadata_path ) + return "" -def __get_path_tree(paths: str, prefix: str) -> Set[str]: +def __get_path_tree(paths: List[str], prefix: str) -> Set[str]: valid_dirs = set() for f in paths: dir_ = os.path.dirname(f) @@ -634,5 +640,7 @@ def __get_path_tree(paths: str, prefix: str) -> Set[str]: def __get_suffix(package_type: str, conf: CharonConfig) -> List[str]: if package_type: - return conf.get_ignore_signature_suffix(package_type) + suffix = conf.get_ignore_signature_suffix(package_type) + if suffix: + return suffix return [] diff --git a/charon/pkgs/pkg_utils.py b/charon/pkgs/pkg_utils.py index 9d57def1..fd9aed89 100644 --- a/charon/pkgs/pkg_utils.py +++ b/charon/pkgs/pkg_utils.py @@ -1,10 +1,11 @@ -from typing import List, Tuple +from typing import List, Optional, Dict from charon.cache import ( CFClient, INVALIDATION_BATCH_DEFAULT, INVALIDATION_BATCH_WILDCARD, INVALIDATION_STATUS_COMPLETED ) +from charon.types import TARGET_TYPE import logging import os @@ -69,16 +70,16 @@ def __post_process( def invalidate_cf_paths( cf_client: CFClient, - bucket: Tuple[str, str, str, str, str], + target: TARGET_TYPE, invalidate_paths: List[str], root="/", batch_size=INVALIDATION_BATCH_DEFAULT ): - logger.info("Invalidating CF cache for %s", bucket[1]) - bucket_name = bucket[1] - prefix = bucket[2] + logger.info("Invalidating CF cache for %s", target[1]) + bucket_name = target[1] + prefix = target[2] prefix = "/" + prefix if not prefix.startswith("/") else prefix - domain = bucket[4] + domain: Optional[str] = target[4] slash_root = root if not root.endswith("/"): slash_root = slash_root + "/" @@ -105,10 +106,10 @@ def invalidate_cf_paths( distr_id, final_paths, real_batch_size ) if result: - output = {} + output: Dict[str, List[str]] = {} for invalidation in result: - status = invalidation.get('Status') - if status not in output: + status = invalidation.get('Status', '') + if status and status not in output: output[status] = [] output[status].append(invalidation["Id"]) non_completed = {} diff --git a/charon/pkgs/signature.py b/charon/pkgs/signature.py index 412aeba8..024b6413 100644 --- a/charon/pkgs/signature.py +++ b/charon/pkgs/signature.py @@ -20,7 +20,7 @@ import logging import shlex from jinja2 import Template -from typing import Awaitable, Callable, List, Tuple +from typing import Callable, List, Tuple from charon.storage import S3Client logger = logging.getLogger(__name__) @@ -34,7 +34,7 @@ def generate_sign( s3_client: S3Client, bucket: str, key: str = None, - command: str = None + command: str = "" ) -> Tuple[List[str], List[str]]: """ This Python function generates a digital signature for a list of metadata files using the GPG library for uploads to an Amazon S3 bucket. @@ -100,14 +100,14 @@ async def sign_file( def __do_path_cut_and( file_paths: List[str], - path_handler: Callable[[str, List[str], List[str], asyncio.Semaphore], Awaitable[bool]], + path_handler: Callable, root="/" -) -> List[str]: +) -> Tuple[List[str], List[str]]: slash_root = root if not root.endswith("/"): slash_root = slash_root + "/" - failed_paths = [] - generated_signs = [] + failed_paths: List[str] = [] + generated_signs: List[str] = [] tasks = [] sem = asyncio.BoundedSemaphore(10) for full_path in file_paths: diff --git a/charon/storage.py b/charon/storage.py index a34a079d..879a5ba7 100644 --- a/charon/storage.py +++ b/charon/storage.py @@ -44,6 +44,8 @@ FILE_REPORT_LIMIT = 1000 +PATH_HANDLER_TYPE = Callable[[str, str, int, int, List[str]], Awaitable[bool]] + class S3Client(object): """The S3Client is a wrapper of the original boto3 s3 client, which will provide @@ -81,7 +83,7 @@ def __init_aws_client( config=config ) - def __get_endpoint(self, extra_conf) -> str: + def __get_endpoint(self, extra_conf) -> Optional[str]: endpoint_url = os.getenv(ENDPOINT_ENV) if not endpoint_url or endpoint_url.strip() == "": if isinstance(extra_conf, Dict): @@ -276,7 +278,7 @@ async def handle_existed( ) if not result: return False - return True + return True return self.__do_path_cut_and( file_paths=file_paths, @@ -918,8 +920,8 @@ async def __update_prod_info( def __path_handler_count_wrapper( self, - path_handler: Callable[[str, str, int, int, List[str], asyncio.Semaphore], Awaitable[bool]] - ) -> Callable[[str, str, int, int, List[str], asyncio.Semaphore], Awaitable[bool]]: + path_handler: PATH_HANDLER_TYPE + ) -> PATH_HANDLER_TYPE: async def wrapper( full_file_path: str, path: str, index: int, total: int, failed: List[str] @@ -933,13 +935,13 @@ async def wrapper( def __do_path_cut_and( self, file_paths: List[str], - path_handler: Callable[[str, str, int, int, List[str], asyncio.Semaphore], Awaitable[bool]], + path_handler: PATH_HANDLER_TYPE, root="/" ) -> List[str]: slash_root = root if not root.endswith("/"): slash_root = slash_root + "/" - failed_paths = [] + failed_paths: List[str] = [] index = 1 file_paths_count = len(file_paths) tasks = [] diff --git a/charon/types.py b/charon/types.py new file mode 100644 index 00000000..db19c2ff --- /dev/null +++ b/charon/types.py @@ -0,0 +1,20 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from typing import Tuple + +# TARGET_TYPE indicate a target, which is tuple as +# (target_name, bucket, prefix, registry, domain) +TARGET_TYPE = Tuple[str, str, str, str, str] diff --git a/charon/utils/archive.py b/charon/utils/archive.py index eca56ebe..4a1f256c 100644 --- a/charon/utils/archive.py +++ b/charon/utils/archive.py @@ -133,6 +133,7 @@ def __parse_npm_package_version_paths(path: str) -> Tuple[dict, list]: return data, package_version_paths except JSONDecodeError: logger.error('Error: Failed to parse json!') + return {}, [] class NpmArchiveType(Enum): diff --git a/charon/utils/logs.py b/charon/utils/logs.py index 7725af58..bcce27e7 100644 --- a/charon/utils/logs.py +++ b/charon/utils/logs.py @@ -80,7 +80,7 @@ def set_logging( hdlr.setFormatter(formatter) -def set_log_file_handler(product: str, version: str, logger: logging): +def set_log_file_handler(product: str, version: str, logger: logging.Logger): prd = product.replace(" ", "_") ver = version.replace(" ", "_") log_loc = os.getenv("ERROR_LOG_LOCATION") diff --git a/tests/test_cf_maven_ops.py b/tests/test_cf_maven_ops.py index 46bb5780..b8cb03c1 100644 --- a/tests/test_cf_maven_ops.py +++ b/tests/test_cf_maven_ops.py @@ -32,7 +32,7 @@ def test_cf_after_upload(self): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - buckets=[('', TEST_BUCKET, 'ga', '', 'maven.repository.redhat.com')], + targets=[('', TEST_BUCKET, 'ga', '', 'maven.repository.redhat.com')], dir_=self.tempdir, do_index=True, cf_enable=True @@ -53,7 +53,7 @@ def test_cf_after_del(self): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, - buckets=[('', TEST_BUCKET, 'ga', '', 'maven.repository.redhat.com')], + targets=[('', TEST_BUCKET, 'ga', '', 'maven.repository.redhat.com')], dir_=self.tempdir, do_index=True ) @@ -61,7 +61,7 @@ def test_cf_after_del(self): product_456 = "commons-client-4.5.6" handle_maven_del( test_zip, product_456, - buckets=[('', TEST_BUCKET, 'ga', '', 'maven.repository.redhat.com')], + targets=[('', TEST_BUCKET, 'ga', '', 'maven.repository.redhat.com')], dir_=self.tempdir, do_index=True, cf_enable=True ) diff --git a/tests/test_cf_npm_ops.py b/tests/test_cf_npm_ops.py index 8b1c11b9..5132484a 100644 --- a/tests/test_cf_npm_ops.py +++ b/tests/test_cf_npm_ops.py @@ -33,7 +33,7 @@ def test_cf_after_upload(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, "/", DEFAULT_REGISTRY, "npm.registry.redhat.com")], + targets=[('', TEST_BUCKET, "/", DEFAULT_REGISTRY, "npm.registry.redhat.com")], dir_=self.tempdir, do_index=True, cf_enable=True ) @@ -53,13 +53,13 @@ def test_cf_after_del(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, '/', DEFAULT_REGISTRY, 'npm.registry.redhat.com')], + targets=[('', TEST_BUCKET, '/', DEFAULT_REGISTRY, 'npm.registry.redhat.com')], dir_=self.tempdir, do_index=True ) handle_npm_del( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, '/', DEFAULT_REGISTRY, 'npm.registry.redhat.com')], + targets=[('', TEST_BUCKET, '/', DEFAULT_REGISTRY, 'npm.registry.redhat.com')], dir_=self.tempdir, do_index=True, cf_enable=True ) diff --git a/tests/test_cf_reindex.py b/tests/test_cf_reindex.py index 0e986af6..5b5a48fe 100644 --- a/tests/test_cf_reindex.py +++ b/tests/test_cf_reindex.py @@ -37,7 +37,7 @@ def test_cf_maven_after_reindex(self): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, - buckets=[('', TEST_BUCKET, 'ga', '', 'maven.repository.redhat.com')], + targets=[('', TEST_BUCKET, 'ga', '', 'maven.repository.redhat.com')], dir_=self.tempdir ) @@ -62,7 +62,7 @@ def test_cf_npm_after_reindex(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, '/', DEFAULT_REGISTRY, 'npm.registry.redhat.com')], + targets=[('', TEST_BUCKET, '/', DEFAULT_REGISTRY, 'npm.registry.redhat.com')], dir_=self.tempdir, do_index=True ) diff --git a/tests/test_manifest_del.py b/tests/test_manifest_del.py index b5d42255..7a81be3c 100644 --- a/tests/test_manifest_del.py +++ b/tests/test_manifest_del.py @@ -43,7 +43,7 @@ def test_maven_manifest_delete(self): product = "commons-client-4.5.6" handle_maven_del( test_zip, product, - buckets=[(TEST_TARGET, TEST_BUCKET, '', '')], + targets=[(TEST_TARGET, TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False, manifest_bucket_name=TEST_MANIFEST_BUCKET @@ -64,7 +64,7 @@ def test_npm_manifest_delete(self): product = "code-frame-7.14.5" handle_npm_del( test_tgz, product, - buckets=[(TEST_TARGET, TEST_BUCKET, '', '')], + targets=[(TEST_TARGET, TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False, manifest_bucket_name=TEST_MANIFEST_BUCKET @@ -78,7 +78,7 @@ def __prepare_maven_content(self): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - buckets=[(TEST_TARGET, TEST_BUCKET, '', '')], + targets=[(TEST_TARGET, TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False, manifest_bucket_name=TEST_MANIFEST_BUCKET @@ -89,7 +89,7 @@ def __prepare_npm_content(self): product = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product, - buckets=[(TEST_TARGET, TEST_BUCKET, '', DEFAULT_REGISTRY)], + targets=[(TEST_TARGET, TEST_BUCKET, '', DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False, manifest_bucket_name=TEST_MANIFEST_BUCKET diff --git a/tests/test_manifest_upload.py b/tests/test_manifest_upload.py index 8a76de8d..c7e801b2 100644 --- a/tests/test_manifest_upload.py +++ b/tests/test_manifest_upload.py @@ -37,7 +37,7 @@ def test_maven_manifest_upload(self): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - buckets=[(TEST_TARGET, TEST_BUCKET, '', '')], + targets=[(TEST_TARGET, TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False, manifest_bucket_name=TEST_MANIFEST_BUCKET @@ -67,7 +67,7 @@ def test_npm_manifest_upload(self): product = "code-frame-7.14.5" handle_npm_uploading( test_zip, product, - buckets=[(TEST_TARGET, TEST_BUCKET, '', DEFAULT_REGISTRY)], + targets=[(TEST_TARGET, TEST_BUCKET, '', DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False, manifest_bucket_name=TEST_MANIFEST_BUCKET diff --git a/tests/test_maven_del.py b/tests/test_maven_del.py index 9ce85eaa..5b565adc 100644 --- a/tests/test_maven_del.py +++ b/tests/test_maven_del.py @@ -55,7 +55,7 @@ def test_ignore_del(self): handle_maven_del( test_zip, product_456, ignore_patterns=[".*.sha1"], - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False ) @@ -104,7 +104,7 @@ def __test_prefix_deletion(self, prefix: str): product_456 = "commons-client-4.5.6" handle_maven_del( test_zip, product_456, - buckets=[('', TEST_BUCKET, prefix, '')], + targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir, do_index=False ) @@ -178,7 +178,7 @@ def __test_prefix_deletion(self, prefix: str): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") handle_maven_del( test_zip, product_459, - buckets=[('', TEST_BUCKET, prefix, '')], + targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir, do_index=False ) @@ -191,7 +191,7 @@ def __prepare_content(self, prefix=None): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, - buckets=[('', TEST_BUCKET, prefix, '')], + targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir, do_index=False ) @@ -200,7 +200,7 @@ def __prepare_content(self, prefix=None): product_459 = "commons-client-4.5.9" handle_maven_uploading( test_zip, product_459, - buckets=[('', TEST_BUCKET, prefix, '')], + targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir, do_index=False ) diff --git a/tests/test_maven_del_multi_tgts.py b/tests/test_maven_del_multi_tgts.py index c3c93713..26fa11cc 100644 --- a/tests/test_maven_del_multi_tgts.py +++ b/tests/test_maven_del_multi_tgts.py @@ -65,7 +65,7 @@ def test_ignore_del(self): handle_maven_del( test_zip, product_456, ignore_patterns=[".*.sha1"], - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False ) @@ -115,7 +115,7 @@ def __test_prefix_deletion(self, prefix: str): product_456 = "commons-client-4.5.6" handle_maven_del( test_zip, product_456, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) @@ -243,7 +243,7 @@ def __test_prefix_deletion(self, prefix: str): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") handle_maven_del( test_zip, product_459, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) @@ -260,7 +260,7 @@ def __prepare_content(self, prefix=None): targets_ = [('', TEST_BUCKET, prefix, ''), ('', TEST_BUCKET_2, prefix, '')] handle_maven_uploading( test_zip, product_456, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) @@ -269,7 +269,7 @@ def __prepare_content(self, prefix=None): product_459 = "commons-client-4.5.9" handle_maven_uploading( test_zip, product_459, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) diff --git a/tests/test_maven_index.py b/tests/test_maven_index.py index 4952c5d7..a5cd1ed2 100644 --- a/tests/test_maven_index.py +++ b/tests/test_maven_index.py @@ -38,7 +38,7 @@ def test_uploading_index(self): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -80,7 +80,7 @@ def test_overlap_upload_index(self): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -88,7 +88,7 @@ def test_overlap_upload_index(self): product_459 = "commons-client-4.5.9" handle_maven_uploading( test_zip, product_459, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -131,7 +131,7 @@ def test_re_index(self): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -222,7 +222,7 @@ def __test_upload_index_with_prefix(self, prefix: str): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - buckets=[('', TEST_BUCKET, prefix, '')], + targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir ) @@ -274,7 +274,7 @@ def test_deletion_index(self): product_456 = "commons-client-4.5.6" handle_maven_del( test_zip, product_456, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -322,7 +322,7 @@ def test_deletion_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") handle_maven_del( test_zip, product_459, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -345,7 +345,7 @@ def __test_deletion_index_with_prefix(self, prefix: str): product_456 = "commons-client-4.5.6" handle_maven_del( test_zip, product_456, - buckets=[('', TEST_BUCKET, prefix, '')], + targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir ) @@ -392,7 +392,7 @@ def __test_deletion_index_with_prefix(self, prefix: str): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") handle_maven_del( test_zip, product_459, - buckets=[('', TEST_BUCKET, prefix, '')], + targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir ) @@ -404,7 +404,7 @@ def __prepare_content(self, prefix=None): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, - buckets=[('', TEST_BUCKET, prefix, '')], + targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir ) @@ -412,6 +412,6 @@ def __prepare_content(self, prefix=None): product_459 = "commons-client-4.5.9" handle_maven_uploading( test_zip, product_459, - buckets=[('', TEST_BUCKET, prefix, '')], + targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir ) diff --git a/tests/test_maven_index_multi_tgts.py b/tests/test_maven_index_multi_tgts.py index ddd7bb12..cc9d0718 100644 --- a/tests/test_maven_index_multi_tgts.py +++ b/tests/test_maven_index_multi_tgts.py @@ -47,7 +47,7 @@ def test_uploading_index(self): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - buckets=targets_, + targets=targets_, dir_=self.tempdir ) @@ -107,7 +107,7 @@ def test_overlap_upload_index(self): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, - buckets=targets_, + targets=targets_, dir_=self.tempdir ) @@ -115,7 +115,7 @@ def test_overlap_upload_index(self): product_459 = "commons-client-4.5.9" handle_maven_uploading( test_zip, product_459, - buckets=targets_, + targets=targets_, dir_=self.tempdir ) @@ -195,7 +195,7 @@ def __test_upload_index_with_prefix(self, prefix: str): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - buckets=targets_, + targets=targets_, dir_=self.tempdir ) @@ -262,7 +262,7 @@ def test_deletion_index(self): product_456 = "commons-client-4.5.6" handle_maven_del( test_zip, product_456, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -310,7 +310,7 @@ def test_deletion_index(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") handle_maven_del( test_zip, product_459, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -333,7 +333,7 @@ def __test_deletion_index_with_prefix(self, prefix: str): product_456 = "commons-client-4.5.6" handle_maven_del( test_zip, product_456, - buckets=targets_, + targets=targets_, dir_=self.tempdir ) @@ -402,7 +402,7 @@ def __test_deletion_index_with_prefix(self, prefix: str): test_zip = os.path.join(INPUTS, "commons-client-4.5.9.zip") handle_maven_del( test_zip, product_459, - buckets=targets_, + targets=targets_, dir_=self.tempdir ) @@ -418,7 +418,7 @@ def __prepare_content(self, prefix=None): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, - buckets=targets_, + targets=targets_, dir_=self.tempdir ) @@ -426,6 +426,6 @@ def __prepare_content(self, prefix=None): product_459 = "commons-client-4.5.9" handle_maven_uploading( test_zip, product_459, - buckets=targets_, + targets=targets_, dir_=self.tempdir ) diff --git a/tests/test_maven_sign.py b/tests/test_maven_sign.py index 52df5690..f60ee54d 100644 --- a/tests/test_maven_sign.py +++ b/tests/test_maven_sign.py @@ -33,7 +33,7 @@ def test_uploading_sign(self): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, gen_sign=True, key="random" @@ -64,7 +64,7 @@ def test_overlap_upload_index(self): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, gen_sign=True, key="random" @@ -74,7 +74,7 @@ def test_overlap_upload_index(self): product_459 = "commons-client-4.5.9" handle_maven_uploading( test_zip, product_459, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, gen_sign=True, key="random" diff --git a/tests/test_maven_upload.py b/tests/test_maven_upload.py index c47d1695..629a9e3f 100644 --- a/tests/test_maven_upload.py +++ b/tests/test_maven_upload.py @@ -48,7 +48,7 @@ def test_overlap_upload(self): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False ) @@ -56,7 +56,7 @@ def test_overlap_upload(self): product_459 = "commons-client-4.5.9" handle_maven_uploading( test_zip, product_459, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False ) @@ -115,7 +115,7 @@ def test_ignore_upload(self): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, [".*.sha1"], - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, do_index=False ) @@ -144,7 +144,7 @@ def __test_prefix_upload(self, prefix: str): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - buckets=[('', TEST_BUCKET, prefix, '')], + targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir, do_index=False ) diff --git a/tests/test_maven_upload_multi_tgts.py b/tests/test_maven_upload_multi_tgts.py index 921e8a9d..35aa49d4 100644 --- a/tests/test_maven_upload_multi_tgts.py +++ b/tests/test_maven_upload_multi_tgts.py @@ -69,7 +69,7 @@ def test_overlap_upload(self): ] handle_maven_uploading( test_zip, product_456, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) @@ -77,7 +77,7 @@ def test_overlap_upload(self): product_459 = "commons-client-4.5.9" handle_maven_uploading( test_zip, product_459, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) @@ -187,7 +187,7 @@ def test_ignore_upload(self): ] handle_maven_uploading( test_zip, product_456, [".*.sha1"], - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) @@ -222,7 +222,7 @@ def __test_prefix_upload(self, targets: List[Tuple[str, str, str, str]]): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - buckets=targets, + targets=targets, dir_=self.tempdir, do_index=False ) diff --git a/tests/test_npm_del.py b/tests/test_npm_del.py index 5f734b26..108d2232 100644 --- a/tests/test_npm_del.py +++ b/tests/test_npm_del.py @@ -44,7 +44,7 @@ def __test_prefix(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_del( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, prefix, '')], + targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir, do_index=False ) @@ -88,7 +88,7 @@ def __test_prefix(self, prefix: str = None): test_tgz = os.path.join(INPUTS, "code-frame-7.15.8.tgz") handle_npm_del( test_tgz, product_7_15_8, - buckets=[('', TEST_BUCKET, prefix, '')], + targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir, do_index=False ) objs = list(test_bucket.objects.all()) @@ -99,7 +99,7 @@ def __prepare_content(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, prefix, DEFAULT_REGISTRY)], + targets=[('', TEST_BUCKET, prefix, DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False ) @@ -107,6 +107,6 @@ def __prepare_content(self, prefix: str = None): product_7_15_8 = "code-frame-7.15.8" handle_npm_uploading( test_tgz, product_7_15_8, - buckets=[('', TEST_BUCKET, prefix, DEFAULT_REGISTRY)], + targets=[('', TEST_BUCKET, prefix, DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False ) diff --git a/tests/test_npm_del_multi_tgts.py b/tests/test_npm_del_multi_tgts.py index a6401db6..1ec5938e 100644 --- a/tests/test_npm_del_multi_tgts.py +++ b/tests/test_npm_del_multi_tgts.py @@ -54,7 +54,7 @@ def __test_prefix(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_del( test_tgz, product_7_14_5, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) @@ -124,7 +124,7 @@ def __test_prefix(self, prefix: str = None): test_tgz = os.path.join(INPUTS, "code-frame-7.15.8.tgz") handle_npm_del( test_tgz, product_7_15_8, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) for target in targets_: @@ -140,7 +140,7 @@ def __prepare_content(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) @@ -148,6 +148,6 @@ def __prepare_content(self, prefix: str = None): product_7_15_8 = "code-frame-7.15.8" handle_npm_uploading( test_tgz, product_7_15_8, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) diff --git a/tests/test_npm_dist_gen.py b/tests/test_npm_dist_gen.py index 7fbf58c0..93189385 100644 --- a/tests/test_npm_dist_gen.py +++ b/tests/test_npm_dist_gen.py @@ -39,7 +39,7 @@ def test_dist_gen_in_single_target(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) test_bucket = self.mock_s3.Bucket(TEST_BUCKET) @@ -82,7 +82,7 @@ def test_dist_gen_in_multi_targets(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) test_bucket_1 = self.mock_s3.Bucket(TEST_BUCKET) @@ -117,7 +117,7 @@ def test_overlapping_registry_dist_gen(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) test_bucket = self.mock_s3.Bucket(TEST_BUCKET) @@ -138,7 +138,7 @@ def test_overlapping_registry_dist_gen(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=targets_overlapping_, + targets=targets_overlapping_, dir_=self.tempdir, do_index=False ) diff --git a/tests/test_npm_index.py b/tests/test_npm_index.py index b435f765..b32e634f 100644 --- a/tests/test_npm_index.py +++ b/tests/test_npm_index.py @@ -49,7 +49,7 @@ def __test_upload_prefix(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, prefix, DEFAULT_REGISTRY)], + targets=[('', TEST_BUCKET, prefix, DEFAULT_REGISTRY)], dir_=self.tempdir, ) @@ -129,7 +129,7 @@ def __test_deletion_prefix(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_del( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, prefix, '')], + targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir ) @@ -160,7 +160,7 @@ def __test_deletion_prefix(self, prefix: str = None): test_tgz = os.path.join(INPUTS, "code-frame-7.15.8.tgz") handle_npm_del( test_tgz, product_7_15_8, - buckets=[('', TEST_BUCKET, prefix, '')], + targets=[('', TEST_BUCKET, prefix, '')], dir_=self.tempdir ) @@ -172,7 +172,7 @@ def __prepare_content(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, prefix, DEFAULT_REGISTRY)], + targets=[('', TEST_BUCKET, prefix, DEFAULT_REGISTRY)], dir_=self.tempdir ) @@ -180,7 +180,7 @@ def __prepare_content(self, prefix: str = None): product_7_15_8 = "code-frame-7.15.8" handle_npm_uploading( test_tgz, product_7_15_8, - buckets=[('', TEST_BUCKET, prefix, DEFAULT_REGISTRY)], + targets=[('', TEST_BUCKET, prefix, DEFAULT_REGISTRY)], dir_=self.tempdir ) @@ -191,7 +191,7 @@ def test_re_index(self): handle_npm_uploading( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, SHORT_TEST_PREFIX, DEFAULT_REGISTRY)], + targets=[('', TEST_BUCKET, SHORT_TEST_PREFIX, DEFAULT_REGISTRY)], dir_=self.tempdir, ) diff --git a/tests/test_npm_index_multi_tgts.py b/tests/test_npm_index_multi_tgts.py index acb882a4..794344fe 100644 --- a/tests/test_npm_index_multi_tgts.py +++ b/tests/test_npm_index_multi_tgts.py @@ -61,7 +61,7 @@ def __test_upload_prefix(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=targets_, + targets=targets_, dir_=self.tempdir, ) @@ -169,7 +169,7 @@ def __test_deletion_prefix(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_del( test_tgz, product_7_14_5, - buckets=targets_, + targets=targets_, dir_=self.tempdir ) @@ -211,7 +211,7 @@ def __test_deletion_prefix(self, prefix: str = None): test_tgz = os.path.join(INPUTS, "code-frame-7.15.8.tgz") handle_npm_del( test_tgz, product_7_15_8, - buckets=targets_, + targets=targets_, dir_=self.tempdir ) @@ -228,7 +228,7 @@ def __prepare_content(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=targets_, + targets=targets_, dir_=self.tempdir ) @@ -236,6 +236,6 @@ def __prepare_content(self, prefix: str = None): product_7_15_8 = "code-frame-7.15.8" handle_npm_uploading( test_tgz, product_7_15_8, - buckets=targets_, + targets=targets_, dir_=self.tempdir ) diff --git a/tests/test_npm_meta.py b/tests/test_npm_meta.py index 6d112efd..75ce22ca 100644 --- a/tests/test_npm_meta.py +++ b/tests/test_npm_meta.py @@ -67,7 +67,7 @@ def test_handle_npm_uploading_for_old_version(self): tarball_test_path = os.path.join(INPUTS, 'kogito-tooling-workspace-0.9.0-3.tgz') handle_npm_uploading( tarball_test_path, "kogito-tooling-workspace-0.9.0-3", - buckets=[('', MY_BUCKET, '', DEFAULT_REGISTRY)], + targets=[('', MY_BUCKET, '', DEFAULT_REGISTRY)], dir_=self.tempdir ) (files, _) = self.s3_client.get_files( @@ -117,7 +117,7 @@ def test_handle_npm_uploading_for_new_version(self): tarball_test_path = os.path.join(INPUTS, 'kogito-tooling-workspace-0.9.0-3.tgz') handle_npm_uploading( tarball_test_path, "kogito-tooling-workspace-0.9.0-3", - buckets=[('', MY_BUCKET, '', DEFAULT_REGISTRY)], + targets=[('', MY_BUCKET, '', DEFAULT_REGISTRY)], dir_=self.tempdir ) (files, _) = self.s3_client.get_files( @@ -168,7 +168,7 @@ def test_handle_npm_meta_wrong_dist_tags(self): tarball_test_path = os.path.join(INPUTS, 'kogito-tooling-workspace-0.9.0-3.tgz') handle_npm_uploading( tarball_test_path, "kogito-tooling-workspace-0.9.0-3", - buckets=[('', MY_BUCKET, '', DEFAULT_REGISTRY)], + targets=[('', MY_BUCKET, '', DEFAULT_REGISTRY)], dir_=self.tempdir ) (files, _) = self.s3_client.get_files( diff --git a/tests/test_npm_upload.py b/tests/test_npm_upload.py index 53767301..6f015ded 100644 --- a/tests/test_npm_upload.py +++ b/tests/test_npm_upload.py @@ -49,14 +49,14 @@ def test_double_uploads(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, '', DEFAULT_REGISTRY)], + targets=[('', TEST_BUCKET, '', DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False ) test_tgz = os.path.join(INPUTS, "code-frame-7.15.8.tgz") product_7_15_8 = "code-frame-7.15.8" handle_npm_uploading( test_tgz, product_7_15_8, - buckets=[('', TEST_BUCKET, '', DEFAULT_REGISTRY)], + targets=[('', TEST_BUCKET, '', DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False ) test_bucket = self.mock_s3.Bucket(TEST_BUCKET) @@ -94,7 +94,7 @@ def __test_prefix(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, prefix, DEFAULT_REGISTRY)], + targets=[('', TEST_BUCKET, prefix, DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False ) diff --git a/tests/test_npm_upload_diff_pkgs.py b/tests/test_npm_upload_diff_pkgs.py index 0dab2e66..f369d4ea 100644 --- a/tests/test_npm_upload_diff_pkgs.py +++ b/tests/test_npm_upload_diff_pkgs.py @@ -49,7 +49,7 @@ def test_npm_uploads_multi_pkgjson_with_root(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, '', DEFAULT_REGISTRY)], + targets=[('', TEST_BUCKET, '', DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False ) test_bucket = self.mock_s3.Bucket(TEST_BUCKET) @@ -81,7 +81,7 @@ def test_npm_uploads_multi_pkgjson_with_no_root(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, '', DEFAULT_REGISTRY)], + targets=[('', TEST_BUCKET, '', DEFAULT_REGISTRY)], dir_=self.tempdir, do_index=False ) test_bucket = self.mock_s3.Bucket(TEST_BUCKET) diff --git a/tests/test_npm_upload_multi_tgts.py b/tests/test_npm_upload_multi_tgts.py index 242937a7..c1432307 100644 --- a/tests/test_npm_upload_multi_tgts.py +++ b/tests/test_npm_upload_multi_tgts.py @@ -60,14 +60,14 @@ def test_double_uploads(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) test_tgz = os.path.join(INPUTS, "code-frame-7.15.8.tgz") product_7_15_8 = "code-frame-7.15.8" handle_npm_uploading( test_tgz, product_7_15_8, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) @@ -132,7 +132,7 @@ def __test_prefix(self, prefix: str = None): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=targets_, + targets=targets_, dir_=self.tempdir, do_index=False ) diff --git a/tests/test_pkgs_dryrun.py b/tests/test_pkgs_dryrun.py index 3b82d1b4..46061734 100644 --- a/tests/test_pkgs_dryrun.py +++ b/tests/test_pkgs_dryrun.py @@ -31,7 +31,7 @@ def test_maven_upload_dry_run(self): product = "commons-client-4.5.6" handle_maven_uploading( test_zip, product, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, dry_run=True ) @@ -47,7 +47,7 @@ def test_maven_delete_dry_run(self): product_456 = "commons-client-4.5.6" handle_maven_del( test_zip, product_456, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, dry_run=True ) @@ -61,7 +61,7 @@ def test_npm_upload_dry_run(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, '', DEFAULT_REGISTRY)], + targets=[('', TEST_BUCKET, '', DEFAULT_REGISTRY)], dir_=self.tempdir, dry_run=True ) @@ -77,7 +77,7 @@ def test_npm_deletion_dry_run(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_del( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir, dry_run=True ) @@ -91,7 +91,7 @@ def __prepare_maven_content(self): product_456 = "commons-client-4.5.6" handle_maven_uploading( test_zip, product_456, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -99,7 +99,7 @@ def __prepare_maven_content(self): product_459 = "commons-client-4.5.9" handle_maven_uploading( test_zip, product_459, - buckets=[('', TEST_BUCKET, '', '')], + targets=[('', TEST_BUCKET, '', '')], dir_=self.tempdir ) @@ -108,7 +108,7 @@ def __prepare_npm_content(self): product_7_14_5 = "code-frame-7.14.5" handle_npm_uploading( test_tgz, product_7_14_5, - buckets=[('', TEST_BUCKET, '', DEFAULT_REGISTRY)], + targets=[('', TEST_BUCKET, '', DEFAULT_REGISTRY)], dir_=self.tempdir ) @@ -116,6 +116,6 @@ def __prepare_npm_content(self): product_7_15_8 = "code-frame-7.15.8" handle_npm_uploading( test_tgz, product_7_15_8, - buckets=[('', TEST_BUCKET, '', DEFAULT_REGISTRY)], + targets=[('', TEST_BUCKET, '', DEFAULT_REGISTRY)], dir_=self.tempdir )