diff --git a/iib/common/pydantic_models.py b/iib/common/pydantic_models.py new file mode 100644 index 000000000..f88deb4d6 --- /dev/null +++ b/iib/common/pydantic_models.py @@ -0,0 +1,414 @@ +from typing import Any, Dict, List, Optional, Union +from typing_extensions import Annotated + +from pydantic import ( + AfterValidator, + BaseModel, + BeforeValidator, + model_validator, + SecretStr, +) + +from iib.exceptions import ValidationError +from iib.common.pydantic_utils import ( + DISTRIBUTION_SCOPE_LITERAL, + GRAPH_MODE_LITERAL, + binary_image_check, + distribution_scope_lower, + get_unique_bundles, + get_unique_deprecation_list_items, + image_format_check, + images_format_check, + length_validator, + from_index_add_arches, + validate_graph_mode_index_image, + validate_overwrite_params, +) + +UnionPydanticRequestType = Union[ + 'AddPydanticModel', + 'CreateEmptyIndexPydanticModel', + 'FbcOperationsPydanticModel', + 'MergeIndexImagePydanticModel', + 'RecursiveRelatedBundlesPydanticModel', + 'RegenerateBundlePydanticModel', + 'RmPydanticModel', +] + + +class PydanticRequestBaseModel(BaseModel): + """Base model representing IIB request.""" + + @classmethod + def _get_all_keys_to_check_in_db(cls): + """Class that returns request specific keys to check.""" + raise NotImplementedError("Not implemented") + + def get_keys_to_check_in_db(self): + """ + Filter keys, which need to be checked in db. + + Return only a keys that are set to values. + """ + return [k for k in self._get_all_keys_to_check_in_db() if getattr(self, k, None)] + + +class AddPydanticModel(PydanticRequestBaseModel): + """Datastructure of the request to /builds/add API point.""" + + add_arches: Optional[List[str]] = None + binary_image: Annotated[ + Optional[str], + AfterValidator(length_validator), + AfterValidator(binary_image_check), + ] = None + build_tags: Optional[List[str]] = None + bundles: Annotated[ + List[str], + AfterValidator(length_validator), + AfterValidator(get_unique_bundles), + AfterValidator(images_format_check), + ] + cnr_token: Optional[SecretStr] = None # deprecated + # TODO remove this comment -> old request without this parameter will not have False but None + check_related_images: Optional[bool] = None + deprecation_list: Annotated[ + Optional[List[str]], + AfterValidator(get_unique_deprecation_list_items), + AfterValidator(images_format_check), + ] = [] # deprecated + distribution_scope: Annotated[ + Optional[DISTRIBUTION_SCOPE_LITERAL], + BeforeValidator(distribution_scope_lower), + ] = None + force_backport: Optional[bool] = False # deprecated + from_index: Annotated[Optional[str], AfterValidator(image_format_check)] = None + graph_update_mode: Optional[GRAPH_MODE_LITERAL] = None + organization: Optional[str] = None # deprecated + overwrite_from_index: Optional[bool] = False + overwrite_from_index_token: Optional[SecretStr] = None + + @model_validator(mode='after') + def verify_from_index_add_arches_combination(self) -> 'AddPydanticModel': + """Check the 'overwrite_from_index' parameter with 'overwrite_from_index_token' param.""" + from_index_add_arches(self.from_index, self.add_arches) + return self + + # TODO remove this comment -> Validator from RequestIndexImageMixin class + @model_validator(mode='after') + def verify_overwrite_from_index_token(self) -> 'AddPydanticModel': + """Check the 'overwrite_from_index' parameter with 'overwrite_from_index_token' param.""" + validate_overwrite_params(self.overwrite_from_index, self.overwrite_from_index_token) + return self + + # TODO remove this comment -> Validator from RequestAdd class + @model_validator(mode='after') + def verify_graph_update_mode_with_index_image(self) -> 'AddPydanticModel': + """Validate graph mode and check if index image is allowed to use different graph mode.""" + validate_graph_mode_index_image(self.graph_update_mode, self.from_index) + return self + + # TODO remove this comment -> Validator from RequestAdd class + @model_validator(mode='after') + def from_index_needed_if_no_bundles(self) -> 'AddPydanticModel': + """ + Check if no bundles and `from_index is specified. + + if no bundles and no from index then an empty index will be created which is a no-op + """ + if not (self.bundles or self.from_index): + raise ValidationError('"from_index" must be specified if no bundles are specified') + return self + + # TODO remove this comment -> Validator from RequestADD class + @model_validator(mode='after') + def bundles_needed_with_check_related_images(self) -> 'AddPydanticModel': + """Verify that `check_related_images` is specified when bundles are specified.""" + if self.check_related_images and not self.bundles: + raise ValidationError( + '"check_related_images" must be specified only when bundles are specified' + ) + return self + + def get_json_for_request(self): + """Return json with the parameters we store in the db.""" + return self.model_dump( + exclude=[ + "add_arches", + "build_tags", + "cnr_token", + "force_backport", + "overwrite_from_index", + "overwrite_from_index_token", + ], + exclude_none=True, + ) + + def _get_all_keys_to_check_in_db(self): + return ["binary_image", "bundles", "deprecation_list", "from_index"] + + +class RmPydanticModel(PydanticRequestBaseModel): + """Datastructure of the request to /builds/rm API point.""" + + add_arches: Optional[List[str]] = None + binary_image: Annotated[ + Optional[str], + AfterValidator(binary_image_check), + ] = None + build_tags: Optional[List[str]] = None + distribution_scope: Annotated[ + Optional[DISTRIBUTION_SCOPE_LITERAL], + BeforeValidator(distribution_scope_lower), + ] = None + from_index: Annotated[Optional[str], AfterValidator(image_format_check)] = None + operators: Annotated[List[str], AfterValidator(length_validator)] + overwrite_from_index: Optional[bool] = False + overwrite_from_index_token: Optional[SecretStr] = None + + @model_validator(mode='after') + def verify_from_index_add_arches_combination(self) -> 'AddPydanticModel': + """Check the 'overwrite_from_index' parameter with 'overwrite_from_index_token' param.""" + from_index_add_arches(self.from_index, self.add_arches) + return self + + @model_validator(mode='after') + def verify_overwrite_from_index_token(self) -> 'RmPydanticModel': + """Validate overwrite_from_index and overwrite_from_index_token param combination.""" + validate_overwrite_params( + self.overwrite_from_index, + self.overwrite_from_index_token, + ) + return self + + def get_json_for_request(self): + """Return json with the parameters we store in the db.""" + return self.model_dump( + exclude=[ + "add_arches", + "build_tags", + "overwrite_from_index", + "overwrite_from_index_token", + ], + exclude_none=True, + ) + + def _get_all_keys_to_check_in_db(self): + return ["binary_image", "from_index", "operators"] + + +class AddRmBatchPydanticModel(BaseModel): + """Datastructure of the request to /builds/add-rm-batch API point.""" + + annotations: Dict[str, Any] + build_requests: List[Union[AddPydanticModel, RmPydanticModel]] + + +class RegistryAuth(BaseModel): + """Datastructure representing private registry token.""" + + auth: SecretStr + + +class RegistryAuths(BaseModel): + """ + Datastructure used within recursive-related-bundles. + + Provide the dockerconfig.json for authentication to private registries. + Non-auth information in the dockerconfig.json is not allowed. + """ + + auths: Annotated[Dict[SecretStr, RegistryAuth], AfterValidator(length_validator)] + + +class RegenerateBundlePydanticModel(PydanticRequestBaseModel): + """Datastructure of the request to /builds/regenerate-bundle API point.""" + + # BUNDLE_IMAGE, from_bundle_image_resolved, build_tags? + bundle_replacements: Optional[Dict[str, str]] = None + from_bundle_image: Annotated[str, AfterValidator(image_format_check)] + organization: Optional[str] = None + registry_auths: Optional[RegistryAuths] = None # not in db + + def get_json_for_request(self): + """Return json with the parameters we store in the db.""" + return self.model_dump( + exclude=["registry_auths"], + exclude_none=True, + ) + + def _get_all_keys_to_check_in_db(self): + return ["from_bundle_image"] + + +class RegenerateBundleBatchPydanticModel(BaseModel): + """Datastructure of the request to /builds/regenerate-bundle-batch API point.""" + + build_requests: List[RegenerateBundlePydanticModel] + annotations: Dict[str, Any] + + +class MergeIndexImagePydanticModel(PydanticRequestBaseModel): + """Datastructure of the request to /builds/merge-index-image API point.""" + + binary_image: Annotated[ + Optional[str], + AfterValidator(image_format_check), + AfterValidator(binary_image_check), + ] = None + build_tags: Optional[List[str]] = None + deprecation_list: Annotated[ + Optional[List[str]], + AfterValidator(get_unique_deprecation_list_items), + AfterValidator(images_format_check), + ] = [] + distribution_scope: Annotated[ + Optional[DISTRIBUTION_SCOPE_LITERAL], + BeforeValidator(distribution_scope_lower), + ] = None + graph_update_mode: Optional[GRAPH_MODE_LITERAL] = None + ignore_bundle_ocp_version: Optional[bool] = None + overwrite_target_index: Optional[bool] = False + overwrite_target_index_token: Optional[SecretStr] = None + source_from_index: Annotated[str, AfterValidator(image_format_check)] + target_index: Annotated[Optional[str], AfterValidator(image_format_check)] = None + batch: Optional[str] = None # TODO Not sure with presence + user: Optional[str] = None # TODO Not sure with presence + + @model_validator(mode='after') + def verify_graph_update_mode_with_target_index(self) -> 'MergeIndexImagePydanticModel': + """Validate graph_update_mode with target_index param combination.""" + validate_graph_mode_index_image(self.graph_update_mode, self.target_index) + return self + + @model_validator(mode='after') + def verify_overwrite_from_index_token(self) -> 'MergeIndexImagePydanticModel': + """Validate overwrite_target_index with overwrite_target_index_token param combination.""" + validate_overwrite_params( + self.overwrite_target_index, + self.overwrite_target_index_token, + disable_auth_check=True, + ) + return self + + def get_json_for_request(self): + """Return json with the parameters we store in the db.""" + return self.model_dump( + exclude=["build_tags", "overwrite_target_index", "overwrite_target_index_token"], + exclude_none=True, + ) + + def _get_all_keys_to_check_in_db(self): + return [ + "binary_image", + "deprecation_list", + "source_from_index", + "target_index", + ] + + +class CreateEmptyIndexPydanticModel(PydanticRequestBaseModel): + """Datastructure of the request to /builds/create-empty-index API point.""" + + binary_image: Annotated[ + Optional[str], + AfterValidator(image_format_check), + AfterValidator(binary_image_check), + ] = None + from_index: Annotated[ + str, + AfterValidator(image_format_check), + AfterValidator(length_validator), + ] + # TODO (remove comment) old request without this parameter will not have empty labels + labels: Optional[Dict[str, str]] = None + # TODO (remove comment) old request without this parameter will not have empty output_fbc + output_fbc: Optional[bool] = None + + def get_json_for_request(self): + """Return json with the parameters we store in the db.""" + return self.model_dump( + exclude_none=True, + ) + + def _get_all_keys_to_check_in_db(self): + return ["binary_image", "from_index"] + + +class RecursiveRelatedBundlesPydanticModel(PydanticRequestBaseModel): + """Datastructure of the request to /builds/recursive-related-bundles API point.""" + + organization: Optional[str] = None + parent_bundle_image: Annotated[ + str, + AfterValidator(image_format_check), + AfterValidator(length_validator), + ] + registry_auths: Optional[RegistryAuths] = None # not in db + + def get_json_for_request(self): + """Return json with the parameters we store in the db.""" + return self.model_dump( + exclude=["registry_auths"], + exclude_none=True, + ) + + def _get_all_keys_to_check_in_db(self): + return ["parent_bundle_image"] + + +class FbcOperationsPydanticModel(PydanticRequestBaseModel): + """Datastructure of the request to /builds/fbc-operations API point.""" + + add_arches: Optional[List[str]] = [] + binary_image: Annotated[ + Optional[str], + AfterValidator(image_format_check), + AfterValidator(binary_image_check), + ] = None + # TODO (remove comment) old request without this parameter will not have empty list but None + bundles: Annotated[ + Optional[List[str]], + AfterValidator(length_validator), + AfterValidator(get_unique_bundles), + AfterValidator(images_format_check), + ] = None + build_tags: Optional[List[str]] = None + distribution_scope: Annotated[ + Optional[DISTRIBUTION_SCOPE_LITERAL], + BeforeValidator(distribution_scope_lower), + ] = None + fbc_fragment: Annotated[ + str, + AfterValidator(image_format_check), + AfterValidator(length_validator), + ] + from_index: Annotated[ + str, + AfterValidator(image_format_check), + AfterValidator(length_validator), + ] + organization: Optional[str] = None + overwrite_from_index: Optional[bool] = False + overwrite_from_index_token: Optional[SecretStr] = None + + @model_validator(mode='after') + def verify_overwrite_from_index_token(self) -> 'FbcOperationsPydanticModel': + """Validate overwrite_from_index and overwrite_from_index_token param combination.""" + validate_overwrite_params(self.overwrite_from_index, self.overwrite_from_index_token) + return self + + def get_json_for_request(self): + """Return json with the parameters we store in the db.""" + return self.model_dump( + exclude=[ + "add_arches", + "build_tags", + "overwrite_from_index", + "overwrite_from_index_token", + ], + exclude_none=True, + ) + + def _get_all_keys_to_check_in_db(self): + return ["binary_image", "bundles", "fbc_fragment", "from_index"] diff --git a/iib/common/pydantic_utils.py b/iib/common/pydantic_utils.py new file mode 100644 index 000000000..61e76d9f0 --- /dev/null +++ b/iib/common/pydantic_utils.py @@ -0,0 +1,125 @@ +from typing import List, Optional, Any, Literal + +import copy +from werkzeug.exceptions import Forbidden +from flask import current_app +from flask_login import current_user + +from iib.exceptions import ValidationError + + +GRAPH_MODE_LITERAL = Literal['replaces', 'semver', 'semver-skippatch'] +DISTRIBUTION_SCOPE_LITERAL = Literal['prod', 'stage', 'dev'] + + +# TODO add regex in future to not allow following values ":s", "s:", ":"? +def image_format_check(image_name: str) -> str: + """Check format of the index image.""" + if '@' not in image_name and ':' not in image_name: + raise ValidationError(f'Image {image_name} should have a tag or a digest specified.') + return image_name + + +def images_format_check(image_list: List[str]) -> List[str]: + """Check multiple image names.""" + for image_name in image_list: + image_format_check(image_name) + return image_list + + +def get_unique_bundles(bundles: List[str]) -> List[str]: + """Check and possibly remove duplicates from a list of bundles.""" + if not bundles: + return bundles + + unique_bundles = list(set(bundles)) + if len(unique_bundles) != len(bundles): + duplicate_bundles = copy.copy(bundles) + for bundle in unique_bundles: + duplicate_bundles.remove(bundle) + + # flask.current_app.logger.info( + # f'Removed duplicate bundles from request: {duplicate_bundles}' + # ) + return unique_bundles + + +# RequestIndexImageMixin +def get_unique_deprecation_list_items(deprecation_list: List[str]) -> List[str]: + """Return a list of unique items.""" + return list(set(deprecation_list)) + + +def validate_graph_mode_index_image( + graph_update_mode: Optional[GRAPH_MODE_LITERAL], + index_image: Optional[str], +) -> Optional[str]: + """ + Validate graph mode and check if index image is allowed to use different graph mode. + + :param str graph_update_mode: one of the graph mode options + :param str index_image: pullspec of index image to which graph mode should be applied to + :raises: ValidationError when incorrect graph_update_mode is set + :raises: Forbidden when graph_mode can't be used for given index image + """ + if graph_update_mode: + allowed_from_indexes: List[str] = current_app.config['IIB_GRAPH_MODE_INDEX_ALLOW_LIST'] + if index_image not in allowed_from_indexes: + raise Forbidden( + '"graph_update_mode" can only be used on the' + f' following index image: {allowed_from_indexes}' + ) + return graph_update_mode + + +# RequestIndexImageMixin +def from_index_add_arches(from_index: Optional[str], add_arches: Optional[List[str]]) -> None: + """Check if both `from_index` and `add_arches` are not specified.""" + if not from_index and not add_arches: + raise ValidationError('One of "from_index" or "add_arches" must be specified') + + +# RequestIndexImageMixin +def binary_image_check(binary_image: str) -> str: + """Validate binary_image is correctly provided.""" + if not binary_image and not current_app.config['IIB_BINARY_IMAGE_CONFIG']: + raise ValidationError('The "binary_image" value must be a non-empty string') + return binary_image + + +# RequestIndexImageMixin +def validate_overwrite_params( + overwrite_index_image: Optional[bool], + overwrite_index_image_token: Optional[str], + disable_auth_check: Optional[bool] = False, +) -> None: + """Check if both `overwrite_index_image` and `overwrite_index_image_token` are specified.""" + if overwrite_index_image_token and not overwrite_index_image: + raise ValidationError( + 'The "overwrite_from_index" parameter is required when' + ' the "overwrite_from_index_token" parameter is used' + ) + + # Verify the user is authorized to use overwrite_from_index + # current_user.is_authenticated is only ever False when auth is disabled + # TODO Remove "1 or" + if 1 or disable_auth_check or current_user.is_authenticated: + if overwrite_index_image and not overwrite_index_image_token: + raise Forbidden( + 'You must set "overwrite_from_index_token" to use "overwrite_from_index"' + ) + + +# RequestIndexImageMixin +def distribution_scope_lower(distribution_scope: str) -> str: + """Transform distribution_scope parameter to lowercase.""" + return distribution_scope.lower() + + +def length_validator(model_property: Any) -> Any: + """Validate length of the given model property.""" + if model_property is not None and len(model_property) == 0: + raise ValidationError( + f"The {type(model_property)} {model_property} should have at least 1 item." + ) + return model_property diff --git a/iib/web/api_v1.py b/iib/web/api_v1.py index 24315a02c..a04272229 100644 --- a/iib/web/api_v1.py +++ b/iib/web/api_v1.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: GPL-3.0-or-later -import copy import logging import os from datetime import datetime @@ -12,7 +11,7 @@ from sqlalchemy.sql import text from sqlalchemy import or_ from werkzeug.exceptions import Forbidden, Gone, NotFound -from typing import Any, cast, Dict, List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Union from iib.common.tracing import instrument_tracing from iib.exceptions import IIBError, ValidationError @@ -25,6 +24,7 @@ Operator, Request, RequestAdd, + RequestCreateEmptyIndex, RequestFbcOperations, RequestMergeIndexImage, RequestRecursiveRelatedBundles, @@ -32,12 +32,22 @@ RequestRm, RequestState, RequestStateMapping, - get_request_query_options, RequestTypeMapping, - RequestCreateEmptyIndex, User, + get_request_query_options, ) from iib.web.s3_utils import get_object_from_s3_bucket +from iib.common.pydantic_models import ( + AddPydanticModel, + RmPydanticModel, + RegenerateBundlePydanticModel, + RegenerateBundleBatchPydanticModel, + AddRmBatchPydanticModel, + CreateEmptyIndexPydanticModel, + RecursiveRelatedBundlesPydanticModel, + FbcOperationsPydanticModel, + MergeIndexImagePydanticModel, +) from botocore.response import StreamingBody from iib.web.utils import pagination_metadata, str_to_bool from iib.workers.tasks.build import ( @@ -52,115 +62,10 @@ from iib.workers.tasks.build_merge_index_image import handle_merge_request from iib.workers.tasks.build_create_empty_index import handle_create_empty_index_request from iib.workers.tasks.general import failed_request_callback -from iib.web.iib_static_types import ( - AddRequestPayload, - AddRmBatchPayload, - CreateEmptyIndexPayload, - FbcOperationRequestPayload, - MergeIndexImagesPayload, - PayloadTypesUnion, - RecursiveRelatedBundlesRequestPayload, - RegenerateBundleBatchPayload, - RegenerateBundlePayload, - RmRequestPayload, -) api_v1 = flask.Blueprint('api_v1', __name__) -def _get_rm_args( - payload: RmRequestPayload, - request: Request, - overwrite_from_index: bool, -) -> List[Union[str, List[str], Dict[str, str], bool, None]]: - """ - Generate arguments for remove request. - - :param RmRequestPayload payload: Payload from the remove request - :param Request request: request saved in the database - :param bool overwrite_from_index: determines if the overwrite should be forced - :return: List with remove arguments - :rtype: list - """ - return [ - payload['operators'], - request.id, - payload['from_index'], - payload.get('binary_image'), - payload.get('add_arches'), - overwrite_from_index, - payload.get('overwrite_from_index_token'), - request.distribution_scope, - flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], - payload.get('build_tags', []), - ] - - -def _get_add_args( - payload: AddRequestPayload, - request: Request, - overwrite_from_index: bool, - celery_queue: Optional[str], -) -> List[Any]: - """ - Generate arguments for add request. - - :param AddRequestPayload payload: Payload from the add request - :param Request request: request saved in the database - :param bool overwrite_from_index: determines if the overwrite should be forced - :param str celery_queue: name of celery queue - :return: List with add arguments - :rtype: list - """ - return [ - payload.get('bundles', []), - request.id, - payload.get('binary_image'), - payload.get('from_index'), - payload.get('add_arches'), - payload.get('cnr_token'), - payload.get('organization'), - payload.get('force_backport'), - overwrite_from_index, - payload.get('overwrite_from_index_token'), - request.distribution_scope, - flask.current_app.config['IIB_GREENWAVE_CONFIG'].get(celery_queue), - flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], - payload.get('deprecation_list', []), - payload.get('build_tags', []), - payload.get('graph_update_mode'), - payload.get('check_related_images', False), - ] - - -def _get_safe_args( - args: List[Any], - payload: PayloadTypesUnion, -) -> List[Union[str, List[str], bool, Dict[str, str]]]: - """ - Generate arguments that are safe to print to stdout or log. - - :param list args: arguments for each api, that are not safe - :param PayloadTypesUnion payload: Payload from the IIB request - :return: List with safe to print arguments - :rtype: list - """ - safe_args = copy.copy(args) - - if payload.get('cnr_token'): - safe_args[safe_args.index(payload['cnr_token'])] = '*****' # type: ignore - if payload.get('overwrite_from_index_token'): - safe_args[safe_args.index(payload['overwrite_from_index_token'])] = '*****' # type: ignore - if payload.get('overwrite_target_index_token'): - safe_args[ - safe_args.index(payload['overwrite_target_index_token']) # type: ignore - ] = '*****' - if payload.get('registry_auths'): - safe_args[safe_args.index(payload['registry_auths'])] = '*****' # type: ignore - - return safe_args - - def get_artifact_file_from_s3_bucket( s3_key_prefix: str, s3_file_name: str, @@ -192,35 +97,6 @@ def get_artifact_file_from_s3_bucket( raise NotFound() -def _get_unique_bundles(bundles: List[str]) -> List[str]: - """ - Return list with unique bundles. - - :param list bundles: bundles given in payload from original request - :return: list of unique bundles preserving order (python 3.6+) - :rtype: list - """ - if not bundles: - return bundles - - # `dict` is preserving order of inserted keys since Python 3.6. - # Keys in dictionary are behaving as a set() therefore can not have same key twice. - # This will create dictionary where keys are taken from `bundles` using `dict.fromkeys()` - # After that we have dictionary with unique keys with same order as it is in `bundles`. - # Last step is to convert the keys from this dictionary to list using `list()` - unique_bundles = list(dict.fromkeys(bundles).keys()) - - if len(unique_bundles) != len(bundles): - duplicate_bundles = copy.copy(bundles) - for bundle in unique_bundles: - duplicate_bundles.remove(bundle) - - flask.current_app.logger.info( - f'Removed duplicate bundles from request: {duplicate_bundles}' - ) - return unique_bundles - - @api_v1.route('/builds/') @instrument_tracing(span_name="web.api_v1.get_build") def get_build(request_id: int) -> flask.Response: @@ -583,34 +459,38 @@ def add_bundles() -> Tuple[flask.Response, int]: :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ - payload: AddRequestPayload = cast(AddRequestPayload, flask.request.get_json()) - if not isinstance(payload, dict): - raise ValidationError('The input data must be a JSON object') - - # Only run `_get_unique_bundles` if it is a list. If it's not, `from_json` - # will raise an error to the user. - if payload.get('bundles') and isinstance(payload['bundles'], list): - payload['bundles'] = _get_unique_bundles(payload['bundles']) + try: + request_payload = AddPydanticModel.model_validate( + flask.request.get_json(), + strict=True, + ) + except ValidationError as e: + # If the JSON data doesn't match the Pydantic model, return a 400 Bad Request response + return flask.jsonify({'Error parsing data': str(e)}), 400 - request = RequestAdd.from_json(payload) + request = RequestAdd.from_json_replacement( + payload=request_payload, + ) db.session.add(request) db.session.commit() messaging.send_message_for_state_change(request, new_batch_msg=True) - overwrite_from_index = payload.get('overwrite_from_index', False) from_index_pull_spec = request.from_index.pull_specification if request.from_index else None celery_queue = _get_user_queue( - serial=overwrite_from_index, from_index_pull_spec=from_index_pull_spec + serial=request_payload.overwrite_from_index, from_index_pull_spec=from_index_pull_spec ) - args = _get_add_args(payload, request, overwrite_from_index, celery_queue) - safe_args = _get_safe_args(args, payload) + args = [ + request_payload, + request.id, + flask.current_app.config['IIB_GREENWAVE_CONFIG'].get(celery_queue), + flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], + ] error_callback = failed_request_callback.s(request.id) try: handle_add_request.apply_async( args=args, link_error=error_callback, - argsrepr=repr(safe_args), queue=celery_queue, headers={'traceparent': flask.request.headers.get('traceparent')}, ) @@ -816,29 +696,35 @@ def rm_operators() -> Tuple[flask.Response, int]: :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ - payload: RmRequestPayload = cast(RmRequestPayload, flask.request.get_json()) - if not isinstance(payload, dict): - raise ValidationError('The input data must be a JSON object') + try: + request_payload = RmPydanticModel.model_validate( + flask.request.get_json(), + strict=True, + ) + except ValidationError as e: + # If the JSON data doesn't match the Pydantic model, return a 400 Bad Request response + return flask.jsonify({'Error parsing data': str(e)}), 400 - request = RequestRm.from_json(payload) + request = RequestRm.from_json_replacement( + payload=request_payload, + ) db.session.add(request) db.session.commit() messaging.send_message_for_state_change(request, new_batch_msg=True) - overwrite_from_index = payload.get('overwrite_from_index', False) - - args = _get_rm_args(payload, request, overwrite_from_index) - safe_args = _get_safe_args(args, payload) - + args = [ + request_payload, + request.id, + flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], + ] error_callback = failed_request_callback.s(request.id) from_index_pull_spec = request.from_index.pull_specification if request.from_index else None try: handle_rm_request.apply_async( args=args, link_error=error_callback, - argsrepr=repr(safe_args), queue=_get_user_queue( - serial=overwrite_from_index, + serial=request_payload.overwrite_from_index, from_index_pull_spec=from_index_pull_spec, ), ) @@ -859,30 +745,33 @@ def regenerate_bundle() -> Tuple[flask.Response, int]: :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ - payload: RegenerateBundlePayload = cast(RegenerateBundlePayload, flask.request.get_json()) - if not isinstance(payload, dict): - raise ValidationError('The input data must be a JSON object') + try: + request_payload = RegenerateBundlePydanticModel.model_validate( + flask.request.get_json(), + strict=True, + ) + except ValidationError as e: + # If the JSON data doesn't match the Pydantic model, return a 400 Bad Request response + return flask.jsonify({'Error parsing data': str(e)}), 400 + + request = RequestRegenerateBundle.from_json_replacement( + payload=request_payload, + ) - request = RequestRegenerateBundle.from_json(payload) db.session.add(request) db.session.commit() messaging.send_message_for_state_change(request, new_batch_msg=True) args = [ - payload['from_bundle_image'], - payload.get('organization'), + request_payload, request.id, - payload.get('registry_auths'), - payload.get('bundle_replacements', dict()), ] - safe_args = _get_safe_args(args, payload) error_callback = failed_request_callback.s(request.id) try: handle_regenerate_bundle_request.apply_async( args=args, link_error=error_callback, - argsrepr=repr(safe_args), queue=_get_user_queue(), ) except kombu.exceptions.OperationalError: @@ -902,27 +791,26 @@ def regenerate_bundle_batch() -> Tuple[flask.Response, int]: :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ - payload: RegenerateBundleBatchPayload = cast( - RegenerateBundleBatchPayload, flask.request.get_json() - ) - Batch.validate_batch_request_params(payload) + try: + request_payload_batch = RegenerateBundleBatchPydanticModel.model_validate( + flask.request.get_json(), + strict=True, + ) + except ValidationError as e: + # If the JSON data doesn't match the Pydantic model, return a 400 Bad Request response + return flask.jsonify({'Error parsing data': str(e)}), 400 - batch = Batch(annotations=payload.get('annotations')) + batch = Batch(annotations=request_payload_batch.annotations) db.session.add(batch) requests = [] # Iterate through all the build requests and verify that the requests are valid before # committing them and scheduling the tasks - for build_request in payload['build_requests']: - try: - request = RequestRegenerateBundle.from_json(build_request, batch) - except ValidationError as e: - # Rollback the transaction if any of the build requests are invalid - db.session.rollback() - raise ValidationError( - f'{str(e).rstrip(".")}. This occurred on the build request in ' - f'index {payload["build_requests"].index(build_request)}.' - ) + for request_payload in request_payload_batch.build_requests: + request = RequestRegenerateBundle.from_json_replacement( + payload=request_payload, + batch=batch, + ) db.session.add(request) requests.append(request) @@ -933,22 +821,17 @@ def regenerate_bundle_batch() -> Tuple[flask.Response, int]: # This list will be used for the log message below and avoids the need of having to iterate # through the list of requests another time processed_request_ids = [] - build_and_requests = zip(payload['build_requests'], requests) + build_and_requests = zip(request_payload.build_requests, requests) try: - for build_request, request in build_and_requests: + for request_payload, request in build_and_requests: args = [ - build_request['from_bundle_image'], - build_request.get('organization'), + request_payload, request.id, - build_request.get('registry_auths'), - build_request.get('bundle_replacements', dict()), ] - safe_args = _get_safe_args(args, build_request) error_callback = failed_request_callback.s(request.id) handle_regenerate_bundle_request.apply_async( args=args, link_error=error_callback, - argsrepr=repr(safe_args), queue=_get_user_queue(), ) @@ -978,34 +861,33 @@ def add_rm_batch() -> Tuple[flask.Response, int]: :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ - payload: AddRmBatchPayload = cast(AddRmBatchPayload, flask.request.get_json()) - Batch.validate_batch_request_params(payload) + try: + request_payload_batch = AddRmBatchPydanticModel.model_validate( + flask.request.get_json(), + strict=True, + ) + except ValidationError as e: + # If the JSON data doesn't match the Pydantic model, return a 400 Bad Request response + return flask.jsonify({'Error parsing data': str(e)}), 400 - batch = Batch(annotations=payload.get('annotations')) + batch = Batch(annotations=request_payload_batch.annotations) db.session.add(batch) - requests: List[Union[RequestAdd, RequestRm]] = [] + requests: List[Union[AddPydanticModel, RmPydanticModel]] = [] # Iterate through all the build requests and verify that the requests are valid before # committing them and scheduling the tasks - for build_request in payload['build_requests']: - try: - if build_request.get('operators'): - # Check for the validity of a RM request - # cast Union[AddRequestPayload, RmRequestPayload] based on presence of 'operators' - request = RequestRm.from_json(cast(RmRequestPayload, build_request), batch) - elif build_request.get('bundles'): - # cast Union[AddRequestPayload, RmRequestPayload] based on presence of 'bundles' - build_request_uniq = cast(AddRequestPayload, copy.deepcopy(build_request)) - build_request_uniq['bundles'] = _get_unique_bundles(build_request_uniq['bundles']) - # Check for the validity of an Add request - request = RequestAdd.from_json(build_request_uniq, batch) - else: - raise ValidationError('Build request is not a valid Add/Rm request.') - except ValidationError as e: - raise ValidationError( - f'{str(e).rstrip(".")}. This occurred on the build request in ' - f'index {payload["build_requests"].index(build_request)}.' + for request_payload in request_payload_batch.build_requests: + if isinstance(request_payload, AddPydanticModel): + request = RequestAdd.from_json_replacement( + payload=request_payload, + batch=batch, + ) + else: + request = RequestRm.from_json_replacement( + payload=request_payload, + batch=batch, ) + db.session.add(request) requests.append(request) @@ -1016,46 +898,36 @@ def add_rm_batch() -> Tuple[flask.Response, int]: # This list will be used for the log message below and avoids the need of having to iterate # through the list of requests another time processed_request_ids = [] - for build_request, request in zip(payload['build_requests'], requests): + for request_payload, request in zip(request_payload_batch.build_requests, requests): request_jsons.append(request.to_json()) - overwrite_from_index = build_request.get('overwrite_from_index', False) from_index_pull_spec = request.from_index.pull_specification if request.from_index else None celery_queue = _get_user_queue( - serial=overwrite_from_index, from_index_pull_spec=from_index_pull_spec + serial=request_payload.overwrite_from_index, from_index_pull_spec=from_index_pull_spec ) - if isinstance(request, RequestAdd): - args: List[Any] = _get_add_args( - # cast Union[AddRequestPayload, RmRequestPayload] based on request variable - cast(AddRequestPayload, build_request), - request, - overwrite_from_index, - celery_queue, - ) - elif isinstance(request, RequestRm): - args = _get_rm_args( - # cast Union[AddRequestPayload, RmRequestPayload] based on request variable - cast(RmRequestPayload, build_request), - request, - overwrite_from_index, - ) - - safe_args = _get_safe_args(args, build_request) - error_callback = failed_request_callback.s(request.id) try: if isinstance(request, RequestAdd): + args = [ + request_payload, + request.id, + flask.current_app.config['IIB_GREENWAVE_CONFIG'].get(celery_queue), + flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], + ] handle_add_request.apply_async( args=args, link_error=error_callback, - argsrepr=repr(safe_args), queue=celery_queue, ) - else: + elif isinstance(request, RequestRm): + args = [ + request_payload, + request.id, + flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], + ] handle_rm_request.apply_async( args=args, link_error=error_callback, - argsrepr=repr(safe_args), queue=celery_queue, ) except kombu.exceptions.OperationalError: @@ -1082,37 +954,33 @@ def merge_index_image() -> Tuple[flask.Response, int]: :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ - payload: MergeIndexImagesPayload = cast(MergeIndexImagesPayload, flask.request.get_json()) - if not isinstance(payload, dict): - raise ValidationError('The input data must be a JSON object') - request = RequestMergeIndexImage.from_json(payload) + try: + request_payload = MergeIndexImagePydanticModel.model_validate( + flask.request.get_json(), + strict=True, + ) + except ValidationError as e: + # If the JSON data doesn't match the Pydantic model, return a 400 Bad Request response + return flask.jsonify({'Error parsing data': str(e)}), 400 + + request = RequestMergeIndexImage.from_json_replacement( + payload=request_payload, + ) + db.session.add(request) db.session.commit() messaging.send_message_for_state_change(request, new_batch_msg=True) - overwrite_target_index = payload.get('overwrite_target_index', False) - celery_queue = _get_user_queue(serial=overwrite_target_index) + celery_queue = _get_user_queue(serial=request_payload.overwrite_target_index) args = [ - payload['source_from_index'], - payload.get('deprecation_list', []), + request_payload, request.id, - payload.get('binary_image'), - payload.get('target_index'), - overwrite_target_index, - payload.get('overwrite_target_index_token'), - request.distribution_scope, flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], - payload.get('build_tags', []), - payload.get('graph_update_mode'), - payload.get('ignore_bundle_ocp_version'), ] - safe_args = _get_safe_args(args, payload) error_callback = failed_request_callback.s(request.id) try: - handle_merge_request.apply_async( - args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=celery_queue - ) + handle_merge_request.apply_async(args=args, link_error=error_callback, queue=celery_queue) except kombu.exceptions.OperationalError: handle_broker_error(request) @@ -1132,29 +1000,33 @@ def create_empty_index() -> Tuple[flask.Response, int]: :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ - payload: CreateEmptyIndexPayload = cast(CreateEmptyIndexPayload, flask.request.get_json()) - if not isinstance(payload, dict): - raise ValidationError('The input data must be a JSON object') + try: + request_payload = CreateEmptyIndexPydanticModel.model_validate( + flask.request.get_json(), + strict=True, + ) + except ValidationError as e: + # If the JSON data doesn't match the Pydantic model, return a 400 Bad Request response + return flask.jsonify({'Error parsing data': str(e)}), 400 + + request = RequestCreateEmptyIndex.from_json_replacement( + payload=request_payload, + ) - request = RequestCreateEmptyIndex.from_json(payload) db.session.add(request) db.session.commit() messaging.send_message_for_state_change(request, new_batch_msg=True) args = [ - payload['from_index'], + request_payload, request.id, - payload.get('output_fbc'), - payload.get('binary_image'), - payload.get('labels'), flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], ] - safe_args = _get_safe_args(args, payload) error_callback = failed_request_callback.s(request.id) try: handle_create_empty_index_request.apply_async( - args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=_get_user_queue() + args=args, link_error=error_callback, queue=_get_user_queue() ) except kombu.exceptions.OperationalError: handle_broker_error(request) @@ -1175,31 +1047,33 @@ def recursive_related_bundles() -> Tuple[flask.Response, int]: :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ - payload: RecursiveRelatedBundlesRequestPayload = cast( - RecursiveRelatedBundlesRequestPayload, flask.request.get_json() + try: + request_payload = RecursiveRelatedBundlesPydanticModel.model_validate( + flask.request.get_json(), + strict=True, + ) + except ValidationError as e: + # If the JSON data doesn't match the Pydantic model, return a 400 Bad Request response + return flask.jsonify({'Error parsing data': str(e)}), 400 + + request = RequestRecursiveRelatedBundles.from_json_replacement( + payload=request_payload, ) - if not isinstance(payload, dict): - raise ValidationError('The input data must be a JSON object') - request = RequestRecursiveRelatedBundles.from_json(payload) db.session.add(request) db.session.commit() messaging.send_message_for_state_change(request, new_batch_msg=True) args = [ - payload['parent_bundle_image'], - payload.get('organization'), + request_payload, request.id, - payload.get('registry_auths'), ] - safe_args = _get_safe_args(args, payload) error_callback = failed_request_callback.s(request.id) try: handle_recursive_related_bundles_request.apply_async( args=args, link_error=error_callback, - argsrepr=repr(safe_args), queue=_get_user_queue(), ) except kombu.exceptions.OperationalError: @@ -1281,38 +1155,37 @@ def fbc_operations() -> Tuple[flask.Response, int]: :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ - payload: FbcOperationRequestPayload = flask.request.get_json() - if not isinstance(payload, dict): - raise ValidationError('The input data must be a JSON object') + try: + request_payload = FbcOperationsPydanticModel.model_validate( + flask.request.get_json(), + strict=True, + ) + except ValidationError as e: + # If the JSON data doesn't match the Pydantic model, return a 400 Bad Request response + return flask.jsonify({'Error parsing data': str(e)}), 400 + + request = RequestFbcOperations.from_json_replacement( + payload=request_payload, + ) - request = RequestFbcOperations.from_json(payload) db.session.add(request) db.session.commit() messaging.send_message_for_state_change(request, new_batch_msg=True) - overwrite_from_index = payload.get('overwrite_from_index', False) from_index_pull_spec = request.from_index.pull_specification if request.from_index else None celery_queue = _get_user_queue( - serial=overwrite_from_index, from_index_pull_spec=from_index_pull_spec + serial=request_payload.overwrite_from_index, from_index_pull_spec=from_index_pull_spec ) args = [ - request.id, - payload['fbc_fragment'], - payload['from_index'], - payload.get('binary_image'), - payload.get('distribution_scope'), - payload.get('overwrite_from_index'), - payload.get('overwrite_from_index_token'), - payload.get('build_tags'), - payload.get('add_arches'), + request_payload, + request_payload, flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], ] - safe_args = _get_safe_args(args, payload) error_callback = failed_request_callback.s(request.id) try: handle_fbc_operation_request.apply_async( - args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=celery_queue + args=args, link_error=error_callback, queue=celery_queue ) except kombu.exceptions.OperationalError: handle_broker_error(request) diff --git a/iib/web/iib_static_types.py b/iib/web/iib_static_types.py index b72520d88..d13d193cc 100644 --- a/iib/web/iib_static_types.py +++ b/iib/web/iib_static_types.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-3.0-or-later -from typing import Any, Dict, List, NamedTuple, Optional, Union, Sequence, Set +from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Set from typing_extensions import NotRequired, TypedDict, Literal from proton._message import Message @@ -35,207 +35,6 @@ class RelatedBundlesMetadata(TypedDict): url: str -# Start of the Payloads Part - -# try inheritance from other payloads - -PayloadTags = Literal[ - 'AddRequestPayload', - 'RmRequestPayload', - 'RegenerateBundlePayload', - 'RegenerateBundleBatchPayload', - 'AddRmBatchPayload', - 'MergeIndexImagesPayload', - 'CreateEmptyIndexPayload', - 'FbcOperationRequestPayload', -] - - -PossiblePayloadParameters = Sequence[ - Literal[ - 'add_arches', - 'annotations', - 'batch', - 'binary_image', - 'build_requests', - 'build_tags', - 'bundles', - 'cnr_token', - 'check_related_images', - 'deprecation_list', - 'distribution_scope', - 'force_backport', - 'from_bundle_image', - 'from_index', - 'graph_update_mode', - 'labels', - 'operators', - 'organization', - 'output_fbc', - 'overwrite_from_index', - 'overwrite_from_index_token', - 'registry_auths', - 'related_bundles', - 'source_from_index', - 'target_index', - 'user', - ] -] - - -class AddRequestPayload(TypedDict): - """Datastructure of the request to /builds/add API point.""" - - add_arches: NotRequired[List[str]] - binary_image: NotRequired[str] - build_tags: NotRequired[List[str]] - bundles: List[str] - cnr_token: NotRequired[str] - check_related_images: NotRequired[bool] - deprecation_list: NotRequired[List[str]] - distribution_scope: NotRequired[str] - force_backport: NotRequired[bool] - from_index: NotRequired[str] - graph_update_mode: NotRequired[GRAPH_MODE_LITERAL] - organization: NotRequired[str] - overwrite_from_index: NotRequired[bool] - overwrite_from_index_token: NotRequired[str] - - -class RmRequestPayload(TypedDict): - """Datastructure of the request to /builds/rm API point.""" - - add_arches: NotRequired[List[str]] - binary_image: NotRequired[str] - build_tags: NotRequired[List[str]] - distribution_scope: NotRequired[str] - from_index: str - operators: List[str] - overwrite_from_index: NotRequired[bool] - overwrite_from_index_token: Optional[str] - - -class FbcOperationRequestPayload(TypedDict): - """Datastructure of the request to /builds/fbc-operation API point.""" - - fbc_fragment: str - from_index: str - binary_image: NotRequired[str] - build_tags: NotRequired[List[str]] - add_arches: NotRequired[List[str]] - overwrite_from_index: NotRequired[bool] - overwrite_from_index_token: NotRequired[str] - batch: NotRequired[str] - distribution_scope: NotRequired[str] - user: NotRequired[str] - - -class RegenerateBundlePayload(TypedDict): - """Datastructure of the request to /builds/regenerate-bundle API point.""" - - from_bundle_image: str - organization: NotRequired[str] - registry_auths: NotRequired[Dict[str, Any]] - related_bundles: NotRequired[RelatedBundlesMetadata] - user: NotRequired[str] - batch: NotRequired[str] - - -class RegenerateBundleBatchPayload(TypedDict): - """Datastructure of the request to /builds/regenerate-bundle-batch API point.""" - - annotations: NotRequired[Dict[str, Any]] - build_requests: List[RegenerateBundlePayload] - - -class AddRmBatchPayload(TypedDict): - """Datastructure of the request to /builds/add-rm-batch API point.""" - - annotations: NotRequired[Dict[str, Any]] - build_requests: List[Union[AddRequestPayload, RmRequestPayload]] - - -class MergeIndexImagesPayload(TypedDict): - """Datastructure of the request to /builds/merge-index-image API point.""" - - binary_image: NotRequired[str] - build_tags: NotRequired[List[str]] - deprecation_list: NotRequired[List[str]] - distribution_scope: NotRequired[str] - graph_update_mode: NotRequired[GRAPH_MODE_LITERAL] - overwrite_target_index: NotRequired[bool] - overwrite_target_index_token: NotRequired[str] - source_from_index: str - target_index: NotRequired[str] - batch: NotRequired[str] - user: NotRequired[str] - - -class CreateEmptyIndexPayload(TypedDict): - """Datastructure of the request to /builds/create-empty-index API point.""" - - binary_image: NotRequired[str] - from_index: str - labels: NotRequired[Dict[str, str]] - output_fbc: NotRequired[bool] - - -class RecursiveRelatedBundlesRequestPayload(TypedDict): - """Datastructure of the request to /builds/recursive-related-bundles API point.""" - - batch: NotRequired[int] - organization: NotRequired[str] - parent_bundle_image: str - registry_auths: NotRequired[Dict[str, Any]] - user: NotRequired[str] - - -class RequestPayload(TypedDict): - """Datastructure with all the possible keys that can API points receive.""" - - add_arches: NotRequired[List[str]] - annotations: NotRequired[Dict[str, Any]] - batch: NotRequired[int] - binary_image: NotRequired[str] - build_requests: NotRequired[ - List[Union[AddRequestPayload, RmRequestPayload, RegenerateBundlePayload]] - ] - build_tags: NotRequired[List[str]] - bundles: NotRequired[Optional[List[str]]] - cnr_token: NotRequired[str] - check_related_images: NotRequired[bool] - deprecation_list: NotRequired[List[str]] - distribution_scope: NotRequired[str] - fbc_fragment: NotRequired[bool] - force_backport: NotRequired[bool] - from_bundle_image: NotRequired[str] - from_index: NotRequired[str] - labels: NotRequired[Dict[str, str]] - operators: NotRequired[List[str]] - organization: NotRequired[str] - output_fbc: NotRequired[bool] - overwrite_from_index: NotRequired[bool] - overwrite_from_index_token: NotRequired[str] - overwrite_target_index: NotRequired[bool] - overwrite_target_index_token: NotRequired[str] - registry_auths: NotRequired[Dict[str, Any]] - related_bundles: NotRequired[RelatedBundlesMetadata] - source_from_index: NotRequired[str] - target_index: NotRequired[str] - user: NotRequired[str] - - -PayloadTypesUnion = Union[ - AddRequestPayload, - CreateEmptyIndexPayload, - FbcOperationRequestPayload, - MergeIndexImagesPayload, - RecursiveRelatedBundlesRequestPayload, - RegenerateBundlePayload, - RmRequestPayload, -] - -# End of the Payloads Part # Start of the RequestResponses Part diff --git a/iib/web/models.py b/iib/web/models.py index b6d36b37d..11c0b0210 100644 --- a/iib/web/models.py +++ b/iib/web/models.py @@ -1,11 +1,9 @@ # SPDX-License-Identifier: GPL-3.0-or-later from __future__ import annotations -from copy import deepcopy from datetime import datetime, timedelta from enum import Enum import json -from typing import Any, cast, Dict, List, Literal, Optional, Sequence, Set, Union -from abc import abstractmethod +from typing import Any, cast, Dict, List, Optional, Set, Union from flask import current_app, url_for from flask_login import UserMixin, current_user @@ -14,34 +12,24 @@ from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import joinedload, load_only, Mapped, validates from sqlalchemy.orm.strategy_options import _AbstractLoad -from werkzeug.exceptions import Forbidden from iib.exceptions import ValidationError from iib.web import db - +from iib.common.pydantic_models import ( + UnionPydanticRequestType, +) from iib.web.iib_static_types import ( - AddRequestPayload, AddRequestResponse, - AddRmBatchPayload, AddRmRequestResponseBase, BaseClassRequestResponse, BuildRequestState, CommonIndexImageResponseBase, - CreateEmptyIndexPayload, CreateEmptyIndexRequestResponse, + FbcOperationRequestResponse, MergeIndexImageRequestResponse, - MergeIndexImagesPayload, - RequestPayload, - PayloadTypesUnion, - RecursiveRelatedBundlesRequestPayload, RecursiveRelatedBundlesRequestResponse, - RegenerateBundleBatchPayload, - RegenerateBundlePayload, RegenerateBundleRequestResponse, - RmRequestPayload, - FbcOperationRequestPayload, - FbcOperationRequestResponse, ) @@ -493,21 +481,71 @@ def add_architecture(self, arch_name: str) -> None: if arch not in self.architectures: self.architectures.append(arch) - @abstractmethod - def from_json( + @classmethod + def from_json_replacement( cls, - kwargs: PayloadTypesUnion, - ) -> Request: + payload: UnionPydanticRequestType, + batch: Optional[Batch] = None, + ): """ - Handle JSON requests for a request API endpoint. - - Child classes MUST override this method. + Handle JSON requests for the builds/* API endpoint. - :param PayloadTypesUnion kwargs: the user provided parameters to create a Request - :return: an object representation of the request - :retype: Request + :param UnionPydanticRequestType payload: the Pydantic model representing the request. + :param Batch batch: the batch to specify with the request. """ - raise NotImplementedError('{} does not implement from_json'.format(cls.__name__)) + keys_to_check = payload.get_keys_to_check_in_db() + for key in keys_to_check: + if key in [ + 'binary_image', + 'fbc_fragment', + 'from_index', + 'from_bundle_image', + 'source_from_index', + 'target_index', + 'parent_bundle_image', + ]: + payload.__setattr__( + key, + Image.get_or_create(pull_specification=payload.__getattribute__(key)), + ) + + elif key in ["bundles", "deprecation_list"]: + payload.__setattr__( + key, + [ + Image.get_or_create(pull_specification=image) + for image in payload.__getattribute__(key) + ], + ) + + elif key == "operators": + payload.__setattr__( + key, + [Operator.get_or_create(name=item) for item in payload.__getattribute__(key)], + ) + + else: + raise ValidationError(f"Unexpected key: {key} during from_json() method.") + + request_kwargs = payload.get_json_for_request() + + # current_user.is_authenticated is only ever False when auth is disabled + if current_user.is_authenticated: + request_kwargs['user'] = current_user + + # Add the request to a new batch + + batch = batch or Batch() + db.session.add(batch) + request_kwargs['batch'] = batch + + request = cls(**request_kwargs) + if payload.model_fields.get("build_tags") and payload.build_tags: + for bt in payload.build_tags: + request.add_build_tag(bt) + + request.add_state('in_progress', 'The request was initiated') + return request # return value is BaseClassRequestResponse, however because of LSP, we need other types here too def to_json( @@ -627,31 +665,6 @@ def annotations(self, annotations: Optional[Dict[str, Any]]) -> None: json.dumps(annotations, sort_keys=True) if annotations is not None else None ) - @staticmethod - def validate_batch_request_params( - payload: Union[AddRmBatchPayload, RegenerateBundleBatchPayload] - ) -> None: - """ - Validate batch specific parameters from the input JSON payload. - - The requests in the "build_requests" key's value are not validated. Those should be - validated separately. - - :raises ValidationError: if the payload is invalid - """ - if ( - not isinstance(payload, dict) - or not isinstance(payload.get('build_requests'), list) - or not payload['build_requests'] - ): - raise ValidationError( - 'The input data must be a JSON object and the "build_requests" value must be a ' - 'non-empty array' - ) - - if not isinstance(payload.get('annotations', {}), dict): - raise ValidationError('The value of "annotations" must be a JSON object') - @property def state(self) -> str: """ @@ -798,30 +811,6 @@ def get_request_query_options(verbose: Optional[bool] = False) -> List[_Abstract return query_options -def validate_graph_mode(graph_update_mode: Optional[str], index_image: Optional[str]): - """ - Validate graph mode and check if index image is allowed to use different graph mode. - - :param str graph_update_mode: one of the graph mode options - :param str index_image: pullspec of index image to which graph mode should be applied to - :raises: ValidationError when incorrect graph_update_mode is set - :raises: Forbidden when graph_mode can't be used for given index image - - """ - if graph_update_mode: - graph_mode_options = current_app.config['IIB_GRAPH_MODE_OPTIONS'] - if graph_update_mode not in graph_mode_options: - raise ValidationError( - f'"graph_update_mode" must be set to one of these: {graph_mode_options}' - ) - allowed_from_indexes: List[str] = current_app.config['IIB_GRAPH_MODE_INDEX_ALLOW_LIST'] - if index_image not in allowed_from_indexes: - raise Forbidden( - '"graph_update_mode" can only be used on the' - f' following index image: {allowed_from_indexes}' - ) - - class RequestIndexImageMixin: """ A class for shared functionality between index image requests. @@ -919,112 +908,6 @@ def distribution_scope(cls: DefaultMeta) -> Mapped[str]: """Return the distribution_scope for the request.""" return db.mapped_column(db.String, nullable=True) - # Union for request_kwargs would require exhausting checking of the request_kwargs in the method - @staticmethod - def _from_json( - request_kwargs: RequestPayload, - additional_required_params: Optional[List[str]] = None, - additional_optional_params: Optional[List[str]] = None, - batch: Optional[Batch] = None, - ) -> None: - """ - Validate and process request agnostic parameters. - - As part of the processing, the input ``request_kwargs`` parameter - is updated to reference database objects where appropriate. - - :param dict request_kwargs: copy of args provided in API request - :param Batch batch: the batch to specify with the request. If one is not specified, one will - be created automatically. - """ - # Validate all required parameters are present - required_params = set(additional_required_params or []) - optional_params = { - 'add_arches', - 'binary_image', - 'overwrite_from_index', - 'overwrite_from_index_token', - 'distribution_scope', - 'build_tags', - 'output_fbc', - } | set(additional_optional_params or []) - - validate_request_params( - request_kwargs, required_params=required_params, optional_params=optional_params - ) - - # Check if both `from_index` and `add_arches` are not specified - if not request_kwargs.get('from_index') and not request_kwargs.get('add_arches'): - raise ValidationError('One of "from_index" or "add_arches" must be specified') - - # Verify that `overwrite_from_index` is the correct type - overwrite = request_kwargs.pop('overwrite_from_index', False) - if not isinstance(overwrite, bool): - raise ValidationError('The "overwrite_from_index" parameter must be a boolean') - - # Verify that `overwrite_from_index_token` is the correct type - overwrite_token = request_kwargs.pop('overwrite_from_index_token', None) - if overwrite_token: - if not isinstance(overwrite_token, str): - raise ValidationError('The "overwrite_from_index_token" parameter must be a string') - if overwrite_token and not overwrite: - raise ValidationError( - 'The "overwrite_from_index" parameter is required when' - ' the "overwrite_from_index_token" parameter is used' - ) - - distribution_scope = request_kwargs.pop('distribution_scope', None) - if distribution_scope: - distribution_scope = distribution_scope.lower() - if distribution_scope not in ['prod', 'stage', 'dev']: - raise ValidationError( - 'The "distribution_scope" value must be one of "dev", "stage", or "prod"' - ) - request_kwargs['distribution_scope'] = distribution_scope - - # Prevent duplicated items in "deprecation_list" - deprecation_list = request_kwargs.pop('deprecation_list', None) - if deprecation_list: - request_kwargs['deprecation_list'] = list(set(deprecation_list)) - - # Verify the user is authorized to use overwrite_from_index - # current_user.is_authenticated is only ever False when auth is disabled - if current_user.is_authenticated: - if overwrite and not overwrite_token: - raise Forbidden( - 'You must set "overwrite_from_index_token" to use "overwrite_from_index"' - ) - - # Validate add_arches are correctly provided - add_arches = request_kwargs.pop('add_arches', []) - Architecture.validate_architecture_json(add_arches) - - # Validate binary_image is correctly provided - binary_image = request_kwargs.pop('binary_image', None) - if binary_image is not None and not isinstance(binary_image, str): - raise ValidationError('The "binary_image" value must be a string') - elif not binary_image and not current_app.config['IIB_BINARY_IMAGE_CONFIG']: - raise ValidationError('The "binary_image" value must be a non-empty string') - - if binary_image: - request_kwargs['binary_image'] = Image.get_or_create(pull_specification=binary_image) - - if 'from_index' in request_kwargs: - if not isinstance(request_kwargs['from_index'], str): - raise ValidationError('"from_index" must be a string') - request_kwargs['from_index'] = Image.get_or_create( - pull_specification=request_kwargs['from_index'] - ) - - # current_user.is_authenticated is only ever False when auth is disabled - if current_user.is_authenticated: - request_kwargs['user'] = current_user - - # Add the request to a new batch - batch = batch or Batch() - db.session.add(batch) - request_kwargs['batch'] = batch - def get_common_index_image_json(self) -> CommonIndexImageResponseBase: """ Return the common set of attributes for an index image request. @@ -1105,101 +988,6 @@ class RequestAdd(Request, RequestIndexImageMixin): __mapper_args__ = {'polymorphic_identity': RequestTypeMapping.__members__['add'].value} - @classmethod - def from_json( # type: ignore[override] # noqa: F821 - cls, - kwargs: AddRequestPayload, - batch: Optional[Batch] = None, - ) -> RequestAdd: - """ - Handle JSON requests for the Add API endpoint. - - :param dict kwargs: the JSON payload of the request. - :param Batch batch: the batch to specify with the request. - """ - request_kwargs = deepcopy(kwargs) - - for key in ('bundles', 'deprecation_list'): - value = request_kwargs.get(key, []) - if not isinstance(value, list) or any( - not item or not isinstance(item, str) for item in value - ): - raise ValidationError( - f'"{key}" should be either an empty array or an array of non-empty strings' - ) - - # Check if no bundles and `from_index is specified - # if no bundles and no from index then an empty index will be created which is a no-op - if not (request_kwargs.get('bundles') or request_kwargs.get('from_index')): - raise ValidationError('"from_index" must be specified if no bundles are specified') - - # Verify that `check_related_images` is specified when bundles are specified - if request_kwargs.get('check_related_images') and not request_kwargs.get('bundles'): - raise ValidationError( - '"check_related_images" must be specified only when bundles are specified' - ) - - # Verify that `check_related_images` is the correct type - check_related_images = request_kwargs.get('check_related_images', False) - if not isinstance(check_related_images, bool): - raise ValidationError('The "check_related_images" parameter must be a boolean') - - ALLOWED_KEYS_1: Sequence[Literal['cnr_token', 'graph_update_mode', 'organization']] = ( - 'cnr_token', - 'graph_update_mode', - 'organization', - ) - for param in ALLOWED_KEYS_1: - if param not in request_kwargs: - continue - - if not isinstance(request_kwargs[param], str): - raise ValidationError(f'"{param}" must be a string') - - if param == 'graph_update_mode': - validate_graph_mode(request_kwargs[param], request_kwargs.get('from_index')) - - if not isinstance(request_kwargs.get('force_backport', False), bool): - raise ValidationError('"force_backport" must be a boolean') - - # Remove attributes that are not stored in the database - request_kwargs.pop('cnr_token', None) - request_kwargs.pop('force_backport', None) - - # cast to more wider type, see _from_json method - cls._from_json( - cast(RequestPayload, request_kwargs), - additional_optional_params=[ - 'from_index', - 'organization', - 'bundles', - 'distribution_scope', - 'deprecation_list', - 'graph_update_mode', - 'build_tags', - 'check_related_images', - ], - batch=batch, - ) - - ALLOWED_KEYS_2: Sequence[Literal['bundles', 'deprecation_list']] = ( - 'bundles', - 'deprecation_list', - ) - for key in ALLOWED_KEYS_2: - request_kwargs[key] = [ - Image.get_or_create(pull_specification=item) - for item in request_kwargs.get(key, []) # type: ignore - ] - build_tags = request_kwargs.pop('build_tags', []) - request = cls(**request_kwargs) - - for bt in build_tags: - request.add_build_tag(bt) - - request.add_state('in_progress', 'The request was initiated') - return request - def to_json(self, verbose: Optional[bool] = True) -> AddRequestResponse: """ Provide the JSON representation of an "add" build request. @@ -1259,46 +1047,6 @@ class RequestRm(Request, RequestIndexImageMixin): __mapper_args__ = {'polymorphic_identity': RequestTypeMapping.__members__['rm'].value} - @classmethod - def from_json( # type: ignore[override] # noqa: F821 - cls, - kwargs: RmRequestPayload, - batch: Optional[Batch] = None, - ) -> RequestRm: - """ - Handle JSON requests for the Remove API endpoint. - - :param dict kwargs: the JSON payload of the request. - :param Batch batch: the batch to specify with the request. - """ - request_kwargs = deepcopy(kwargs) - - operators = request_kwargs.get('operators', []) - if ( - not isinstance(operators, list) - or len(operators) == 0 - or any(not item or not isinstance(item, str) for item in operators) - ): - raise ValidationError('"operators" should be a non-empty array of strings') - - # cast to more wider type, see _from_json method - cls._from_json( - cast(RequestPayload, request_kwargs), - additional_required_params=['operators', 'from_index'], - batch=batch, - ) - - request_kwargs['operators'] = [Operator.get_or_create(name=item) for item in operators] - - build_tags = request_kwargs.pop('build_tags', []) - request = cls(**request_kwargs) - request.add_state('in_progress', 'The request was initiated') - - for bt in build_tags: - request.add_build_tag(bt) - - return request - def to_json(self, verbose: Optional[bool] = True) -> AddRmRequestResponseBase: """ Provide the JSON representation of an "rm" build request. @@ -1381,70 +1129,6 @@ def bundle_replacements(self, bundle_replacements: Dict[str, str]) -> None: json.dumps(bundle_replacements, sort_keys=True) if bundle_replacements else None ) - @classmethod - def from_json( # type: ignore[override] # noqa: F821 - cls, - kwargs: RegenerateBundlePayload, - batch: Optional[Batch] = None, - ) -> RequestRegenerateBundle: - """ - Handle JSON requests for the Regenerate Bundle API endpoint. - - :param dict kwargs: the JSON payload of the request. - :param Batch batch: the batch to specify with the request. If one is not specified, one will - be created automatically. - """ - batch = batch or Batch() - request_kwargs = deepcopy(kwargs) - - validate_request_params( - request_kwargs, - required_params={'from_bundle_image'}, - optional_params={'bundle_replacements', 'organization', 'registry_auths'}, - ) - # Validate bundle_replacements is correctly provided - bundle_replacements = request_kwargs.get('bundle_replacements', {}) - if bundle_replacements: - if not isinstance(bundle_replacements, dict): - raise ValidationError('The value of "bundle_replacements" must be a JSON object') - - for key, value in bundle_replacements.items(): - if not isinstance(value, str) or not isinstance(key, str): - raise ValidationError(f'The key and value of "{key}" must be a string') - - # Validate organization is correctly provided - organization = request_kwargs.get('organization') - if organization and not isinstance(organization, str): - raise ValidationError('"organization" must be a string') - - # Validate from_bundle_image is correctly provided - from_bundle_image = request_kwargs.get('from_bundle_image') - if not isinstance(from_bundle_image, str): - raise ValidationError('"from_bundle_image" must be a string') - - # Remove attributes that are not stored in the database - registry_auths = request_kwargs.pop('registry_auths', None) - - # Check that registry_auths were provided in valid format - if registry_auths: - validate_registry_auths(registry_auths) - - request_kwargs['from_bundle_image'] = Image.get_or_create( - pull_specification=from_bundle_image - ) - - # current_user.is_authenticated is only ever False when auth is disabled - if current_user.is_authenticated: - request_kwargs['user'] = current_user - - # Add the request to a new batch - db.session.add(batch) - request_kwargs['batch'] = batch - - request = cls(**request_kwargs) - request.add_state('in_progress', 'The request was initiated') - return request - def to_json(self, verbose: Optional[bool] = True) -> RegenerateBundleRequestResponse: """ Provide the JSON representation of a "regenerate-bundle" build request. @@ -1540,114 +1224,6 @@ class RequestMergeIndexImage(Request): 'polymorphic_identity': RequestTypeMapping.__members__['merge_index_image'].value } - @classmethod - def from_json( # type: ignore[override] # noqa: F821 - cls, - kwargs: MergeIndexImagesPayload, - batch: Optional[Batch] = None, - ) -> RequestMergeIndexImage: - """ - Handle JSON requests for the merge-index-image API endpoint. - - :param dict kwargs: the JSON payload of the request. - :param Batch batch: the batch to specify with the request. - """ - request_kwargs = deepcopy(kwargs) - - deprecation_list = request_kwargs.pop('deprecation_list', []) - if not isinstance(deprecation_list, list) or any( - not item or not isinstance(item, str) for item in deprecation_list - ): - raise ValidationError( - 'The "deprecation_list" value should be an empty array or an array of strings' - ) - - request_kwargs['deprecation_list'] = [ - Image.get_or_create(pull_specification=item) for item in deprecation_list - ] - - source_from_index = request_kwargs.get('source_from_index', None) - if not (isinstance(source_from_index, str) and source_from_index): - raise ValidationError('The "source_from_index" value must be a string') - request_kwargs['source_from_index'] = Image.get_or_create( - pull_specification=source_from_index - ) - - graph_update_mode = request_kwargs.get('graph_update_mode') - validate_graph_mode(graph_update_mode, request_kwargs.get('target_index')) - - target_index = request_kwargs.pop('target_index', None) - if target_index: - if not isinstance(target_index, str): - raise ValidationError('The "target_index" value must be a string') - request_kwargs['target_index'] = Image.get_or_create(pull_specification=target_index) - - # Verify that `overwrite_target_index` is the correct type - overwrite = request_kwargs.pop('overwrite_target_index', False) - if not isinstance(overwrite, bool): - raise ValidationError('The "overwrite_target_index" value must be a boolean') - - # Verify that `overwrite_target_index_token` is the correct type - overwrite_token = request_kwargs.pop('overwrite_target_index_token', None) - if overwrite_token: - if not isinstance(overwrite_token, str): - raise ValidationError('The "overwrite_target_index_token" value must be a string') - if overwrite_token and not overwrite: - raise ValidationError( - 'The "overwrite_target_index" value is required when' - ' the "overwrite_target_index_token" value is used' - ) - elif overwrite: - raise ValidationError( - 'The "overwrite_target_index_token" value is required when' - ' the "overwrite_target_index" value is set' - ) - - # Validate binary_image is correctly provided - binary_image = request_kwargs.pop('binary_image', None) - if binary_image is not None and not isinstance(binary_image, str): - raise ValidationError('The "binary_image" value must be a string') - elif not binary_image and not current_app.config['IIB_BINARY_IMAGE_CONFIG']: - raise ValidationError('The "binary_image" value must be a non-empty string') - - if binary_image: - request_kwargs['binary_image'] = Image.get_or_create(pull_specification=binary_image) - - distribution_scope = request_kwargs.pop('distribution_scope', None) - if distribution_scope: - distribution_scope = distribution_scope.lower() - if distribution_scope not in ['prod', 'stage', 'dev']: - raise ValidationError( - 'The "distribution_scope" value must be one of "dev", "stage", or "prod"' - ) - request_kwargs['distribution_scope'] = distribution_scope - - if not isinstance(request_kwargs.get('build_tags', []), list) or any( - not item or not isinstance(item, str) for item in request_kwargs.get('build_tags', []) - ): - raise ValidationError( - '"build_tags" should be either an empty array or an array of non-empty strings' - ) - - # current_user.is_authenticated is only ever False when auth is disabled - if current_user.is_authenticated: - request_kwargs['user'] = current_user - - # Add the request to a new batch - batch = batch or Batch() - db.session.add(batch) - request_kwargs['batch'] = batch - - request = cls(**request_kwargs) - - build_tags = request_kwargs.pop('build_tags', []) - - for bt in build_tags: - request.add_build_tag(bt) - - request.add_state('in_progress', 'The request was initiated') - return request - def to_json(self, verbose: Optional[bool] = True) -> MergeIndexImageRequestResponse: """ Provide the JSON representation of an "merge-index-image" build request. @@ -1770,78 +1346,6 @@ def get_or_create(cls, username: str) -> User: return user -def validate_request_params( - request_params: Union[RequestPayload, PayloadTypesUnion], - required_params: Set[str], - optional_params: Set[str], -) -> None: - """ - Validate parameters for a build request. - - All required parameters must be set in the request_params and - unknown parameters are not allowed. - - :param Union[RequestPayload, PayloadTypesUnion] request_params: the request parameters - provided by the user - :param set required_params: the set of required parameters - :param set optional_params: the set of optional parameters - :raises iib.exceptions.ValidationError: if validation of parameters fails - """ - missing_params = required_params - request_params.keys() - if missing_params: - raise ValidationError('Missing required parameter(s): {}'.format(', '.join(missing_params))) - - # Don't allow the user to set arbitrary columns or relationships - invalid_params = request_params.keys() - required_params - optional_params - if invalid_params: - raise ValidationError( - 'The following parameters are invalid: {}'.format(', '.join(invalid_params)) - ) - - # Verify that all the required parameters are set and not empty - for param in required_params: - if not request_params.get(param): - raise ValidationError(f'"{param}" must be set') - - # If any optional parameters are set but are empty, just remove them since they are - # treated as null values - for param in optional_params: - if ( - param in request_params - and not isinstance(request_params.get(param), bool) - and not request_params[param] # type: ignore - ): - del request_params[param] # type: ignore - - -def validate_registry_auths(registry_auths: Dict[str, Any]) -> None: - """ - Validate registry_auths for a build request. - - Only auth item in dockerconfig.json is supported for iib. - - :param dict registry_auths: User provided dockerconfig for authentication - to private registries - :raises ValidationError: if registry_auths are not in valid format - """ - auths = 'auths' - if not isinstance(registry_auths, dict): - raise ValidationError('"registry_auths" must be a dict') - if list(registry_auths.keys()) != [auths]: - raise ValidationError(f'"registry_auths" must contain single key "{auths}"') - if not registry_auths[auths] or not isinstance(registry_auths[auths], dict): - raise ValidationError(f'"registry_auths.{auths}" must be a non-empty dict') - for reg, auth_dict in registry_auths[auths].items(): - err_msg = ( - f'{reg} in registry_auths has auth value in incorrect format. ' - 'See the API docs for details on the expected format' - ) - if not isinstance(auth_dict, dict) or len(auth_dict) != 1: - raise ValidationError(err_msg) - if not all(k == 'auth' and isinstance(v, str) for (k, v) in auth_dict.items()): - raise ValidationError(err_msg) - - class RequestCreateEmptyIndex(Request, RequestIndexImageMixin): """An "create-empty-index" image build request.""" @@ -1874,64 +1378,6 @@ def labels(self, labels: Optional[Dict[str, Any]]) -> None: """ self._labels = json.dumps(labels, sort_keys=True) if labels is not None else None - @classmethod - def from_json( # type: ignore[override] # noqa: F821 - cls, - kwargs: CreateEmptyIndexPayload, - batch: Optional[Batch] = None, - ) -> RequestCreateEmptyIndex: - """ - Handle JSON requests for the create-empty-index API endpoint. - - :param dict kwargs: the JSON payload of the request. - :param Batch batch: the batch to specify with the request. - """ - request_kwargs = deepcopy(kwargs) - if request_kwargs.get('from_index') is None: - raise ValidationError('"from_index" must be a specified') - if ( - not isinstance(request_kwargs.get('from_index'), str) - or len(str(request_kwargs.get('from_index'))) == 0 - ): - raise ValidationError('"from_index" must be a non-empty string') - if request_kwargs.get('output_fbc') and not isinstance( - request_kwargs.get('output_fbc'), bool - ): - raise ValidationError('"output_fbc" should be boolean') - - new_labels = request_kwargs.get('labels') - if new_labels is not None: - if not isinstance(new_labels, dict): - raise ValidationError('The value of "labels" must be a JSON object') - - for key, value in new_labels.items(): - if not isinstance(value, str) or not isinstance(key, str): - raise ValidationError(f'The key and value of "{key}" must be a string') - - for arg in ( - 'add_arches', - 'overwrite_from_index', - 'overwrite_from_index_token', - 'build_tags', - ): - if arg in request_kwargs: - raise ValidationError( - f'The "{arg}" arg is invalid for the create-empty-index endpoint.' - ) - - # cast to more wider type, see _from_json method - cls._from_json( - cast(RequestPayload, request_kwargs), - additional_required_params=['from_index'], - additional_optional_params=['labels'], - batch=batch, - ) - - request = cls(**request_kwargs) - request.add_state('in_progress', 'The request was initiated') - - return request - def to_json(self, verbose: Optional[bool] = True) -> CreateEmptyIndexRequestResponse: """ Provide the JSON representation of an "create-empty-index" build request. @@ -2001,61 +1447,6 @@ class RequestRecursiveRelatedBundles(Request): } build_tags = None - @classmethod - def from_json( # type: ignore[override] # noqa: F821 - cls, - kwargs: RecursiveRelatedBundlesRequestPayload, - batch: Optional[Batch] = None, - ): - """ - Handle JSON requests for the Recursive Related Bundles API endpoint. - - :param dict kwargs: the JSON payload of the request. - :param Batch batch: the batch to specify with the request. If one is not specified, one will - be created automatically. - """ - batch = batch or Batch() - request_kwargs = deepcopy(kwargs) - - validate_request_params( - request_kwargs, - required_params={'parent_bundle_image'}, - optional_params={'organization', 'registry_auths'}, - ) - - # Validate organization is correctly provided - organization = request_kwargs.get('organization') - if organization and not isinstance(organization, str): - raise ValidationError('"organization" must be a string') - - # Validate parent_bundle_image is correctly provided - parent_bundle_image = request_kwargs.get('parent_bundle_image') - if not isinstance(parent_bundle_image, str): - raise ValidationError('"parent_bundle_image" must be a string') - - # Remove attributes that are not stored in the database - registry_auths = request_kwargs.pop('registry_auths', None) - - # Check that registry_auths were provided in valid format - if registry_auths: - validate_registry_auths(registry_auths) - - request_kwargs['parent_bundle_image'] = Image.get_or_create( - pull_specification=parent_bundle_image - ) - - # current_user.is_authenticated is only ever False when auth is disabled - if current_user.is_authenticated: - request_kwargs['user'] = current_user - - # Add the request to a new batch - db.session.add(batch) - request_kwargs['batch'] = batch - - request = cls(**request_kwargs) - request.add_state('in_progress', 'The request was initiated') - return request - def to_json(self, verbose: Optional[bool] = True) -> RecursiveRelatedBundlesRequestResponse: """ Provide the JSON representation of a "recursive-related-bundles" build request. @@ -2113,57 +1504,6 @@ class RequestFbcOperations(Request, RequestIndexImageMixin): 'polymorphic_identity': RequestTypeMapping.__members__['fbc_operations'].value } - @classmethod - def from_json( # type: ignore[override] # noqa: F821 - cls, - kwargs: FbcOperationRequestPayload, - ): - """ - Handle JSON requests for the fbc-operations API endpoint. - - :param dict kwargs: the JSON payload of the request. - """ - request_kwargs = deepcopy(kwargs) - - validate_request_params( - request_kwargs, - required_params={'fbc_fragment', 'from_index'}, - optional_params={ - 'add_arches', - 'binary_image', - 'distribution_scope', - 'build_tags', - 'overwrite_from_index', - 'overwrite_from_index_token', - }, - ) - - # Validate parent_bundle_image is correctly provided - fbc_fragment = request_kwargs.get('fbc_fragment') - if not isinstance(fbc_fragment, str): - raise ValidationError('The "fbc_fragment" must be a string') - request_kwargs['fbc_fragment'] = Image.get_or_create(pull_specification=fbc_fragment) - - # cast to more wider type, see _from_json method - cls._from_json( - cast(RequestPayload, request_kwargs), - additional_optional_params=[ - 'bundles', - 'fbc_fragment', - 'from_index', - 'organization', - ], - ) - - build_tags = request_kwargs.pop('build_tags', []) - request = cls(**request_kwargs) - - for bt in build_tags: - request.add_build_tag(bt) - - request.add_state('in_progress', 'The request was initiated') - return request - def to_json(self, verbose: Optional[bool] = True) -> FbcOperationRequestResponse: """ Provide the JSON representation of a "fbc-operation" build request. diff --git a/iib/workers/tasks/build.py b/iib/workers/tasks/build.py index 8e525a8e3..0f0cc6b74 100644 --- a/iib/workers/tasks/build.py +++ b/iib/workers/tasks/build.py @@ -16,8 +16,8 @@ wait_exponential, wait_incrementing, ) - from iib.common.common_utils import get_binary_versions +from iib.common.pydantic_models import AddPydanticModel, RmPydanticModel from iib.common.tracing import instrument_tracing from iib.exceptions import IIBError, ExternalServiceError from iib.workers.api_utils import set_request_state, update_request @@ -292,16 +292,20 @@ def _update_index_image_pull_spec( else: index_image = output_pull_spec - payload: UpdateRequestPayload = {'arches': list(arches), 'index_image': index_image} + update_payload: UpdateRequestPayload = {'arches': list(arches), 'index_image': index_image} if add_or_rm: with set_registry_token(overwrite_from_index_token, from_index, append=True): index_image_resolved = get_resolved_image(index_image) - payload['index_image_resolved'] = index_image_resolved - payload['internal_index_image_copy'] = output_pull_spec - payload['internal_index_image_copy_resolved'] = get_resolved_image(output_pull_spec) + update_payload['index_image_resolved'] = index_image_resolved + update_payload['internal_index_image_copy'] = output_pull_spec + update_payload['internal_index_image_copy_resolved'] = get_resolved_image(output_pull_spec) - update_request(request_id, payload, exc_msg='Failed setting the index image on the request') + update_request( + request_id, + update_payload, + exc_msg='Failed setting the index image on the request', + ) def _get_external_arch_pull_spec( @@ -671,7 +675,7 @@ def _update_index_image_build_state( image. """ arches_str = ', '.join(sorted(prebuild_info['arches'])) - payload: UpdateRequestPayload = { + update_payload: UpdateRequestPayload = { 'binary_image': prebuild_info['binary_image'], 'binary_image_resolved': prebuild_info['binary_image_resolved'], 'state': 'in_progress', @@ -681,26 +685,26 @@ def _update_index_image_build_state( bundle_mapping: Optional[Dict[str, List[str]]] = prebuild_info.get('bundle_mapping') if bundle_mapping: - payload['bundle_mapping'] = bundle_mapping + update_payload['bundle_mapping'] = bundle_mapping from_index_resolved = prebuild_info.get('from_index_resolved') if from_index_resolved: - payload['from_index_resolved'] = from_index_resolved + update_payload['from_index_resolved'] = from_index_resolved source_from_index_resolved = prebuild_info.get('source_from_index_resolved') if source_from_index_resolved: - payload['source_from_index_resolved'] = source_from_index_resolved + update_payload['source_from_index_resolved'] = source_from_index_resolved target_index_resolved = prebuild_info.get('target_index_resolved') if target_index_resolved: - payload['target_index_resolved'] = target_index_resolved + update_payload['target_index_resolved'] = target_index_resolved fbc_fragment_resolved = prebuild_info.get('fbc_fragment_resolved') if fbc_fragment_resolved: - payload['fbc_fragment_resolved'] = fbc_fragment_resolved + update_payload['fbc_fragment_resolved'] = fbc_fragment_resolved exc_msg = 'Failed setting the resolved images on the request' - update_request(request_id, payload, exc_msg) + update_request(request_id, update_payload, exc_msg) @retry( @@ -835,24 +839,10 @@ def inspect_related_images(bundles: List[str], request_id) -> None: @request_logger @instrument_tracing(span_name="workers.tasks.handle_add_request", attributes=get_binary_versions()) def handle_add_request( - bundles: List[str], + payload: AddPydanticModel, request_id: int, - binary_image: Optional[str] = None, - from_index: Optional[str] = None, - add_arches: Optional[Set[str]] = None, - cnr_token: Optional[str] = None, - organization: Optional[str] = None, - force_backport: bool = False, - overwrite_from_index: bool = False, - overwrite_from_index_token: Optional[str] = None, - distribution_scope: Optional[str] = None, greenwave_config: Optional[GreenwaveConfig] = None, binary_image_config: Optional[Dict[str, Dict[str, str]]] = None, - deprecation_list: Optional[List[str]] = None, - build_tags: Optional[List[str]] = None, - graph_update_mode: Optional[str] = None, - check_related_images: bool = False, - traceparent: Optional[str] = None, ) -> None: """ Coordinate the the work needed to build the index image with the input bundles. @@ -896,10 +886,14 @@ def handle_add_request( # Resolve bundles to their digests set_request_state(request_id, 'in_progress', 'Resolving the bundles') - with set_registry_token(overwrite_from_index_token, from_index, append=True): - resolved_bundles = get_resolved_bundles(bundles) + with set_registry_token( + payload.overwrite_from_index_token, + payload.from_index, + append=True, + ): + resolved_bundles = get_resolved_bundles(payload.bundles) verify_labels(resolved_bundles) - if check_related_images: + if payload.check_related_images: inspect_related_images(resolved_bundles, request_id) # Check if Gating passes for all the bundles @@ -911,23 +905,23 @@ def handle_add_request( prebuild_info = prepare_request_for_build( request_id, RequestConfigAddRm( - _binary_image=binary_image, - from_index=from_index, - overwrite_from_index_token=overwrite_from_index_token, - add_arches=add_arches, - bundles=bundles, - distribution_scope=distribution_scope, + _binary_image=payload.binary_image, + from_index=payload.from_index, + overwrite_from_index_token=payload.overwrite_from_index_token, + add_arches=payload.add_arches, + bundles=payload.bundles, + distribution_scope=payload.distribution_scope, binary_image_config=binary_image_config, ), ) from_index_resolved = prebuild_info['from_index_resolved'] - with set_registry_token(overwrite_from_index_token, from_index_resolved): - is_fbc = is_image_fbc(from_index_resolved) if from_index else False + with set_registry_token(payload.overwrite_from_index_token, from_index_resolved): + is_fbc = is_image_fbc(from_index_resolved) if payload.from_index else False if is_fbc: # logging requested by stakeholders do not delete log.info("Processing File-Based Catalog image") - if (cnr_token and organization) or force_backport: + if (payload.cnr_token and payload.organization) or payload.force_backport: log.warning( "Legacy support is deprecated in IIB. " "cnr_token, organization and force_backport parameters will be ignored." @@ -937,12 +931,16 @@ def handle_add_request( present_bundles: List[BundleImage] = [] present_bundles_pull_spec: List[str] = [] with tempfile.TemporaryDirectory(prefix=f'iib-{request_id}-') as temp_dir: - if from_index: + if payload.from_index: msg = 'Checking if bundles are already present in index image' log.info(msg) set_request_state(request_id, 'in_progress', msg) - with set_registry_token(overwrite_from_index_token, from_index_resolved, append=True): + with set_registry_token( + payload.overwrite_from_index_token, + from_index_resolved, + append=True, + ): present_bundles, present_bundles_pull_spec = _get_present_bundles( from_index_resolved, temp_dir ) @@ -965,8 +963,8 @@ def handle_add_request( bundles=resolved_bundles, binary_image=prebuild_info['binary_image_resolved'], from_index=from_index_resolved, - graph_update_mode=graph_update_mode, - overwrite_from_index_token=overwrite_from_index_token, + graph_update_mode=payload.graph_update_mode, + overwrite_from_index_token=payload.overwrite_from_index_token, overwrite_csv=(prebuild_info['distribution_scope'] in ['dev', 'stage']), ) else: @@ -975,8 +973,8 @@ def handle_add_request( bundles=resolved_bundles, binary_image=prebuild_info['binary_image_resolved'], from_index=from_index_resolved, - graph_update_mode=graph_update_mode, - overwrite_from_index_token=overwrite_from_index_token, + graph_update_mode=payload.graph_update_mode, + overwrite_from_index_token=payload.overwrite_from_index_token, overwrite_csv=(prebuild_info['distribution_scope'] in ['dev', 'stage']), container_tool='podman', ) @@ -989,7 +987,7 @@ def handle_add_request( add_max_ocp_version_property(resolved_bundles, temp_dir) deprecation_bundles = get_bundles_from_deprecation_list( - present_bundles_pull_spec + resolved_bundles, deprecation_list or [] + present_bundles_pull_spec + resolved_bundles, payload.deprecation_list or [] ) arches = prebuild_info['arches'] @@ -1013,7 +1011,7 @@ def handle_add_request( ) with set_registry_token( - overwrite_from_index_token, from_index_resolved, append=True + payload.overwrite_from_index_token, from_index_resolved, append=True ): deprecate_bundles( bundles=deprecation_bundles, @@ -1049,7 +1047,11 @@ def handle_add_request( ) # get catalog with opted-in operators os.makedirs(os.path.join(temp_dir, 'from_index'), exist_ok=True) - with set_registry_token(overwrite_from_index_token, from_index_resolved, append=True): + with set_registry_token( + payload.overwrite_from_index_token, + from_index_resolved, + append=True, + ): catalog_from_index = get_catalog_dir( from_index=from_index_resolved, base_dir=os.path.join(temp_dir, 'from_index') ) @@ -1102,15 +1104,15 @@ def handle_add_request( ) set_request_state(request_id, 'in_progress', 'Creating the manifest list') - output_pull_spec = _create_and_push_manifest_list(request_id, arches, build_tags) + output_pull_spec = _create_and_push_manifest_list(request_id, arches, payload.build_tags) _update_index_image_pull_spec( output_pull_spec, request_id, arches, - from_index, - overwrite_from_index, - overwrite_from_index_token, + payload.from_index, + payload.overwrite_from_index, + payload.overwrite_from_index_token, from_index_resolved, add_or_rm=True, ) @@ -1124,16 +1126,9 @@ def handle_add_request( @request_logger @instrument_tracing(span_name="workers.tasks.handle_rm_request", attributes=get_binary_versions()) def handle_rm_request( - operators: List[str], + payload: RmPydanticModel, request_id: int, - from_index: str, - binary_image: Optional[str] = None, - add_arches: Optional[Set[str]] = None, - overwrite_from_index: bool = False, - overwrite_from_index_token: Optional[str] = None, - distribution_scope: Optional[str] = None, binary_image_config: Optional[Dict[str, Dict[str, str]]] = None, - build_tags: Optional[List[str]] = None, ) -> None: """ Coordinate the work needed to remove the input operators and rebuild the index image. @@ -1163,11 +1158,11 @@ def handle_rm_request( prebuild_info = prepare_request_for_build( request_id, RequestConfigAddRm( - _binary_image=binary_image, - from_index=from_index, - overwrite_from_index_token=overwrite_from_index_token, - add_arches=add_arches, - distribution_scope=distribution_scope, + _binary_image=payload.binary_image, + from_index=payload.from_index, + overwrite_from_index_token=payload.overwrite_from_index_token, + add_arches=payload.add_arches, + distribution_scope=payload.distribution_scope, binary_image_config=binary_image_config, ), ) @@ -1176,7 +1171,11 @@ def handle_rm_request( from_index_resolved = prebuild_info['from_index_resolved'] with tempfile.TemporaryDirectory(prefix=f'iib-{request_id}-') as temp_dir: - with set_registry_token(overwrite_from_index_token, from_index_resolved, append=True): + with set_registry_token( + payload.overwrite_from_index_token, + from_index_resolved, + append=True, + ): image_is_fbc = is_image_fbc(from_index_resolved) if image_is_fbc: @@ -1184,9 +1183,9 @@ def handle_rm_request( fbc_dir, _ = opm_registry_rm_fbc( base_dir=temp_dir, from_index=from_index_resolved, - operators=operators, + operators=payload.operators, binary_image=prebuild_info['binary_image'], - overwrite_from_index_token=overwrite_from_index_token, + overwrite_from_index_token=payload.overwrite_from_index_token, generate_cache=False, ) @@ -1197,12 +1196,16 @@ def handle_rm_request( os.makedirs(os.path.join(temp_dir, 'from_index'), exist_ok=True) # get catalog with opted-in operators - with set_registry_token(overwrite_from_index_token, from_index_resolved, append=True): + with set_registry_token( + payload.overwrite_from_index_token, + from_index_resolved, + append=True, + ): catalog_from_index = get_catalog_dir( from_index=from_index_resolved, base_dir=os.path.join(temp_dir, 'from_index') ) # remove operators from from_index file-based catalog - for operator in operators: + for operator in payload.operators: operator_path = os.path.join(catalog_from_index, operator) if os.path.exists(operator_path): log.debug('Removing operator from from_index FBC %s', operator_path) @@ -1227,10 +1230,10 @@ def handle_rm_request( else: _opm_index_rm( base_dir=temp_dir, - operators=operators, + operators=payload.operators, binary_image=prebuild_info['binary_image'], from_index=from_index_resolved, - overwrite_from_index_token=overwrite_from_index_token, + overwrite_from_index_token=payload.overwrite_from_index_token, container_tool='podman', ) @@ -1263,15 +1266,15 @@ def handle_rm_request( ) set_request_state(request_id, 'in_progress', 'Creating the manifest list') - output_pull_spec = _create_and_push_manifest_list(request_id, arches, build_tags) + output_pull_spec = _create_and_push_manifest_list(request_id, arches, payload.build_tags) _update_index_image_pull_spec( output_pull_spec, request_id, arches, - from_index, - overwrite_from_index, - overwrite_from_index_token, + payload.from_index, + payload.overwrite_from_index, + payload.overwrite_from_index_token, from_index_resolved, add_or_rm=True, ) diff --git a/iib/workers/tasks/build_create_empty_index.py b/iib/workers/tasks/build_create_empty_index.py index 413bba100..b1605378e 100644 --- a/iib/workers/tasks/build_create_empty_index.py +++ b/iib/workers/tasks/build_create_empty_index.py @@ -5,6 +5,7 @@ import re from typing import Dict, List, Optional +from iib.common.pydantic_models import CreateEmptyIndexPydanticModel from iib.common.tracing import instrument_tracing from iib.exceptions import IIBError from iib.workers.api_utils import set_request_state @@ -66,11 +67,8 @@ def _get_present_operators(from_index: str, base_dir: str) -> List[str]: span_name="workers.tasks.handle_create_empty_index_request", attributes=get_binary_versions() ) def handle_create_empty_index_request( - from_index: str, + payload: CreateEmptyIndexPydanticModel, request_id: int, - output_fbc: bool = False, - binary_image: Optional[str] = None, - labels: Optional[Dict[str, str]] = None, binary_image_config: Optional[Dict[str, Dict[str, str]]] = None, ) -> None: """Coordinate the the work needed to create the index image with labels. @@ -89,15 +87,15 @@ def handle_create_empty_index_request( prebuild_info: PrebuildInfo = prepare_request_for_build( request_id, RequestConfigCreateIndexImage( - _binary_image=binary_image, - from_index=from_index, + _binary_image=payload.binary_image, + from_index=payload.from_index, binary_image_config=binary_image_config, ), ) from_index_resolved = prebuild_info['from_index_resolved'] - prebuild_info['labels'] = labels + prebuild_info['labels'] = payload.labels - if not output_fbc and is_image_fbc(from_index_resolved): + if not payload.output_fbc and is_image_fbc(from_index_resolved): log.debug('%s is FBC index image', from_index_resolved) err_msg = 'Cannot create SQLite index image from File-Based Catalog index image' log.error(err_msg) @@ -112,13 +110,13 @@ def handle_create_empty_index_request( # if output_fbc parameter is true, create an empty FBC index image # else create empty SQLite index image - if output_fbc: - log.debug('Creating empty FBC index image from %s', from_index) + if payload.output_fbc: + log.debug('Creating empty FBC index image from %s', payload.from_index) opm_create_empty_fbc( request_id=request_id, temp_dir=temp_dir, from_index_resolved=from_index_resolved, - from_index=from_index, + from_index=payload.from_index, binary_image=prebuild_info['binary_image'], operators=operators, ) @@ -141,8 +139,8 @@ def handle_create_empty_index_request( 'com.redhat.index.delivery.distribution_scope': prebuild_info['distribution_scope'], } - if labels: - iib_labels.update(labels) + if payload.labels: + iib_labels.update(payload.labels) for index_label, value in iib_labels.items(): _add_label_to_index(index_label, value, temp_dir, 'index.Dockerfile') @@ -159,7 +157,7 @@ def handle_create_empty_index_request( output_pull_spec=output_pull_spec, request_id=request_id, arches=arches, - from_index=from_index, + from_index=payload.from_index, resolved_prebuild_from_index=from_index_resolved, ) _cleanup() diff --git a/iib/workers/tasks/build_fbc_operations.py b/iib/workers/tasks/build_fbc_operations.py index 1c4639873..48b79d8f5 100644 --- a/iib/workers/tasks/build_fbc_operations.py +++ b/iib/workers/tasks/build_fbc_operations.py @@ -1,9 +1,10 @@ # SPDX-License-Identifier: GPL-3.0-or-later import logging import tempfile -from typing import Dict, Optional, Set +from typing import Dict, Optional from iib.common.common_utils import get_binary_versions +from iib.common.pydantic_models import FbcOperationsPydanticModel from iib.common.tracing import instrument_tracing from iib.workers.api_utils import set_request_state from iib.workers.tasks.build import ( @@ -36,15 +37,8 @@ span_name="workers.tasks.build.handle_fbc_operation_request", attributes=get_binary_versions() ) def handle_fbc_operation_request( + payload: FbcOperationsPydanticModel, request_id: int, - fbc_fragment: str, - from_index: Optional[str] = None, - binary_image: Optional[str] = None, - distribution_scope: Optional[str] = None, - overwrite_from_index: bool = False, - overwrite_from_index_token: Optional[str] = None, - build_tags: Optional[Set[str]] = None, - add_arches: Optional[Set[str]] = None, binary_image_config: Optional[Dict[str, Dict[str, str]]] = None, ) -> None: """ @@ -63,18 +57,18 @@ def handle_fbc_operation_request( _cleanup() set_request_state(request_id, 'in_progress', 'Resolving the fbc fragment') - with set_registry_token(overwrite_from_index_token, fbc_fragment, append=True): - resolved_fbc_fragment = get_resolved_image(fbc_fragment) + with set_registry_token(payload.overwrite_from_index_token, payload.fbc_fragment, append=True): + resolved_fbc_fragment = get_resolved_image(payload.fbc_fragment) prebuild_info = prepare_request_for_build( request_id, RequestConfigFBCOperation( - _binary_image=binary_image, - from_index=from_index, - overwrite_from_index_token=overwrite_from_index_token, - add_arches=add_arches, - fbc_fragment=fbc_fragment, - distribution_scope=distribution_scope, + _binary_image=payload.binary_image, + from_index=payload.from_index, + overwrite_from_index_token=payload.overwrite_from_index_token, + add_arches=payload.add_arches, + fbc_fragment=payload.fbc_fragment, + distribution_scope=payload.distribution_scope, binary_image_config=binary_image_config, ), ) @@ -93,7 +87,7 @@ def handle_fbc_operation_request( from_index_resolved, binary_image_resolved, resolved_fbc_fragment, - overwrite_from_index_token, + payload.overwrite_from_index_token, ) _add_label_to_index( @@ -116,15 +110,15 @@ def handle_fbc_operation_request( _push_image(request_id, arch) set_request_state(request_id, 'in_progress', 'Creating the manifest list') - output_pull_spec = _create_and_push_manifest_list(request_id, arches, build_tags) + output_pull_spec = _create_and_push_manifest_list(request_id, arches, payload.build_tags) _update_index_image_pull_spec( output_pull_spec, request_id, arches, - from_index, - overwrite_from_index, - overwrite_from_index_token, + payload.from_index, + payload.overwrite_from_index, + payload.overwrite_from_index_token, from_index_resolved, add_or_rm=True, ) diff --git a/iib/workers/tasks/build_merge_index_image.py b/iib/workers/tasks/build_merge_index_image.py index 153cdbc34..10cc9ecec 100644 --- a/iib/workers/tasks/build_merge_index_image.py +++ b/iib/workers/tasks/build_merge_index_image.py @@ -17,6 +17,7 @@ ) from packaging.version import Version +from iib.common.pydantic_models import MergeIndexImagePydanticModel from iib.exceptions import IIBError from iib.workers.api_utils import set_request_state from iib.workers.tasks.build import ( @@ -197,18 +198,9 @@ def _add_bundles_missing_in_source( span_name="workers.tasks.build.handle_merge_request", attributes=get_binary_versions() ) def handle_merge_request( - source_from_index: str, - deprecation_list: List[str], + payload: MergeIndexImagePydanticModel, request_id: int, - binary_image: Optional[str] = None, - target_index: Optional[str] = None, - overwrite_target_index: bool = False, - overwrite_target_index_token: Optional[str] = None, - distribution_scope: Optional[str] = None, binary_image_config: Optional[str] = None, - build_tags: Optional[List[str]] = None, - graph_update_mode: Optional[str] = None, - ignore_bundle_ocp_version: Optional[bool] = False, ) -> None: """ Coordinate the work needed to merge old (N) index image with new (N+1) index image. @@ -237,15 +229,19 @@ def handle_merge_request( :raises IIBError: if the index image merge fails. """ _cleanup() - with set_registry_token(overwrite_target_index_token, target_index, append=True): + with set_registry_token( + payload.overwrite_target_index_token, + payload.target_index, + append=True, + ): prebuild_info = prepare_request_for_build( request_id, RequestConfigMerge( - _binary_image=binary_image, - overwrite_target_index_token=overwrite_target_index_token, - source_from_index=source_from_index, - target_index=target_index, - distribution_scope=distribution_scope, + _binary_image=payload.binary_image, + overwrite_target_index_token=payload.overwrite_target_index_token, + source_from_index=payload.source_from_index, + target_index=payload.target_index, + distribution_scope=payload.distribution_scope, binary_image_config=binary_image_config, ), ) @@ -255,7 +251,11 @@ def handle_merge_request( dockerfile_name = 'index.Dockerfile' with tempfile.TemporaryDirectory(prefix=f'iib-{request_id}-') as temp_dir: - with set_registry_token(overwrite_target_index_token, target_index, append=True): + with set_registry_token( + payload.overwrite_target_index_token, + payload.target_index, + append=True, + ): source_fbc = is_image_fbc(source_from_index_resolved) target_fbc = is_image_fbc(target_index_resolved) @@ -275,16 +275,20 @@ def handle_merge_request( set_request_state(request_id, 'in_progress', 'Getting bundles present in the index images') log.info('Getting bundles present in the source index image') - with set_registry_token(overwrite_target_index_token, target_index, append=True): + with set_registry_token( + payload.overwrite_target_index_token, + payload.target_index, + append=True, + ): source_index_bundles, source_index_bundles_pull_spec = _get_present_bundles( source_from_index_resolved, temp_dir ) target_index_bundles: List[BundleImage] = [] - if target_index: + if payload.target_index: log.info('Getting bundles present in the target index image') with set_registry_token( - overwrite_target_index_token, target_index_resolved, append=True + payload.overwrite_target_index_token, target_index_resolved, append=True ): target_index_bundles, _ = _get_present_bundles(target_index_resolved, temp_dir) @@ -300,11 +304,11 @@ def handle_merge_request( request_id=request_id, arch=arch, ocp_version=prebuild_info['target_ocp_version'], - graph_update_mode=graph_update_mode, - target_index=target_index, - overwrite_target_index_token=overwrite_target_index_token, + graph_update_mode=payload.graph_update_mode, + target_index=payload.target_index, + overwrite_target_index_token=payload.overwrite_target_index_token, distribution_scope=prebuild_info['distribution_scope'], - ignore_bundle_ocp_version=ignore_bundle_ocp_version, + ignore_bundle_ocp_version=payload.ignore_bundle_ocp_version, ) missing_bundle_paths = [bundle['bundlePath'] for bundle in missing_bundles] @@ -314,7 +318,7 @@ def handle_merge_request( log.info('Deprecating bundles in the deprecation list') intermediate_bundles = missing_bundle_paths + source_index_bundles_pull_spec deprecation_bundles = get_bundles_from_deprecation_list( - intermediate_bundles, deprecation_list + intermediate_bundles, payload.deprecation_list ) # We do not need to pass the invalid_version_bundles through the # get_bundles_from_deprecation_list function because we already know @@ -351,7 +355,7 @@ def handle_merge_request( base_dir=temp_dir, binary_image=prebuild_info['binary_image'], from_index=intermediate_image_name, - overwrite_target_index_token=overwrite_target_index_token, + overwrite_target_index_token=payload.overwrite_target_index_token, ) if target_fbc: @@ -401,15 +405,15 @@ def handle_merge_request( ) output_pull_spec = _create_and_push_manifest_list( - request_id, prebuild_info['arches'], build_tags + request_id, prebuild_info['arches'], payload.build_tags ) _update_index_image_pull_spec( output_pull_spec, request_id, prebuild_info['arches'], - target_index, - overwrite_target_index, - overwrite_target_index_token, + payload.target_index, + payload.overwrite_target_index, + payload.overwrite_target_index_token, target_index_resolved, ) _cleanup() diff --git a/iib/workers/tasks/build_recursive_related_bundles.py b/iib/workers/tasks/build_recursive_related_bundles.py index 535041207..494a1825a 100644 --- a/iib/workers/tasks/build_recursive_related_bundles.py +++ b/iib/workers/tasks/build_recursive_related_bundles.py @@ -3,12 +3,13 @@ import logging import os import tempfile -from typing import Any, Dict, List, Optional +from typing import List, Optional from operator_manifest.operator import OperatorManifest import ruamel.yaml from iib.common.common_utils import get_binary_versions +from iib.common.pydantic_models import RecursiveRelatedBundlesPydanticModel from iib.common.tracing import instrument_tracing from iib.exceptions import IIBError from iib.workers.api_utils import set_request_state, update_request @@ -55,10 +56,8 @@ attributes=get_binary_versions(), ) def handle_recursive_related_bundles_request( - parent_bundle_image: str, - organization: str, + payload: RecursiveRelatedBundlesPydanticModel, request_id: int, - registry_auths: Optional[Dict[str, Any]] = None, ) -> None: """ Coordinate the work needed to find recursive related bundles of the operator bundle image. @@ -75,17 +74,17 @@ def handle_recursive_related_bundles_request( set_request_state(request_id, 'in_progress', 'Resolving parent_bundle_image') - with set_registry_auths(registry_auths): - parent_bundle_image_resolved = get_resolved_image(parent_bundle_image) + with set_registry_auths(payload.registry_auths): + parent_bundle_image_resolved = get_resolved_image(payload.parent_bundle_image) - payload: UpdateRequestPayload = { + update_payload: UpdateRequestPayload = { 'parent_bundle_image_resolved': parent_bundle_image_resolved, 'state': 'in_progress', 'state_reason': ( - f'Finding recursive related bundles for the bundle: {parent_bundle_image}' + f'Finding recursive related bundles for the bundle: {payload.parent_bundle_image}' ), } - update_request(request_id, payload) + update_request(request_id, update_payload) recursive_related_bundles = [parent_bundle_image_resolved] current_level_related_bundles = [parent_bundle_image_resolved] @@ -97,7 +96,7 @@ def handle_recursive_related_bundles_request( current_level_related_bundles = [] for bundle in temp_current_level_related_bundles: children_related_bundles = process_parent_bundle_image( - bundle, request_id, organization + bundle, request_id, payload.organization ) current_level_related_bundles.extend(children_related_bundles) @@ -109,11 +108,15 @@ def handle_recursive_related_bundles_request( if not current_level_related_bundles: traversal_completed = True - payload = { + update_payload = { 'state': 'in_progress', 'state_reason': 'Writing recursive related bundles to a file', } - update_request(request_id, payload, exc_msg='Failed setting the bundle image on the request') + update_request( + request_id, + update_payload, + exc_msg='Failed setting the bundle image on the request', + ) # Reverse the list while writing because we did a top to bottom level traversal of a tree. # The return value should be a bottom to top level traversal. write_related_bundles_file( @@ -123,12 +126,16 @@ def handle_recursive_related_bundles_request( 'recursive_related_bundles', ) - payload = { + update_payload = { 'state': 'complete', 'state_reason': 'The request completed successfully', } _cleanup() - update_request(request_id, payload, exc_msg='Failed setting the bundle image on the request') + update_request( + request_id, + update_payload, + exc_msg='Failed setting the bundle image on the request', + ) def process_parent_bundle_image( diff --git a/iib/workers/tasks/build_regenerate_bundle.py b/iib/workers/tasks/build_regenerate_bundle.py index 653500535..b42a2c138 100644 --- a/iib/workers/tasks/build_regenerate_bundle.py +++ b/iib/workers/tasks/build_regenerate_bundle.py @@ -10,6 +10,7 @@ import ruamel.yaml from iib.common.common_utils import get_binary_versions +from iib.common.pydantic_models import RegenerateBundlePydanticModel from iib.common.tracing import instrument_tracing from iib.exceptions import IIBError from iib.workers.s3_utils import upload_file_to_s3_bucket @@ -58,11 +59,8 @@ attributes=get_binary_versions(), ) def handle_regenerate_bundle_request( - from_bundle_image: str, - organization: str, + payload: RegenerateBundlePydanticModel, request_id: int, - registry_auths: Optional[Dict[str, Any]] = None, - bundle_replacements: Optional[Dict[str, str]] = {}, ) -> None: """ Coordinate the work needed to regenerate the operator bundle image. @@ -80,8 +78,8 @@ def handle_regenerate_bundle_request( set_request_state(request_id, 'in_progress', 'Resolving from_bundle_image') - with set_registry_auths(registry_auths): - from_bundle_image_resolved = get_resolved_image(from_bundle_image) + with set_registry_auths(payload.registry_auths): + from_bundle_image_resolved = get_resolved_image(payload.from_bundle_image) arches: Set[str] = get_image_arches(from_bundle_image_resolved) if not arches: @@ -97,13 +95,13 @@ def handle_regenerate_bundle_request( arches_str = ', '.join(sorted(arches)) log.debug('Set to regenerate the bundle image for the following arches: %s', arches_str) - payload: UpdateRequestPayload = { + update_payload: UpdateRequestPayload = { 'from_bundle_image_resolved': from_bundle_image_resolved, 'state': 'in_progress', 'state_reason': f'Regenerating the bundle image for the following arches: {arches_str}', } exc_msg = 'Failed setting the resolved "from_bundle_image" on the request' - update_request(request_id, payload, exc_msg=exc_msg) + update_request(request_id, update_payload, exc_msg=exc_msg) # Pull the from_bundle_image to ensure steps later on don't fail due to registry timeouts podman_pull(from_bundle_image_resolved) @@ -117,9 +115,9 @@ def handle_regenerate_bundle_request( manifests_path, metadata_path, request_id, - organization=organization, + organization=payload.organization, pinned_by_iib=pinned_by_iib, - bundle_replacements=bundle_replacements, + bundle_replacements=payload.bundle_replacements, ) with open(os.path.join(temp_dir, 'Dockerfile'), 'w') as dockerfile: @@ -154,14 +152,18 @@ def handle_regenerate_bundle_request( output_pull_spec, ) - payload = { + update_payload = { 'arches': list(arches), 'bundle_image': output_pull_spec, 'state': 'complete', 'state_reason': 'The request completed successfully', } _cleanup() - update_request(request_id, payload, exc_msg='Failed setting the bundle image on the request') + update_request( + request_id, + update_payload, + exc_msg='Failed setting the bundle image on the request', + ) def _apply_package_name_suffix( diff --git a/requirements.txt b/requirements.txt index d4760942d..acd40369e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -960,6 +960,8 @@ typing-extensions==4.10.0 \ # iib (setup.py) # kombu # opentelemetry-sdk + # pydantic + # pydantic-core # sqlalchemy tzdata==2024.1 \ --hash=sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd \ diff --git a/setup.py b/setup.py index e17d9eef5..0d46ec03c 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,8 @@ 'tenacity', 'typing-extensions', 'packaging', + 'pydantic', + 'annotated-types', 'opentelemetry-api', 'opentelemetry-sdk', 'opentelemetry-exporter-otlp', diff --git a/tests/test_web/test_migrations.py b/tests/test_web/test_migrations.py index 85e38a16f..d7abad139 100644 --- a/tests/test_web/test_migrations.py +++ b/tests/test_web/test_migrations.py @@ -5,13 +5,20 @@ import pytest from iib.web.models import ( + Request, RequestAdd, RequestMergeIndexImage, RequestRegenerateBundle, RequestRm, RequestCreateEmptyIndex, ) - +from iib.common.pydantic_models import ( + AddPydanticModel, + RmPydanticModel, + MergeIndexImagePydanticModel, + RegenerateBundlePydanticModel, + CreateEmptyIndexPydanticModel, +) INITIAL_DB_REVISION = '274ba38408e8' @@ -25,19 +32,19 @@ def test_migrate_to_polymorphic_requests(app, auth_env, client, db): for i in range(total_requests): request_class = random.choice((RequestAdd, RequestRm)) if request_class == RequestAdd: - data = { - 'binary_image': 'quay.io/namespace/binary_image:latest', - 'bundles': [f'quay.io/namespace/bundle:{i}'], - 'from_index': f'quay.io/namespace/repo:{i}', - } - request = RequestAdd.from_json(data) + data = AddPydanticModel( + binary_image='quay.io/namespace/binary_image:latest', + bundles=[f'quay.io/namespace/bundle:{i}'], + from_index=f'quay.io/namespace/repo:{i}', + ) + request = RequestAdd.from_json_replacement(payload=data) elif request_class == RequestRm: - data = { - 'binary_image': 'quay.io/namespace/binary_image:latest', - 'operators': [f'operator-{i}'], - 'from_index': f'quay.io/namespace/repo:{i}', - } - request = RequestRm.from_json(data) + data = RmPydanticModel( + binary_image='quay.io/namespace/binary_image:latest', + operators=[f'operator-{i}'], + from_index=f'quay.io/namespace/repo:{i}', + ) + request = RequestRm.from_json_replacement(data) if i % 5 == 0: # Simulate failed request @@ -63,26 +70,26 @@ def test_migrate_to_merge_index_endpoints(app, auth_env, client, db): for i in range(total_requests): request_class = random.choice((RequestAdd, RequestMergeIndexImage, RequestRm)) if request_class == RequestAdd: - data = { - 'binary_image': 'quay.io/namespace/binary_image:latest', - 'bundles': [f'quay.io/namespace/bundle:{i}'], - 'from_index': f'quay.io/namespace/repo:{i}', - } - request = RequestAdd.from_json(data) + data = AddPydanticModel( + binary_image='quay.io/namespace/binary_image:latest', + bundles=[f'quay.io/namespace/bundle:{i}'], + from_index=f'quay.io/namespace/repo:{i}', + ) + request = RequestAdd.from_json_replacement(data) elif request_class == RequestRm: - data = { - 'binary_image': 'quay.io/namespace/binary_image:latest', - 'operators': [f'operator-{i}'], - 'from_index': f'quay.io/namespace/repo:{i}', - } - request = RequestRm.from_json(data) + data = RmPydanticModel( + binary_image='quay.io/namespace/binary_image:latest', + operators=[f'operator-{i}'], + from_index=f'quay.io/namespace/repo:{i}', + ) + request = RequestRm.from_json_replacement(data) elif request_class == RequestMergeIndexImage: - data = { - 'source_from_index': f'quay.io/namespace/repo:{i}', - 'target_index': f'quay.io/namespace/repo:{i}', - 'binary_image': 'quay.io/namespace/binary_image:latest', - } - request = RequestMergeIndexImage.from_json(data) + data = MergeIndexImagePydanticModel( + source_from_index=f'quay.io/namespace/repo:{i}', + target_index=f'quay.io/namespace/repo:{i}', + binary_image='quay.io/namespace/binary_image:latest', + ) + request = RequestMergeIndexImage.from_json_replacement(data) if i % 5 == 0: # Simulate failed request @@ -104,31 +111,35 @@ def test_abort_when_downgrading_from_regenerate_bundle_request(app, auth_env, cl # flask_login.current_user is used in Request*.from_json which requires a request context with app.test_request_context(environ_base=auth_env): # Always add a RequestRegenerateBundle to ensure sufficient test data is available - data = {'from_bundle_image': 'quay.io/namespace/bundle-image:latest'} - request = RequestRegenerateBundle.from_json(data) + data = RegenerateBundlePydanticModel( + from_bundle_image='quay.io/namespace/bundle-image:latest' + ) + request = RequestRegenerateBundle.from_json_replacement(data) db.session.add(request) # One request was already added, let's add the remaining ones for i in range(total_requests - 1): request_class = random.choice((RequestAdd, RequestRm, RequestRegenerateBundle)) if request_class == RequestAdd: - data = { - 'binary_image': 'quay.io/namespace/binary_image:latest', - 'bundles': [f'quay.io/namespace/bundle:{i}'], - 'from_index': f'quay.io/namespace/repo:{i}', - } - request = RequestAdd.from_json(data) + data = AddPydanticModel( + binary_image='quay.io/namespace/binary_image:latest', + bundles=[f'quay.io/namespace/bundle:{i}'], + from_index=f'quay.io/namespace/repo:{i}', + ) + request = RequestAdd.from_json_replacement(data) elif request_class == RequestRm: - data = { - 'binary_image': 'quay.io/namespace/binary_image:latest', - 'operators': [f'operator-{i}'], - 'from_index': f'quay.io/namespace/repo:{i}', - } - request = RequestRm.from_json(data) + data = RmPydanticModel( + binary_image='quay.io/namespace/binary_image:latest', + operators=[f'operator-{i}'], + from_index=f'quay.io/namespace/repo:{i}', + ) + request = RequestRm.from_json_replacement(data) else: - data = {'from_bundle_image': 'quay.io/namespace/bundle-image:latest'} - request = RequestRegenerateBundle.from_json(data) + data = RegenerateBundlePydanticModel( + from_bundle_image='quay.io/namespace/bundle-image:latest' + ) + request = RequestRegenerateBundle.from_json_replacement(data) db.session.add(request) db.session.commit() @@ -148,35 +159,35 @@ def test_create_empty_index_image_request(app, auth_env, client, db): # which requires a request context with app.test_request_context(environ_base=auth_env): # Generate some data to verify migration - data = { - 'from_index': 'quay.io/namespace/index_image:latest', - 'binary_image': 'quay.io/namespace/binary_image:latest', - } - request = RequestCreateEmptyIndex.from_json(data) + data = CreateEmptyIndexPydanticModel( + from_index='quay.io/namespace/index_image:latest', + binary_image='quay.io/namespace/binary_image:latest', + ) + request = RequestCreateEmptyIndex.from_json_replacement(data) db.session.add(request) for i in range(total_requests): request_class = random.choice((RequestAdd, RequestRm, RequestCreateEmptyIndex)) if request_class == RequestAdd: - data = { - 'binary_image': 'quay.io/namespace/binary_image:latest', - 'bundles': [f'quay.io/namespace/bundle:{i}'], - 'from_index': f'quay.io/namespace/repo:{i}', - } - request = RequestAdd.from_json(data) + data = AddPydanticModel( + binary_image='quay.io/namespace/binary_image:latest', + bundles=[f'quay.io/namespace/bundle:{i}'], + from_index=f'quay.io/namespace/repo:{i}', + ) + request = RequestAdd.from_json_replacement(data) elif request_class == RequestRm: - data = { - 'binary_image': 'quay.io/namespace/binary_image:latest', - 'operators': [f'operator-{i}'], - 'from_index': f'quay.io/namespace/repo:{i}', - } - request = RequestRm.from_json(data) + data = RmPydanticModel( + binary_image='quay.io/namespace/binary_image:latest', + operators=[f'operator-{i}'], + from_index=f'quay.io/namespace/repo:{i}', + ) + request = RequestRm.from_json_replacement(data) elif request_class == RequestCreateEmptyIndex: - data = { - 'from_index': f'quay.io/namespace/index_image:{i}', - 'binary_image': 'quay.io/namespace/binary_image:latest', - } - request = RequestCreateEmptyIndex.from_json(data) + data = CreateEmptyIndexPydanticModel( + from_index=f'quay.io/namespace/index_image:{i}', + binary_image='quay.io/namespace/binary_image:latest', + ) + request = RequestCreateEmptyIndex.from_json_replacement(data) if i % 5 == 0: # Simulate failed request diff --git a/tests/test_workers/test_tasks/test_build.py b/tests/test_workers/test_tasks/test_build.py index de178f148..e72866c6b 100644 --- a/tests/test_workers/test_tasks/test_build.py +++ b/tests/test_workers/test_tasks/test_build.py @@ -12,6 +12,7 @@ from iib.workers.tasks import build from iib.workers.tasks.utils import RequestConfigAddRm from iib.workers.config import get_worker_config +from iib.common.pydantic_models import AddPydanticModel, RmPydanticModel from operator_manifest.operator import ImageName worker_config = get_worker_config() @@ -622,7 +623,9 @@ def test_buildah_fail_max_retries(mock_run_cmd: mock.MagicMock) -> None: @mock.patch('iib.workers.tasks.build._get_present_bundles') @mock.patch('iib.workers.tasks.build.set_registry_token') @mock.patch('iib.workers.tasks.build.is_image_fbc') +@mock.patch('iib.common.pydantic_models.binary_image_check') def test_handle_add_request( + mock_binary_image_check, mock_iifbc, mock_srt, mock_gpb, @@ -692,24 +695,24 @@ def side_effect(*args, base_dir, **kwargs): mock_ors.return_value = (port, my_mock) mock_run_cmd.return_value = '{"packageName": "package1", "version": "v1.0", \ "bundlePath": "bundle1"\n}' - - build.handle_add_request( - bundles, - 3, - binary_image, - 'from-index:latest', - ['s390x'], - cnr_token, - organization, - force_backport, - False, - None, - None, - greenwave_config, - binary_image_config=binary_image_config, + add_pydantic_model = AddPydanticModel.model_construct( + bundles=bundles, + binary_image=binary_image, + from_index='from_index:latest', + cnr_token=cnr_token, + organization=organization, + force_backport=force_backport, + overwrite_from_index=False, + overwrite_from_index_token=None, deprecation_list=deprecation_list, build_tags=["extra_tag1", "extra_tag2"], ) + build.handle_add_request( + payload=add_pydantic_model, + request_id=3, + greenwave_config=greenwave_config, + binary_image_config=binary_image_config, + ) mock_ors.assert_called_once() mock_run_cmd.assert_called_once() @@ -778,21 +781,24 @@ def side_effect(*args, base_dir, **kwargs): def test_handle_add_request_raises(mock_iifbc, mock_runcmd, mock_c): mock_iifbc.return_value = True with pytest.raises(IIBError): - build.handle_add_request( + add_pydantic_model = AddPydanticModel.model_construct( bundles=['some-bundle:2.3-1', 'some-deprecation-bundle:1.1-1'], - request_id=3, binary_image='binary-image:latest', - from_index='from-index:latest', add_arches=['s390x'], + from_index='from_index:latest', cnr_token='token', organization='org', force_backport=True, overwrite_from_index=False, overwrite_from_index_token=None, distribution_scope=None, + deprecation_list=[], + ) + build.handle_add_request( + payload=add_pydantic_model, + request_id=3, greenwave_config={'some_key': 'other_value'}, binary_image_config={'prod': {'v4.5': 'some_image'}}, - deprecation_list=[], ) @@ -897,21 +903,24 @@ def deprecate_bundles_mock(*args, **kwargs): ] mock_sqlite.execute.return_value = 200 + add_pydantic_model = AddPydanticModel.model_construct( + bundles=bundles, + binary_image='binary-image:latest', + add_arches=['s390x'], + from_index='from_index:latest', + cnr_token=cnr_token, + organization=organization, + force_backport=True, + overwrite_from_index=False, + overwrite_from_index_token=None, + distribution_scope=None, + deprecation_list=deprecation_list, + ) build.handle_add_request( - bundles, - 3, - 'binary-image:latest', - 'from-index:latest', - ['s390x'], - cnr_token, - organization, - True, - False, - None, - None, - greenwave_config, + payload=add_pydantic_model, + request_id=3, + greenwave_config=greenwave_config, binary_image_config=binary_image_config, - deprecation_list=deprecation_list, ) mock_ors.assert_called_once() @@ -983,19 +992,21 @@ def test_handle_add_request_gating_failure( organization = 'org' greenwave_config = {'some_key': 'other_value'} with pytest.raises(IIBError, match=error_msg): + add_pydantic_model = AddPydanticModel.model_construct( + bundles=bundles, + binary_image='binary-image:latest', + add_arches=['s390x'], + from_index='from_index:latest', + cnr_token=cnr_token, + organization=organization, + overwrite_from_index=False, + overwrite_from_index_token=None, + distribution_scope=None, + ) build.handle_add_request( - bundles, - 'binary-image:latest', - 3, - 'from-index:latest', - ['s390x'], - cnr_token, - organization, - None, - False, - None, - None, - greenwave_config, + payload=add_pydantic_model, + request_id=3, + greenwave_config=greenwave_config, ) assert mock_cleanup.call_count == 1 mock_srs2.assert_called_once() @@ -1014,17 +1025,20 @@ def test_handle_add_request_bundle_resolution_failure(mock_grb, mock_srs, mock_c organization = 'org' greenwave_config = {'some_key': 'other_value'} with pytest.raises(IIBError, match=error_msg): + add_pydantic_model = AddPydanticModel.model_construct( + bundles=bundles, + binary_image='binary-image:latest', + add_arches=['s390x'], + from_index='from_index:latest', + cnr_token=cnr_token, + organization=organization, + force_backport=False, + overwrite_from_index=False, + overwrite_from_index_token=None, + ) build.handle_add_request( - bundles, - 'binary-image:latest', - 3, - 'from-index:latest', - ['s390x'], - cnr_token, - organization, - False, - False, - None, + payload=add_pydantic_model, + request_id=3, greenwave_config=greenwave_config, ) assert mock_cleanup.call_count == 1 @@ -1073,11 +1087,14 @@ def test_handle_rm_request( 'distribution_scope': 'PROD', } binary_image_config = {'prod': {'v4.6': 'some_image'}} + rm_pydantic_model = RmPydanticModel.model_construct( + operators=['some_operator'], + from_index='from-index:latest', + binary_image=binary_image, + ) build.handle_rm_request( - ['some-operator'], - 3, - 'from-index:latest', - binary_image, + payload=rm_pydantic_model, + request_id=3, binary_image_config=binary_image_config, ) @@ -1162,11 +1179,14 @@ def test_handle_rm_request_fbc( mock_om.return_value = "/tmp/xyz/catalog" mock_orrf.return_value = "/tmp/fbc_dir", "/tmp/cache_dir" mock_gcd.return_value = "/some/path" - build.handle_rm_request( - operators=['some-operator'], - request_id=5, + rm_pydantic_model = RmPydanticModel.model_construct( + operators=['some_operator'], from_index='from-index:latest', binary_image='binary-image:latest', + ) + build.handle_rm_request( + payload=rm_pydantic_model, + request_id=5, binary_image_config={'prod': {'v4.6': 'some_image'}}, ) mock_prfb.assert_called_once_with( @@ -1446,9 +1466,8 @@ def test_handle_add_request_check_related_images_fail( mock_grb.return_value = ['some-bundle@sha256:123'] mock_iri.side_effect = IIBError(error_msg) with pytest.raises(IIBError, match=re.escape(error_msg)): - build.handle_add_request( + add_pydantic_model = AddPydanticModel.model_construct( bundles=bundles, - request_id=3, binary_image='binary-image:latest', from_index='from-index:latest', add_arches=['s390x'], @@ -1458,13 +1477,17 @@ def test_handle_add_request_check_related_images_fail( overwrite_from_index=False, overwrite_from_index_token=None, distribution_scope=None, - greenwave_config=None, - binary_image_config={'prod': {'v4.5': 'some_image'}}, deprecation_list=[], build_tags=None, graph_update_mode=None, check_related_images=True, ) + build.handle_add_request( + payload=add_pydantic_model, + request_id=3, + greenwave_config=None, + binary_image_config={'prod': {'v4.5': 'some_image'}}, + ) assert mock_cleanup.call_count == 1 mock_srs.assert_called_once() mock_grb.assert_called_once_with(bundles) diff --git a/tests/test_workers/test_tasks/test_build_create_empty_index.py b/tests/test_workers/test_tasks/test_build_create_empty_index.py index 5b06b07be..1b8f0c64f 100644 --- a/tests/test_workers/test_tasks/test_build_create_empty_index.py +++ b/tests/test_workers/test_tasks/test_build_create_empty_index.py @@ -6,6 +6,7 @@ from iib.exceptions import IIBError from iib.workers.tasks import build_create_empty_index from iib.workers.tasks.utils import RequestConfigCreateIndexImage +from iib.common.pydantic_models import CreateEmptyIndexPydanticModel @mock.patch('iib.workers.tasks.build_create_empty_index.grpcurl_get_db_data') @@ -81,12 +82,15 @@ def test_handle_create_empty_index_request( output_pull_spec = 'quay.io/namespace/some-image:3' mock_capml.return_value = output_pull_spec - build_create_empty_index.handle_create_empty_index_request( + create_empty_index_pydantic_model = CreateEmptyIndexPydanticModel.model_construct( from_index=from_index, - request_id=3, output_fbc=False, binary_image=binary_image, labels=labels, + ) + build_create_empty_index.handle_create_empty_index_request( + payload=create_empty_index_pydantic_model, + request_id=3, binary_image_config=binary_image_config, ) @@ -141,12 +145,16 @@ def test_handle_create_empty_index_request_raises(mock_prfb, mock_iifbc, mock_c) IIBError, match=('Cannot create SQLite index image from File-Based Catalog index image') ): mock_iifbc.return_value = True - build_create_empty_index.handle_create_empty_index_request( + + create_empty_index_pydantic_model = CreateEmptyIndexPydanticModel.model_construct( from_index=from_index, - request_id=3, output_fbc=False, binary_image=binary_image, labels={"version": "v4.5"}, + ) + build_create_empty_index.handle_create_empty_index_request( + payload=create_empty_index_pydantic_model, + request_id=3, binary_image_config={'prod': {'v4.5': 'some_image'}}, ) @@ -194,12 +202,15 @@ def test_handle_create_empty_index_request_fbc( output_pull_spec = 'quay.io/namespace/some-image:3' mock_capml.return_value = output_pull_spec - build_create_empty_index.handle_create_empty_index_request( + create_empty_index_pydantic_model = CreateEmptyIndexPydanticModel.model_construct( from_index=from_index, - request_id=3, output_fbc=True, binary_image=binary_image, labels={"version": "v4.5"}, + ) + build_create_empty_index.handle_create_empty_index_request( + payload=create_empty_index_pydantic_model, + request_id=3, binary_image_config={'prod': {'v4.5': 'some_image'}}, ) diff --git a/tests/test_workers/test_tasks/test_build_fbc_operations.py b/tests/test_workers/test_tasks/test_build_fbc_operations.py index 3488fbc44..7b835f26d 100644 --- a/tests/test_workers/test_tasks/test_build_fbc_operations.py +++ b/tests/test_workers/test_tasks/test_build_fbc_operations.py @@ -3,6 +3,7 @@ from iib.workers.tasks import build_fbc_operations from iib.workers.tasks.utils import RequestConfigFBCOperation +from iib.common.pydantic_models import FbcOperationsPydanticModel @mock.patch('iib.workers.tasks.build_fbc_operations._update_index_image_pull_spec') @@ -48,11 +49,15 @@ def test_handle_fbc_operation_request( } mock_gri.return_value = 'fbc-fragment@sha256:qwerty' - build_fbc_operations.handle_fbc_operation_request( - request_id=request_id, - fbc_fragment=fbc_fragment, + fbc_operations_pydantic_model = FbcOperationsPydanticModel.model_construct( from_index=from_index, binary_image=binary_image, + fbc_fragment=fbc_fragment, + + ) + build_fbc_operations.handle_fbc_operation_request( + payload=fbc_operations_pydantic_model, + request_id=request_id, binary_image_config=binary_image_config, ) mock_prfb.assert_called_once_with(