diff --git a/README.md b/README.md index 3f5f6b2..7834753 100644 --- a/README.md +++ b/README.md @@ -37,3 +37,8 @@ To give you the path to python for your virtual env run: ``` echo "$(poetry env info | grep Path | awk '{print $2}')/bin/python" ``` + +Run unit tests: +```shell +poetry run python -m pytest +``` diff --git a/appconfig/config.py b/appconfig/config.py index ce535d4..3c75687 100644 --- a/appconfig/config.py +++ b/appconfig/config.py @@ -6,21 +6,14 @@ class Config: blaise_api_url: str blaise_server_park: str - project_id: str - topic_name: str - env: str @classmethod def from_env(cls): return cls( blaise_api_url=os.getenv("BLAISE_API_URL"), blaise_server_park=os.getenv("BLAISE_SERVER_PARK"), - project_id=os.getenv("PROJECT_ID"), - topic_name=os.getenv("TOPIC_NAME"), - env=os.getenv("ENV"), ) def log(self): - print(f"Configuration: Project ID: {self.project_id}") - print(f"Configuration: Topic Name: {self.topic_name}") - print(f"Configuration: Env: {self.env}") + print(f"Configuration: Blaise API Url: {self.blaise_api_url}") + print(f"Configuration: Blaise Server Park: {self.blaise_server_park}") diff --git a/appconfig/message.py b/appconfig/message.py deleted file mode 100644 index 658ac23..0000000 --- a/appconfig/message.py +++ /dev/null @@ -1,172 +0,0 @@ -import json -import pathlib -from dataclasses import asdict, dataclass -from typing import List - -from google.cloud import pubsub_v1 - -from utils import ( - InvalidFileExtension, - InvalidFileType, - md5hash_to_md5sum, - size_in_megabytes, -) - -SUPPORTED_FILE_EXTENSIONS = [".zip"] - -SUPPORTED_FILE_TYPES = ["dd", "mi"] - - -@dataclass -class File: - name: str - sizeBytes: str - md5sum: str - relativePath: str = ".\\" - - def extension(self): - return pathlib.Path(self.filename()).suffix - - def filename(self): - return self.name.split(":")[0] - - def type(self): - return self.name.split("_")[0] - - def survey_tla(self): - return self.filename().split("_")[1][0:3].upper() - - def instrument_name(self): - file_prefix = pathlib.Path(self.filename()).stem - parsed_prefix = file_prefix.split("_")[1:] - instrument_name = [ - instrument_name_part - for instrument_name_part in parsed_prefix - if not instrument_name_part.isnumeric() - ] - return "_".join(instrument_name).upper() - - def is_lms(self): - return self.survey_tla().startswith("LM") - - def is_frs(self): - return self.survey_tla().startswith("FRS") - - @classmethod - def from_event(cls, event): - return cls( - name=f"{event['name']}:{event['bucket']}", - sizeBytes=event["size"], - md5sum=md5hash_to_md5sum(event["md5Hash"]), - ) - - -@dataclass -class Message: - files: List[File] - sourceName: str - manifestCreated: str - fullSizeMegabytes: str - version: int = 3 - schemaVersion: int = 1 - description: str = "" - dataset: str = "" - sensitivity: str = "High" - iterationL1: str = "" - iterationL2: str = "" - iterationL3: str = "" - iterationL4: str = "" - - def json(self): - return json.dumps(asdict(self)) - - def first_file(self): - return self.files[0] - - def management_information(self, config): - file = self.first_file() - self.description = ( - "Management Information files uploaded to GCP bucket from Blaise5" - ) - self.dataset = "blaise_mi" - self.iterationL1 = f"BL5-{config.env}" - self.iterationL2 = file.survey_tla() - self.iterationL3 = file.instrument_name() - return self - - def data_delivery_default(self, config): - file = self.first_file() - survey_tla = file.survey_tla() - self.description = ( - f"Data Delivery files for {survey_tla} uploaded to GCP bucket from Blaise5" - ) - self.dataset = "blaise_dde" - self.iterationL1 = "SYSTEMS" - self.iterationL2 = config.on_prem_subfolder - self.iterationL3 = survey_tla - self.iterationL4 = file.instrument_name() - return self - - def data_delivery_lms(self, config): - file = self.first_file() - survey_tla = file.survey_tla() - environment = config.env - self.description = ( - f"Data Delivery files for {survey_tla} uploaded to GCP bucket from Blaise5" - ) - self.dataset = "blaise_dde_lms" - self.iterationL1 = "CLOUD" - self.iterationL2 = environment - self.iterationL3 = file.instrument_name() - return self - - def data_delivery_frs(self, config): - file = self.first_file() - survey_tla = file.survey_tla() - environment = config.env - self.description = ( - f"Data Delivery files for {survey_tla} uploaded to GCP bucket from Blaise5" - ) - self.dataset = "blaise_dde_frs" - self.iterationL1 = "ingress" - self.iterationL2 = "survey_data" - self.iterationL3 = f"bl5-{environment}" - self.iterationL4 = file.instrument_name() - return self - - -def create_message(event, config): - file = File.from_event(event) - - msg = Message( - sourceName=f"gcp_blaise_{config.env}", - manifestCreated=event["timeCreated"], - fullSizeMegabytes=size_in_megabytes(event["size"]), - files=[file], - ) - - if file.extension() not in SUPPORTED_FILE_EXTENSIONS: - raise InvalidFileExtension( - f"File extension '{file.extension()}' is invalid, supported extensions: {SUPPORTED_FILE_EXTENSIONS}" # noqa:E501 - ) - - if file.type() == "mi": - return msg.management_information(config) - if file.type() == "dd" and file.is_lms(): - return msg.data_delivery_lms(config) - if file.type() == "dd" and file.is_frs(): - return msg.data_delivery_frs(config) - if file.type() == "dd": - return msg.data_delivery_default(config) - - raise InvalidFileType( - f"File type '{file.type()}' is invalid, supported extensions: {SUPPORTED_FILE_TYPES}" # noqa:E501 - ) - - -def send_pub_sub_message(config, message): - client = pubsub_v1.PublisherClient() - topic_path = client.topic_path(config.project_id, config.topic_name) - msg_bytes = bytes(message.json(), encoding="utf-8") - client.publish(topic_path, data=msg_bytes) - print("Message published") diff --git a/main.py b/main.py index 5147684..538484b 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,6 @@ import logging import utils -from services.ingest_service import IngestService from services.validation_service import ValidationService from appconfig.config import Config from services.blaise_service import BlaiseService @@ -10,7 +9,6 @@ BlaiseError, ConfigError, GuidError, - QuestionnaireNotFound, RequestError, UsersError, IngestError, @@ -20,7 +18,7 @@ setup_logger() -def process_zip_file(data, context): +def process_zip_file(data): try: logging.info("Running Cloud Function - 'ingest data'") validation_service = ValidationService() @@ -34,7 +32,7 @@ def process_zip_file(data, context): print(f"File {file_name} is not a zip file, skipping.") return - print(f"Processing ZIP file: {file_name}") + print(f"Processing ZIP file: {file_name} from bucket {bucket_name}") # Initialize the client storage_client = storage.Client() @@ -56,17 +54,19 @@ def process_zip_file(data, context): questionnaire_name = utils.get_questionnaire_name(file_name) - # Blaise Handler TODO: Get questionnaire_name from somewhere and validate it exists + # Blaise Handler validation_service.validate_questionnaire_exists( questionnaire_name, blaise_config ) # Ingest Handler blaise_service = BlaiseService(blaise_config) - ingest_service = IngestService(blaise_service) - logging.info(f"Calling Ingest Service with server park: {blaise_server_park} and questionnaire name: {questionnaire_name}") - ingest_service.ingest(blaise_server_park, questionnaire_name) + logging.info(f"Calling Ingest Service with " + f"server park: {blaise_server_park}, " + f"questionnaire name: {questionnaire_name}, " + f"file name: {file_name}") + blaise_service.get_ingest(blaise_server_park, questionnaire_name, file_name) logging.info("Finished Running Cloud Function - 'ingest data'") return f"Successfully ingested file from bucket", 200 @@ -83,3 +83,6 @@ def process_zip_file(data, context): error_message = f"Error occurred during Ingest: {e}" logging.error(error_message) return error_message, 500 + +# Testing +process_zip_file() \ No newline at end of file diff --git a/mypi.ini b/mypi.ini new file mode 100644 index 0000000..a6fe838 --- /dev/null +++ b/mypi.ini @@ -0,0 +1,7 @@ +[mypy] +disallow_untyped_defs = False +namespace_packages = True +exclude = "scripts/" + +[mypy-blaise_restapi.*] +ignore_missing_imports = True diff --git a/poetry.lock b/poetry.lock index 6a6a2c4..d394b46 100644 --- a/poetry.lock +++ b/poetry.lock @@ -62,7 +62,7 @@ requests = "^2.26.0" type = "git" url = "https://github.com/ONSdigital/blaise-api-python-client" reference = "BLAIS5-4597" -resolved_reference = "08c718de643703157e7e3f973d4d9d7b336ec5a4" +resolved_reference = "c2ae4ef1e0afacb42cbf6ac7b66674e3032b44c9" [[package]] name = "blinker" @@ -694,85 +694,85 @@ protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4 [[package]] name = "grpcio" -version = "1.69.0" +version = "1.70.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.8" files = [ - {file = "grpcio-1.69.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:2060ca95a8db295ae828d0fc1c7f38fb26ccd5edf9aa51a0f44251f5da332e97"}, - {file = "grpcio-1.69.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:2e52e107261fd8fa8fa457fe44bfadb904ae869d87c1280bf60f93ecd3e79278"}, - {file = "grpcio-1.69.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:316463c0832d5fcdb5e35ff2826d9aa3f26758d29cdfb59a368c1d6c39615a11"}, - {file = "grpcio-1.69.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:26c9a9c4ac917efab4704b18eed9082ed3b6ad19595f047e8173b5182fec0d5e"}, - {file = "grpcio-1.69.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90b3646ced2eae3a0599658eeccc5ba7f303bf51b82514c50715bdd2b109e5ec"}, - {file = "grpcio-1.69.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3b75aea7c6cb91b341c85e7c1d9db1e09e1dd630b0717f836be94971e015031e"}, - {file = "grpcio-1.69.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5cfd14175f9db33d4b74d63de87c64bb0ee29ce475ce3c00c01ad2a3dc2a9e51"}, - {file = "grpcio-1.69.0-cp310-cp310-win32.whl", hash = "sha256:9031069d36cb949205293cf0e243abd5e64d6c93e01b078c37921493a41b72dc"}, - {file = "grpcio-1.69.0-cp310-cp310-win_amd64.whl", hash = "sha256:cc89b6c29f3dccbe12d7a3b3f1b3999db4882ae076c1c1f6df231d55dbd767a5"}, - {file = "grpcio-1.69.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:8de1b192c29b8ce45ee26a700044717bcbbd21c697fa1124d440548964328561"}, - {file = "grpcio-1.69.0-cp311-cp311-macosx_10_14_universal2.whl", hash = "sha256:7e76accf38808f5c5c752b0ab3fd919eb14ff8fafb8db520ad1cc12afff74de6"}, - {file = "grpcio-1.69.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:d5658c3c2660417d82db51e168b277e0ff036d0b0f859fa7576c0ffd2aec1442"}, - {file = "grpcio-1.69.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5494d0e52bf77a2f7eb17c6da662886ca0a731e56c1c85b93505bece8dc6cf4c"}, - {file = "grpcio-1.69.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ed866f9edb574fd9be71bf64c954ce1b88fc93b2a4cbf94af221e9426eb14d6"}, - {file = "grpcio-1.69.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c5ba38aeac7a2fe353615c6b4213d1fbb3a3c34f86b4aaa8be08baaaee8cc56d"}, - {file = "grpcio-1.69.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f79e05f5bbf551c4057c227d1b041ace0e78462ac8128e2ad39ec58a382536d2"}, - {file = "grpcio-1.69.0-cp311-cp311-win32.whl", hash = "sha256:bf1f8be0da3fcdb2c1e9f374f3c2d043d606d69f425cd685110dd6d0d2d61258"}, - {file = "grpcio-1.69.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb9302afc3a0e4ba0b225cd651ef8e478bf0070cf11a529175caecd5ea2474e7"}, - {file = "grpcio-1.69.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:fc18a4de8c33491ad6f70022af5c460b39611e39578a4d84de0fe92f12d5d47b"}, - {file = "grpcio-1.69.0-cp312-cp312-macosx_10_14_universal2.whl", hash = "sha256:0f0270bd9ffbff6961fe1da487bdcd594407ad390cc7960e738725d4807b18c4"}, - {file = "grpcio-1.69.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:dc48f99cc05e0698e689b51a05933253c69a8c8559a47f605cff83801b03af0e"}, - {file = "grpcio-1.69.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e925954b18d41aeb5ae250262116d0970893b38232689c4240024e4333ac084"}, - {file = "grpcio-1.69.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87d222569273720366f68a99cb62e6194681eb763ee1d3b1005840678d4884f9"}, - {file = "grpcio-1.69.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b62b0f41e6e01a3e5082000b612064c87c93a49b05f7602fe1b7aa9fd5171a1d"}, - {file = "grpcio-1.69.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:db6f9fd2578dbe37db4b2994c94a1d9c93552ed77dca80e1657bb8a05b898b55"}, - {file = "grpcio-1.69.0-cp312-cp312-win32.whl", hash = "sha256:b192b81076073ed46f4b4dd612b8897d9a1e39d4eabd822e5da7b38497ed77e1"}, - {file = "grpcio-1.69.0-cp312-cp312-win_amd64.whl", hash = "sha256:1227ff7836f7b3a4ab04e5754f1d001fa52a730685d3dc894ed8bc262cc96c01"}, - {file = "grpcio-1.69.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:a78a06911d4081a24a1761d16215a08e9b6d4d29cdbb7e427e6c7e17b06bcc5d"}, - {file = "grpcio-1.69.0-cp313-cp313-macosx_10_14_universal2.whl", hash = "sha256:dc5a351927d605b2721cbb46158e431dd49ce66ffbacb03e709dc07a491dde35"}, - {file = "grpcio-1.69.0-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:3629d8a8185f5139869a6a17865d03113a260e311e78fbe313f1a71603617589"}, - {file = "grpcio-1.69.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9a281878feeb9ae26db0622a19add03922a028d4db684658f16d546601a4870"}, - {file = "grpcio-1.69.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cc614e895177ab7e4b70f154d1a7c97e152577ea101d76026d132b7aaba003b"}, - {file = "grpcio-1.69.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:1ee76cd7e2e49cf9264f6812d8c9ac1b85dda0eaea063af07292400f9191750e"}, - {file = "grpcio-1.69.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:0470fa911c503af59ec8bc4c82b371ee4303ececbbdc055f55ce48e38b20fd67"}, - {file = "grpcio-1.69.0-cp313-cp313-win32.whl", hash = "sha256:b650f34aceac8b2d08a4c8d7dc3e8a593f4d9e26d86751ebf74ebf5107d927de"}, - {file = "grpcio-1.69.0-cp313-cp313-win_amd64.whl", hash = "sha256:028337786f11fecb5d7b7fa660475a06aabf7e5e52b5ac2df47414878c0ce7ea"}, - {file = "grpcio-1.69.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:b7f693db593d6bf285e015d5538bf1c86cf9c60ed30b6f7da04a00ed052fe2f3"}, - {file = "grpcio-1.69.0-cp38-cp38-macosx_10_14_universal2.whl", hash = "sha256:8b94e83f66dbf6fd642415faca0608590bc5e8d30e2c012b31d7d1b91b1de2fd"}, - {file = "grpcio-1.69.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:b634851b92c090763dde61df0868c730376cdb73a91bcc821af56ae043b09596"}, - {file = "grpcio-1.69.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bf5f680d3ed08c15330d7830d06bc65f58ca40c9999309517fd62880d70cb06e"}, - {file = "grpcio-1.69.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:200e48a6e7b00f804cf00a1c26292a5baa96507c7749e70a3ec10ca1a288936e"}, - {file = "grpcio-1.69.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:45a4704339b6e5b24b0e136dea9ad3815a94f30eb4f1e1d44c4ac484ef11d8dd"}, - {file = "grpcio-1.69.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:85d347cb8237751b23539981dbd2d9d8f6e9ff90082b427b13022b948eb6347a"}, - {file = "grpcio-1.69.0-cp38-cp38-win32.whl", hash = "sha256:60e5de105dc02832dc8f120056306d0ef80932bcf1c0e2b4ca3b676de6dc6505"}, - {file = "grpcio-1.69.0-cp38-cp38-win_amd64.whl", hash = "sha256:282f47d0928e40f25d007f24eb8fa051cb22551e3c74b8248bc9f9bea9c35fe0"}, - {file = "grpcio-1.69.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:dd034d68a2905464c49479b0c209c773737a4245d616234c79c975c7c90eca03"}, - {file = "grpcio-1.69.0-cp39-cp39-macosx_10_14_universal2.whl", hash = "sha256:01f834732c22a130bdf3dc154d1053bdbc887eb3ccb7f3e6285cfbfc33d9d5cc"}, - {file = "grpcio-1.69.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:a7f4ed0dcf202a70fe661329f8874bc3775c14bb3911d020d07c82c766ce0eb1"}, - {file = "grpcio-1.69.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd7ea241b10bc5f0bb0f82c0d7896822b7ed122b3ab35c9851b440c1ccf81588"}, - {file = "grpcio-1.69.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f03dc9b4da4c0dc8a1db7a5420f575251d7319b7a839004d8916257ddbe4816"}, - {file = "grpcio-1.69.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ca71d73a270dff052fe4edf74fef142d6ddd1f84175d9ac4a14b7280572ac519"}, - {file = "grpcio-1.69.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5ccbed100dc43704e94ccff9e07680b540d64e4cc89213ab2832b51b4f68a520"}, - {file = "grpcio-1.69.0-cp39-cp39-win32.whl", hash = "sha256:1514341def9c6ec4b7f0b9628be95f620f9d4b99331b7ef0a1845fd33d9b579c"}, - {file = "grpcio-1.69.0-cp39-cp39-win_amd64.whl", hash = "sha256:c1fea55d26d647346acb0069b08dca70984101f2dc95066e003019207212e303"}, - {file = "grpcio-1.69.0.tar.gz", hash = "sha256:936fa44241b5379c5afc344e1260d467bee495747eaf478de825bab2791da6f5"}, + {file = "grpcio-1.70.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:95469d1977429f45fe7df441f586521361e235982a0b39e33841549143ae2851"}, + {file = "grpcio-1.70.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:ed9718f17fbdb472e33b869c77a16d0b55e166b100ec57b016dc7de9c8d236bf"}, + {file = "grpcio-1.70.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:374d014f29f9dfdb40510b041792e0e2828a1389281eb590df066e1cc2b404e5"}, + {file = "grpcio-1.70.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2af68a6f5c8f78d56c145161544ad0febbd7479524a59c16b3e25053f39c87f"}, + {file = "grpcio-1.70.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce7df14b2dcd1102a2ec32f621cc9fab6695effef516efbc6b063ad749867295"}, + {file = "grpcio-1.70.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c78b339869f4dbf89881e0b6fbf376313e4f845a42840a7bdf42ee6caed4b11f"}, + {file = "grpcio-1.70.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:58ad9ba575b39edef71f4798fdb5c7b6d02ad36d47949cd381d4392a5c9cbcd3"}, + {file = "grpcio-1.70.0-cp310-cp310-win32.whl", hash = "sha256:2b0d02e4b25a5c1f9b6c7745d4fa06efc9fd6a611af0fb38d3ba956786b95199"}, + {file = "grpcio-1.70.0-cp310-cp310-win_amd64.whl", hash = "sha256:0de706c0a5bb9d841e353f6343a9defc9fc35ec61d6eb6111802f3aa9fef29e1"}, + {file = "grpcio-1.70.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:17325b0be0c068f35770f944124e8839ea3185d6d54862800fc28cc2ffad205a"}, + {file = "grpcio-1.70.0-cp311-cp311-macosx_10_14_universal2.whl", hash = "sha256:dbe41ad140df911e796d4463168e33ef80a24f5d21ef4d1e310553fcd2c4a386"}, + {file = "grpcio-1.70.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:5ea67c72101d687d44d9c56068328da39c9ccba634cabb336075fae2eab0d04b"}, + {file = "grpcio-1.70.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb5277db254ab7586769e490b7b22f4ddab3876c490da0a1a9d7c695ccf0bf77"}, + {file = "grpcio-1.70.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7831a0fc1beeeb7759f737f5acd9fdcda520e955049512d68fda03d91186eea"}, + {file = "grpcio-1.70.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:27cc75e22c5dba1fbaf5a66c778e36ca9b8ce850bf58a9db887754593080d839"}, + {file = "grpcio-1.70.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d63764963412e22f0491d0d32833d71087288f4e24cbcddbae82476bfa1d81fd"}, + {file = "grpcio-1.70.0-cp311-cp311-win32.whl", hash = "sha256:bb491125103c800ec209d84c9b51f1c60ea456038e4734688004f377cfacc113"}, + {file = "grpcio-1.70.0-cp311-cp311-win_amd64.whl", hash = "sha256:d24035d49e026353eb042bf7b058fb831db3e06d52bee75c5f2f3ab453e71aca"}, + {file = "grpcio-1.70.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:ef4c14508299b1406c32bdbb9fb7b47612ab979b04cf2b27686ea31882387cff"}, + {file = "grpcio-1.70.0-cp312-cp312-macosx_10_14_universal2.whl", hash = "sha256:aa47688a65643afd8b166928a1da6247d3f46a2784d301e48ca1cc394d2ffb40"}, + {file = "grpcio-1.70.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:880bfb43b1bb8905701b926274eafce5c70a105bc6b99e25f62e98ad59cb278e"}, + {file = "grpcio-1.70.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e654c4b17d07eab259d392e12b149c3a134ec52b11ecdc6a515b39aceeec898"}, + {file = "grpcio-1.70.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2394e3381071045a706ee2eeb6e08962dd87e8999b90ac15c55f56fa5a8c9597"}, + {file = "grpcio-1.70.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b3c76701428d2df01964bc6479422f20e62fcbc0a37d82ebd58050b86926ef8c"}, + {file = "grpcio-1.70.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ac073fe1c4cd856ebcf49e9ed6240f4f84d7a4e6ee95baa5d66ea05d3dd0df7f"}, + {file = "grpcio-1.70.0-cp312-cp312-win32.whl", hash = "sha256:cd24d2d9d380fbbee7a5ac86afe9787813f285e684b0271599f95a51bce33528"}, + {file = "grpcio-1.70.0-cp312-cp312-win_amd64.whl", hash = "sha256:0495c86a55a04a874c7627fd33e5beaee771917d92c0e6d9d797628ac40e7655"}, + {file = "grpcio-1.70.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:aa573896aeb7d7ce10b1fa425ba263e8dddd83d71530d1322fd3a16f31257b4a"}, + {file = "grpcio-1.70.0-cp313-cp313-macosx_10_14_universal2.whl", hash = "sha256:d405b005018fd516c9ac529f4b4122342f60ec1cee181788249372524e6db429"}, + {file = "grpcio-1.70.0-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:f32090238b720eb585248654db8e3afc87b48d26ac423c8dde8334a232ff53c9"}, + {file = "grpcio-1.70.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dfa089a734f24ee5f6880c83d043e4f46bf812fcea5181dcb3a572db1e79e01c"}, + {file = "grpcio-1.70.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f19375f0300b96c0117aca118d400e76fede6db6e91f3c34b7b035822e06c35f"}, + {file = "grpcio-1.70.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:7c73c42102e4a5ec76608d9b60227d917cea46dff4d11d372f64cbeb56d259d0"}, + {file = "grpcio-1.70.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:0a5c78d5198a1f0aa60006cd6eb1c912b4a1520b6a3968e677dbcba215fabb40"}, + {file = "grpcio-1.70.0-cp313-cp313-win32.whl", hash = "sha256:fe9dbd916df3b60e865258a8c72ac98f3ac9e2a9542dcb72b7a34d236242a5ce"}, + {file = "grpcio-1.70.0-cp313-cp313-win_amd64.whl", hash = "sha256:4119fed8abb7ff6c32e3d2255301e59c316c22d31ab812b3fbcbaf3d0d87cc68"}, + {file = "grpcio-1.70.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:8058667a755f97407fca257c844018b80004ae8035565ebc2812cc550110718d"}, + {file = "grpcio-1.70.0-cp38-cp38-macosx_10_14_universal2.whl", hash = "sha256:879a61bf52ff8ccacbedf534665bb5478ec8e86ad483e76fe4f729aaef867cab"}, + {file = "grpcio-1.70.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:0ba0a173f4feacf90ee618fbc1a27956bfd21260cd31ced9bc707ef551ff7dc7"}, + {file = "grpcio-1.70.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:558c386ecb0148f4f99b1a65160f9d4b790ed3163e8610d11db47838d452512d"}, + {file = "grpcio-1.70.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:412faabcc787bbc826f51be261ae5fa996b21263de5368a55dc2cf824dc5090e"}, + {file = "grpcio-1.70.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3b0f01f6ed9994d7a0b27eeddea43ceac1b7e6f3f9d86aeec0f0064b8cf50fdb"}, + {file = "grpcio-1.70.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7385b1cb064734005204bc8994eed7dcb801ed6c2eda283f613ad8c6c75cf873"}, + {file = "grpcio-1.70.0-cp38-cp38-win32.whl", hash = "sha256:07269ff4940f6fb6710951116a04cd70284da86d0a4368fd5a3b552744511f5a"}, + {file = "grpcio-1.70.0-cp38-cp38-win_amd64.whl", hash = "sha256:aba19419aef9b254e15011b230a180e26e0f6864c90406fdbc255f01d83bc83c"}, + {file = "grpcio-1.70.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:4f1937f47c77392ccd555728f564a49128b6a197a05a5cd527b796d36f3387d0"}, + {file = "grpcio-1.70.0-cp39-cp39-macosx_10_14_universal2.whl", hash = "sha256:0cd430b9215a15c10b0e7d78f51e8a39d6cf2ea819fd635a7214fae600b1da27"}, + {file = "grpcio-1.70.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:e27585831aa6b57b9250abaf147003e126cd3a6c6ca0c531a01996f31709bed1"}, + {file = "grpcio-1.70.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c1af8e15b0f0fe0eac75195992a63df17579553b0c4af9f8362cc7cc99ccddf4"}, + {file = "grpcio-1.70.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbce24409beaee911c574a3d75d12ffb8c3e3dd1b813321b1d7a96bbcac46bf4"}, + {file = "grpcio-1.70.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ff4a8112a79464919bb21c18e956c54add43ec9a4850e3949da54f61c241a4a6"}, + {file = "grpcio-1.70.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5413549fdf0b14046c545e19cfc4eb1e37e9e1ebba0ca390a8d4e9963cab44d2"}, + {file = "grpcio-1.70.0-cp39-cp39-win32.whl", hash = "sha256:b745d2c41b27650095e81dea7091668c040457483c9bdb5d0d9de8f8eb25e59f"}, + {file = "grpcio-1.70.0-cp39-cp39-win_amd64.whl", hash = "sha256:a31d7e3b529c94e930a117b2175b2efd179d96eb3c7a21ccb0289a8ab05b645c"}, + {file = "grpcio-1.70.0.tar.gz", hash = "sha256:8d1584a68d5922330025881e63a6c1b54cc8117291d382e4fa69339b6d914c56"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.69.0)"] +protobuf = ["grpcio-tools (>=1.70.0)"] [[package]] name = "grpcio-status" -version = "1.69.0" +version = "1.70.0" description = "Status proto mapping for gRPC" optional = false python-versions = ">=3.8" files = [ - {file = "grpcio_status-1.69.0-py3-none-any.whl", hash = "sha256:d6b2a3c9562c03a817c628d7ba9a925e209c228762d6d7677ae5c9401a542853"}, - {file = "grpcio_status-1.69.0.tar.gz", hash = "sha256:595ef84e5178d6281caa732ccf68ff83259241608d26b0e9c40a5e66eee2a2d2"}, + {file = "grpcio_status-1.70.0-py3-none-any.whl", hash = "sha256:fc5a2ae2b9b1c1969cc49f3262676e6854aa2398ec69cb5bd6c47cd501904a85"}, + {file = "grpcio_status-1.70.0.tar.gz", hash = "sha256:0e7b42816512433b18b9d764285ff029bde059e9d41f8fe10a60631bd8348101"}, ] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.69.0" +grpcio = ">=1.70.0" protobuf = ">=5.26.1,<6.0dev" [[package]] diff --git a/services/blaise_service.py b/services/blaise_service.py index e5b2985..2a65b6e 100644 --- a/services/blaise_service.py +++ b/services/blaise_service.py @@ -1,10 +1,9 @@ import logging -from typing import Any, Dict - import blaise_restapi +from typing import Any, Dict from appconfig.config import Config -from utilities.custom_exceptions import BlaiseError +from utilities.custom_exceptions import BlaiseError, IngestError from utilities.logging import function_name @@ -15,62 +14,17 @@ def __init__(self, config: Config) -> None: f"http://{self._config.blaise_api_url}" ) - self.cma_serverpark_name = "cma" - self.cma_questionnaire = "CMA_Launcher" - - def get_questionnaire( - self, server_park: str, questionnaire_name: str - ) -> Dict[str, Any]: - try: - questionnaire = self.restapi_client.get_questionnaire_for_server_park( - server_park, questionnaire_name - ) - logging.info(f"Got questionnaire '{questionnaire_name}'") - return questionnaire - except Exception as e: - error_message = ( - f"Exception caught in {function_name()}. " - f"Error getting questionnaire '{questionnaire_name}': {e}" - ) - logging.error(error_message) - raise BlaiseError(error_message) - - def get_users(self, server_park: str) -> dict[str, Any]: - try: - return self.restapi_client.get_users() - except Exception as e: - error_message = ( - f"Exception caught in {function_name()}. " - f"Error getting users from server park {server_park}: {e}" - ) - logging.error(error_message) - raise BlaiseError(error_message) - - def get_questionnaire_cases(self, guid: str) -> dict[str, Any]: - try: - cases = self.restapi_client.get_questionnaire_data( - self.cma_serverpark_name, - self.cma_questionnaire, - ["MainSurveyID", "id", "CMA_IsDonorCase"], - f"MainSurveyID='{guid}'", - ) - return cases - except Exception as e: - error_message = ( - f"Exception caught in {function_name()}. " - f"Error getting questionnaire cases from server park {self.cma_serverpark_name}: {e}" - ) - logging.error(error_message) - raise BlaiseError(error_message) - - def get_ingest(self, server_park: str, questionnaire_name: str): # TODO: stuff + def get_ingest(self, server_park: str, questionnaire_name: str, bucket_file_path: str): try: - result = self.restapi_client.get_ingest(server_park, questionnaire_name) + data_fields: Dict[str, Any] = { + "bucketFilePath": bucket_file_path, + } + result = self.restapi_client.get_ingest(server_park, questionnaire_name, data_fields) logging.info(f"Got ingest from server park {server_park}: {result}") except Exception as e: error_message = ( f"Exception caught in {function_name()}. " - f"Error getting existing get ingest: {e}" + f"Error when ingesting zip file: {e}" ) logging.error(error_message) - raise BlaiseError(error_message) + raise IngestError(error_message) diff --git a/services/guid_service.py b/services/guid_service.py deleted file mode 100644 index ec4091d..0000000 --- a/services/guid_service.py +++ /dev/null @@ -1,28 +0,0 @@ -import logging - -from services.blaise_service import BlaiseService -from utilities.custom_exceptions import BlaiseError, GuidError -from utilities.logging import function_name - - -class GUIDService: - def __init__(self, blaise_service: BlaiseService) -> None: - self._blaise_service = blaise_service - - def get_guid(self, server_park: str, questionnaire_name: str) -> str: - try: - questionnaire = self._blaise_service.get_questionnaire( - server_park, questionnaire_name - ) - guid = questionnaire["id"] - logging.info(f"Got GUID {guid} for questionnaire {questionnaire_name}") - return guid - except BlaiseError as e: - raise BlaiseError(e.message) - except Exception as e: - error_message = ( - f"Exception caught in {function_name()}. " - f"Error getting GUID for questionnaire {questionnaire_name}: {e}" - ) - logging.error(error_message) - raise GuidError(error_message) \ No newline at end of file diff --git a/services/ingest_service.py b/services/ingest_service.py deleted file mode 100644 index 400d7a7..0000000 --- a/services/ingest_service.py +++ /dev/null @@ -1,42 +0,0 @@ -import logging - -from services.blaise_service import BlaiseService -from utilities.custom_exceptions import BlaiseError, IngestError -from utilities.logging import function_name - - -class IngestService: - def __init__(self, blaise_service: BlaiseService) -> None: - self._blaise_service = blaise_service - - @staticmethod - def assert_expected_number_of_things( # TODO: what is in the zip? - expected_number_of_things: int, total_things: int - ): - if expected_number_of_things != total_things: - logging.info( - f"Expected to create {expected_number_of_things} things. Only created {total_things}" - ) - else: - logging.info( - f"Expected to create {expected_number_of_things} things. Successfully Created {total_things} things" - ) - - def ingest( # TODO: what is in the zip? - self, server_park_name: str, questionnaire_name: str - ) -> None: - try: - self._blaise_service.get_ingest(server_park_name, questionnaire_name) - except BlaiseError as e: - raise BlaiseError(e.message) - except IngestError as e: - raise IngestError(e.message) - except Exception as e: - error_message = ( - f"Exception caught in {function_name()}. " - f"Error when checking and creating zip thing: {e}" - ) - logging.error(error_message) - raise IngestError(error_message) - - \ No newline at end of file diff --git a/services/mypi.ini b/services/mypi.ini deleted file mode 100644 index c0f154d..0000000 --- a/services/mypi.ini +++ /dev/null @@ -1,19 +0,0 @@ -[mypy] -disallow_untyped_defs = False -namespace_packages = True -exclude = "scripts/" - -[mypy-blaise_restapi.*] -ignore_missing_imports = True - -[mypy-flask_httpauth.*] -ignore_missing_imports = True - -[mypy-google.auth.transport.requests.*] -ignore_missing_imports = True - -[mypy-google.oauth2.*] -ignore_missing_imports = True - -[mypy-dataclass_wizard.*] -ignore_missing_imports = True diff --git a/services/user_service.py b/services/user_service.py deleted file mode 100644 index 5cfe761..0000000 --- a/services/user_service.py +++ /dev/null @@ -1,57 +0,0 @@ -import logging - -from services.blaise_service import BlaiseService -from utilities.custom_exceptions import BlaiseError, UsersError, UsersWithRoleNotFound -from utilities.logging import function_name - - -class UserService: - def __init__(self, blaise_service: BlaiseService): - self._blaise_service = blaise_service - - def get_users_by_role(self, blaise_server_park: str, role: str) -> list[str]: - try: - blaise_users = self._blaise_service.get_users(blaise_server_park) - ips_users = [user["name"] for user in blaise_users if user["role"] == role] - logging.info( - f"Got {len(ips_users)} users from server park {blaise_server_park} for role {role}" - ) - return ips_users - except BlaiseError as e: - raise BlaiseError(e.message) from e - except UsersWithRoleNotFound as e: - raise UsersWithRoleNotFound(e.message) from e - except Exception as e: - error_message = ( - f"Exception caught in {function_name()}. " - f"Error getting users by role for server park {blaise_server_park}: {e}" - ) - logging.error(error_message) - raise UsersError(error_message) - - def get_user_by_name(self, blaise_server_park: str, username: str) -> dict: - try: - blaise_users = self._blaise_service.get_users(blaise_server_park) - user = next( - (user for user in blaise_users if user["name"] == username), None - ) - if user: - logging.info( - f"Got user {username} from server park {blaise_server_park}" - ) - return user - else: - error_message = ( - f"User {username} not found in server park {blaise_server_park}" - ) - logging.error(error_message) - raise UsersError(error_message) - except BlaiseError as e: - raise BlaiseError(e.message) from e - except Exception as e: - error_message = ( - f"Exception caught in {function_name()}. " - f"Error getting user by username for server park {blaise_server_park}: {e}" - ) - logging.error(error_message) - raise UsersError(error_message) \ No newline at end of file diff --git a/services/validation_service.py b/services/validation_service.py index b2b7faa..8c121b5 100644 --- a/services/validation_service.py +++ b/services/validation_service.py @@ -24,15 +24,11 @@ def get_valid_request_values_for_ingest_service( self.validate_request_is_json(request) self.validate_request_values_are_not_empty() self.validate_questionnaire_name() - self.validate_role() return (self.request_json["serverParkName"], self.request_json["questionnaireName"], self.request_json["tempFilePath"]) - # IngestDataDto ingestDataDto, string serverParkName, string questionnaireName, - # string tempFilePath) - def validate_request_is_json(self, request): try: self.request_json = request.get_json() @@ -47,29 +43,13 @@ def validate_request_is_json(self, request): def validate_request_values_are_not_empty(self): missing_values = [] questionnaire_name = self.request_json["questionnaire_name"] - role = self.request_json["role"] + server_park = self.request_json["server_park"] if questionnaire_name is None or questionnaire_name == "": missing_values.append("questionnaire_name") - if role is None or role == "": - missing_values.append("role") - - if missing_values: - error_message = f"Missing required values from request: {missing_values}" - logging.error(error_message) - raise RequestError(error_message) - - def validate_request_values_are_not_empty_for_ingest_service(self): - missing_values = [] - questionnaire_name = self.request_json["questionnaire_name"] - user = self.request_json["user"] - - if questionnaire_name is None or questionnaire_name == "": - missing_values.append("questionnaire_name") - - if user is None or user == "": - missing_values.append("user") + if server_park is None or server_park == "": + missing_values.append("server_park") if missing_values: error_message = f"Missing required values from request: {missing_values}" @@ -88,16 +68,6 @@ def validate_questionnaire_name(self): logging.error(error_message) raise RequestError(error_message) - def validate_role(self): - valid_roles = ["IPS Manager", "IPS Field Interviewer", "IPS Pilot Interviewer"] - if self.request_json["role"] not in valid_roles: - error_message = ( - f"{self.request_json['role']} is not a valid role. " - f"Please choose one of the following roles: {valid_roles}" - ) - logging.error(error_message) - raise RequestError(error_message) - @staticmethod def validate_config(config): missing_configs = [] @@ -127,10 +97,3 @@ def validate_questionnaire_exists(questionnaire_name: str, config: Config): ) logging.error(error_message) raise BlaiseError(error_message) - - @staticmethod - def validate_users_with_role_exist(users: list, role: str): - if not users: - error_message = f"No users found with role '{role}'" - logging.error(error_message) - raise UsersWithRoleNotFound(error_message) \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index b441cb2..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,194 +0,0 @@ -import pytest - -from models.config import Config -from models.message import File, Message - - -@pytest.fixture -def md5hash(): - return "0a14db6e48b947b57988a2f61469f228" - - -@pytest.fixture -def event(md5hash): - def wrapper(filename): - return { - "name": f"{filename}.zip", - "bucket": "ons-blaise-v2-nifi", - "md5Hash": md5hash, - "size": "20", - "timeCreated": "0103202021_16428", - } - - return wrapper - - -@pytest.fixture -def dd_event(md5hash): - def wrapper(instrument): - return { - "name": f"dd_{instrument}_0103202021_16428.zip", - "bucket": "ons-blaise-v2-nifi", - "md5Hash": md5hash, - "size": "20", - "timeCreated": "0103202021_16428", - } - - return wrapper - - -@pytest.fixture -def mi_event(md5hash): - def wrapper(instrument): - return { - "name": f"mi_{instrument}_0103202021_16428.zip", - "bucket": "ons-blaise-v2-nifi", - "md5Hash": md5hash, - "size": "20", - "timeCreated": "0103202021_16428", - } - - return wrapper - - -@pytest.fixture -def config(): - return Config( - on_prem_subfolder="survey_on_prem_subfolder", - project_id="survey_project_id", - topic_name="topic_name", - env="test", - ) - - -@pytest.fixture -def file(): - return File( - name="dd_file.zip:my-bucket-name", - sizeBytes="20", - md5sum="dasdasd", - relativePath="./", - ) - - -@pytest.fixture -def message(file, md5hash, config): - return Message( - files=file, - sourceName="foo", - manifestCreated="bar", - fullSizeMegabytes="foobar", - version=3, - schemaVersion=1, - description="barfoo", - dataset="foobarfoo", - sensitivity="High", - iterationL1=config.on_prem_subfolder, - iterationL2="", - iterationL3="", - iterationL4="", - ) - - -@pytest.fixture -def expected_pubsub_message_dd_opn(): - return { - "version": 3, - "schemaVersion": 1, - "files": [ - { - "sizeBytes": "20", - "name": "dd_OPN2102R_0103202021_16428.zip:ons-blaise-v2-nifi", - "md5sum": "d1ad7875be9ee3c6fde3b6f9efdf3c6b67fad78ebd7f6dbc", - "relativePath": ".\\", - } - ], - "sensitivity": "High", - "sourceName": "gcp_blaise_test", - "description": "Data Delivery files for OPN uploaded to GCP bucket from Blaise5", - "dataset": "blaise_dde", - "iterationL1": "SYSTEMS", - "iterationL2": "DEV", - "iterationL3": "OPN", - "iterationL4": "OPN2102R", - "manifestCreated": "0103202021_16428", - "fullSizeMegabytes": "0.000020", - } - - -@pytest.fixture -def expected_pubsub_message_dd_lms(): - return { - "version": 3, - "schemaVersion": 1, - "files": [ - { - "sizeBytes": "20", - "name": "dd_LMS2102R_0103202021_16428.zip:ons-blaise-v2-nifi", - "md5sum": "d1ad7875be9ee3c6fde3b6f9efdf3c6b67fad78ebd7f6dbc", - "relativePath": ".\\", - } - ], - "sensitivity": "High", - "sourceName": "gcp_blaise_test", - "description": "Data Delivery files for LMS uploaded to GCP bucket from Blaise5", - "dataset": "blaise_dde_lms", - "iterationL1": "CLOUD", - "iterationL2": "test", - "iterationL3": "LMS2102R", - "iterationL4": "", - "manifestCreated": "0103202021_16428", - "fullSizeMegabytes": "0.000020", - } - - -@pytest.fixture -def expected_pubsub_message_dd_lmc(): - return { - "version": 3, - "schemaVersion": 1, - "files": [ - { - "sizeBytes": "20", - "name": "dd_LMC2102R_0103202021_16428.zip:ons-blaise-v2-nifi", - "md5sum": "d1ad7875be9ee3c6fde3b6f9efdf3c6b67fad78ebd7f6dbc", - "relativePath": ".\\", - } - ], - "sensitivity": "High", - "sourceName": "gcp_blaise_test", - "description": "Data Delivery files for LMC uploaded to GCP bucket from Blaise5", - "dataset": "blaise_dde_lms", - "iterationL1": "CLOUD", - "iterationL2": "test", - "iterationL3": "LMC2102R", - "iterationL4": "", - "manifestCreated": "0103202021_16428", - "fullSizeMegabytes": "0.000020", - } - - -@pytest.fixture -def expected_pubsub_message_mi(): - return { - "version": 3, - "schemaVersion": 1, - "files": [ - { - "sizeBytes": "20", - "name": "mi_OPN2101A_0103202021_16428.zip:ons-blaise-v2-nifi", - "md5sum": "d1ad7875be9ee3c6fde3b6f9efdf3c6b67fad78ebd7f6dbc", - "relativePath": ".\\", - } - ], - "sensitivity": "High", - "sourceName": "gcp_blaise_test", - "description": "Management Information files uploaded to GCP bucket from Blaise5", - "dataset": "blaise_mi", - "iterationL1": "BL5-test", - "iterationL2": "OPN", - "iterationL3": "OPN2101A", - "iterationL4": "", - "manifestCreated": "0103202021_16428", - "fullSizeMegabytes": "0.000020", - } diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..2255a57 --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,5 @@ +from appconfig.config import Config + + +def get_default_config() -> Config: + return Config(blaise_api_url="blaise_api_url", blaise_server_park="gusty") diff --git a/tests/models/test_config.py b/tests/models/test_config.py index aa3f904..b036b5c 100644 --- a/tests/models/test_config.py +++ b/tests/models/test_config.py @@ -1,63 +1,28 @@ import os from unittest import mock -import blaise_dds import pytest -from main import publishMsg -from models.config import Config +from appconfig.config import Config def test_config(): config = Config( - on_prem_subfolder="OPN", project_id="foobar", topic_name="barfoo", env="test" + blaise_api_url="foobar", + blaise_server_park="foobar", ) - assert config.on_prem_subfolder == "OPN" - assert config.project_id == "foobar" - assert config.topic_name == "barfoo" - assert config.env == "test" + assert config.blaise_api_url == "foobar" + assert config.blaise_server_park == "foobar" @mock.patch.dict( os.environ, { - "PROJECT_ID": "test_project_id", - "ENV": "test", - "TOPIC_NAME": "nifi-notify", - "ON-PREM-SUBFOLDER": "DEV", + "BLAISE_API_URL": "test_blaise_api_url", + "BLAISE_SERVER_PARK": "test_blaise_server_park", }, ) def test_config_from_env(): config = Config.from_env() - assert config.on_prem_subfolder == "DEV" - assert config.project_id == "test_project_id" - assert config.topic_name == "nifi-notify" - assert config.env == "test" - - -@mock.patch.dict( - os.environ, - {"TOPIC_NAME": "nifi-notify"}, -) -@mock.patch.object(blaise_dds.Client, "update_state") -@pytest.mark.parametrize( - "instrument", - [ - ("LMC2102R"), - ("OPN2102R"), - ("LMS2102R"), - ], -) -def test_project_id_not_set(_mock_update_state, dd_event, capsys, instrument): - dd_event = dd_event(instrument) - publishMsg(dd_event, None) - captured = capsys.readouterr() - assert captured.out == ( - "Configuration: Project ID: None\n" - + "Configuration: Topic Name: nifi-notify\n" - + "Configuration: ON-PREM-SUBFOLDER: None\n" - + "Configuration: Env: None\n" - + f"Configuration: File name: dd_{instrument}_0103202021_16428.zip\n" - + "Configuration: Bucket Name: ons-blaise-v2-nifi\n" - + "project_id not set, publish failed\n" - ) + assert config.blaise_api_url == "test_blaise_api_url" + assert config.blaise_server_park == "test_blaise_server_park" diff --git a/tests/models/test_message.py b/tests/models/test_message.py deleted file mode 100644 index 6205e3a..0000000 --- a/tests/models/test_message.py +++ /dev/null @@ -1,234 +0,0 @@ -import json -from dataclasses import asdict -from unittest import mock - -import pytest -from google.cloud.pubsub_v1 import PublisherClient - -from models.message import File, create_message, send_pub_sub_message -from utils import InvalidFileExtension, InvalidFileType - - -def test_file_extension(file): - assert file.extension() == ".zip" - - -def test_file_filename(file): - assert file.filename() == "dd_file.zip" - - -@pytest.mark.parametrize( - "file_name,file_type", - [ - ("dd_file.zip", "dd"), - ("mi_file.zip", "mi"), - ], -) -def test_file_type(file, file_name, file_type): - file.name = f"{file_name}:my-bucket-name" - assert file.type() == file_type - - -@pytest.mark.parametrize( - "file_name, expected", - [ - ("dd_opn2101a.zip", "OPN"), - ("dd_lms2102_a1.zip", "LMS"), - ("dd_lms2102_bk1.zip", "LMS"), - ("dd_lmc2102_bk1.zip", "LMC"), - ("dd_lmb21021_bk2.zip", "LMB"), - ("dd_frs2411a.zip", "FRS"), - ], -) -def test_file_survey_tla(file, file_name, expected): - file.name = f"{file_name}:my-bucket-name" - assert file.survey_tla() == expected - - -@pytest.mark.parametrize( - "file_name, expected", - [ - ("dd_opn2101a.zip", "OPN2101A"), - ("dd_lms2102_a1.zip", "LMS2102_A1"), - ("dd_lms2102_bk1.zip", "LMS2102_BK1"), - ("dd_lmc2102_bk1.zip", "LMC2102_BK1"), - ("dd_frs2411a.zip", "FRS2411A"), - ], -) -def test_file_instrument_name(file, file_name, expected): - file.name = f"{file_name}:my-bucket-name" - assert file.instrument_name() == expected - - -@pytest.mark.parametrize( - "survey_tla, expected", - [ - ("OPN", False), - ("OLS", False), - ("LMS", True), - ("LMB", True), - ("IPS", False), - ("LMC", True), - ("LMO", True), - ("QWERTY", False), - ("LMNOP", True), - ("LBS", False), - ("FRS", False), - ], -) -def test_file_is_lms(file, survey_tla, expected): - file.name = f"dd_{survey_tla}2101a.zip:my-bucket-name" - assert file.is_lms() is expected - -@pytest.mark.parametrize( - "survey_tla, expected", - [ - ("OPN", False), - ("OLS", False), - ("LMS", False), - ("LMB", False), - ("IPS", False), - ("LMC", False), - ("LMO", False), - ("QWERTY", False), - ("LMNOP", False), - ("LBS", False), - ("FRS", True), - ], -) -def test_file_is_frs(file, survey_tla, expected): - file.name = f"dd_{survey_tla}2101a.zip:my-bucket-name" - assert file.is_frs() is expected - - -def test_file_from_event(dd_event): - file = File.from_event(dd_event("OPN2102R")) - assert file.name == "dd_OPN2102R_0103202021_16428.zip:ons-blaise-v2-nifi" - assert file.sizeBytes == "20" - assert file.md5sum == "d1ad7875be9ee3c6fde3b6f9efdf3c6b67fad78ebd7f6dbc" - assert file.relativePath == ".\\" - - -@pytest.mark.parametrize( - "instrument, expected_tla", - [ - ("opn2101A", "OPN"), - ("lms2102_bk1", "LMS"), - ("frs2102a", "FRS"), - ], -) -def test_create_message_for_management_information( - instrument, expected_tla, mi_event, config -): - mi_event = mi_event(instrument) - actual_message = create_message(mi_event, config) - - assert ( - actual_message.description - == "Management Information files uploaded to GCP bucket from Blaise5" - ) - assert actual_message.dataset == "blaise_mi" - assert actual_message.iterationL1 == "BL5-test" - assert actual_message.iterationL2 == expected_tla - assert actual_message.iterationL3 == instrument.upper() - - -def test_create_message_for_data_delivery_opn(dd_event, config): - dd_event = dd_event("OPN2101A") - actual_message = create_message(dd_event, config) - - assert ( - actual_message.description - == "Data Delivery files for OPN uploaded to GCP bucket from Blaise5" - ) - assert actual_message.dataset == "blaise_dde" - assert actual_message.iterationL1 == "SYSTEMS" - assert actual_message.iterationL2 == "survey_on_prem_subfolder" - assert actual_message.iterationL3 == "OPN" - assert actual_message.iterationL4 == "OPN2101A" - - -def test_create_message_for_data_delivery_frs(dd_event, config): - dd_event = dd_event("frs2411a") - actual_message = create_message(dd_event, config) - - assert ( - actual_message.description - == "Data Delivery files for FRS uploaded to GCP bucket from Blaise5" - ) - - assert actual_message.dataset == "blaise_dde_frs" - assert actual_message.iterationL1 == "ingress" - assert actual_message.iterationL2 == "survey_data" - assert actual_message.iterationL3 == "bl5-test" - assert actual_message.iterationL4 == "FRS2411A" - - -@pytest.mark.parametrize( - "instrument,expected_survey_tla", - [ - ("LMS2102_A1", "LMS"), - ("lms2102_bk1", "LMS"), - ("lmc2102_bk1", "LMC"), - ("lmb21021_bk2", "LMB"), - ], -) -def test_create_message_for_data_delivery_lms( - instrument, expected_survey_tla, dd_event, config -): - dd_event = dd_event(instrument) - actual_message = create_message(dd_event, config) - - assert ( - actual_message.description - == f"Data Delivery files for {expected_survey_tla} uploaded to GCP bucket from Blaise5" - ) - assert actual_message.dataset == "blaise_dde_lms" - assert actual_message.iterationL1 == "CLOUD" - assert actual_message.iterationL2 == "test" - assert actual_message.iterationL3 == instrument.upper() - assert actual_message.iterationL4 == "" - - -@pytest.mark.parametrize( - "spicy_file_extension", - [ - ("avi"), - ("dat"), - ("nth"), - ("zoo"), - ("qxd"), - ], -) -def test_create_message_with_an_invalid_file_extension( - spicy_file_extension, dd_event, config -): - dd_event = dd_event("OPN2101A") - dd_event["name"] = f"dd_opn2101a.{spicy_file_extension}:my-bucket-name" - - with pytest.raises(InvalidFileExtension): - create_message(dd_event, config) - - -@pytest.mark.parametrize( - "spicy_file_types", - [("notMI"), ("notDD"), ("ddfoo"), ("mibar"), ("mmmm_spicy")], -) -def test_create_message_with_an_invalid_file_type(spicy_file_types, event, config): - event = event(spicy_file_types) - - with pytest.raises(InvalidFileType): - create_message(event, config) - - -@mock.patch.object(PublisherClient, "publish") -def test_send_pub_sub_message(mock_pubsub, config, message): - send_pub_sub_message(config, message) - - assert len(mock_pubsub.call_args_list) == 1 - assert ( - mock_pubsub.call_args_list[0][0][0] - == "projects/survey_project_id/topics/topic_name" - ) - pubsub_message = mock_pubsub.call_args_list[0][1]["data"] - assert json.loads(pubsub_message) == asdict(message) diff --git a/tests/services/test_blaise_service.py b/tests/services/test_blaise_service.py new file mode 100644 index 0000000..f4e6f1d --- /dev/null +++ b/tests/services/test_blaise_service.py @@ -0,0 +1,41 @@ +import logging +from unittest import mock + +import blaise_restapi +import pytest + +from appconfig.config import Config +from services.blaise_service import BlaiseService +from tests.helpers import get_default_config +from utilities.custom_exceptions import BlaiseError + + +@pytest.fixture() +def config() -> Config: + return get_default_config() + + +@pytest.fixture() +def blaise_service(config) -> BlaiseService: + return BlaiseService(config=config) + + +class TestIngest: + @mock.patch.object(blaise_restapi.Client, "get_ingest") + def test_get_ingest_calls_the_rest_api_endpoint_with_the_correct_parameters( + self, _mock_rest_api_client, blaise_service + ): + # arrange + blaise_server_park = "gusty" + questionnaire_name = "IPS2306a" + bucket_file_path = 'IPS2306a.zip' + expected_bucket_file_path = { + 'bucketFilePath': 'IPS2306a.zip' + } + + # act + blaise_service.get_ingest(blaise_server_park, questionnaire_name, bucket_file_path) + + # assert + _mock_rest_api_client.assert_called_with(blaise_server_park, questionnaire_name, expected_bucket_file_path) + diff --git a/tests/test_main.py b/tests/test_main.py deleted file mode 100644 index 5a19d8b..0000000 --- a/tests/test_main.py +++ /dev/null @@ -1,85 +0,0 @@ -import json -import os -from unittest import mock - -import blaise_dds -import pytest -from google.cloud.pubsub_v1 import PublisherClient - -from main import publishMsg - - -@mock.patch.dict( - os.environ, - { - "PROJECT_ID": "test_project_id", - "ENV": "test", - "TOPIC_NAME": "nifi-notify", - "ON-PREM-SUBFOLDER": "DEV", - }, -) -@mock.patch.object(blaise_dds.Client, "update_state") -@mock.patch.object(PublisherClient, "publish") -@pytest.mark.parametrize( - "instrument, expected_message", - [ - ("OPN2102R", pytest.lazy_fixture("expected_pubsub_message_dd_opn")), - ("LMS2102R", pytest.lazy_fixture("expected_pubsub_message_dd_lms")), - ("LMC2102R", pytest.lazy_fixture("expected_pubsub_message_dd_lmc")), - ], -) -def test_publishMsg_for_data_delivery( - mock_pubsub, _mock_update_state, dd_event, instrument, expected_message -): - dd_event = dd_event(instrument) - publishMsg(dd_event, None) - - pubsub_message = mock_pubsub.call_args_list[0][1]["data"] - assert json.loads(pubsub_message) == expected_message - - -@mock.patch.dict( - os.environ, - { - "PROJECT_ID": "test_project_id", - "ENV": "test", - "TOPIC_NAME": "nifi-notify", - "ON-PREM-SUBFOLDER": "DEV", - }, -) -@mock.patch.object(blaise_dds.Client, "update_state") -@mock.patch.object(PublisherClient, "publish") -def test_publishMsg_for_management_information( - mock_pubsub, _mock_update_state, mi_event, expected_pubsub_message_mi -): - mi_event = mi_event("OPN2101A") - publishMsg(mi_event, None) - pubsub_message = mock_pubsub.call_args_list[0][1]["data"] - assert json.loads(pubsub_message) == expected_pubsub_message_mi - - -@mock.patch.dict( - os.environ, - {"PROJECT_ID": "test_project_id", "ENV": "test", "TOPIC_NAME": "nifi-notify"}, -) -@mock.patch.object(blaise_dds.Client, "update_state") -@mock.patch.object(PublisherClient, "publish") -@pytest.mark.parametrize( - "instrument", - [ - ("LMC2102R"), - ("OPN2102R"), - ("LMS2102R"), - ], -) -def test_publishMsg_error(mock_pubsub, mock_update_state, dd_event, instrument): - mock_pubsub.side_effect = Exception( - "Explosions occurred when sending message to pubsub" - ) - dd_event = dd_event(instrument) - publishMsg(dd_event, None) - assert mock_update_state.call_args_list[1] == mock.call( - dd_event["name"], - "errored", - "Exception('Explosions occurred when sending message to pubsub')", - ) diff --git a/tests/test_utils.py b/tests/test_utils.py deleted file mode 100644 index f36e443..0000000 --- a/tests/test_utils.py +++ /dev/null @@ -1,70 +0,0 @@ -from unittest import mock - -import blaise_dds -import pytest - -from utils import md5hash_to_md5sum, size_in_megabytes, update_data_delivery_state - - -def test_md5hash_to_md5sum(md5hash): - assert ( - md5hash_to_md5sum(md5hash) == "d1ad7875be9ee3c6fde3b6f9efdf3c6b67fad78ebd7f6dbc" - ) - - -@pytest.mark.parametrize( - "size_in_bytes,size_in_megs", - [ - ("20", "0.000020"), - ("320", "0.000320"), - ("4783", "0.004783"), - ("12004783", "12.004783"), - ("3475231", "3.475231"), - ], -) -def test_size_in_megabytes(size_in_bytes, size_in_megs): - assert size_in_megabytes(size_in_bytes) == size_in_megs - - -@mock.patch.object(blaise_dds.Client, "update_state") -@pytest.mark.parametrize( - "instrument,state", - [ - ("LMC2102R", "in_nifi_bucket"), - ("OPN2102R", "nifi_notified"), - ("LMS2102R", "in_arc"), - ], -) -def test_update_data_delivery_state(mock_update_state, dd_event, instrument, state): - dd_event = dd_event(instrument) - update_data_delivery_state(dd_event, state) - assert mock_update_state.call_count == 1 - assert mock_update_state.call_args_list[0] == mock.call( - dd_event["name"], - state, - None, - ) - - -@mock.patch.object(blaise_dds.Client, "update_state") -@pytest.mark.parametrize( - "instrument,state", - [ - ("LMC2102R", "in_nifi_bucket"), - ("OPN2102R", "nifi_notified"), - ("LMS2102R", "in_arc"), - ], -) -def test_update_data_delivery_state_fail( - mock_update_state, dd_event, capsys, instrument, state -): - mock_update_state.side_effect = Exception( - "Computer says no. Do not pass Go. Do not collect £200" - ) - dd_event = dd_event(instrument) - update_data_delivery_state(dd_event, state) - captured = capsys.readouterr() - assert ( - captured.out - == "failed to update dds state: Computer says no. Do not pass Go. Do not collect £200\n" - ) diff --git a/utils.py b/utils.py index c7a6126..ad5c72d 100644 --- a/utils.py +++ b/utils.py @@ -9,11 +9,6 @@ def log_event(event): print(f"Configuration: Bucket Name: {event['bucket']}") -def md5hash_to_md5sum(md5hash): - decode_hash = base64.b64decode(md5hash) - encoded_hash = binascii.hexlify(decode_hash) - return str(encoded_hash, "utf-8") - def get_questionnaire_name(zip_filename): match = re.search(r"^([a-zA-Z]+)(\d{4})([a-zA-Z]*)(?:edit)?\.zip", zip_filename)