From 414fe8c52fb455f75fea78a9c27e538da33a40a7 Mon Sep 17 00:00:00 2001 From: CarlosCoelhoSL <110818364+CarlosCoelhoSL@users.noreply.github.com> Date: Tue, 4 Feb 2025 15:56:17 +0000 Subject: [PATCH] change default of refills todays logs to true (#27) * changes default of refills todays logs to true * regenerate-log-override -> False * removes refill-todays-log button --- dags/collection_generator.py | 6 ++---- dags/utils.py | 2 -- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/dags/collection_generator.py b/dags/collection_generator.py index ca09534..f0410cc 100644 --- a/dags/collection_generator.py +++ b/dags/collection_generator.py @@ -53,8 +53,7 @@ "transformed-jobs": Param(default=8, type="integer"), "dataset-jobs": Param(default=8, type="integer"), "incremental-loading-override": Param(default=False, type="boolean"), - "regenerate-log-override": Param(default=True, type="boolean"), - "refill-todays-logs": Param(default=False, type="boolean"), + "regenerate-log-override": Param(default=False, type="boolean"), }, render_template_as_native_obj=True, is_paused_upon_creation=False, @@ -94,8 +93,7 @@ {"name": "TRANSFORMED_JOBS", "value":"'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"transformed-jobs\") | string }}'"}, {"name": "DATASET_JOBS", "value": "'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"dataset-jobs\") | string }}'"}, {"name": "INCREMENTAL_LOADING_OVERRIDE", "value": "'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"incremental-loading-override\") | string }}'"}, - {"name": "REGENERATE_LOG_OVERRIDE", "value": "'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"regenerate-log-override\") | string }}'"}, - {"name": "REFILL_TODAYS_LOGS", "value": "'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"refill-todays-logs\") | string }}'"} + {"name": "REGENERATE_LOG_OVERRIDE", "value": "'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"regenerate-log-override\") | string }}'"} ], }, ] diff --git a/dags/utils.py b/dags/utils.py index fe36ce3..e49eecc 100644 --- a/dags/utils.py +++ b/dags/utils.py @@ -90,7 +90,6 @@ def configure_dag(**kwargs): dataset_jobs = str(kwargs['params'].get('dataset-jobs')) incremental_loading_override = bool(kwargs['params'].get('incremental-loading-override')) regenerate_log_override = bool(kwargs['params'].get('regenerate-log-override')) - refill_todays_logs = bool(kwargs['params'].get('refill-todays-logs')) # get ecs-task logging configuration ecs_client = boto3.client('ecs') @@ -117,6 +116,5 @@ def configure_dag(**kwargs): ti.xcom_push(key='collection-dataset-bucket-name', value=collection_dataset_bucket_name) ti.xcom_push(key='incremental-loading-override', value=incremental_loading_override) ti.xcom_push(key='regenerate-log-override', value=regenerate_log_override) - ti.xcom_push(key='refill-todays-logs', value=refill_todays_logs) return configure_dag