change default of refills todays logs to true (#27)

* changes default of refills todays logs to true * regenerate-log-override -> False * removes refill-todays-log button
digital-land · Feb 4, 2025 · 414fe8c · 414fe8c
1 parent 4f02274
commit 414fe8c
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 6 deletions.
diff --git a/dags/collection_generator.py b/dags/collection_generator.py
@@ -53,8 +53,7 @@
             "transformed-jobs": Param(default=8, type="integer"),
             "dataset-jobs": Param(default=8, type="integer"),
             "incremental-loading-override": Param(default=False, type="boolean"),
-            "regenerate-log-override": Param(default=True, type="boolean"),
-            "refill-todays-logs": Param(default=False, type="boolean"),
+            "regenerate-log-override": Param(default=False, type="boolean"),
         },
         render_template_as_native_obj=True,
         is_paused_upon_creation=False,
@@ -94,8 +93,7 @@
                             {"name": "TRANSFORMED_JOBS", "value":"'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"transformed-jobs\") | string }}'"},
                             {"name": "DATASET_JOBS", "value": "'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"dataset-jobs\") | string }}'"},
                             {"name": "INCREMENTAL_LOADING_OVERRIDE", "value": "'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"incremental-loading-override\") | string }}'"},
-                            {"name": "REGENERATE_LOG_OVERRIDE", "value": "'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"regenerate-log-override\") | string }}'"},
-                            {"name": "REFILL_TODAYS_LOGS", "value": "'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"refill-todays-logs\") | string }}'"}
+                            {"name": "REGENERATE_LOG_OVERRIDE", "value": "'{{ task_instance.xcom_pull(task_ids=\"configure-dag\", key=\"regenerate-log-override\") | string }}'"}
                         ],
                     },
                 ]

diff --git a/dags/utils.py b/dags/utils.py
@@ -90,7 +90,6 @@ def configure_dag(**kwargs):
         dataset_jobs = str(kwargs['params'].get('dataset-jobs'))
         incremental_loading_override = bool(kwargs['params'].get('incremental-loading-override'))
         regenerate_log_override = bool(kwargs['params'].get('regenerate-log-override'))
-        refill_todays_logs = bool(kwargs['params'].get('refill-todays-logs'))
 
         # get ecs-task logging configuration
         ecs_client = boto3.client('ecs')
@@ -117,6 +116,5 @@ def configure_dag(**kwargs):
         ti.xcom_push(key='collection-dataset-bucket-name', value=collection_dataset_bucket_name)
         ti.xcom_push(key='incremental-loading-override', value=incremental_loading_override)
         ti.xcom_push(key='regenerate-log-override', value=regenerate_log_override)
-        ti.xcom_push(key='refill-todays-logs', value=refill_todays_logs)
 
     return configure_dag