Airflow Slack notifications (#22)

* Trying test DAG for Slack notifications

* Corrected BashOperator import

* Corrected DAG name argument

* Correcting characters in DAG name

* Removed schedule from slack-notifcation-tester DAG

* Trying to create slack api default connection needed by Slack provider

* Trying out conditional version of slack notification on failure

* Compiled dependencies since test context was missing new airflow slack provider dependency

* Change handling of failure callbacks

* Added Slack notification on failure to collection DAG

* Removed unused import

dags/collection_generator.py

@@ -9,8 +9,13 @@
     EcsRegisterTaskDefinitionOperator,
     EcsRunTaskOperator,
 )
+
+from airflow.operators.bash import BashOperator
 from airflow.operators.python import PythonOperator
 from airflow.models.param import Param
+from airflow.models import Variable
+from airflow.models.connection import Connection
+from airflow.providers.slack.notifications.slack import send_slack_notification
 
 from utils import dag_default_args, get_config, load_specification_datasets, setup_configure_dag_callable
 
@@ -23,6 +28,16 @@
 
 collections = load_specification_datasets()
 
+failure_callbacks = []
+if config['env'] == 'production':
+    failure_callbacks.append(
+        send_slack_notification(
+            text="The DAG {{ dag.dag_id }} failed",
+            channel="#planning-data-platform",
+            username="Airflow"
+        )
+    )
+
 for collection, datasets in collections.items():
     dag_id = f"{collection}-collection"
 
@@ -40,7 +55,8 @@
             "incremental-loading-override": Param(default=False, type="boolean"),
         },
         render_template_as_native_obj=True,
-        is_paused_upon_creation=False
+        is_paused_upon_creation=False,
+        on_failure_callback=failure_callbacks,
     ) as dag:
         configure_dag_task = PythonOperator(
             task_id="configure-dag",

requirements/dev-requirements.txt

@@ -39,6 +39,7 @@ apache-airflow==2.9.2
     #   apache-airflow-providers-ftp
     #   apache-airflow-providers-http
     #   apache-airflow-providers-imap
+    #   apache-airflow-providers-slack
     #   apache-airflow-providers-smtp
     #   apache-airflow-providers-sqlite
 apache-airflow-providers-amazon==8.28.0
@@ -56,6 +57,7 @@ apache-airflow-providers-common-sql==1.16.0
     #   -r requirements/requirements.txt
     #   apache-airflow
     #   apache-airflow-providers-amazon
+    #   apache-airflow-providers-slack
     #   apache-airflow-providers-sqlite
 apache-airflow-providers-fab==1.3.0
     # via
@@ -74,6 +76,8 @@ apache-airflow-providers-imap==3.7.0
     # via
     #   -r requirements/requirements.txt
     #   apache-airflow
+apache-airflow-providers-slack==8.9.1
+    # via -r requirements/requirements.txt
 apache-airflow-providers-smtp==1.8.0
     # via
     #   -r requirements/requirements.txt
@@ -684,6 +688,10 @@ six==1.16.0
     #   python-dateutil
     #   rfc3339-validator
     #   wirerope
+slack-sdk==3.33.5
+    # via
+    #   -r requirements/requirements.txt
+    #   apache-airflow-providers-slack
 sniffio==1.3.1
     # via
     #   -r requirements/requirements.txt
@@ -776,6 +784,8 @@ urllib3==2.2.2
     #   botocore
     #   requests
     #   responses
+utils==1.0.2
+    # via -r requirements/requirements.txt
 watchtower==3.3.1
     # via
     #   -r requirements/requirements.txt

requirements/requirements.in

@@ -1,5 +1,6 @@
 apache-airflow==2.9.2
 apache-airflow-providers-amazon
+apache-airflow-providers-slack
 boto
 pydantic
 click

requirements/requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.12
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile --output-file=requirements/requirements.txt requirements/requirements.in
@@ -27,6 +27,7 @@ apache-airflow==2.9.2
     #   apache-airflow-providers-ftp
     #   apache-airflow-providers-http
     #   apache-airflow-providers-imap
+    #   apache-airflow-providers-slack
     #   apache-airflow-providers-smtp
     #   apache-airflow-providers-sqlite
 apache-airflow-providers-amazon==8.28.0
@@ -39,6 +40,7 @@ apache-airflow-providers-common-sql==1.16.0
     # via
     #   apache-airflow
     #   apache-airflow-providers-amazon
+    #   apache-airflow-providers-slack
     #   apache-airflow-providers-sqlite
 apache-airflow-providers-fab==1.3.0
     # via apache-airflow
@@ -50,6 +52,8 @@ apache-airflow-providers-http==4.13.0
     #   apache-airflow-providers-amazon
 apache-airflow-providers-imap==3.7.0
     # via apache-airflow
+apache-airflow-providers-slack==8.9.1
+    # via -r requirements/requirements.in
 apache-airflow-providers-smtp==1.8.0
     # via apache-airflow
 apache-airflow-providers-sqlite==3.9.0
@@ -65,6 +69,8 @@ asgiref==3.8.1
     #   apache-airflow-providers-http
 asn1crypto==1.5.1
     # via scramp
+async-timeout==4.0.3
+    # via aiohttp
 attrs==24.2.0
     # via
     #   aiohttp
@@ -143,6 +149,8 @@ docutils==0.21.2
     # via python-daemon
 email-validator==2.2.0
     # via flask-appbuilder
+exceptiongroup==1.2.2
+    # via anyio
 flask==2.2.5
     # via
     #   apache-airflow
@@ -196,6 +204,8 @@ googleapis-common-protos==1.63.2
     # via
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http
+greenlet==3.1.1
+    # via sqlalchemy
 grpcio==1.65.5
     # via opentelemetry-exporter-otlp-proto-grpc
 gunicorn==23.0.0
@@ -214,7 +224,9 @@ idna==3.7
     #   requests
     #   yarl
 importlib-metadata==8.0.0
-    # via opentelemetry-api
+    # via
+    #   apache-airflow
+    #   opentelemetry-api
 importlib-resources==6.4.4
     # via limits
 inflection==0.5.1
@@ -437,6 +449,8 @@ six==1.16.0
     #   python-dateutil
     #   rfc3339-validator
     #   wirerope
+slack-sdk==3.33.5
+    # via apache-airflow-providers-slack
 sniffio==1.3.1
     # via
     #   anyio
@@ -476,6 +490,8 @@ time-machine==2.15.0
 typing-extensions==4.12.2
     # via
     #   alembic
+    #   anyio
+    #   asgiref
     #   flask-limiter
     #   limits
     #   opentelemetry-sdk