Skip to content

Commit 0bc9e82

Browse files
authored
Delete the CloudWatch log group left by example_mwaa_serverless (#76810)
MWAA Serverless creates a log group named `/aws/mwaa-serverless/<workflow id>/` for every workflow that does not bring its own, and `DeleteWorkflow` leaves it in place. The `example_mwaa_serverless` system test never removed it, so each run leaked one log group into the test account. The workflow ID ends in a service-generated suffix, so the log group name is derived at run time from the ARN returned by `create_workflow`. Teardown copies the workflow logs into the task log and then deletes the group, whether or not the run passed, as `example_dms` does. The role running the test needs `logs:FilterLogEvents` and `logs:DeleteLogGroup` on `log-group:/aws/mwaa-serverless/*`.
1 parent c03cb84 commit 0bc9e82

1 file changed

Lines changed: 18 additions & 1 deletion

File tree

‎providers/amazon/tests/system/amazon/aws/example_mwaa_serverless.py‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818

1919
from datetime import datetime
2020

21+
import boto3
22+
2123
from airflow.providers.amazon.aws.operators.mwaa_serverless import (
2224
MwaaServerlessCreateWorkflowOperator,
2325
MwaaServerlessDeleteWorkflowOperator,
@@ -31,7 +33,7 @@
3133
S3DeleteBucketOperator,
3234
)
3335
from airflow.providers.amazon.aws.sensors.mwaa_serverless import MwaaServerlessWorkflowRunSensor
34-
from airflow.providers.common.compat.sdk import DAG, chain
36+
from airflow.providers.common.compat.sdk import DAG, chain, task
3537

3638
from system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder
3739
from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS
@@ -77,6 +79,20 @@
7779

7880
sys_test_context_task = SystemTestContextBuilder().add_variable(ROLE_ARN_KEY).build()
7981

82+
83+
@task(trigger_rule=TriggerRule.ALL_DONE)
84+
def print_and_delete_workflow_logs(workflow_arn: str):
85+
"""Copy the workflow logs into this task's log, then delete their CloudWatch log group."""
86+
# The service names the log group after the workflow ID, the last ARN segment, which ends in
87+
# a generated suffix. The name is therefore only known once the workflow exists.
88+
log_group_name = f"/aws/mwaa-serverless/{workflow_arn.rsplit('/', 1)[-1]}/"
89+
logs_client = boto3.client("logs")
90+
for page in logs_client.get_paginator("filter_log_events").paginate(logGroupName=log_group_name):
91+
for event in page["events"]:
92+
print(event["message"])
93+
logs_client.delete_log_group(logGroupName=log_group_name)
94+
95+
8096
with DAG(
8197
dag_id=DAG_ID,
8298
schedule=None,
@@ -210,6 +226,7 @@
210226
# TEST TEARDOWN
211227
delete_workflow,
212228
delete_bucket,
229+
print_and_delete_workflow_logs(workflow_arn),
213230
)
214231

215232
from tests_common.test_utils.watcher import watcher

0 commit comments

Comments
 (0)