Skip to content

Commit a09364a

Browse files
jozefbakusjozefbakus
authored
#818: Regularly execute archive_dag_instances function (#828)
* #818: Regularly execute archive_dag_instances function
1 parent 03fca8e commit a09364a

File tree

5 files changed

+136
-0
lines changed

5 files changed

+136
-0
lines changed

‎src/main/resources/db_scripts/db_script_latest.sql‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -363,3 +363,11 @@ BEGIN
363363
END;
364364
$$ LANGUAGE plpgsql;
365365

366+
create table if not exists housekeepinglock
367+
(
368+
locked boolean not null,
369+
started_at timestamp
370+
);
371+
372+
insert into "housekeepinglock" ("locked", "started_at")
373+
values (false, null);

‎src/main/resources/db_scripts/housekeeping/cleanUpRuns.sh‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
#! /bin/bash
2+
#
3+
# Copyright 2018 ABSA Group Limited
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
17+
RECIPIENTS=$1
18+
RETENTION_DAYS=$2
19+
ENV=$3
20+
CONFIG_FILE_LOCATION=${4:-"../../application.properties"}
21+
22+
if [[ -n "$RECIPIENTS" && -n "$RETENTION_DAYS" && -n "$ENV" ]]; then
23+
echo "Starting runs clean up with retention days: $RETENTION_DAYS, env: $ENV and recipients: $RECIPIENTS"
24+
else
25+
echo "Incorrect input. Exiting the program."
26+
exit 1
27+
fi
28+
29+
db_url=$(grep '^db.url=' "$CONFIG_FILE_LOCATION" | awk -F'jdbc:postgresql://' '{print $2}' | awk -F'\\?' '{print $1}')
30+
db_user=$(grep '^db.user=' "$CONFIG_FILE_LOCATION" | awk -F'=' '{print $2}')
31+
db_password=$(grep '^db.password=' "$CONFIG_FILE_LOCATION" | awk -F'=' '{print $2}')
32+
connection_url="postgresql://$db_user:$db_password@$db_url"
33+
34+
obtain_lock_response=$(psql -X $connection_url -c "UPDATE housekeepinglock SET locked = 'true', started_at = now() WHERE locked = false AND started_at is null;" 2>&1)
35+
36+
if [[ $obtain_lock_response == "UPDATE 1" ]]; then
37+
echo "Exclusive lock for running runs clean up job obtained."
38+
39+
archive_dag_instances_response=$(psql -X $connection_url -c "CALL archive_dag_instances(i_to_ts => (now() - '$RETENTION_DAYS days'::interval)::timestamp without time zone);" 2>&1)
40+
release_lock_response=$(psql -X $connection_url -c "UPDATE housekeepinglock SET locked = 'false', started_at = null;" 2>&1)
41+
42+
message=""
43+
subject=""
44+
if [[ $archive_dag_instances_response == *"ERROR"* ]]; then
45+
subject="Hyperdrive Notifications - $ENV - Runs clean up failed!"
46+
message+="Runs clean up job failed with following output:"
47+
else
48+
subject="Hyperdrive Notifications - $ENV - Runs clean up succeeded"
49+
message+="Runs clean up job succeeded with following output:"
50+
fi
51+
message+="\n$archive_dag_instances_response"
52+
53+
if [[ $release_lock_response == "UPDATE 1" ]]; then
54+
message+="\n\nLock for running runs clean job was successfully released"
55+
else
56+
message+="\n\nLock for running runs clean job was not released!"
57+
fi
58+
echo -e "$message"
59+
echo -e "$message" | mailx -s "$subject" -r "[email protected]" "$RECIPIENTS"
60+
61+
else
62+
obtain_lock_time_response=$(psql -X $connection_url -c "SELECT started_at FROM housekeepinglock;" 2>&1)
63+
64+
timestamp_string=$(echo "$obtain_lock_time_response" | grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}')
65+
timestamp_unix=$(date -jf "%Y-%m-%d %H:%M:%S" "$timestamp_string" +%s)
66+
current_unix=$(date +%s)
67+
time_difference=$((current_unix - timestamp_unix))
68+
days_difference=$((time_difference / 86400))
69+
70+
if [ "$days_difference" -gt "2" ]; then
71+
message="Runs clean up has been running for more than 3 days."
72+
echo -e "$message"
73+
echo -e "$message" | mailx -s "Hyperdrive Notifications - $ENV - Runs clean up locked" -r "[email protected]" "$RECIPIENTS"
74+
else
75+
echo "Could not execute runs clean up job because there is another instance currently running for ($((time_difference / 60)) minutes)."
76+
fi
77+
fi

‎src/main/resources/db_scripts/liquibase/db.changelog.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,3 +98,6 @@ databaseChangeLog:
9898
- include:
9999
relativeToChangelogFile: true
100100
file: v0.5.20.add-diagnostics-field.yml
101+
- include:
102+
relativeToChangelogFile: true
103+
file: v0.5.20.add-housekeeping-lock-table.yml

‎src/main/resources/db_scripts/liquibase/v0.5.20.add-housekeeping-lock-table.sql‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Copyright 2018 ABSA Group Limited
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
create table if not exists housekeepinglock
17+
(
18+
locked boolean not null,
19+
started_at timestamp
20+
);
21+
22+
insert into "housekeepinglock" ("locked", "started_at")
23+
values (false, null);

‎src/main/resources/db_scripts/liquibase/v0.5.20.add-housekeeping-lock-table.yml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#
2+
# Copyright 2018 ABSA Group Limited
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
16+
databaseChangeLog:
17+
- changeSet:
18+
id: v0.5.20.add-housekeeping-lock-table
19+
logicalFilePath: v0.5.20.add-housekeeping-lock-table
20+
author: [email protected]
21+
context: default
22+
changes:
23+
- sqlFile:
24+
relativeToChangelogFile: true
25+
path: v0.5.20.add-housekeeping-lock-table.sql

0 commit comments

Comments
 (0)