Merge pull request #23 from alliance-genome/SCRUM-2180_additional_sch…

…eduled_backups

SCRUM-2180 additional scheduled backups

aws_infra/cdk_classes/lambda_stack.py

@@ -23,14 +23,6 @@ def __init__(self, scope: Stack, ecs_cluster_arn: str, ecs_task_def_arn: str,
 			]
 		)
 
-		# Create nightly backup event rule (scheduled backup trigger)
-		nightly_backups_event_rule = aws_events.Rule(scope, "nightly-backup-schedule",
-			rule_name='NightlyBackupSchedule',
-			description='Run nightly at 9 PM UTC',
-			enabled=True,
-			schedule=aws_events.Schedule.cron(minute='0', hour='21', day='*', month='*', year='*')
-		)
-
 		# Copy helper file into lambda bundle
 		dirname = os.path.dirname(os.path.realpath(__file__))
 		shutil.copyfile(os.path.join(dirname, '..','..','app','interfaces','helper.py'),
@@ -56,10 +48,19 @@ def __init__(self, scope: Stack, ecs_cluster_arn: str, ecs_task_def_arn: str,
 		os.remove(os.path.join(dirname, '..', 'lambda_bundle', 'helper.py'))
 
 		# Add targets to nightly backup event rule for every DB requiring nightly backup
-		nightly_backup_targets = json.load(open(os.path.join(dirname, '..', 'resources', 'backup_list.json'), 'r'))
+		backup_list = json.load(open(os.path.join(dirname, '..', 'resources', 'backup_list.json'), 'r'))
+
+		for backup_target in backup_list:
+			# Create nightly backup event rule (scheduled backup trigger)
+			# for each target, as each event only support max 5 targets
+			rule_name = "agrdb-"+backup_target['name']
+			backup_event_rule = aws_events.Rule(scope, rule_name,
+				rule_name=rule_name,
+				enabled=True,
+				schedule=aws_events.Schedule.expression(backup_target['schedule'])
+			)
 
-		for event_obj in nightly_backup_targets:
-			nightly_backups_event_rule.add_target(aws_events_targets.LambdaFunction(
+			backup_event_rule.add_target(aws_events_targets.LambdaFunction(
 				handler=aws_lambda_fn,
-				event=aws_events.RuleTargetInput.from_object(event_obj)
+				event=aws_events.RuleTargetInput.from_object(backup_target['payload'])
 			))

aws_infra/resources/backup_list.json

@@ -1,42 +1,74 @@
 [
 	{
-		"action": "backup",
-		"target_env": "dev",
-		"identifier": "chipmunk"
+		"name": "chipmunk-dev-nightly-backup",
+		"schedule": "cron(0 21 * * ? *)",
+		"payload": {
+			"action": "backup",
+			"target_env": "dev",
+			"identifier": "chipmunk"
+		}
 	},
 	{
-		"action": "backup",
-		"target_env": "production",
-		"identifier": "chipmunk"
+		"name": "chipmunk-prod-nightly-backup",
+		"schedule": "cron(0 21 * * ? *)",
+		"payload": {
+				"action": "backup",
+				"target_env": "production",
+				"identifier": "chipmunk"
+		}
 	},
 	{
-		"action": "backup",
-		"identifier": "curation",
-		"target_env": "alpha"
+		"name": "curation-alpha-nightly-backup",
+		"schedule": "cron(0 21 * * ? *)",
+		"payload": {
+			"action": "backup",
+			"identifier": "curation",
+			"target_env": "alpha"
+		}
 	},
 	{
-		"action": "backup",
-		"identifier": "curation",
-		"target_env": "beta"
+		"name": "curation-beta-nightly-backup",
+		"schedule": "cron(0 21 * * ? *)",
+		"payload": {
+			"action": "backup",
+			"identifier": "curation",
+			"target_env": "beta"
+		}
 	},
 	{
-		"action": "backup",
-		"identifier": "curation",
-		"target_env": "production"
+		"name": "curation-prod-nightly-backup",
+		"schedule": "cron(0 21 * * ? *)",
+		"payload": {
+			"action": "backup",
+			"identifier": "curation",
+			"target_env": "production"
+		}
 	},
 	{
-		"action": "backup",
-		"target_env": "mati",
-		"identifier": "alpha"
+		"name": "mati-alpha-nightly-backup",
+		"schedule": "cron(0 21 * * ? *)",
+		"payload": {
+			"action": "backup",
+			"target_env": "mati",
+			"identifier": "alpha"
+		}
 	},
 	{
-		"action": "backup",
-		"target_env": "mati",
-		"identifier": "beta"
+		"name": "mati-beta-nightly-backup",
+		"schedule": "cron(0 21 * * ? *)",
+		"payload": {
+			"action": "backup",
+			"target_env": "mati",
+			"identifier": "beta"
+		}
 	},
 	{
-		"action": "backup",
-		"target_env": "mati",
-		"identifier": "production"
+		"name": "mati-prod-nightly-backup",
+		"schedule": "cron(0 21 * * ? *)",
+		"payload": {
+			"action": "backup",
+			"target_env": "mati",
+			"identifier": "production"
+		}
 	}
 ]

img/AWS_diagram.drawio.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<mxfile modified="2022-12-01T18:46:57.619Z" host="app.diagrams.net" agent="5.0 (Windows)" etag="Vutw6MRz8OMn0z3aT5De" version="20.0.3" type="google"><diagram id="Ht1M8jgEwFfnCIfOTk4-" name="Page-1">7Vtbc9soFP41ntl9iEcXdHv0tZvZdDbbTCfdfckQC9tqZOFFOHH66wsSyBJg1W58yaZxmlQcLoLDdw4fB9xxB4v1BwKX8484RmnHseJ1xx12HCd0XPaXC55Lgef6pWBGkrgU2RvBTfINCaElpKskRnmjIMU4pcmyKZzgLEMT2pBBQvBTs9gUp823LuEMaYKbCUx16W0S07mQ2n60yfgDJbM5lQMOyowFlIXFSPI5jPFTTeSOOu6AYEzLp8V6gFKuO6mXst54S27VMYIyukuFWz/9Cv79+AH2J/Hfl1e99WfkXYhRPMJ0JQbcW8BvOGOy0eCG/R2kq5wiIoZAn6VepkmaDnCKSZF0R+OxNXY7bj+nBD8gmZPhjJXvxzCfI94NiyUeEaEJU3AvTWYZk1G8ZNIpzuiNaN4S6Vr77GfMHty+6C9rA623KsKu1MtgifACUfLMiogKviVGLSDJsNb1SsnTZoqreZvXZteVQihgNata32iePQjlmyfi8+jb9ede769LMr5I8nDmDYL/Lmwn0nSMYgZFkcSEzvEMZzAdbaR9gldZXGl2U+YKc6UObSb8iih9FnYFVxQz0ZwuUpGbU0hoj9sJf8cSZVI2Tnj/i2ZRFislmKSWX065tA5HAwGbvtDiP7zmOqFfxLv58z+8jS7wZXq4li/lieda4hqRhKkaESHbMutMgXhFJqhN1XL+2ThniLaVtAUu+Ey0QougFNLksek0TCApqjJ1wudagSVOMprXWr7mgg1i3chqINYNFFPfrzx7KHtgru1YzdrAVkBfaljUUnBf6eQFpiCXiDOagnBcRzWFjKnpS9UwS5TG4MnkxhaK1HM9dUBrkBivW4NxrQhfhS14oYJtP2i1Ba28tY8t+KotWM5OtqA35DYbihSbKrV/PJuyDCv9bbHAX11q1oZXNE0yhlrJqTjEZgTGCdqsysJElIXaA17fB9uIgMIawiCyI38bQbiC9yi9xnlCE8yJwj2mFC+2MggoUhPWR24Xddsu2EVp+LZjYBswX5YDnSZr3o8+I2tLnrlYzzit7cKnHHRZ5xkZIpD3546x0ofyxQViERk9Ig7c8oUp73sfTh5mhV+qjXlafNpMViM3W5mMA4Imtp3AwGRsy8Bkoha7fBmRkf6khrQruLiPYcfxU/b2/j1jk/6MP01X2aSY3J0AaKKEoxZgNrE2tLyBHezFUM8JQBOojKBMC93eVao8FfZAULHmVvQFoQ4+KTsC+HzdzckNzY2rI7C/mjwgelj8sTzf6rluUMsbJgSJ+WGQI1w5KkCdIODdN1CGatpeHUqNiCSoXBQvJwUcWbJ8apbK3ZNBNVQpQGR1Q6AhNbS6Ej51sNbEL8GrkVJ5vwLRNdLGH5JVo75kBGjHjdupuSkACtC8fbhmtc/axCV+jmsGW7pxMrIpg34GL6w64EGKV/EtpJM5K3+FZ7le5CjsYA/HPHYBCHWIv1bmYPTJE67mp0LNp/K6fgQaKPQkCmsu1zMwUxWsB3O2LdTg0/BGx90S53TGVi+WP9RBmDPewLVZTIJwDCzD67N/bASD8tdjRQdc0nU8g9AkC3ShrRdj/9mmN6hCkyzQhbZejKdkr5tCkyzw9B6rtW1DbVup7fEg70vpFxgGThGN3sPK2Qf0o1+GfpE47xwkmq6SK082cYI9gJkjgHOQKhnYFvE7EdreP6xdw4C/I1fbL+jYzusOyNVCnauZ58t5IVV7EVoCbVF49/T/I08/DsKRBfbz9FsiQW/V06PJgTz9buemroHSuceidHZwDl//Ez56t/20yWPr3v2Q+2ln1w11uaU7l5e2zx8oOcHh+FkmtoxtHS9Ssj1KEURqANmyu8D2IscGEfDDSGGIu4Y/IvXMTr20cbjwh1n35z++PsVNjsMfXx8N+dvDAaeOEIa+gk3QfpNDLe/5tgLZtohi4Ntdq/Zxmu92ra7lhaHjek4EbLmW7mtuYWAe0snMzdEofHl1jcL8QY/qMIZEIaOcpJ3nH+MEEjAK/QZPICuN3tkHIpph81gbMKIp0Vljmj7QmSawu657JBISGraKJbri5FHCq7xUMWaQhxTJfPa6WpH3Deb7BvPVWfsLNphTAfYD2b5yCMaWsNNdKmg7dNRuThWe8Z4knLVoywzimazQb5QRnYt7mKP49+OvOBrgqgoKRN8kEAud33GF3xUKPwwiga1edDEHPnxD4MM/WuBDX4wGwz8vFjCDMzZwVjubEqgBro6CbZDxev4gbFzKs7fO/A9u7VdNqSg4wLS4QXNSbKnr2pQA3zNQhPBYk1J90+M8Jw+d+pV6eQqx+8nDZkPXDYDT3NQBL/yZbZ067XtGykSJ1xEpM3xnoMUyzxYo23bNdzT4pC9Sn9CSO3ksXvzOSd856VuhAhNymGVGPd4+Kh9lyc33Acv4yOZLle7oOw==</diagram></mxfile>