From 5e38e7221bfbd460e2dea67492c694995f2d478a Mon Sep 17 00:00:00 2001
From: Barry Halper <69764062+barryhalper@users.noreply.github.com>
Date: Tue, 18 Feb 2025 12:00:03 +0000
Subject: [PATCH] OD-12 create resources for Glue

---
 service-infrastructure/aurora_rds/outputs.tf  |  5 ++
 service-infrastructure/glue/catalogue.tf      |  0
 service-infrastructure/glue/etl.tf            |  0
 .../glue/glue_connection.tf                   | 17 +++++
 service-infrastructure/glue/iam.tf            | 62 +++++++++++++++++++
 service-infrastructure/glue/main.tf           | 10 +++
 service-infrastructure/glue/outputs.tf        |  3 +
 service-infrastructure/glue/s3.tf             | 13 ++++
 service-infrastructure/glue/secret.tf         | 14 +++++
 .../glue/security_groups.tf                   | 34 ++++++++++
 service-infrastructure/glue/variables.tf      | 30 +++++++++
 service-infrastructure/modules.tf             | 14 ++++-
 service-infrastructure/networking/outputs.tf  |  8 +++
 13 files changed, 207 insertions(+), 3 deletions(-)
 create mode 100644 service-infrastructure/glue/catalogue.tf
 create mode 100644 service-infrastructure/glue/etl.tf
 create mode 100644 service-infrastructure/glue/glue_connection.tf
 create mode 100644 service-infrastructure/glue/iam.tf
 create mode 100644 service-infrastructure/glue/main.tf
 create mode 100644 service-infrastructure/glue/outputs.tf
 create mode 100644 service-infrastructure/glue/s3.tf
 create mode 100644 service-infrastructure/glue/secret.tf
 create mode 100644 service-infrastructure/glue/security_groups.tf
 create mode 100644 service-infrastructure/glue/variables.tf

diff --git a/service-infrastructure/aurora_rds/outputs.tf b/service-infrastructure/aurora_rds/outputs.tf
index bc6c8ca..2fc37b8 100644
--- a/service-infrastructure/aurora_rds/outputs.tf
+++ b/service-infrastructure/aurora_rds/outputs.tf
@@ -33,3 +33,8 @@ output "rds_db_reader_connection_string" {
   value       = local.reader_connection_string
   sensitive   = true
 }
+
+output "rds_db_reader_endpoint" {
+  value     = aws_rds_cluster.this.reader_endpoint
+  sensitive = true
+}
diff --git a/service-infrastructure/glue/catalogue.tf b/service-infrastructure/glue/catalogue.tf
new file mode 100644
index 0000000..e69de29
diff --git a/service-infrastructure/glue/etl.tf b/service-infrastructure/glue/etl.tf
new file mode 100644
index 0000000..e69de29
diff --git a/service-infrastructure/glue/glue_connection.tf b/service-infrastructure/glue/glue_connection.tf
new file mode 100644
index 0000000..695bf72
--- /dev/null
+++ b/service-infrastructure/glue/glue_connection.tf
@@ -0,0 +1,17 @@
+
+resource "aws_glue_connection" "example" {
+  name = "${var.prefix}-datawarehouse-db-connection"
+  connection_properties = {
+    JDBC_CONNECTION_URL = "jdbc:postgresql://${var.db_instance}:5432/epb"
+    SECRET_ID           = aws_secretsmanager_secret.glue_db_creds.id
+  }
+
+
+  physical_connection_requirements {
+    availability_zone      = var.subnet_group_az
+    security_group_id_list = [aws_security_group.glue_security_group.id]
+    subnet_id              = var.subnet_group_id
+
+  }
+
+}
\ No newline at end of file
diff --git a/service-infrastructure/glue/iam.tf b/service-infrastructure/glue/iam.tf
new file mode 100644
index 0000000..dabc4a1
--- /dev/null
+++ b/service-infrastructure/glue/iam.tf
@@ -0,0 +1,62 @@
+resource "aws_iam_role" "glueServiceRole" {
+  name = "AWSGlueServiceRole-${var.prefix}-glue"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Action = "sts:AssumeRole"
+      Effect = "Allow"
+      Principal = {
+        Service = "glue.amazonaws.com"
+      }
+    }]
+  })
+}
+
+
+resource "aws_iam_role_policy_attachment" "default" {
+  role       = aws_iam_role.glueServiceRole.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
+}
+
+resource "aws_iam_role_policy" "s3_bucket_policy" {
+  name = "${var.prefix}-glue-role-s3-policy"
+  role = aws_iam_role.glueServiceRole.id
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = [
+          "s3:PutObject*",
+          "s3:ListBucket",
+          "s3:GetObject*",
+          "s3:DeleteObject*",
+          "s3:GetBucketLocation"
+        ]
+        Effect = "Allow"
+        Resource = [
+          aws_s3_bucket.this.arn,
+          "${aws_s3_bucket.this.arn}/*"
+        ]
+      }
+    ]
+  })
+
+}
+
+resource "aws_iam_role_policy" "secret_access" {
+  name = "${var.prefix}-glue-role-secret-access-db-creds-policy"
+  role = aws_iam_role.glueServiceRole.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = [
+          "secretsmanager:GetSecretValue"
+        ]
+        Effect   = "Allow"
+        Resource = aws_secretsmanager_secret.glue_db_creds.id
+      }
+    ]
+  })
+}
diff --git a/service-infrastructure/glue/main.tf b/service-infrastructure/glue/main.tf
new file mode 100644
index 0000000..f505f42
--- /dev/null
+++ b/service-infrastructure/glue/main.tf
@@ -0,0 +1,10 @@
+terraform {
+  required_version = "~>1.3"
+
+  required_providers {
+    aws = {
+      version = "~>5.63"
+      source  = "hashicorp/aws"
+    }
+  }
+}
diff --git a/service-infrastructure/glue/outputs.tf b/service-infrastructure/glue/outputs.tf
new file mode 100644
index 0000000..4a79d45
--- /dev/null
+++ b/service-infrastructure/glue/outputs.tf
@@ -0,0 +1,3 @@
+output "glue_security_group_id" {
+  value = aws_security_group.glue_security_group.id
+}
diff --git a/service-infrastructure/glue/s3.tf b/service-infrastructure/glue/s3.tf
new file mode 100644
index 0000000..f080755
--- /dev/null
+++ b/service-infrastructure/glue/s3.tf
@@ -0,0 +1,13 @@
+resource "aws_s3_bucket" "this" {
+  bucket        = "${var.prefix}-data-lake"
+  force_destroy = false
+}
+
+resource "aws_s3_bucket_public_access_block" "block_public_access" {
+  bucket                  = aws_s3_bucket.this.id
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
diff --git a/service-infrastructure/glue/secret.tf b/service-infrastructure/glue/secret.tf
new file mode 100644
index 0000000..0c125ed
--- /dev/null
+++ b/service-infrastructure/glue/secret.tf
@@ -0,0 +1,14 @@
+
+resource "aws_secretsmanager_secret" "glue_db_creds" {
+  name = "GLUE-DATAWAREHOUSE-CREDS"
+}
+
+resource "aws_secretsmanager_secret_version" "glue_db_creds_varsion" {
+  secret_id     = aws_secretsmanager_secret.glue_db_creds.id
+  secret_string = <<EOF
+   {
+    "username": ${var.db_user}",
+    "password":  ${var.db_password}"
+   }
+EOF
+}
\ No newline at end of file
diff --git a/service-infrastructure/glue/security_groups.tf b/service-infrastructure/glue/security_groups.tf
new file mode 100644
index 0000000..c0892ef
--- /dev/null
+++ b/service-infrastructure/glue/security_groups.tf
@@ -0,0 +1,34 @@
+locals {
+  sg_name = "${var.prefix}-glue-sg"
+}
+
+resource "aws_security_group" "glue_security_group" {
+  name   = local.sg_name
+  vpc_id = var.vpc_id
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = local.sg_name
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+}
+
+resource "aws_security_group_rule" "self" {
+  type              = "ingress"
+  security_group_id = aws_security_group.glue_security_group.id
+  from_port         = 0
+  protocol          = "tcp"
+  to_port           = 65535
+  self              = true
+}
+
diff --git a/service-infrastructure/glue/variables.tf b/service-infrastructure/glue/variables.tf
new file mode 100644
index 0000000..e66deff
--- /dev/null
+++ b/service-infrastructure/glue/variables.tf
@@ -0,0 +1,30 @@
+variable "prefix" {
+  type = string
+}
+
+variable "vpc_id" {
+  type = string
+}
+
+variable "subnet_group_id" {
+  type = string
+}
+
+variable "subnet_group_az" {
+  type = string
+}
+
+variable "db_instance" {
+  type = string
+}
+
+variable "db_user" {
+  type = string
+}
+
+variable "db_password" {
+  type = string
+}
+
+
+
diff --git a/service-infrastructure/modules.tf b/service-infrastructure/modules.tf
index 0c2b635..5d6ab4c 100644
--- a/service-infrastructure/modules.tf
+++ b/service-infrastructure/modules.tf
@@ -746,7 +746,6 @@ module "warehouse_api_application" {
   cloudwatch_ecs_events_arn = module.logging.cloudwatch_ecs_events_arn
 }
 
-
 module "warehouse_database" {
   source = "./aurora_rds"
 
@@ -756,7 +755,7 @@ module "warehouse_database" {
   instance_parameter_group_name = module.data_warehouse_parameter_groups.rds_pg_param_group_name
   postgres_version              = var.postgres_aurora_version
   prefix                        = "${local.prefix}-warehouse"
-  security_group_ids            = [module.warehouse_application.ecs_security_group_id, module.bastion.security_group_id, module.warehouse_scheduled_tasks_application.ecs_security_group_id, module.warehouse_api_application.ecs_security_group_id]
+  security_group_ids            = [module.warehouse_application.ecs_security_group_id, module.bastion.security_group_id, module.warehouse_scheduled_tasks_application.ecs_security_group_id, module.warehouse_api_application.ecs_security_group_id, module.data_warehouse_glue.glue_security_group_id]
   storage_backup_period         = var.storage_backup_period
   subnet_group_name             = local.db_subnet
   vpc_id                        = module.networking.vpc_id
@@ -774,7 +773,6 @@ module "warehouse_redis" {
   vpc_id                        = module.networking.vpc_id
 }
 
-
 module "bastion" {
   source    = "./bastion"
   subnet_id = module.networking.private_subnet_ids[0]
@@ -1007,3 +1005,13 @@ module "rds_kms_key" {
   environment = var.environment
 }
 
+module "data_warehouse_glue" {
+  source          = "./glue"
+  prefix          = local.prefix
+  subnet_group_id = module.networking.private_db_subnet_first_id
+  db_instance     = module.warehouse_database.rds_db_reader_endpoint
+  db_user         = module.warehouse_database.rds_db_username
+  db_password     = module.warehouse_database.rds_db_password
+  subnet_group_az = module.networking.private_db_subnet_first_az
+  vpc_id          = module.networking.vpc_id
+}
\ No newline at end of file
diff --git a/service-infrastructure/networking/outputs.tf b/service-infrastructure/networking/outputs.tf
index 6c8f7e7..22453b9 100644
--- a/service-infrastructure/networking/outputs.tf
+++ b/service-infrastructure/networking/outputs.tf
@@ -33,3 +33,11 @@ output "private_db_subnet_cidr" {
 output "private_db_subnet_ids" {
   value = aws_subnet.private_db[*].id
 }
+
+output "private_db_subnet_first_id" {
+  value = aws_subnet.private_db[0].id
+}
+
+output "private_db_subnet_first_az" {
+  value = aws_subnet.private_db[0].availability_zone
+}