From 071122db2509c478dafb9708ad047a696f2b3f93 Mon Sep 17 00:00:00 2001
From: Graham Wright <graham.wright@seqera.io>
Date: Sat, 15 Jun 2024 16:32:42 -0400
Subject: [PATCH 1/8] Working through POC to pass objects to external scripts.

---
 .../example_import_tfvars_via_extractor.py    | 61 -------------
 .../generate_db_connection_string.py          | 23 ++++-
 .../data_external/generate_dns_values.py      | 54 +++++++++++
 .../template_get_tfvars_and_query.py          | 40 +++++++++
 000_main.tf                                   | 90 +++++++++----------
 001_vpc.tf                                    | 16 ----
 6 files changed, 157 insertions(+), 127 deletions(-)
 delete mode 100644 .githooks/data_external/example_import_tfvars_via_extractor.py
 create mode 100644 .githooks/data_external/generate_dns_values.py
 create mode 100644 .githooks/data_external/template_get_tfvars_and_query.py

diff --git a/.githooks/data_external/example_import_tfvars_via_extractor.py b/.githooks/data_external/example_import_tfvars_via_extractor.py
deleted file mode 100644
index 84874bf6..00000000
--- a/.githooks/data_external/example_import_tfvars_via_extractor.py
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/usr/bin/env python3
-import os
-import json
-import sys
-from types import SimpleNamespace
-from typing import List
-
-sys.dont_write_bytecode = True
-
-# NOTE! This part is a bit convoluted but there's a method to the madness:
-# - Script executes from `/~/cx-field-tools-installer`. 
-# - Need to change dir into .githooks (to avoid the `.` import problem), import, then return to project root so the extractor can find tfvars.
-# - Error this is solving: `ImportError: attempted relative import with no known parent package``
-project_root = os.getcwd()
-os.chdir(f"{project_root}/.githooks")
-sys.path.append(".")
-from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
-
-# Extract tfvars just like we do with the Python validation script
-os.chdir(project_root)
-data_dictionary = get_tfvars_as_json()
-data = SimpleNamespace(**data_dictionary)
-
-# Connection string modifiers
-mysql8_connstring = "allowPublicKeyRetrieval=true&useSSL=false"
-v24plus_connstring = "permitMysqlScheme=true"
-
-
-def return_tf_payload(status: str, value: str):
-    payload = {'status': status, 'value': value}
-    print(json.dumps(payload))
-
-
-def generate_connection_string(mysql8: str, v24plus: str):
-    connection_string = ""
-    add_mysql8 = False
-    add_v24plus = False
-
-    if mysql8.startswith("8."): 
-        add_mysql8 = True
-
-    if v24plus >= "v24":
-        add_v24plus = True
-
-    if add_mysql8 and add_v24plus:
-        connection_string = f"?{mysql8_connstring}&{v24plus_connstring}"
-    elif add_mysql8 and not add_v24plus:
-        connection_string = f"?{mysql8_connstring}"
-    elif not add_mysql8 and add_v24plus:
-        connection_string = f"?{v24plus_connstring}"
-
-    return connection_string
-
-
-if data.flag_use_container_db:
-    connection_string = generate_connection_string(data.db_container_engine_version, data.tower_container_version)
-else:
-    connection_string = generate_connection_string(data.db_engine_version, data.tower_container_version)
-
-return_tf_payload("0", connection_string)
-exit(0)
\ No newline at end of file
diff --git a/.githooks/data_external/generate_db_connection_string.py b/.githooks/data_external/generate_db_connection_string.py
index 882e528e..9d9c1c8f 100644
--- a/.githooks/data_external/generate_db_connection_string.py
+++ b/.githooks/data_external/generate_db_connection_string.py
@@ -7,9 +7,30 @@
 
 sys.dont_write_bytecode = True
 
+
+## ------------------------------------------------------------------------------------
+## Extract TFvars and get query values
+## ------------------------------------------------------------------------------------
+# NOTE! This part is a bit convoluted but there's a method to the madness:
+#   - Script executes from `~/cx-field-tools-installer`. 
+#   - Change into .githooks (to avoid the `.` import problem), import, return to project root to extract tfvars.
+#   - Error thrown without workaround: `ImportError: attempted relative import with no known parent package`
+#   - Query object to receive objects via TF data call (i.e. resources)
+project_root = os.getcwd()
+os.chdir(f"{project_root}/.githooks")
+sys.path.append(".")
+from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
+
+# Extract tfvars just like we do with the Python validation script
+os.chdir(project_root)
+data_dictionary = get_tfvars_as_json()
+data = SimpleNamespace(**data_dictionary)
+
 # Much simpler way to get variable passed in (via Terraform sending to stdin)
 query = json.load(sys.stdin)
-data = SimpleNamespace(**query)
+query = SimpleNamespace(**query)
+## ------------------------------------------------------------------------------------
+
 
 # Get engine_version depending on whether container DB or RDS instance is in play
 engine_version = data.db_container_engine_version if data.flag_use_container_db else data.db_engine_version
diff --git a/.githooks/data_external/generate_dns_values.py b/.githooks/data_external/generate_dns_values.py
new file mode 100644
index 00000000..1a97a70f
--- /dev/null
+++ b/.githooks/data_external/generate_dns_values.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+import os
+import json
+import sys
+from types import SimpleNamespace
+from typing import List
+
+sys.dont_write_bytecode = True
+
+
+## ------------------------------------------------------------------------------------
+## Extract TFvars and get query values
+## ------------------------------------------------------------------------------------
+# NOTE! This part is a bit convoluted but there's a method to the madness:
+#   - Script executes from `~/cx-field-tools-installer`. 
+#   - Change into .githooks (to avoid the `.` import problem), import, return to project root to extract tfvars.
+#   - Error thrown without workaround: `ImportError: attempted relative import with no known parent package`
+#   - Query object to receive objects via TF data call (i.e. resources)
+project_root = os.getcwd()
+os.chdir(f"{project_root}/.githooks")
+sys.path.append(".")
+from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
+
+# Extract tfvars just like we do with the Python validation script
+os.chdir(project_root)
+data_dictionary = get_tfvars_as_json()
+data = SimpleNamespace(**data_dictionary)
+
+# Much simpler way to get variable passed in (via Terraform sending to stdin)
+query = json.load(sys.stdin)
+# query = SimpleNamespace(**query)
+## ------------------------------------------------------------------------------------
+
+
+with open("query.json", "w") as file:
+    file.write(str(query))
+
+# Determine kinda of DNS record to create
+dns_create_alb_record = True if (data.flag_create_load_balancer and not data.flag_create_hosts_file_entry) else False
+dns_create_ec2_record = True if (not data.flag_create_load_balancer and not data.flag_create_hosts_file_entry) else False
+
+if data.flag_create_route53_private_zone == True:
+    dns_zone_id = query.aws_route53_zone
+
+
+def return_tf_payload(status: str, value: str):
+    payload = {'status': status, 'value': value}
+    print(json.dumps(payload))
+
+
+if __name__ == '__main__':
+    return_tf_payload("0", "abc")
+    
+    exit(0)
\ No newline at end of file
diff --git a/.githooks/data_external/template_get_tfvars_and_query.py b/.githooks/data_external/template_get_tfvars_and_query.py
new file mode 100644
index 00000000..7cc33652
--- /dev/null
+++ b/.githooks/data_external/template_get_tfvars_and_query.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+import os
+import json
+import sys
+from types import SimpleNamespace
+from typing import List
+
+sys.dont_write_bytecode = True
+
+
+## ------------------------------------------------------------------------------------
+## Extract TFvars and get query values
+## ------------------------------------------------------------------------------------
+# NOTE! This part is a bit convoluted but there's a method to the madness:
+#   - Script executes from `~/cx-field-tools-installer`. 
+#   - Change into .githooks (to avoid the `.` import problem), import, return to project root to extract tfvars.
+#   - Error thrown without workaround: `ImportError: attempted relative import with no known parent package`
+#   - Query object to receive objects via TF data call (i.e. resources)
+project_root = os.getcwd()
+os.chdir(f"{project_root}/.githooks")
+sys.path.append(".")
+from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
+
+# Extract tfvars just like we do with the Python validation script
+os.chdir(project_root)
+data_dictionary = get_tfvars_as_json()
+data = SimpleNamespace(**data_dictionary)
+
+# Much simpler way to get variable passed in (via Terraform sending to stdin)
+query = json.load(sys.stdin)
+query = SimpleNamespace(**query)
+## ------------------------------------------------------------------------------------
+
+
+def return_tf_payload(status: str, value: str):
+    payload = {'status': status, 'value': value}
+    print(json.dumps(payload))
+
+
+
diff --git a/000_main.tf b/000_main.tf
index ad37321e..e0bcf7a9 100644
--- a/000_main.tf
+++ b/000_main.tf
@@ -49,15 +49,36 @@ resource "random_pet" "stackname" {
 
 data "aws_caller_identity" "current" {}
 
+data "aws_vpc" "preexisting" { 
+  id = local.vpc_id 
+}
+
+# https://stackoverflow.com/questions/67562197/terraform-loop-through-ids-list-and-generate-data-blocks-from-it-and-access-it
+data "aws_subnet" "existing" {
+  # Creates a map with the keys being the CIDRs --  e.g. `data.aws_subnet.public["10.0.0.0/20"].id
+  for_each = toset(local.subnets_all)
+  vpc_id     = local.vpc_id
+  cidr_block = each.key
+}
 
 # https://medium.com/@leslie.alldridge/terraform-external-data-source-using-custom-python-script-with-example-cea5e618d83e
 data "external" "generate_db_connection_string" {
   program = ["python3", "${path.module}/.githooks/data_external/generate_db_connection_string.py"]
   query = {
-    tower_container_version = var.tower_container_version
-    flag_use_container_db = var.flag_use_container_db
-    db_container_engine_version = var.db_container_engine_version
-    db_engine_version = var.db_engine_version
+    # tower_container_version = var.tower_container_version
+    # flag_use_container_db = var.flag_use_container_db
+    # db_container_engine_version = var.db_container_engine_version
+    # db_engine_version = var.db_engine_version
+  }
+}
+
+data "external" "generate_dns_valeus" {
+  program = ["python3", "${path.module}/.githooks/data_external/generate_dns_values.py"]
+  query = {
+    r53_privatezone = jsonencode(aws_route53_zone.private) # [0])
+    r53_public =  jsonencode(data.aws_route53_zone.public) # [0])
+    subnets = jsonencode(data.aws_subnet.existing)       # Adding in these values wont be known til created.
+    # ec2 = jsonencode(aws_instance.ec2)                     # Adding in these values wont be known til created.
   }
 }
 
@@ -71,56 +92,27 @@ locals {
   # ---------------------------------------------------------------------------------------
   global_prefix = var.flag_use_custom_resource_naming_prefix == true ? var.custom_resource_naming_prefix : "tf-${var.app_name}-${random_pet.stackname.id}"
 
-
   # Networking
   # ---------------------------------------------------------------------------------------
   vpc_id = var.flag_create_new_vpc == true ? module.vpc[0].vpc_id : var.vpc_existing_id
-  vpc_private_route_table_ids = var.flag_create_new_vpc == true ? module.vpc[0].private_route_table_ids : data.aws_route_tables.preexisting.ids
-
-  # If creating VPC from scratch, map all subnet CIDRS to corresponding subnet ID
-  #  zipmap -- turn 2 lists into a dictionary. https://developer.hashicorp.com/terraform/language/functions/zipmap
-  #  merge  -- join  two dictionaries. https://developer.hashicorp.com/terraform/language/functions/merge
-  vpc_new_cidr_block_to_id_public  = var.flag_create_new_vpc == true ? zipmap(module.vpc[0].public_subnets_cidr_blocks, module.vpc[0].public_subnets) : {}
-  vpc_new_cidr_block_to_id_private = var.flag_create_new_vpc == true ? zipmap(module.vpc[0].private_subnets_cidr_blocks, module.vpc[0].private_subnets) : {}
-  vpc_new_cidr_block_to_id_unified = var.flag_create_new_vpc == true ? merge(local.vpc_new_cidr_block_to_id_public, local.vpc_new_cidr_block_to_id_private) : {}
-
-  # Regardless of whether we build a new VPC or use existing, assign the subnet CIDRs to a common variable for subsequent subnet ID lookup. 
-  #  concat -- join lists of strings. https://developer.hashicorp.com/terraform/language/functions/concat
-  subnets_ec2   = var.flag_create_new_vpc == true ? var.vpc_new_ec2_subnets : var.vpc_existing_ec2_subnets
-  subnets_batch = var.flag_create_new_vpc == true ? var.vpc_new_batch_subnets : var.vpc_existing_batch_subnets
-  subnets_db    = var.flag_create_new_vpc == true ? var.vpc_new_db_subnets : var.vpc_existing_db_subnets
-  subnets_redis = var.flag_create_new_vpc == true ? var.vpc_new_redis_subnets : var.vpc_existing_redis_subnets
-  subnets_alb   = var.flag_create_new_vpc == true ? var.vpc_new_alb_subnets : var.vpc_existing_alb_subnets
 
-  subnets_all = concat(local.subnets_ec2, local.subnets_batch, local.subnets_db, local.subnets_redis, local.subnets_alb)
+  vpc_private_route_table_ids = data.aws_route_tables.preexisting.ids
 
-  # If using existing VPC, get subnet IDs by querying datasources with subnet CIDR.
-  # If building new VPC, make dictionary from cidr_block and subnet id (two different list outputs from VPC module).
-  subnet_ids_ec2 = (var.flag_create_new_vpc == true ?
-    [for cidr in local.subnets_ec2 : lookup(local.vpc_new_cidr_block_to_id_unified, cidr)] :
-    [for cidr in local.subnets_ec2 : data.aws_subnet.existing[cidr].id]
-  )
-
-  subnet_ids_batch = (var.flag_create_new_vpc == true ?
-    [for cidr in local.subnets_batch : lookup(local.vpc_new_cidr_block_to_id_unified, cidr)] :
-    [for cidr in local.subnets_batch : data.aws_subnet.existing[cidr].id]
-  )
-
-  subnet_ids_db = (var.flag_create_new_vpc == true ?
-    [for cidr in local.subnets_db : lookup(local.vpc_new_cidr_block_to_id_unified, cidr)] :
-    [for cidr in local.subnets_db : data.aws_subnet.existing[cidr].id]
-  )
-
-  subnet_ids_redis = (var.flag_create_new_vpc == true ?
-    [for cidr in local.subnets_redis : lookup(local.vpc_new_cidr_block_to_id_unified, cidr)] :
-    [for cidr in local.subnets_redis : data.aws_subnet.existing[cidr].id]
-  )
-
-  subnet_ids_alb = (
-    var.flag_create_load_balancer == true && var.flag_create_new_vpc == true ?
-      [for cidr in var.vpc_new_alb_subnets : lookup(local.vpc_new_cidr_block_to_id_unified, cidr)] :
-      var.flag_create_load_balancer == true && var.flag_use_existing_vpc == true ?
-        [for cidr in var.vpc_existing_alb_subnets : data.aws_subnet.existing[cidr].id] : []
+  # Map CIDR blocks to subnet IDs (depending on tf resource, either/or needed). 
+  # Cant delegate this to Python due to need to make multiple data calls.
+  subnets_ec2   = var.flag_create_new_vpc == true ? var.vpc_new_ec2_subnets   : var.vpc_existing_ec2_subnets
+  subnets_batch = var.flag_create_new_vpc == true ? var.vpc_new_batch_subnets : var.vpc_existing_batch_subnets
+  subnets_db    = var.flag_create_new_vpc == true ? var.vpc_new_db_subnets    : var.vpc_existing_db_subnets
+  subnets_redis = var.flag_create_new_vpc == true ? var.vpc_new_redis_subnets : var.vpc_existing_redis_subnets
+  subnets_alb   = var.flag_create_new_vpc == true ? var.vpc_new_alb_subnets   : var.vpc_existing_alb_subnets
+  subnets_all   = concat(local.subnets_ec2, local.subnets_batch, local.subnets_db, local.subnets_redis, local.subnets_alb)
+
+  subnet_ids_ec2    = [for cidr in local.subnets_ec2 : data.aws_subnet.existing[cidr].id]
+  subnet_ids_batch  = [for cidr in local.subnets_batch : data.aws_subnet.existing[cidr].id]
+  subnet_ids_db     = [for cidr in local.subnets_db : data.aws_subnet.existing[cidr].id]
+  subnet_ids_redis  = [for cidr in local.subnets_redis : data.aws_subnet.existing[cidr].id]
+  subnet_ids_alb    = (var.flag_create_load_balancer == true ? 
+    [for cidr in local.subnets_alb : data.aws_subnet.existing[cidr].id] : []
   )
 
 
diff --git a/001_vpc.tf b/001_vpc.tf
index a5ae23b9..c1c2806d 100644
--- a/001_vpc.tf
+++ b/001_vpc.tf
@@ -35,22 +35,6 @@ module "vpc" {
 }
 
 
-# https://stackoverflow.com/questions/67562197/terraform-loop-through-ids-list-and-generate-data-blocks-from-it-and-access-it
-data "aws_subnet" "existing" {
-  # Creates a map with the keys being the CIDRs --  e.g. `data.aws_subnet.public["10.0.0.0/20"].id
-  # Only make a data query if we are using an existing VPC
-  for_each = var.flag_use_existing_vpc == true ? toset(local.subnets_all) : []
-
-  vpc_id     = local.vpc_id
-  cidr_block = each.key
-}
-
-
-# Needed to add this to get existing CIDR range to limit ALB listeners
-data "aws_vpc" "preexisting" {
-  id = local.vpc_id
-}
-
 # Needed to grab route tables from pre-existing VPC to create VPC endpoints.
 data "aws_route_tables" "preexisting" {
   vpc_id = local.vpc_id

From bc966c6618915ca9eb2da9af418265b57cc883e9 Mon Sep 17 00:00:00 2001
From: Graham Wright <graham.wright@seqera.io>
Date: Sun, 16 Jun 2024 22:25:57 -0400
Subject: [PATCH 2/8] Broke output somehow.

---
 .../generate_db_connection_string.py          |  5 +-
 .../data_external/generate_dns_values.py      | 67 +++++++++++++++++--
 .githooks/utils/logger.py                     |  2 +-
 000_main.tf                                   | 36 +++++++---
 4 files changed, 90 insertions(+), 20 deletions(-)

diff --git a/.githooks/data_external/generate_db_connection_string.py b/.githooks/data_external/generate_db_connection_string.py
index 9d9c1c8f..5ef35228 100644
--- a/.githooks/data_external/generate_db_connection_string.py
+++ b/.githooks/data_external/generate_db_connection_string.py
@@ -19,7 +19,8 @@
 project_root = os.getcwd()
 os.chdir(f"{project_root}/.githooks")
 sys.path.append(".")
-from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
+from utils.extractors import get_tfvars_as_json
+from utils.logger import logger
 
 # Extract tfvars just like we do with the Python validation script
 os.chdir(project_root)
@@ -41,7 +42,9 @@
 
 
 def return_tf_payload(status: str, value: str):
+    logger.debug(f"Value is: {value}")
     payload = {'status': status, 'value': value}
+    logger.debug(f"Payload is: {payload}")
     print(json.dumps(payload))
 
 
diff --git a/.githooks/data_external/generate_dns_values.py b/.githooks/data_external/generate_dns_values.py
index 1a97a70f..6dacb696 100644
--- a/.githooks/data_external/generate_dns_values.py
+++ b/.githooks/data_external/generate_dns_values.py
@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
+import ast
 import os
 import json
 import sys
 from types import SimpleNamespace
-from typing import List
+from typing import Any, List
 
 sys.dont_write_bytecode = True
 
@@ -20,6 +21,7 @@
 os.chdir(f"{project_root}/.githooks")
 sys.path.append(".")
 from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
+from utils.logger import logger
 
 # Extract tfvars just like we do with the Python validation script
 os.chdir(project_root)
@@ -31,24 +33,75 @@
 # query = SimpleNamespace(**query)
 ## ------------------------------------------------------------------------------------
 
+# For debugging purposes
+# with open("query.json", "w") as file:
+#     file.write(str(query))
 
-with open("query.json", "w") as file:
-    file.write(str(query))
+# https://www.reddit.com/r/learnpython/comments/y02net/is_there_a_better_way_to_store_full_dictionary/
+def getDVal(d : dict, listPath : list) -> Any:
+    '''Recursively loop through a dictionary object to get to the target nested key.'''
+    for key in listPath:
+        d = d[key]
+    return d
+
+# Vars
+dns_zone_id = ""
+dns_instance_ip = ""
 
 # Determine kinda of DNS record to create
 dns_create_alb_record = True if (data.flag_create_load_balancer and not data.flag_create_hosts_file_entry) else False
 dns_create_ec2_record = True if (not data.flag_create_load_balancer and not data.flag_create_hosts_file_entry) else False
 
-if data.flag_create_route53_private_zone == True:
-    dns_zone_id = query.aws_route53_zone
+dns_zone_mappings = {
+    "flag_create_route53_private_zone":             "zone_private_new",
+    "flag_use_existing_route53_private_zone":       "zone_private_existing",
+    "flag_use_existing_route53_public_zone":        "zone_public_existing"
+}
+
+# Transformed dictionary nested into flat dicionary so value can be passed around.
+# I don't love this approach but can't find a cleaner/simpler way right now.
+dns_instance_ip_mappings = {
+    "flag_make_instance_private":                   ("tower_host_instance", ['private_ip']),
+    "flag_make_instance_private_behind_public_alb": ("tower_host_instance", ['private_ip']),
+    "flag_private_tower_without_eice":              ("tower_host_instance", ['private_ip']),
+    "flag_make_instance_public":                    ("tower_host_eip",      [0, 'public_ip'])
+}
+
+# Figure out DNS Zone Id
+logger.debug(f"Query is: {query}")
+
+for k,v in dns_zone_mappings.items():
+    logger.debug(f"k is : {k}; and v is: {v}")
+    dns_zone_id = json.loads(query[v])[0]["id"] if data_dictionary[k] else dns_zone_id
+
+for k,v in dns_instance_ip_mappings.items():
+    '''`aws_instance.ec2.private_ip` vs `aws_eip.towerhost[0].public_ip`'''
+    tf_obj, dpath = v
+    tf_obj_json = json.loads(query[tf_obj])
+    dns_instance_ip = getDVal(tf_obj_json, dpath) if data_dictionary[k] else dns_instance_ip
 
 
-def return_tf_payload(status: str, value: str):
+value = {
+    "dns_create_alb_record": dns_create_alb_record,
+    "dns_create_ec2_record": dns_create_ec2_record,
+
+    "dns_zone_id": dns_zone_id,
+    "dns_instance_ip": dns_instance_ip
+}
+
+
+def return_tf_payload(status: str, value: dict):
+    logger.debug(f"Payload is: {value}")
+
     payload = {'status': status, 'value': value}
     print(json.dumps(payload))
 
+    #  Result Error: invalid character '-' after top-level value
+    # with open("payload.json", "w") as file:
+    #     file.write(str(query))
+
 
 if __name__ == '__main__':
-    return_tf_payload("0", "abc")
+    return_tf_payload("0", value)
     
     exit(0)
\ No newline at end of file
diff --git a/.githooks/utils/logger.py b/.githooks/utils/logger.py
index df7efbdd..77cb5aeb 100644
--- a/.githooks/utils/logger.py
+++ b/.githooks/utils/logger.py
@@ -7,7 +7,7 @@
 handlers = [file_handler, stdout_handler]
 
 logging.basicConfig(
-    level=logging.INFO, 
+    level=logging.DEBUG, 
     # format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
     # https://stackoverflow.com/questions/57925917/python-logging-left-align-with-brackets
     format='%(asctime)s  %(filename)-15s:%(lineno)-4d %(levelname)-12s %(message)s',
diff --git a/000_main.tf b/000_main.tf
index e0bcf7a9..d1e53ffb 100644
--- a/000_main.tf
+++ b/000_main.tf
@@ -64,23 +64,29 @@ data "aws_subnet" "existing" {
 # https://medium.com/@leslie.alldridge/terraform-external-data-source-using-custom-python-script-with-example-cea5e618d83e
 data "external" "generate_db_connection_string" {
   program = ["python3", "${path.module}/.githooks/data_external/generate_db_connection_string.py"]
-  query = {
+  #query = {
     # tower_container_version = var.tower_container_version
     # flag_use_container_db = var.flag_use_container_db
     # db_container_engine_version = var.db_container_engine_version
     # db_engine_version = var.db_engine_version
-  }
+  #}
 }
 
-data "external" "generate_dns_valeus" {
-  program = ["python3", "${path.module}/.githooks/data_external/generate_dns_values.py"]
-  query = {
-    r53_privatezone = jsonencode(aws_route53_zone.private) # [0])
-    r53_public =  jsonencode(data.aws_route53_zone.public) # [0])
-    subnets = jsonencode(data.aws_subnet.existing)       # Adding in these values wont be known til created.
-    # ec2 = jsonencode(aws_instance.ec2)                     # Adding in these values wont be known til created.
-  }
-}
+# data "external" "generate_dns_values" {
+#   program = ["python3", "${path.module}/.githooks/data_external/generate_dns_values.py"]
+#   query = {
+#     # jsonencoding necessary for empty objects
+#     zone_private_new          = jsonencode(aws_route53_zone.private)
+#     zone_private_existing     = jsonencode(data.aws_route53_zone.private)
+#     zone_public_existing      = jsonencode(data.aws_route53_zone.public)
+
+#     tower_host_instance       = jsonencode(aws_instance.ec2)
+#     tower_host_eip            = jsonencode(aws_eip.towerhost)
+
+#     # subnets = data.aws_subnet.existing          # Adding in these values wont be known til created.
+#     # ec2 = jsonencode(aws_instance.ec2)                     # Adding in these values wont be known til created.
+#   }
+# }
 
 
 ## ------------------------------------------------------------------------------------
@@ -140,6 +146,7 @@ locals {
   # ---------------------------------------------------------------------------------------
   # All values here refer to Route53 in same AWS account as Tower instance.
   # If R53 record not generated, will create entry in EC2 hosts file.
+
   dns_create_alb_record = var.flag_create_load_balancer == true && var.flag_create_hosts_file_entry == false ? true : false
   dns_create_ec2_record = var.flag_create_load_balancer == false && var.flag_create_hosts_file_entry == false ? true : false
 
@@ -158,6 +165,13 @@ locals {
     "No_Match_Found"
   )
 
+  # dns_create_alb_record = data.external.generate_dns_values.result.dns_create_alb_record
+  # dns_create_ec2_record = data.external.generate_dns_values.result.dns_create_ec2_record
+
+  # dns_zone_id = data.external.generate_dns_values.result.dns_zone_id
+  # dns_instance_ip = data.external.generate_dns_values.result.dns_instance_ip
+
+
   # If no HTTPS and no load-balancer, use `http` prefix and expose port in URL. Otherwise, use `https` prefix and no port.
   tower_server_url = (
     var.flag_create_load_balancer == false && var.flag_do_not_use_https == true ?

From baafd773ad61532c42215ad9c329bc44afb96800 Mon Sep 17 00:00:00 2001
From: Graham Wright <graham.wright@seqera.io>
Date: Mon, 17 Jun 2024 10:59:11 -0400
Subject: [PATCH 3/8] Code half-working. Checkpointing now in case I break
 something.

---
 .../generate_db_connection_string.py          | 15 ++--
 .../data_external/generate_dns_values.py      | 16 ++--
 .githooks/utils/extractors.py                 |  3 +-
 .githooks/utils/logger.py                     | 41 ++++++++--
 000_main.tf                                   | 76 +++++++++----------
 assets/src/tower_config/tower.env.tpl         |  1 +
 6 files changed, 90 insertions(+), 62 deletions(-)

diff --git a/.githooks/data_external/generate_db_connection_string.py b/.githooks/data_external/generate_db_connection_string.py
index 5ef35228..786d4a54 100644
--- a/.githooks/data_external/generate_db_connection_string.py
+++ b/.githooks/data_external/generate_db_connection_string.py
@@ -20,7 +20,8 @@
 os.chdir(f"{project_root}/.githooks")
 sys.path.append(".")
 from utils.extractors import get_tfvars_as_json
-from utils.logger import logger
+#from utils.logger import logger
+from utils.logger import external_logger #as logger
 
 # Extract tfvars just like we do with the Python validation script
 os.chdir(project_root)
@@ -42,10 +43,14 @@
 
 
 def return_tf_payload(status: str, value: str):
-    logger.debug(f"Value is: {value}")
+    external_logger.debug(f"Value is: {value}")
     payload = {'status': status, 'value': value}
-    logger.debug(f"Payload is: {payload}")
+    external_logger.debug(f"Payload is: {payload}")
+
     print(json.dumps(payload))
+    # MemoryHandler configured to flush on `logger.error`. Flush once after TF gets its payload (above).
+    external_logger.error("Flushing.")     #external_logger.flush()
+    exit(0)
 
 
 def generate_connection_string(mysql8: str, v24plus: str):
@@ -71,6 +76,4 @@ def generate_connection_string(mysql8: str, v24plus: str):
 
 if __name__ == '__main__':
     connection_string = generate_connection_string(engine_version, data.tower_container_version)
-    return_tf_payload("0", connection_string)
-    
-    exit(0)
\ No newline at end of file
+    return_tf_payload("0", connection_string)
\ No newline at end of file
diff --git a/.githooks/data_external/generate_dns_values.py b/.githooks/data_external/generate_dns_values.py
index 6dacb696..448b4f0b 100644
--- a/.githooks/data_external/generate_dns_values.py
+++ b/.githooks/data_external/generate_dns_values.py
@@ -21,7 +21,7 @@
 os.chdir(f"{project_root}/.githooks")
 sys.path.append(".")
 from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
-from utils.logger import logger
+from utils.logger import external_logger
 
 # Extract tfvars just like we do with the Python validation script
 os.chdir(project_root)
@@ -30,12 +30,9 @@
 
 # Much simpler way to get variable passed in (via Terraform sending to stdin)
 query = json.load(sys.stdin)
-# query = SimpleNamespace(**query)
+query = SimpleNamespace(**query)
 ## ------------------------------------------------------------------------------------
 
-# For debugging purposes
-# with open("query.json", "w") as file:
-#     file.write(str(query))
 
 # https://www.reddit.com/r/learnpython/comments/y02net/is_there_a_better_way_to_store_full_dictionary/
 def getDVal(d : dict, listPath : list) -> Any:
@@ -68,10 +65,10 @@ def getDVal(d : dict, listPath : list) -> Any:
 }
 
 # Figure out DNS Zone Id
-logger.debug(f"Query is: {query}")
+external_logger.debug(f"Query is: {query}")
 
 for k,v in dns_zone_mappings.items():
-    logger.debug(f"k is : {k}; and v is: {v}")
+    external_logger.debug(f"k is : {k}; and v is: {v}")
     dns_zone_id = json.loads(query[v])[0]["id"] if data_dictionary[k] else dns_zone_id
 
 for k,v in dns_instance_ip_mappings.items():
@@ -91,14 +88,11 @@ def getDVal(d : dict, listPath : list) -> Any:
 
 
 def return_tf_payload(status: str, value: dict):
-    logger.debug(f"Payload is: {value}")
+    external_logger.debug(f"Payload is: {value}")
 
     payload = {'status': status, 'value': value}
     print(json.dumps(payload))
 
-    #  Result Error: invalid character '-' after top-level value
-    # with open("payload.json", "w") as file:
-    #     file.write(str(query))
 
 
 if __name__ == '__main__':
diff --git a/.githooks/utils/extractors.py b/.githooks/utils/extractors.py
index 37234e8b..88ae5699 100644
--- a/.githooks/utils/extractors.py
+++ b/.githooks/utils/extractors.py
@@ -65,7 +65,8 @@ def convert_tfvars_to_dictionary(file):
             elif flag_skip_block_comment:
                 indices_to_pop.append(i)
 
-        logger.debug(f"Indices to pop: {indices_to_pop}")
+        # This breaks my `external.data` implementation when logging set to debug. Commenting out.
+        # logger.debug(f"Indices to pop: {indices_to_pop}")
         purge_indices_in_reverse(indices_to_pop)
 
 
diff --git a/.githooks/utils/logger.py b/.githooks/utils/logger.py
index 77cb5aeb..707eea5c 100644
--- a/.githooks/utils/logger.py
+++ b/.githooks/utils/logger.py
@@ -1,17 +1,46 @@
 
 import logging
+import logging.handlers
 import sys
 
-file_handler = logging.FileHandler(filename='tmp.log')
-stdout_handler = logging.StreamHandler(stream=sys.stdout)
-handlers = [file_handler, stdout_handler]
+validation_file_handler = logging.FileHandler(filename='verify.log')
+validation_stdout_handler = logging.StreamHandler(stream=sys.stdout)
 
+data_external_file_handler = logging.FileHandler(filename="data_external.log")
+data_external_memory_handler = logging.handlers.MemoryHandler(
+    capacity=1024*100,
+    flushLevel=logging.ERROR,
+    target=data_external_file_handler,
+    #target=logging.FileHandler(filename="data_external.log"),
+    #flushOnClose=False,
+)
+
+# WARNING!! 
+# `handler[...]` population here works for `make verify` but breaks TF `data.external`
+# I have removed and attach formatting one-by-one. It's stupid, but it works.
+# handlers = [validation_file_handler, validation_stdout_handler]  # DONT UNCOMMENT
 logging.basicConfig(
     level=logging.DEBUG, 
-    # format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
     # https://stackoverflow.com/questions/57925917/python-logging-left-align-with-brackets
     format='%(asctime)s  %(filename)-15s:%(lineno)-4d %(levelname)-12s %(message)s',
-    handlers=handlers
+    # handlers=handlers  # DONT UNCOMMENT
 )
 
-logger = logging.getLogger('LOGGER_NAME')
\ No newline at end of file
+
+# https://docs.python.org/3/library/logging.handlers.html
+# MemoryHandler is needed due to how data flows to/from TF `data.external`. Using 2 loggers to keep 
+# better isolation between tfvars verification logic and content-generating scripts.
+
+logger = logging.getLogger('VALIDATION')
+logger.addHandler(validation_file_handler)
+logger.addHandler(validation_stdout_handler)
+
+external_logger = logging.getLogger('EXTERNAL')
+external_logger.addHandler(data_external_memory_handler)
+
+# data_external_memory_handler.setFormatter(formatter)  # No timestamp. Dunno why.
+formatter = logging.Formatter('%(asctime)s  %(filename)-15s:%(lineno)-4d %(levelname)-12s %(message)s')
+validation_file_handler.setFormatter(formatter)
+validation_stdout_handler.setFormatter(formatter)
+data_external_file_handler.setFormatter(formatter)
+
diff --git a/000_main.tf b/000_main.tf
index d1e53ffb..10444d5c 100644
--- a/000_main.tf
+++ b/000_main.tf
@@ -72,21 +72,21 @@ data "external" "generate_db_connection_string" {
   #}
 }
 
-# data "external" "generate_dns_values" {
-#   program = ["python3", "${path.module}/.githooks/data_external/generate_dns_values.py"]
-#   query = {
-#     # jsonencoding necessary for empty objects
-#     zone_private_new          = jsonencode(aws_route53_zone.private)
-#     zone_private_existing     = jsonencode(data.aws_route53_zone.private)
-#     zone_public_existing      = jsonencode(data.aws_route53_zone.public)
-
-#     tower_host_instance       = jsonencode(aws_instance.ec2)
-#     tower_host_eip            = jsonencode(aws_eip.towerhost)
-
-#     # subnets = data.aws_subnet.existing          # Adding in these values wont be known til created.
-#     # ec2 = jsonencode(aws_instance.ec2)                     # Adding in these values wont be known til created.
-#   }
-# }
+data "external" "generate_dns_values" {
+  program = ["python3", "${path.module}/.githooks/data_external/generate_dns_values.py"]
+  query = {
+    # jsonencoding necessary for empty objects
+    zone_private_new          = jsonencode(aws_route53_zone.private)
+    zone_private_existing     = jsonencode(data.aws_route53_zone.private)
+    zone_public_existing      = jsonencode(data.aws_route53_zone.public)
+
+    tower_host_instance       = jsonencode(aws_instance.ec2)
+    tower_host_eip            = jsonencode(aws_eip.towerhost)
+
+    # subnets = data.aws_subnet.existing          # Adding in these values wont be known til created.
+    # ec2 = jsonencode(aws_instance.ec2)                     # Adding in these values wont be known til created.
+  }
+}
 
 
 ## ------------------------------------------------------------------------------------
@@ -147,29 +147,29 @@ locals {
   # All values here refer to Route53 in same AWS account as Tower instance.
   # If R53 record not generated, will create entry in EC2 hosts file.
 
-  dns_create_alb_record = var.flag_create_load_balancer == true && var.flag_create_hosts_file_entry == false ? true : false
-  dns_create_ec2_record = var.flag_create_load_balancer == false && var.flag_create_hosts_file_entry == false ? true : false
-
-  dns_zone_id = (
-    var.flag_create_route53_private_zone == true ? aws_route53_zone.private[0].id :
-    var.flag_use_existing_route53_public_zone == true ? data.aws_route53_zone.public[0].id :
-    var.flag_use_existing_route53_private_zone == true ? data.aws_route53_zone.private[0].id :
-    "No_Match_Found"
-  )
-
-  dns_instance_ip = (
-    var.flag_make_instance_private == true ? aws_instance.ec2.private_ip :
-    var.flag_make_instance_private_behind_public_alb == true ? aws_instance.ec2.private_ip :
-    var.flag_private_tower_without_eice == true ? aws_instance.ec2.private_ip :
-    var.flag_make_instance_public == true ? aws_eip.towerhost[0].public_ip :
-    "No_Match_Found"
-  )
-
-  # dns_create_alb_record = data.external.generate_dns_values.result.dns_create_alb_record
-  # dns_create_ec2_record = data.external.generate_dns_values.result.dns_create_ec2_record
-
-  # dns_zone_id = data.external.generate_dns_values.result.dns_zone_id
-  # dns_instance_ip = data.external.generate_dns_values.result.dns_instance_ip
+  # dns_create_alb_record = var.flag_create_load_balancer == true && var.flag_create_hosts_file_entry == false ? true : false
+  # dns_create_ec2_record = var.flag_create_load_balancer == false && var.flag_create_hosts_file_entry == false ? true : false
+
+  # dns_zone_id = (
+  #   var.flag_create_route53_private_zone == true ? aws_route53_zone.private[0].id :
+  #   var.flag_use_existing_route53_public_zone == true ? data.aws_route53_zone.public[0].id :
+  #   var.flag_use_existing_route53_private_zone == true ? data.aws_route53_zone.private[0].id :
+  #   "No_Match_Found"
+  # )
+
+  # dns_instance_ip = (
+  #   var.flag_make_instance_private == true ? aws_instance.ec2.private_ip :
+  #   var.flag_make_instance_private_behind_public_alb == true ? aws_instance.ec2.private_ip :
+  #   var.flag_private_tower_without_eice == true ? aws_instance.ec2.private_ip :
+  #   var.flag_make_instance_public == true ? aws_eip.towerhost[0].public_ip :
+  #   "No_Match_Found"
+  # )
+
+  dns_create_alb_record = data.external.generate_dns_values.result.dns_create_alb_record
+  dns_create_ec2_record = data.external.generate_dns_values.result.dns_create_ec2_record
+
+  dns_zone_id = data.external.generate_dns_values.result.dns_zone_id
+  dns_instance_ip = data.external.generate_dns_values.result.dns_instance_ip
 
 
   # If no HTTPS and no load-balancer, use `http` prefix and expose port in URL. Otherwise, use `https` prefix and no port.
diff --git a/assets/src/tower_config/tower.env.tpl b/assets/src/tower_config/tower.env.tpl
index 874df340..80c4001f 100755
--- a/assets/src/tower_config/tower.env.tpl
+++ b/assets/src/tower_config/tower.env.tpl
@@ -2,6 +2,7 @@
 # Generic Tower configuration values
 # ------------------------------------------------
 TOWER_ENABLE_AWS_SSM=true
+TOWER_ENABLE_ARM64=true
 
 LICENSE_SERVER_URL=https://licenses.seqera.io
 

From a71e7285fe0b1698538760620071b3e2490dc857 Mon Sep 17 00:00:00 2001
From: Graham Wright <graham.wright@seqera.io>
Date: Mon, 17 Jun 2024 12:07:31 -0400
Subject: [PATCH 4/8] External scripts working.

---
 .../data_external/generate_dns_values.py      | 31 +++++++++++++------
 .gitignore                                    |  2 ++
 000_main.tf                                   | 20 ++++++++++--
 3 files changed, 42 insertions(+), 11 deletions(-)

diff --git a/.githooks/data_external/generate_dns_values.py b/.githooks/data_external/generate_dns_values.py
index 448b4f0b..a6de88c2 100644
--- a/.githooks/data_external/generate_dns_values.py
+++ b/.githooks/data_external/generate_dns_values.py
@@ -30,7 +30,7 @@
 
 # Much simpler way to get variable passed in (via Terraform sending to stdin)
 query = json.load(sys.stdin)
-query = SimpleNamespace(**query)
+# query = SimpleNamespace(**query)
 ## ------------------------------------------------------------------------------------
 
 
@@ -50,9 +50,9 @@ def getDVal(d : dict, listPath : list) -> Any:
 dns_create_ec2_record = True if (not data.flag_create_load_balancer and not data.flag_create_hosts_file_entry) else False
 
 dns_zone_mappings = {
-    "flag_create_route53_private_zone":             "zone_private_new",
-    "flag_use_existing_route53_private_zone":       "zone_private_existing",
-    "flag_use_existing_route53_public_zone":        "zone_public_existing"
+    "flag_create_route53_private_zone":             ("zone_private_new",        [0, 'id']),
+    "flag_use_existing_route53_private_zone":       ("zone_private_existing",   [0, 'id']),
+    "flag_use_existing_route53_public_zone":        ("zone_public_existing",    [0, 'id']),
 }
 
 # Transformed dictionary nested into flat dicionary so value can be passed around.
@@ -69,7 +69,12 @@ def getDVal(d : dict, listPath : list) -> Any:
 
 for k,v in dns_zone_mappings.items():
     external_logger.debug(f"k is : {k}; and v is: {v}")
-    dns_zone_id = json.loads(query[v])[0]["id"] if data_dictionary[k] else dns_zone_id
+    tf_obj, dpath = v
+    tf_obj_json = json.loads(query[tf_obj])
+    dns_zone_id = getDVal(tf_obj_json, dpath) if data_dictionary[k] else dns_instance_ip
+    # tf_obj_json = json.loads(query[v])
+    # dns_zone_id = getDVal(tf_obj_json, dpath) if data_dictionary[k] else dns_zone_id
+    # dns_zone_id = json.loads(query[v])[0]["id"] if data_dictionary[k] else dns_zone_id
 
 for k,v in dns_instance_ip_mappings.items():
     '''`aws_instance.ec2.private_ip` vs `aws_eip.towerhost[0].public_ip`'''
@@ -87,15 +92,23 @@ def getDVal(d : dict, listPath : list) -> Any:
 }
 
 
+
 def return_tf_payload(status: str, value: dict):
     external_logger.debug(f"Payload is: {value}")
 
-    payload = {'status': status, 'value': value}
+    payload = {'status': status, **value} # 'value': 'a'}  #value}
+    payload["dns_create_alb_record"] = "true"
+    payload["dns_create_ec2_record"] = "false"
+    #payload = json.dumps(payload)
+    #external_logger.debug(f"Dumped payload is: {payload}")
+
     print(json.dumps(payload))
+    #print(payload)
+
+    external_logger.error("Flushing.")     #external_logger.flush()
+    exit(0)
 
 
 
 if __name__ == '__main__':
-    return_tf_payload("0", value)
-    
-    exit(0)
\ No newline at end of file
+    return_tf_payload("0", value)
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index d425bd48..6d5a264f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,5 @@ tmp.log
 **/venv
 **/.terraform.lock.hcl
 **/__pycache__
+verify.log
+**/data_external.log
\ No newline at end of file
diff --git a/000_main.tf b/000_main.tf
index 10444d5c..5c8cb7d0 100644
--- a/000_main.tf
+++ b/000_main.tf
@@ -165,12 +165,28 @@ locals {
   #   "No_Match_Found"
   # )
 
-  dns_create_alb_record = data.external.generate_dns_values.result.dns_create_alb_record
-  dns_create_ec2_record = data.external.generate_dns_values.result.dns_create_ec2_record
+  ## while calling jsondecode(str)
+  ##    │ data.external.generate_dns_values.result is map of string with 5 elements
+  # dns_vars = jsondecode(data.external.generate_dns_values.result)
+  # dns_create_alb_record = local.dns_vars["dns_create_alb_record"]
+  # dns_create_ec2_record = local.dns_vars["dns_create_ec2_record"]
+
+  # dns_zone_id = local.dns_vars["dns_zone_id"]
+  # dns_instance_ip = local.dns_vars["dns_instance_ip"]
+
+  # Need to jsondecode to handle true/false. Don't need to decode for the others.
+  dns_create_alb_record = jsondecode(data.external.generate_dns_values.result.dns_create_alb_record)
+  dns_create_ec2_record = jsondecode(data.external.generate_dns_values.result.dns_create_ec2_record)
 
   dns_zone_id = data.external.generate_dns_values.result.dns_zone_id
   dns_instance_ip = data.external.generate_dns_values.result.dns_instance_ip
 
+  # dns_create_alb_record = data.external.generate_dns_values.result.dns_create_alb_record
+  # dns_create_ec2_record = data.external.generate_dns_values.result.dns_create_ec2_record
+
+  # dns_zone_id = data.external.generate_dns_values.result.dns_zone_id
+  # dns_instance_ip = data.external.generate_dns_values.result.dns_instance_ip
+
 
   # If no HTTPS and no load-balancer, use `http` prefix and expose port in URL. Otherwise, use `https` prefix and no port.
   tower_server_url = (

From 8df4432329ebde92db1723270e1c6c07b81b9651 Mon Sep 17 00:00:00 2001
From: Graham Wright <graham.wright@seqera.io>
Date: Mon, 17 Jun 2024 16:52:33 -0400
Subject: [PATCH 5/8] Going to fix double log on Verifying 'terraform.tfvars'.
 Beginning tfvars configuration check. ----- Verifying TFVARS file ----- -----
 Verifying Tower configurations ----- ----- Verifying AWS Integrations -----
 Retrieving subnet information from AWS Account. `sg_egress_eice` allows
 egress everywhere. Consider tightening. `sg_egress_tower_ec2` allows egress
 everywhere. Consider tightening. `sg_egress_tower_alb` allows egress
 everywhere. Consider tightening. `sg_egress_batch_ec2` allows egress
 everywhere. Consider tightening. `sg_egress_interface_endpoint` allows egress
 everywhere. Consider tightening. ----- Verifying Database settings ----- Your
 EC2 AMI will update as newer images are available. This means your VM will
 occasionally be destroyed and recreated. Your docker db container will
 destroyed when you EC2 AMI is updated. Ensure this fits your intention. You
 have not enabled Deletion Protection on your external DB. This is HIGHLY
 recommended for Production instances. If you want this, set
 `db_deletion_protection` to true. You have disabled a final snapshot of your
 external DB. Enablement of this feature is recommended for Production.
 Finished tfvars configuration check.

---
 .../data_external/generate_dns_values.py      | 83 ++++++++++---------
 .githooks/data_external/generate_flags.py     | 74 +++++++++++++++++
 .githooks/utils/logger.py                     |  6 +-
 000_main.tf                                   | 11 ++-
 4 files changed, 129 insertions(+), 45 deletions(-)
 create mode 100644 .githooks/data_external/generate_flags.py

diff --git a/.githooks/data_external/generate_dns_values.py b/.githooks/data_external/generate_dns_values.py
index a6de88c2..7af5b5b2 100644
--- a/.githooks/data_external/generate_dns_values.py
+++ b/.githooks/data_external/generate_dns_values.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-import ast
 import os
 import json
 import sys
@@ -42,13 +41,6 @@ def getDVal(d : dict, listPath : list) -> Any:
     return d
 
 # Vars
-dns_zone_id = ""
-dns_instance_ip = ""
-
-# Determine kinda of DNS record to create
-dns_create_alb_record = True if (data.flag_create_load_balancer and not data.flag_create_hosts_file_entry) else False
-dns_create_ec2_record = True if (not data.flag_create_load_balancer and not data.flag_create_hosts_file_entry) else False
-
 dns_zone_mappings = {
     "flag_create_route53_private_zone":             ("zone_private_new",        [0, 'id']),
     "flag_use_existing_route53_private_zone":       ("zone_private_existing",   [0, 'id']),
@@ -64,51 +56,62 @@ def getDVal(d : dict, listPath : list) -> Any:
     "flag_make_instance_public":                    ("tower_host_eip",      [0, 'public_ip'])
 }
 
-# Figure out DNS Zone Id
-external_logger.debug(f"Query is: {query}")
 
-for k,v in dns_zone_mappings.items():
-    external_logger.debug(f"k is : {k}; and v is: {v}")
-    tf_obj, dpath = v
-    tf_obj_json = json.loads(query[tf_obj])
-    dns_zone_id = getDVal(tf_obj_json, dpath) if data_dictionary[k] else dns_instance_ip
-    # tf_obj_json = json.loads(query[v])
-    # dns_zone_id = getDVal(tf_obj_json, dpath) if data_dictionary[k] else dns_zone_id
-    # dns_zone_id = json.loads(query[v])[0]["id"] if data_dictionary[k] else dns_zone_id
+def populate_values(query):
 
-for k,v in dns_instance_ip_mappings.items():
-    '''`aws_instance.ec2.private_ip` vs `aws_eip.towerhost[0].public_ip`'''
-    tf_obj, dpath = v
-    tf_obj_json = json.loads(query[tf_obj])
-    dns_instance_ip = getDVal(tf_obj_json, dpath) if data_dictionary[k] else dns_instance_ip
+    external_logger.debug(f"Query is: {query}")
 
+    # Baseline variables
+    dns_zone_id = ""
+    dns_instance_ip = ""
 
-value = {
-    "dns_create_alb_record": dns_create_alb_record,
-    "dns_create_ec2_record": dns_create_ec2_record,
+    # Determine kinda of DNS record to create
+    # dns_create_alb_record = True if (data.flag_create_load_balancer and not data.flag_create_hosts_file_entry) else False
+    # dns_create_ec2_record = True if (not data.flag_create_load_balancer and not data.flag_create_hosts_file_entry) else False
+
+    for k,v in dns_zone_mappings.items():
+        external_logger.debug(f"k is : {k}; and v is: {v}")
+        tf_obj, dpath = v
+        tf_obj_json = json.loads(query[tf_obj])
+        dns_zone_id = getDVal(tf_obj_json, dpath) if data_dictionary[k] else dns_instance_ip
 
-    "dns_zone_id": dns_zone_id,
-    "dns_instance_ip": dns_instance_ip
-}
 
+    for k,v in dns_instance_ip_mappings.items():
+        '''`aws_instance.ec2.private_ip` vs `aws_eip.towerhost[0].public_ip`'''
+        tf_obj, dpath = v
+        tf_obj_json = json.loads(query[tf_obj])
+        dns_instance_ip = getDVal(tf_obj_json, dpath) if data_dictionary[k] else dns_instance_ip
 
+    values = {
+        # "dns_create_alb_record": dns_create_alb_record,
+        # "dns_create_ec2_record": dns_create_ec2_record,
 
-def return_tf_payload(status: str, value: dict):
-    external_logger.debug(f"Payload is: {value}")
+        "dns_zone_id": dns_zone_id,
+        "dns_instance_ip": dns_instance_ip
+    }
 
-    payload = {'status': status, **value} # 'value': 'a'}  #value}
-    payload["dns_create_alb_record"] = "true"
-    payload["dns_create_ec2_record"] = "false"
-    #payload = json.dumps(payload)
-    #external_logger.debug(f"Dumped payload is: {payload}")
+    return values
 
+
+def convert_booleans_to_strings(payload: dict) -> dict: 
+    for k,v in payload.items():
+        external_logger.debug(f"k is {k} and v is {v}.")
+        if isinstance(v, bool):
+            payload[k] = "true" if v == True else "false"
+    return payload
+        
+
+def return_tf_payload(status: str, values: dict):
+    external_logger.debug(f"Payload is: {values}")
+
+    payload = {'status': status, **values}
+    payload = convert_booleans_to_strings(payload)
     print(json.dumps(payload))
-    #print(payload)
 
-    external_logger.error("Flushing.")     #external_logger.flush()
+    external_logger.error("Flushing.")
     exit(0)
 
 
-
 if __name__ == '__main__':
-    return_tf_payload("0", value)
\ No newline at end of file
+    values = populate_values(query)
+    return_tf_payload("0", values)
\ No newline at end of file
diff --git a/.githooks/data_external/generate_flags.py b/.githooks/data_external/generate_flags.py
new file mode 100644
index 00000000..d7205550
--- /dev/null
+++ b/.githooks/data_external/generate_flags.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+import os
+import json
+import sys
+from types import SimpleNamespace
+from typing import Any, List
+
+sys.dont_write_bytecode = True
+
+
+## ------------------------------------------------------------------------------------
+## Extract TFvars and get query values
+## ------------------------------------------------------------------------------------
+# NOTE! This part is a bit convoluted but there's a method to the madness:
+#   - Script executes from `~/cx-field-tools-installer`. 
+#   - Change into .githooks (to avoid the `.` import problem), import, return to project root to extract tfvars.
+#   - Error thrown without workaround: `ImportError: attempted relative import with no known parent package`
+#   - Query object to receive objects via TF data call (i.e. resources)
+project_root = os.getcwd()
+os.chdir(f"{project_root}/.githooks")
+sys.path.append(".")
+from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
+from utils.logger import external_logger
+
+# Extract tfvars just like we do with the Python validation script
+os.chdir(project_root)
+data_dictionary = get_tfvars_as_json()
+data = SimpleNamespace(**data_dictionary)
+
+# Much simpler way to get variable passed in (via Terraform sending to stdin)
+query = json.load(sys.stdin)
+# query = SimpleNamespace(**query)
+## ------------------------------------------------------------------------------------
+
+
+
+def populate_values(query):
+
+    external_logger.debug(f"Query is: {query}")
+
+    # Determine kinda of DNS record to create
+    dns_create_alb_record = True if (data.flag_create_load_balancer and not data.flag_create_hosts_file_entry) else False
+    dns_create_ec2_record = True if (not data.flag_create_load_balancer and not data.flag_create_hosts_file_entry) else False
+
+    values = {
+        "dns_create_alb_record": dns_create_alb_record,
+        "dns_create_ec2_record": dns_create_ec2_record,
+    }
+
+    return values
+
+
+def convert_booleans_to_strings(payload: dict) -> dict: 
+    for k,v in payload.items():
+        external_logger.debug(f"k is {k} and v is {v}.")
+        if isinstance(v, bool):
+            payload[k] = "true" if v == True else "false"
+    return payload
+        
+
+def return_tf_payload(status: str, values: dict):
+    external_logger.debug(f"Payload is: {values}")
+
+    payload = {'status': status, **values}
+    payload = convert_booleans_to_strings(payload)
+    print(json.dumps(payload))
+
+    external_logger.error("Flushing.")
+    exit(0)
+
+
+if __name__ == '__main__':
+    values = populate_values(query)
+    return_tf_payload("0", values)
\ No newline at end of file
diff --git a/.githooks/utils/logger.py b/.githooks/utils/logger.py
index 707eea5c..13da5337 100644
--- a/.githooks/utils/logger.py
+++ b/.githooks/utils/logger.py
@@ -20,7 +20,7 @@
 # I have removed and attach formatting one-by-one. It's stupid, but it works.
 # handlers = [validation_file_handler, validation_stdout_handler]  # DONT UNCOMMENT
 logging.basicConfig(
-    level=logging.DEBUG, 
+    level=logging.INFO, 
     # https://stackoverflow.com/questions/57925917/python-logging-left-align-with-brackets
     format='%(asctime)s  %(filename)-15s:%(lineno)-4d %(levelname)-12s %(message)s',
     # handlers=handlers  # DONT UNCOMMENT
@@ -40,7 +40,7 @@
 
 # data_external_memory_handler.setFormatter(formatter)  # No timestamp. Dunno why.
 formatter = logging.Formatter('%(asctime)s  %(filename)-15s:%(lineno)-4d %(levelname)-12s %(message)s')
-validation_file_handler.setFormatter(formatter)
-validation_stdout_handler.setFormatter(formatter)
+#validation_file_handler.setFormatter(formatter)
+# validation_stdout_handler.setFormatter(formatter)
 data_external_file_handler.setFormatter(formatter)
 
diff --git a/000_main.tf b/000_main.tf
index 5c8cb7d0..8bc2d52f 100644
--- a/000_main.tf
+++ b/000_main.tf
@@ -72,6 +72,11 @@ data "external" "generate_db_connection_string" {
   #}
 }
 
+data "external" "generate_flags" {
+  program = ["python3", "${path.module}/.githooks/data_external/generate_flags.py"]
+  query = {}
+}
+
 data "external" "generate_dns_values" {
   program = ["python3", "${path.module}/.githooks/data_external/generate_dns_values.py"]
   query = {
@@ -175,8 +180,10 @@ locals {
   # dns_instance_ip = local.dns_vars["dns_instance_ip"]
 
   # Need to jsondecode to handle true/false. Don't need to decode for the others.
-  dns_create_alb_record = jsondecode(data.external.generate_dns_values.result.dns_create_alb_record)
-  dns_create_ec2_record = jsondecode(data.external.generate_dns_values.result.dns_create_ec2_record)
+  # dns_create_alb_record = jsondecode(data.external.generate_dns_values.result.dns_create_alb_record)
+  # dns_create_ec2_record = jsondecode(data.external.generate_dns_values.result.dns_create_ec2_record)
+  dns_create_alb_record = jsondecode(data.external.generate_flags.result.dns_create_alb_record)
+  dns_create_ec2_record = jsondecode(data.external.generate_flags.result.dns_create_ec2_record)
 
   dns_zone_id = data.external.generate_dns_values.result.dns_zone_id
   dns_instance_ip = data.external.generate_dns_values.result.dns_instance_ip

From e13ef036bd1347f49cb1b79ece05410dc1c41ef3 Mon Sep 17 00:00:00 2001
From: Graham Wright <graham.wright@seqera.io>
Date: Mon, 17 Jun 2024 17:23:22 -0400
Subject: [PATCH 6/8] Fixed up logger code.

---
 .githooks/utils/logger.py | 58 +++++++++++++++++++++------------------
 1 file changed, 32 insertions(+), 26 deletions(-)

diff --git a/.githooks/utils/logger.py b/.githooks/utils/logger.py
index 13da5337..73feec0a 100644
--- a/.githooks/utils/logger.py
+++ b/.githooks/utils/logger.py
@@ -3,8 +3,38 @@
 import logging.handlers
 import sys
 
+
+# WARNING (June 17, 2024)
+#
+# Original logger configuration logic worked well for verification logic but broke when using TF `data.external` mechanism.
+# (broke due to how the `data.external` mechanism reads stdout/stderr ... which log events wrote to by defaul). This was resolved
+# by splitting the logger into two: one to handle verification via stdout/file, the other to handle TF external mechanism by
+# buffering all log events in memory until a proper payload is returned to TF (with the buffered logs emitted afterwards).
+# https://docs.python.org/3/library/logging.handlers.html
+#
+# Trial-and-error reworking logic finally got that working, but allowing logging.basicConfig to remain resulted in 
+# double-entry log events. Not sure why and don't care to spend more time right now investigating. 
+# 
+# Ultimately made things more granular and less DRY, but it works so .....
+
+
+formatter = logging.Formatter('%(asctime)s  %(filename)-15s:%(lineno)-4d %(levelname)-12s %(message)s')
+
+# Validation Logger
+logger = logging.getLogger('VALIDATION')
+logger.setLevel(logging.DEBUG)
+
 validation_file_handler = logging.FileHandler(filename='verify.log')
 validation_stdout_handler = logging.StreamHandler(stream=sys.stdout)
+validation_file_handler.setFormatter(formatter)
+validation_stdout_handler.setFormatter(formatter)
+logger.addHandler(validation_file_handler)
+logger.addHandler(validation_stdout_handler)
+
+
+# External Logger
+external_logger = logging.getLogger('EXTERNAL')
+external_logger.setLevel(logging.DEBUG)
 
 data_external_file_handler = logging.FileHandler(filename="data_external.log")
 data_external_memory_handler = logging.handlers.MemoryHandler(
@@ -14,33 +44,9 @@
     #target=logging.FileHandler(filename="data_external.log"),
     #flushOnClose=False,
 )
-
-# WARNING!! 
-# `handler[...]` population here works for `make verify` but breaks TF `data.external`
-# I have removed and attach formatting one-by-one. It's stupid, but it works.
-# handlers = [validation_file_handler, validation_stdout_handler]  # DONT UNCOMMENT
-logging.basicConfig(
-    level=logging.INFO, 
-    # https://stackoverflow.com/questions/57925917/python-logging-left-align-with-brackets
-    format='%(asctime)s  %(filename)-15s:%(lineno)-4d %(levelname)-12s %(message)s',
-    # handlers=handlers  # DONT UNCOMMENT
-)
+data_external_file_handler.setFormatter(formatter)
+external_logger.addHandler(data_external_memory_handler)
 
 
-# https://docs.python.org/3/library/logging.handlers.html
-# MemoryHandler is needed due to how data flows to/from TF `data.external`. Using 2 loggers to keep 
-# better isolation between tfvars verification logic and content-generating scripts.
 
-logger = logging.getLogger('VALIDATION')
-logger.addHandler(validation_file_handler)
-logger.addHandler(validation_stdout_handler)
-
-external_logger = logging.getLogger('EXTERNAL')
-external_logger.addHandler(data_external_memory_handler)
-
-# data_external_memory_handler.setFormatter(formatter)  # No timestamp. Dunno why.
-formatter = logging.Formatter('%(asctime)s  %(filename)-15s:%(lineno)-4d %(levelname)-12s %(message)s')
-#validation_file_handler.setFormatter(formatter)
-# validation_stdout_handler.setFormatter(formatter)
-data_external_file_handler.setFormatter(formatter)
 

From dc6bf1f15042392545c48ec73a640242f4876583 Mon Sep 17 00:00:00 2001
From: Graham Wright <graham.wright@seqera.io>
Date: Mon, 17 Jun 2024 18:11:02 -0400
Subject: [PATCH 7/8] Rationalized data_external functions.

---
 .../generate_db_connection_string.py          | 41 ++++++-----------
 .../data_external/generate_dns_values.py      | 37 +--------------
 .githooks/data_external/generate_flags.py     | 24 +---------
 .../utils/common_data_external_functions.py   | 45 +++++++++++++++++++
 000_main.tf                                   |  9 +---
 012_outputs.tf                                |  2 +-
 6 files changed, 66 insertions(+), 92 deletions(-)
 create mode 100644 .githooks/utils/common_data_external_functions.py

diff --git a/.githooks/data_external/generate_db_connection_string.py b/.githooks/data_external/generate_db_connection_string.py
index 786d4a54..00a8646c 100644
--- a/.githooks/data_external/generate_db_connection_string.py
+++ b/.githooks/data_external/generate_db_connection_string.py
@@ -3,7 +3,6 @@
 import json
 import sys
 from types import SimpleNamespace
-from typing import List
 
 sys.dont_write_bytecode = True
 
@@ -19,18 +18,18 @@
 project_root = os.getcwd()
 os.chdir(f"{project_root}/.githooks")
 sys.path.append(".")
-from utils.extractors import get_tfvars_as_json
-#from utils.logger import logger
-from utils.logger import external_logger #as logger
 
-# Extract tfvars just like we do with the Python validation script
+from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
+from utils.logger import external_logger
+from utils.common_data_external_functions import getDVal, return_tf_payload
+
 os.chdir(project_root)
 data_dictionary = get_tfvars_as_json()
 data = SimpleNamespace(**data_dictionary)
 
 # Much simpler way to get variable passed in (via Terraform sending to stdin)
 query = json.load(sys.stdin)
-query = SimpleNamespace(**query)
+# query = SimpleNamespace(**query)
 ## ------------------------------------------------------------------------------------
 
 
@@ -42,27 +41,11 @@
 v24plus_connstring = "permitMysqlScheme=true"
 
 
-def return_tf_payload(status: str, value: str):
-    external_logger.debug(f"Value is: {value}")
-    payload = {'status': status, 'value': value}
-    external_logger.debug(f"Payload is: {payload}")
-
-    print(json.dumps(payload))
-    # MemoryHandler configured to flush on `logger.error`. Flush once after TF gets its payload (above).
-    external_logger.error("Flushing.")     #external_logger.flush()
-    exit(0)
-
-
 def generate_connection_string(mysql8: str, v24plus: str):
     connection_string = ""
-    add_mysql8 = False
-    add_v24plus = False
-
-    if mysql8.startswith("8."): 
-        add_mysql8 = True
 
-    if v24plus >= "v24":
-        add_v24plus = True
+    add_mysql8 = True if mysql8.startswith("8.") else False
+    add_v24plus = True if v24plus >= "v24" else False
 
     if add_mysql8 and add_v24plus:
         connection_string = f"?{mysql8_connstring}&{v24plus_connstring}"
@@ -71,9 +54,13 @@ def generate_connection_string(mysql8: str, v24plus: str):
     elif not add_mysql8 and add_v24plus:
         connection_string = f"?{v24plus_connstring}"
 
-    return connection_string
+    values = {
+        "connection_string": connection_string
+    }
+
+    return values
 
 
 if __name__ == '__main__':
-    connection_string = generate_connection_string(engine_version, data.tower_container_version)
-    return_tf_payload("0", connection_string)
\ No newline at end of file
+    values = generate_connection_string(engine_version, data.tower_container_version)
+    return_tf_payload("0", values)
\ No newline at end of file
diff --git a/.githooks/data_external/generate_dns_values.py b/.githooks/data_external/generate_dns_values.py
index 7af5b5b2..9dc2b1c8 100644
--- a/.githooks/data_external/generate_dns_values.py
+++ b/.githooks/data_external/generate_dns_values.py
@@ -3,7 +3,6 @@
 import json
 import sys
 from types import SimpleNamespace
-from typing import Any, List
 
 sys.dont_write_bytecode = True
 
@@ -19,10 +18,11 @@
 project_root = os.getcwd()
 os.chdir(f"{project_root}/.githooks")
 sys.path.append(".")
+
 from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
 from utils.logger import external_logger
+from utils.common_data_external_functions import getDVal, return_tf_payload
 
-# Extract tfvars just like we do with the Python validation script
 os.chdir(project_root)
 data_dictionary = get_tfvars_as_json()
 data = SimpleNamespace(**data_dictionary)
@@ -33,13 +33,6 @@
 ## ------------------------------------------------------------------------------------
 
 
-# https://www.reddit.com/r/learnpython/comments/y02net/is_there_a_better_way_to_store_full_dictionary/
-def getDVal(d : dict, listPath : list) -> Any:
-    '''Recursively loop through a dictionary object to get to the target nested key.'''
-    for key in listPath:
-        d = d[key]
-    return d
-
 # Vars
 dns_zone_mappings = {
     "flag_create_route53_private_zone":             ("zone_private_new",        [0, 'id']),
@@ -65,10 +58,6 @@ def populate_values(query):
     dns_zone_id = ""
     dns_instance_ip = ""
 
-    # Determine kinda of DNS record to create
-    # dns_create_alb_record = True if (data.flag_create_load_balancer and not data.flag_create_hosts_file_entry) else False
-    # dns_create_ec2_record = True if (not data.flag_create_load_balancer and not data.flag_create_hosts_file_entry) else False
-
     for k,v in dns_zone_mappings.items():
         external_logger.debug(f"k is : {k}; and v is: {v}")
         tf_obj, dpath = v
@@ -83,9 +72,6 @@ def populate_values(query):
         dns_instance_ip = getDVal(tf_obj_json, dpath) if data_dictionary[k] else dns_instance_ip
 
     values = {
-        # "dns_create_alb_record": dns_create_alb_record,
-        # "dns_create_ec2_record": dns_create_ec2_record,
-
         "dns_zone_id": dns_zone_id,
         "dns_instance_ip": dns_instance_ip
     }
@@ -93,25 +79,6 @@ def populate_values(query):
     return values
 
 
-def convert_booleans_to_strings(payload: dict) -> dict: 
-    for k,v in payload.items():
-        external_logger.debug(f"k is {k} and v is {v}.")
-        if isinstance(v, bool):
-            payload[k] = "true" if v == True else "false"
-    return payload
-        
-
-def return_tf_payload(status: str, values: dict):
-    external_logger.debug(f"Payload is: {values}")
-
-    payload = {'status': status, **values}
-    payload = convert_booleans_to_strings(payload)
-    print(json.dumps(payload))
-
-    external_logger.error("Flushing.")
-    exit(0)
-
-
 if __name__ == '__main__':
     values = populate_values(query)
     return_tf_payload("0", values)
\ No newline at end of file
diff --git a/.githooks/data_external/generate_flags.py b/.githooks/data_external/generate_flags.py
index d7205550..f95c6923 100644
--- a/.githooks/data_external/generate_flags.py
+++ b/.githooks/data_external/generate_flags.py
@@ -3,7 +3,6 @@
 import json
 import sys
 from types import SimpleNamespace
-from typing import Any, List
 
 sys.dont_write_bytecode = True
 
@@ -19,10 +18,11 @@
 project_root = os.getcwd()
 os.chdir(f"{project_root}/.githooks")
 sys.path.append(".")
+
 from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
 from utils.logger import external_logger
+from utils.common_data_external_functions import getDVal, return_tf_payload
 
-# Extract tfvars just like we do with the Python validation script
 os.chdir(project_root)
 data_dictionary = get_tfvars_as_json()
 data = SimpleNamespace(**data_dictionary)
@@ -33,7 +33,6 @@
 ## ------------------------------------------------------------------------------------
 
 
-
 def populate_values(query):
 
     external_logger.debug(f"Query is: {query}")
@@ -50,25 +49,6 @@ def populate_values(query):
     return values
 
 
-def convert_booleans_to_strings(payload: dict) -> dict: 
-    for k,v in payload.items():
-        external_logger.debug(f"k is {k} and v is {v}.")
-        if isinstance(v, bool):
-            payload[k] = "true" if v == True else "false"
-    return payload
-        
-
-def return_tf_payload(status: str, values: dict):
-    external_logger.debug(f"Payload is: {values}")
-
-    payload = {'status': status, **values}
-    payload = convert_booleans_to_strings(payload)
-    print(json.dumps(payload))
-
-    external_logger.error("Flushing.")
-    exit(0)
-
-
 if __name__ == '__main__':
     values = populate_values(query)
     return_tf_payload("0", values)
\ No newline at end of file
diff --git a/.githooks/utils/common_data_external_functions.py b/.githooks/utils/common_data_external_functions.py
new file mode 100644
index 00000000..569f9b23
--- /dev/null
+++ b/.githooks/utils/common_data_external_functions.py
@@ -0,0 +1,45 @@
+# Centralizes common functions which will be needed across scripts called by TF `data.external`.
+import json
+from typing import Any
+
+from utils.logger import external_logger
+
+
+# https://www.reddit.com/r/learnpython/comments/y02net/is_there_a_better_way_to_store_full_dictionary/
+def getDVal(d : dict, listPath : list) -> Any:
+    '''
+    Recursively loop through a dictionary object to get to the target nested key.
+    This allows us to specify a TF objecy and the exact subpath to slice from.
+    Note: The path must be defined as discrete elements in a list.
+    '''
+    for key in listPath:
+        d = d[key]
+    return d
+
+
+def convert_booleans_to_strings(payload: dict) -> dict: 
+    '''
+    TF data.external freaks out if JSON true/false values not returned as strings.
+    Unfortunately, json.dumps will convert True/False to true/false, but not turn to string.
+    This function fixes the json.dumps payload so it's acceptable to TF.
+    '''
+    for k,v in payload.items():
+        external_logger.debug(f"k is {k} and v is {v}.")
+        if isinstance(v, bool):
+            payload[k] = "true" if v == True else "false"
+    return payload
+
+
+def return_tf_payload(status: str, values: dict):
+    '''
+    Package the payload to return to TF data.external.
+    Note: Using external_logger due to the stdout/stderr problem.
+    '''
+    external_logger.debug(f"Payload is: {values}")
+
+    payload = {'status': status, **values}
+    payload = convert_booleans_to_strings(payload)
+    print(json.dumps(payload))
+
+    external_logger.error("Flushing.")
+    exit(0)
\ No newline at end of file
diff --git a/000_main.tf b/000_main.tf
index 8bc2d52f..83ede8f6 100644
--- a/000_main.tf
+++ b/000_main.tf
@@ -64,12 +64,7 @@ data "aws_subnet" "existing" {
 # https://medium.com/@leslie.alldridge/terraform-external-data-source-using-custom-python-script-with-example-cea5e618d83e
 data "external" "generate_db_connection_string" {
   program = ["python3", "${path.module}/.githooks/data_external/generate_db_connection_string.py"]
-  #query = {
-    # tower_container_version = var.tower_container_version
-    # flag_use_container_db = var.flag_use_container_db
-    # db_container_engine_version = var.db_container_engine_version
-    # db_engine_version = var.db_engine_version
-  #}
+  query = {}
 }
 
 data "external" "generate_flags" {
@@ -241,7 +236,7 @@ locals {
 
   # tower_db_url = var.flag_create_external_db == true ? module.rds[0].db_instance_address : var.tower_db_url
   tower_db_root = ( var.flag_use_container_db == true? var.tower_db_url : module.rds[0].db_instance_address )
-  tower_db_url = "${local.tower_db_root}/${var.db_database_name}${data.external.generate_db_connection_string.result.value}"
+  tower_db_url = "${local.tower_db_root}/${var.db_database_name}${data.external.generate_db_connection_string.result.connection_string}"
 
 
   # Redis
diff --git a/012_outputs.tf b/012_outputs.tf
index 568e1753..2ff3bed9 100644
--- a/012_outputs.tf
+++ b/012_outputs.tf
@@ -27,5 +27,5 @@ output "redis_endpoint" {
 # Example for how to get values dynamically generated by `data.external`
 output "database_connection_string" {
   description = "Dynamically generated db connectino string based on tfvars selections."
-  value = data.external.generate_db_connection_string.result.value
+  value = data.external.generate_db_connection_string.result.connection_string
 }
\ No newline at end of file

From 0b75aaf26ac55ed2677eb94f0a88dc55ddbdc889 Mon Sep 17 00:00:00 2001
From: Graham Wright <graham.wright@seqera.io>
Date: Mon, 17 Jun 2024 18:22:19 -0400
Subject: [PATCH 8/8] Cleaned up data external template.

---
 .../template_get_tfvars_and_query.py          | 24 ++++++++---
 000_main.tf                                   | 41 +------------------
 2 files changed, 20 insertions(+), 45 deletions(-)

diff --git a/.githooks/data_external/template_get_tfvars_and_query.py b/.githooks/data_external/template_get_tfvars_and_query.py
index 7cc33652..b40fd0a6 100644
--- a/.githooks/data_external/template_get_tfvars_and_query.py
+++ b/.githooks/data_external/template_get_tfvars_and_query.py
@@ -3,7 +3,6 @@
 import json
 import sys
 from types import SimpleNamespace
-from typing import List
 
 sys.dont_write_bytecode = True
 
@@ -19,22 +18,35 @@
 project_root = os.getcwd()
 os.chdir(f"{project_root}/.githooks")
 sys.path.append(".")
+
 from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
+from utils.logger import external_logger
+from utils.common_data_external_functions import getDVal, return_tf_payload
 
-# Extract tfvars just like we do with the Python validation script
 os.chdir(project_root)
 data_dictionary = get_tfvars_as_json()
 data = SimpleNamespace(**data_dictionary)
 
 # Much simpler way to get variable passed in (via Terraform sending to stdin)
 query = json.load(sys.stdin)
-query = SimpleNamespace(**query)
+# query = SimpleNamespace(**query)
 ## ------------------------------------------------------------------------------------
 
 
-def return_tf_payload(status: str, value: str):
-    payload = {'status': status, 'value': value}
-    print(json.dumps(payload))
+def populate_values(query):
+
+    external_logger.debug(f"Query is: {query}")
+
+    # Add logic here
+
+    values = {
+        "key": "value"
+    }
+
+    return values
 
 
+if __name__ == '__main__':
+    values = populate_values(query)
+    return_tf_payload("0", values)
 
diff --git a/000_main.tf b/000_main.tf
index 83ede8f6..d54b2b43 100644
--- a/000_main.tf
+++ b/000_main.tf
@@ -146,48 +146,11 @@ locals {
   # ---------------------------------------------------------------------------------------
   # All values here refer to Route53 in same AWS account as Tower instance.
   # If R53 record not generated, will create entry in EC2 hosts file.
-
-  # dns_create_alb_record = var.flag_create_load_balancer == true && var.flag_create_hosts_file_entry == false ? true : false
-  # dns_create_ec2_record = var.flag_create_load_balancer == false && var.flag_create_hosts_file_entry == false ? true : false
-
-  # dns_zone_id = (
-  #   var.flag_create_route53_private_zone == true ? aws_route53_zone.private[0].id :
-  #   var.flag_use_existing_route53_public_zone == true ? data.aws_route53_zone.public[0].id :
-  #   var.flag_use_existing_route53_private_zone == true ? data.aws_route53_zone.private[0].id :
-  #   "No_Match_Found"
-  # )
-
-  # dns_instance_ip = (
-  #   var.flag_make_instance_private == true ? aws_instance.ec2.private_ip :
-  #   var.flag_make_instance_private_behind_public_alb == true ? aws_instance.ec2.private_ip :
-  #   var.flag_private_tower_without_eice == true ? aws_instance.ec2.private_ip :
-  #   var.flag_make_instance_public == true ? aws_eip.towerhost[0].public_ip :
-  #   "No_Match_Found"
-  # )
-
-  ## while calling jsondecode(str)
-  ##    │ data.external.generate_dns_values.result is map of string with 5 elements
-  # dns_vars = jsondecode(data.external.generate_dns_values.result)
-  # dns_create_alb_record = local.dns_vars["dns_create_alb_record"]
-  # dns_create_ec2_record = local.dns_vars["dns_create_ec2_record"]
-
-  # dns_zone_id = local.dns_vars["dns_zone_id"]
-  # dns_instance_ip = local.dns_vars["dns_instance_ip"]
-
-  # Need to jsondecode to handle true/false. Don't need to decode for the others.
-  # dns_create_alb_record = jsondecode(data.external.generate_dns_values.result.dns_create_alb_record)
-  # dns_create_ec2_record = jsondecode(data.external.generate_dns_values.result.dns_create_ec2_record)
   dns_create_alb_record = jsondecode(data.external.generate_flags.result.dns_create_alb_record)
   dns_create_ec2_record = jsondecode(data.external.generate_flags.result.dns_create_ec2_record)
 
-  dns_zone_id = data.external.generate_dns_values.result.dns_zone_id
-  dns_instance_ip = data.external.generate_dns_values.result.dns_instance_ip
-
-  # dns_create_alb_record = data.external.generate_dns_values.result.dns_create_alb_record
-  # dns_create_ec2_record = data.external.generate_dns_values.result.dns_create_ec2_record
-
-  # dns_zone_id = data.external.generate_dns_values.result.dns_zone_id
-  # dns_instance_ip = data.external.generate_dns_values.result.dns_instance_ip
+  dns_zone_id           = data.external.generate_dns_values.result.dns_zone_id
+  dns_instance_ip       = data.external.generate_dns_values.result.dns_instance_ip
 
 
   # If no HTTPS and no load-balancer, use `http` prefix and expose port in URL. Otherwise, use `https` prefix and no port.