ECI-395 Search Resources for Logging

datadog-logs-oci-orm/data.tf

@@ -0,0 +1,3 @@
+data "external" "logging_services" {
+    program = ["bash", "logging_services.sh"]
+}

datadog-logs-oci-orm/locals.tf

@@ -4,3 +4,32 @@ locals {
     datadog-terraform = "true"
   }
 }
+
+locals {
+  # Decode the uploaded CSV file into a map
+  logging_csv_content = base64decode(var.logging_compartments_csv)
+  logging_compartments = csvdecode(local.logging_csv_content)
+
+  # Extract only the compartment IDs into a list
+  logging_compartment_ids = [for row in local.logging_compartments : row.compartment_id]
+
+  # Parse the content from the external data source
+  logging_services = jsondecode(data.external.logging_services.result["content"])
+
+  # Filter services to exclude those in exclude_services
+  filtered_services = [
+    for service in local.logging_services : service
+    if !contains(var.exclude_services, service.id)
+  ]
+
+  # Generate a Cartesian product of compartments and filtered services
+  logging_targets = flatten([
+    for compartment_id in local.logging_compartment_ids : [
+      for service in local.filtered_services : {
+        compartment_id = compartment_id
+        service_id     = service.id
+        resource_types = service.resourceTypes
+      }
+    ]
+  ])
+}

datadog-logs-oci-orm/logging_services.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+output_file="oci_logging_services.json"
+echo "[]" > $output_file # Initialize the output file with an empty JSON array
+
+# Fetch logging services using OCI CLI
+response=$(oci logging service list --all --query "data[].{id:id, resourceTypes:\"resource-types\"[].{name:name, categories:categories[].{name:name}}}" --output json)
+
+# Write the response to the output file
+echo "$response" > "$output_file"
+
+# Output the response in a valid JSON map for Terraform's external data source
+content=$(jq -c . < "$output_file") # Ensure the file's content is compact JSON
+rm -f "$output_file"
+echo "{\"content\": \"$(echo "$content" | sed 's/"/\\"/g')\"}" # Escape quotes for JSON compatibility

datadog-logs-oci-orm/main.tf

@@ -47,3 +47,11 @@ module "function" {
     function_app_ocid = module.functionapp.function_app_details.function_app_ocid
     function_image_path = var.function_image_path == "" ? module.containerregistry[0].containerregistry_details.function_image_path : var.function_image_path
 }
+
+module "resourcediscovery" {
+    for_each = { for target in local.logging_targets : "${target.compartment_id}_${target.service_id}" => target }
+    source = "./modules/resourcediscovery"
+    compartment_ocid = each.value.compartment_id
+    group_id = each.value.service_id
+    resource_types = [for rt in each.value.resource_types : rt.name]
+}

datadog-logs-oci-orm/modules/resourcediscovery/data.tf

@@ -0,0 +1,3 @@
+data "external" "find_resources" {
+    program = ["bash", "modules/resourcediscovery/search_resources.sh", var.compartment_ocid, var.group_id, local.resource_types_string]
+}

datadog-logs-oci-orm/modules/resourcediscovery/locals.tf

@@ -0,0 +1,3 @@
+locals {
+    resource_types_string = join(",", [for rt in var.resource_types : rt])
+}

datadog-logs-oci-orm/modules/resourcediscovery/outputs.tf

@@ -0,0 +1,3 @@
+output "response" {
+  value = jsondecode(data.external.find_resources.result["content"])
+}

datadog-logs-oci-orm/modules/resourcediscovery/search_resources.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+export COMPARTMENT_ID="${1}"
+export GROUP_ID="${2}"
+export RESOURCE_TYPES="${3}"
+
+output_file="oci_resources_${GROUP_ID}_${COMPARTMENT_ID}.json"
+echo "[]" > $output_file # Initialize the output file with an empty JSON array
+IFS=',' read -ra types <<< "$RESOURCE_TYPES"
+
+# Perform OCI CLI query for each resource type
+for rt in "${types[@]}"; do
+    # Initialize variables for pagination
+    next_page="init"
+
+    # Loop through pages
+    while [ "$next_page" != "null" ]; do
+        if [ "$next_page" == "init" ]; then
+            # First query without next page token
+            response=$(oci search resource structured-search --query-text \
+                "QUERY $rt resources where compartmentId = '$COMPARTMENT_ID' && lifeCycleState != 'TERMINATED' && lifeCycleState != 'FAILED'" \
+                --output json)
+        else
+            # Query with next page token
+            response=$(oci search resource structured-search --query-text \
+                "QUERY $rt resources where compartmentId = '$COMPARTMENT_ID' && lifeCycleState != 'TERMINATED' && lifeCycleState != 'FAILED'" \
+                --page "$next_page" \
+                --output json)
+        fi
+
+        # Check if the response contains an error
+        if [ -z "$response" ]; then
+            # Log ServiceError responses
+            echo "ServiceError for $GROUP_ID, resource type: $rt" >> error.log
+            break
+        fi
+
+        # Extract next page token
+        next_page=$(echo "$response" | jq -r '."opc-next-page" // "null"')
+
+        # Extract and transform items
+        items=$(echo "$response" | jq --arg group_id "$GROUP_ID" -c '[.data.items[] | {compartmentId: ."compartment-id", displayName: ."display-name", identifier: ."identifier", resourceType: ."resource-type", groupId: $group_id}]')
+        # Append items to the output file if not empty
+        if [ "$(echo "$items" | jq length)" -gt 0 ]; then
+            temp_items_file="temp_items_$${GROUP_ID}_$${COMPARTMENT_ID}.json"
+            echo "$items" > "$temp_items_file"
+            jq -s '.[0] + .[1]' "$output_file" "$temp_items_file" > tmp_${GROUP_ID}_${COMPARTMENT_ID}.json && mv tmp_${GROUP_ID}_${COMPARTMENT_ID}.json "$output_file"
+            rm -f "$temp_items_file"
+        fi
+    done
+done
+
+# Read the file's content and return it as a JSON-encoded string
+content=$(jq -c . < "$output_file")
+rm -f "$output_file"
+echo "{\"content\": \"$(echo "$content" | sed 's/"/\\"/g')\"}"

datadog-logs-oci-orm/modules/resourcediscovery/variables.tf

@@ -0,0 +1,14 @@
+variable "group_id" {
+  type        = string
+  description = "Unique Name for the group of resource types"
+}
+
+variable "compartment_ocid" {
+  type        = string
+  description = "The OCID of the compartment where resources exist"
+}
+
+variable "resource_types" {
+  type        = list(string)
+  description = "List of resource types"
+}

datadog-logs-oci-orm/outputs.tf

@@ -23,3 +23,9 @@ output "function_details" {
   description = "Output of function creation"
   value = module.function.function_details
 }
+
+output "resources" {
+    value = {
+      for k, v in module.resourcediscovery : k => v.response if length(v.response) > 0
+    }
+}

datadog-logs-oci-orm/schema.yaml

@@ -36,6 +36,10 @@ variableGroups:
       - ${auth_token_description}
       - ${auth_token}
     visible: ${create_new_function_image}
+  - title: "Logging"
+    variables:
+      - ${exclude_services}
+      - ${logging_compartments_csv}
 
 variables:
   current_user_ocid:
@@ -183,3 +187,50 @@ variables:
     visible:
       not:
         - ${create_auth_token}
+
+#Logging
+  exclude_services:
+    title: Exclude Services from Logging
+    type: enum
+    description: List of services to exclude from logging.
+    required: false
+    default: []
+    additionalProps:
+      allowMultiple: true
+    enum:
+      - "oacnativeproduction"
+      - "apigateway"
+      - "adm"
+      - "apm"
+      - "cloud_guard_query_results_prod"
+      - "cloud_guard_raw_logs_prod"
+      - "och"
+      - "oke-k8s-cp-prod"
+      - "contentdeliverynetwork"
+      - "dataflow"
+      - "dataintegration"
+      - "datascience"
+      - "delegateaccessprod"
+      - "devops"
+      - "emaildelivery"
+      - "cloudevents"
+      - "filestorage"
+      - "functions"
+      - "goldengate"
+      - "integration"
+      - "loadbalancer"
+      - "mediaflow"
+      - "ocinetworkfirewall"
+      - "objectstorage"
+      - "operatoraccessprod"
+      - "postgresql"
+      - "oci_c3_vpn"
+      - "flowlogs"
+      - "waa"
+      - "waf"
+
+  logging_compartments_csv:
+    title: Compartments CSV
+    description: "Upload a CSV file containing the list of compartments. The file must include a column named 'compartment_id'."
+    type: file
+    required: true

datadog-logs-oci-orm/variables.tf

@@ -110,3 +110,17 @@ variable "service_user_ocid" {
   default     = ""
   description = "The OCID of the service user to be used for Docker login and pushing images."
 }
+
+#***************
+#    Logging    
+#***************
+variable "exclude_services" {
+  type        = list(string)
+  default     = []
+  description = "List of services to be excluded from logging"
+}
+
+variable "logging_compartments_csv" {
+  description = "Base64-encoded CSV file containing compartment IDs."
+  type        = string
+}