Skip to content

Commit 178ccf0

Browse files
sva91sva91
authored
ECI-395 Search Resources for Logging (#24)
* working draft commit * Logging Module * uncomment modules * simplify file removal * Rename Logging Module to ResourceDiscovery * reviewer comments ECI-395
1 parent a6a999a commit 178ccf0

File tree

12 files changed

+205
-0
lines changed

12 files changed

+205
-0
lines changed

Diff for: ‎datadog-logs-oci-orm/data.tf

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
data "external" "logging_services" {
2+
program = ["bash", "logging_services.sh"]
3+
}

Diff for: ‎datadog-logs-oci-orm/locals.tf

+29
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,32 @@ locals {
44
datadog-terraform = "true"
55
}
66
}
7+
8+
locals {
9+
# Decode the uploaded CSV file into a map
10+
logging_csv_content = base64decode(var.logging_compartments_csv)
11+
logging_compartments = csvdecode(local.logging_csv_content)
12+
13+
# Extract only the compartment IDs into a list
14+
logging_compartment_ids = [for row in local.logging_compartments : row.compartment_id]
15+
16+
# Parse the content from the external data source
17+
logging_services = jsondecode(data.external.logging_services.result["content"])
18+
19+
# Filter services to exclude those in exclude_services
20+
filtered_services = [
21+
for service in local.logging_services : service
22+
if !contains(var.exclude_services, service.id)
23+
]
24+
25+
# Generate a Cartesian product of compartments and filtered services
26+
logging_targets = flatten([
27+
for compartment_id in local.logging_compartment_ids : [
28+
for service in local.filtered_services : {
29+
compartment_id = compartment_id
30+
service_id = service.id
31+
resource_types = service.resourceTypes
32+
}
33+
]
34+
])
35+
}

Diff for: ‎datadog-logs-oci-orm/logging_services.sh

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
3+
output_file="oci_logging_services.json"
4+
echo "[]" > $output_file # Initialize the output file with an empty JSON array
5+
6+
# Fetch logging services using OCI CLI
7+
response=$(oci logging service list --all --query "data[].{id:id, resourceTypes:\"resource-types\"[].{name:name, categories:categories[].{name:name}}}" --output json)
8+
9+
# Write the response to the output file
10+
echo "$response" > "$output_file"
11+
12+
# Output the response in a valid JSON map for Terraform's external data source
13+
content=$(jq -c . < "$output_file") # Ensure the file's content is compact JSON
14+
rm -f "$output_file"
15+
echo "{\"content\": \"$(echo "$content" | sed 's/"/\\"/g')\"}" # Escape quotes for JSON compatibility

Diff for: ‎datadog-logs-oci-orm/main.tf

+8
Original file line numberDiff line numberDiff line change
@@ -47,3 +47,11 @@ module "function" {
4747
function_app_ocid = module.functionapp.function_app_details.function_app_ocid
4848
function_image_path = var.function_image_path == "" ? module.containerregistry[0].containerregistry_details.function_image_path : var.function_image_path
4949
}
50+
51+
module "resourcediscovery" {
52+
for_each = { for target in local.logging_targets : "${target.compartment_id}_${target.service_id}" => target }
53+
source = "./modules/resourcediscovery"
54+
compartment_ocid = each.value.compartment_id
55+
group_id = each.value.service_id
56+
resource_types = [for rt in each.value.resource_types : rt.name]
57+
}

Diff for: ‎datadog-logs-oci-orm/modules/resourcediscovery/data.tf

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
data "external" "find_resources" {
2+
program = ["bash", "modules/resourcediscovery/search_resources.sh", var.compartment_ocid, var.group_id, local.resource_types_string]
3+
}

Diff for: ‎datadog-logs-oci-orm/modules/resourcediscovery/locals.tf

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
locals {
2+
resource_types_string = join(",", [for rt in var.resource_types : rt])
3+
}

Diff for: ‎datadog-logs-oci-orm/modules/resourcediscovery/outputs.tf

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
output "response" {
2+
value = jsondecode(data.external.find_resources.result["content"])
3+
}

Diff for: ‎datadog-logs-oci-orm/modules/resourcediscovery/search_resources.sh

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/bin/bash
2+
3+
export COMPARTMENT_ID="${1}"
4+
export GROUP_ID="${2}"
5+
export RESOURCE_TYPES="${3}"
6+
7+
output_file="oci_resources_${GROUP_ID}_${COMPARTMENT_ID}.json"
8+
echo "[]" > $output_file # Initialize the output file with an empty JSON array
9+
IFS=',' read -ra types <<< "$RESOURCE_TYPES"
10+
11+
# Perform OCI CLI query for each resource type
12+
for rt in "${types[@]}"; do
13+
# Initialize variables for pagination
14+
next_page="init"
15+
16+
# Loop through pages
17+
while [ "$next_page" != "null" ]; do
18+
if [ "$next_page" == "init" ]; then
19+
# First query without next page token
20+
response=$(oci search resource structured-search --query-text \
21+
"QUERY $rt resources where compartmentId = '$COMPARTMENT_ID' && lifeCycleState != 'TERMINATED' && lifeCycleState != 'FAILED'" \
22+
--output json)
23+
else
24+
# Query with next page token
25+
response=$(oci search resource structured-search --query-text \
26+
"QUERY $rt resources where compartmentId = '$COMPARTMENT_ID' && lifeCycleState != 'TERMINATED' && lifeCycleState != 'FAILED'" \
27+
--page "$next_page" \
28+
--output json)
29+
fi
30+
31+
# Check if the response contains an error
32+
if [ -z "$response" ]; then
33+
# Log ServiceError responses
34+
echo "ServiceError for $GROUP_ID, resource type: $rt" >> error.log
35+
break
36+
fi
37+
38+
# Extract next page token
39+
next_page=$(echo "$response" | jq -r '."opc-next-page" // "null"')
40+
41+
# Extract and transform items
42+
items=$(echo "$response" | jq --arg group_id "$GROUP_ID" -c '[.data.items[] | {compartmentId: ."compartment-id", displayName: ."display-name", identifier: ."identifier", resourceType: ."resource-type", groupId: $group_id}]')
43+
# Append items to the output file if not empty
44+
if [ "$(echo "$items" | jq length)" -gt 0 ]; then
45+
temp_items_file="temp_items_$${GROUP_ID}_$${COMPARTMENT_ID}.json"
46+
echo "$items" > "$temp_items_file"
47+
jq -s '.[0] + .[1]' "$output_file" "$temp_items_file" > tmp_${GROUP_ID}_${COMPARTMENT_ID}.json && mv tmp_${GROUP_ID}_${COMPARTMENT_ID}.json "$output_file"
48+
rm -f "$temp_items_file"
49+
fi
50+
done
51+
done
52+
53+
# Read the file's content and return it as a JSON-encoded string
54+
content=$(jq -c . < "$output_file")
55+
rm -f "$output_file"
56+
echo "{\"content\": \"$(echo "$content" | sed 's/"/\\"/g')\"}"

Diff for: ‎datadog-logs-oci-orm/modules/resourcediscovery/variables.tf

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
variable "group_id" {
2+
type = string
3+
description = "Unique Name for the group of resource types"
4+
}
5+
6+
variable "compartment_ocid" {
7+
type = string
8+
description = "The OCID of the compartment where resources exist"
9+
}
10+
11+
variable "resource_types" {
12+
type = list(string)
13+
description = "List of resource types"
14+
}

Diff for: ‎datadog-logs-oci-orm/outputs.tf

+6
Original file line numberDiff line numberDiff line change
@@ -23,3 +23,9 @@ output "function_details" {
2323
description = "Output of function creation"
2424
value = module.function.function_details
2525
}
26+
27+
output "resources" {
28+
value = {
29+
for k, v in module.resourcediscovery : k => v.response if length(v.response) > 0
30+
}
31+
}

Diff for: ‎datadog-logs-oci-orm/schema.yaml

+51
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ variableGroups:
3636
- ${auth_token_description}
3737
- ${auth_token}
3838
visible: ${create_new_function_image}
39+
- title: "Logging"
40+
variables:
41+
- ${exclude_services}
42+
- ${logging_compartments_csv}
3943

4044
variables:
4145
current_user_ocid:
@@ -183,3 +187,50 @@ variables:
183187
visible:
184188
not:
185189
- ${create_auth_token}
190+
191+
#Logging
192+
exclude_services:
193+
title: Exclude Services from Logging
194+
type: enum
195+
description: List of services to exclude from logging.
196+
required: false
197+
default: []
198+
additionalProps:
199+
allowMultiple: true
200+
enum:
201+
- "oacnativeproduction"
202+
- "apigateway"
203+
- "adm"
204+
- "apm"
205+
- "cloud_guard_query_results_prod"
206+
- "cloud_guard_raw_logs_prod"
207+
- "och"
208+
- "oke-k8s-cp-prod"
209+
- "contentdeliverynetwork"
210+
- "dataflow"
211+
- "dataintegration"
212+
- "datascience"
213+
- "delegateaccessprod"
214+
- "devops"
215+
- "emaildelivery"
216+
- "cloudevents"
217+
- "filestorage"
218+
- "functions"
219+
- "goldengate"
220+
- "integration"
221+
- "loadbalancer"
222+
- "mediaflow"
223+
- "ocinetworkfirewall"
224+
- "objectstorage"
225+
- "operatoraccessprod"
226+
- "postgresql"
227+
- "oci_c3_vpn"
228+
- "flowlogs"
229+
- "waa"
230+
- "waf"
231+
232+
logging_compartments_csv:
233+
title: Compartments CSV
234+
description: "Upload a CSV file containing the list of compartments. The file must include a column named 'compartment_id'."
235+
type: file
236+
required: true

Diff for: ‎datadog-logs-oci-orm/variables.tf

+14
Original file line numberDiff line numberDiff line change
@@ -110,3 +110,17 @@ variable "service_user_ocid" {
110110
default = ""
111111
description = "The OCID of the service user to be used for Docker login and pushing images."
112112
}
113+
114+
#***************
115+
# Logging
116+
#***************
117+
variable "exclude_services" {
118+
type = list(string)
119+
default = []
120+
description = "List of services to be excluded from logging"
121+
}
122+
123+
variable "logging_compartments_csv" {
124+
description = "Base64-encoded CSV file containing compartment IDs."
125+
type = string
126+
}

0 commit comments

Comments
 (0)