INSIGHTS-465 - Improve BigQuery logs on Cloud Costs plugin (#978)

* INSIGHTS-465 Improve BigQuery logs on Cloud Costs plugin * Adding more debug * Adding more debug * Adding more debug * Adding more debug * Adding more debug * Adding more debug * Adding more debug * Adding more debug * Adding more debug * Adding more debug * Adding more debug * Fixing issue * Update plugins/cloud-costs/cloud-costs.sh Co-authored-by: Vitor Rodrigo Vezani <[email protected]> --------- Co-authored-by: Vitor Rodrigo Vezani <[email protected]>
FairwindsOps · Nov 8, 2024 · dc73fe9 · dc73fe9
1 parent 57e1d3f
commit dc73fe9
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 3 deletions.
diff --git a/plugins/cloud-costs/CHANGELOG.md b/plugins/cloud-costs/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.3.10
+* improved big query logs
+
 ## 0.3.9
 * upgraded google cloud sdk
 

diff --git a/plugins/cloud-costs/cloud-costs.sh b/plugins/cloud-costs/cloud-costs.sh
@@ -177,9 +177,21 @@ if [[ "$provider" == "gcp" ]]; then
   echo "Google bigquey is running......"
 
   sql="SELECT cost, 0.0 AS cost_at_list, '' AS cost_type, service, sku, usage, usage_start_time, usage_end_time, CASE WHEN machine_spec IS NOT NULL THEN [ STRUCT('compute.googleapis.com/machine_spec' AS key, machine_spec AS value), STRUCT('compute.googleapis.com/cores' AS key, total_cores AS value), STRUCT('compute.googleapis.com/memory' AS key, total_memory AS value)  ] ELSE NULL END AS system_labels FROM (  SELECT SUM(main.cost) AS cost, STRUCT(main.service.description) AS service, STRUCT(main.sku.description AS description) AS sku, main.usage_start_time, main.usage_end_time, STRUCT(SUM(main.usage.amount) AS amount, SUM(main.usage.amount_in_pricing_units) AS amount_in_pricing_units, '' AS pricing_unit, '' AS unit) AS usage, (SELECT value FROM UNNEST(system_labels) WHERE key = 'compute.googleapis.com/machine_spec') AS machine_spec, (SELECT value FROM UNNEST(system_labels) WHERE key = 'compute.googleapis.com/cores') AS total_cores, (SELECT value FROM UNNEST(system_labels) WHERE key = 'compute.googleapis.com/memory') AS total_memory FROM \`$table\` AS main LEFT JOIN UNNEST(labels) AS labels WHERE labels.key = '$tagkey' AND labels.value = '$tagvalue' and usage_start_time >= '$initial_date_time' AND usage_start_time < '$final_date_time' AND TIMESTAMP_TRUNC(_PARTITIONTIME, DAY) >= '$initial_date_time' AND TIMESTAMP_TRUNC(_PARTITIONTIME, DAY) <= '$final_date_time' GROUP BY main.service.description, usage_start_time, usage_end_time, main.sku.description, machine_spec, total_memory, total_cores) order by usage_start_time desc"
-  /google-cloud-sdk/bin/bq --format=prettyjson query --max_rows=10000000 --nouse_legacy_sql "$sql" > /output/cloudcosts-tmp.json
-  echo "Google bigquey finished......"
 
+  /google-cloud-sdk/bin/bq --format=prettyjson query --max_rows=10000000 --nouse_legacy_sql "$sql" > /output/cloudcosts-tmp.json  && echo "Executing..."
+  if grep -q "BigQuery error" /output/cloudcosts-tmp.json; then
+    echo "Error executing the query"
+    cat /output/cloudcosts-tmp.json
+    exit 1
+  fi
+
+  if ! grep -q "\[" /output/cloudcosts-tmp.json; then
+    echo "No data found for the given tag value"
+    cat /output/cloudcosts-tmp.json
+    exit 1
+  fi
+
+  echo "Google BigQuey finished..."
   awk '$0 == "[" {p=1} p' < /output/cloudcosts-tmp.json > /output/cloudcosts-tmp-clean.json
 
   mv /output/cloudcosts-tmp-clean.json /output/cloudcosts.json

diff --git a/plugins/cloud-costs/version.txt b/plugins/cloud-costs/version.txt
@@ -1 +1 @@
-0.3.9
+0.3.10