Merge pull request #133 from utilitywarehouse/as-tf-alert-fix

fix terraform alert log link

common/terraform-applier.yaml.tmpl

@@ -12,7 +12,10 @@ groups:
         annotations:
           summary: "terraform-applier backlog is not being drained for last 20m, Queue depth for {{ $labels.name }} has reached {{ $value }}."
           impact: "terraform-applier is probably frozen and not processing/running any modules"
-          action: "Check logs and restart controller to fix"
+          action: |
+            Restart controller to fix the issue.
+            Please also collect `goroutine` info before restart for debugging the issue.
+            `https://terraform-applier-system.$ENVIRONMENT.$PROVIDER.uw.systems/debug/pprof/goroutine?debug=1`
           command: "`kubectl --context $ENVIRONMENT-$PROVIDER --namespace {{ $labels.kubernetes_namespace }} rollout restart sts {{ $labels.kubernetes_name }}`"
           dashboard: <https://grafana.$ENVIRONMENT.$PROVIDER.uw.systems/d/9kH3Tk0Zzd/terraform-applier-v2?orgId=1&refresh=5s"|link>
           logs: <https://grafana.$ENVIRONMENT.aws.uw.systems/explore?left=["now-1h","now","Loki",{"expr":"{kubernetes_cluster=\"{{$labels.kubernetes_cluster}}\",kubernetes_namespace=\"{{$labels.kubernetes_namespace}}\",kubernetes_pod_name=\"{{$labels.kubernetes_pod_name}}\"}"}]|link>
@@ -25,4 +28,4 @@ groups:
           summary: "terraform-applier has not been able to fetch {{ $labels.repo }} repository in the last 10m"
           impact: "terraform-applier will not be running modules from this repository"
           dashboard: <https://grafana.$ENVIRONMENT.$PROVIDER.uw.systems/d/9kH3Tk0Zzd/terraform-applier-v2?orgId=1&refresh=5s"|link>
-          logs: <https://grafana.$ENVIRONMENT.aws.uw.systems/explore?left=["now-1h","now","Loki",{"expr":"{kubernetes_cluster=\"{{$labels.kubernetes_cluster}}\",kubernetes_namespace=\"{{$labels.kubernetes_namespace}}\",kubernetes_pod_name=\"{{$labels.kubernetes_pod_name}}\"}"}]|link>
+          logs: <https://grafana.$ENVIRONMENT.aws.uw.systems/explore?left=["now-1h","now","Loki",{"expr":"{kubernetes_cluster=\"$ENVIRONMENT-$PROVIDER\",kubernetes_namespace=\"sys-terraform-applier\"}"}]|link>