implement azuretracing metrics

Signed-off-by: Markus Blaschke <[email protected]>

Author: mblaschke

README.md

@@ -1,9 +1,9 @@
-Azurer Kubernetes Autopilot
-============================
+# Azure Kubernetes Autopilot
 
 [![license](https://img.shields.io/github/license/webdevops/azure-k8s-autopilot.svg)](https://github.com/webdevops/azure-k8s-autopilot/blob/master/LICENSE)
 [![DockerHub](https://img.shields.io/badge/DockerHub-webdevops%2Fazure--k8s--autopilot-blue)](https://hub.docker.com/r/webdevops/azure-k8s-autopilot/)
 [![Quay.io](https://img.shields.io/badge/Quay.io-webdevops%2Fazure--k8s--autopilot-blue)](https://quay.io/repository/webdevops/azure-k8s-autopilot)
+[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/azure-k8s-autopilot)](https://artifacthub.io/packages/search?repo=azure-k8s-autopilot)
 
 Kubernetess service for automatic maintenance of an Azure cluster.
 
@@ -16,8 +16,7 @@ Supports [shoutrrr](https://containrrr.github.io/shoutrrr/) notifications.
 
 (Successor of `azure-k8s-autorepair`)
 
-Configuration
--------------
+## Configuration
 
 ```
 Usage:
@@ -134,8 +133,7 @@ for Azure API authentication (using ENV vars) see https://docs.microsoft.com/en-
 
 for Kubernetes ServiceAccont is discoverd automatically (or you can use env path `KUBECONFIG` to specify path to your kubeconfig file)
 
-Metrics
--------
+## Metrics
 
  (see `:8080/metrics`)
 
@@ -146,3 +144,22 @@ Metrics
 | `autopilot_repair_duration`    | Duration of repair task                         |
 | `autopilot_update_count`       | Count of update actions                         |
 | `autopilot_update_duration`    | Duration of last exec                           |
+
+### Azuretracing metrics
+
+(with 22.2.0 and later)
+
+Azuretracing metrics collects latency and latency from azure-sdk-for-go and creates metrics and is controllable using
+environment variables (eg. setting buckets, disabling metrics or disable autoreset).
+
+| Metric                                   | Description                                                                            |
+|------------------------------------------|----------------------------------------------------------------------------------------|
+| `azurerm_api_ratelimit`                  | Azure ratelimit metrics (only on /metrics, resets after query due to limited validity) |
+| `azurerm_api_request_*`                  | Azure request count and latency as histogram                                           |
+
+| Environment variable                     | Example                          | Description                                              |
+|------------------------------------------|----------------------------------|----------------------------------------------------------|
+| `METRIC_AZURERM_API_REQUEST_BUCKETS`     | `1, 2.5, 5, 10, 30, 60, 90, 120` | Sets buckets for `azurerm_api_request` histogram metric  |
+| `METRIC_AZURERM_API_REQUEST_DISABLE`     | `false`                          | Disables `azurerm_api_request_*` metric                  |
+| `METRIC_AZURERM_API_RATELIMIT_DISABLE`   | `false`                          | Disables `azurerm_api_ratelimit` metric                  |
+| `METRIC_AZURERM_API_RATELIMIT_AUTORESET` | `false`                          | Disables `azurerm_api_ratelimit` autoreset after fetch   |

autopilot/lib.azure.go

@@ -19,10 +19,10 @@ func (r *AzureK8sAutopilot) azureVmssInstanceRepair(contextLogger *log.Entry, no
 	}
 
 	vmssClient := compute.NewVirtualMachineScaleSetsClientWithBaseURI(r.azureEnvironment.ResourceManagerEndpoint, nodeInfo.Subscription)
-	vmssClient.Authorizer = r.azureAuthorizer
+	r.decorateAzureAutoRest(&vmssClient.BaseClient.Client)
 
 	vmssVmClient := compute.NewVirtualMachineScaleSetVMsClientWithBaseURI(r.azureEnvironment.ResourceManagerEndpoint, nodeInfo.Subscription)
-	vmssVmClient.Authorizer = r.azureAuthorizer
+	r.decorateAzureAutoRest(&vmssVmClient.BaseClient.Client)
 
 	// fetch instances
 	vmInstance, err := vmssVmClient.Get(r.ctx, nodeInfo.ResourceGroup, nodeInfo.VMScaleSetName, nodeInfo.VMInstanceID, "")
@@ -87,7 +87,7 @@ func (r *AzureK8sAutopilot) azureVmRepair(contextLogger *log.Entry, nodeInfo k8s
 	var err error
 
 	client := compute.NewVirtualMachinesClientWithBaseURI(r.azureEnvironment.ResourceManagerEndpoint, nodeInfo.Subscription)
-	client.Authorizer = r.azureAuthorizer
+	r.decorateAzureAutoRest(&client.BaseClient.Client)
 
 	// fetch instances
 	vmInstance, err := client.Get(r.ctx, nodeInfo.ResourceGroup, nodeInfo.VMname, "")
@@ -132,10 +132,10 @@ func (r *AzureK8sAutopilot) azureVmssInstanceUpdate(contextLogger *log.Entry, no
 	var err error
 
 	vmssClient := compute.NewVirtualMachineScaleSetsClientWithBaseURI(r.azureEnvironment.ResourceManagerEndpoint, nodeInfo.Subscription)
-	vmssClient.Authorizer = r.azureAuthorizer
+	r.decorateAzureAutoRest(&vmssClient.BaseClient.Client)
 
 	vmssVmClient := compute.NewVirtualMachineScaleSetVMsClientWithBaseURI(r.azureEnvironment.ResourceManagerEndpoint, nodeInfo.Subscription)
-	vmssVmClient.Authorizer = r.azureAuthorizer
+	r.decorateAzureAutoRest(&vmssVmClient.BaseClient.Client)
 
 	// fetch instances
 	vmInstance, err := vmssVmClient.Get(r.ctx, nodeInfo.ResourceGroup, nodeInfo.VMScaleSetName, nodeInfo.VMInstanceID, "")

autopilot/main.go

@@ -13,6 +13,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/webdevopos/azure-k8s-autopilot/config"
 	"github.com/webdevopos/azure-k8s-autopilot/k8s"
+	"github.com/webdevops/go-prometheus-common/azuretracing"
 	"golang.org/x/net/context"
 	"k8s.io/api/policy/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -33,6 +34,8 @@ type (
 		ctx    context.Context
 		Config config.Opts
 
+		UserAgent string
+
 		cron struct {
 			repair *cron.Cron
 			update *cron.Cron
@@ -93,6 +96,7 @@ func (r *AzureK8sAutopilot) Init() {
 		AzureAuthorizer:   r.azureAuthorizer,
 		AzureEnvironment:  r.azureEnvironment,
 		Client:            r.k8sClient,
+		UserAgent:         r.UserAgent,
 	}
 
 	r.Config.Repair.ProvisioningStateAll = false
@@ -451,3 +455,12 @@ func (r *AzureK8sAutopilot) autoUncordonExpiredNodes(contextLogger *log.Entry, n
 		}
 	}
 }
+
+func (r *AzureK8sAutopilot) decorateAzureAutoRest(client *autorest.Client) {
+	client.Authorizer = r.azureAuthorizer
+	if err := client.AddToUserAgent(r.UserAgent); err != nil {
+		log.Panic(err)
+	}
+
+	azuretracing.DecoreAzureAutoRest(client)
+}

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/prometheus/client_golang v1.12.1
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/sirupsen/logrus v1.8.1
+	github.com/webdevops/go-prometheus-common v0.0.0-20220211101814-4c9913788fe1
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2
 	k8s.io/api v0.23.0
 	k8s.io/apimachinery v0.23.0
@@ -61,7 +62,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
+	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
 	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect

go.sum

@@ -600,6 +600,8 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/webdevops/go-prometheus-common v0.0.0-20220211101814-4c9913788fe1 h1:wIf6O43jGEarp8ojgInXRIt6jHeQ94JUvsy4O3wqKHY=
+github.com/webdevops/go-prometheus-common v0.0.0-20220211101814-4c9913788fe1/go.mod h1:naEkgDRh6L+Ef0qzaS9Q60m1GkwN5Mu6xaNd/Ha84Sg=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -865,8 +867,10 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

k8s/nodelist.go

@@ -8,6 +8,7 @@ import (
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/patrickmn/go-cache"
 	log "github.com/sirupsen/logrus"
+	"github.com/webdevops/go-prometheus-common/azuretracing"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/watch"
@@ -26,6 +27,8 @@ type (
 		AzureAuthorizer  autorest.Authorizer
 		AzureEnvironment azure.Environment
 
+		UserAgent string
+
 		nodeWatcher watch.Interface
 		azureCache  *cache.Cache
 		ctx         context.Context
@@ -188,7 +191,7 @@ func (n *NodeList) refreshAzureVmssCache() error {
 		})
 
 		vmssVmClient := compute.NewVirtualMachineScaleSetVMsClientWithBaseURI(n.AzureEnvironment.ResourceManagerEndpoint, vmssInfo.Subscription)
-		vmssVmClient.Authorizer = n.AzureAuthorizer
+		n.decorateAzureAutoRest(&vmssVmClient.BaseClient.Client)
 
 		vmssInstanceList, err := vmssVmClient.List(n.ctx, vmssInfo.ResourceGroup, vmssInfo.VMScaleSetName, "", "", "")
 		if err != nil {
@@ -249,3 +252,12 @@ func (n *NodeList) GetAzureVmssList() (vmssList map[string]*NodeInfo, err error)
 
 	return
 }
+
+func (n *NodeList) decorateAzureAutoRest(client *autorest.Client) {
+	client.Authorizer = n.AzureAuthorizer
+	if err := client.AddToUserAgent(n.UserAgent); err != nil {
+		log.Panic(err)
+	}
+
+	azuretracing.DecoreAzureAutoRest(client)
+}

main.go

@@ -7,6 +7,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/webdevopos/azure-k8s-autopilot/autopilot"
 	"github.com/webdevopos/azure-k8s-autopilot/config"
+	"github.com/webdevops/go-prometheus-common/azuretracing"
 	"net/http"
 	"os"
 	"os/signal"
@@ -18,6 +19,8 @@ import (
 
 const (
 	Author = "webdevops.io"
+
+	UserAgent = "azure-k8s-autopilot/"
 )
 
 var (
@@ -37,7 +40,8 @@ func main() {
 	log.Info(string(opts.GetJson()))
 
 	pilot := autopilot.AzureK8sAutopilot{
-		Config: opts,
+		Config:    opts,
+		UserAgent: UserAgent + gitTag,
 	}
 	pilot.Init()
 	pilot.Start()
@@ -115,7 +119,8 @@ func startHttpServer() {
 		}
 	})
 
-	http.Handle("/metrics", promhttp.Handler())
+	http.Handle("/metrics", azuretracing.RegisterAzureMetricAutoClean(promhttp.Handler()))
+
 	go func() {
 		log.Fatal(http.ListenAndServe(opts.ServerBind, nil))
 	}()