forked from newrelic/nri-flex
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathconfluent-kafka.yml
133 lines (133 loc) · 4.37 KB
/
confluent-kafka.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
---
integrations:
- name: nri-flex
# interval: 30s
config:
name: confluentKafkaFlex
global:
base_url: https://api.telemetry.confluent.cloud
user: <[email protected]>
pass: <password>
headers:
accept: application/json
apis:
# https://docs.confluent.io/current/cloud/metrics-api.html
- event_type: confluentKafkaSample
url: /v1/metrics/cloud/query
rename_keys:
value: received_bytes
metric.label.topic: topic_name
method: POST
payload: >
{
"aggregations": [
{
"agg": "SUM",
"metric": "io.confluent.kafka.server/received_bytes/delta"
}
],
"filter": {
"filters": [
{
"field": "metric.label.cluster_id",
"op": "EQ",
"value": "<clusterID>"
}
],
"op": "AND"
},
"granularity": "PT1M",
"group_by": [
"metric.label.topic"
],
"intervals": [
"${timestamp:datetimeutctz-2min}/${timestamp:datetimeutctz-1min}"
],
"limit": 1000
}
- event_type: confluentKafkaSample
url: /v1/metrics/cloud/query
rename_keys:
value: retained_bytes
metric.label.topic: topic_name
method: POST
payload: >
{
"aggregations": [
{
"agg": "SUM",
"metric": "io.confluent.kafka.server/retained_bytes"
}
],
"filter": {
"filters": [
{
"field": "metric.label.cluster_id",
"op": "EQ",
"value": "<clusterID>"
}
],
"op": "AND"
},
"granularity": "PT1M",
"group_by": [
"metric.label.topic"
],
"intervals": [
"${timestamp:datetimeutctz-2min}/${timestamp:datetimeutctz-1min}"
],
"limit": 1000
}
- event_type: confluentKafkaSample
url: /v1/metrics/cloud/query
rename_keys:
value: sent_bytes
metric.label.topic: topic_name
method: POST
payload: >
{
"aggregations": [
{
"agg": "SUM",
"metric": "io.confluent.kafka.server/sent_bytes/delta"
}
],
"filter": {
"filters": [
{
"field": "metric.label.cluster_id",
"op": "EQ",
"value": "<clusterID>"
}
],
"op": "AND"
},
"granularity": "PT1M",
"group_by": [
"metric.label.topic"
],
"intervals": [
"${timestamp:datetimeutctz-2min}/${timestamp:datetimeutctz-1min}"
],
"limit": 1000
}
- event_type: confluentKafkaLagSample
commands:
# Requires the installation of Java and Confluent Platform to work
# https://docs.confluent.io/current/cloud/using/monitor-lag.html
- run: <pathToConfluentPlatform>/bin/kafka-consumer-groups --bootstrap-server "<bootstrapEndpoint>" --command-config client-ssl.properties --describe --group my-group
line_start: 6
split: horizontal
split_by: \s+
set_header:
[
GROUP,
TOPIC,
PARTITION,
CURRENT-OFFSET,
LOG-END-OFFSET,
LAG,
CONSUMER-ID,
HOST,
CLIENT-ID,
]