Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for complex counter values #260

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .vscode/config-schema.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,19 @@ properties:
type: string
labels:
$ref: "#/definitions/labels"
values:
type: array
items:
type: object
additionalProperties: false
required:
- name
- help
properties:
name:
type: string
help:
type: string
histograms:
type: array
items:
Expand Down
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -759,10 +759,13 @@ perf_event_array: <whether map is a BPF_MAP_TYPE_PERF_EVENT_ARRAY map: bool>
flush_interval: <how often should we flush metrics from the perf_event_array: time.Duration>
labels:
[ - label ]
[ values: { name: "...", help: "..."}]
```

An example of `perf_map` can be found [here](examples/oomkill.yaml).

An example of `values` can be found [here](examples/complex-value.yaml).

#### `histogram`

See [Histograms](#histograms) section for more details.
Expand Down
7 changes: 7 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ type Counter struct {
PerfEventArray bool `yaml:"perf_event_array"`
FlushInterval time.Duration `yaml:"flush_interval"`
Labels []Label `yaml:"labels"`
Values []Value `yaml:"values"`
}

// Histogram is a metric defining prometheus histogram
Expand Down Expand Up @@ -75,6 +76,12 @@ type Decoder struct {
AllowUnknown bool `yaml:"allow_unknown"`
}

// Value describes a metric in when it's split across multiple u64
type Value struct {
Name string `yaml:"name"`
Help string `yaml:"help"`
}

// HistogramBucketType is an enum to define how to interpret histogram
type HistogramBucketType string

Expand Down
83 changes: 83 additions & 0 deletions examples/complex-value.bpf.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
#include <vmlinux.h>
#include <bpf/bpf_core_read.h>
#include <bpf/bpf_tracing.h>

#define MKDEV(ma, mi) ((mi & 0xff) | (ma << 8) | ((mi & ~0xff) << 12))

/**
* commit d152c682f03c ("block: add an explicit ->disk backpointer to the
* request_queue") and commit f3fa33acca9f ("block: remove the ->rq_disk
* field in struct request") make some changes to `struct request` and
* `struct request_queue`. Now, to get the `struct gendisk *` field in a CO-RE
* way, we need both `struct request` and `struct request_queue`.
* see:
* https://github.com/torvalds/linux/commit/d152c682f03c
* https://github.com/torvalds/linux/commit/f3fa33acca9f
*/
struct request_queue___x {
struct gendisk *disk;
} __attribute__((preserve_access_index));

struct request___x {
struct request_queue___x *q;
struct gendisk *rq_disk;
} __attribute__((preserve_access_index));

struct key_t {
u32 dev;
};

struct value_t {
u64 count;
u64 bytes;
};

struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 1024);
__type(key, struct key_t);
__type(value, struct value_t);
} block_rq_completions SEC(".maps");

static __always_inline struct gendisk *get_disk(void *request)
{
struct request___x *r = request;

if (bpf_core_field_exists(r->rq_disk))
return r->rq_disk;
return r->q->disk;
}

static struct value_t *get_value(void *map, struct key_t *key)
{
struct value_t *value = bpf_map_lookup_elem(map, key);
if (!value) {
struct value_t zero = { .count = 0, .bytes = 0 };
bpf_map_update_elem(map, key, &zero, BPF_NOEXIST);
value = bpf_map_lookup_elem(map, key);
if (!value) {
return NULL;
}
}

return value;
}

SEC("tp_btf/block_rq_complete")
int BPF_PROG(block_rq_complete, struct request *rq, blk_status_t error, unsigned int nr_bytes)
{
struct gendisk *disk = get_disk(rq);
struct key_t key = { .dev = disk ? MKDEV(disk->major, disk->first_minor) : 0 };
struct value_t *value = get_value(&block_rq_completions, &key);

if (!value) {
return 0;
}

__sync_fetch_and_add(&value->count, 1);
__sync_fetch_and_add(&value->bytes, nr_bytes);

return 0;
}

char LICENSE[] SEC("license") = "GPL";
14 changes: 14 additions & 0 deletions examples/complex-value.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
metrics:
counters:
- name: block_rq_completions
help: Block request completions split into count and bytes
labels:
- name: device
size: 4
decoders:
- name: majorminor
values:
- name: block_rq_completions_total
help: Total number of block request completions
- name: block_rq_completed_bytes_total
help: Total number of bytes served by block requests completions
63 changes: 43 additions & 20 deletions exporter/exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,13 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
e.perfEventArrayCollectors = append(e.perfEventArrayCollectors, perfSink)
}

addDescs(cfg.Name, counter.Name, counter.Help, counter.Labels)
if counter.Values != nil {
for _, value := range counter.Values {
addDescs(cfg.Name, value.Name, value.Help, counter.Labels)
}
} else {
addDescs(cfg.Name, counter.Name, counter.Help, counter.Labels)
}
}

for _, histogram := range cfg.Metrics.Histograms {
Expand Down Expand Up @@ -477,10 +483,14 @@ func (e *Exporter) collectCounters(ch chan<- prometheus.Metric) {

aggregatedMapValues := aggregateMapValues(mapValues)

desc := e.descs[cfg.Name][counter.Name]

for _, metricValue := range aggregatedMapValues {
ch <- prometheus.MustNewConstMetric(desc, prometheus.CounterValue, metricValue.value, metricValue.labels...)
if counter.Values != nil {
for i, value := range counter.Values {
ch <- prometheus.MustNewConstMetric(e.descs[cfg.Name][value.Name], prometheus.CounterValue, metricValue.value[i], metricValue.labels...)
}
} else {
ch <- prometheus.MustNewConstMetric(e.descs[cfg.Name][counter.Name], prometheus.CounterValue, metricValue.value[0], metricValue.labels...)
}
}
}
}
Expand Down Expand Up @@ -531,7 +541,7 @@ func (e *Exporter) collectHistograms(ch chan<- prometheus.Metric) {
break
}

histograms[key].buckets[float64(leUint)] = uint64(metricValue.value)
histograms[key].buckets[float64(leUint)] = uint64(metricValue.value[0])
}

if skip {
Expand Down Expand Up @@ -673,29 +683,33 @@ func (e *Exporter) MapsHandler(w http.ResponseWriter, r *http.Request) {
}

func validateMaps(module *libbpfgo.Module, cfg config.Config) error {
maps := []string{}
sizes := map[string]int{}

for _, counter := range cfg.Metrics.Counters {
if counter.Name != "" && !counter.PerfEventArray {
maps = append(maps, counter.Name)
if counter.Values != nil {
sizes[counter.Name] = len(counter.Values) * 8
} else {
sizes[counter.Name] = 8
}
}
}

for _, histogram := range cfg.Metrics.Histograms {
if histogram.Name != "" {
maps = append(maps, histogram.Name)
sizes[histogram.Name] = 8
}
}

for _, name := range maps {
for name, expected := range sizes {
m, err := module.GetMap(name)
if err != nil {
return fmt.Errorf("failed to get map %q: %w", name, err)
}

valueSize := m.ValueSize()
if valueSize != 8 {
return fmt.Errorf("value size for map %q is not expected 8 bytes (u64), it is %d bytes", name, valueSize)
if valueSize != expected {
return fmt.Errorf("value size for map %q is not expected %d bytes (8 bytes per u64 value), it is %d bytes", name, expected, valueSize)
}
}

Expand All @@ -721,7 +735,9 @@ func aggregateMapValues(values []metricValue) []aggregatedMetricValue {
value: value.value,
}
} else {
existing.value += value.value
for i := range existing.value {
existing.value[i] += value.value[i]
}
}
}

Expand Down Expand Up @@ -785,17 +801,18 @@ func readMapValues(m *libbpfgo.BPFMap, labels []config.Label) ([]metricValue, er
return metricValues, nil
}

func mapValue(m *libbpfgo.BPFMap, key []byte) (float64, error) {
func mapValue(m *libbpfgo.BPFMap, key []byte) ([]float64, error) {
v, err := m.GetValue(unsafe.Pointer(&key[0]))
if err != nil {
return 0.0, err
return []float64{0.0}, err
}

return decodeValue(v), nil
}

func mapValuePerCPU(m *libbpfgo.BPFMap, key []byte) ([]float64, error) {
values := []float64{}
func mapValuePerCPU(m *libbpfgo.BPFMap, key []byte) ([][]float64, error) {
values := [][]float64{}

size := m.ValueSize()

value, err := m.GetValue(unsafe.Pointer(&key[0]))
Expand All @@ -811,8 +828,14 @@ func mapValuePerCPU(m *libbpfgo.BPFMap, key []byte) ([]float64, error) {
}

// Assuming counter's value type is always u64
func decodeValue(value []byte) float64 {
return float64(util.GetHostByteOrder().Uint64(value))
func decodeValue(value []byte) []float64 {
values := make([]float64, len(value)/8)

for i := range values {
values[i] = float64(util.GetHostByteOrder().Uint64(value[i*8:]))
}

return values
}

// metricValue is a row in a kernel map
Expand All @@ -822,13 +845,13 @@ type metricValue struct {
// labels are decoded from the raw key
labels []string
// value is the kernel map value
value float64
value []float64
}

// aggregatedMetricValue is a value after aggregation of equal label sets
type aggregatedMetricValue struct {
// labels are decoded from the raw key
labels []string
// value is the kernel map value
value float64
value []float64
}
12 changes: 6 additions & 6 deletions exporter/exporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,32 +10,32 @@ func TestAggregatedMetricValues(t *testing.T) {
values := []metricValue{
{
labels: []string{"foo"},
value: 8,
value: []float64{8},
},
{
labels: []string{"bar"},
value: 1,
value: []float64{1},
},
{
labels: []string{"foo"},
value: 3,
value: []float64{3},
},
}

aggregated := aggregateMapValues(values)

sort.Slice(aggregated, func(i, j int) bool {
return aggregated[i].value > aggregated[j].value
return aggregated[i].value[0] > aggregated[j].value[0]
})

expected := []aggregatedMetricValue{
{
labels: []string{"foo"},
value: 11,
value: []float64{11},
},
{
labels: []string{"bar"},
value: 1,
value: []float64{1},
},
}

Expand Down