From fc0414a684e2c4c3fcf7d40d1e2f417e3e3bb9a0 Mon Sep 17 00:00:00 2001 From: Hugo Labrador Date: Mon, 19 Aug 2024 14:58:45 +0200 Subject: [PATCH] add quotas exporter and minor refactoring of package module (#37) --- .gitlab-ci.yml | 15 +++- collector/fs.go | 2 +- collector/fsck.go | 2 +- collector/fusex.go | 2 +- collector/group.go | 2 +- collector/inspector.go | 2 +- collector/io.go | 2 +- collector/node.go | 2 +- collector/ns.go | 2 +- collector/quotas.go | 164 +++++++++++++++++++++++++++++++++++++++++ collector/recycle.go | 2 +- collector/space.go | 2 +- collector/who.go | 2 +- eos_exporter.go | 3 +- eos_exporter.spec | 4 +- eosclient/eos.go | 87 ++++++++++++++++++---- go.mod | 5 +- 17 files changed, 264 insertions(+), 36 deletions(-) create mode 100644 collector/quotas.go diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c10f8e..1b5fd2f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,10 +17,16 @@ stages: el-7: image: gitlab-registry.cern.ch/linuxsupport/cc7-base - variables: - PKG_MGR: yum - extends: .build_eos-exporter-template - + stage: build:rpm + script: + - yum-config-manager --save --setopt=epel.baseurl=https://linuxsoft.cern.ch/internal/archive/epel/7/x86_64/ + - yum install -y sudo sssd-client createrepo tree + - tree + - yum install -y golang rpm-devel rpm-build make rpmdevtools sudo createrepo git + - make rpm + - mkdir -p ${CI_JOB_NAME}_artifacts + - cp --recursive rpmbuild/SRPMS/ ${CI_JOB_NAME}_artifacts + - cp --recursive rpmbuild/RPMS/ ${CI_JOB_NAME}_artifacts el-8: image: gitlab-registry.cern.ch/linuxsupport/alma8-base variables: @@ -37,6 +43,7 @@ rpm_artifacts: stage: publish image: gitlab-registry.cern.ch/linuxsupport/cc7-base script: + - yum-config-manager --save --setopt=epel.baseurl=https://linuxsoft.cern.ch/internal/archive/epel/7/x86_64/ - yum install -y sudo sssd-client createrepo tree - tree - sudo -u stci -H ./gitlab-ci/publish_rpms.sh diff --git a/collector/fs.go b/collector/fs.go index 773b84a..40dba6f 100644 --- a/collector/fs.go +++ b/collector/fs.go @@ -10,7 +10,7 @@ import ( "strings" "github.com/prometheus/client_golang/prometheus" - "gitlab.cern.ch/rvalverd/eos_exporter/eosclient" + "github.com/cern-eos/eos_exporter/eosclient" ) type FSCollector struct { diff --git a/collector/fsck.go b/collector/fsck.go index 755621d..5ebb2e2 100644 --- a/collector/fsck.go +++ b/collector/fsck.go @@ -6,7 +6,7 @@ import ( "strconv" "github.com/prometheus/client_golang/prometheus" - "gitlab.cern.ch/rvalverd/eos_exporter/eosclient" + "github.com/cern-eos/eos_exporter/eosclient" ) type FsckCollector struct { diff --git a/collector/fusex.go b/collector/fusex.go index 22ebd94..3410e72 100644 --- a/collector/fusex.go +++ b/collector/fusex.go @@ -5,7 +5,7 @@ import ( "log" "github.com/prometheus/client_golang/prometheus" - "gitlab.cern.ch/rvalverd/eos_exporter/eosclient" + "github.com/cern-eos/eos_exporter/eosclient" ) // sample line diff --git a/collector/group.go b/collector/group.go index ee71625..bcdca98 100644 --- a/collector/group.go +++ b/collector/group.go @@ -6,7 +6,7 @@ import ( "strconv" "github.com/prometheus/client_golang/prometheus" - "gitlab.cern.ch/rvalverd/eos_exporter/eosclient" + "github.com/cern-eos/eos_exporter/eosclient" ) type GroupCollector struct { diff --git a/collector/inspector.go b/collector/inspector.go index 4a01e8e..6109352 100644 --- a/collector/inspector.go +++ b/collector/inspector.go @@ -6,7 +6,7 @@ import ( "strconv" "github.com/prometheus/client_golang/prometheus" - "gitlab.cern.ch/rvalverd/eos_exporter/eosclient" + "github.com/cern-eos/eos_exporter/eosclient" ) type InspectorLayoutCollector struct { diff --git a/collector/io.go b/collector/io.go index 5b44520..6d4132c 100644 --- a/collector/io.go +++ b/collector/io.go @@ -7,7 +7,7 @@ import ( "strconv" "github.com/prometheus/client_golang/prometheus" - "gitlab.cern.ch/rvalverd/eos_exporter/eosclient" + "github.com/cern-eos/eos_exporter/eosclient" ) type IOInfoCollector struct { diff --git a/collector/node.go b/collector/node.go index be13142..850c9b0 100644 --- a/collector/node.go +++ b/collector/node.go @@ -6,7 +6,7 @@ import ( "strconv" "github.com/prometheus/client_golang/prometheus" - "gitlab.cern.ch/rvalverd/eos_exporter/eosclient" + "github.com/cern-eos/eos_exporter/eosclient" ) const ( diff --git a/collector/ns.go b/collector/ns.go index 9fb93e1..ef64074 100644 --- a/collector/ns.go +++ b/collector/ns.go @@ -6,7 +6,7 @@ import ( "strconv" "github.com/prometheus/client_golang/prometheus" - "gitlab.cern.ch/rvalverd/eos_exporter/eosclient" + "github.com/cern-eos/eos_exporter/eosclient" //"os" //"bufio" diff --git a/collector/quotas.go b/collector/quotas.go new file mode 100644 index 0000000..4b63085 --- /dev/null +++ b/collector/quotas.go @@ -0,0 +1,164 @@ +package collector + +import ( + "context" + "fmt" + + // "time" + "log" + + "github.com/cern-eos/eos_exporter/eosclient" + "github.com/prometheus/client_golang/prometheus" +) + +type QuotasCollector struct { + *CollectorOpts + QuotaUsedBytes *prometheus.GaugeVec + QuotaMaxBytes *prometheus.GaugeVec + QuotaUsedLogicalBytes *prometheus.GaugeVec + QuotaMaxLogicalBytes *prometheus.GaugeVec + QuotaUsedFiles *prometheus.GaugeVec + QuotaMaxFiles *prometheus.GaugeVec +} + +// NewQuotasCollector creates an cluster of the QuotasCollector +func NewQuotasCollector(opts *CollectorOpts) *QuotasCollector { + cluster := opts.Cluster + labels := make(prometheus.Labels) + labels["cluster"] = cluster + + namespace := "eos" + + return &QuotasCollector{ + //file: f, + CollectorOpts: opts, + QuotaUsedBytes: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "quota_used_bytes", + Help: "Quota used bytes", + ConstLabels: labels, + }, + []string{"uid", "gid", "space"}, + ), + QuotaMaxBytes: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "quota_max_bytes", + Help: "Quota max bytes", + ConstLabels: labels, + }, + []string{"uid", "gid", "space"}, + ), + QuotaUsedLogicalBytes: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "quota_used_logical_bytes", + Help: "Quota used logical bytes", + ConstLabels: labels, + }, + []string{"uid", "gid", "space"}, + ), + QuotaMaxLogicalBytes: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "quota_max_logical_bytes", + Help: "Quota maxlogical bytes", + ConstLabels: labels, + }, + []string{"uid", "gid", "space"}, + ), + QuotaUsedFiles: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "quota_used_files", + Help: "Quota used files", + ConstLabels: labels, + }, + []string{"uid", "gid", "space"}, + ), + QuotaMaxFiles: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "quota_max_files", + Help: "Quota max files", + ConstLabels: labels, + }, + []string{"uid", "gid", "space"}, + ), + } +} + +func (o *QuotasCollector) collectorList() []prometheus.Collector { + return []prometheus.Collector{ + o.QuotaUsedBytes, + o.QuotaMaxBytes, + o.QuotaUsedFiles, + o.QuotaMaxFiles, + o.QuotaUsedLogicalBytes, + o.QuotaMaxLogicalBytes, + } +} + +func (o *QuotasCollector) collectQuotaDF() error { + ins := getEOSInstance() + url := "root://" + ins + opt := &eosclient.Options{URL: url, Timeout: o.Timeout} + client, err := eosclient.New(opt) + if err != nil { + panic(err) + } + + quotas, err := client.Quotas(context.Background(), "root") + if err != nil { + return err + } + + // a GaugeVector keeps its state for any combination of labels until the process is restarted. + // For metrics obtained from systems like EOS that do not produce a complete set of metrics (only the active metrics) + // then we risk to expose these metrics forever until the next process restart. + // To workaround this, we reset the gauge vectors when collecting metrics. + + // output is like this: + // quota=node uid=9218 space=/eos/user/ usedbytes=158090138 usedlogicalbytes=79045069 usedfiles=1546 maxbytes=0 maxlogicalbytes=0 maxfiles=0 percentageusedbytes=100.00 statusbytes=ignored statusfiles=ignored + + o.QuotaUsedBytes.Reset() + o.QuotaMaxBytes.Reset() + o.QuotaUsedLogicalBytes.Reset() + o.QuotaMaxLogicalBytes.Reset() + o.QuotaUsedFiles.Reset() + o.QuotaMaxFiles.Reset() + + for _, q := range quotas { + o.QuotaUsedBytes.WithLabelValues(q.Uid, q.Gid, q.Space).Set(float64(q.UsedBytes)) + o.QuotaMaxBytes.WithLabelValues(q.Uid, q.Gid, q.Space).Set(float64(q.MaxBytes)) + o.QuotaUsedLogicalBytes.WithLabelValues(q.Uid, q.Gid, q.Space).Set(float64(q.UsedLogicalBytes)) + o.QuotaMaxLogicalBytes.WithLabelValues(q.Uid, q.Gid, q.Space).Set(float64(q.MaxLogicalBytes)) + o.QuotaUsedFiles.WithLabelValues(q.Uid, q.Gid, q.Space).Set(float64(q.UsedFiles)) + o.QuotaMaxFiles.WithLabelValues(q.Uid, q.Gid, q.Space).Set(float64(q.MaxFiles)) + } + + return nil + +} // collectQuotaDF() + +// Describe sends the descriptors of each SpaceCollector related metrics we have defined +func (o *QuotasCollector) Describe(ch chan<- *prometheus.Desc) { + for _, metric := range o.collectorList() { + fmt.Print(metric) + metric.Describe(ch) + } +} + +// Collect sends all the collected metrics to the provided prometheus channel. +func (o *QuotasCollector) Collect(ch chan<- prometheus.Metric) { + + if err := o.collectQuotaDF(); err != nil { + log.Println("failed collecting quota metrics:", err) + return + } + + for _, collector := range o.collectorList() { + collector.Collect(ch) + } +} diff --git a/collector/recycle.go b/collector/recycle.go index 380a6d6..c377834 100644 --- a/collector/recycle.go +++ b/collector/recycle.go @@ -7,7 +7,7 @@ import ( "strconv" "github.com/prometheus/client_golang/prometheus" - "gitlab.cern.ch/rvalverd/eos_exporter/eosclient" + "github.com/cern-eos/eos_exporter/eosclient" //"os" //"bufio" //"strings" diff --git a/collector/space.go b/collector/space.go index ae21e9f..836defa 100644 --- a/collector/space.go +++ b/collector/space.go @@ -6,7 +6,7 @@ import ( "strconv" "github.com/prometheus/client_golang/prometheus" - "gitlab.cern.ch/rvalverd/eos_exporter/eosclient" + "github.com/cern-eos/eos_exporter/eosclient" ) /* diff --git a/collector/who.go b/collector/who.go index c365177..505d740 100644 --- a/collector/who.go +++ b/collector/who.go @@ -10,7 +10,7 @@ import ( "strings" "github.com/prometheus/client_golang/prometheus" - "gitlab.cern.ch/rvalverd/eos_exporter/eosclient" + "github.com/cern-eos/eos_exporter/eosclient" ) // eos who -a -m provides 3 clusters of information diff --git a/eos_exporter.go b/eos_exporter.go index 39ad4f9..087813a 100644 --- a/eos_exporter.go +++ b/eos_exporter.go @@ -31,7 +31,7 @@ import ( // "github.com/prometheus/common/log" /* // For enabling profile mode in Go "github.com/pkg/profile"*/ - "gitlab.cern.ch/rvalverd/eos_exporter/collector" + "github.com/cern-eos/eos_exporter/collector" _ "embed" ) @@ -72,6 +72,7 @@ func NewEOSExporter(opts *collector.CollectorOpts) *EOSExporter { collector.NewNSBatchCollector(opts), // eos namespace potential batch overload information collector.NewRecycleCollector(opts), // eos recycle bin information collector.NewWhoCollector(opts), // eos who information + collector.NewQuotasCollector(opts), // eos quota information collector.NewFsckCollector(opts), // eos fsck information collector.NewFusexCollector(opts), // eos fusex information collector.NewInspectorLayoutCollector(opts), // eos inspector layout information diff --git a/eos_exporter.spec b/eos_exporter.spec index c01090b..98d40d9 100644 --- a/eos_exporter.spec +++ b/eos_exporter.spec @@ -1,7 +1,7 @@ # # eos_exporter spec file # -%define version 0.1.8 +%define version 0.1.9 Name: eos_exporter Summary: The Prometheus EOS exporter exposes EOS metrics. @@ -58,6 +58,8 @@ rm -rf %buildroot/ %systemd_preun %{name}.service %changelog +* Mon Aug 19 2024 Gianmaria Del Monte 0.1.9-1 +- Add quotas exporter * Mon Jun 3 2024 Cedric Caffy 0.1.8-1 - Adds new eos inspector metrics as access time volume and files, birthtime and cost per group. - Adds qclient metrics diff --git a/eosclient/eos.go b/eosclient/eos.go index 38560c0..48ead10 100644 --- a/eosclient/eos.go +++ b/eosclient/eos.go @@ -428,21 +428,6 @@ func (c *Client) ListFS(ctx context.Context, username string) ([]*FSInfo, error) return c.parseFSsInfo(stdout) } -func (c *Client) getEosMGMVersion(ctx context.Context) (string, error) { - out, _, err := c.execute(exec.CommandContext(ctx, "/usr/bin/eos", "version")) - if err != nil { - return "", err - } - stdo_mgm := strings.Split(out, "\n") - for _, l := range stdo_mgm { - if strings.HasPrefix(l, "EOS_SERVER_VERSION=") { - s := strings.Split(l, " ") - return strings.Split(s[0], "EOS_SERVER_VERSION=")[1], nil - } - } - return "", errors.New("version not found") -} - // List the activity of different users in the instance func (c *Client) ListNS(ctx context.Context) ([]*NSInfo, []*NSActivityInfo, []*NSBatchInfo, error) { // eos ns stat, without -a will exclude batch users info (this adds to much latency in the instance where the exporter is deployed) @@ -883,7 +868,7 @@ func (c *Client) parseNSsInfo(raw string, raw_batch string, ctx context.Context) // Check that user has stall operation and is actually that operation to be exposed, plus is legitimate user if UidLetter(kv["uid"]) && onlyUsers(kv["uid"], excl_uids) && batchMetrics[kv["uid"]+"-"+kv["cmd"]] { var eos_instance string = "homecanary" - var level int = 0 + level := 0 ctx, cancel := c.getTimeout(ctx) defer cancel() @@ -1079,6 +1064,76 @@ func (c *Client) parseRecycleLineInfo(line string) (*RecycleInfo, error) { return rb, nil } +// Data struct // +type QuotaInfo struct { + Uid string + Gid string + Space string + UsedBytes int64 + MaxBytes int64 + UsedLogicalBytes int64 + MaxLogicalBytes int64 + UsedFiles int64 + MaxFiles int64 +} + +// Launch who command // +func (c *Client) Quotas(ctx context.Context, username string) ([]*QuotaInfo, error) { + unixUser, err := getUnixUser(username) + if err != nil { + return nil, err + } + ctxWt, cancel := c.getTimeout(ctx) + defer cancel() + + cmd := exec.CommandContext(ctxWt, "/usr/bin/eos", "-r", unixUser.Uid, unixUser.Gid, "quota", "ls", "-m") + stdout, _, err := c.execute(cmd) + if err != nil { + return nil, err + } + return c.parseQuotaInfo(stdout) +} + +// Parse information from recycle bin // +func (c *Client) parseQuotaInfo(raw string) ([]*QuotaInfo, error) { + whoInfo := []*QuotaInfo{} + rawLines := strings.Split(raw, "\n") + for _, rl := range rawLines { + if rl == "" { + continue + } + + kv := c.getMap(rl) + uid, okuid := kv["uid"] + gid, okgid := kv["gid"] + if !okuid && !okgid { + continue + } + + usedBytes, _ := strconv.ParseInt(kv["usedbytes"], 10, 64) + maxBytes, _ := strconv.ParseInt(kv["maxbytes"], 10, 64) + usedLogicalBytes, _ := strconv.ParseInt(kv["usedlogicalbytes"], 10, 64) + maxLogicalBytes, _ := strconv.ParseInt(kv["maxlogicalbytes"], 10, 64) + usedFiles, _ := strconv.ParseInt(kv["usedfiles"], 10, 64) + maxFiles, _ := strconv.ParseInt(kv["maxfiles"], 10, 64) + + who := &QuotaInfo{ + Uid: uid, + Gid: gid, + Space: kv["space"], + UsedBytes: usedBytes, + MaxBytes: maxBytes, + UsedLogicalBytes: usedLogicalBytes, + MaxLogicalBytes: maxLogicalBytes, + UsedFiles: usedFiles, + MaxFiles: maxFiles, + } + + whoInfo = append(whoInfo, who) + } + return whoInfo, nil +} + // ----------------------------------------// // EOS WHO INFORMATION // // ----------------------------------------// diff --git a/go.mod b/go.mod index de92dec..da8daaf 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module gitlab.cern.ch/rvalverd/eos_exporter +module github.com/cern-eos/eos_exporter go 1.18 @@ -7,14 +7,13 @@ require ( go.uber.org/zap v1.21.0 ) -require github.com/prometheus/common v0.34.0 // indirect - require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.34.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect