diff --git a/README.md b/README.md index f8f980f..020b008 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,21 @@ problematic the collection of logs can be configured as follows: To disable log collection you can set: ```yaml -collectlogs: false +hosts: + default: + username: username + password: password + collectlogs: false +``` + +or for a group: + +```yaml +groups: + group1: + username: username + password: password + collectlogs: false ``` 2) via the `collectlogs` query parameter @@ -60,6 +74,47 @@ curl '127.0.0.1:9123/redfish?target=10.10.12.23&collectlogs=true' The `collectlogs` query parameter can be included in Prometheus config. + +### Log collection count + +To restrict the number of logs collected for all log services: + +``` +hosts: + default: + username: username + password: password + collectlogs: true + logcount: + DEFAULT: 10 +``` + +For a particular log service, `Sel`: + +``` +hosts: + default: + username: username + password: password + collectlogs: true + logcount: + Sel: 20 +``` + +Collect all logs for one service, `Sel` , but limit others: + +``` +hosts: + default: + username: username + password: password + collectlogs: true + logcount: + DEFAULT: 10 + Sel: -1 +``` + + ## Building To build the redfish_exporter executable run the command: diff --git a/collector/chassis_collector.go b/collector/chassis_collector.go index ad49dc4..cb6f633 100755 --- a/collector/chassis_collector.go +++ b/collector/chassis_collector.go @@ -7,8 +7,8 @@ import ( "sync" "github.com/apex/log" + "github.com/jenningsloy318/redfish_exporter/common" "github.com/prometheus/client_golang/prometheus" - "github.com/stmcginnis/gofish" "github.com/stmcginnis/gofish/redfish" ) @@ -33,9 +33,8 @@ var ( // ChassisCollector implements the prometheus.Collector. type ChassisCollector struct { - redfishClient *gofish.APIClient + Ctx *common.CollectionContext metrics map[string]Metric - collectLogs bool collectorScrapeStatus *prometheus.GaugeVec Log *log.Entry } @@ -91,13 +90,12 @@ func createChassisMetricMap() map[string]Metric { } // NewChassisCollector returns a collector that collecting chassis statistics -func NewChassisCollector(redfishClient *gofish.APIClient, collectLogs bool, logger *log.Entry) *ChassisCollector { +func NewChassisCollector(ctx *common.CollectionContext, logger *log.Entry) *ChassisCollector { // get service from redfish client return &ChassisCollector{ - redfishClient: redfishClient, - metrics: chassisMetrics, - collectLogs: collectLogs, + metrics: chassisMetrics, + Ctx: ctx, Log: logger.WithFields(log.Fields{ "collector": "ChassisCollector", }), @@ -124,8 +122,8 @@ func (c *ChassisCollector) Describe(ch chan<- *prometheus.Desc) { // Collect implemented prometheus.Collector func (c *ChassisCollector) Collect(ch chan<- prometheus.Metric) { collectorLogContext := c.Log - collectLogs := c.collectLogs - service := c.redfishClient.Service + collectLogs := c.Ctx.CollectLogs + service := c.Ctx.RedfishClient.Service // get a list of chassis from service if chassises, err := service.Chassis(); err != nil { @@ -252,7 +250,7 @@ func (c *ChassisCollector) Collect(ch chan<- prometheus.Metric) { wg6.Add(len(logServices)) for _, logService := range logServices { - if err = parseLogService(ch, chassisMetrics, ChassisSubsystem, chassisID, logService, wg6); err != nil { + if err = parseLogService(ch, chassisMetrics, c.Ctx, chassisLogContext, ChassisSubsystem, chassisID, logService, wg6); err != nil { chassisLogContext.WithField("operation", "chassis.LogServices()").WithError(err).Error("error getting log entries from log service") } } diff --git a/collector/common_collector.go b/collector/common_collector.go index 7d83328..47c4f21 100644 --- a/collector/common_collector.go +++ b/collector/common_collector.go @@ -4,7 +4,10 @@ import ( "fmt" "sync" + "github.com/apex/log" + redfish_common "github.com/jenningsloy318/redfish_exporter/common" "github.com/prometheus/client_golang/prometheus" + "github.com/stmcginnis/gofish/common" "github.com/stmcginnis/gofish/redfish" ) @@ -33,7 +36,7 @@ func addToMetricMap(metricMap map[string]Metric, subsystem, name, help string, v } } -func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, subsystem, collectorID string, logService *redfish.LogService, wg *sync.WaitGroup) (err error) { +func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, ctx *redfish_common.CollectionContext, logger *log.Entry, subsystem, collectorID string, logService *redfish.LogService, wg *sync.WaitGroup) (err error) { defer wg.Done() logServiceName := logService.Name logServiceID := logService.ID @@ -50,15 +53,40 @@ func parseLogService(ch chan<- prometheus.Metric, metrics map[string]Metric, sub if logServiceHealthStateValue, ok := parseCommonStatusHealth(logServiceHealthState); ok { ch <- prometheus.MustNewConstMetric(metrics[fmt.Sprintf("%s_%s", subsystem, "log_service_health_state")].desc, prometheus.GaugeValue, logServiceHealthStateValue, logServiceLabelValues...) } + var ( + logEntries []*redfish.LogEntry + ) + logCount, ok := ctx.LogCount[logServiceID] + if !ok { + logCount, ok = ctx.LogCount["DEFAULT"] + if !ok { + logCount = -1 + } + } + + logger.WithField("operation", "parseLogService").Info(fmt.Sprintf("logServiceID=%s, logcount=%d", logServiceID, logCount)) - logEntries, err := logService.Entries() + if logCount > 0 { + logEntries, err = logService.FilteredEntries(common.WithTop(logCount)) + } else { + logEntries, err = logService.Entries() + } if err != nil { return } wg2 := &sync.WaitGroup{} wg2.Add(len(logEntries)) + processed := make(map[string]bool) for _, logEntry := range logEntries { - go parseLogEntry(ch, metrics[fmt.Sprintf("%s_%s", subsystem, "log_entry_severity_state")].desc, collectorID, logServiceName, logServiceID, logEntry, wg2) + _, exists := processed[logEntry.MessageID] + if exists { + wg2.Done() + continue + } else { + go parseLogEntry(ch, metrics[fmt.Sprintf("%s_%s", subsystem, "log_entry_severity_state")].desc, collectorID, logServiceName, logServiceID, logEntry, wg2) + } + + processed[logEntry.MessageID] = true } return } diff --git a/collector/manager_collector.go b/collector/manager_collector.go index e8591d8..7e40fc7 100755 --- a/collector/manager_collector.go +++ b/collector/manager_collector.go @@ -5,8 +5,8 @@ import ( "sync" "github.com/apex/log" + "github.com/jenningsloy318/redfish_exporter/common" "github.com/prometheus/client_golang/prometheus" - "github.com/stmcginnis/gofish" ) // ManagerSubmanager is the manager subsystem @@ -22,9 +22,8 @@ var ( // ManagerCollector implements the prometheus.Collector. type ManagerCollector struct { - redfishClient *gofish.APIClient + Ctx *common.CollectionContext metrics map[string]Metric - collectLogs bool collectorScrapeStatus *prometheus.GaugeVec Log *log.Entry } @@ -43,11 +42,10 @@ func createManagerMetricMap() map[string]Metric { } // NewManagerCollector returns a collector that collecting memory statistics -func NewManagerCollector(redfishClient *gofish.APIClient, collectLogs bool, logger *log.Entry) *ManagerCollector { +func NewManagerCollector(ctx *common.CollectionContext, logger *log.Entry) *ManagerCollector { return &ManagerCollector{ - redfishClient: redfishClient, - metrics: managerMetrics, - collectLogs: collectLogs, + metrics: managerMetrics, + Ctx: ctx, Log: logger.WithFields(log.Fields{ "collector": "ManagerCollector", }), @@ -74,9 +72,9 @@ func (m *ManagerCollector) Describe(ch chan<- *prometheus.Desc) { // Collect implemented prometheus.Collector func (m *ManagerCollector) Collect(ch chan<- prometheus.Metric) { collectorLogContext := m.Log - collectLogs := m.collectLogs + collectLogs := m.Ctx.CollectLogs //get service - service := m.redfishClient.Service + service := m.Ctx.RedfishClient.Service // get a list of managers from service if managers, err := service.Managers(); err != nil { @@ -118,7 +116,7 @@ func (m *ManagerCollector) Collect(ch chan<- prometheus.Metric) { wg.Add(len(logServices)) for _, logService := range logServices { - if err = parseLogService(ch, managerMetrics, ManagerSubmanager, ManagerID, logService, wg); err != nil { + if err = parseLogService(ch, managerMetrics, m.Ctx, managerLogContext, ManagerSubmanager, ManagerID, logService, wg); err != nil { managerLogContext.WithField("operation", "manager.LogServices()").WithError(err).Error("error getting log entries from log service") } } diff --git a/collector/redfish_collector.go b/collector/redfish_collector.go index 95b1dc5..30da33d 100755 --- a/collector/redfish_collector.go +++ b/collector/redfish_collector.go @@ -2,11 +2,11 @@ package collector import ( "bytes" - "fmt" "sync" "time" - "github.com/apex/log" + alog "github.com/apex/log" + "github.com/jenningsloy318/redfish_exporter/common" "github.com/prometheus/client_golang/prometheus" gofish "github.com/stmcginnis/gofish" gofishcommon "github.com/stmcginnis/gofish/common" @@ -40,22 +40,16 @@ type RedfishCollector struct { } // NewRedfishCollector return RedfishCollector -func NewRedfishCollector(host string, username string, password string, collectLogs bool, logger *log.Entry) *RedfishCollector { +func NewRedfishCollector(ctx *common.CollectionContext, logger *alog.Entry) *RedfishCollector { var collectors map[string]prometheus.Collector - collectorLogCtx := logger - redfishClient, err := newRedfishClient(host, username, password) - if err != nil { - collectorLogCtx.WithError(err).Error("error creating redfish client") - } else { - chassisCollector := NewChassisCollector(redfishClient, collectLogs, collectorLogCtx) - systemCollector := NewSystemCollector(redfishClient, collectLogs, collectorLogCtx) - managerCollector := NewManagerCollector(redfishClient, collectLogs, collectorLogCtx) + chassisCollector := NewChassisCollector(ctx, logger) + systemCollector := NewSystemCollector(ctx, logger) + managerCollector := NewManagerCollector(ctx, logger) - collectors = map[string]prometheus.Collector{"chassis": chassisCollector, "system": systemCollector, "manager": managerCollector} - } + collectors = map[string]prometheus.Collector{"chassis": chassisCollector, "system": systemCollector, "manager": managerCollector} return &RedfishCollector{ - redfishClient: redfishClient, + redfishClient: ctx.RedfishClient, collectors: collectors, redfishUp: prometheus.NewGauge( prometheus.GaugeOpts{ @@ -101,23 +95,6 @@ func (r *RedfishCollector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(totalScrapeDurationDesc, prometheus.GaugeValue, time.Since(scrapeTime).Seconds()) } -func newRedfishClient(host string, username string, password string) (*gofish.APIClient, error) { - - url := fmt.Sprintf("https://%s", host) - - config := gofish.ClientConfig{ - Endpoint: url, - Username: username, - Password: password, - Insecure: true, - } - redfishClient, err := gofish.Connect(config) - if err != nil { - return nil, err - } - return redfishClient, nil -} - func parseCommonStatusHealth(status gofishcommon.Health) (float64, bool) { if bytes.Equal([]byte(status), []byte("OK")) { return float64(1), true diff --git a/collector/system_collector.go b/collector/system_collector.go index 9ed8a4f..dd869ee 100755 --- a/collector/system_collector.go +++ b/collector/system_collector.go @@ -5,8 +5,8 @@ import ( "sync" "github.com/apex/log" + "github.com/jenningsloy318/redfish_exporter/common" "github.com/prometheus/client_golang/prometheus" - "github.com/stmcginnis/gofish" "github.com/stmcginnis/gofish/redfish" ) @@ -33,9 +33,8 @@ var ( // SystemCollector implements the prometheus.Collector. type SystemCollector struct { - redfishClient *gofish.APIClient - metrics map[string]Metric - collectLogs bool + Ctx *common.CollectionContext + metrics map[string]Metric prometheus.Collector collectorScrapeStatus *prometheus.GaugeVec Log *log.Entry @@ -101,11 +100,10 @@ func createSystemMetricMap() map[string]Metric { } // NewSystemCollector returns a collector that collecting memory statistics -func NewSystemCollector(redfishClient *gofish.APIClient, collectLogs bool, logger *log.Entry) *SystemCollector { +func NewSystemCollector(ctx *common.CollectionContext, logger *log.Entry) *SystemCollector { return &SystemCollector{ - redfishClient: redfishClient, - metrics: systemMetrics, - collectLogs: collectLogs, + metrics: systemMetrics, + Ctx: ctx, Log: logger.WithFields(log.Fields{ "collector": "SystemCollector", }), @@ -131,9 +129,9 @@ func (s *SystemCollector) Describe(ch chan<- *prometheus.Desc) { // Collect implements prometheus.Collector. func (s *SystemCollector) Collect(ch chan<- prometheus.Metric) { collectorLogContext := s.Log - collectLogs := s.collectLogs + collectLogs := s.Ctx.CollectLogs //get service - service := s.redfishClient.Service + service := s.Ctx.RedfishClient.Service // get a list of systems from service if systems, err := service.Systems(); err != nil { @@ -379,7 +377,7 @@ func (s *SystemCollector) Collect(ch chan<- prometheus.Metric) { wg10.Add(len(logServices)) for _, logService := range logServices { - if err = parseLogService(ch, systemMetrics, SystemSubsystem, SystemID, logService, wg10); err != nil { + if err = parseLogService(ch, systemMetrics, s.Ctx, systemLogContext, SystemSubsystem, SystemID, logService, wg10); err != nil { systemLogContext.WithField("operation", "system.LogServices()").WithError(err).Error("error getting log entries from log service") } } diff --git a/config.go b/common/config.go similarity index 61% rename from config.go rename to common/config.go index 071b722..1b18fd2 100755 --- a/config.go +++ b/common/config.go @@ -1,4 +1,4 @@ -package main +package common import ( "fmt" @@ -9,10 +9,9 @@ import ( ) type Config struct { - Hosts map[string]HostConfig `yaml:"hosts"` - Groups map[string]HostConfig `yaml:"groups"` - Loglevel string `yaml:"loglevel"` - Collectlogs *bool `yaml:"collectlogs,omitempty"` + Hosts map[string]HostConfig `yaml:"hosts"` + Groups map[string]HostConfig `yaml:"groups"` + Loglevel string `yaml:"loglevel"` } type SafeConfig struct { @@ -21,8 +20,10 @@ type SafeConfig struct { } type HostConfig struct { - Username string `yaml:"username"` - Password string `yaml:"password"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Collectlogs bool `yaml:"collectlogs,omitempty"` + Logcount map[string]int `yaml:"logcount,omitempty"` } func (sc *SafeConfig) ReloadConfig(configFile string) error { @@ -48,14 +49,18 @@ func (sc *SafeConfig) HostConfigForTarget(target string) (*HostConfig, error) { defer sc.Unlock() if hostConfig, ok := sc.C.Hosts[target]; ok { return &HostConfig{ - Username: hostConfig.Username, - Password: hostConfig.Password, + Username: hostConfig.Username, + Password: hostConfig.Password, + Collectlogs: hostConfig.Collectlogs, + Logcount: hostConfig.Logcount, }, nil } if hostConfig, ok := sc.C.Hosts["default"]; ok { return &HostConfig{ - Username: hostConfig.Username, - Password: hostConfig.Password, + Username: hostConfig.Username, + Password: hostConfig.Password, + Collectlogs: hostConfig.Collectlogs, + Logcount: hostConfig.Logcount, }, nil } return &HostConfig{}, fmt.Errorf("no credentials found for target %s", target) @@ -67,7 +72,12 @@ func (sc *SafeConfig) HostConfigForGroup(group string) (*HostConfig, error) { sc.Lock() defer sc.Unlock() if hostConfig, ok := sc.C.Groups[group]; ok { - return &hostConfig, nil + return &HostConfig{ + Username: hostConfig.Username, + Password: hostConfig.Password, + Collectlogs: hostConfig.Collectlogs, + Logcount: hostConfig.Logcount, + }, nil } return &HostConfig{}, fmt.Errorf("no credentials found for group %s", group) } @@ -81,12 +91,3 @@ func (sc *SafeConfig) AppLogLevel() string { } return "info" } - -func (sc *SafeConfig) CollectLogs() bool { - sc.Lock() - defer sc.Unlock() - if sc.C.Collectlogs == nil { - return true - } - return *sc.C.Collectlogs -} diff --git a/common/context.go b/common/context.go new file mode 100644 index 0000000..2e28456 --- /dev/null +++ b/common/context.go @@ -0,0 +1,57 @@ +package common + +import ( + "fmt" + "net/http" + "strconv" + + alog "github.com/apex/log" + gofish "github.com/stmcginnis/gofish" +) + +type CollectionContext struct { + Request *http.Request + RedfishClient *gofish.APIClient + CollectLogs bool + LogCount map[string]int +} + +func NewCollectionContext(r *http.Request, target string, hostconfig *HostConfig, logger *alog.Entry) (*CollectionContext, error) { + client, err := newRedfishClient(target, hostconfig.Username, hostconfig.Password) + if err != nil { + logger.WithError(err).Error("error creating redfish client") + return nil, err + } + + logCount := hostconfig.Logcount + // TODO.. query parameter could logcount_=10 + + // Support optionally overriding collectlogs setting using a query parameter + collectLogs := hostconfig.Collectlogs + collectLogsOverride := r.URL.Query().Get("collectlogs") + if collectLogsOverride != "" { + if collectLogsQuery, err := strconv.ParseBool(collectLogsOverride); err != nil { + logger.WithError(err).Error("error parsing collectlogs query parameter as a boolean") + } else { + collectLogs = collectLogsQuery + } + } + return &CollectionContext{Request: r, RedfishClient: client, CollectLogs: collectLogs, LogCount: logCount}, nil +} + +func newRedfishClient(host string, username string, password string) (*gofish.APIClient, error) { + + url := fmt.Sprintf("https://%s", host) + + config := gofish.ClientConfig{ + Endpoint: url, + Username: username, + Password: password, + Insecure: true, + } + redfishClient, err := gofish.Connect(config) + if err != nil { + return nil, err + } + return redfishClient, nil +} diff --git a/main.go b/main.go index 2e571ac..2d4eab8 100755 --- a/main.go +++ b/main.go @@ -5,12 +5,12 @@ import ( "net/http" "os" "os/signal" - "strconv" "syscall" alog "github.com/apex/log" kitlog "github.com/go-kit/log" "github.com/jenningsloy318/redfish_exporter/collector" + "github.com/jenningsloy318/redfish_exporter/common" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/log" @@ -36,8 +36,8 @@ var ( "web.listen-address", "Address to listen on for web interface and telemetry.", ).Default(":9610").String() - sc = &SafeConfig{ - C: &Config{}, + sc = &common.SafeConfig{ + C: &common.Config{}, } reloadCh chan chan error ) @@ -95,7 +95,7 @@ func metricsHandler() http.HandlerFunc { targetLoggerCtx.Info("scraping target host") var ( - hostConfig *HostConfig + hostConfig *common.HostConfig err error ok bool group []string @@ -118,18 +118,12 @@ func metricsHandler() http.HandlerFunc { return } } - - // Support optionally overriding collectlogs setting using a query parameter - collectLogs := sc.CollectLogs() - collectLogsOverride := r.URL.Query().Get("collectlogs") - if collectLogsOverride != "" { - if collectLogs, err = strconv.ParseBool(collectLogsOverride); err != nil { - targetLoggerCtx.WithError(err).Error("error parsing collectlogs query parameter as a boolean") - return - } + collectionCtx, err := common.NewCollectionContext(r, target, hostConfig, targetLoggerCtx) + if err != nil { + targetLoggerCtx.WithError(err).Error("error creating collection context") + return } - - collector := collector.NewRedfishCollector(target, hostConfig.Username, hostConfig.Password, collectLogs, targetLoggerCtx) + collector := collector.NewRedfishCollector(collectionCtx, targetLoggerCtx) registry.MustRegister(collector) gatherers := prometheus.Gatherers{ prometheus.DefaultGatherer, @@ -147,7 +141,6 @@ func main() { kingpin.HelpFlag.Short('h') kingpin.Parse() kitlogger := kitlog.NewLogfmtLogger(os.Stderr) - configLoggerCtx := rootLoggerCtx.WithField("config", *configFile) configLoggerCtx.Info("starting app") // load config first time