Skip to content

Commit

Permalink
Refactor metrics
Browse files Browse the repository at this point in the history
The two things that gather metrics (the debug server and the periodic
Metron notifier) now take a map of metric keys to things that provide
them.

This is a prefactor before only enabling certain metrics when an image
plugin is not configured.

[#140552475]

Signed-off-by: Craig Furman <[email protected]>
  • Loading branch information
teddyking authored and Craig Furman committed Jun 6, 2017
1 parent 434c2eb commit 51dc3eb
Show file tree
Hide file tree
Showing 9 changed files with 129 additions and 452 deletions.
22 changes: 18 additions & 4 deletions guardiancmd/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -412,12 +412,26 @@ func (cmd *ServerCommand) Run(signals <-chan os.Signal, ready chan<- struct{}) e

metricsProvider := cmd.wireMetricsProvider(logger, cmd.Containers.Dir, cmd.Graph.Dir)

metronNotifier := cmd.wireMetronNotifier(logger, metricsProvider)
debugServerMetrics := map[string]func() int{
"numCPUS": metricsProvider.NumCPU,
"numGoRoutines": metricsProvider.NumGoroutine,
"loopDevices": metricsProvider.LoopDevices,
"backingStores": metricsProvider.BackingStores,
"depotDirs": metricsProvider.DepotDirs,
}

periodicMetronMetrics := map[string]func() int{
"LoopDevices": metricsProvider.LoopDevices,
"BackingStores": metricsProvider.BackingStores,
"DepotDirs": metricsProvider.DepotDirs,
}

metronNotifier := cmd.wireMetronNotifier(logger, periodicMetronMetrics)
metronNotifier.Start()

if cmd.Server.DebugBindIP != nil {
addr := fmt.Sprintf("%s:%d", cmd.Server.DebugBindIP.IP(), cmd.Server.DebugBindPort)
metrics.StartDebugServer(addr, reconfigurableSink, metricsProvider)
metrics.StartDebugServer(addr, reconfigurableSink, debugServerMetrics)
}

err = gardenServer.Start()
Expand Down Expand Up @@ -724,13 +738,13 @@ func (cmd *ServerCommand) wireContainerizer(log lager.Logger,
return rundmc.New(depot, template, runcrunner, &goci.BndlLoader{}, nstar, stopper, eventStore, stateStore, &preparerootfs.SymlinkRefusingFileCreator{})
}

func (cmd *ServerCommand) wireMetricsProvider(log lager.Logger, depotPath, graphRoot string) metrics.Metrics {
func (cmd *ServerCommand) wireMetricsProvider(log lager.Logger, depotPath, graphRoot string) *metrics.MetricsProvider {
var backingStoresPath string
if graphRoot != "" {
backingStoresPath = filepath.Join(graphRoot, "backing_stores")
}

return metrics.NewMetrics(log, backingStoresPath, depotPath)
return metrics.NewMetricsProvider(log, backingStoresPath, depotPath)
}

func (cmd *ServerCommand) wireMetronNotifier(log lager.Logger, metricsProvider metrics.Metrics) *metrics.PeriodicMetronNotifier {
Expand Down
27 changes: 8 additions & 19 deletions metrics/debug.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,25 +13,14 @@ import (
)

func StartDebugServer(address string, sink *lager.ReconfigurableSink, metrics Metrics) (ifrit.Process, error) {
expvar.Publish("numCPUS", expvar.Func(func() interface{} {
return metrics.NumCPU()
}))

expvar.Publish("numGoRoutines", expvar.Func(func() interface{} {
return metrics.NumGoroutine()
}))

expvar.Publish("loopDevices", expvar.Func(func() interface{} {
return metrics.LoopDevices()
}))

expvar.Publish("backingStores", expvar.Func(func() interface{} {
return metrics.BackingStores()
}))

expvar.Publish("depotDirs", expvar.Func(func() interface{} {
return metrics.DepotDirs()
}))
for key, metric := range metrics {
// https://github.com/golang/go/wiki/CommonMistakes
captureKey := key
captureMetric := metric
expvar.Publish(captureKey, expvar.Func(func() interface{} {
return captureMetric()
}))
}

server := http_server.New(address, handler(sink))
p := ifrit.Invoke(server)
Expand Down
25 changes: 9 additions & 16 deletions metrics/debug_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"os"

"code.cloudfoundry.org/guardian/metrics"
fakes "code.cloudfoundry.org/guardian/metrics/metricsfakes"
"code.cloudfoundry.org/lager"
"github.com/tedsuo/ifrit"

Expand All @@ -16,40 +15,34 @@ import (

var _ = Describe("Debug", func() {
var (
serverProc ifrit.Process
fakeMetrics *fakes.FakeMetrics
serverProc ifrit.Process
)

BeforeEach(func() {
var err error

fakeMetrics = new(fakes.FakeMetrics)
fakeMetrics.NumCPUReturns(11)
fakeMetrics.NumGoroutineReturns(888)
fakeMetrics.LoopDevicesReturns(33)
fakeMetrics.BackingStoresReturns(12)
fakeMetrics.DepotDirsReturns(3)
testMetrics := map[string]func() int{
"metric1": func() int { return 33 },
"metric2": func() int { return 12 },
}

sink := lager.NewReconfigurableSink(lager.NewWriterSink(GinkgoWriter, lager.DEBUG), lager.DEBUG)
serverProc, err = metrics.StartDebugServer("127.0.0.1:5123", sink, fakeMetrics)
serverProc, err = metrics.StartDebugServer("127.0.0.1:5123", sink, testMetrics)
Expect(err).ToNot(HaveOccurred())
})

AfterEach(func() {
serverProc.Signal(os.Kill)
})

It("should report the number of loop devices, backing store files and depotDirs", func() {
It("should report the configured metrics", func() {
resp, err := http.Get("http://127.0.0.1:5123/debug/vars")
Expect(err).ToNot(HaveOccurred())

defer resp.Body.Close()
Expect(resp.StatusCode).To(Equal(http.StatusOK))

Expect(expvar.Get("loopDevices").String()).To(Equal("33"))
Expect(expvar.Get("backingStores").String()).To(Equal("12"))
Expect(expvar.Get("depotDirs").String()).To(Equal("3"))
Expect(expvar.Get("numCPUS").String()).To(Equal("11"))
Expect(expvar.Get("numGoRoutines").String()).To(Equal("888"))
Expect(expvar.Get("metric1").String()).To(Equal("33"))
Expect(expvar.Get("metric2").String()).To(Equal("12"))
})
})
76 changes: 1 addition & 75 deletions metrics/metrics.go
Original file line number Diff line number Diff line change
@@ -1,77 +1,3 @@
package metrics

import (
"bytes"
"fmt"
"io/ioutil"
"os/exec"
"runtime"

"code.cloudfoundry.org/lager"
)

//go:generate counterfeiter . Metrics

type Metrics interface {
NumCPU() int
NumGoroutine() int
LoopDevices() int
BackingStores() int
DepotDirs() int
}

type metrics struct {
backingStoresPath string
depotPath string
logger lager.Logger
}

func NewMetrics(logger lager.Logger, backingStoresPath, depotPath string) Metrics {
return &metrics{
backingStoresPath: backingStoresPath,
depotPath: depotPath,
logger: logger.Session("metrics"),
}
}

func (m *metrics) NumCPU() int {
return runtime.NumCPU()
}

func (m *metrics) NumGoroutine() int {
return runtime.NumGoroutine()
}

func (m *metrics) LoopDevices() int {
devices, err := exec.Command("losetup", "-a").CombinedOutput()
if err != nil {
m.logger.Error("cannot-get-loop-devices", fmt.Errorf("%s, out: %s", err, string(devices)))
return -1
}
return bytes.Count(devices, []byte("\n"))
}

func (m *metrics) BackingStores() int {
if m.backingStoresPath == "" {
// graph is disabled
return -1
}

entries, err := ioutil.ReadDir(m.backingStoresPath)
if err != nil {
m.logger.Error("cannot-get-backing-stores", err)
return -1
}

return len(entries)
}

func (m *metrics) DepotDirs() int {
entries, err := ioutil.ReadDir(m.depotPath)
if err != nil {
m.logger.Error("cannot-get-depot-dirs", err)
return -1
}

return len(entries)
}
type Metrics map[string]func() int
67 changes: 67 additions & 0 deletions metrics/metrics_provider.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package metrics

import (
"bytes"
"fmt"
"io/ioutil"
"os/exec"
"runtime"

"code.cloudfoundry.org/lager"
)

type MetricsProvider struct {
backingStoresPath string
depotPath string
logger lager.Logger
}

func NewMetricsProvider(logger lager.Logger, backingStoresPath, depotPath string) *MetricsProvider {
return &MetricsProvider{
backingStoresPath: backingStoresPath,
depotPath: depotPath,
logger: logger.Session("metrics"),
}
}

func (m *MetricsProvider) NumCPU() int {
return runtime.NumCPU()
}

func (m *MetricsProvider) NumGoroutine() int {
return runtime.NumGoroutine()
}

func (m *MetricsProvider) LoopDevices() int {
devices, err := exec.Command("losetup", "-a").CombinedOutput()
if err != nil {
m.logger.Error("cannot-get-loop-devices", fmt.Errorf("%s, out: %s", err, string(devices)))
return -1
}
return bytes.Count(devices, []byte("\n"))
}

func (m *MetricsProvider) BackingStores() int {
if m.backingStoresPath == "" {
// graph is disabled
return -1
}

entries, err := ioutil.ReadDir(m.backingStoresPath)
if err != nil {
m.logger.Error("cannot-get-backing-stores", err)
return -1
}

return len(entries)
}

func (m *MetricsProvider) DepotDirs() int {
entries, err := ioutil.ReadDir(m.depotPath)
if err != nil {
m.logger.Error("cannot-get-depot-dirs", err)
return -1
}

return len(entries)
}
8 changes: 4 additions & 4 deletions metrics/metrics_test.go → metrics/metrics_provider_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,13 @@ import (
. "github.com/onsi/gomega"
)

var _ = Describe("Metrics", func() {
var _ = Describe("MetricsProvider", func() {
var (
logger *lagertest.TestLogger
backingStorePath string
depotPath string

m metrics.Metrics
m *metrics.MetricsProvider
)

BeforeEach(func() {
Expand All @@ -42,7 +42,7 @@ var _ = Describe("Metrics", func() {

Expect(err).ToNot(HaveOccurred())
logger = lagertest.NewTestLogger("test")
m = metrics.NewMetrics(logger, backingStorePath, depotPath)
m = metrics.NewMetricsProvider(logger, backingStorePath, depotPath)
})

AfterEach(func() {
Expand All @@ -60,7 +60,7 @@ var _ = Describe("Metrics", func() {

Context("when the backing store path is empty", func() {
It("reports BackingStores as -1 without doing any funny business", func() {
m := metrics.NewMetrics(logger, "", depotPath)
m := metrics.NewMetricsProvider(logger, "", depotPath)
Expect(m.BackingStores()).To(Equal(-1))

Expect(logger.LogMessages()).To(BeEmpty())
Expand Down
Loading

0 comments on commit 51dc3eb

Please sign in to comment.