diff --git a/README.md b/README.md index 879ee4f..bb17b96 100644 --- a/README.md +++ b/README.md @@ -25,24 +25,25 @@ Let's go through an example leveraging `go test` flow: 1. Implement the workload by embedding`e2e.Runnable` or `*e2e.InstrumentedRunnable`. Or you can use existing ones in [e2edb](db/) package. For example implementing Thanos Querier with our desired configuration could look like this: - ```go mdox-exec="sed -n '47,64p' examples/thanos/standalone.go" + ```go mdox-exec="sed -n '49,67p' examples/thanos/standalone.go" func newThanosSidecar(env e2e.Environment, name string, prom e2e.Linkable) *e2e.InstrumentedRunnable { ports := map[string]int{ "http": 9090, "grpc": 9091, } - return e2e.NewInstrumentedRunnable(env, name, ports, "http", e2e.StartOptions{ - Image: "quay.io/thanos/thanos:v0.21.1", - Command: e2e.NewCommand("sidecar", e2e.BuildArgs(map[string]string{ - "--debug.name": name, - "--grpc-address": fmt.Sprintf(":%d", ports["grpc"]), - "--http-address": fmt.Sprintf(":%d", ports["http"]), - "--prometheus.url": "http://" + prom.InternalEndpoint(e2edb.AccessPortName), - "--log.level": "info", - })...), - Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), - User: strconv.Itoa(os.Getuid()), - }) + return e2e.NewInstrumentedRunnable(env, name, ports, "http").Init( + e2e.StartOptions{ + Image: "quay.io/thanos/thanos:v0.21.1", + Command: e2e.NewCommand("sidecar", e2e.BuildArgs(map[string]string{ + "--debug.name": name, + "--grpc-address": fmt.Sprintf(":%d", ports["grpc"]), + "--http-address": fmt.Sprintf(":%d", ports["http"]), + "--prometheus.url": "http://" + prom.InternalEndpoint(e2edb.AccessPortName), + "--log.level": "info", + })...), + Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), + User: strconv.Itoa(os.Getuid()), + }) } ``` @@ -119,6 +120,30 @@ Let's go through an example leveraging `go test` flow: } ``` +### Monitoring + +Each instrumented workload have programmatic access to latest metrics with `WaitSumMetricsWithOptions` methods family. Yet, especially for standalone mode it's often useful to query yourself and visualisate metrics provided by your workloads and environment. In order to do so just start monitoring from `e2emontioring` package: + +```go +mon, err := e2emonitoring.Start(e) +if err != nil { + return err +} +``` + +This will start Prometheus with automatic discovery for every new and old instrumented runnables being scraped. It also runs cadvisor that monitors docker itself if `env.DockerEnvironment` is started. Run `OpenUserInterfaceInBrowser()` to open Prometheus UI in browser. + +```go + // Open monitoring page with all metrics. + if err := mon.OpenUserInterfaceInBrowser(); err != nil { + return err + } +``` + +To see how it works in practice run our example code in [standalone.go](examples/thanos/standalone.go) by running `make run-example`. At the end two UI should show in your browser. Thanos one and monitoring one. You can then e.g query docker container metrics using `sum(container_memory_working_set_bytes{name!=""}) by (name)` metric e.g: + +![mem metric](monitoring.png) + ## Credits * Initial Authors: [@pracucci](https://github.com/pracucci), [@bwplotka](https://github.com/bwplotka), [@pstibrany](https://github.com/pstibrany) diff --git a/db/db.go b/db/db.go index a4eee10..081e5cf 100644 --- a/db/db.go +++ b/db/db.go @@ -61,7 +61,7 @@ func NewMinio(env e2e.Environment, name, bktName string, opts ...Option) *e2e.In "MINIO_KMS_KES_CERT_FILE=" + "root.cert", "MINIO_KMS_KES_KEY_NAME=" + "my-minio-key", } - f := e2e.NewFutureInstrumentedRunnable(env, name, ports, AccessPortName) + f := e2e.NewInstrumentedRunnable(env, name, ports, AccessPortName) return f.Init( e2e.StartOptions{ Image: o.image, @@ -87,11 +87,7 @@ func NewConsul(env e2e.Environment, name string, opts ...Option) *e2e.Instrument } e2e.MergeFlags() - return e2e.NewInstrumentedRunnable( - env, - name, - map[string]int{AccessPortName: 8500}, - AccessPortName, + return e2e.NewInstrumentedRunnable(env, name, map[string]int{AccessPortName: 8500}, AccessPortName).Init( e2e.StartOptions{ Image: o.image, // Run consul in "dev" mode so that the initial leader election is immediate. @@ -107,11 +103,7 @@ func NewDynamoDB(env e2e.Environment, name string, opts ...Option) *e2e.Instrume opt(&o) } - return e2e.NewInstrumentedRunnable( - env, - name, - map[string]int{AccessPortName: 8000}, - AccessPortName, + return e2e.NewInstrumentedRunnable(env, name, map[string]int{AccessPortName: 8000}, AccessPortName).Init( e2e.StartOptions{ Image: o.image, Command: e2e.NewCommand("-jar", "DynamoDBLocal.jar", "-inMemory", "-sharedDb"), @@ -127,11 +119,7 @@ func NewBigtable(env e2e.Environment, name string, opts ...Option) *e2e.Instrume opt(&o) } - return e2e.NewInstrumentedRunnable( - env, - name, - nil, - AccessPortName, + return e2e.NewInstrumentedRunnable(env, name, nil, AccessPortName).Init( e2e.StartOptions{ Image: o.image, }, @@ -144,11 +132,7 @@ func NewCassandra(env e2e.Environment, name string, opts ...Option) *e2e.Instrum opt(&o) } - return e2e.NewInstrumentedRunnable( - env, - name, - map[string]int{AccessPortName: 9042}, - AccessPortName, + return e2e.NewInstrumentedRunnable(env, name, map[string]int{AccessPortName: 9042}, AccessPortName).Init( e2e.StartOptions{ Image: o.image, // Readiness probe inspired from https://github.com/kubernetes/examples/blob/b86c9d50be45eaf5ce74dee7159ce38b0e149d38/cassandra/image/files/ready-probe.sh @@ -163,11 +147,7 @@ func NewSwiftStorage(env e2e.Environment, name string, opts ...Option) *e2e.Inst opt(&o) } - return e2e.NewInstrumentedRunnable( - env, - name, - map[string]int{AccessPortName: 8080}, - AccessPortName, + return e2e.NewInstrumentedRunnable(env, name, map[string]int{AccessPortName: 8080}, AccessPortName).Init( e2e.StartOptions{ Image: o.image, Readiness: e2e.NewHTTPReadinessProbe(AccessPortName, "/", 404, 404), @@ -181,9 +161,7 @@ func NewMemcached(env e2e.Environment, name string, opts ...Option) e2e.Runnable opt(&o) } - return env.Runnable( - name, - map[string]int{AccessPortName: 11211}, + return env.Runnable(name).WithPorts(map[string]int{AccessPortName: 11211}).Init( e2e.StartOptions{ Image: o.image, Readiness: e2e.NewTCPReadinessProbe(AccessPortName), @@ -197,14 +175,7 @@ func NewETCD(env e2e.Environment, name string, opts ...Option) *e2e.Instrumented opt(&o) } - return e2e.NewInstrumentedRunnable( - env, - name, - map[string]int{ - AccessPortName: 2379, - "metrics": 9000, - }, - "metrics", + return e2e.NewInstrumentedRunnable(env, name, map[string]int{AccessPortName: 2379, "metrics": 9000}, "metrics").Init( e2e.StartOptions{ Image: o.image, Command: e2e.NewCommand("/usr/local/bin/etcd", "--listen-client-urls=http://0.0.0.0:2379", "--advertise-client-urls=http://0.0.0.0:2379", "--listen-metrics-urls=http://0.0.0.0:9000", "--log-level=error"), diff --git a/db/prometheus.go b/db/prometheus.go index e9b6455..9e17517 100644 --- a/db/prometheus.go +++ b/db/prometheus.go @@ -26,7 +26,7 @@ func NewPrometheus(env e2e.Environment, name string, opts ...Option) *Prometheus ports := map[string]int{"http": 9090} - f := e2e.NewFutureInstrumentedRunnable(env, name, ports, "http") + f := e2e.NewInstrumentedRunnable(env, name, ports, "http") config := fmt.Sprintf(` global: external_labels: diff --git a/db/thanos.go b/db/thanos.go index 443e433..72a6bf7 100644 --- a/db/thanos.go +++ b/db/thanos.go @@ -38,7 +38,7 @@ func NewThanosQuerier(env e2e.Environment, name string, endpointsAddresses []str args = e2e.MergeFlagsWithoutRemovingEmpty(args, o.flagOverride) } - return e2e.NewInstrumentedRunnable(env, name, ports, "http", e2e.StartOptions{ + return e2e.NewInstrumentedRunnable(env, name, ports, "http").Init(e2e.StartOptions{ Image: o.image, Command: e2e.NewCommand("query", e2e.BuildKingpinArgs(args)...), Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), @@ -68,7 +68,7 @@ func NewThanosSidecar(env e2e.Environment, name string, prom e2e.Linkable, opts args = e2e.MergeFlagsWithoutRemovingEmpty(args, o.flagOverride) } - return e2e.NewInstrumentedRunnable(env, name, ports, "http", e2e.StartOptions{ + return e2e.NewInstrumentedRunnable(env, name, ports, "http").Init(e2e.StartOptions{ Image: o.image, Command: e2e.NewCommand("sidecar", e2e.BuildKingpinArgs(args)...), Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), @@ -87,7 +87,7 @@ func NewThanosStore(env e2e.Environment, name string, bktConfigYaml []byte, opts "grpc": 9091, } - f := e2e.NewFutureInstrumentedRunnable(env, name, ports, "http") + f := e2e.NewInstrumentedRunnable(env, name, ports, "http") args := map[string]string{ "--debug.name": name, "--grpc-address": fmt.Sprintf(":%d", ports["grpc"]), diff --git a/env.go b/env.go index 5e8d8c2..9fff011 100644 --- a/env.go +++ b/env.go @@ -42,23 +42,31 @@ func WithVerbose() EnvironmentOption { // Environment defines how to run Runnable in isolated area e.g via docker in isolated docker network. type Environment interface { + // SharedDir returns host directory that will be shared with all runnables. SharedDir() string - // Runnable returns instance of runnable which can be started and stopped within this environment. - Runnable(name string, Ports map[string]int, opts StartOptions) Runnable - // FutureRunnable returns instance of runnable which can be started and stopped within this environment. - FutureRunnable(name string, Ports map[string]int) FutureRunnable - // Close shutdowns isolated environment and cleans it's resources. + // Runnable returns runnable builder which can build runnables that can be started and stopped within this environment. + Runnable(name string) RunnableBuilder + // AddListener registers given listener to be notified on environment runnable changes. + AddListener(listener EnvironmentListener) + // Close shutdowns isolated environment and cleans its resources. Close() } +type EnvironmentListener interface { + OnRunnableChange(started []Runnable) error +} + type StartOptions struct { Image string EnvVars map[string]string User string Command Command Readiness ReadinessProbe - // WaitReadyBackoff represents backoff used for WaitReady. + // WaitReadyBackofff represents backoff used for WaitReady. WaitReadyBackoff *backoff.Config + Volumes []string + UserNs string + Privileged bool } // Linkable is the entity that one can use to link runnable to other runnables before started. @@ -84,8 +92,24 @@ type FutureRunnable interface { Init(opts StartOptions) Runnable } +type RunnableBuilder interface { + WithPorts(map[string]int) RunnableBuilder + WithConcreteType(r Runnable) RunnableBuilder + + // Future returns future runnable + Future() FutureRunnable + // Init returns runnable. + Init(opts StartOptions) Runnable +} + +type identificable interface { + id() uintptr +} + // Runnable is the entity that environment returns to manage single instance. type Runnable interface { + identificable + Linkable // IsRunning returns if runnable was started. diff --git a/env_docker.go b/env_docker.go index 4458e0d..d7908bb 100644 --- a/env_docker.go +++ b/env_docker.go @@ -16,6 +16,7 @@ import ( "strconv" "strings" "time" + "unsafe" "github.com/efficientgo/tools/core/pkg/backoff" "github.com/pkg/errors" @@ -36,6 +37,7 @@ type DockerEnvironment struct { networkName string registered map[string]struct{} + listeners []EnvironmentListener started []Runnable verbose bool @@ -85,58 +87,63 @@ func NewDockerEnvironment(name string, opts ...EnvironmentOption) (*DockerEnviro return d, nil } -func (e *DockerEnvironment) Runnable(name string, ports map[string]int, opts StartOptions) Runnable { - return e.FutureRunnable(name, ports).Init(opts) -} - -func (e *DockerEnvironment) FutureRunnable(name string, ports map[string]int) FutureRunnable { +func (e *DockerEnvironment) Runnable(name string) RunnableBuilder { if e.closed { - return ErrRunnable{name: name, err: errors.New("environment close was invoked already.")} + return Errorer{name: name, err: errors.New("environment close was invoked already.")} } if e.isRegistered(name) { - return ErrRunnable{name: name, err: errors.Errorf("there is already one runnable created with the same name %v", name)} + return Errorer{name: name, err: errors.Errorf("there is already one runnable created with the same name %v", name)} } d := &dockerRunnable{ env: e, name: name, - ports: ports, logger: e.logger, + ports: map[string]int{}, hostPorts: map[string]int{}, } + d.concreteType = d if err := os.MkdirAll(d.Dir(), 0750); err != nil { - return ErrRunnable{name: name, err: err} + return Errorer{name: name, err: err} } - e.register(name) return d } -type ErrRunnable struct { +// AddListener registers given listener to be notified on environment runnable changes. +func (e *DockerEnvironment) AddListener(listener EnvironmentListener) { + e.listeners = append(e.listeners, listener) +} + +type Errorer struct { name string err error } -func NewErrRunnable(name string, err error) ErrRunnable { - return ErrRunnable{ +func NewErrorer(name string, err error) Errorer { + return Errorer{ name: name, err: err, } } -func (r ErrRunnable) Name() string { return r.name } -func (ErrRunnable) Dir() string { return "" } -func (ErrRunnable) InternalDir() string { return "" } -func (r ErrRunnable) Start() error { return r.err } -func (r ErrRunnable) WaitReady() error { return r.err } -func (r ErrRunnable) Kill() error { return r.err } -func (r ErrRunnable) Stop() error { return r.err } -func (r ErrRunnable) Exec(Command) (string, string, error) { return "", "", r.err } -func (ErrRunnable) Endpoint(string) string { return "" } -func (ErrRunnable) InternalEndpoint(string) string { return "" } -func (ErrRunnable) IsRunning() bool { return false } -func (r ErrRunnable) Init(StartOptions) Runnable { return r } +func (e Errorer) id() uintptr { return 0 } +func (e Errorer) Name() string { return e.name } +func (Errorer) Dir() string { return "" } +func (Errorer) InternalDir() string { return "" } +func (e Errorer) Start() error { return e.err } +func (e Errorer) WaitReady() error { return e.err } +func (e Errorer) Kill() error { return e.err } +func (e Errorer) Stop() error { return e.err } +func (e Errorer) Exec(Command) (string, string, error) { return "", "", e.err } +func (Errorer) Endpoint(string) string { return "" } +func (Errorer) InternalEndpoint(string) string { return "" } +func (Errorer) IsRunning() bool { return false } +func (e Errorer) Init(StartOptions) Runnable { return e } +func (e Errorer) WithPorts(map[string]int) RunnableBuilder { return e } +func (e Errorer) WithConcreteType(Runnable) RunnableBuilder { return e } +func (e Errorer) Future() FutureRunnable { return e } func (e *DockerEnvironment) isRegistered(name string) bool { _, ok := e.registered[name] @@ -147,16 +154,30 @@ func (e *DockerEnvironment) register(name string) { e.registered[name] = struct{}{} } -func (e *DockerEnvironment) registerStarted(r Runnable) { +func (e *DockerEnvironment) registerStarted(r Runnable) error { e.started = append(e.started, r) + + for _, l := range e.listeners { + if err := l.OnRunnableChange(e.started); err != nil { + return err + } + } + return nil } -func (e *DockerEnvironment) registerStopped(name string) { +func (e *DockerEnvironment) registerStopped(name string) error { for i, r := range e.started { if r.Name() == name { e.started = append(e.started[:i], e.started[i+1:]...) + for _, l := range e.listeners { + if err := l.OnRunnableChange(e.started); err != nil { + return err + } + } + return nil } } + return nil } func (e *DockerEnvironment) SharedDir() string { @@ -166,9 +187,13 @@ func (e *DockerEnvironment) SharedDir() string { func (e *DockerEnvironment) buildDockerRunArgs(name string, ports map[string]int, opts StartOptions) []string { args := []string{"run", "--rm", "--net=" + e.networkName, "--name=" + dockerNetworkContainerHost(e.networkName, name), "--hostname=" + name} - // Mount the shared/ directory into the container. We share all containers dir to each othe to allow easier scenarios. + // Mount the shared/ directory into the container. We share all containers dir to each other to allow easier scenarios. args = append(args, "-v", fmt.Sprintf("%s:%s:z", e.dir, dockerLocalSharedDir)) + for _, v := range opts.Volumes { + args = append(args, "-v", v) + } + // Environment variables for name, value := range opts.EnvVars { args = append(args, "-e", name+"="+value) @@ -178,6 +203,13 @@ func (e *DockerEnvironment) buildDockerRunArgs(name string, ports map[string]int args = append(args, "--user", opts.User) } + if opts.UserNs != "" { + args = append(args, "--userns", opts.UserNs) + } + + if opts.Privileged { + args = append(args, "--privileged") + } // Published ports. for _, port := range ports { args = append(args, "-p", strconv.Itoa(port)) @@ -213,6 +245,8 @@ type dockerRunnable struct { // hostPorts Maps port name to dynamically binded local ports. hostPorts map[string]int + + concreteType Runnable } func (d *dockerRunnable) Name() string { @@ -241,6 +275,24 @@ func (d *dockerRunnable) Init(opts StartOptions) Runnable { return d } +func (d *dockerRunnable) WithPorts(ports map[string]int) RunnableBuilder { + d.ports = ports + return d +} + +func (d *dockerRunnable) WithConcreteType(r Runnable) RunnableBuilder { + d.concreteType = r + return d +} + +func (d *dockerRunnable) id() uintptr { + return uintptr(unsafe.Pointer(d)) +} + +func (d *dockerRunnable) Future() FutureRunnable { + return d +} + func (d *dockerRunnable) IsRunning() bool { return d.usedNetworkName != "" } @@ -251,6 +303,14 @@ func (d *dockerRunnable) Start() (err error) { return errors.Errorf("%v is running. Stop or kill it first to restart.", d.Name()) } + i, ok := d.concreteType.(identificable) + if !ok { + return errors.Errorf("concrete type has at least embed runnable or future runnable instance provided by Runnable builder, got %T; not implementing identificable", d.concreteType) + } + if i.id() != d.id() { + return errors.Errorf("concrete type has at least embed runnable or future runnable instance provided by Runnable builder, got %T; id %v, expected %v", d.concreteType, i.id(), d.id()) + } + d.logger.Log("Starting", d.Name()) // In case of any error, if the container was already created, we @@ -276,7 +336,9 @@ func (d *dockerRunnable) Start() (err error) { return err } - d.env.registerStarted(d) + if err := d.env.registerStarted(d.concreteType); err != nil { + return err + } // Get the dynamic local ports mapped to the container. for portName, containerPort := range d.ports { @@ -319,8 +381,7 @@ func (d *dockerRunnable) Stop() error { return err } d.usedNetworkName = "" - d.env.registerStopped(d.Name()) - return nil + return d.env.registerStopped(d.Name()) } func (d *dockerRunnable) Kill() error { @@ -340,8 +401,7 @@ func (d *dockerRunnable) Kill() error { _, _ = d.env.exec("docker", "wait", d.containerName()).CombinedOutput() d.usedNetworkName = "" - d.env.registerStopped(d.Name()) - return nil + return d.env.registerStopped(d.Name()) } // Endpoint returns external (from host perspective) service endpoint (host:port) for given port name. @@ -463,7 +523,6 @@ func (d *dockerRunnable) WaitReady() (err error) { d.waitBackoff.Wait() } - return errors.Wrapf(err, "the service %s is not ready", d.Name()) } @@ -557,8 +616,9 @@ func (e *DockerEnvironment) close() { // Kill the services in the opposite order. for i := len(e.started) - 1; i >= 0; i-- { + n := e.started[i].Name() if err := e.started[i].Kill(); err != nil { - e.logger.Log("Unable to kill service", e.started[i].Name(), ":", err.Error()) + e.logger.Log("Unable to kill service", n, ":", err.Error()) } } diff --git a/examples/thanos/go.sum b/examples/thanos/go.sum index 0295ac7..cf7b76d 100644 --- a/examples/thanos/go.sum +++ b/examples/thanos/go.sum @@ -22,11 +22,13 @@ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -127,6 +129,7 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -170,6 +173,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= @@ -232,6 +236,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -309,6 +314,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -336,8 +342,10 @@ golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/examples/thanos/standalone.go b/examples/thanos/standalone.go index 18777ab..a1c9f6e 100644 --- a/examples/thanos/standalone.go +++ b/examples/thanos/standalone.go @@ -14,6 +14,7 @@ import ( "github.com/efficientgo/e2e" e2edb "github.com/efficientgo/e2e/db" e2einteractive "github.com/efficientgo/e2e/interactive" + e2emonitoring "github.com/efficientgo/e2e/monitoring" "github.com/efficientgo/tools/core/pkg/merrors" "github.com/oklog/run" ) @@ -36,12 +37,13 @@ func newThanosQuerier(env e2e.Environment, name string, endpointsAddresses ...st for _, e := range endpointsAddresses { args = append(args, "--store="+e) } - return e2e.NewInstrumentedRunnable(env, name, ports, "http", e2e.StartOptions{ - Image: "quay.io/thanos/thanos:v0.21.1", - Command: e2e.NewCommand("query", args...), - Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), - User: strconv.Itoa(os.Getuid()), - }) + return e2e.NewInstrumentedRunnable(env, name, ports, "http").Init( + e2e.StartOptions{ + Image: "quay.io/thanos/thanos:v0.21.1", + Command: e2e.NewCommand("query", args...), + Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), + User: strconv.Itoa(os.Getuid()), + }) } func newThanosSidecar(env e2e.Environment, name string, prom e2e.Linkable) *e2e.InstrumentedRunnable { @@ -49,21 +51,22 @@ func newThanosSidecar(env e2e.Environment, name string, prom e2e.Linkable) *e2e. "http": 9090, "grpc": 9091, } - return e2e.NewInstrumentedRunnable(env, name, ports, "http", e2e.StartOptions{ - Image: "quay.io/thanos/thanos:v0.21.1", - Command: e2e.NewCommand("sidecar", e2e.BuildArgs(map[string]string{ - "--debug.name": name, - "--grpc-address": fmt.Sprintf(":%d", ports["grpc"]), - "--http-address": fmt.Sprintf(":%d", ports["http"]), - "--prometheus.url": "http://" + prom.InternalEndpoint(e2edb.AccessPortName), - "--log.level": "info", - })...), - Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), - User: strconv.Itoa(os.Getuid()), - }) + return e2e.NewInstrumentedRunnable(env, name, ports, "http").Init( + e2e.StartOptions{ + Image: "quay.io/thanos/thanos:v0.21.1", + Command: e2e.NewCommand("sidecar", e2e.BuildArgs(map[string]string{ + "--debug.name": name, + "--grpc-address": fmt.Sprintf(":%d", ports["grpc"]), + "--http-address": fmt.Sprintf(":%d", ports["http"]), + "--prometheus.url": "http://" + prom.InternalEndpoint(e2edb.AccessPortName), + "--log.level": "info", + })...), + Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), + User: strconv.Itoa(os.Getuid()), + }) } -func deploy(ctx context.Context) error { +func deployWithMonitoring(ctx context.Context) error { // Start isolated environment with given ref. e, err := e2e.NewDockerEnvironment("e2e_example") if err != nil { @@ -72,6 +75,11 @@ func deploy(ctx context.Context) error { // Make sure resources (e.g docker containers, network, dir) are cleaned. defer e.Close() + mon, err := e2emonitoring.Start(e) + if err != nil { + return err + } + // Create structs for Prometheus containers scraping itself. p1 := e2edb.NewPrometheus(e, "prometheus-1") s1 := newThanosSidecar(e, "sidecar-1", p1) @@ -103,6 +111,10 @@ func deploy(ctx context.Context) error { if err := e2einteractive.OpenInBrowser("http://" + t1.Endpoint("http")); err != nil { return err } + // Open monitoring page with all metrics. + if err := mon.OpenUserInterfaceInBrowser(); err != nil { + return err + } // For interactive mode, wait until user interrupt. fmt.Println("Waiting on user interrupt (e.g Ctrl+C") <-ctx.Done() @@ -115,7 +127,7 @@ func main() { g.Add(run.SignalHandler(context.Background(), syscall.SIGINT, syscall.SIGTERM)) { ctx, cancel := context.WithCancel(context.Background()) - g.Add(func() error { return deploy(ctx) }, func(error) { cancel() }) + g.Add(func() error { return deployWithMonitoring(ctx) }, func(error) { cancel() }) } if err := g.Run(); err != nil { log.Fatal(err) diff --git a/go.mod b/go.mod index 1faa4ae..b6df87b 100644 --- a/go.mod +++ b/go.mod @@ -7,4 +7,5 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.15.0 + gopkg.in/yaml.v2 v2.3.0 ) diff --git a/go.sum b/go.sum index dc91c73..e7d6efb 100644 --- a/go.sum +++ b/go.sum @@ -22,11 +22,13 @@ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -127,6 +129,7 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -167,6 +170,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= @@ -208,6 +212,7 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -226,6 +231,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -303,6 +309,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -330,8 +337,10 @@ golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/http.go b/instrumented.go similarity index 87% rename from http.go rename to instrumented.go index 6bbbc8f..2f73288 100644 --- a/http.go +++ b/instrumented.go @@ -19,12 +19,25 @@ import ( var errMissingMetric = errors.New("metric not found") +type MetricTarget struct { + InternalEndpoint string + MetricPath string +} + +// Instrumented is implemented by all instrumented runnables. +type Instrumented interface { + MetricTargets() []MetricTarget +} + +var _ Instrumented = &InstrumentedRunnable{} + // InstrumentedRunnable represents opinionated microservice with one port marked as HTTP port with metric endpoint. type InstrumentedRunnable struct { Runnable name string metricPortName string + metricPath string ports map[string]int waitBackoff *backoff.Backoff @@ -36,31 +49,27 @@ type FutureInstrumentedRunnable struct { r *InstrumentedRunnable } -func NewFutureInstrumentedRunnable( +func NewInstrumentedRunnable( env Environment, name string, ports map[string]int, metricPortName string, ) *FutureInstrumentedRunnable { f := &FutureInstrumentedRunnable{ - r: &InstrumentedRunnable{name: name, ports: ports, metricPortName: metricPortName}, + r: &InstrumentedRunnable{name: name, ports: ports, metricPortName: metricPortName, metricPath: "/metrics"}, } if _, ok := ports[metricPortName]; !ok { - f.FutureRunnable = NewErrRunnable(name, errors.Errorf("metric port name %v does not exists in given ports", metricPortName)) + f.FutureRunnable = NewErrorer(name, errors.Errorf("metric port name %v does not exists in given ports", metricPortName)) return f } - f.FutureRunnable = env.FutureRunnable(name, ports) + f.FutureRunnable = env.Runnable(name).WithPorts(ports).WithConcreteType(f.r).Future() return f } -func NewInstrumentedRunnable(env Environment, name string, ports map[string]int, metricPortName string, opts StartOptions) *InstrumentedRunnable { - return NewFutureInstrumentedRunnable(env, name, ports, metricPortName).Init(opts) -} - func NewErrInstrumentedRunnable(name string, err error) *InstrumentedRunnable { return &InstrumentedRunnable{ - Runnable: NewErrRunnable(name, err), + Runnable: NewErrorer(name, err), } } @@ -78,6 +87,10 @@ func (r *FutureInstrumentedRunnable) Init(opts StartOptions) *InstrumentedRunnab return r.r } +func (r *InstrumentedRunnable) MetricTargets() []MetricTarget { + return []MetricTarget{{MetricPath: r.metricPath, InternalEndpoint: r.InternalEndpoint(r.metricPortName)}} +} + func (r *InstrumentedRunnable) Metrics() (_ string, err error) { if !r.IsRunning() { return "", errors.Errorf("%s is not running", r.Name()) diff --git a/http_composite.go b/instrumented_composite.go similarity index 79% rename from http_composite.go rename to instrumented_composite.go index 22cb5cc..8049921 100644 --- a/http_composite.go +++ b/instrumented_composite.go @@ -30,25 +30,32 @@ func NewCompositeInstrumentedRunnable(runnables ...*InstrumentedRunnable) *Compo } } -func (s *CompositeInstrumentedRunnable) Instances() []*InstrumentedRunnable { - return s.runnables +func (r *CompositeInstrumentedRunnable) Instances() []*InstrumentedRunnable { + return r.runnables +} + +func (r *CompositeInstrumentedRunnable) MetricTargets() (ret []MetricTarget) { + for _, inst := range r.runnables { + ret = append(ret, inst.MetricTargets()...) + } + return ret } // WaitSumMetrics waits for at least one instance of each given metric names to be present and their sums, returning true // when passed to given expected(...). -func (s *CompositeInstrumentedRunnable) WaitSumMetrics(expected MetricValueExpectation, metricNames ...string) error { - return s.WaitSumMetricsWithOptions(expected, metricNames) +func (r *CompositeInstrumentedRunnable) WaitSumMetrics(expected MetricValueExpectation, metricNames ...string) error { + return r.WaitSumMetricsWithOptions(expected, metricNames) } -func (s *CompositeInstrumentedRunnable) WaitSumMetricsWithOptions(expected MetricValueExpectation, metricNames []string, opts ...MetricsOption) error { +func (r *CompositeInstrumentedRunnable) WaitSumMetricsWithOptions(expected MetricValueExpectation, metricNames []string, opts ...MetricsOption) error { var ( sums []float64 err error options = buildMetricsOptions(opts) ) - for s.backoff.Reset(); s.backoff.Ongoing(); { - sums, err = s.SumMetrics(metricNames, opts...) + for r.backoff.Reset(); r.backoff.Ongoing(); { + sums, err = r.SumMetrics(metricNames, opts...) if options.waitMissingMetrics && errors.Is(err, errMissingMetric) { continue } @@ -60,7 +67,7 @@ func (s *CompositeInstrumentedRunnable) WaitSumMetricsWithOptions(expected Metri return nil } - s.backoff.Wait() + r.backoff.Wait() } return errors.Errorf("unable to find metrics %s with expected values. Last error: %v. Last values: %v", metricNames, err, sums) diff --git a/http_test.go b/instrumented_test.go similarity index 100% rename from http_test.go rename to instrumented_test.go diff --git a/monitoring.png b/monitoring.png new file mode 100644 index 0000000..bfd818e Binary files /dev/null and b/monitoring.png differ diff --git a/monitoring/monitoring.go b/monitoring/monitoring.go index a543d25..6a774f1 100644 --- a/monitoring/monitoring.go +++ b/monitoring/monitoring.go @@ -3,6 +3,113 @@ package e2emonitoring -// TODO(bwplotka): Add easy to use service showing container metrics. -type MonitoringService struct { +import ( + "fmt" + "time" + + "github.com/efficientgo/e2e" + e2edb "github.com/efficientgo/e2e/db" + e2einteractive "github.com/efficientgo/e2e/interactive" + "github.com/efficientgo/e2e/monitoring/promconfig" + sdconfig "github.com/efficientgo/e2e/monitoring/promconfig/discovery/config" + "github.com/efficientgo/e2e/monitoring/promconfig/discovery/targetgroup" + "github.com/prometheus/common/model" + "gopkg.in/yaml.v2" +) + +type Service struct { + p *e2edb.Prometheus +} + +type listener struct { + p *e2edb.Prometheus +} + +func (l *listener) updateConfig(started map[string]e2e.Instrumented) error { + cfg := promconfig.Config{ + GlobalConfig: promconfig.GlobalConfig{ + ExternalLabels: map[model.LabelName]model.LabelValue{"prometheus": model.LabelValue(l.p.Name())}, + ScrapeInterval: model.Duration(15 * time.Second), + }, + } + + add := func(name string, instr e2e.Instrumented) { + scfg := &promconfig.ScrapeConfig{ + JobName: name, + ServiceDiscoveryConfig: sdconfig.ServiceDiscoveryConfig{StaticConfigs: []*targetgroup.Group{{}}}, + } + for _, t := range instr.MetricTargets() { + scfg.ServiceDiscoveryConfig.StaticConfigs[0].Targets = append(scfg.ServiceDiscoveryConfig.StaticConfigs[0].Targets, map[model.LabelName]model.LabelValue{ + model.AddressLabel: model.LabelValue(t.InternalEndpoint), + }) + + if t.MetricPath != "/metrics" { + // TODO(bwplotka) Add relabelling rule to change `__path__`. + panic("Different metrics endpoints are not implemented yet") + } + } + cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scfg) + } + + add("e2emonitoring-prometheus", l.p) + for name, s := range started { + add(name, s) + } + + o, err := yaml.Marshal(cfg) + if err != nil { + return err + } + return l.p.SetConfig(string(o)) +} + +func (l *listener) OnRunnableChange(started []e2e.Runnable) error { + s := map[string]e2e.Instrumented{} + for _, r := range started { + instr, ok := r.(e2e.Instrumented) + if !ok { + fmt.Printf("NOT INSTRUMENTABLE %s %T\n", r.Name(), r) // To fix. + continue + } + s[r.Name()] = instr + } + + return l.updateConfig(s) +} + +// Start deploys monitoring service which deploys Prometheus that monitors all registered InstrumentedServices +// in environment. +func Start(env e2e.Environment) (*Service, error) { + p := e2edb.NewPrometheus(env, "monitoring") + l := &listener{p: p} + if err := l.updateConfig(map[string]e2e.Instrumented{}); err != nil { + return nil, err + } + env.AddListener(l) + + if err := newCadvisor(env, "cadvisor").Start(); err != nil { + return nil, err + } + + // TODO(bwplotka): Run cadvisor. + return &Service{p: p}, e2e.StartAndWaitReady(p) +} + +func (s *Service) OpenUserInterfaceInBrowser() error { + return e2einteractive.OpenInBrowser("http://" + s.p.Endpoint(e2edb.AccessPortName)) +} + +func newCadvisor(env e2e.Environment, name string) *e2e.InstrumentedRunnable { + return e2e.NewInstrumentedRunnable(env, name, map[string]int{"http": 8080}, "http").Init(e2e.StartOptions{ + Image: "gcr.io/cadvisor/cadvisor:v0.37.5", + // See https://github.com/google/cadvisor/blob/master/docs/running.md. + Volumes: []string{ + "/:/rootfs:ro", + "/var/run:/var/run:rw", + "/sys:/sys:ro", + "/var/lib/docker/:/var/lib/docker:ro", + }, + UserNs: "host", + Privileged: true, + }) } diff --git a/monitoring/promconfig/discovery/azure/azure.go b/monitoring/promconfig/discovery/azure/azure.go new file mode 100644 index 0000000..a2aebf4 --- /dev/null +++ b/monitoring/promconfig/discovery/azure/azure.go @@ -0,0 +1,36 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Copyright (c) bwplotka/mimic Authors +// Licensed under the Apache License 2.0. + +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + config_util "github.com/prometheus/common/config" + "github.com/prometheus/common/model" +) + +// SDConfig is the configuration for Azure based service discovery. +type SDConfig struct { + Environment string `yaml:"environment,omitempty"` + Port int `yaml:"port"` + SubscriptionID string `yaml:"subscription_id"` + TenantID string `yaml:"tenant_id,omitempty"` + ClientID string `yaml:"client_id,omitempty"` + ClientSecret config_util.Secret `yaml:"client_secret,omitempty"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` +} diff --git a/monitoring/promconfig/discovery/config/config.go b/monitoring/promconfig/discovery/config/config.go new file mode 100644 index 0000000..9160494 --- /dev/null +++ b/monitoring/promconfig/discovery/config/config.go @@ -0,0 +1,65 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Copyright (c) bwplotka/mimic Authors +// Licensed under the Apache License 2.0. + +// Copyright 2016 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdconfig + +import ( + "github.com/efficientgo/e2e/monitoring/promconfig/discovery/azure" + "github.com/efficientgo/e2e/monitoring/promconfig/discovery/consul" + "github.com/efficientgo/e2e/monitoring/promconfig/discovery/dns" + "github.com/efficientgo/e2e/monitoring/promconfig/discovery/ec2" + "github.com/efficientgo/e2e/monitoring/promconfig/discovery/file" + "github.com/efficientgo/e2e/monitoring/promconfig/discovery/gce" + "github.com/efficientgo/e2e/monitoring/promconfig/discovery/kubernetes" + "github.com/efficientgo/e2e/monitoring/promconfig/discovery/marathon" + "github.com/efficientgo/e2e/monitoring/promconfig/discovery/openstack" + "github.com/efficientgo/e2e/monitoring/promconfig/discovery/targetgroup" + "github.com/efficientgo/e2e/monitoring/promconfig/discovery/triton" + "github.com/efficientgo/e2e/monitoring/promconfig/discovery/zookeeper" +) + +// ServiceDiscoveryConfig configures lists of different service discovery mechanisms. +type ServiceDiscoveryConfig struct { + // List of labeled target groups for this job. + StaticConfigs []*targetgroup.Group `yaml:"static_configs,omitempty"` + // List of DNS service discovery configurations. + DNSSDConfigs []*dns.SDConfig `yaml:"dns_sd_configs,omitempty"` + // List of file service discovery configurations. + FileSDConfigs []*file.SDConfig `yaml:"file_sd_configs,omitempty"` + // List of Consul service discovery configurations. + ConsulSDConfigs []*consul.SDConfig `yaml:"consul_sd_configs,omitempty"` + // List of Serverset service discovery configurations. + ServersetSDConfigs []*zookeeper.ServersetSDConfig `yaml:"serverset_sd_configs,omitempty"` + // NerveSDConfigs is a list of Nerve service discovery configurations. + NerveSDConfigs []*zookeeper.NerveSDConfig `yaml:"nerve_sd_configs,omitempty"` + // MarathonSDConfigs is a list of Marathon service discovery configurations. + MarathonSDConfigs []*marathon.SDConfig `yaml:"marathon_sd_configs,omitempty"` + // List of Kubernetes service discovery configurations. + KubernetesSDConfigs []*kubernetes.SDConfig `yaml:"kubernetes_sd_configs,omitempty"` + // List of GCE service discovery configurations. + GCESDConfigs []*gce.SDConfig `yaml:"gce_sd_configs,omitempty"` + // List of EC2 service discovery configurations. + EC2SDConfigs []*ec2.SDConfig `yaml:"ec2_sd_configs,omitempty"` + // List of OpenStack service discovery configurations. + OpenstackSDConfigs []*openstack.SDConfig `yaml:"openstack_sd_configs,omitempty"` + // List of Azure service discovery configurations. + AzureSDConfigs []*azure.SDConfig `yaml:"azure_sd_configs,omitempty"` + // List of Triton service discovery configurations. + TritonSDConfigs []*triton.SDConfig `yaml:"triton_sd_configs,omitempty"` +} diff --git a/monitoring/promconfig/discovery/consul/consul.go b/monitoring/promconfig/discovery/consul/consul.go new file mode 100644 index 0000000..7e44a2c --- /dev/null +++ b/monitoring/promconfig/discovery/consul/consul.go @@ -0,0 +1,57 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Copyright (c) bwplotka/mimic Authors +// Licensed under the Apache License 2.0. + +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consul + +import ( + config_util "github.com/prometheus/common/config" + "github.com/prometheus/common/model" +) + +// SDConfig is the configuration for Consul service discovery. +type SDConfig struct { + Server string `yaml:"server,omitempty"` + Token config_util.Secret `yaml:"token,omitempty"` + Datacenter string `yaml:"datacenter,omitempty"` + TagSeparator string `yaml:"tag_separator,omitempty"` + Scheme string `yaml:"scheme,omitempty"` + Username string `yaml:"username,omitempty"` + Password config_util.Secret `yaml:"password,omitempty"` + + // See https://www.consul.io/docs/internals/consensus.html#consistency-modes, + // stale reads are a lot cheaper and are a necessity if you have >5k targets. + AllowStale bool `yaml:"allow_stale"` + // By default use blocking queries (https://www.consul.io/api/index.html#blocking-queries) + // but allow users to throttle updates if necessary. This can be useful because of "bugs" like + // https://github.com/hashicorp/consul/issues/3712 which cause an un-necessary + // amount of requests on consul. + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + + // See https://www.consul.io/api/catalog.html#list-services + // The list of services for which targets are discovered. + // Defaults to all services if empty. + Services []string `yaml:"services,omitempty"` + // An optional tag used to filter instances inside a service. A single tag is supported + // here to match the Consul API. + ServiceTag string `yaml:"tag,omitempty"` + // Desired node metadata. + NodeMeta map[string]string `yaml:"node_meta,omitempty"` + + TLSConfig config_util.TLSConfig `yaml:"tls_config,omitempty"` +} diff --git a/monitoring/promconfig/discovery/dns/dns.go b/monitoring/promconfig/discovery/dns/dns.go new file mode 100644 index 0000000..9cda5dd --- /dev/null +++ b/monitoring/promconfig/discovery/dns/dns.go @@ -0,0 +1,32 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Copyright (c) bwplotka/mimic Authors +// Licensed under the Apache License 2.0. + +// Copyright 2016 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dns + +import ( + "github.com/prometheus/common/model" +) + +// SDConfig is the configuration for DNS based service discovery. +type SDConfig struct { + Names []string `yaml:"names"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + Type string `yaml:"type"` + Port int `yaml:"port"` // Ignored for SRV records +} diff --git a/monitoring/promconfig/discovery/ec2/ec2.go b/monitoring/promconfig/discovery/ec2/ec2.go new file mode 100644 index 0000000..f731cd4 --- /dev/null +++ b/monitoring/promconfig/discovery/ec2/ec2.go @@ -0,0 +1,45 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Copyright (c) bwplotka/mimic Authors +// Licensed under the Apache License 2.0. + +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ec2 + +import ( + "github.com/prometheus/common/model" + + config_util "github.com/prometheus/common/config" +) + +// Filter is the configuration for filtering EC2 instances. +type Filter struct { + Name string `yaml:"name"` + Values []string `yaml:"values"` +} + +// SDConfig is the configuration for EC2 based service discovery. +type SDConfig struct { + Endpoint string `yaml:"endpoint"` + Region string `yaml:"region"` + AccessKey string `yaml:"access_key,omitempty"` + SecretKey config_util.Secret `yaml:"secret_key,omitempty"` + Profile string `yaml:"profile,omitempty"` + RoleARN string `yaml:"role_arn,omitempty"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + Port int `yaml:"port"` + Filters []*Filter `yaml:"filters"` +} diff --git a/monitoring/promconfig/discovery/file/file.go b/monitoring/promconfig/discovery/file/file.go new file mode 100644 index 0000000..4e5ffea --- /dev/null +++ b/monitoring/promconfig/discovery/file/file.go @@ -0,0 +1,30 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Copyright (c) bwplotka/mimic Authors +// Licensed under the Apache License 2.0. + +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package file + +import ( + "github.com/prometheus/common/model" +) + +// SDConfig is the configuration for file based discovery. +type SDConfig struct { + Files []string `yaml:"files"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` +} diff --git a/monitoring/promconfig/discovery/gce/gce.go b/monitoring/promconfig/discovery/gce/gce.go new file mode 100644 index 0000000..174a4ac --- /dev/null +++ b/monitoring/promconfig/discovery/gce/gce.go @@ -0,0 +1,43 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Copyright (c) bwplotka/mimic Authors +// Licensed under the Apache License 2.0. + +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gce + +import ( + "github.com/prometheus/common/model" +) + +// SDConfig is the configuration for GCE based service discovery. +type SDConfig struct { + // Project: The Google Cloud Project ID + Project string `yaml:"project"` + + // Zone: The zone of the scrape targets. + // If you need to configure multiple zones use multiple gce_sd_configs + Zone string `yaml:"zone"` + + // Filter: Can be used optionally to filter the instance list by other criteria. + // Syntax of this filter string is described here in the filter query parameter section: + // https://cloud.google.com/compute/docs/reference/latest/instances/list + Filter string `yaml:"filter,omitempty"` + + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + Port int `yaml:"port"` + TagSeparator string `yaml:"tag_separator,omitempty"` +} diff --git a/monitoring/promconfig/discovery/kubernetes/kubernetes.go b/monitoring/promconfig/discovery/kubernetes/kubernetes.go new file mode 100644 index 0000000..3eb340f --- /dev/null +++ b/monitoring/promconfig/discovery/kubernetes/kubernetes.go @@ -0,0 +1,60 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Copyright (c) bwplotka/mimic Authors +// Licensed under the Apache License 2.0. + +// Copyright 2016 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + config_util "github.com/prometheus/common/config" +) + +// Role is role of the service in Kubernetes. +type Role string + +// The valid options for Role. +const ( + RoleNode Role = "node" + RolePod Role = "pod" + RoleService Role = "service" + RoleEndpoint Role = "endpoints" + RoleIngress Role = "ingress" +) + +// SDConfig is the configuration for Kubernetes service discovery. +type SDConfig struct { + APIServer config_util.URL `yaml:"api_server,omitempty"` + Role Role `yaml:"role"` + BasicAuth *config_util.BasicAuth `yaml:"basic_auth,omitempty"` + BearerToken config_util.Secret `yaml:"bearer_token,omitempty"` + BearerTokenFile string `yaml:"bearer_token_file,omitempty"` + TLSConfig config_util.TLSConfig `yaml:"tls_config,omitempty"` + NamespaceDiscovery NamespaceDiscovery `yaml:"namespaces,omitempty"` +} + +// NamespaceDiscovery is the configuration for discovering +// Kubernetes namespaces. +type NamespaceDiscovery struct { + Names []string `yaml:"names"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *NamespaceDiscovery) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = NamespaceDiscovery{} + type plain NamespaceDiscovery + return unmarshal((*plain)(c)) +} diff --git a/monitoring/promconfig/discovery/marathon/marathon.go b/monitoring/promconfig/discovery/marathon/marathon.go new file mode 100644 index 0000000..46d2654 --- /dev/null +++ b/monitoring/promconfig/discovery/marathon/marathon.go @@ -0,0 +1,34 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Copyright (c) bwplotka/mimic Authors +// Licensed under the Apache License 2.0. + +// Copyright 2016 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package marathon + +import ( + config_util "github.com/prometheus/common/config" + "github.com/prometheus/common/model" +) + +// SDConfig is the configuration for services running on Marathon. +type SDConfig struct { + Servers []string `yaml:"servers,omitempty"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + AuthToken config_util.Secret `yaml:"auth_token,omitempty"` + AuthTokenFile string `yaml:"auth_token_file,omitempty"` + HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"` +} diff --git a/monitoring/promconfig/discovery/openstack/openstack.go b/monitoring/promconfig/discovery/openstack/openstack.go new file mode 100644 index 0000000..9953bb7 --- /dev/null +++ b/monitoring/promconfig/discovery/openstack/openstack.go @@ -0,0 +1,56 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Copyright (c) bwplotka/mimic Authors +// Licensed under the Apache License 2.0. + +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openstack + +import ( + config_util "github.com/prometheus/common/config" + "github.com/prometheus/common/model" +) + +// SDConfig is the configuration for OpenStack based service discovery. +type SDConfig struct { + IdentityEndpoint string `yaml:"identity_endpoint"` + Username string `yaml:"username"` + UserID string `yaml:"userid"` + Password config_util.Secret `yaml:"password"` + ProjectName string `yaml:"project_name"` + ProjectID string `yaml:"project_id"` + DomainName string `yaml:"domain_name"` + DomainID string `yaml:"domain_id"` + Role Role `yaml:"role"` + Region string `yaml:"region"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + Port int `yaml:"port"` + AllTenants bool `yaml:"all_tenants,omitempty"` + TLSConfig config_util.TLSConfig `yaml:"tls_config,omitempty"` +} + +// OpenStackRole is role of the target in OpenStack. +type Role string + +// The valid options for OpenStackRole. +const ( + // OpenStack document reference + // https://docs.openstack.org/nova/pike/admin/arch.html#hypervisors + OpenStackRoleHypervisor Role = "hypervisor" + // OpenStack document reference + // https://docs.openstack.org/horizon/pike/user/launch-instances.html + OpenStackRoleInstance Role = "instance" +) diff --git a/monitoring/promconfig/discovery/targetgroup/targetgroup.go b/monitoring/promconfig/discovery/targetgroup/targetgroup.go new file mode 100644 index 0000000..e96ae43 --- /dev/null +++ b/monitoring/promconfig/discovery/targetgroup/targetgroup.go @@ -0,0 +1,99 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Copyright (c) bwplotka/mimic Authors +// Licensed under the Apache License 2.0. + +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package targetgroup + +import ( + "bytes" + "encoding/json" + + "github.com/prometheus/common/model" +) + +// Group is a set of targets with a common label set(production , test, staging etc.). +type Group struct { + // Targets is a list of targets identified by a label set. Each target is + // uniquely identifiable in the group by its address label. + Targets []model.LabelSet + // Labels is a set of labels that is common across all targets in the group. + Labels model.LabelSet + + // Source is an identifier that describes a group of targets. + Source string +} + +func (tg Group) String() string { + return tg.Source +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (tg *Group) UnmarshalYAML(unmarshal func(interface{}) error) error { + g := struct { + Targets []string `yaml:"targets"` + Labels model.LabelSet `yaml:"labels"` + }{} + if err := unmarshal(&g); err != nil { + return err + } + tg.Targets = make([]model.LabelSet, 0, len(g.Targets)) + for _, t := range g.Targets { + tg.Targets = append(tg.Targets, model.LabelSet{ + model.AddressLabel: model.LabelValue(t), + }) + } + tg.Labels = g.Labels + return nil +} + +// MarshalYAML implements the yaml.Marshaler interface. +func (tg Group) MarshalYAML() (interface{}, error) { + g := &struct { + Targets []string `yaml:"targets"` + Labels model.LabelSet `yaml:"labels,omitempty"` + }{ + Targets: make([]string, 0, len(tg.Targets)), + Labels: tg.Labels, + } + for _, t := range tg.Targets { + g.Targets = append(g.Targets, string(t[model.AddressLabel])) + } + return g, nil +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (tg *Group) UnmarshalJSON(b []byte) error { + g := struct { + Targets []string `json:"targets"` + Labels model.LabelSet `json:"labels"` + }{} + + dec := json.NewDecoder(bytes.NewReader(b)) + dec.DisallowUnknownFields() + if err := dec.Decode(&g); err != nil { + return err + } + tg.Targets = make([]model.LabelSet, 0, len(g.Targets)) + for _, t := range g.Targets { + tg.Targets = append(tg.Targets, model.LabelSet{ + model.AddressLabel: model.LabelValue(t), + }) + } + tg.Labels = g.Labels + return nil +} diff --git a/monitoring/promconfig/discovery/triton/triton.go b/monitoring/promconfig/discovery/triton/triton.go new file mode 100644 index 0000000..807fd9e --- /dev/null +++ b/monitoring/promconfig/discovery/triton/triton.go @@ -0,0 +1,38 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Copyright (c) bwplotka/mimic Authors +// Licensed under the Apache License 2.0. + +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package triton + +import ( + "github.com/prometheus/common/model" + + config_util "github.com/prometheus/common/config" +) + +// SDConfig is the configuration for Triton based service discovery. +type SDConfig struct { + Account string `yaml:"account"` + DNSSuffix string `yaml:"dns_suffix"` + Endpoint string `yaml:"endpoint"` + Groups []string `yaml:"groups,omitempty"` + Port int `yaml:"port"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + TLSConfig config_util.TLSConfig `yaml:"tls_config,omitempty"` + Version int `yaml:"version"` +} diff --git a/monitoring/promconfig/discovery/zookeeper/zookeeper.go b/monitoring/promconfig/discovery/zookeeper/zookeeper.go new file mode 100644 index 0000000..513dbcc --- /dev/null +++ b/monitoring/promconfig/discovery/zookeeper/zookeeper.go @@ -0,0 +1,38 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +// Copyright (c) bwplotka/mimic Authors +// Licensed under the Apache License 2.0. + +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zookeeper + +import ( + "github.com/prometheus/common/model" +) + +// ServersetSDConfig is the configuration for Twitter serversets in Zookeeper based discovery. +type ServersetSDConfig struct { + Servers []string `yaml:"servers"` + Paths []string `yaml:"paths"` + Timeout model.Duration `yaml:"timeout,omitempty"` +} + +// NerveSDConfig is the configuration for AirBnB's Nerve in Zookeeper based discovery. +type NerveSDConfig struct { + Servers []string `yaml:"servers"` + Paths []string `yaml:"paths"` + Timeout model.Duration `yaml:"timeout,omitempty"` +} diff --git a/monitoring/promconfig/prometheus.go b/monitoring/promconfig/prometheus.go new file mode 100644 index 0000000..e9a2132 --- /dev/null +++ b/monitoring/promconfig/prometheus.go @@ -0,0 +1,261 @@ +// Copyright (c) The EfficientGo Authors. +// Licensed under the Apache License 2.0. + +package promconfig + +import ( + "net/url" + "regexp" + "strings" + + sdconfig "github.com/efficientgo/e2e/monitoring/promconfig/discovery/config" + "github.com/pkg/errors" + config_util "github.com/prometheus/common/config" + "github.com/prometheus/common/model" +) + +// NOTE(bwplotka): Stripped out Prometheus configuration from: https://github.com/prometheus/prometheus/blob/master/config/config.go +// We cannot import them directly easily as config package has cosmic number of dependencies (50+), dependency hell. +// TODO(bwplotka): Contribute and strip deps in Prometheus upstream once we agree as Prometheus maintainers if that use case we want to be support. + +// Config is the top-level configuration for Prometheus's config files. +type Config struct { + GlobalConfig GlobalConfig `yaml:"global"` + AlertingConfig AlertingConfig `yaml:"alerting,omitempty"` + RuleFiles []string `yaml:"rule_files,omitempty"` + ScrapeConfigs []*ScrapeConfig `yaml:"scrape_configs,omitempty"` + + RemoteWriteConfigs []*RemoteWriteConfig `yaml:"remote_write,omitempty"` + RemoteReadConfigs []*RemoteReadConfig `yaml:"remote_read,omitempty"` +} + +// GlobalConfig configures values that are used across other configuration +// objects. +type GlobalConfig struct { + // How frequently to scrape targets by default. + ScrapeInterval model.Duration `yaml:"scrape_interval,omitempty"` + // The default timeout when scraping targets. + ScrapeTimeout model.Duration `yaml:"scrape_timeout,omitempty"` + // How frequently to evaluate rules by default. + EvaluationInterval model.Duration `yaml:"evaluation_interval,omitempty"` + // The labels to add to any timeseries that this Prometheus instance scrapes. + ExternalLabels model.LabelSet `yaml:"external_labels,omitempty"` +} + +// ScrapeConfig configures a scraping unit for Prometheus. +type ScrapeConfig struct { + // The job name to which the job label is set by default. + JobName string `yaml:"job_name"` + // Indicator whether the scraped metrics should remain unmodified. + HonorLabels bool `yaml:"honor_labels,omitempty"` + // A set of query parameters with which the target is scraped. + Params url.Values `yaml:"params,omitempty"` + // How frequently to scrape the targets of this scrape config. + ScrapeInterval model.Duration `yaml:"scrape_interval,omitempty"` + // The timeout for scraping targets of this config. + ScrapeTimeout model.Duration `yaml:"scrape_timeout,omitempty"` + // The HTTP resource path on which to fetch metrics from targets. + MetricsPath string `yaml:"metrics_path,omitempty"` + // The URL scheme with which to fetch metrics from targets. + Scheme string `yaml:"scheme,omitempty"` + // More than this many samples post metric-relabelling will cause the scrape to fail. + SampleLimit uint `yaml:"sample_limit,omitempty"` + + // We cannot do proper Go type embedding below as the parser will then parse + // values arbitrarily into the overflow maps of further-down types. + + ServiceDiscoveryConfig sdconfig.ServiceDiscoveryConfig `yaml:",inline"` + HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"` + + // List of target relabel configurations. + RelabelConfigs []*RelabelConfig `yaml:"relabel_configs,omitempty"` + // List of metric relabel configurations. + MetricRelabelConfigs []*RelabelConfig `yaml:"metric_relabel_configs,omitempty"` +} + +// AlertingConfig configures alerting and alertmanager related configs. +type AlertingConfig struct { + AlertRelabelConfigs []*RelabelConfig `yaml:"alert_relabel_configs,omitempty"` + AlertmanagerConfigs []*AlertmanagerConfig `yaml:"alertmanagers,omitempty"` +} + +// AlertmanagerConfig configures how Alertmanagers can be discovered and communicated with. +type AlertmanagerConfig struct { + // We cannot do proper Go type embedding below as the parser will then parse + // values arbitrarily into the overflow maps of further-down types. + + ServiceDiscoveryConfig sdconfig.ServiceDiscoveryConfig `yaml:",inline"` + HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"` + + // The URL scheme to use when talking to Alertmanagers. + Scheme string `yaml:"scheme,omitempty"` + // Path prefix to add in front of the push endpoint path. + PathPrefix string `yaml:"path_prefix,omitempty"` + // The timeout used when sending alerts. + Timeout model.Duration `yaml:"timeout,omitempty"` + + // List of Alertmanager relabel configurations. + RelabelConfigs []*RelabelConfig `yaml:"relabel_configs,omitempty"` +} + +// ClientCert contains client cert credentials. +type ClientCert struct { + Cert string `yaml:"cert"` + Key config_util.Secret `yaml:"key"` +} + +// FileSDConfig is the configuration for file based discovery. +type FileSDConfig struct { + Files []string `yaml:"files"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` +} + +// RelabelAction is the action to be performed on relabeling. +type RelabelAction string + +const ( + // RelabelReplace performs a regex replacement. + RelabelReplace RelabelAction = "replace" + // RelabelKeep drops targets for which the input does not match the regex. + RelabelKeep RelabelAction = "keep" + // RelabelDrop drops targets for which the input does match the regex. + RelabelDrop RelabelAction = "drop" + // RelabelHashMod sets a label to the modulus of a hash of labels. + RelabelHashMod RelabelAction = "hashmod" + // RelabelLabelMap copies labels to other labelnames based on a regex. + RelabelLabelMap RelabelAction = "labelmap" + // RelabelLabelDrop drops any label matching the regex. + RelabelLabelDrop RelabelAction = "labeldrop" + // RelabelLabelKeep drops any label not matching the regex. + RelabelLabelKeep RelabelAction = "labelkeep" +) + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (a *RelabelAction) UnmarshalYAML(unmarshal func(interface{}) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + switch act := RelabelAction(strings.ToLower(s)); act { + case RelabelReplace, RelabelKeep, RelabelDrop, RelabelHashMod, RelabelLabelMap, RelabelLabelDrop, RelabelLabelKeep: + *a = act + return nil + } + return errors.Errorf("unknown relabel action %q", s) +} + +// RelabelConfig is the configuration for relabeling of target label sets. +type RelabelConfig struct { + // A list of labels from which values are taken and concatenated + // with the configured separator in order. + SourceLabels model.LabelNames `yaml:"source_labels,flow,omitempty"` + // Separator is the string between concatenated values from the source labels. + Separator string `yaml:"separator,omitempty"` + // Regex against which the concatenation is matched. + Regex Regexp `yaml:"regex,omitempty"` + // Modulus to take of the hash of concatenated values from the source labels. + Modulus uint64 `yaml:"modulus,omitempty"` + // TargetLabel is the label to which the resulting string is written in a replacement. + // Regexp interpolation is allowed for the replace action. + TargetLabel string `yaml:"target_label,omitempty"` + // Replacement is the regex replacement pattern to be used. + Replacement string `yaml:"replacement,omitempty"` + // Action is the action to be performed for the relabeling. + Action RelabelAction `yaml:"action,omitempty"` +} + +// Regexp encapsulates a regexp.Regexp and makes it YAML marshallable. +type Regexp struct { + *regexp.Regexp + original string +} + +// NewRegexp creates a new anchored Regexp and returns an error if the +// passed-in regular expression does not compile. +func NewRegexp(s string) (Regexp, error) { + regex, err := regexp.Compile("^(?:" + s + ")$") + return Regexp{ + Regexp: regex, + original: s, + }, err +} + +// MustNewRegexp works like NewRegexp, but panics if the regular expression does not compile. +func MustNewRegexp(s string) Regexp { + re, err := NewRegexp(s) + if err != nil { + panic(err) + } + return re +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (re *Regexp) UnmarshalYAML(unmarshal func(interface{}) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + r, err := NewRegexp(s) + if err != nil { + return err + } + *re = r + return nil +} + +// MarshalYAML implements the yaml.Marshaler interface. +func (re Regexp) MarshalYAML() (interface{}, error) { + if re.original != "" { + return re.original, nil + } + return nil, nil +} + +// RemoteWriteConfig is the configuration for writing to remote storage. +type RemoteWriteConfig struct { + URL *config_util.URL `yaml:"url"` + RemoteTimeout model.Duration `yaml:"remote_timeout,omitempty"` + WriteRelabelConfigs []*RelabelConfig `yaml:"write_relabel_configs,omitempty"` + + // We cannot do proper Go type embedding below as the parser will then parse + // values arbitrarily into the overflow maps of further-down types. + HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"` + QueueConfig QueueConfig `yaml:"queue_config,omitempty"` +} + +// QueueConfig is the configuration for the queue used to write to remote +// storage. +type QueueConfig struct { + // Number of samples to buffer per shard before we start dropping them. + Capacity int `yaml:"capacity,omitempty"` + + // Max number of shards, i.e. amount of concurrency. + MaxShards int `yaml:"max_shards,omitempty"` + + // Maximum number of samples per send. + MaxSamplesPerSend int `yaml:"max_samples_per_send,omitempty"` + + // Maximum time sample will wait in buffer. + BatchSendDeadline model.Duration `yaml:"batch_send_deadline,omitempty"` + + // Max number of times to retry a batch on recoverable errors. + MaxRetries int `yaml:"max_retries,omitempty"` + + // On recoverable errors, backoff exponentially. + MinBackoff model.Duration `yaml:"min_backoff,omitempty"` + MaxBackoff model.Duration `yaml:"max_backoff,omitempty"` +} + +// RemoteReadConfig is the configuration for reading from remote storage. +type RemoteReadConfig struct { + URL *config_util.URL `yaml:"url"` + RemoteTimeout model.Duration `yaml:"remote_timeout,omitempty"` + ReadRecent bool `yaml:"read_recent,omitempty"` + // We cannot do proper Go type embedding below as the parser will then parse + // values arbitrarily into the overflow maps of further-down types. + HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"` + + // RequiredMatchers is an optional list of equality matchers which have to + // be present in a selector to query the remote read endpoint. + RequiredMatchers model.LabelSet `yaml:"required_matchers,omitempty"` +}