-
Notifications
You must be signed in to change notification settings - Fork 44
feat(dashmate): add prometheus service discovery labels #2818
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: v2.2-dev
Are you sure you want to change the base?
Changes from all commits
4e1c115
208608b
62337dd
57a0a67
1a6e248
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| # Prometheus Configuration | ||
|
|
||
| This directory contains a minimal Prometheus stack for scraping metrics from a Dashmate-managed Platform network. It is meant for local debugging and pairs with the metric endpoints that Dashmate exposes when you enable them in your node configuration. | ||
|
|
||
| ## Files | ||
| - `docs/prometheus/docker-compose.yml` brings up two containers: a read-only Docker Socket Proxy that exposes `tcp://127.0.0.1:2375`, and Prometheus itself listening on `:9080`. | ||
| - `docs/prometheus/prometheus.yml` configures Prometheus to use Docker service discovery against the proxy and scrape every container that carries Dashmate’s `prometheus.io/*` labels. | ||
|
|
||
| ## Prepare Dashmate | ||
| 1. Enable metrics on the services you want Prometheus to monitor (examples): | ||
| - `dashmate config set platform.drive.abci.metrics.enabled true` | ||
| - `dashmate config set platform.drive.tenderdash.metrics.enabled true` | ||
| - `dashmate config set platform.dapi.rsDapi.metrics.enabled true` | ||
| - `dashmate config set platform.gateway.metrics.enabled true` | ||
| - `dashmate config set platform.gateway.rateLimiter.metrics.enabled true` | ||
| 2. Restart your network so the containers are recreated with the Prometheus labels: `yarn restart`. | ||
|
|
||
| The labels Dashmate adds map cleanly onto the Prometheus discovery config: | ||
| - `prometheus.io/scrape=true` enables scraping. | ||
| - `prometheus.io/path` overrides the metrics path (defaults to `/metrics`). | ||
| - `prometheus.io/port` is rewritten to `127.0.0.1:<port>` so scraping stays on the host network. | ||
| - `org_dashmate_service_title` and `org_dashmate_config_name` become `service` and `config` labels on each time series. | ||
|
|
||
| ## Run Prometheus | ||
| ```bash | ||
| docker compose -f docs/prometheus/docker-compose.yml up -d | ||
| ``` | ||
|
|
||
| Prometheus stores data in the `prometheus-data` volume and exposes its UI at `http://127.0.0.1:9080`. Reload the configuration after any edits with: | ||
| ```bash | ||
| curl -X POST http://127.0.0.1:9080/-/reload | ||
| ``` | ||
|
|
||
| ### Customisation Tips | ||
| - Adjust `scrape_interval` or add more `scrape_configs` in `prometheus.yml` as needed. | ||
| - To scrape additional containers, attach the same `prometheus.io/*` labels to them, or add dedicated jobs to the configuration. | ||
| - If your Docker daemon is not local, update the proxy service or mount a different socket before starting the stack. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| --- | ||
|
|
||
| services: | ||
| docker_socket_proxy: | ||
| image: tecnativa/docker-socket-proxy:latest | ||
| container_name: prometheus-docker-socket-proxy | ||
| restart: unless-stopped | ||
| environment: | ||
| - CONTAINERS=1 | ||
| - NETWORKS=1 | ||
| volumes: | ||
| - /var/run/docker.sock:/var/run/docker.sock:ro | ||
| ports: | ||
| - "127.0.0.1:2375:2375" | ||
|
|
||
| prometheus: | ||
| image: prom/prometheus:latest | ||
| container_name: prometheus-rsdapi | ||
| network_mode: host | ||
| depends_on: | ||
| - docker_socket_proxy | ||
| command: | ||
| - --config.file=/etc/prometheus/prometheus.yml | ||
| - --storage.tsdb.path=/prometheus | ||
| - --web.enable-lifecycle | ||
| - --web.listen-address=:9080 | ||
| volumes: | ||
| - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro | ||
| - prometheus-data:/prometheus | ||
| restart: unless-stopped | ||
|
|
||
| volumes: | ||
| prometheus-data: | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| --- | ||
|
|
||
| global: | ||
| scrape_interval: 15s | ||
| scrape_timeout: 10s | ||
|
|
||
| scrape_configs: | ||
| - job_name: dashmate | ||
| metrics_path: /metrics | ||
| docker_sd_configs: | ||
| - host: tcp://127.0.0.1:2375 | ||
| refresh_interval: 15s | ||
| relabel_configs: | ||
| - source_labels: [__meta_docker_container_label_prometheus_io_scrape] | ||
| action: keep | ||
| regex: true | ||
| - source_labels: [__meta_docker_container_label_prometheus_io_path] | ||
| target_label: __metrics_path__ | ||
| regex: (.+) | ||
| - source_labels: [__meta_docker_container_label_prometheus_io_port] | ||
| action: replace | ||
| regex: (.+) | ||
| replacement: 127.0.0.1:$1 | ||
| target_label: __address__ | ||
| - source_labels: [__meta_docker_container_label_org_dashmate_service_title] | ||
| target_label: service | ||
| regex: (.+) | ||
| - source_labels: [__meta_docker_container_label_org_dashmate_config_name] | ||
| target_label: config | ||
| regex: (.+) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -16,17 +16,6 @@ export default function analyseConfigFactory() { | |
| const problems = []; | ||
|
|
||
| if (config?.get('platform.enable')) { | ||
| // Gateway admin is disabled while metrics are enabled | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why do you remove this check? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This case will never happen, see https://github.com/dashpay/platform/pull/2818/files#r2472878855 |
||
| if (config.get('platform.gateway.metrics.enabled') && !config.get('platform.gateway.admin.enabled')) { | ||
| const problem = new Problem( | ||
| 'Gateway admin is disabled while metrics are enabled', | ||
| chalk`Please enable gateway admin: {bold.cyanBright dashmate config set platform.gateway.admin.enabled true}`, | ||
| SEVERITY.HIGH, | ||
| ); | ||
|
|
||
| problems.push(problem); | ||
| } | ||
|
|
||
| // Platform Node ID | ||
| const masternodeStatus = samples.getServiceInfo('core', 'masternodeStatus'); | ||
| const platformNodeId = masternodeStatus?.dmnState?.platformNodeId; | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -527,7 +527,7 @@ static_resources: | |
| address: gateway_rate_limiter | ||
| port_value: 8081 | ||
| {{?}} | ||
| {{? it.platform.gateway.metrics.enabled && it.platform.gateway.admin.enabled }} | ||
| {{? it.platform.gateway.metrics.enabled }} | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why do you remove this condition? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Metrics without admin don't work. Assuming the intention of an user who enabled metrics is to get them really working ;), I've changed logic so that when metrics are enabled, always enable admin automatically but don't expose it to public. See line 546-550 below. When admin is not enabled, adminHost will be 127.0.0.1, what means it will listen on loopback of docker container (not docker host). |
||
| - name: admin | ||
| connect_timeout: 0.25s | ||
| type: STATIC | ||
|
|
@@ -542,11 +542,12 @@ static_resources: | |
| port_value: 9901 | ||
| {{?}} | ||
|
|
||
| {{? it.platform.gateway.admin.enabled }} | ||
| {{? it.platform.gateway.admin.enabled || it.platform.gateway.metrics.enabled }} | ||
| {{ adminHost = it.platform.gateway.admin.enabled ? '0.0.0.0' : '127.0.0.1'; }} | ||
| admin: | ||
| address: | ||
| socket_address: | ||
| address: 0.0.0.0 # For docker container only. Must be a local/private interface. | ||
| address: {{= adminHost }} # defaults to loopback when not explicitly enabled | ||
| port_value: 9901 | ||
| {{?}} | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| import HomeDir from '../../../src/config/HomeDir.js'; | ||
| import getBaseConfigFactory from '../../../configs/defaults/getBaseConfigFactory.js'; | ||
| import renderTemplateFactory from '../../../src/templates/renderTemplateFactory.js'; | ||
| import renderServiceTemplatesFactory from '../../../src/templates/renderServiceTemplatesFactory.js'; | ||
lklimek marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| function getRsDapiBlock(dynamicComposeContent) { | ||
| const match = dynamicComposeContent.match(/rs_dapi:\n((?: {2}.*\n)+)/); | ||
| return match ? match[1] : ''; | ||
| } | ||
|
|
||
| describe('dynamic compose template', () => { | ||
| let getBaseConfig; | ||
| let renderServiceTemplates; | ||
|
|
||
| beforeEach(() => { | ||
| getBaseConfig = getBaseConfigFactory(HomeDir.createTemp()); | ||
| const renderTemplate = renderTemplateFactory(); | ||
| renderServiceTemplates = renderServiceTemplatesFactory(renderTemplate); | ||
| }); | ||
|
|
||
| it('should not publish metrics port when rs-dapi metrics are disabled', () => { | ||
| const config = getBaseConfig(); | ||
|
|
||
| const renderedConfigs = renderServiceTemplates(config); | ||
| const rsDapiBlock = getRsDapiBlock(renderedConfigs['dynamic-compose.yml']); | ||
|
|
||
| expect(rsDapiBlock).to.not.include('ports:'); | ||
| expect(rsDapiBlock).to.not.include(':0'); | ||
| }); | ||
|
|
||
| it('should publish metrics port when rs-dapi metrics are enabled', () => { | ||
| const config = getBaseConfig(); | ||
|
|
||
| config.set('platform.dapi.rsDapi.metrics.enabled', true); | ||
| config.set('platform.dapi.rsDapi.metrics.port', 29091); | ||
| config.set('platform.dapi.rsDapi.metrics.host', '127.0.0.1'); | ||
|
|
||
| const renderedConfigs = renderServiceTemplates(config); | ||
| const rsDapiBlock = getRsDapiBlock(renderedConfigs['dynamic-compose.yml']); | ||
|
|
||
| expect(rsDapiBlock).to.include('ports:\n - 127.0.0.1:29091:29091'); | ||
| expect(rsDapiBlock).to.include('- 29091'); | ||
| }); | ||
| }); | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| import getBaseConfigFactory from '../../../configs/defaults/getBaseConfigFactory.js'; | ||
| import HomeDir from '../../../src/config/HomeDir.js'; | ||
| import renderServiceTemplatesFactory from '../../../src/templates/renderServiceTemplatesFactory.js'; | ||
| import renderTemplateFactory from '../../../src/templates/renderTemplateFactory.js'; | ||
lklimek marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| describe('envoy template', () => { | ||
| it('should render admin interface when metrics are enabled even if admin is disabled', () => { | ||
| const getBaseConfig = getBaseConfigFactory(HomeDir.createTemp()); | ||
| const config = getBaseConfig(); | ||
|
|
||
| config.set('platform.gateway.metrics.enabled', true); | ||
| config.set('platform.gateway.admin.enabled', false); | ||
|
|
||
| const renderTemplate = renderTemplateFactory(); | ||
| const renderServiceTemplates = renderServiceTemplatesFactory(renderTemplate); | ||
| const renderedConfigs = renderServiceTemplates(config); | ||
|
|
||
| const envoyConfig = renderedConfigs['platform/gateway/envoy.yaml']; | ||
|
|
||
| expect(envoyConfig).to.include('cluster_name: admin'); | ||
| expect(envoyConfig).to.include('address: 127.0.0.1'); | ||
| expect(envoyConfig).to.include('port_value: 9901'); | ||
| }); | ||
| }); | ||
Uh oh!
There was an error while loading. Please reload this page.