Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file removed .yarn/cache/fsevents-patch-19706e7e35-10.zip
Binary file not shown.
4 changes: 4 additions & 0 deletions packages/dashmate/docker-compose.rate_limiter.metrics.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ services:
image: ${PLATFORM_GATEWAY_RATE_LIMITER_METRICS_DOCKER_IMAGE:?err}
labels:
org.dashmate.service.title: "Gateway rate limiter metrics exporter"
org.dashmate.config.name: "${CONFIG_NAME:?err}"
prometheus.io/scrape: "${PLATFORM_GATEWAY_RATE_LIMITER_METRICS_ENABLED:-false}"
prometheus.io/port: "${PLATFORM_GATEWAY_RATE_LIMITER_METRICS_PORT:?err}"
prometheus.io/path: "/metrics"
restart: unless-stopped
logging: *default-logging
entrypoint: /bin/statsd_exporter
Expand Down
16 changes: 16 additions & 0 deletions packages/dashmate/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ services:
image: ${PLATFORM_DRIVE_ABCI_DOCKER_IMAGE:?err}
labels:
org.dashmate.service.title: "Drive ABCI"
org.dashmate.config.name: "${CONFIG_NAME:?err}"
prometheus.io/scrape: "${PLATFORM_DRIVE_ABCI_METRICS_ENABLED:-false}"
prometheus.io/port: "${PLATFORM_DRIVE_ABCI_METRICS_PORT:?err}"
prometheus.io/path: "/metrics"
restart: unless-stopped
logging: *default-logging
volumes:
Expand Down Expand Up @@ -111,6 +115,10 @@ services:
image: ${PLATFORM_DRIVE_TENDERDASH_DOCKER_IMAGE:?err}
labels:
org.dashmate.service.title: "Drive Tenderdash"
org.dashmate.config.name: "${CONFIG_NAME:?err}"
prometheus.io/scrape: "${PLATFORM_DRIVE_TENDERDASH_METRICS_ENABLED:-false}"
prometheus.io/port: "${PLATFORM_DRIVE_TENDERDASH_METRICS_PORT:?err}"
prometheus.io/path: "/metrics"
restart: unless-stopped
logging: *default-logging
depends_on:
Expand Down Expand Up @@ -196,6 +204,10 @@ services:
image: ${PLATFORM_DAPI_RS_DAPI_DOCKER_IMAGE:?err}
labels:
org.dashmate.service.title: "rs-dapi (Rust DAPI)"
org.dashmate.config.name: "${CONFIG_NAME:?err}"
prometheus.io/scrape: "${PLATFORM_DAPI_RS_DAPI_METRICS_ENABLED:-false}"
prometheus.io/port: "${PLATFORM_DAPI_RS_DAPI_METRICS_PORT:?err}"
prometheus.io/path: "/metrics"
restart: unless-stopped
logging: *default-logging
deploy:
Expand Down Expand Up @@ -236,6 +248,10 @@ services:
image: ${PLATFORM_GATEWAY_DOCKER_IMAGE:?err}
labels:
org.dashmate.service.title: "Gateway"
org.dashmate.config.name: "${CONFIG_NAME:?err}"
prometheus.io/scrape: "${PLATFORM_GATEWAY_METRICS_ENABLED:-false}"
prometheus.io/port: "${PLATFORM_GATEWAY_METRICS_PORT:?err}"
prometheus.io/path: "/metrics"
restart: unless-stopped
logging: *default-logging
ports:
Expand Down
2 changes: 1 addition & 1 deletion packages/dashmate/docs/config/gateway.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ These settings control the metrics endpoint for monitoring the Gateway:
| `platform.gateway.metrics.port` | Port for metrics server | `9090` | `9091` |

Metrics provide performance and health information about the Gateway service.
Admin must be enabled to access the metrics endpoint.
Dashmate automatically enables the Envoy admin endpoint whenever metrics are enabled so that the Prometheus listener can proxy `/stats/prometheus`; if admin itself is still disabled, the listener is not exposed outside of Docker.

## Admin

Expand Down
37 changes: 37 additions & 0 deletions packages/dashmate/docs/prometheus/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Prometheus Configuration

This directory contains a minimal Prometheus stack for scraping metrics from a Dashmate-managed Platform network. It is meant for local debugging and pairs with the metric endpoints that Dashmate exposes when you enable them in your node configuration.

## Files
- `docs/prometheus/docker-compose.yml` brings up two containers: a read-only Docker Socket Proxy that exposes `tcp://127.0.0.1:2375`, and Prometheus itself listening on `:9080`.
- `docs/prometheus/prometheus.yml` configures Prometheus to use Docker service discovery against the proxy and scrape every container that carries Dashmate’s `prometheus.io/*` labels.

## Prepare Dashmate
1. Enable metrics on the services you want Prometheus to monitor (examples):
- `dashmate config set platform.drive.abci.metrics.enabled true`
- `dashmate config set platform.drive.tenderdash.metrics.enabled true`
- `dashmate config set platform.dapi.rsDapi.metrics.enabled true`
- `dashmate config set platform.gateway.metrics.enabled true`
- `dashmate config set platform.gateway.rateLimiter.metrics.enabled true`
2. Restart your network so the containers are recreated with the Prometheus labels: `yarn restart`.

The labels Dashmate adds map cleanly onto the Prometheus discovery config:
- `prometheus.io/scrape=true` enables scraping.
- `prometheus.io/path` overrides the metrics path (defaults to `/metrics`).
- `prometheus.io/port` is rewritten to `127.0.0.1:<port>` so scraping stays on the host network.
- `org_dashmate_service_title` and `org_dashmate_config_name` become `service` and `config` labels on each time series.

## Run Prometheus
```bash
docker compose -f docs/prometheus/docker-compose.yml up -d
```

Prometheus stores data in the `prometheus-data` volume and exposes its UI at `http://127.0.0.1:9080`. Reload the configuration after any edits with:
```bash
curl -X POST http://127.0.0.1:9080/-/reload
```

### Customisation Tips
- Adjust `scrape_interval` or add more `scrape_configs` in `prometheus.yml` as needed.
- To scrape additional containers, attach the same `prometheus.io/*` labels to them, or add dedicated jobs to the configuration.
- If your Docker daemon is not local, update the proxy service or mount a different socket before starting the stack.
33 changes: 33 additions & 0 deletions packages/dashmate/docs/prometheus/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
---

services:
docker_socket_proxy:
image: tecnativa/docker-socket-proxy:latest
container_name: prometheus-docker-socket-proxy
restart: unless-stopped
environment:
- CONTAINERS=1
- NETWORKS=1
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
ports:
- "127.0.0.1:2375:2375"

prometheus:
image: prom/prometheus:latest
container_name: prometheus-rsdapi
network_mode: host
depends_on:
- docker_socket_proxy
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --web.enable-lifecycle
- --web.listen-address=:9080
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
- prometheus-data:/prometheus
restart: unless-stopped

volumes:
prometheus-data:
30 changes: 30 additions & 0 deletions packages/dashmate/docs/prometheus/prometheus.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
---

global:
scrape_interval: 15s
scrape_timeout: 10s

scrape_configs:
- job_name: dashmate
metrics_path: /metrics
docker_sd_configs:
- host: tcp://127.0.0.1:2375
refresh_interval: 15s
relabel_configs:
- source_labels: [__meta_docker_container_label_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_docker_container_label_prometheus_io_path]
target_label: __metrics_path__
regex: (.+)
- source_labels: [__meta_docker_container_label_prometheus_io_port]
action: replace
regex: (.+)
replacement: 127.0.0.1:$1
target_label: __address__
- source_labels: [__meta_docker_container_label_org_dashmate_service_title]
target_label: service
regex: (.+)
- source_labels: [__meta_docker_container_label_org_dashmate_config_name]
target_label: config
regex: (.+)
2 changes: 1 addition & 1 deletion packages/dashmate/docs/services/gateway.md
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@ Metrics include:
- Host: `platform.gateway.metrics.host` (default: 127.0.0.1)
- Port: `platform.gateway.metrics.port` (default: 9090)

**Note:** admin interface must be enabled too.
**Note:** Dashmate automatically enables the Envoy admin endpoint whenever metrics are turned on so the Prometheus listener can proxy `/stats/prometheus`. If the admin service remains disabled, this socket is not exposed outside Docker; once you explicitly enable admin, it uses the host you configure.

### Security Considerations

Expand Down
3 changes: 3 additions & 0 deletions packages/dashmate/docs/services/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -119,3 +119,6 @@ Most services provide metrics endpoints:
- Rate Limiter: Rate limiting metrics

These can be integrated with monitoring systems like Prometheus and Grafana.
All containers that expose Prometheus metrics also advertise the Docker label `org.dashmate.config.name`, which Dashmate sets to the active config name. When you run multiple nodes on the same host, you can use that label in Prometheus relabeling rules or dashboards to distinguish each instance.

You can find example Prometheus config for local devnet monitoring in [../prometheus/](../prometheus/README.md).
11 changes: 0 additions & 11 deletions packages/dashmate/src/doctor/analyse/analyseConfigFactory.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,6 @@ export default function analyseConfigFactory() {
const problems = [];

if (config?.get('platform.enable')) {
// Gateway admin is disabled while metrics are enabled
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do you remove this check?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if (config.get('platform.gateway.metrics.enabled') && !config.get('platform.gateway.admin.enabled')) {
const problem = new Problem(
'Gateway admin is disabled while metrics are enabled',
chalk`Please enable gateway admin: {bold.cyanBright dashmate config set platform.gateway.admin.enabled true}`,
SEVERITY.HIGH,
);

problems.push(problem);
}

// Platform Node ID
const masternodeStatus = samples.getServiceInfo('core', 'masternodeStatus');
const platformNodeId = masternodeStatus?.dmnState?.platformNodeId;
Expand Down
2 changes: 0 additions & 2 deletions packages/dashmate/templates/dynamic-compose.yml.dot
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,6 @@ services:
{{? it.platform.dapi.rsDapi.metrics.enabled }}
ports:
- {{=it.platform.dapi.rsDapi.metrics.host}}:{{=it.platform.dapi.rsDapi.metrics.port}}:{{=it.platform.dapi.rsDapi.metrics.port}}
{{??}}
ports: []
{{?}}
{{?}}

Expand Down
7 changes: 4 additions & 3 deletions packages/dashmate/templates/platform/gateway/envoy.yaml.dot
Original file line number Diff line number Diff line change
Expand Up @@ -527,7 +527,7 @@ static_resources:
address: gateway_rate_limiter
port_value: 8081
{{?}}
{{? it.platform.gateway.metrics.enabled && it.platform.gateway.admin.enabled }}
{{? it.platform.gateway.metrics.enabled }}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do you remove this condition?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Metrics without admin don't work.

Assuming the intention of an user who enabled metrics is to get them really working ;), I've changed logic so that when metrics are enabled, always enable admin automatically but don't expose it to public.

See line 546-550 below. When admin is not enabled, adminHost will be 127.0.0.1, what means it will listen on loopback of docker container (not docker host).

- name: admin
connect_timeout: 0.25s
type: STATIC
Expand All @@ -542,11 +542,12 @@ static_resources:
port_value: 9901
{{?}}

{{? it.platform.gateway.admin.enabled }}
{{? it.platform.gateway.admin.enabled || it.platform.gateway.metrics.enabled }}
{{ adminHost = it.platform.gateway.admin.enabled ? '0.0.0.0' : '127.0.0.1'; }}
admin:
address:
socket_address:
address: 0.0.0.0 # For docker container only. Must be a local/private interface.
address: {{= adminHost }} # defaults to loopback when not explicitly enabled
port_value: 9901
{{?}}

Expand Down
44 changes: 44 additions & 0 deletions packages/dashmate/test/unit/templates/dynamicCompose.spec.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import HomeDir from '../../../src/config/HomeDir.js';
import getBaseConfigFactory from '../../../configs/defaults/getBaseConfigFactory.js';
import renderTemplateFactory from '../../../src/templates/renderTemplateFactory.js';
import renderServiceTemplatesFactory from '../../../src/templates/renderServiceTemplatesFactory.js';

function getRsDapiBlock(dynamicComposeContent) {
const match = dynamicComposeContent.match(/rs_dapi:\n((?: {2}.*\n)+)/);
return match ? match[1] : '';
}

describe('dynamic compose template', () => {
let getBaseConfig;
let renderServiceTemplates;

beforeEach(() => {
getBaseConfig = getBaseConfigFactory(HomeDir.createTemp());
const renderTemplate = renderTemplateFactory();
renderServiceTemplates = renderServiceTemplatesFactory(renderTemplate);
});

it('should not publish metrics port when rs-dapi metrics are disabled', () => {
const config = getBaseConfig();

const renderedConfigs = renderServiceTemplates(config);
const rsDapiBlock = getRsDapiBlock(renderedConfigs['dynamic-compose.yml']);

expect(rsDapiBlock).to.not.include('ports:');
expect(rsDapiBlock).to.not.include(':0');
});

it('should publish metrics port when rs-dapi metrics are enabled', () => {
const config = getBaseConfig();

config.set('platform.dapi.rsDapi.metrics.enabled', true);
config.set('platform.dapi.rsDapi.metrics.port', 29091);
config.set('platform.dapi.rsDapi.metrics.host', '127.0.0.1');

const renderedConfigs = renderServiceTemplates(config);
const rsDapiBlock = getRsDapiBlock(renderedConfigs['dynamic-compose.yml']);

expect(rsDapiBlock).to.include('ports:\n - 127.0.0.1:29091:29091');
expect(rsDapiBlock).to.include('- 29091');
});
});
24 changes: 24 additions & 0 deletions packages/dashmate/test/unit/templates/envoyTemplate.spec.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import getBaseConfigFactory from '../../../configs/defaults/getBaseConfigFactory.js';
import HomeDir from '../../../src/config/HomeDir.js';
import renderServiceTemplatesFactory from '../../../src/templates/renderServiceTemplatesFactory.js';
import renderTemplateFactory from '../../../src/templates/renderTemplateFactory.js';

describe('envoy template', () => {
it('should render admin interface when metrics are enabled even if admin is disabled', () => {
const getBaseConfig = getBaseConfigFactory(HomeDir.createTemp());
const config = getBaseConfig();

config.set('platform.gateway.metrics.enabled', true);
config.set('platform.gateway.admin.enabled', false);

const renderTemplate = renderTemplateFactory();
const renderServiceTemplates = renderServiceTemplatesFactory(renderTemplate);
const renderedConfigs = renderServiceTemplates(config);

const envoyConfig = renderedConfigs['platform/gateway/envoy.yaml'];

expect(envoyConfig).to.include('cluster_name: admin');
expect(envoyConfig).to.include('address: 127.0.0.1');
expect(envoyConfig).to.include('port_value: 9901');
});
});
Loading