diff --git a/.yarn/cache/fsevents-patch-19706e7e35-10.zip b/.yarn/cache/fsevents-patch-19706e7e35-10.zip deleted file mode 100644 index aff1ab12ce5..00000000000 Binary files a/.yarn/cache/fsevents-patch-19706e7e35-10.zip and /dev/null differ diff --git a/packages/dashmate/docker-compose.rate_limiter.metrics.yml b/packages/dashmate/docker-compose.rate_limiter.metrics.yml index 8d0c3a85de6..62fb25dae32 100644 --- a/packages/dashmate/docker-compose.rate_limiter.metrics.yml +++ b/packages/dashmate/docker-compose.rate_limiter.metrics.yml @@ -12,6 +12,10 @@ services: image: ${PLATFORM_GATEWAY_RATE_LIMITER_METRICS_DOCKER_IMAGE:?err} labels: org.dashmate.service.title: "Gateway rate limiter metrics exporter" + org.dashmate.config.name: "${CONFIG_NAME:?err}" + prometheus.io/scrape: "${PLATFORM_GATEWAY_RATE_LIMITER_METRICS_ENABLED:-false}" + prometheus.io/port: "${PLATFORM_GATEWAY_RATE_LIMITER_METRICS_PORT:?err}" + prometheus.io/path: "/metrics" restart: unless-stopped logging: *default-logging entrypoint: /bin/statsd_exporter diff --git a/packages/dashmate/docker-compose.yml b/packages/dashmate/docker-compose.yml index 5ca9175d830..33d21c4cd8a 100644 --- a/packages/dashmate/docker-compose.yml +++ b/packages/dashmate/docker-compose.yml @@ -54,6 +54,10 @@ services: image: ${PLATFORM_DRIVE_ABCI_DOCKER_IMAGE:?err} labels: org.dashmate.service.title: "Drive ABCI" + org.dashmate.config.name: "${CONFIG_NAME:?err}" + prometheus.io/scrape: "${PLATFORM_DRIVE_ABCI_METRICS_ENABLED:-false}" + prometheus.io/port: "${PLATFORM_DRIVE_ABCI_METRICS_PORT:?err}" + prometheus.io/path: "/metrics" restart: unless-stopped logging: *default-logging volumes: @@ -111,6 +115,10 @@ services: image: ${PLATFORM_DRIVE_TENDERDASH_DOCKER_IMAGE:?err} labels: org.dashmate.service.title: "Drive Tenderdash" + org.dashmate.config.name: "${CONFIG_NAME:?err}" + prometheus.io/scrape: "${PLATFORM_DRIVE_TENDERDASH_METRICS_ENABLED:-false}" + prometheus.io/port: "${PLATFORM_DRIVE_TENDERDASH_METRICS_PORT:?err}" + prometheus.io/path: "/metrics" restart: unless-stopped logging: *default-logging depends_on: @@ -196,6 +204,10 @@ services: image: ${PLATFORM_DAPI_RS_DAPI_DOCKER_IMAGE:?err} labels: org.dashmate.service.title: "rs-dapi (Rust DAPI)" + org.dashmate.config.name: "${CONFIG_NAME:?err}" + prometheus.io/scrape: "${PLATFORM_DAPI_RS_DAPI_METRICS_ENABLED:-false}" + prometheus.io/port: "${PLATFORM_DAPI_RS_DAPI_METRICS_PORT:?err}" + prometheus.io/path: "/metrics" restart: unless-stopped logging: *default-logging deploy: @@ -236,6 +248,10 @@ services: image: ${PLATFORM_GATEWAY_DOCKER_IMAGE:?err} labels: org.dashmate.service.title: "Gateway" + org.dashmate.config.name: "${CONFIG_NAME:?err}" + prometheus.io/scrape: "${PLATFORM_GATEWAY_METRICS_ENABLED:-false}" + prometheus.io/port: "${PLATFORM_GATEWAY_METRICS_PORT:?err}" + prometheus.io/path: "/metrics" restart: unless-stopped logging: *default-logging ports: diff --git a/packages/dashmate/docs/config/gateway.md b/packages/dashmate/docs/config/gateway.md index ef782969ed9..7e6c86111fd 100644 --- a/packages/dashmate/docs/config/gateway.md +++ b/packages/dashmate/docs/config/gateway.md @@ -54,7 +54,7 @@ These settings control the metrics endpoint for monitoring the Gateway: | `platform.gateway.metrics.port` | Port for metrics server | `9090` | `9091` | Metrics provide performance and health information about the Gateway service. -Admin must be enabled to access the metrics endpoint. +Dashmate automatically enables the Envoy admin endpoint whenever metrics are enabled so that the Prometheus listener can proxy `/stats/prometheus`; if admin itself is still disabled, the listener is not exposed outside of Docker. ## Admin diff --git a/packages/dashmate/docs/prometheus/README.md b/packages/dashmate/docs/prometheus/README.md new file mode 100644 index 00000000000..d40458c0301 --- /dev/null +++ b/packages/dashmate/docs/prometheus/README.md @@ -0,0 +1,37 @@ +# Prometheus Configuration + +This directory contains a minimal Prometheus stack for scraping metrics from a Dashmate-managed Platform network. It is meant for local debugging and pairs with the metric endpoints that Dashmate exposes when you enable them in your node configuration. + +## Files +- `docs/prometheus/docker-compose.yml` brings up two containers: a read-only Docker Socket Proxy that exposes `tcp://127.0.0.1:2375`, and Prometheus itself listening on `:9080`. +- `docs/prometheus/prometheus.yml` configures Prometheus to use Docker service discovery against the proxy and scrape every container that carries Dashmate’s `prometheus.io/*` labels. + +## Prepare Dashmate +1. Enable metrics on the services you want Prometheus to monitor (examples): + - `dashmate config set platform.drive.abci.metrics.enabled true` + - `dashmate config set platform.drive.tenderdash.metrics.enabled true` + - `dashmate config set platform.dapi.rsDapi.metrics.enabled true` + - `dashmate config set platform.gateway.metrics.enabled true` + - `dashmate config set platform.gateway.rateLimiter.metrics.enabled true` +2. Restart your network so the containers are recreated with the Prometheus labels: `yarn restart`. + +The labels Dashmate adds map cleanly onto the Prometheus discovery config: +- `prometheus.io/scrape=true` enables scraping. +- `prometheus.io/path` overrides the metrics path (defaults to `/metrics`). +- `prometheus.io/port` is rewritten to `127.0.0.1:` so scraping stays on the host network. +- `org_dashmate_service_title` and `org_dashmate_config_name` become `service` and `config` labels on each time series. + +## Run Prometheus +```bash +docker compose -f docs/prometheus/docker-compose.yml up -d +``` + +Prometheus stores data in the `prometheus-data` volume and exposes its UI at `http://127.0.0.1:9080`. Reload the configuration after any edits with: +```bash +curl -X POST http://127.0.0.1:9080/-/reload +``` + +### Customisation Tips +- Adjust `scrape_interval` or add more `scrape_configs` in `prometheus.yml` as needed. +- To scrape additional containers, attach the same `prometheus.io/*` labels to them, or add dedicated jobs to the configuration. +- If your Docker daemon is not local, update the proxy service or mount a different socket before starting the stack. diff --git a/packages/dashmate/docs/prometheus/docker-compose.yml b/packages/dashmate/docs/prometheus/docker-compose.yml new file mode 100644 index 00000000000..84739f3d5fd --- /dev/null +++ b/packages/dashmate/docs/prometheus/docker-compose.yml @@ -0,0 +1,33 @@ +--- + +services: + docker_socket_proxy: + image: tecnativa/docker-socket-proxy:latest + container_name: prometheus-docker-socket-proxy + restart: unless-stopped + environment: + - CONTAINERS=1 + - NETWORKS=1 + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + ports: + - "127.0.0.1:2375:2375" + + prometheus: + image: prom/prometheus:latest + container_name: prometheus-rsdapi + network_mode: host + depends_on: + - docker_socket_proxy + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --web.enable-lifecycle + - --web.listen-address=:9080 + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + restart: unless-stopped + +volumes: + prometheus-data: diff --git a/packages/dashmate/docs/prometheus/prometheus.yml b/packages/dashmate/docs/prometheus/prometheus.yml new file mode 100644 index 00000000000..70f1a0f0497 --- /dev/null +++ b/packages/dashmate/docs/prometheus/prometheus.yml @@ -0,0 +1,30 @@ +--- + +global: + scrape_interval: 15s + scrape_timeout: 10s + +scrape_configs: + - job_name: dashmate + metrics_path: /metrics + docker_sd_configs: + - host: tcp://127.0.0.1:2375 + refresh_interval: 15s + relabel_configs: + - source_labels: [__meta_docker_container_label_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_docker_container_label_prometheus_io_path] + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__meta_docker_container_label_prometheus_io_port] + action: replace + regex: (.+) + replacement: 127.0.0.1:$1 + target_label: __address__ + - source_labels: [__meta_docker_container_label_org_dashmate_service_title] + target_label: service + regex: (.+) + - source_labels: [__meta_docker_container_label_org_dashmate_config_name] + target_label: config + regex: (.+) diff --git a/packages/dashmate/docs/services/gateway.md b/packages/dashmate/docs/services/gateway.md index 499c18cc492..bf995692a8b 100644 --- a/packages/dashmate/docs/services/gateway.md +++ b/packages/dashmate/docs/services/gateway.md @@ -261,7 +261,7 @@ Metrics include: - Host: `platform.gateway.metrics.host` (default: 127.0.0.1) - Port: `platform.gateway.metrics.port` (default: 9090) -**Note:** admin interface must be enabled too. +**Note:** Dashmate automatically enables the Envoy admin endpoint whenever metrics are turned on so the Prometheus listener can proxy `/stats/prometheus`. If the admin service remains disabled, this socket is not exposed outside Docker; once you explicitly enable admin, it uses the host you configure. ### Security Considerations diff --git a/packages/dashmate/docs/services/index.md b/packages/dashmate/docs/services/index.md index ec16757e948..6f396da4e18 100644 --- a/packages/dashmate/docs/services/index.md +++ b/packages/dashmate/docs/services/index.md @@ -119,3 +119,6 @@ Most services provide metrics endpoints: - Rate Limiter: Rate limiting metrics These can be integrated with monitoring systems like Prometheus and Grafana. +All containers that expose Prometheus metrics also advertise the Docker label `org.dashmate.config.name`, which Dashmate sets to the active config name. When you run multiple nodes on the same host, you can use that label in Prometheus relabeling rules or dashboards to distinguish each instance. + +You can find example Prometheus config for local devnet monitoring in [../prometheus/](../prometheus/README.md). \ No newline at end of file diff --git a/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js b/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js index 32c4355bdec..92458dd62fb 100644 --- a/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js +++ b/packages/dashmate/src/doctor/analyse/analyseConfigFactory.js @@ -16,17 +16,6 @@ export default function analyseConfigFactory() { const problems = []; if (config?.get('platform.enable')) { - // Gateway admin is disabled while metrics are enabled - if (config.get('platform.gateway.metrics.enabled') && !config.get('platform.gateway.admin.enabled')) { - const problem = new Problem( - 'Gateway admin is disabled while metrics are enabled', - chalk`Please enable gateway admin: {bold.cyanBright dashmate config set platform.gateway.admin.enabled true}`, - SEVERITY.HIGH, - ); - - problems.push(problem); - } - // Platform Node ID const masternodeStatus = samples.getServiceInfo('core', 'masternodeStatus'); const platformNodeId = masternodeStatus?.dmnState?.platformNodeId; diff --git a/packages/dashmate/templates/dynamic-compose.yml.dot b/packages/dashmate/templates/dynamic-compose.yml.dot index 9db8a26827c..b0e723cfac0 100644 --- a/packages/dashmate/templates/dynamic-compose.yml.dot +++ b/packages/dashmate/templates/dynamic-compose.yml.dot @@ -39,8 +39,6 @@ services: {{? it.platform.dapi.rsDapi.metrics.enabled }} ports: - {{=it.platform.dapi.rsDapi.metrics.host}}:{{=it.platform.dapi.rsDapi.metrics.port}}:{{=it.platform.dapi.rsDapi.metrics.port}} - {{??}} - ports: [] {{?}} {{?}} diff --git a/packages/dashmate/templates/platform/gateway/envoy.yaml.dot b/packages/dashmate/templates/platform/gateway/envoy.yaml.dot index e9ceb7953ec..df5941f86f0 100644 --- a/packages/dashmate/templates/platform/gateway/envoy.yaml.dot +++ b/packages/dashmate/templates/platform/gateway/envoy.yaml.dot @@ -527,7 +527,7 @@ static_resources: address: gateway_rate_limiter port_value: 8081 {{?}} - {{? it.platform.gateway.metrics.enabled && it.platform.gateway.admin.enabled }} + {{? it.platform.gateway.metrics.enabled }} - name: admin connect_timeout: 0.25s type: STATIC @@ -542,11 +542,12 @@ static_resources: port_value: 9901 {{?}} -{{? it.platform.gateway.admin.enabled }} +{{? it.platform.gateway.admin.enabled || it.platform.gateway.metrics.enabled }} +{{ adminHost = it.platform.gateway.admin.enabled ? '0.0.0.0' : '127.0.0.1'; }} admin: address: socket_address: - address: 0.0.0.0 # For docker container only. Must be a local/private interface. + address: {{= adminHost }} # defaults to loopback when not explicitly enabled port_value: 9901 {{?}} diff --git a/packages/dashmate/test/unit/templates/dynamicCompose.spec.js b/packages/dashmate/test/unit/templates/dynamicCompose.spec.js new file mode 100644 index 00000000000..610cd801b08 --- /dev/null +++ b/packages/dashmate/test/unit/templates/dynamicCompose.spec.js @@ -0,0 +1,44 @@ +import HomeDir from '../../../src/config/HomeDir.js'; +import getBaseConfigFactory from '../../../configs/defaults/getBaseConfigFactory.js'; +import renderTemplateFactory from '../../../src/templates/renderTemplateFactory.js'; +import renderServiceTemplatesFactory from '../../../src/templates/renderServiceTemplatesFactory.js'; + +function getRsDapiBlock(dynamicComposeContent) { + const match = dynamicComposeContent.match(/rs_dapi:\n((?: {2}.*\n)+)/); + return match ? match[1] : ''; +} + +describe('dynamic compose template', () => { + let getBaseConfig; + let renderServiceTemplates; + + beforeEach(() => { + getBaseConfig = getBaseConfigFactory(HomeDir.createTemp()); + const renderTemplate = renderTemplateFactory(); + renderServiceTemplates = renderServiceTemplatesFactory(renderTemplate); + }); + + it('should not publish metrics port when rs-dapi metrics are disabled', () => { + const config = getBaseConfig(); + + const renderedConfigs = renderServiceTemplates(config); + const rsDapiBlock = getRsDapiBlock(renderedConfigs['dynamic-compose.yml']); + + expect(rsDapiBlock).to.not.include('ports:'); + expect(rsDapiBlock).to.not.include(':0'); + }); + + it('should publish metrics port when rs-dapi metrics are enabled', () => { + const config = getBaseConfig(); + + config.set('platform.dapi.rsDapi.metrics.enabled', true); + config.set('platform.dapi.rsDapi.metrics.port', 29091); + config.set('platform.dapi.rsDapi.metrics.host', '127.0.0.1'); + + const renderedConfigs = renderServiceTemplates(config); + const rsDapiBlock = getRsDapiBlock(renderedConfigs['dynamic-compose.yml']); + + expect(rsDapiBlock).to.include('ports:\n - 127.0.0.1:29091:29091'); + expect(rsDapiBlock).to.include('- 29091'); + }); +}); diff --git a/packages/dashmate/test/unit/templates/envoyTemplate.spec.js b/packages/dashmate/test/unit/templates/envoyTemplate.spec.js new file mode 100644 index 00000000000..e654d62a6f7 --- /dev/null +++ b/packages/dashmate/test/unit/templates/envoyTemplate.spec.js @@ -0,0 +1,24 @@ +import getBaseConfigFactory from '../../../configs/defaults/getBaseConfigFactory.js'; +import HomeDir from '../../../src/config/HomeDir.js'; +import renderServiceTemplatesFactory from '../../../src/templates/renderServiceTemplatesFactory.js'; +import renderTemplateFactory from '../../../src/templates/renderTemplateFactory.js'; + +describe('envoy template', () => { + it('should render admin interface when metrics are enabled even if admin is disabled', () => { + const getBaseConfig = getBaseConfigFactory(HomeDir.createTemp()); + const config = getBaseConfig(); + + config.set('platform.gateway.metrics.enabled', true); + config.set('platform.gateway.admin.enabled', false); + + const renderTemplate = renderTemplateFactory(); + const renderServiceTemplates = renderServiceTemplatesFactory(renderTemplate); + const renderedConfigs = renderServiceTemplates(config); + + const envoyConfig = renderedConfigs['platform/gateway/envoy.yaml']; + + expect(envoyConfig).to.include('cluster_name: admin'); + expect(envoyConfig).to.include('address: 127.0.0.1'); + expect(envoyConfig).to.include('port_value: 9901'); + }); +});