From fb368c71e70581019bdfe6bac30bcfbd65e900da Mon Sep 17 00:00:00 2001 From: Amin Rezaei Date: Thu, 25 Sep 2025 16:51:57 -0400 Subject: [PATCH 1/5] feat: grafana automatic dashboard and auth set --- benchmark/Grafana-dashboard.json | 82 ++++++++-------------- benchmark/setup-scripts/install-grafana.sh | 38 ++++++++++ 2 files changed, 69 insertions(+), 51 deletions(-) diff --git a/benchmark/Grafana-dashboard.json b/benchmark/Grafana-dashboard.json index 718216cf75..195df7b241 100644 --- a/benchmark/Grafana-dashboard.json +++ b/benchmark/Grafana-dashboard.json @@ -1,35 +1,4 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "11.5.1" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], "annotations": { "list": [ { @@ -68,7 +37,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -154,7 +123,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", @@ -174,7 +143,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -261,7 +230,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", @@ -277,7 +246,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "editorMode": "code", "expr": "100 * sum(increase(firewood_cache_freelist{type=\"hit\"}[$__rate_interval])) by (name) / sum(increase(firewood_cache_freelist[$__rate_interval])) by (name)", @@ -307,7 +276,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -394,7 +363,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" } } ], @@ -404,7 +373,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -491,7 +460,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "editorMode": "code", "expr": "irate(firewood_remove[$__rate_interval])", @@ -504,7 +473,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(irate(firewood_insert{merkle!=\"update\"}[$__rate_interval]))", @@ -517,7 +486,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "editorMode": "code", "expr": "irate(firewood_insert{merkle=\"update\"}[$__rate_interval])", @@ -547,7 +516,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -629,7 +598,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", @@ -649,7 +618,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -732,7 +701,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "editorMode": "code", "expr": "node_memory_Dirty_bytes", @@ -748,7 +717,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -839,7 +808,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" } } ], @@ -849,7 +818,7 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" }, "fieldConfig": { "defaults": { @@ -946,7 +915,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "${datasource}" } } ], @@ -958,7 +927,18 @@ "schemaVersion": 40, "tags": [], "templating": { - "list": [] + "list": [ + { + "includeAll": false, + "label": "Data Source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] }, "time": { "from": "now-1h", diff --git a/benchmark/setup-scripts/install-grafana.sh b/benchmark/setup-scripts/install-grafana.sh index 36e05c81e0..33b5bd9b02 100644 --- a/benchmark/setup-scripts/install-grafana.sh +++ b/benchmark/setup-scripts/install-grafana.sh @@ -46,6 +46,44 @@ if ! grep -q '^http_port = 80$' /etc/grafana/grafana.ini; then perl -pi -e 's/^;?http_port = .*/http_port = 80/' /etc/grafana/grafana.ini fi +# configure username and password +# TODO(amin): auto-generate some password for more security +# TODO(amin): another possible option here is enabling google oauth, and this could give access +# to anyone within our org emails +sed -i -E "s|^;?\s*admin_user\s*=.*|admin_user = admin|" /etc/grafana/grafana.ini +sed -i -E "s|^;?\s*admin_password\s*=.*|admin_password = firewoodisfast|" /etc/grafana/grafana.ini + +# provision data source and dashboards +{ + echo 'apiVersion: 1' + echo 'datasources:' + echo ' - name: Prometheus' + echo ' type: prometheus' + echo ' access: proxy' + echo ' orgId: 1' + echo ' url: http://localhost:9090' + echo ' isDefault: true' + echo ' editable: true' +} | sudo tee /etc/grafana/provisioning/datasources/prometheus.yml >/dev/null + +{ + echo 'apiVersion: 1' + echo 'providers:' + echo " - name: 'files'" + echo ' orgId: 1' + echo " folder: 'Provisioned'" + echo ' type: file' + echo ' disableDeletion: false' + echo ' editable: true' + echo ' options:' + echo ' path: /var/lib/grafana/dashboards' +} | sudo tee /etc/grafana/provisioning/dashboards/dashboards.yaml >/dev/null + +# add firewood's dashboard and also node exporter full +sudo mkdir -p /var/lib/grafana/dashboards +sudo wget -O /var/lib/grafana/dashboards/firewood.json https://github.com/ava-labs/firewood/raw/refs/heads/main/benchmark/Grafana-dashboard.json +sudo wget -O /var/lib/grafana/dashboards/node_exporter_full.json https://grafana.com/api/dashboards/1860/revisions/latest/download + # configure prometheus to scrape firewood if ! grep -q '^ - job_name: firewood$' /etc/prometheus/prometheus.yml; then cat >> /etc/prometheus/prometheus.yml < Date: Thu, 25 Sep 2025 16:58:14 -0400 Subject: [PATCH 2/5] feat: add folder structure and a note --- benchmark/setup-scripts/install-grafana.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmark/setup-scripts/install-grafana.sh b/benchmark/setup-scripts/install-grafana.sh index 33b5bd9b02..e29dddc577 100644 --- a/benchmark/setup-scripts/install-grafana.sh +++ b/benchmark/setup-scripts/install-grafana.sh @@ -77,10 +77,12 @@ sed -i -E "s|^;?\s*admin_password\s*=.*|admin_password = firewoodisfast|" /etc/g echo ' editable: true' echo ' options:' echo ' path: /var/lib/grafana/dashboards' + echo ' foldersFromFilesStructure: true' } | sudo tee /etc/grafana/provisioning/dashboards/dashboards.yaml >/dev/null # add firewood's dashboard and also node exporter full sudo mkdir -p /var/lib/grafana/dashboards +# TODO(amin): replace this with script dir sudo wget -O /var/lib/grafana/dashboards/firewood.json https://github.com/ava-labs/firewood/raw/refs/heads/main/benchmark/Grafana-dashboard.json sudo wget -O /var/lib/grafana/dashboards/node_exporter_full.json https://grafana.com/api/dashboards/1860/revisions/latest/download From f47ed5dda96d06b2376e46e8d9d9f514f2329acc Mon Sep 17 00:00:00 2001 From: Amin Rezaei Date: Thu, 25 Sep 2025 17:26:37 -0400 Subject: [PATCH 3/5] review: apply suggestions --- benchmark/setup-scripts/install-grafana.sh | 54 +++++++++++----------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/benchmark/setup-scripts/install-grafana.sh b/benchmark/setup-scripts/install-grafana.sh index e29dddc577..829fd22d2c 100644 --- a/benchmark/setup-scripts/install-grafana.sh +++ b/benchmark/setup-scripts/install-grafana.sh @@ -54,37 +54,37 @@ sed -i -E "s|^;?\s*admin_user\s*=.*|admin_user = admin|" /etc/grafana/grafana.in sed -i -E "s|^;?\s*admin_password\s*=.*|admin_password = firewoodisfast|" /etc/grafana/grafana.ini # provision data source and dashboards -{ - echo 'apiVersion: 1' - echo 'datasources:' - echo ' - name: Prometheus' - echo ' type: prometheus' - echo ' access: proxy' - echo ' orgId: 1' - echo ' url: http://localhost:9090' - echo ' isDefault: true' - echo ' editable: true' -} | sudo tee /etc/grafana/provisioning/datasources/prometheus.yml >/dev/null +cat > /etc/grafana/provisioning/datasources/prometheus.yml </dev/null +cat > /etc/grafana/provisioning/dashboards/dashboards.yaml < Date: Thu, 9 Oct 2025 18:27:02 -0400 Subject: [PATCH 4/5] review: Ron's suggestions --- benchmark/Grafana-dashboard.json | 5 +++++ benchmark/README.md | 18 ++++++------------ benchmark/setup-scripts/install-grafana.sh | 6 +++--- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/benchmark/Grafana-dashboard.json b/benchmark/Grafana-dashboard.json index 195df7b241..3ea5ce585f 100644 --- a/benchmark/Grafana-dashboard.json +++ b/benchmark/Grafana-dashboard.json @@ -929,6 +929,11 @@ "templating": { "list": [ { + "current": { + "selected": false, + "text": "Prometheus", + "value": "prometheus" + }, "includeAll": false, "label": "Data Source", "name": "datasource", diff --git a/benchmark/README.md b/benchmark/README.md index 75d7ec8002..c98ce0d58c 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -128,18 +128,12 @@ If you want to install grafana and prometheus on an AWS host (using Ubuntu as a 8. Launch the instance 3. ssh ubuntu@AWS-IP 4. Run the scripts as described above, including the grafana installation. -5. Log in to grafana on - a. username: admin, password: admin -6. When prompted, change the password (`firewood_is_fast`) -7. On the left panel, click "Data Sources" - a. Select "Prometheus" - b. For the URL, use - c. click "Save and test" -8. On the left panel, click Dashboards - a. On the right top pulldown, click New->Import - b. Import the dashboard from the Grafana-dashboard.json file - c. Set the data source to Prometheus -9. \[optional] Install a stock dashboard from [grafana](https://grafana.com/grafana/dashboards/1860-node-exporter-full/) +5. Log in to Grafana at + a. Username: `admin`, password: `firewood_is_fast` +6. A Prometheus data source, the Firewood dashboard, and a [system statistics dashboard](https://grafana.com/grafana/dashboards/1860-node-exporter-full/) are preconfigured and ready to use. + +### Updating the dashboard +If you want to update the dashboard and commit it, do not enable "Export for sharing externally" when exporting. These dashboards are provisioned and reference the default Prometheus data source, with the option to pick from available data sources. ## Usage diff --git a/benchmark/setup-scripts/install-grafana.sh b/benchmark/setup-scripts/install-grafana.sh index 829fd22d2c..f3c5157aad 100644 --- a/benchmark/setup-scripts/install-grafana.sh +++ b/benchmark/setup-scripts/install-grafana.sh @@ -6,8 +6,8 @@ if ! [ -d /etc/apt/keyrings ]; then mkdir -p /etc/apt/keyrings/ fi if ! [ -f /etc/apt/keyrings/grafana.gpg ]; then - wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/grafana.gpg > /dev/null - echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list + wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor > /etc/apt/keyrings/grafana.gpg + echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" >> /etc/apt/sources.list.d/grafana.list fi apt-get update @@ -51,7 +51,7 @@ fi # TODO(amin): another possible option here is enabling google oauth, and this could give access # to anyone within our org emails sed -i -E "s|^;?\s*admin_user\s*=.*|admin_user = admin|" /etc/grafana/grafana.ini -sed -i -E "s|^;?\s*admin_password\s*=.*|admin_password = firewoodisfast|" /etc/grafana/grafana.ini +sed -i -E "s|^;?\s*admin_password\s*=.*|admin_password = firewood_is_fast|" /etc/grafana/grafana.ini # provision data source and dashboards cat > /etc/grafana/provisioning/datasources/prometheus.yml < Date: Thu, 9 Oct 2025 22:09:40 -0400 Subject: [PATCH 5/5] fix: doc lint --- benchmark/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/README.md b/benchmark/README.md index c98ce0d58c..453775c253 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -133,6 +133,7 @@ If you want to install grafana and prometheus on an AWS host (using Ubuntu as a 6. A Prometheus data source, the Firewood dashboard, and a [system statistics dashboard](https://grafana.com/grafana/dashboards/1860-node-exporter-full/) are preconfigured and ready to use. ### Updating the dashboard + If you want to update the dashboard and commit it, do not enable "Export for sharing externally" when exporting. These dashboards are provisioned and reference the default Prometheus data source, with the option to pick from available data sources. ## Usage