From aff0ebf704b37cd6cd8b2a7a2696e923b39b738f Mon Sep 17 00:00:00 2001 From: Cyber-SiKu Date: Wed, 25 Oct 2023 14:29:19 +0800 Subject: [PATCH 1/6] [feat]add cluster usage Signed-off-by: Cyber-SiKu --- .../provisioning/dashboards/cluster.json | 1014 +++++++++++++++++ curvefs/src/mds/BUILD | 1 + curvefs/src/mds/fs_manager.cpp | 19 +- curvefs/src/mds/fs_storage.cpp | 4 + curvefs/src/mds/metric/fs_metric.cpp | 49 +- curvefs/src/mds/metric/fs_metric.h | 14 +- curvefs/src/mds/metric/metric.cpp | 6 + curvefs/src/mds/metric/metric.h | 21 +- curvefs/test/mds/mds_metric_test.cpp | 48 + 9 files changed, 1160 insertions(+), 16 deletions(-) create mode 100644 curvefs/monitor/grafana/provisioning/dashboards/cluster.json create mode 100644 curvefs/test/mds/mds_metric_test.cpp diff --git a/curvefs/monitor/grafana/provisioning/dashboards/cluster.json b/curvefs/monitor/grafana/provisioning/dashboards/cluster.json new file mode 100644 index 0000000000..2dcae0ec8e --- /dev/null +++ b/curvefs/monitor/grafana/provisioning/dashboards/cluster.json @@ -0,0 +1,1014 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Curvefs cluster", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 7, + "interval": "1s", + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "{__name__=~\"fs_usage_info_fs_[[fsName:regex]]_used\", job=\"mds\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "fs used", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "cluster total used", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 5, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum({__name__=~\"fs_usage_info_fs_[[fsName:regex]]_used\",job=\"mds\"})", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "cluster total used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 4, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "{__name__=~\"fs_usage_info_fs_[[fsName:regex]]_used\", job=\"mds\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "fs used", + "type": "timeseries" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 8, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"topology_fs_id_[[fsId:regex]]_.*inode_num\",job=\"mds\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "inode_num_by_type", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [ + { + "id": "sortBy", + "options": {} + } + ], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "hiddenSeries": false, + "id": 13, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum({__name__=~\"topology_fs_id_[0-9]*_inode_num\",job=\"mds\"})", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cluster inode", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [ + { + "id": "sortBy", + "options": {} + } + ], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 10, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum({__name__=~\"topology_.*_type_file_inode_num\",job=\"mds\"})", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cluster file inode", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [ + { + "id": "sortBy", + "options": {} + } + ], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "hiddenSeries": false, + "id": 9, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum({__name__=~\"topology_.*_type_directory_inode_num\",job=\"mds\"})", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cluster directory inode", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [ + { + "id": "sortBy", + "options": {} + } + ], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 12, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum({__name__=~\"topology_.*_type_sym_link_inode_num\",job=\"mds\"})", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cluster sym_link inode", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [ + { + "id": "sortBy", + "options": {} + } + ], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "hiddenSeries": false, + "id": 11, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum({__name__=~\"topology_.*_type_s3_inode_num\",job=\"mds\"})", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "cluster s3 inode", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [ + { + "id": "sortBy", + "options": {} + } + ], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "fs usage", + "type": "row" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "{__name__=~\"fs_usage_info_fs_.*_used\",job=\"mds\"}", + "hide": 0, + "includeAll": true, + "label": "fsName", + "multi": false, + "name": "fsName", + "options": [], + "query": { + "query": "{__name__=~\"fs_usage_info_fs_.*_used\",job=\"mds\"}", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/fs_usage_info_fs_(.*)_used.*/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "{__name__=~\"topology_fs_id_[0-9]*_inode_num\",job=\"mds\"}", + "hide": 0, + "includeAll": true, + "label": "fsId", + "multi": false, + "name": "fsId", + "options": [], + "query": { + "query": "{__name__=~\"topology_fs_id_[0-9]*_inode_num\",job=\"mds\"}", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/topology_fs_id_([0-9]*)_inode_num.*/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "cluster", + "uid": "c57406c1-8262-4bf8-85c4-f4f65f336f96", + "version": 1, + "weekStart": "" +} diff --git a/curvefs/src/mds/BUILD b/curvefs/src/mds/BUILD index bf46e8d63c..4a26b34bae 100644 --- a/curvefs/src/mds/BUILD +++ b/curvefs/src/mds/BUILD @@ -33,6 +33,7 @@ cc_library( "//curvefs/src/common:curvefs_common", "//curvefs/src/mds/codec:fs_mds_codec", "//curvefs/src/mds/idgenerator:fs_mds_idgenerator", + "//curvefs/src/mds/metric:fs_mds_metric", "//src/common:curve_common", "//src/common/concurrent:curve_concurrent", "//src/idgenerator", diff --git a/curvefs/src/mds/fs_manager.cpp b/curvefs/src/mds/fs_manager.cpp index 1fafbb1a1b..6ab8090946 100644 --- a/curvefs/src/mds/fs_manager.cpp +++ b/curvefs/src/mds/fs_manager.cpp @@ -592,6 +592,7 @@ FSStatusCode FsManager::DeleteFs(const std::string& fsName) { << ", ret = " << FSStatusCode_Name(ret); // do not abort deletion } + FsMetric::GetInstance().DeleteFsUsage(fsName); } return FSStatusCode::OK; @@ -832,7 +833,11 @@ FSStatusCode FsManager::UpdateFsInfo( wrapper.SetCapacity(req->capacity()); } - return fsStorage_->Update(wrapper); + ret = fsStorage_->Update(wrapper); + if (ret == FSStatusCode::OK) { + FsMetric::GetInstance().SetCapacity(fsName, req->capacity()); + } + return ret; } int FsManager::IsExactlySameOrCreateUnComplete(const std::string& fsName, @@ -904,7 +909,11 @@ FSStatusCode FsManager::UpdateFsUsedBytes( auto ret = fsStorage_->GetFsUsage(fsName, &usage, true); if (ret == FSStatusCode::NOT_FOUND) { usage.set_usedbytes(deltaBytes); - return fsStorage_->SetFsUsage(fsName, usage); + ret = fsStorage_->SetFsUsage(fsName, usage); + if (ret == FSStatusCode::OK) { + FsMetric::GetInstance().SetFsUsage(fsName, usage); + } + return ret; } if (ret != FSStatusCode::OK) { @@ -918,7 +927,11 @@ FSStatusCode FsManager::UpdateFsUsedBytes( } else { usage.set_usedbytes(usage.usedbytes() + deltaBytes); } - return fsStorage_->SetFsUsage(fsName, usage); + ret = fsStorage_->SetFsUsage(fsName, usage); + if (ret == FSStatusCode::OK) { + FsMetric::GetInstance().SetFsUsage(fsName, usage); + } + return ret; } void FsManager::RefreshSession(const RefreshSessionRequest* request, diff --git a/curvefs/src/mds/fs_storage.cpp b/curvefs/src/mds/fs_storage.cpp index 304e2a30f5..b38b9f9803 100644 --- a/curvefs/src/mds/fs_storage.cpp +++ b/curvefs/src/mds/fs_storage.cpp @@ -29,6 +29,7 @@ #include #include "curvefs/src/mds/codec/codec.h" +#include "curvefs/src/mds/metric/fs_metric.h" namespace curvefs { namespace mds { @@ -409,6 +410,9 @@ bool PersisKVStorage::LoadAllFs() { { WriteLockGuard lock(fsUsageCacheMutex_); fsUsageCache_[fsInfo.fsname()] = fsUsage; + FsMetric::GetInstance().SetFsUsage(fsInfo.fsname(), fsUsage); + FsMetric::GetInstance().SetCapacity( + fsInfo.fsname(), fsInfo.capacity()); } } diff --git a/curvefs/src/mds/metric/fs_metric.cpp b/curvefs/src/mds/metric/fs_metric.cpp index 56ae90b381..9b84c92645 100644 --- a/curvefs/src/mds/metric/fs_metric.cpp +++ b/curvefs/src/mds/metric/fs_metric.cpp @@ -26,11 +26,11 @@ namespace curvefs { namespace mds { void FsMetric::OnMount(const std::string& fsname, const Mountpoint& mp) { - std::lock_guard lock(mtx_); + std::lock_guard lock(mountMtx_); - auto iter = metrics_.find(fsname); - if (iter == metrics_.end()) { - auto r = metrics_.emplace(fsname, new FsMountMetric(fsname)); + auto iter = mountMetrics_.find(fsname); + if (iter == mountMetrics_.end()) { + auto r = mountMetrics_.emplace(fsname, new FsMountMetric(fsname)); iter = r.first; } @@ -38,15 +38,50 @@ void FsMetric::OnMount(const std::string& fsname, const Mountpoint& mp) { } void FsMetric::OnUnMount(const std::string& fsname, const Mountpoint& mp) { - std::lock_guard lock(mtx_); + std::lock_guard lock(mountMtx_); - auto iter = metrics_.find(fsname); - if (iter == metrics_.end()) { + auto iter = mountMetrics_.find(fsname); + if (iter == mountMetrics_.end()) { return; } iter->second->OnUnMount(mp); } +std::unordered_map>::iterator +FsMetric::GetFsnameUsageMetricIter(const std::string& fsname) { + auto iter = usageMetrics_.find(fsname); + if (iter == usageMetrics_.end()) { + auto r = usageMetrics_.emplace(fsname, new FsUsageMetric(fsname)); + if (!r.second) { + LOG(ERROR) << "insert fs usage metric failed, fsname = " << fsname; + return usageMetrics_.end(); + } + iter = r.first; + } + return iter; +} + +void FsMetric::SetFsUsage(const std::string& fsname, const FsUsage& usage) { + std::lock_guard lock(usageMtx_); + auto iter = GetFsnameUsageMetricIter(fsname); + if (iter != usageMetrics_.end()) { + iter->second->SetUsage(usage); + } +} + +void FsMetric::SetCapacity(const std::string& fsname, uint64_t capacity) { + std::lock_guard lock(usageMtx_); + auto iter = GetFsnameUsageMetricIter(fsname); + if (iter != usageMetrics_.end()) { + iter->second->SetCapacity(capacity); + } +} + +void FsMetric::DeleteFsUsage(const std::string& fsname) { + std::lock_guard lock(usageMtx_); + usageMetrics_.erase(fsname); +} + } // namespace mds } // namespace curvefs diff --git a/curvefs/src/mds/metric/fs_metric.h b/curvefs/src/mds/metric/fs_metric.h index b5137efa88..6b8349e332 100644 --- a/curvefs/src/mds/metric/fs_metric.h +++ b/curvefs/src/mds/metric/fs_metric.h @@ -43,6 +43,9 @@ class FsMetric { void OnMount(const std::string& fsname, const Mountpoint& mp); void OnUnMount(const std::string& fsname, const Mountpoint& mp); + void SetFsUsage(const std::string& fsname, const FsUsage& usage); + void SetCapacity(const std::string& fsname, uint64_t capacity); + void DeleteFsUsage(const std::string& fsname); private: FsMetric() = default; @@ -51,9 +54,16 @@ class FsMetric { FsMetric(const FsMetric&) = delete; FsMetric& operator=(const FsMetric&) = delete; + std::unordered_map>::iterator + GetFsnameUsageMetricIter(const std::string& fsname); + private: - Mutex mtx_; - std::unordered_map> metrics_; + Mutex mountMtx_; + std::unordered_map> + mountMetrics_; + Mutex usageMtx_; + std::unordered_map> + usageMetrics_; }; } // namespace mds diff --git a/curvefs/src/mds/metric/metric.cpp b/curvefs/src/mds/metric/metric.cpp index b948a1ec56..d828c2923f 100644 --- a/curvefs/src/mds/metric/metric.cpp +++ b/curvefs/src/mds/metric/metric.cpp @@ -34,6 +34,8 @@ namespace curvefs { namespace mds { +const std::string FsUsageMetric::prefix = "fs_usage_info"; // NOLINT + void FsMountMetric::OnMount(const Mountpoint& mp) { std::string key = Key(mp); @@ -68,5 +70,9 @@ std::string FsMountMetric::Key(const Mountpoint& mp) { std::to_string(mp.port()) + "_" + mp.path(); } +void FsUsageMetric::SetUsage(const FsUsage& usage) { + usedBytes_.set_value(usage.usedbytes()); +} + } // namespace mds } // namespace curvefs diff --git a/curvefs/src/mds/metric/metric.h b/curvefs/src/mds/metric/metric.h index 947a57dcb3..53488a2f51 100644 --- a/curvefs/src/mds/metric/metric.h +++ b/curvefs/src/mds/metric/metric.h @@ -25,6 +25,7 @@ #include +#include #include #include #include @@ -34,7 +35,6 @@ namespace curvefs { namespace mds { - // Metric for a filesystem // includes filesystem mount number and filesystem mountpoint lists class FsMountMetric { @@ -59,15 +59,28 @@ class FsMountMetric { // current number of fs mountpoints bvar::Adder count_; - using MountPointMetric = - std::unordered_map>>; + using MountPointMetric = std::unordered_map>>; // protect mps_ Mutex mtx_; MountPointMetric mps_; }; +struct FsUsageMetric { + static const std::string prefix; + + std::string fsname_; + bvar::Status usedBytes_; + bvar::Status capacityBytes_; + + explicit FsUsageMetric(const std::string& fsname) + : fsname_(fsname), usedBytes_(prefix + "_fs_" + fsname + "_used", 0) {} + + void SetUsage(const FsUsage& usage); + void SetCapacity(uint64_t capacity) { capacityBytes_.set_value(capacity); } +}; + } // namespace mds } // namespace curvefs diff --git a/curvefs/test/mds/mds_metric_test.cpp b/curvefs/test/mds/mds_metric_test.cpp new file mode 100644 index 0000000000..7377179e22 --- /dev/null +++ b/curvefs/test/mds/mds_metric_test.cpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: Fri Apr 21 2023 + * Author: Xinlong-Chen + */ + +#include + +#include "curvefs/proto/mds.pb.h" +#include "curvefs/src/mds/metric/fs_metric.h" + +namespace curvefs { +namespace mds { + +class MdsMetricTest : public ::testing::Test { + protected: + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(MdsMetricTest, UpdateFsUsage) { + FsUsage usage; + usage.set_usedbytes(100); + FsMetric::GetInstance().SetFsUsage("test", usage); + usage.set_usedbytes(200); + FsMetric::GetInstance().SetFsUsage("test", usage); + FsMetric::GetInstance().DeleteFsUsage("test"); + FsMetric::GetInstance().DeleteFsUsage("test"); +} + +} // namespace mds +} // namespace curvefs From b8c46ce693ff5a960ede988f273f1f22260c12a8 Mon Sep 17 00:00:00 2001 From: Hanqing Wu Date: Thu, 26 Oct 2023 15:15:20 +0800 Subject: [PATCH 2/6] Revert "add clang-format action" This reverts commit 0a6be1253e7e53458dbb4801033c47e8c0ff9492. Signed-off-by: Hanqing Wu --- .clang-format | 153 ++++++++++++++++++++++++++--- .github/workflows/clang-format.yml | 30 ------ Makefile | 5 +- developers_guide.md | 15 --- developers_guide_cn.md | 14 --- util/format.sh | 2 - 6 files changed, 143 insertions(+), 76 deletions(-) delete mode 100644 .github/workflows/clang-format.yml delete mode 100644 util/format.sh diff --git a/.clang-format b/.clang-format index da0d029bcd..e7f2f7fce4 100644 --- a/.clang-format +++ b/.clang-format @@ -1,16 +1,147 @@ --- -Language: Cpp -BasedOnStyle: Google +Language: Cpp AccessModifierOffset: -3 +AlignAfterOpenBracket: Align +AlignConsecutiveMacros: true +AlignConsecutiveAssignments: false +AlignConsecutiveBitFields: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Right +AlignOperands: Align +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortEnumsOnASingleLine: true +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: MultiLine +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: Never + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DeriveLineEnding: true DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + SortPriority: 0 + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + SortPriority: 0 + - Regex: '.*' + Priority: 1 + SortPriority: 0 +IncludeIsMainRegex: '(Test)?$' +IncludeIsMainSourceRegex: '' +IndentCaseLabels: false +IndentCaseBlocks: false +IndentGotoLabels: true +IndentPPDirectives: None +IndentExternBlock: AfterExternBlock IndentWidth: 4 -AlignAfterOpenBracket: DontAlign ---- -Language: Proto -BasedOnStyle: Google -IndentWidth: 4 -ColumnLimit: 120 ---- -Language: Json -DisableFormat: true +IndentWrappedFunctionNames: false +InsertTrailingCommas: None +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: true +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 2 +NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 2 +ObjCBreakBeforeNestedBlockParam: true +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PointerAlignment: Right +ReflowComments: true +SortIncludes: false +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpaceBeforeSquareBrackets: false +Standard: Cpp11 +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 4 +UseCRLF: false +UseTab: Never +WhitespaceSensitiveMacros: + - STRINGIZE + - PP_STRINGIZE + - BOOST_PP_STRINGIZE ... diff --git a/.github/workflows/clang-format.yml b/.github/workflows/clang-format.yml deleted file mode 100644 index fa4c88720e..0000000000 --- a/.github/workflows/clang-format.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Code Formatter - -# run on pull requests to develop -on: - pull_request - -jobs: - format: - name: Clang Formatter - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - # fetch everything to be able to compare with any ref - fetch-depth: 0 - - - name: Check - env: - LANG: "C.UTF-8" - LC_ALL: "C.UTF-8" - LANGUAGE: "C.UTF-8" - run: | - sudo apt-get install -y clang-format-14 python3 git - diff_output_file=$(mktemp) - git --no-pager diff -U0 --diff-filter=ACMRT ${{github.event.pull_request.base.sha}}...${{ github.event.pull_request.head.sha }} | clang-format-diff-14 -p1 2>&1 | tee $diff_output_file - if [ -s $diff_output_file ]; then - exit 1 - else - exit 0 - fi diff --git a/Makefile b/Makefile index 749e838ba2..aefeeb374d 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ tag?= "curvebs:unknown" case?= "*" os?= "debian11" ci?=0 -commit_id="HEAD^" + define help_msg ## build curvebs make build stor=bs dep=1 @@ -122,9 +122,6 @@ test: docker: @bash util/docker.sh --os=$(os) --ci=$(ci) -format: - @bash util/format.sh $(commit_id) - init-hadoop: @bash util/init-hadoop.sh diff --git a/developers_guide.md b/developers_guide.md index 0ba0136263..a5ece6ba46 100644 --- a/developers_guide.md +++ b/developers_guide.md @@ -115,21 +115,6 @@ Curve Ci use ```cpplint``` check what your changed. $ cpplint --filter=-build/c++11 --quiet --recursive your_path ``` -- use clang-format-diff to check submitted code - ``` - $ make format - # Or - $ make format commit_id=$(commit_id) # commit_id is a sha of a commit, default HEAD^ - ``` -> Clang-format understands also special comments that switch formatting in a delimited range. The code between a comment `// clang-format off` or `/* clang-format off */` up to a comment `// clang-format on` or `/* clang-format on */` will not be formatted. - > ```cpp - > int formatted_code; - > // clang-format off - > void unformatted_code ; - > // clang-format on - > void formatted_code_again; - > ``` - For PR we have the following requirements: - The Curve coding standard strictly follows the [Google C++ Open Source Project Coding Guide](https://google.github.io/styleguide/cppguide.html), but we use 4 spaces to indent, Clang-format will more helpful for you. Of course, CI will check what your changed. diff --git a/developers_guide_cn.md b/developers_guide_cn.md index 62e768ffce..1c36a8c0e3 100644 --- a/developers_guide_cn.md +++ b/developers_guide_cn.md @@ -70,20 +70,6 @@ Curve CI 使用```cpplint```检查更改的代码, ```bash $ cpplint --filter=-build/c++11 --quiet --recursive your_path ``` -- 本地执行clang-format-diff对提交部分代码进行检查: - ``` - $ make format - # or - $ make format commit_id=$(commit_id) # commit_id为某一个提交的sha，默认commit_id为HEAD^ - ``` - > 如果需要对部分代码禁用格式检查，可以在开始处添加 `// clang-format off` 或 `/* clang-format off */` 注释，结尾处添加 `// clang-format on` 或 `/* clang-format on */` 注释 - > ```cpp - > int formatted_code; - > // clang-format off - > void unformatted_code ; - > // clang-format on - > void formatted_code_again; - > ``` 对于 PR 我们有如下要求： diff --git a/util/format.sh b/util/format.sh deleted file mode 100644 index 81432e6039..0000000000 --- a/util/format.sh +++ /dev/null @@ -1,2 +0,0 @@ -git --no-pager diff -U0 --diff-filter=ACMRT $1 HEAD | clang-format-diff-14 -p1 -i -style file - From a38f8e65f1e4a16662a5078d3af7cbbb8a09faf9 Mon Sep 17 00:00:00 2001 From: Hanqing Wu Date: Thu, 26 Oct 2023 15:16:37 +0800 Subject: [PATCH 3/6] remove clang-format/clangd config Signed-off-by: Hanqing Wu --- .clang-format | 147 -------------------------------------------------- .clangd | 12 ----- 2 files changed, 159 deletions(-) delete mode 100644 .clang-format delete mode 100644 .clangd diff --git a/.clang-format b/.clang-format deleted file mode 100644 index e7f2f7fce4..0000000000 --- a/.clang-format +++ /dev/null @@ -1,147 +0,0 @@ ---- -Language: Cpp -AccessModifierOffset: -3 -AlignAfterOpenBracket: Align -AlignConsecutiveMacros: true -AlignConsecutiveAssignments: false -AlignConsecutiveBitFields: false -AlignConsecutiveDeclarations: false -AlignEscapedNewlines: Right -AlignOperands: Align -AlignTrailingComments: true -AllowAllArgumentsOnNextLine: true -AllowAllConstructorInitializersOnNextLine: true -AllowAllParametersOfDeclarationOnNextLine: true -AllowShortEnumsOnASingleLine: true -AllowShortBlocksOnASingleLine: Never -AllowShortCaseLabelsOnASingleLine: false -AllowShortFunctionsOnASingleLine: All -AllowShortLambdasOnASingleLine: All -AllowShortIfStatementsOnASingleLine: Never -AllowShortLoopsOnASingleLine: false -AlwaysBreakAfterDefinitionReturnType: None -AlwaysBreakAfterReturnType: None -AlwaysBreakBeforeMultilineStrings: false -AlwaysBreakTemplateDeclarations: MultiLine -BinPackArguments: true -BinPackParameters: true -BraceWrapping: - AfterCaseLabel: false - AfterClass: false - AfterControlStatement: Never - AfterEnum: false - AfterFunction: false - AfterNamespace: false - AfterObjCDeclaration: false - AfterStruct: false - AfterUnion: false - AfterExternBlock: false - BeforeCatch: false - BeforeElse: false - BeforeLambdaBody: false - BeforeWhile: false - IndentBraces: false - SplitEmptyFunction: true - SplitEmptyRecord: true - SplitEmptyNamespace: true -BreakBeforeBinaryOperators: None -BreakBeforeBraces: Attach -BreakBeforeInheritanceComma: false -BreakInheritanceList: BeforeColon -BreakBeforeTernaryOperators: true -BreakConstructorInitializersBeforeComma: false -BreakConstructorInitializers: BeforeColon -BreakAfterJavaFieldAnnotations: false -BreakStringLiterals: true -ColumnLimit: 80 -CommentPragmas: '^ IWYU pragma:' -CompactNamespaces: false -ConstructorInitializerAllOnOneLineOrOnePerLine: false -ConstructorInitializerIndentWidth: 4 -ContinuationIndentWidth: 4 -Cpp11BracedListStyle: true -DeriveLineEnding: true -DerivePointerAlignment: false -DisableFormat: false -ExperimentalAutoDetectBinPacking: false -FixNamespaceComments: true -ForEachMacros: - - foreach - - Q_FOREACH - - BOOST_FOREACH -IncludeBlocks: Preserve -IncludeCategories: - - Regex: '^"(llvm|llvm-c|clang|clang-c)/' - Priority: 2 - SortPriority: 0 - - Regex: '^(<|"(gtest|gmock|isl|json)/)' - Priority: 3 - SortPriority: 0 - - Regex: '.*' - Priority: 1 - SortPriority: 0 -IncludeIsMainRegex: '(Test)?$' -IncludeIsMainSourceRegex: '' -IndentCaseLabels: false -IndentCaseBlocks: false -IndentGotoLabels: true -IndentPPDirectives: None -IndentExternBlock: AfterExternBlock -IndentWidth: 4 -IndentWrappedFunctionNames: false -InsertTrailingCommas: None -JavaScriptQuotes: Leave -JavaScriptWrapImports: true -KeepEmptyLinesAtTheStartOfBlocks: true -MacroBlockBegin: '' -MacroBlockEnd: '' -MaxEmptyLinesToKeep: 2 -NamespaceIndentation: None -ObjCBinPackProtocolList: Auto -ObjCBlockIndentWidth: 2 -ObjCBreakBeforeNestedBlockParam: true -ObjCSpaceAfterProperty: false -ObjCSpaceBeforeProtocolList: true -PenaltyBreakAssignment: 2 -PenaltyBreakBeforeFirstCallParameter: 19 -PenaltyBreakComment: 300 -PenaltyBreakFirstLessLess: 120 -PenaltyBreakString: 1000 -PenaltyBreakTemplateDeclaration: 10 -PenaltyExcessCharacter: 1000000 -PenaltyReturnTypeOnItsOwnLine: 60 -PointerAlignment: Right -ReflowComments: true -SortIncludes: false -SortUsingDeclarations: true -SpaceAfterCStyleCast: false -SpaceAfterLogicalNot: false -SpaceAfterTemplateKeyword: true -SpaceBeforeAssignmentOperators: true -SpaceBeforeCpp11BracedList: false -SpaceBeforeCtorInitializerColon: true -SpaceBeforeInheritanceColon: true -SpaceBeforeParens: ControlStatements -SpaceBeforeRangeBasedForLoopColon: true -SpaceInEmptyBlock: false -SpaceInEmptyParentheses: false -SpacesBeforeTrailingComments: 2 -SpacesInAngles: false -SpacesInConditionalStatement: false -SpacesInContainerLiterals: true -SpacesInCStyleCastParentheses: false -SpacesInParentheses: false -SpacesInSquareBrackets: false -SpaceBeforeSquareBrackets: false -Standard: Cpp11 -StatementMacros: - - Q_UNUSED - - QT_REQUIRE_VERSION -TabWidth: 4 -UseCRLF: false -UseTab: Never -WhitespaceSensitiveMacros: - - STRINGIZE - - PP_STRINGIZE - - BOOST_PP_STRINGIZE -... diff --git a/.clangd b/.clangd deleted file mode 100644 index efdc056fe2..0000000000 --- a/.clangd +++ /dev/null @@ -1,12 +0,0 @@ -Diagnostics: - ClangTidy: - Add: [performance-*, modernize-*, readability-*,bugprone-*,] - Remove: [modernize-use-trailing-return-type] - CheckOptions: - readability-identifier-naming.VariableCase: camelBack - UnusedIncludes: Strict - -InlayHints: - Enabled: No - ParameterNames: No - DeducedTypes: No From d19e7e9bf38d5f0a03afd68553a38869b109d84b Mon Sep 17 00:00:00 2001 From: victorseptember <2452508171@qq.com> Date: Sun, 16 Jul 2023 22:44:21 +0800 Subject: [PATCH 4/6] [feat]tools-v2: add check file consistency Signed-off-by: victorseptember <2452508171@qq.com> --- tools-v2/README.md | 25 ++- tools-v2/internal/error/error.go | 6 + .../pkg/cli/command/curvebs/check/check.go | 2 + .../curvebs/check/consistency/consistency.go | 193 ++++++++++++++++++ .../command/curvebs/query/chunk/chunkhash.go | 182 +++++++++++++++++ .../command/curvebs/query/seginfo/seginfo.go | 21 ++ .../command/curvebs/status/copyset/agent.go | 87 ++++++++ tools-v2/pkg/config/bs.go | 24 +++ 8 files changed, 538 insertions(+), 2 deletions(-) create mode 100644 tools-v2/pkg/cli/command/curvebs/check/consistency/consistency.go create mode 100644 tools-v2/pkg/cli/command/curvebs/query/chunk/chunkhash.go diff --git a/tools-v2/README.md b/tools-v2/README.md index 5ee19eb6ae..b9fea277f8 100644 --- a/tools-v2/README.md +++ b/tools-v2/README.md @@ -89,6 +89,7 @@ A tool for CurveFS & CurveBs. - [check copyset](#check-copyset-1) - [check chunkserver](#check-chunkserver) - [check server](#check-server) + - [check consistency](#check-consistency) - [snapshot](#snapshot) - [snapshot copyset](#snapshot-copyset) - [Comparison of old and new commands](#comparison-of-old-and-new-commands) @@ -1871,6 +1872,26 @@ Output: +--------+-----------+-------+------------------+ ``` +##### check consistency + +check the file consistency + +Usage: + +```shell +curve bs check consistency --path /test +``` + +Output: + +```shell ++--------+--------+------------+ +| NAME | STATUS | EXPLAIN | ++--------+--------+------------+ +| /test | ok | - | ++--------+--------+------------+ +``` + #### snapshot ##### snapshot copyset @@ -1971,8 +1992,8 @@ Output: | curve_ops_tool list-may-broken-vol | curve bs list may-broken-vol | | curve_ops_tool rapid-leader-schedule | curve bs update leader-schedule | | curve_ops_tool do-snapshot-all | curve bs snapshot --all | -| curve_ops_tool check-chunkserver | curbe bs check chunkserver | +| curve_ops_tool check-chunkserver | curve bs check chunkserver | | curve_ops_tool status | | -| curve_ops_tool check-consistency | | +| curve_ops_tool check-consistency | curve bs check consistency | | curve_ops_tool check-server | curve bs check server | diff --git a/tools-v2/internal/error/error.go b/tools-v2/internal/error/error.go index a4737ba33d..a8d91a9cc9 100644 --- a/tools-v2/internal/error/error.go +++ b/tools-v2/internal/error/error.go @@ -487,6 +487,12 @@ var ( ErrBsGetFormatStatus = func() *CmdError { return NewInternalCmdError(75, "get format status fail, err: %s") } + ErrBsGetSegmentInfo = func() *CmdError { + return NewInternalCmdError(76, "get segment info fail, err: %s") + } + ErrBsGetChunkHash = func() *CmdError { + return NewInternalCmdError(77, "get chunk hash fail, err: %s") + } // http error ErrHttpUnreadableResult = func() *CmdError { diff --git a/tools-v2/pkg/cli/command/curvebs/check/check.go b/tools-v2/pkg/cli/command/curvebs/check/check.go index 17617c529e..0e5bd26d49 100644 --- a/tools-v2/pkg/cli/command/curvebs/check/check.go +++ b/tools-v2/pkg/cli/command/curvebs/check/check.go @@ -25,6 +25,7 @@ package check import ( basecmd "github.com/opencurve/curve/tools-v2/pkg/cli/command" "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/check/chunkserver" + "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/check/consistency" "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/check/copyset" "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/check/operator" "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/check/server" @@ -43,6 +44,7 @@ func (checkCmd *CheckCommand) AddSubCommands() { operator.NewOperatorCommand(), server.NewServerCommand(), chunkserver.NewChunkserverCommand(), + consistency.NewConsistencyCommand(), ) } diff --git a/tools-v2/pkg/cli/command/curvebs/check/consistency/consistency.go b/tools-v2/pkg/cli/command/curvebs/check/consistency/consistency.go new file mode 100644 index 0000000000..a0f71ee0ae --- /dev/null +++ b/tools-v2/pkg/cli/command/curvebs/check/consistency/consistency.go @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: tools-v2 + * Created Date: 2023-07-13 + * Author: victorseptember + */ + +package consistency + +import ( + "fmt" + "strconv" + "strings" + + mapset "github.com/deckarep/golang-set/v2" + cmderror "github.com/opencurve/curve/tools-v2/internal/error" + cobrautil "github.com/opencurve/curve/tools-v2/internal/utils" + basecmd "github.com/opencurve/curve/tools-v2/pkg/cli/command" + "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/query/chunk" + "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/query/chunkserver" + "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/query/seginfo" + "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/status/copyset" + "github.com/opencurve/curve/tools-v2/pkg/config" + "github.com/opencurve/curve/tools-v2/pkg/output" + "github.com/spf13/cobra" +) + +const ( + consistencyExample = `$ curve bs check consistency --path file_path` +) + +type ConsistencyCommand struct { + basecmd.FinalCurveCmd +} + +var _ basecmd.FinalCurveCmdFunc = (*ConsistencyCommand)(nil) + +func NewConsistencyCommand() *cobra.Command { + return NewCheckConsistencyCommand().Cmd +} + +func NewCheckConsistencyCommand() *ConsistencyCommand { + consistencyCmd := &ConsistencyCommand{ + FinalCurveCmd: basecmd.FinalCurveCmd{ + Use: "consistency", + Short: "check the file consistency", + Example: consistencyExample, + }} + basecmd.NewFinalCurveCli(&consistencyCmd.FinalCurveCmd, consistencyCmd) + return consistencyCmd +} + +func (cCmd *ConsistencyCommand) AddFlags() { + config.AddBsMdsFlagOption(cCmd.Cmd) + config.AddRpcTimeoutFlag(cCmd.Cmd) + config.AddRpcRetryTimesFlag(cCmd.Cmd) + config.AddBsPathRequiredFlag(cCmd.Cmd) + config.AddHttpTimeoutFlag(cCmd.Cmd) + config.AddBsCheckHashOptionFlag(cCmd.Cmd) +} + +func (cCmd *ConsistencyCommand) Init(cmd *cobra.Command, args []string) error { + segmentInfoResponse, err := seginfo.GetSegmentInfo(cCmd.Cmd) + if err.TypeCode() != cmderror.CODE_SUCCESS { + return err.ToError() + } + logicalpoolIds := make([]string, 0) + copysetIds := make([]string, 0) + chunkIds := make([]string, 0) + + for _, segment := range segmentInfoResponse { + for _, chunk := range segment.GetChunks() { + logicalpoolId := segment.GetLogicalPoolID() + copysetId := chunk.GetCopysetID() + chunkId := chunk.GetChunkID() + logicalpoolIds = append(logicalpoolIds, fmt.Sprintf("%d", logicalpoolId)) + copysetIds = append(copysetIds, fmt.Sprintf("%d", copysetId)) + chunkIds = append(chunkIds, fmt.Sprintf("%d", chunkId)) + } + } + + if len(logicalpoolIds) == 0 || len(copysetIds) == 0 { + return fmt.Errorf("the number of logicalpoolid or copysetid is empty") + } + config.AddBsCopysetIdSliceRequiredFlag(cCmd.Cmd) + config.AddBsLogicalPoolIdSliceRequiredFlag(cCmd.Cmd) + cCmd.Cmd.ParseFlags([]string{ + fmt.Sprintf("--%s", config.CURVEBS_LOGIC_POOL_ID), strings.Join(logicalpoolIds, ","), + fmt.Sprintf("--%s", config.CURVEBS_COPYSET_ID), strings.Join(copysetIds, ","), + }) + + key2Location, err := chunkserver.GetChunkServerListInCopySets(cCmd.Cmd) + if err.TypeCode() != cmderror.CODE_SUCCESS { + return err.ToError() + } + logicalpoolidList, errParse := cobrautil.StringList2Uint32List(logicalpoolIds) + if errParse != nil { + return fmt.Errorf("parse logicalpoolid%v fail", logicalpoolIds) + } + copysetidList, errParse := cobrautil.StringList2Uint32List(copysetIds) + if errParse != nil { + return fmt.Errorf("parse copysetid%v fail", copysetIds) + } + + // check copyset raft status + chunkServerLocationSet := mapset.NewSet[string]() + key2Address := make(map[uint64][]string) + copysetKeysSet := mapset.NewSet[uint64]() + addr2Id := make(map[string]uint32) + for i := 0; i < len(logicalpoolidList); i++ { + logicpoolid := logicalpoolidList[i] + copysetid := copysetidList[i] + key := cobrautil.GetCopysetKey(uint64(logicpoolid), uint64(copysetid)) + if copysetKeysSet.Contains(key) { + continue + } + copysetKeysSet.Add(key) + for _, cs := range key2Location[key] { + address := fmt.Sprintf("%s:%d", *cs.HostIp, *cs.Port) + key2Address[key] = append(key2Address[key], address) + chunkServerLocationSet.Add(address) + addr2Id[address] = *cs.ChunkServerID + } + } + + cs := copyset.NewCopyset() + filename := config.GetBsFlagString(cmd, config.CURVEBS_PATH) + status, explain, err2 := copyset.CheckCopysetsInChunkServers( + chunkServerLocationSet.ToSlice(), addr2Id, cs, copysetKeysSet, cmd) + if err2 != nil { + return fmt.Errorf("CheckCopysetsInChunkServers fail") + } + + // chech chunk hash + checkHash := config.GetBsFlagBool(cCmd.Cmd, config.CURVEBS_CHECK_HASH) + if checkHash { + config.AddBsPeersAddressFlag(cCmd.Cmd) + config.AddBsChunkIdRequiredFlag(cCmd.Cmd) + config.AddBsChunkSizeRequiredFlag(cCmd.Cmd) + chunksize := segmentInfoResponse[0].GetChunkSize() + for i := 0; i < len(logicalpoolIds); i++ { + key := cobrautil.GetCopysetKey(uint64(logicalpoolidList[i]), uint64(copysetidList[i])) + config.ResetStringSliceFlag(cCmd.Cmd.Flag(config.CURVEBS_LOGIC_POOL_ID), logicalpoolIds[i]) + config.ResetStringSliceFlag(cCmd.Cmd.Flag(config.CURVEBS_COPYSET_ID), copysetIds[i]) + cCmd.Cmd.ParseFlags([]string{ + fmt.Sprintf("--%s", config.CURVEBS_CHUNK_ID), chunkIds[i], + fmt.Sprintf("--%s", config.CURVEBS_CHUNK_SIZE), fmt.Sprintf("%d", chunksize), + fmt.Sprintf("--%s", config.CURVEBS_PEERS_ADDRESS), strings.Join(key2Address[key], ","), + }) + result, checkHashExplain, err := chunk.GetChunkHash(cCmd.Cmd) + if err != nil { + return fmt.Errorf("GetChunkHash fail") + } + if !result { + status = cobrautil.CopysetHealthStatus_Str[int32(cobrautil.HEALTH_ERROR)] + explain += fmt.Sprintf("copyset %s chunkID %s %s.\n", + strconv.FormatUint(key, 10), chunkIds[i], checkHashExplain) + } + } + } + cCmd.TableNew.Append([]string{filename, status, explain}) + header := []string{cobrautil.ROW_NAME, cobrautil.ROW_STATUS, cobrautil.ROW_EXPLAIN} + cCmd.SetHeader(header) + + return nil +} + +func (cCmd *ConsistencyCommand) RunCommand(cmd *cobra.Command, args []string) error { + return nil +} + +func (cCmd *ConsistencyCommand) Print(cmd *cobra.Command, args []string) error { + return output.FinalCmdOutputPlain(&cCmd.FinalCurveCmd) +} + +func (cCmd *ConsistencyCommand) ResultPlainOutput() error { + return output.FinalCmdOutputPlain(&cCmd.FinalCurveCmd) +} diff --git a/tools-v2/pkg/cli/command/curvebs/query/chunk/chunkhash.go b/tools-v2/pkg/cli/command/curvebs/query/chunk/chunkhash.go new file mode 100644 index 0000000000..5745472fe7 --- /dev/null +++ b/tools-v2/pkg/cli/command/curvebs/query/chunk/chunkhash.go @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: tools-v2 + * Created Date: 2023-10-03 + * Author: victorseptember + */ + +package chunk + +import ( + "context" + "fmt" + "strings" + + cmderror "github.com/opencurve/curve/tools-v2/internal/error" + basecmd "github.com/opencurve/curve/tools-v2/pkg/cli/command" + "github.com/opencurve/curve/tools-v2/pkg/config" + "github.com/opencurve/curve/tools-v2/pkg/output" + "github.com/opencurve/curve/tools-v2/proto/proto/chunk" + "github.com/spf13/cobra" + "google.golang.org/grpc" +) + +type GetChunkHashRpc struct { + Info *basecmd.Rpc + Request *chunk.GetChunkHashRequest + mdsClient chunk.ChunkServiceClient +} + +var _ basecmd.RpcFunc = (*GetChunkHashRpc)(nil) // check interface + +func (gRpc *GetChunkHashRpc) NewRpcClient(cc grpc.ClientConnInterface) { + gRpc.mdsClient = chunk.NewChunkServiceClient(cc) +} + +func (gRpc *GetChunkHashRpc) Stub_Func(ctx context.Context) (interface{}, error) { + return gRpc.mdsClient.GetChunkHash(ctx, gRpc.Request) +} + +type GetChunkHashCommand struct { + basecmd.FinalCurveCmd + Rpc []*GetChunkHashRpc + result bool + explain string +} + +var _ basecmd.FinalCurveCmdFunc = (*GetChunkHashCommand)(nil) + +func NewChunkHashCommand() *cobra.Command { + return NewGetChunkHashCommand().Cmd +} + +func NewGetChunkHashCommand() *GetChunkHashCommand { + chunkHashCmd := &GetChunkHashCommand{ + FinalCurveCmd: basecmd.FinalCurveCmd{}, + result: true, + } + basecmd.NewFinalCurveCli(&chunkHashCmd.FinalCurveCmd, chunkHashCmd) + return chunkHashCmd +} + +func (cCmd *GetChunkHashCommand) Init(cmd *cobra.Command, args []string) error { + timeout := config.GetFlagDuration(cCmd.Cmd, config.RPCTIMEOUT) + retrytimes := config.GetFlagInt32(cCmd.Cmd, config.RPCRETRYTIMES) + copysetid := config.GetBsFlagUint32(cCmd.Cmd, config.CURVEBS_COPYSET_ID) + logicalpoolid := config.GetBsFlagUint32(cCmd.Cmd, config.CURVEBS_LOGIC_POOL_ID) + chunkid := config.GetBsFlagUint64(cCmd.Cmd, config.CURVEBS_CHUNK_ID) + chunksize := config.GetBsFlagUint32(cCmd.Cmd, config.CURVEBS_CHUNK_SIZE) + peers := config.GetBsFlagString(cCmd.Cmd, config.CURVEBS_PEERS_ADDRESS) + + var offset uint32 = 0 + peersAddress := strings.Split(peers, ",") + for _, address := range peersAddress { + rpc := &GetChunkHashRpc{ + Request: &chunk.GetChunkHashRequest{ + LogicPoolId: &logicalpoolid, + CopysetId: ©setid, + ChunkId: &chunkid, + Offset: &offset, + Length: &chunksize, + }, + } + rpc.Info = basecmd.NewRpc([]string{address}, timeout, retrytimes, "GetChunkHash") + cCmd.Rpc = append(cCmd.Rpc, rpc) + } + return nil +} + +func (cCmd *GetChunkHashCommand) RunCommand(cmd *cobra.Command, args []string) error { + var infos []*basecmd.Rpc + var funcs []basecmd.RpcFunc + for _, rpc := range cCmd.Rpc { + infos = append(infos, rpc.Info) + funcs = append(funcs, rpc) + } + results, errs := basecmd.GetRpcListResponse(infos, funcs) + if len(errs) == len(infos) { + mergeErr := cmderror.MergeCmdErrorExceptSuccess(errs) + return mergeErr.ToError() + } + + var preHash string + var curHash string + flag := true + for _, result := range results { + response, ok := result.(*chunk.GetChunkHashResponse) + if !ok { + return fmt.Errorf("response is nil") + } + if *response.Status != chunk.CHUNK_OP_STATUS_CHUNK_OP_STATUS_SUCCESS { + cCmd.result = false + cCmd.explain += chunk.CHUNK_OP_STATUS_name[int32(*response.Status)] + break + } + curHash = *response.Hash + if flag { + flag = false + preHash = curHash + continue + } + if preHash != curHash { + cCmd.result = false + cCmd.explain += "chunk hash inconsistent" + break + } + } + return nil +} + +func (cCmd *GetChunkHashCommand) Print(cmd *cobra.Command, args []string) error { + return output.FinalCmdOutput(&cCmd.FinalCurveCmd, cCmd) +} + +func (cCmd *GetChunkHashCommand) ResultPlainOutput() error { + return output.FinalCmdOutputPlain(&cCmd.FinalCurveCmd) +} + +func (cCmd *GetChunkHashCommand) AddFlags() { + config.AddBsMdsFlagOption(cCmd.Cmd) + config.AddRpcRetryTimesFlag(cCmd.Cmd) + config.AddRpcTimeoutFlag(cCmd.Cmd) + + config.AddBsCopysetIdRequiredFlag(cCmd.Cmd) + config.AddBsLogicalPoolIdRequiredFlag(cCmd.Cmd) + config.AddBsPeersAddressFlag(cCmd.Cmd) + config.AddBsChunkIdRequiredFlag(cCmd.Cmd) + config.AddBsChunkSizeRequiredFlag(cCmd.Cmd) +} + +func GetChunkHash(caller *cobra.Command) (bool, string, *cmderror.CmdError) { + cCmd := NewGetChunkHashCommand() + config.AlignFlagsValue(caller, cCmd.Cmd, []string{ + config.RPCRETRYTIMES, config.RPCTIMEOUT, config.CURVEBS_MDSADDR, + config.CURVEBS_LOGIC_POOL_ID, config.CURVEBS_PEERS_ADDRESS, config.CURVEBS_CHUNK_ID, + config.CURVEBS_CHUNK_SIZE, config.CURVEBS_COPYSET_ID, + }) + cCmd.Cmd.SilenceErrors = true + cCmd.Cmd.SilenceUsage = true + cCmd.Cmd.SetArgs([]string{"--format", config.FORMAT_NOOUT}) + err := cCmd.Cmd.Execute() + if err != nil { + retErr := cmderror.ErrBsGetChunkHash() + retErr.Format(err.Error()) + return false, "", retErr + } + return cCmd.result, cCmd.explain, nil +} diff --git a/tools-v2/pkg/cli/command/curvebs/query/seginfo/seginfo.go b/tools-v2/pkg/cli/command/curvebs/query/seginfo/seginfo.go index 30b9da3653..2a1b672178 100644 --- a/tools-v2/pkg/cli/command/curvebs/query/seginfo/seginfo.go +++ b/tools-v2/pkg/cli/command/curvebs/query/seginfo/seginfo.go @@ -150,3 +150,24 @@ TaverSegment: func (sCmd *SeginfoCommand) ResultPlainOutput() error { return output.FinalCmdOutputPlain(&sCmd.FinalCurveCmd) } + + +//add for check file consistency (by victorseptember) +func GetSegmentInfo(caller *cobra.Command) ([]*nameserver2.PageFileSegment, *cmderror.CmdError) { + getCmd := NewQuerySeginfoCommand() + config.AlignFlagsValue(caller, getCmd.Cmd, []string{ + config.RPCRETRYTIMES, config.RPCTIMEOUT, config.CURVEBS_MDSADDR, + config.CURVEBS_PATH, + }) + getCmd.Cmd.SilenceErrors = true + getCmd.Cmd.SilenceUsage = true + getCmd.Cmd.SetArgs([]string{"--format", config.FORMAT_NOOUT}) + err := getCmd.Cmd.Execute() + if err != nil { + retErr := cmderror.ErrBsGetSegmentInfo() + retErr.Format(err.Error()) + return nil, retErr + } + result := getCmd.Result.([]*nameserver2.PageFileSegment) + return result, cmderror.Success() +} \ No newline at end of file diff --git a/tools-v2/pkg/cli/command/curvebs/status/copyset/agent.go b/tools-v2/pkg/cli/command/curvebs/status/copyset/agent.go index 8f3fe0b2ab..2a95901659 100644 --- a/tools-v2/pkg/cli/command/curvebs/status/copyset/agent.go +++ b/tools-v2/pkg/cli/command/curvebs/status/copyset/agent.go @@ -564,3 +564,90 @@ func MaxUint64(first, second uint64) uint64 { } return first } + +//check file consistency +func CheckCopysetsInChunkServers(address []string, addr2Id map[string]uint32, cs *Copyset, + copysetKeysSet set.Set[uint64], cmd *cobra.Command) (string, string, error) { + healthyStatus := cobrautil.HEALTH_OK + explain := "" + status, err := GetCopysetRaftStatus(address, cmd) + if err != nil { + return "", explain, err + } + for k, v := range status { + cs.updateChunkServerCopysets(k, v) + } + + for k, v := range status { + copysetsList := make([]map[string]string, 0) + for _, copysetStatusMap := range v { + key, err2 := strconv.ParseUint(copysetStatusMap[RAFT_STATUS_KEY_GROUPID], 10, 64) + if err2 != nil { + return "", explain, fmt.Errorf(COPYSET_CHECK_PARSE_ERROR) + } + if copysetKeysSet.Contains(key) { + copysetsList = append(copysetsList, copysetStatusMap) + } + } + cs.checkCopysetsOnChunkServer(k, copysetsList, addr2Id, cmd) + } + if _, ok := cs.copysetStat[int32(cobrautil.COPYSET_ERROR)]; ok && + cs.copysetStat[int32(cobrautil.COPYSET_ERROR)].Cardinality() > 0 { + healthyStatus = cobrautil.HEALTH_ERROR + explain = getErrorCopysetsInfo(cs) + } else if _, ok := cs.copysetStat[int32(cobrautil.COPYSET_WARN)]; ok && + cs.copysetStat[int32(cobrautil.COPYSET_WARN)].Cardinality() > 0 { + healthyStatus = cobrautil.HEALTH_WARN + explain = getWarnCopysetsInfo(cs) + } else { + healthyStatus = cobrautil.HEALTH_OK + explain = "-" + } + + return cobrautil.CopysetHealthStatus_Str[int32(healthyStatus)], explain, nil +} + +func getErrorCopysetsInfo(cs *Copyset) string { + message := "" + if _, ok := cs.copyset[COPYSET_CHECK_NO_LEADER]; ok { + for _, key := range(cs.copyset[COPYSET_CHECK_NO_LEADER].ToSlice()) { + message += fmt.Sprintf("copyset %s no leader peer.\n", key) + } + } + if _, ok := cs.copyset[COPYSET_CHECK_MAJORITY_PEER_OFFLINE]; ok { + for _, key := range(cs.copyset[COPYSET_CHECK_MAJORITY_PEER_OFFLINE].ToSlice()) { + message += fmt.Sprintf("copyset %s majority peer offline.\n", key) + } + } + if _, ok := cs.copyset[COPYSET_CHECK_INCONSISTENT]; ok { + for _, key := range(cs.copyset[COPYSET_CHECK_INCONSISTENT].ToSlice()) { + message += fmt.Sprintf("copyset %s check inconsistent.\n", key) + } + } + return message +} + +func getWarnCopysetsInfo(cs *Copyset) string { + message := "" + if _, ok := cs.copyset[COPYSET_CHECK_PEERS_NO_SUFFICIENT]; ok { + for _, key := range(cs.copyset[COPYSET_CHECK_PEERS_NO_SUFFICIENT].ToSlice()) { + message += fmt.Sprintf("copyset %s peers no sufficient.\n", key) + } + } + if _, ok := cs.copyset[COPYSET_CHECK_LOG_INDEX_TOO_BIG]; ok { + for _, key := range(cs.copyset[COPYSET_CHECK_LOG_INDEX_TOO_BIG].ToSlice()) { + message += fmt.Sprintf("copyset %s log index gap too big.\n", key) + } + } + if _, ok := cs.copyset[COPYSET_CHECK_INSTALLING_SNAPSHOT]; ok { + for _, key := range(cs.copyset[COPYSET_CHECK_INSTALLING_SNAPSHOT].ToSlice()) { + message += fmt.Sprintf("copyset %s installing snapshot.\n", key) + } + } + if _, ok := cs.copyset[COPYSET_CHECK_MINORITY_PEER_OFFLINE]; ok { + for _, key := range(cs.copyset[COPYSET_CHECK_MINORITY_PEER_OFFLINE].ToSlice()) { + message += fmt.Sprintf("copyset %s minority peers offline.\n", key) + } + } + return message +} \ No newline at end of file diff --git a/tools-v2/pkg/config/bs.go b/tools-v2/pkg/config/bs.go index 6b1da40887..08b80dd179 100644 --- a/tools-v2/pkg/config/bs.go +++ b/tools-v2/pkg/config/bs.go @@ -144,6 +144,11 @@ const ( VIPER_CURVEBS_TASKID = "curvebs.taskid" CURVEBS_FAILED = "failed" VIPER_CURVEBS_FAILED = "curvebs.failed" + CURVEBS_CHUNK_SIZE = "chunksize" + VIPER_CURVEBS_CHUNK_SIZE = "curvebs.chunksize" + CURVEBS_CHECK_HASH = "checkhash" + VIPER_CURVEBS_CHECK_HASH = "curvebs.checkhash" + CURVEBS_DEFAULT_CHECK_HASH = false ) var ( @@ -198,6 +203,8 @@ var ( CURVEBS_DEST: VIPER_CURVEBS_DEST, CURVEBS_TASKID: VIPER_CURVEBS_TASKID, CURVEBS_FAILED: VIPER_CURVEBS_FAILED, + CURVEBS_CHUNK_SIZE: VIPER_CURVEBS_CHUNK_SIZE, + CURVEBS_CHECK_HASH: VIPER_CURVEBS_CHECK_HASH, } BSFLAG2DEFAULT = map[string]interface{}{ @@ -221,6 +228,7 @@ var ( CURVEBS_ALL: CURVEBS_DEFAULT_ALL, CURVEBS_LOGIC_POOL_ID: CURVEBS_DEFAULT_LOGIC_POOL_ID, CURVEBS_COPYSET_ID: CURVEBS_DEFAULT_COPYSET_ID, + CURVEBS_CHECK_HASH: CURVEBS_DEFAULT_CHECK_HASH, } ) @@ -558,6 +566,10 @@ func AddBsPeersConfFlag(cmd *cobra.Command) { AddBsStringSliceRequiredFlag(cmd, CURVEBS_PEERS_ADDRESS, "peers info.") } +func AddBsPeersAddressFlag(cmd *cobra.Command) { + AddBsStringRequiredFlag(cmd, CURVEBS_PEERS_ADDRESS, "peers address.") +} + func AddBsForceDeleteOptionFlag(cmd *cobra.Command) { AddBsBoolOptionFlag(cmd, CURVEBS_FORCE, "whether to force delete the file") } @@ -614,6 +626,18 @@ func AddBsChunkIdSliceRequiredFlag(cmd *cobra.Command) { AddBsStringSliceRequiredFlag(cmd, CURVEBS_CHUNK_ID, "chunk ids") } +func AddBsChunkIdRequiredFlag(cmd *cobra.Command) { + AddBsUint64RequiredFlag(cmd, CURVEBS_CHUNK_ID, "chunk id") +} + +func AddBsChunkSizeRequiredFlag(cmd *cobra.Command) { + AddBsUint32RequiredFlag(cmd, CURVEBS_CHUNK_SIZE, "chunk size") +} + +func AddBsCheckHashOptionFlag(cmd *cobra.Command) { + AddBsBoolOptionFlag(cmd, CURVEBS_CHECK_HASH, "whether to check chunk hash(take longer time)") +} + func AddBsChunkServerAddressSliceRequiredFlag(cmd *cobra.Command) { AddBsStringSliceRequiredFlag(cmd, CURVEBS_CHUNKSERVER_ADDRESS, "chunk server address") } From c63296b57c24e164eda4412f13a3f0484f9b7510 Mon Sep 17 00:00:00 2001 From: Xu Yifeng Date: Thu, 19 Oct 2023 08:55:16 +0000 Subject: [PATCH 5/6] Add ARC, an Adaptive Replacement Cache. Signed-off-by: Xu Yifeng --- src/common/arc_cache.h | 460 +++++++++++++++++++++++++++++++++++++++++ src/common/lru_cache.h | 2 + 2 files changed, 462 insertions(+) create mode 100644 src/common/arc_cache.h diff --git a/src/common/arc_cache.h b/src/common/arc_cache.h new file mode 100644 index 0000000000..98e4eb7bfc --- /dev/null +++ b/src/common/arc_cache.h @@ -0,0 +1,460 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_COMMON_ARC_CACHE_H_ +#define SRC_COMMON_ARC_CACHE_H_ + +/* + * Project: curve + * Created Date: 20231018 + * Author: xuyifeng + */ + +#include +#include + +#include +#include +#include +#include + +template , + typename ValueTraits = CacheTraits> +class ARCCache : public LRUCacheInterface { + public: + ARCCache(int64_t max_count, + std::shared_ptr cacheMetrics = nullptr) + : c_(max_count/2), p_(0), cacheMetrics_(cacheMetrics) {} + + void Put(const K& key, const V& value); + bool Put(const K& key, const V& value, V* eliminated); + bool Get(const K& key, V* value); + void Remove(const K& key); + uint64_t Size(); + std::shared_ptr GetCacheMetrics() const; + + bool GetLast(const V value, K* key); + bool GetLast(K* key, V* value); + bool GetLast(K* key, V* value, bool (*f)(const V& value)); + + private: + struct BMapVal; + struct TMapVal; + struct BListVal; + struct TListVal; + + typedef std::unordered_map BMap; + typedef typename BMap::iterator bmap_iter; + typedef std::unordered_map TMap; + typedef typename TMap::iterator tmap_iter; + + typedef typename std::list BList; + typedef typename BList::iterator blist_iter; + typedef typename std::list TList; + typedef typename TList::iterator tlist_iter; + + struct BMapVal { + blist_iter list_iter; + + BMapVal() {} + BMapVal(const BMapVal& o) { list_iter = o.list_iter; } + BMapVal& operator=(const BMapVal& o) { list_iter = o.list_iter; } + BMapVal(const blist_iter& iter) // NOLINT + : list_iter(iter) {} + }; + struct TMapVal { + tlist_iter list_iter; + + TMapVal() {} + TMapVal(const TMapVal& o) { list_iter = o.list_iter; } + TMapVal& operator=(const TMapVal& o) { + list_iter = o.list_iter; + return *this; + } + TMapVal(const tlist_iter& iter) // NOLINT + : list_iter(iter) {} + }; + struct BListVal { + bmap_iter map_iter; + + BListVal() {} + BListVal(const BListVal& o) { map_iter = o.map_iter; } + BListVal& operator=(const BListVal& o) { + map_iter = o.map_iter; + return *this; + } + BListVal(const bmap_iter& iter) // NOLINT + :map_iter(iter) {} + }; + struct TListVal { + tmap_iter map_iter; + V value; + + TListVal() {} + TListVal(const TListVal& o) { + map_iter = o.map_iter; + value = o.value; + } + TListVal(const tmap_iter& iter, const V& v) + : map_iter(iter), value(v) {} + TListVal(const V& v) // NOLINT + : value(v) {} + TListVal& operator=(const TListVal& o) { + map_iter = o.map_iter; + value = o.value; + return *this; + } + }; + + struct B { + BMap map; + BList list; + + bool Find(const K& k, bmap_iter* iter) { + auto it = map.find(k); + if (it != map.end()) { + if (iter != nullptr) *iter = it; + return true; + } + return false; + } + + void Insert(const K& k) { + auto r = map.insert({k, blist_iter()}); + assert(r.second); + list.push_back(r.first); + r.first->second.list_iter = --list.end(); + } + + void RemoveLRU(CacheMetrics* m) { + if (list.empty()) return; + if (m) { + m->UpdateRemoveFromCacheBytes( + KeyTraits::CountBytes(list.front().map_iter->first)); + } + map.erase(list.front().map_iter); + list.pop_front(); + } + + void Remove(bmap_iter&& map_iter, CacheMetrics* m) { + if (m) { + m->UpdateRemoveFromCacheBytes( + KeyTraits::CountBytes(map_iter->first)); + } + list.erase(map_iter->second.list_iter); + map.erase(map_iter); + } + + int Count() const { return map.size(); } + }; + + struct T { + TMap map; + TList list; + + void Insert(const K& k, const V& v, CacheMetrics* m) { + auto r = map.insert({k, tlist_iter()}); + assert(r.second); + list.emplace_back(r.first, v); + r.first->second.list_iter = --list.end(); + if (m != nullptr) { + m->UpdateAddToCacheCount(); + m->UpdateAddToCacheBytes(KeyTraits::CountBytes(k) + + ValueTraits::CountBytes(v)); + } + } + + bool Find(const K& k, tmap_iter* map_iter) { + auto it = map.find(k); + if (it != map.end()) { + if (map_iter != nullptr) *map_iter = it; + return true; + } + + return false; + } + + void Remove(tmap_iter&& map_iter, CacheMetrics* m) { + if (m != nullptr) { + m->UpdateRemoveFromCacheCount(); + m->UpdateRemoveFromCacheBytes( + KeyTraits::CountBytes(map_iter->first) + + ValueTraits::CountBytes(map_iter->second.list_iter->value)); + } + list.erase(map_iter->second.list_iter); + map.erase(map_iter); + } + + bool RemoveLRU(V* eliminated, CacheMetrics* m) { + if (list.empty()) return false; + if (eliminated != nullptr) { + *eliminated = list.front().value; + } + if (m != nullptr) { + m->UpdateRemoveFromCacheCount(); + m->UpdateRemoveFromCacheBytes( + KeyTraits::CountBytes(list.front().map_iter->first) + + ValueTraits::CountBytes(list.front().value)); + } + map.erase(list.front().map_iter); + list.pop_front(); + return true; + } + + void Touch(const tmap_iter& map_iter, const V* v, CacheMetrics* m) { + auto list_iter = --list.end(); + if (v && m) { + uint64_t oldSize = + ValueTraits::CountBytes(map_iter->second.list_iter->value); + uint64_t newSize = ValueTraits::CountBytes(*v); + if (oldSize != newSize) { + m->UpdateRemoveFromCacheBytes(oldSize); + m->UpdateAddToCacheBytes(newSize); + } + } + if (list_iter == map_iter->second.list_iter) { + if (v != nullptr) list_iter->value = *v; + return; + } + if (v != nullptr) { + list.push_back(*v); + } else { + list.push_back(map_iter->second.list_iter->value); + } + list.erase(map_iter->second.list_iter); + map_iter->second.list_iter = --list.end(); + } + + int Count() const { return map.size(); } + + tmap_iter GetLRU() const { return list.begin()->map_iter; } + }; + + bool Move_T_B(T* t, B* b, V* eliminated) { + // move t's LRU item to b as MRU item + if (t->Count() == 0) return false; + + auto map_iter = t->GetLRU(); + if (cacheMetrics_ != nullptr) { + cacheMetrics_->UpdateRemoveFromCacheCount(); + // Key is not removed, only value is. + cacheMetrics_->UpdateRemoveFromCacheBytes( + ValueTraits::CountBytes(map_iter->second.list_iter->value)); + } + if (eliminated) { + *eliminated = map_iter->second.list_iter->value; + } + b->Insert(map_iter->first); + t->Remove(std::move(map_iter), nullptr); + return true; + } + + bool Replace(const K& k, V* eliminated); + + ::curve::common::RWLock lock_; + int64_t c_; + int64_t p_; + B b1_, b2_; + T t1_, t2_; + std::shared_ptr cacheMetrics_; +}; + +template +bool ARCCache::Get(const K& key, V* value) { + ::curve::common::WriteLockGuard guard(lock_); + tmap_iter it; + + if (t1_.Find(key, &it)) { + if (value) *value = it->second.list_iter->value; + t2_.Insert(key, it->second.list_iter->value, nullptr); + t1_.Remove(std::move(it), nullptr); + if (cacheMetrics_ != nullptr) { + cacheMetrics_->OnCacheHit(); + } + return true; + } + if (t2_.Find(key, &it)) { + if (value) *value = it->second.list_iter->value; + t2_.Touch(it, nullptr, nullptr); + if (cacheMetrics_ != nullptr) { + cacheMetrics_->OnCacheHit(); + } + return true; + } + if (cacheMetrics_ != nullptr) { + cacheMetrics_->OnCacheMiss(); + } + return false; +} + +template +void ARCCache::Put(const K& key, const V& value) { + Put(key, value, nullptr); +} + +template +bool ARCCache::Put(const K& key, const V& value, + V* eliminated) { + ::curve::common::WriteLockGuard guard(lock_); + tmap_iter it; + bool ret = false; + + if (t1_.Find(key, &it)) { + t2_.Insert(key, value, nullptr); + t1_.Remove(std::move(it), nullptr); + return false; + } + if (t2_.Find(key, &it)) { + t2_.Touch(it, &value, cacheMetrics_.get()); + return false; + } + + bmap_iter b_it; + if (b1_.Find(key, &b_it)) { + if (b1_.Count() >= b2_.Count()) { + p_ += 1; + } else { + p_ += b2_.Count() / b1_.Count(); + } + if (p_ > c_) p_ = c_; + + ret = Replace(key, eliminated); + b1_.Remove(std::move(b_it), cacheMetrics_.get()); + t2_.Insert(key, value, cacheMetrics_.get()); + return ret; + } + + if (b2_.Find(key, &b_it)) { + if (b2_.Count() >= b1_.Count()) { + p_ -= 1; + } else { + p_ -= b1_.Count() / b2_.Count(); + } + if (p_ < 0) p_ = 0; + + ret = Replace(key, eliminated); + b2_.Remove(std::move(b_it), cacheMetrics_.get()); + t2_.Insert(key, value, cacheMetrics_.get()); + return ret; + } + + if (t1_.Count() + b1_.Count() == c_) { + if (t1_.Count() < c_) { + b1_.RemoveLRU(cacheMetrics_.get()); + ret = Replace(key, eliminated); + } else { + ret = t1_.RemoveLRU(eliminated, cacheMetrics_.get()); + } + } else if (t1_.Count() + b1_.Count() < c_) { + auto total = t1_.Count() + b1_.Count() + t2_.Count() + b2_.Count(); + if (total >= c_) { + if (total == 2 * c_) b2_.RemoveLRU(cacheMetrics_.get()); + Replace(key, eliminated); + } + } + t1_.Insert(key, value, cacheMetrics_.get()); + return ret; +} + +template +bool ARCCache::Replace(const K& k, + V* eliminated) { + if (t1_.Count() != 0 && + ((t1_.Count() > p_) || (b2_.Find(k, nullptr) && t1_.Count() == p_))) { + return Move_T_B(&t1_, &b1_, eliminated); + } else { + return Move_T_B(&t2_, &b2_, eliminated); + } +} + +template +uint64_t ARCCache::Size() { + ::curve::common::ReadLockGuard guard(lock_); + return t1_.Count() + t2_.Count(); +} + +template +void ARCCache::Remove(const K& key) { + ::curve::common::WriteLockGuard guard(lock_); + T* ts[]{&t1_, &t2_}; + for (auto t : ts) { + tmap_iter it; + if (t->Find(key, &it)) { + t->Remove(std::move(it), cacheMetrics_.get()); + return; + } + } +} + +template +std::shared_ptr +ARCCache::GetCacheMetrics() const { + return cacheMetrics_; +} + +template +bool ARCCache::GetLast(const V value, K* key) { + // Note! this is not very precise, since ARC is not strictly + // time ordered LRU + ::curve::common::ReadLockGuard guard(lock_); + T* ts[]{&t1_, &t2_}; + for (auto t : ts) { + for (const auto& item : t->list) { + if (item.value == value) { + *key = item->map_iter->first; + return true; + } + } + } + return false; +} + +template +bool ARCCache::GetLast(K* key, V* value) { + ::curve::common::ReadLockGuard guard(lock_); + + T* ts[]{&t1_, &t2_}; + for (auto t : ts) { + if (!t->list.empty()) { + *key = t->list.front().map_iter->first; + *value = t->list.front().value; + return true; + } + } + + return false; +} + +template +bool ARCCache::GetLast( + K* key, V* value, bool (*f)(const V& value)) { + ::curve::common::ReadLockGuard guard(lock_); + + T* ts[]{&t1_, &t2_}; + for (auto t : ts) { + for (const auto& item : t->list) { + bool ok = f(item.value); + if (ok) { + *key = item.map_iter->first; + *value = item.value; + return true; + } + } + } + return false; +} + +#endif // SRC_COMMON_ARC_CACHE_H_ diff --git a/src/common/lru_cache.h b/src/common/lru_cache.h index 6f648264ec..484e166a6b 100644 --- a/src/common/lru_cache.h +++ b/src/common/lru_cache.h @@ -839,6 +839,8 @@ void SglLRUCache::RemoveElement( size_--; } +#include "src/common/arc_cache.h" + } // namespace common } // namespace curve From f9ca5d8ad986bc6add1c3f1a5544c710529ed524 Mon Sep 17 00:00:00 2001 From: Cyber-SiKu Date: Fri, 27 Oct 2023 10:27:12 +0800 Subject: [PATCH 6/6] [fix]curvefs/monitor: target_json.py Signed-off-by: Cyber-SiKu --- curvefs/monitor/target_json.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/curvefs/monitor/target_json.py b/curvefs/monitor/target_json.py index ebb726229d..1447219f84 100644 --- a/curvefs/monitor/target_json.py +++ b/curvefs/monitor/target_json.py @@ -102,11 +102,14 @@ def loadPlugin(): # load *.json file in plugin dir # merge to one json data = [] - for filename in os.listdir(PLUGIN_PATH): - if filename.endswith('.json'): - with open(os.path.join(PLUGIN_PATH, filename)) as f: - plugin_data = json.load(f) - data.append(unitValue(plugin_data["labels"], plugin_data["targets"]) + if os.path.isdir(PLUGIN_PATH): + for filename in os.listdir(PLUGIN_PATH): + if filename.endswith('.json'): + with open(os.path.join(PLUGIN_PATH, filename)) as f: + plugin_data = json.load(f) + if len(plugin_data) == 0: + continue + data.append(unitValue(plugin_data["labels"], plugin_data["targets"])) return data def refresh():