diff --git a/src/shadowbox/Taskfile.yml b/src/shadowbox/Taskfile.yml index fb72a9b87..432dc58e3 100644 --- a/src/shadowbox/Taskfile.yml +++ b/src/shadowbox/Taskfile.yml @@ -60,6 +60,7 @@ tasks: SB_PUBLIC_IP: localhost SB_CERTIFICATE_FILE: '{{joinPath .RUN_DIR "/shadowbox-selfsigned-dev.crt"}}' SB_PRIVATE_KEY_FILE: '{{joinPath .RUN_DIR "/shadowbox-selfsigned-dev.key"}}' + LOG_LEVEL: '{{.LOG_LEVEL}}' cmds: - echo Target platform is {{.TARGET_OS}}/{{.TARGET_ARCH}} - echo "Using directory {{.RUN_DIR}}" diff --git a/src/shadowbox/server/manager_metrics.spec.ts b/src/shadowbox/server/manager_metrics.spec.ts index ed3107ff7..68b163c47 100644 --- a/src/shadowbox/server/manager_metrics.spec.ts +++ b/src/shadowbox/server/manager_metrics.spec.ts @@ -41,20 +41,7 @@ describe('PrometheusManagerMetrics', () => { const managerMetrics = new PrometheusManagerMetrics( new QueryMapPrometheusClient( { - 'sum(rate(shadowsocks_data_bytes_per_location{dir=~"ct"}[300s]))': { - resultType: 'vector', - result: [ - { - metric: { - location: 'US', - asn: '49490', - asorg: 'Test AS Org', - }, - value: [1739284734, '1234'], - }, - ], - }, - 'sum(increase(shadowsocks_data_bytes_per_location{dir=~"ct"}[0s])) by (location, asn, asorg)': + 'sum(increase(shadowsocks_tunnel_time_seconds_per_location[0s])) by (location, asn, asorg)': { resultType: 'vector', result: [ @@ -68,9 +55,11 @@ describe('PrometheusManagerMetrics', () => { }, ], }, - 'sum(increase(shadowsocks_tunnel_time_seconds_per_location[0s])) by (location, asn, asorg)': + }, + { + 'sum(increase(shadowsocks_data_bytes_per_location{dir=~"ct"}[300s])) by (location, asn, asorg)': { - resultType: 'vector', + resultType: 'matrix', result: [ { metric: { @@ -78,50 +67,13 @@ describe('PrometheusManagerMetrics', () => { asn: '49490', asorg: 'Test AS Org', }, - value: [1738959398, '1000'], + values: [ + [1738959398, '9000'], + [1739284734, '3000'], + ], }, ], }, - 'sum(increase(shadowsocks_data_bytes{dir=~"ct"}[0s])) by (access_key)': { - resultType: 'vector', - result: [ - { - metric: { - access_key: '0', - }, - value: [1738959398, '1000'], - }, - ], - }, - 'sum(increase(shadowsocks_tunnel_time_seconds[0s])) by (access_key)': { - resultType: 'vector', - result: [ - { - metric: { - access_key: '0', - }, - value: [1738959398, '1000'], - }, - ], - }, - }, - { - 'sum(rate(shadowsocks_data_bytes_per_location{dir=~"ct"}[300s]))': { - resultType: 'matrix', - result: [ - { - metric: { - location: 'US', - asn: '49490', - asorg: 'Test AS Org', - }, - values: [ - [1738959398, '5678'], - [1739284734, '1234'], - ], - }, - ], - }, 'sum(increase(shadowsocks_data_bytes{dir=~"ct"}[300s])) by (access_key)': { resultType: 'matrix', result: [ @@ -130,8 +82,8 @@ describe('PrometheusManagerMetrics', () => { access_key: '0', }, values: [ - [1738959398, '1000'], - [1739284734, '2000'], + [1738959398, '9000'], + [1739284734, '3000'], ], }, ], @@ -145,7 +97,7 @@ describe('PrometheusManagerMetrics', () => { }, values: [ [1738959398, '1000'], - [1739284734, '0'], + [1738959398, '0'], ], }, ], @@ -162,18 +114,18 @@ describe('PrometheusManagerMetrics', () => { "seconds": 1000 }, "dataTransferred": { - "bytes": 1000 + "bytes": 12000 }, "bandwidth": { "current": { "data": { - "bytes": 1234 + "bytes": 10 }, "timestamp": 1739284734 }, "peak": { "data": { - "bytes": 5678 + "bytes": 30 }, "timestamp": 1738959398 } @@ -184,7 +136,7 @@ describe('PrometheusManagerMetrics', () => { "asn": 49490, "asOrg": "Test AS Org", "dataTransferred": { - "bytes": 1000 + "bytes": 12000 }, "tunnelTime": { "seconds": 1000 @@ -196,7 +148,7 @@ describe('PrometheusManagerMetrics', () => { { "accessKeyId": 0, "dataTransferred": { - "bytes": 1000 + "bytes": 12000 }, "tunnelTime": { "seconds": 1000 @@ -214,24 +166,11 @@ describe('PrometheusManagerMetrics', () => { done(); }); - it('getServerMetrics - does a full outer join on metric data', async (done) => { + it('getServerMetrics - combines sources', async (done) => { const managerMetrics = new PrometheusManagerMetrics( new QueryMapPrometheusClient( { - 'sum(rate(shadowsocks_data_bytes_per_location{dir=~"ct"}[300s]))': { - resultType: 'vector', - result: [ - { - metric: { - location: 'US', - asn: '49490', - asorg: 'Test AS Org', - }, - value: [1739284734, '1234'], - }, - ], - }, - 'sum(increase(shadowsocks_data_bytes_per_location{dir=~"ct"}[0s])) by (location, asn, asorg)': + 'sum(increase(shadowsocks_tunnel_time_seconds_per_location[0s])) by (location, asn, asorg)': { resultType: 'vector', result: [ @@ -245,58 +184,23 @@ describe('PrometheusManagerMetrics', () => { }, ], }, - 'sum(increase(shadowsocks_tunnel_time_seconds_per_location[0s])) by (location, asn, asorg)': + }, + { + 'sum(increase(shadowsocks_data_bytes_per_location{dir=~"ct"}[300s])) by (location, asn, asorg)': { - resultType: 'vector', + resultType: 'matrix', result: [ { metric: { location: 'CA', }, - value: [1738959398, '1000'], + values: [ + [1738959398, '9000'], + [1739284734, '3000'], + ], }, ], }, - 'sum(increase(shadowsocks_data_bytes{dir=~"ct"}[0s])) by (access_key)': { - resultType: 'vector', - result: [ - { - metric: { - access_key: '0', - }, - value: [1738959398, '1000'], - }, - ], - }, - 'sum(increase(shadowsocks_tunnel_time_seconds[0s])) by (access_key)': { - resultType: 'vector', - result: [ - { - metric: { - access_key: '1', - }, - value: [1738959398, '1000'], - }, - ], - }, - }, - { - 'sum(rate(shadowsocks_data_bytes_per_location{dir=~"ct"}[300s]))': { - resultType: 'matrix', - result: [ - { - metric: { - location: 'US', - asn: '49490', - asorg: 'Test AS Org', - }, - values: [ - [1738959398, '5678'], - [1739284734, '1234'], - ], - }, - ], - }, 'sum(increase(shadowsocks_data_bytes{dir=~"ct"}[300s])) by (access_key)': { resultType: 'matrix', result: [ @@ -305,8 +209,8 @@ describe('PrometheusManagerMetrics', () => { access_key: '0', }, values: [ - [1738959398, '1000'], - [1738959398, '2000'], + [1738959398, '9000'], + [1739284734, '3000'], ], }, ], @@ -316,7 +220,7 @@ describe('PrometheusManagerMetrics', () => { result: [ { metric: { - access_key: '0', + access_key: '1', }, values: [ [1738959398, '1000'], @@ -337,18 +241,18 @@ describe('PrometheusManagerMetrics', () => { "seconds": 1000 }, "dataTransferred": { - "bytes": 1000 + "bytes": 12000 }, "bandwidth": { "current": { "data": { - "bytes": 1234 + "bytes": 10 }, "timestamp": 1739284734 }, "peak": { "data": { - "bytes": 5678 + "bytes": 30 }, "timestamp": 1738959398 } @@ -359,10 +263,10 @@ describe('PrometheusManagerMetrics', () => { "asn": null, "asOrg": null, "dataTransferred": { - "bytes": 0 + "bytes": 12000 }, "tunnelTime": { - "seconds": 1000 + "seconds": 0 } }, { @@ -370,25 +274,25 @@ describe('PrometheusManagerMetrics', () => { "asn": 49490, "asOrg": "Test AS Org", "dataTransferred": { - "bytes": 1000 + "bytes": 0 }, "tunnelTime": { - "seconds": 0 + "seconds": 1000 } } ] }, "accessKeys": [ { - "accessKeyId": 1, + "accessKeyId": 0, "dataTransferred": { - "bytes": 0 + "bytes": 12000 }, "tunnelTime": { - "seconds": 1000 + "seconds": 0 }, "connection": { - "lastTrafficSeen": null, + "lastTrafficSeen": 1739284734, "peakDeviceCount": { "data": 0, "timestamp": null @@ -396,15 +300,15 @@ describe('PrometheusManagerMetrics', () => { } }, { - "accessKeyId": 0, + "accessKeyId": 1, "dataTransferred": { - "bytes": 1000 + "bytes": 0 }, "tunnelTime": { - "seconds": 0 + "seconds": 1000 }, "connection": { - "lastTrafficSeen": 1738959398, + "lastTrafficSeen": null, "peakDeviceCount": { "data": 4, "timestamp": 1738959398 diff --git a/src/shadowbox/server/manager_metrics.ts b/src/shadowbox/server/manager_metrics.ts index 7cbc90ffb..643e910eb 100644 --- a/src/shadowbox/server/manager_metrics.ts +++ b/src/shadowbox/server/manager_metrics.ts @@ -115,38 +115,19 @@ export class PrometheusManagerMetrics implements ManagerMetrics { this.prunePrometheusCache(); const [ - bandwidth, - bandwidthRange, - dataTransferredByLocation, - tunnelTimeByLocation, - dataTransferredByAccessKey, - tunnelTimeByAccessKey, dataTransferredByAccessKeyRange, + dataTransferredByLocationRange, tunnelTimeByAccessKeyRange, + tunnelTimeByLocation, ] = await Promise.all([ - this.cachedPrometheusClient.query( - `sum(rate(shadowsocks_data_bytes_per_location{dir=~"ct"}[${PROMETHEUS_RANGE_QUERY_STEP_SECONDS}s]))` - ), this.cachedPrometheusClient.queryRange( - `sum(rate(shadowsocks_data_bytes_per_location{dir=~"ct"}[${PROMETHEUS_RANGE_QUERY_STEP_SECONDS}s]))`, + `sum(increase(shadowsocks_data_bytes{dir=~"ct"}[${PROMETHEUS_RANGE_QUERY_STEP_SECONDS}s])) by (access_key)`, start, end, `${PROMETHEUS_RANGE_QUERY_STEP_SECONDS}s` ), - this.cachedPrometheusClient.query( - `sum(increase(shadowsocks_data_bytes_per_location{dir=~"ct"}[${timeframe.seconds}s])) by (location, asn, asorg)` - ), - this.cachedPrometheusClient.query( - `sum(increase(shadowsocks_tunnel_time_seconds_per_location[${timeframe.seconds}s])) by (location, asn, asorg)` - ), - this.cachedPrometheusClient.query( - `sum(increase(shadowsocks_data_bytes{dir=~"ct"}[${timeframe.seconds}s])) by (access_key)` - ), - this.cachedPrometheusClient.query( - `sum(increase(shadowsocks_tunnel_time_seconds[${timeframe.seconds}s])) by (access_key)` - ), this.cachedPrometheusClient.queryRange( - `sum(increase(shadowsocks_data_bytes{dir=~"ct"}[${PROMETHEUS_RANGE_QUERY_STEP_SECONDS}s])) by (access_key)`, + `sum(increase(shadowsocks_data_bytes_per_location{dir=~"ct"}[${PROMETHEUS_RANGE_QUERY_STEP_SECONDS}s])) by (location, asn, asorg)`, start, end, `${PROMETHEUS_RANGE_QUERY_STEP_SECONDS}s` @@ -157,6 +138,9 @@ export class PrometheusManagerMetrics implements ManagerMetrics { end, `${PROMETHEUS_RANGE_QUERY_STEP_SECONDS}s` ), + this.cachedPrometheusClient.query( + `sum(increase(shadowsocks_tunnel_time_seconds_per_location[${timeframe.seconds}s])) by (location, asn, asorg)` + ), ]); const serverMetrics: ServerMetricsServerEntry = { @@ -168,51 +152,56 @@ export class PrometheusManagerMetrics implements ManagerMetrics { }, locations: [], }; - for (const result of bandwidth.result) { - if (result.value) { - serverMetrics.bandwidth.current.data.bytes = parseFloat(result.value[1]); - serverMetrics.bandwidth.current.timestamp = result.value[0]; - } - break; // There should only be one result. - } - for (const result of bandwidthRange.result) { - const peakDataTransferred = findPeak(result.values ?? []); - if (peakDataTransferred !== null) { - const peakValue = parseFloat(peakDataTransferred[1]); - if (peakValue > 0) { - serverMetrics.bandwidth.peak.data.bytes = peakValue; - serverMetrics.bandwidth.peak.timestamp = Math.min(now, peakDataTransferred[0]); - } - } - break; // There should only be one result. - } - const locationMap = new Map(); - for (const result of tunnelTimeByLocation.result) { - const entry = getServerMetricsLocationEntry(locationMap, result.metric); - const tunnelTime = result.value ? parseFloat(result.value[1]) : 0; - entry.tunnelTime.seconds = tunnelTime; - serverMetrics.tunnelTime.seconds += tunnelTime; - } - for (const result of dataTransferredByLocation.result) { - const entry = getServerMetricsLocationEntry(locationMap, result.metric); - const bytes = result.value ? parseFloat(result.value[1]) : 0; - entry.dataTransferred.bytes = bytes; - serverMetrics.dataTransferred.bytes += bytes; - } - serverMetrics.locations = Array.from(locationMap.values()); + const bandwidthTimeseriesIndex = new Map(); const accessKeyMap = new Map(); - for (const result of tunnelTimeByAccessKey.result) { + for (const result of dataTransferredByAccessKeyRange.result) { const entry = getServerMetricsAccessKeyEntry(accessKeyMap, result.metric); - entry.tunnelTime.seconds = result.value ? parseFloat(result.value[1]) : 0; + const lastTrafficSeen = findLastNonZero(result.values ?? []); + + entry.connection.lastTrafficSeen = lastTrafficSeen ? Math.min(now, lastTrafficSeen[0]) : null; + entry.dataTransferred.bytes = findSum(result.values ?? []); + + for (const entryIndex in result.values) { + const [timestamp, value] = result.values[entryIndex]; + + bandwidthTimeseriesIndex.set( + timestamp, + bandwidthTimeseriesIndex.has(timestamp) + ? String( + parseFloat(bandwidthTimeseriesIndex.get(timestamp) as string) + parseFloat(value) + ) + : value + ); + } } - for (const result of dataTransferredByAccessKey.result) { - const entry = getServerMetricsAccessKeyEntry(accessKeyMap, result.metric); - entry.dataTransferred.bytes = result.value ? parseFloat(result.value[1]) : 0; + + const bandwidthRangeValues = [...bandwidthTimeseriesIndex.entries()].sort( + (a, b) => a[0] - b[0] + ); + + const currentBandwidth = bandwidthRangeValues[bandwidthRangeValues.length - 1] ?? [0, '0']; + + // convert increase() into rate() + serverMetrics.bandwidth.current.data.bytes = + parseFloat(currentBandwidth[1]) / PROMETHEUS_RANGE_QUERY_STEP_SECONDS; + serverMetrics.bandwidth.current.timestamp = currentBandwidth[0]; + + const peakDataTransferred = findPeak(bandwidthRangeValues); + if (peakDataTransferred !== null) { + const peakValue = parseFloat(peakDataTransferred[1]); + + if (peakValue > 0) { + // convert increase() into rate() + serverMetrics.bandwidth.peak.data.bytes = peakValue / PROMETHEUS_RANGE_QUERY_STEP_SECONDS; + serverMetrics.bandwidth.peak.timestamp = Math.min(now, peakDataTransferred[0]); + } } + for (const result of tunnelTimeByAccessKeyRange.result) { const entry = getServerMetricsAccessKeyEntry(accessKeyMap, result.metric); + const peakTunnelTimeSec = findPeak(result.values ?? []); if (peakTunnelTimeSec !== null) { const peakValue = parseFloat(peakTunnelTimeSec[1]); @@ -222,13 +211,29 @@ export class PrometheusManagerMetrics implements ManagerMetrics { entry.connection.peakDeviceCount.timestamp = Math.min(now, peakTunnelTimeSec[0]); } } + + entry.tunnelTime.seconds = findSum(result.values ?? []); } - for (const result of dataTransferredByAccessKeyRange.result) { - const entry = getServerMetricsAccessKeyEntry(accessKeyMap, result.metric); - const lastTrafficSeen = findLastNonZero(result.values ?? []); - entry.connection.lastTrafficSeen = lastTrafficSeen ? Math.min(now, lastTrafficSeen[0]) : null; + + const locationMap = new Map(); + for (const result of dataTransferredByLocationRange.result) { + const entry = getServerMetricsLocationEntry(locationMap, result.metric); + const bytes = findSum(result.values ?? []); + + entry.dataTransferred.bytes += bytes; + serverMetrics.dataTransferred.bytes += bytes; } + for (const result of tunnelTimeByLocation.result) { + const entry = getServerMetricsLocationEntry(locationMap, result.metric); + const tunnelTime = result.value ? parseFloat(result.value[1]) : 0; + + entry.tunnelTime.seconds = tunnelTime; + serverMetrics.tunnelTime.seconds += tunnelTime; + } + + serverMetrics.locations = Array.from(locationMap.values()); + return { server: serverMetrics, accessKeys: Array.from(accessKeyMap.values()), @@ -350,3 +355,14 @@ function findLastNonZero(values: PrometheusValue[]): PrometheusValue | null { } return null; } + +/** + * Finds the sum of the values in an array of PrometheusValues. + */ +function findSum(values: PrometheusValue[]): number { + let sum = 0; + for (const value of values) { + sum += parseFloat(value[1]); + } + return sum; +} diff --git a/src/shadowbox/server/manager_service.ts b/src/shadowbox/server/manager_service.ts index 91bd25f4b..a246ef791 100644 --- a/src/shadowbox/server/manager_service.ts +++ b/src/shadowbox/server/manager_service.ts @@ -628,7 +628,7 @@ export class ShadowsocksManagerService { } async getServerMetrics(req: RequestType, res: ResponseType, next: restify.Next) { - logging.debug(`getServerMetrics request ${JSON.stringify(req.params)}`); + logging.debug(`getServerMetrics request ${JSON.stringify(req.query)}`); let seconds; try {