-
Notifications
You must be signed in to change notification settings - Fork 454
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add aws.firehose.arn, aws.firehose.request_id and aws.metrics_names_f…
…ingerprint fields (#11239) This PR adds aws.firehose.arn and aws.firehose.request_id field definitions for the firehose integration. This PR also adds aws.metrics_names_fingerprint which is a hash of the list of metric names exist in each document. This way we don't have to count request_id as a dimension.
- Loading branch information
1 parent
610739d
commit d93fd8f
Showing
19 changed files
with
526 additions
and
115 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
82 changes: 82 additions & 0 deletions
82
packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-multiple-metrics.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| { | ||
| "events": [ | ||
| { | ||
| "@timestamp": "2024-09-25T23:12:00.000Z", | ||
| "agent.type": "firehose", | ||
| "cloud.provider": "aws", | ||
| "cloud.account.id": "627286350134", | ||
| "cloud.region": "eu-central-1", | ||
| "aws.exporter.arn": "arn:aws:cloudwatch:eu-central-1:627286350134:metric-stream/CustomFull-KefpMG", | ||
| "aws.cloudwatch.namespace": "AWS/Kinesis", | ||
| "aws.firehose.arn": "arn:aws:firehose:eu-central-1:627286350134:deliverystream/KS-PUT-ELI-2", | ||
| "aws.firehose.request_id": "7628c293-ba54-4ea4-9abe-d256976c1dbd", | ||
| "aws.dimensions": { | ||
| "StreamName": "test-esf-encrypted" | ||
| }, | ||
| "aws.kinesis.metrics.GetRecords_Success": { | ||
| "count": 240, | ||
| "sum": 240, | ||
| "avg": 1, | ||
| "max": 1, | ||
| "min": 1 | ||
| }, | ||
| "aws.kinesis.metrics.ReadProvisionedThroughputExceeded": { | ||
| "count": 240, | ||
| "sum": 0, | ||
| "avg": 0, | ||
| "max": 0, | ||
| "min": 0 | ||
| }, | ||
| "aws.kinesis.metrics.GetRecords_Bytes": { | ||
| "count": 240, | ||
| "sum": 0, | ||
| "avg": 0, | ||
| "max": 0, | ||
| "min": 0 | ||
| }, | ||
| "aws.firehose.parameters.X-Found-Cluster": "bb8e51259abe4c21996954f5cfe90af1", | ||
| "data_stream.type": "metrics", | ||
| "data_stream.dataset": "aws.cloudwatch", | ||
| "data_stream.namespace": "default" | ||
| }, | ||
| { | ||
| "@timestamp": "2024-09-25T23:12:00.000Z", | ||
| "agent.type": "firehose", | ||
| "cloud.provider": "aws", | ||
| "cloud.account.id": "627286350134", | ||
| "cloud.region": "eu-central-1", | ||
| "aws.exporter.arn": "arn:aws:cloudwatch:eu-central-1:627286350134:metric-stream/CustomFull-KefpMG", | ||
| "aws.cloudwatch.namespace": "AWS/Kinesis", | ||
| "aws.firehose.arn": "arn:aws:firehose:eu-central-1:627286350134:deliverystream/KS-PUT-ELI-2", | ||
| "aws.firehose.request_id": "7628c293-ba54-4ea4-9abe-d256976c1dbd", | ||
| "aws.dimensions": { | ||
| "StreamName": "test-esf-encrypted" | ||
| }, | ||
| "aws.kinesis.metrics.ReadProvisionedThroughputExceeded": { | ||
| "count": 240, | ||
| "sum": 0, | ||
| "avg": 0, | ||
| "max": 0, | ||
| "min": 0 | ||
| }, | ||
| "aws.kinesis.metrics.GetRecords_Bytes": { | ||
| "count": 240, | ||
| "sum": 0, | ||
| "avg": 0, | ||
| "max": 0, | ||
| "min": 0 | ||
| }, | ||
| "aws.kinesis.metrics.GetRecords_Success": { | ||
| "count": 240, | ||
| "sum": 240, | ||
| "avg": 1, | ||
| "max": 1, | ||
| "min": 1 | ||
| }, | ||
| "aws.firehose.parameters.X-Found-Cluster": "bb8e51259abe4c21996954f5cfe90af1", | ||
| "data_stream.type": "metrics", | ||
| "data_stream.dataset": "aws.cloudwatch", | ||
| "data_stream.namespace": "default" | ||
| } | ||
| ] | ||
| } |
134 changes: 134 additions & 0 deletions
134
...sfirehose/data_stream/metrics/_dev/test/pipeline/test-multiple-metrics.json-expected.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,134 @@ | ||
| { | ||
| "expected": [ | ||
| { | ||
| "@timestamp": "2024-09-25T23:12:00.000Z", | ||
| "agent": { | ||
| "type": "firehose" | ||
| }, | ||
| "aws": { | ||
| "cloudwatch": { | ||
| "namespace": "AWS/Kinesis" | ||
| }, | ||
| "dimensions": { | ||
| "StreamName": "test-esf-encrypted" | ||
| }, | ||
| "exporter": { | ||
| "arn": "arn:aws:cloudwatch:eu-central-1:627286350134:metric-stream/CustomFull-KefpMG" | ||
| }, | ||
| "firehose": { | ||
| "arn": "arn:aws:firehose:eu-central-1:627286350134:deliverystream/KS-PUT-ELI-2", | ||
| "parameters": { | ||
| "X-Found-Cluster": "bb8e51259abe4c21996954f5cfe90af1" | ||
| }, | ||
| "request_id": "7628c293-ba54-4ea4-9abe-d256976c1dbd" | ||
| }, | ||
| "kinesis": { | ||
| "metrics": { | ||
| "GetRecords_Bytes": { | ||
| "avg": 0, | ||
| "count": 240, | ||
| "max": 0, | ||
| "min": 0, | ||
| "sum": 0 | ||
| }, | ||
| "GetRecords_Success": { | ||
| "avg": 1, | ||
| "count": 240, | ||
| "max": 1, | ||
| "min": 1, | ||
| "sum": 240 | ||
| }, | ||
| "ReadProvisionedThroughputExceeded": { | ||
| "avg": 0, | ||
| "count": 240, | ||
| "max": 0, | ||
| "min": 0, | ||
| "sum": 0 | ||
| } | ||
| } | ||
| }, | ||
| "metrics_names_fingerprint": "jayZkk20Wp5tUj8uJIH97j1aJMU=" | ||
| }, | ||
| "cloud": { | ||
| "account": { | ||
| "id": "627286350134" | ||
| }, | ||
| "provider": "aws", | ||
| "region": "eu-central-1" | ||
| }, | ||
| "data_stream": { | ||
| "dataset": "aws.kinesis", | ||
| "namespace": "default", | ||
| "type": "metrics" | ||
| }, | ||
| "ecs": { | ||
| "version": "8.11.0" | ||
| } | ||
| }, | ||
| { | ||
| "@timestamp": "2024-09-25T23:12:00.000Z", | ||
| "agent": { | ||
| "type": "firehose" | ||
| }, | ||
| "aws": { | ||
| "cloudwatch": { | ||
| "namespace": "AWS/Kinesis" | ||
| }, | ||
| "dimensions": { | ||
| "StreamName": "test-esf-encrypted" | ||
| }, | ||
| "exporter": { | ||
| "arn": "arn:aws:cloudwatch:eu-central-1:627286350134:metric-stream/CustomFull-KefpMG" | ||
| }, | ||
| "firehose": { | ||
| "arn": "arn:aws:firehose:eu-central-1:627286350134:deliverystream/KS-PUT-ELI-2", | ||
| "parameters": { | ||
| "X-Found-Cluster": "bb8e51259abe4c21996954f5cfe90af1" | ||
| }, | ||
| "request_id": "7628c293-ba54-4ea4-9abe-d256976c1dbd" | ||
| }, | ||
| "kinesis": { | ||
| "metrics": { | ||
| "GetRecords_Bytes": { | ||
| "avg": 0, | ||
| "count": 240, | ||
| "max": 0, | ||
| "min": 0, | ||
| "sum": 0 | ||
| }, | ||
| "GetRecords_Success": { | ||
| "avg": 1, | ||
| "count": 240, | ||
| "max": 1, | ||
| "min": 1, | ||
| "sum": 240 | ||
| }, | ||
| "ReadProvisionedThroughputExceeded": { | ||
| "avg": 0, | ||
| "count": 240, | ||
| "max": 0, | ||
| "min": 0, | ||
| "sum": 0 | ||
| } | ||
| } | ||
| }, | ||
| "metrics_names_fingerprint": "jayZkk20Wp5tUj8uJIH97j1aJMU=" | ||
| }, | ||
| "cloud": { | ||
| "account": { | ||
| "id": "627286350134" | ||
| }, | ||
| "provider": "aws", | ||
| "region": "eu-central-1" | ||
| }, | ||
| "data_stream": { | ||
| "dataset": "aws.kinesis", | ||
| "namespace": "default", | ||
| "type": "metrics" | ||
| }, | ||
| "ecs": { | ||
| "version": "8.11.0" | ||
| } | ||
| } | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.