Skip to content

Commit 6a3521a

Browse files
GuyMosesGuyMoses
authored
adding s3 & cloudfront event parsers (#147)
1 parent 8335d06 commit 6a3521a

File tree

2 files changed

+257
-5
lines changed

2 files changed

+257
-5
lines changed

‎src/lumigo_tracer/parsers/event_parser.py‎

Lines changed: 91 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,32 @@
2121
"requestContext",
2222
"headers",
2323
]
24+
25+
CLOUDFRONT_KEYS_ORDER = str_to_list(os.environ.get("LUMIGO_CLOUDFRONT_KEYS_ORDER", "")) or [
26+
"config"
27+
]
28+
29+
CLOUDFRONT_REQUEST_KEYS_ORDER = str_to_list(
30+
os.environ.get("LUMIGO_CLOUDFRONT_REQUEST_KEYS_ORDER", "")
31+
) or ["body", "clientIp", "method", "querystring", "uri"]
32+
33+
S3_KEYS_ORDER = str_to_list(os.environ.get("LUMIGO_S3_KEYS_ORDER", "")) or [
34+
"awsRegion",
35+
"eventTime",
36+
"eventName",
37+
"userIdentity",
38+
"requestParameters",
39+
]
40+
41+
S3_BUCKET_KEYS_ORDER = str_to_list(os.environ.get("LUMIGO_S3_BUCKET_KEYS_ORDER", "")) or [
42+
"arn",
43+
]
44+
45+
S3_OBJECT_KEYS_ORDER = str_to_list(os.environ.get("LUMIGO_S3_OBJECT_KEYS_ORDER", "")) or [
46+
"key",
47+
"size",
48+
]
49+
2450
API_GW_PREFIX_KEYS_HEADERS_DELETE_KEYS = str_to_list(
2551
os.environ.get("LUMIGO_API_GW_PREFIX_KEYS_HEADERS_DELETE_KEYS", "")
2652
) or ("cookie", "x-amz", "accept", "cloudfront", "via", "x-forwarded", "sec-")
@@ -57,12 +83,67 @@ def parse(event) -> Dict:
5783
raise NotImplementedError()
5884

5985

86+
class S3Handler(EventParseHandler):
87+
@staticmethod
88+
def is_supported(event) -> bool:
89+
return safe_get(event, ["Records", 0, "eventSource"]) == "aws:s3"
90+
91+
@staticmethod
92+
def parse(event) -> Dict:
93+
new_event: OrderedDict = OrderedDict()
94+
new_event["Records"] = []
95+
96+
for rec in event.get("Records", []):
97+
new_s3_record_event: OrderedDict = OrderedDict()
98+
for key in S3_KEYS_ORDER:
99+
if rec.get(key) is not None:
100+
new_s3_record_event[key] = rec.get(key)
101+
if rec.get("s3"):
102+
new_s3_record_event["s3"] = {}
103+
if rec["s3"].get("bucket") is not None:
104+
new_s3_record_event["s3"]["bucket"] = {}
105+
for key in S3_BUCKET_KEYS_ORDER:
106+
new_s3_record_event["s3"]["bucket"][key] = rec["s3"]["bucket"].get(key)
107+
if rec["s3"].get("object") is not None:
108+
new_s3_record_event["s3"]["object"] = {}
109+
for key in S3_OBJECT_KEYS_ORDER:
110+
new_s3_record_event["s3"]["object"][key] = rec["s3"]["object"].get(key)
111+
new_event["Records"].append(new_s3_record_event)
112+
return new_event
113+
114+
115+
class CloudfrontHandler(EventParseHandler):
116+
@staticmethod
117+
def is_supported(event) -> bool:
118+
return bool(safe_get(event, ["Records", 0, "cf", "config", "distributionId"], {}))
119+
120+
@staticmethod
121+
def parse(event) -> Dict:
122+
new_event: OrderedDict = OrderedDict()
123+
new_event["Records"] = []
124+
125+
for rec in event.get("Records", []):
126+
cf_record = rec.get("cf", {})
127+
new_cloudfront_record_event: OrderedDict = OrderedDict()
128+
new_cloudfront_record_event["cf"] = {}
129+
for key in CLOUDFRONT_KEYS_ORDER:
130+
if cf_record.get(key):
131+
new_cloudfront_record_event["cf"][key] = cf_record.get(key)
132+
if cf_record.get("request") is not None:
133+
new_cloudfront_record_event["cf"]["request"] = {}
134+
for key in CLOUDFRONT_REQUEST_KEYS_ORDER:
135+
if cf_record["request"].get(key) is not None:
136+
new_cloudfront_record_event["cf"]["request"][key] = cf_record[
137+
"request"
138+
].get(key)
139+
new_event["Records"].append(new_cloudfront_record_event)
140+
return new_event
141+
142+
60143
class ApiGWHandler(EventParseHandler):
61144
@staticmethod
62145
def is_supported(event) -> bool:
63-
if is_api_gw_event(event=event): # noqa
64-
return True
65-
return False
146+
return is_api_gw_event(event=event)
66147

67148
@staticmethod
68149
def parse(event) -> Dict:
@@ -129,7 +210,13 @@ def parse(event) -> Dict:
129210
class EventParser:
130211
@staticmethod
131212
def parse_event(event: Dict, handlers: List[EventParseHandler] = None):
132-
handlers = handlers or [ApiGWHandler(), SNSHandler(), SQSHandler()]
213+
handlers = handlers or [
214+
ApiGWHandler(),
215+
SNSHandler(),
216+
SQSHandler(),
217+
S3Handler(),
218+
CloudfrontHandler(),
219+
]
133220
for handler in handlers:
134221
try:
135222
if handler.is_supported(event):

‎src/test/unit/parsers/test_event_parser.py‎

Lines changed: 166 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,14 @@
22
from collections import OrderedDict
33
from typing import Dict
44

5-
from lumigo_tracer.parsers.event_parser import EventParser, EventParseHandler
5+
import pytest
6+
7+
from lumigo_tracer.parsers.event_parser import (
8+
EventParser,
9+
EventParseHandler,
10+
CloudfrontHandler,
11+
S3Handler,
12+
)
613

714

815
class ExceptionHandler(EventParseHandler):
@@ -404,3 +411,161 @@ def test_parse_event_sqs():
404411
}
405412
)
406413
)
414+
415+
416+
def test_is_s3_event(s3_event):
417+
assert S3Handler().is_supported(s3_event) is True
418+
419+
420+
def test_parse_s3_event(s3_event):
421+
ordered_s3_event = EventParser.parse_event(event=s3_event)
422+
assert json.dumps(ordered_s3_event) == json.dumps(
423+
OrderedDict(
424+
{
425+
"Records": [
426+
{
427+
"awsRegion": "us-west-2",
428+
"eventTime": "2020-09-24T12:00:12.137Z",
429+
"eventName": "ObjectCreated:Put",
430+
"userIdentity": {"principalId": "A2QVTU9T5VMOU3"},
431+
"requestParameters": {"sourceIPAddress": "77.127.93.97"},
432+
"s3": {
433+
"bucket": {"arn": "arn:aws:s3:::testingbuckets3testing"},
434+
"object": {
435+
"key": "Screen+Shot+2020-05-27+at+12.37.36.png",
436+
"size": 61211,
437+
},
438+
},
439+
}
440+
]
441+
}
442+
)
443+
)
444+
445+
446+
def test_is_cloudfront_event(cloudfront_event):
447+
assert CloudfrontHandler().is_supported(cloudfront_event) is True
448+
449+
450+
def test_parse_cloudfront_event(cloudfront_event):
451+
ordered_cloudfront_event = EventParser.parse_event(event=cloudfront_event)
452+
assert json.dumps(ordered_cloudfront_event) == json.dumps(
453+
OrderedDict(
454+
{
455+
"Records": [
456+
{
457+
"cf": {
458+
"config": {
459+
"distributionDomainName": "d3f1hyel7d5adt.cloudfront.net",
460+
"distributionId": "E8PDQHVQH1V0Q",
461+
"eventType": "origin-request",
462+
"requestId": "hnql0vH8VDvTTLGwmKn337OH08mMiV5sTPsYGyBqCKgCXPZbfNqYlw==",
463+
},
464+
"request": {
465+
"body": {
466+
"action": "read-only",
467+
"data": "",
468+
"encoding": "base64",
469+
"inputTruncated": False,
470+
},
471+
"clientIp": "176.12.196.206",
472+
"method": "GET",
473+
"querystring": "",
474+
"uri": "/favicon.ico",
475+
},
476+
}
477+
}
478+
]
479+
}
480+
)
481+
)
482+
483+
484+
@pytest.fixture
485+
def s3_event():
486+
return {
487+
"Records": [
488+
{
489+
"eventVersion": "2.1",
490+
"eventSource": "aws:s3",
491+
"awsRegion": "us-west-2",
492+
"eventTime": "2020-09-24T12:00:12.137Z",
493+
"eventName": "ObjectCreated:Put",
494+
"userIdentity": {"principalId": "A2QVTU9T5VMOU3"},
495+
"requestParameters": {"sourceIPAddress": "77.127.93.97"},
496+
"responseElements": {
497+
"x-amz-request-id": "318F33BA8C4CBDC5",
498+
"x-amz-id-2": "VyRyYV/2vjikRUkRoH2WPH6M5WcAjNSGXG8Qtd1uEfbklnTusaDEz/jQPdLQgf2tZLjRuq4MgZFcVFpQJgZLJfiGUoh7IBhU",
499+
},
500+
"s3": {
501+
"s3SchemaVersion": "1.0",
502+
"configurationId": "a078ca2d-53a8-45d4-a621-260a439876d8",
503+
"bucket": {
504+
"name": "testingbuckets3testing",
505+
"ownerIdentity": {"principalId": "A2QVTU9T5VMOU3"},
506+
"arn": "arn:aws:s3:::testingbuckets3testing",
507+
},
508+
"object": {
509+
"key": "Screen+Shot+2020-05-27+at+12.37.36.png",
510+
"size": 61211,
511+
"eTag": "714ee5196a5c0a6e6b9019caa7b6e970",
512+
"sequencer": "005F6C8A510EE02021",
513+
},
514+
},
515+
}
516+
]
517+
}
518+
519+
520+
@pytest.fixture
521+
def cloudfront_event():
522+
return {
523+
"Records": [
524+
{
525+
"cf": {
526+
"config": {
527+
"distributionDomainName": "d3f1hyel7d5adt.cloudfront.net",
528+
"distributionId": "E8PDQHVQH1V0Q",
529+
"eventType": "origin-request",
530+
"requestId": "hnql0vH8VDvTTLGwmKn337OH08mMiV5sTPsYGyBqCKgCXPZbfNqYlw==",
531+
},
532+
"request": {
533+
"body": {
534+
"action": "read-only",
535+
"data": "",
536+
"encoding": "base64",
537+
"inputTruncated": False,
538+
},
539+
"clientIp": "176.12.196.206",
540+
"headers": {
541+
"x-forwarded-for": [
542+
{"key": "X-Forwarded-For", "value": "176.12.196.206"}
543+
],
544+
"user-agent": [{"key": "User-Agent", "value": "Amazon CloudFront"}],
545+
"via": [
546+
{
547+
"key": "Via",
548+
"value": "1.1 67f7ae71b3a190dab6b84c5ceb7fd5e0.cloudfront.net (CloudFront)",
549+
}
550+
],
551+
"accept-encoding": [{"key": "Accept-Encoding", "value": "gzip"}],
552+
"host": [
553+
{"key": "Host", "value": "my-cloudfront-demo-test.s3.amazonaws.com"}
554+
],
555+
},
556+
"method": "GET",
557+
"origin": {
558+
"s3": {
559+
"authMethod": "none",
560+
"customHeaders": {},
561+
"domainName": "my-cloudfront-demo-test.s3.amazonaws.com",
562+
"path": "",
563+
}
564+
},
565+
"querystring": "",
566+
"uri": "/favicon.ico",
567+
},
568+
}
569+
}
570+
]
571+
}

0 commit comments

Comments
 (0)