Skip to content

Commit

Permalink
Update aws.py (#38)
Browse files Browse the repository at this point in the history
* Update aws.py

add function for users to send message to aws sqs

* Update aws.py

fix the format

* Update aws.py

reformat
  • Loading branch information
xiaowei-zillow authored Jul 19, 2023
1 parent 7f687fb commit 0503846
Showing 1 changed file with 46 additions and 9 deletions.
55 changes: 46 additions & 9 deletions zdatasets/utils/aws.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,17 +37,14 @@ def get_aws_session(role_arn: str = None, profile_name: str = None) -> Session:
role_arn=role_arn,
)

# Retrieve crednetials of the assumed role and auto-refresh
credentials = DeferredRefreshableCredentials(
method="assume-role", refresh_using=fetcher.fetch_credentials
# Create new session with assumed role and auto-refresh
botocore_session = Session()
botocore_session._credentials = DeferredRefreshableCredentials(
method="assume-role",
refresh_using=fetcher.fetch_credentials,
)

return boto3.Session(
aws_access_key_id=credentials.access_key,
aws_secret_access_key=credentials.secret_key,
aws_session_token=credentials.token,
region_name=region_name,
)
return boto3.Session(botocore_session=botocore_session, region_name=region_name)


def get_aws_client(role_arn: str, service: str):
Expand All @@ -63,6 +60,46 @@ def get_aws_client(role_arn: str, service: str):
return session.client(service)


def send_sqs_message(
queue_url: str,
message_body: str,
*, # keyword-only parameters
role_arn: str = None,
profile_name: str = None,
) -> None:
"""
Args:
queue_url: the url of SQS to write messages to,
i.e.'https://sqs.<region>.amazonaws.com/<account_id>/<queue_name>'
message_body: i.e. ‘{"date_key": "2023-01-01", "accuracy_threshold": 0.5}'
role_arn: optional
profile_name: optional, mainly for testing/debugging purposes
Returns: None
Exceptions:
botocore.exceptions.ClientError # no permissions to assume role or to SendMessage to SQS
AWS.SimpleQueueService.NonExistentQueue
SQS.Client.exceptions.InvalidMessageContents
SQS.Client.exceptions.UnsupportedOperation
"""
# Create session from given iam role and/or aws profile
session = get_aws_session(role_arn, profile_name)

# Create an SQS client from the session
sqs = session.client("sqs")

# Send message to SQS queue
try:
response = sqs.send_message(QueueUrl=queue_url, MessageBody=message_body)

_logger.debug(
f"Successfully sent the message {message_body} "
f"to sqs {queue_url} with MessageId {response['MessageId']}"
)
except Exception as err:
_logger.error(f"Failed to send the message {message_body} to sqs {queue_url}")
raise err


def get_paginated_list_objects_iterator(
s3_client: boto3.session.Session.client, search: Optional[str] = "Content", **kwargs
) -> Iterable:
Expand Down

0 comments on commit 0503846

Please sign in to comment.