diff --git a/cloud-trail-lookup/README.md b/cloud-trail-lookup/README.md new file mode 100644 index 0000000..12fcaf3 --- /dev/null +++ b/cloud-trail-lookup/README.md @@ -0,0 +1,35 @@ +# Cloud Trail Lookup + +This script will lookup Cloud Trail Event History and find events with "AccessDenied" or "Unauthorized" errors. + +## Why? + +Throttling error may occur with cli commands, the rate limit is 2 per region per account, and it can't be raised upon request. That's where this script comes in handy. Keep in mind the API call uses the same quota in your account, so please keep the time window for lookup as short as possible. + +## What the script does. + +This script iterates through all events found by parameters and list events with "AccessDenied" or "Unauthorized" errors. + +## Usage + +```bash +usage: enable-lambda-log-group.py [-h] [--debug] [--error] [--timestamp] + [--region REGION] [--profile PROFILE] + [--actually-do-it] + +optional arguments: + -h, --help show this help message and exit + --debug print debugging info + --error print error info only + --timestamp Output log with timestamp and toolname + --minutes minutes Minutes to look up till now, default is 30, optional + --user User name to look up in the events, can be email or any format shown in Event History, optional +``` + + +## AWS Docs + +* [boto3 lookup_events()](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudtrail.html#CloudTrail.Client.lookup_events) + + + diff --git a/cloud-trail-lookup/lookup-events.py b/cloud-trail-lookup/lookup-events.py new file mode 100644 index 0000000..6dfd12a --- /dev/null +++ b/cloud-trail-lookup/lookup-events.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 + +import boto3 +from botocore.exceptions import ClientError +import os +import logging +import time +from datetime import datetime as dt, timedelta, time +import json + +# python3 ./lookup-events.py --debug --minutes 10 --user email@warnermedia.com + +def get_arg_minutes(args): + minutes_to_lookup = 30 + try: + if args.minutes: + minutes_to_lookup = int(args.minutes) + except Exception as e: + logger.info("An exception occurred converting arg minutes: ", e) + logger.info(f"minutes to look up: {minutes_to_lookup}") + return minutes_to_lookup + +def get_arg_user(args): + user_to_lookup = "" + if args.user: + user_to_lookup = args.user + logger.info(f"user to look up: {user_to_lookup}") + return user_to_lookup + +def main(args, logger): + # just want to quickly run "aws cloudtrail lookup-events" without ThrottleException + # only care about us-east-1, use default profile + client = boto3.client('cloudtrail') + + minutes_to_lookup = get_arg_minutes(args) + user_to_lookup = get_arg_user(args) + + lastHourDateTime = dt.utcnow() - timedelta(minutes=minutes_to_lookup) + now = dt.utcnow() + startTime = lastHourDateTime + endTime = now + + logger.info(f"start Time: {startTime}") + logger.info(f"end time: {endTime}") + + maxResult = 200 + events_found = [] + # no NextToken at first + event_arg = {"StartTime": startTime, "EndTime": endTime, "MaxResults": maxResult} + if user_to_lookup: + event_arg["LookupAttributes"] = [{ + 'AttributeKey': 'Username', + 'AttributeValue': user_to_lookup + }] + response = client.lookup_events(**event_arg) + + # process response, add to events array + events_found = events_found + search_events(response["Events"]) + + if "NextToken" in response: + nextToken = response["NextToken"] + + logger.debug(nextToken) + + while nextToken: + event_arg["NextToken"] = nextToken + response = client.lookup_events(**event_arg) + + events_found = events_found + search_events(response["Events"]) + + if "NextToken" in response: + nextToken = response["NextToken"] + logger.debug(nextToken) + else: + # loop ends + nextToken = "" + logger.info(f"events_found: {events_found}") + logger.info(f"{len(events_found)} events found") + return events_found + + +def search_events(events): + # list events with error code of "accessdenied" or "unauthorized" + filtered = [] + for event in events: + event_detail_str = event["CloudTrailEvent"] + event_detail = json.loads(event_detail_str) + if "errorCode" in event_detail: + if event_detail["errorCode"].lower() == "accessdenied" or event_detail["errorCode"].lower() == "unauthorized": + filtered.append(event) + return filtered + + +def do_args(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument( + "--debug", help="print debugging info", action='store_true') + parser.add_argument( + "--error", help="print error info only", action='store_true') + parser.add_argument( + "--timestamp", help="Output log with timestamp and toolname", action='store_true') + parser.add_argument( + "--minutes", help="minutes to look up till now") + parser.add_argument( + "--user", help="user name to look up") + + args = parser.parse_args() + + return (args) + + +if __name__ == '__main__': + + args = do_args() + + # create console handler and set level to debug + logger = logging.getLogger('cloud-trail-lookup-events') + ch = logging.StreamHandler() + if args.debug: + logger.setLevel(logging.DEBUG) + elif args.error: + logger.setLevel(logging.ERROR) + else: + logger.setLevel(logging.INFO) + + # Silence Boto3 & Friends + logging.getLogger('botocore').setLevel(logging.WARNING) + logging.getLogger('boto3').setLevel(logging.WARNING) + logging.getLogger('urllib3').setLevel(logging.WARNING) + + # create formatter + if args.timestamp: + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + else: + formatter = logging.Formatter('%(levelname)s - %(message)s') + # add formatter to ch + ch.setFormatter(formatter) + # add ch to logger + logger.addHandler(ch) + + try: + main(args, logger) + except KeyboardInterrupt: + exit(1) diff --git a/ecr-basic-scan/README.md b/ecr-basic-scan/README.md new file mode 100644 index 0000000..e0b816d --- /dev/null +++ b/ecr-basic-scan/README.md @@ -0,0 +1,40 @@ +# ECR Registry Basic Scan + +This script will lookup ECR repos and images given an ECR registry id, then trigger scan manually for each image given that the registry scan configuration is set to Basic scan. + +## Why? + +Manual scans using enhanced scanning isn’t supported. +When continuous scanning is enabled for a repository, if an image hasn’t been updated in the past 30 days based on the image push timestamp, then continuous scanning is suspended for that image. + + +## What the script does not do +Registry scan configuration as Basic scan needs to be set up ahead of time. Note that as of this writing, basic scanning is not able to detect the log4j 2 vulnerability. + + +## Errors +The following error could occur as an image can only be scanned once per day: +"An error occurred (LimitExceededException) when calling the StartImageScan operation: The scan quota per image has been exceeded. Wait and try again." + +## Usage + +```bash +usage: automate-manual-scan.py [-h] [--debug] [--error] [--timestamp] + [--registry-id] + [--actually-do-it] + +optional arguments: + -h, --help show this help message and exit + --debug print debugging info + --error print error info only + --timestamp Output log with timestamp and toolname + --minutes minutes Minutes to look up till now, default is 30, optional + --user User name to look up in the events, can be email or any format shown in Event History, optional +``` + +## AWS Docs + +* [boto3 ECR](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ecr.html) + + + diff --git a/ecr-basic-scan/automate-manual-scan.py b/ecr-basic-scan/automate-manual-scan.py new file mode 100644 index 0000000..f73f64f --- /dev/null +++ b/ecr-basic-scan/automate-manual-scan.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 + +import boto3 +from botocore.exceptions import ClientError +import os +import logging +import time +from datetime import datetime as dt, timedelta, time +import json + +# Assume basic scan setting has been set up in the private registry, otherwise the script throws error +# Trigger manual scan for all images in a repos, catch and display if any errors for each, continue with loop +# python3 automate-manual-scan.py --registry-id +def start_image_scan(client, registry_id, repo_name, image_digest): + try: + response = client.start_image_scan( + registryId=registry_id, + repositoryName=repo_name, + imageId={ + 'imageDigest': image_digest + } + ) + logger.info(response) + except Exception as e: + logger.info(f'An exception occurred starting image scan in repo {repo_name} with image digest {image_digest}') + + +def list_ecr_repos(client, registry_id): + MAX_RESULT = 100 + repo_names = [] + payload = { + 'registryId': registry_id, + 'maxResults': MAX_RESULT + } + try: + response = client.describe_repositories(**payload) + repos = response['repositories'] + repo_names = [repo['repositoryName'] for repo in repos] + if "NextToken" in response: + nextToken = response["NextToken"] + + while nextToken: + payload["NextToken"] = nextToken + response = client.describe_repositories(**payload) + repo_names = repo_names + [repo['repositoryName'] for repo in repos] + + if "NextToken" in response: + nextToken = response["NextToken"] + logger.debug(nextToken) + else: + # loop ends + nextToken = "" + return repo_names + except Exception as e: + logger.info(f'An exception occurred in list_ecr_repos') + + +""" + returned array in this format + [ + { + 'imageDigest': 'string', + 'imageTag': 'string' + }, + ], +""" +def list_images(client, registry_id, repo_name): + MAX_RESULT = 100 + image_ids = [] + payload = { + 'registryId': registry_id, + 'repositoryName': repo_name, + 'maxResults': MAX_RESULT, + "filter": { + 'tagStatus': 'ANY' + } + } + try: + response = client.list_images(**payload) + image_ids = response['imageIds'] + if "NextToken" in response: + nextToken = response["NextToken"] + + while nextToken: + payload["NextToken"] = nextToken + response = client.list_images(**payload) + image_ids = image_ids + response['imageIds'] + + if "NextToken" in response: + nextToken = response["NextToken"] + logger.debug(nextToken) + else: + # loop ends + nextToken = "" + return image_ids + except Exception as e: + logger.info(f'An exception occurred listing images in repo {repo_name}') + + +def main(args, logger): + # use default profile + client = boto3.client('ecr') + logger.info(f'registry_id: {args.registry_id}') + repo_names = list_ecr_repos(client, args.registry_id) + for repo_name in repo_names: + logger.info(f'repo_name: {repo_name}') + image_ids = list_images(client, args.registry_id, repo_name) + for image_id in image_ids: + if args.actually_do_it is True: + start_image_scan(client, args.registry_id, repo_name, image_id['imageDigest']) + + +def do_args(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument( + "--debug", help="print debugging info", action='store_true') + parser.add_argument( + "--error", help="print error info only", action='store_true') + parser.add_argument( + "--timestamp", help="Output log with timestamp and toolname", action='store_true') + parser.add_argument( + "--actually-do-it", help="Actually Perform the action", action='store_true') + parser.add_argument( + "--registry-id", help="private registry id (account number)") + + args = parser.parse_args() + + return (args) + + +if __name__ == '__main__': + + args = do_args() + + # create console handler and set level to debug + logger = logging.getLogger('cloud-trail-lookup-events') + ch = logging.StreamHandler() + if args.debug: + logger.setLevel(logging.DEBUG) + elif args.error: + logger.setLevel(logging.ERROR) + else: + logger.setLevel(logging.INFO) + + # Silence Boto3 & Friends + logging.getLogger('botocore').setLevel(logging.WARNING) + logging.getLogger('boto3').setLevel(logging.WARNING) + logging.getLogger('urllib3').setLevel(logging.WARNING) + + # create formatter + if args.timestamp: + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + else: + formatter = logging.Formatter('%(levelname)s - %(message)s') + # add formatter to ch + ch.setFormatter(formatter) + # add ch to logger + logger.addHandler(ch) + + try: + main(args, logger) + except KeyboardInterrupt: + exit(1) diff --git a/lambda-logging/README.md b/lambda-logging/README.md new file mode 100644 index 0000000..56161ad --- /dev/null +++ b/lambda-logging/README.md @@ -0,0 +1,46 @@ +# Lambda Logging + +This script will enable Lambda function log groups for all regions in your account. + +## Why? + +Lambda automatically integrates with CloudWatch Logs and pushes all logs from your code to a CloudWatch Logs group associated with a Lambda function, which is named /aws/lambda/[function name]. If the log group associated with a Lambda function is not found, you'll find the following error in AWS console: + +```Log group does not exist +The specific log group: /aws/lambda/ does not exist in this account or region. +``` + +This script will enable the log groups for corresponding Lambda functions if they don't exist in all regions. + +## What the script does. + +Thsi script iterates through all the regions returned by ec2:DescribeRegions and if logs:describe_log_groups() with the specified log group name doesn't exist calls logs:create_log_group() to enable with the specified log group name. + +## Usage + +```bash +usage: enable-lambda-log-group.py [-h] [--debug] [--error] [--timestamp] + [--region REGION] [--profile PROFILE] + [--actually-do-it] + +optional arguments: + -h, --help show this help message and exit + --debug print debugging info + --error print error info only + --timestamp Output log with timestamp and toolname + --region REGION Only Process Specified Region + --profile PROFILE Use this CLI profile (instead of default or env credentials) + --actually-do-it Actually Perform the action +``` + +You must specify `--actually-do-it` for the changes to be made. Otherwise the script runs in dry-run mode only. + + +## AWS Docs + +* [EnableEbsEncryptionByDefault API](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_EnableEbsEncryptionByDefault.html) +* [boto3 list_functions()](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/lambda.html#Lambda.Client.list_functions) +* [boto3 describe_log_groups()](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/logs.html#CloudWatchLogs.Client.describe_log_groups) +* [boto3 create_log_group()](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/logs.html#CloudWatchLogs.Client.create_log_group) + + diff --git a/lambda-logging/enable-lambda-log-group.py b/lambda-logging/enable-lambda-log-group.py new file mode 100644 index 0000000..71378cb --- /dev/null +++ b/lambda-logging/enable-lambda-log-group.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 + +import boto3 +from botocore.exceptions import ClientError +import os +import logging +import time + + +def main(args, logger): + '''Executes the Primary Logic of the Fast Fix''' + + # If they specify a profile use it. Otherwise do the normal thing + if args.profile: + session = boto3.Session(profile_name=args.profile) + else: + session = boto3.Session() + + # Get all the Regions for this account + for region in get_regions(session, args): + lambda_client = session.client("lambda", region_name=region) + cwlog_client = session.client("logs", region_name=region) + + # Create a reusable Paginator + paginator = lambda_client.get_paginator('list_functions') + + # Create a PageIterator from the Paginator, parameters here? + page_iterator = paginator.paginate() + + for page in page_iterator: + func_list = page['Functions'] + count = 0 + for func in func_list: + count = count + 1 + if count > 5: + # sleep 1 sec after 5 call (limit 5/sec) + time.sleep(1) + count = 0 + func_name = func.get('FunctionName') + log_group_name=f"/aws/lambda/{func_name}" + logs = cwlog_client.describe_log_groups(logGroupNamePrefix=log_group_name) + if not logs['logGroups']: + # log group doesn't exist for this lambda function, creat log group + if args.actually_do_it is True: + logger.info(f"Creating log group {log_group_name} in {region}") + create_log_group(cwlog_client, region, log_group_name) + else: + logger.info(f"You Need To create log group {log_group_name} in {region}") + + + +def create_log_group(cwlog_client, region, log_group_name): + '''Actually perform the creation of log group''' + try: + cwlog_client.create_log_group(logGroupName=log_group_name) + + except ClientError as e: + logger.error(f"Attempt to create log group {log_group_name} in {region} returned \"{e}\"") + + + +def get_regions(session, args): + '''Return a list of regions with us-east-1 first. If --region was specified, return a list wth just that''' + + # If we specifed a region on the CLI, return a list of just that + if args.region: + return([args.region]) + + # otherwise return all the regions, us-east-1 first + ec2 = session.client('ec2') + response = ec2.describe_regions() + output = ['us-east-1'] + for r in response['Regions']: + # return us-east-1 first, but dont return it twice + if r['RegionName'] == "us-east-1": + continue + output.append(r['RegionName']) + return(output) + + + +def do_args(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--debug", help="print debugging info", action='store_true') + parser.add_argument("--error", help="print error info only", action='store_true') + parser.add_argument("--timestamp", help="Output log with timestamp and toolname", action='store_true') + parser.add_argument("--region", help="Only Process Specified Region") + parser.add_argument("--profile", help="Use this CLI profile (instead of default or env credentials)") + parser.add_argument("--actually-do-it", help="Actually Perform the action", action='store_true') + + args = parser.parse_args() + + return(args) + +if __name__ == '__main__': + + args = do_args() + + # create console handler and set level to debug + logger = logging.getLogger('enable-lambda-log-group') + ch = logging.StreamHandler() + if args.debug: + logger.setLevel(logging.DEBUG) + elif args.error: + logger.setLevel(logging.ERROR) + else: + logger.setLevel(logging.INFO) + + # Silence Boto3 & Friends + logging.getLogger('botocore').setLevel(logging.WARNING) + logging.getLogger('boto3').setLevel(logging.WARNING) + logging.getLogger('urllib3').setLevel(logging.WARNING) + + # create formatter + if args.timestamp: + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + else: + formatter = logging.Formatter('%(levelname)s - %(message)s') + # add formatter to ch + ch.setFormatter(formatter) + # add ch to logger + logger.addHandler(ch) + + try: + main(args, logger) + except KeyboardInterrupt: + exit(1) \ No newline at end of file