From 7ab2ffaf1cff82d9cde28569d0d6a8f8fa9f0bf1 Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Mon, 9 Nov 2020 18:30:00 +0100 Subject: [PATCH 01/23] Add terraform to create Lambda --- .gitignore | 14 +++++++ terraform/main.tf | 93 +++++++++++++++++++++++++++++++++++++++++++++++ terraform/meta.tf | 21 +++++++++++ terraform/vars.tf | 20 ++++++++++ 4 files changed, 148 insertions(+) create mode 100644 terraform/main.tf create mode 100644 terraform/meta.tf create mode 100644 terraform/vars.tf diff --git a/.gitignore b/.gitignore index be50c1e..b89f486 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,17 @@ s3transfer-0.1.9.dist-info botocore-1.4.68.dist-info docutils-0.12.dist-info .idea + + +### Terraform ### +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log + +*.zip diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..89f80bd --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,93 @@ +resource "aws_iam_role_policy" "ecr-cleanup-lambda_role_policy" { + name = "ecr-cleanup-lambda" + role = aws_iam_role.ecr-cleanup-lambda_role.id + + policy = <<-EOF + { + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": "arn:aws:logs:*:*:*" + }, + { + "Action": [ + "ecr:BatchDeleteImage", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:DescribeImages" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ecs:DescribeClusters", + "ecs:DescribeTaskDefinition", + "ecs:DescribeTasks", + "ecs:ListClusters", + "ecs:ListTaskDefinitions", + "ecs:ListTasks" + ], + "Effect": "Allow", + "Resource": "*" + } + ] + } + EOF +} + +resource "aws_iam_role" "ecr-cleanup-lambda_role" { + name = "ecr-cleanup-lambda" + + assume_role_policy = <<-EOF + { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Principal": { + "Service": "lambda.amazonaws.com" + }, + "Effect": "Allow", + "Sid": "" + } + ] + } + EOF +} + +module "lambda_function_existing_package_local" { + source = "terraform-aws-modules/lambda/aws" + version = "~> 1.0" + + function_name = var.NAME_OF_FUNCTION + description = "Lifecycle for ECR images" + handler = "main.handler" + runtime = "python3.8" + + lambda_role = aws_iam_role.ecr-cleanup-lambda_role.arn + create_package = false + create_role = false + local_existing_package = "../${var.NAME_OF_FUNCTION}.zip" + + environment_variables = { + IMAGES_TO_KEEP = tonumber(var.IMAGES_TO_KEEP), + IGNORE_TAGS_REGEX = var.IGNORE_TAGS_REGEX, + REGION = var.REGION + } + + // allowed_triggers = { + // OneRule = { + // principal = "events.amazonaws.com" + // source_arn = "arn:aws:events:eu-west-1:135367859851:rule/RunDaily" + // } + // } +} + + diff --git a/terraform/meta.tf b/terraform/meta.tf new file mode 100644 index 0000000..db0aa7a --- /dev/null +++ b/terraform/meta.tf @@ -0,0 +1,21 @@ +terraform { + required_version = ">= 0.13.0, < 0.14.0" + + required_providers { + aws = { + version = "~> 3.14.1" + source = "hashicorp/aws" + } + } + +/* backend "s3" { + bucket = "hopin-terraform-remote-state" + key = "evil/eu-west-1/iam/terraform.tfstate" + region = "eu-west-1" + encrypt = true + }*/ +} + +provider "aws" { + region = "eu-west-1" +} \ No newline at end of file diff --git a/terraform/vars.tf b/terraform/vars.tf new file mode 100644 index 0000000..fc66f83 --- /dev/null +++ b/terraform/vars.tf @@ -0,0 +1,20 @@ +variable "DRYRUN" { + default = false +} + +variable "IMAGES_TO_KEEP" { + type = string + default = 1 +} + +variable "IGNORE_TAGS_REGEX" { + default = "whatever" +} + +variable "NAME_OF_FUNCTION" { + default = "ecr-cleanup-lambda" +} + +variable "REGION" { + default = "eu-west-1" +} \ No newline at end of file From 6bccce4829340bcf3095ab696ac3bd1c0880a6a5 Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Tue, 10 Nov 2020 08:19:01 +0100 Subject: [PATCH 02/23] Add GHA pipeline --- .github/workflows/deploy.yml | 50 ++++++++++++++++++++++++++++++++++++ .gitignore | 3 ++- terraform/vars.tf | 19 ++++---------- 3 files changed, 57 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..bc48694 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,50 @@ +name: Deploy code to Lambda +on: push + +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: 'eu-west-1' + REPO_NAME: ${{ github.event.repository.name }} + IMAGES_TO_KEEP: 1 + IGNORE_TAGS_REGEX: 'whatever' + +jobs: + terraform: + runs-on: ubuntu-latest + steps: + - name: 'checkout' + uses: actions/checkout@v2 + - name: 'setup python 3.8' + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: 'zip it all' + run: | + mkdir zip_folder + pip install -r requirements.txt -t zip_folder + pip install . -t zip_folder + zip --quiet -r ${REPO_NAME}.zip zip_folder + - name: 'setup terraform' + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: '>=0.13.0 <0.14.0' + - name: 'terraform init' + working-directory: terraform + run: terraform init + - name: 'terraform validate' + working-directory: terraform + run: terraform validate + - name: 'terraform plan' + working-directory: terraform + run: | + terraform plan \ + -out="${REPO_NAME}.tfplan" \ + -var="DRYRUN=false" \ + -var="IMAGES_TO_KEEP=${IMAGES_TO_KEEP}" \ + -var="IGNORE_TAGS_REGEX=${IGNORE_TAGS_REGEX}" \ + -var="NAME_OF_FUNCTION=${REPO_NAME}" \ + -var="REGION=${AWS_REGION}" + - name: 'terraform apply' + working-directory: terraform + run: terraform apply -auto-approve "${REPO_NAME}.tfplan" diff --git a/.gitignore b/.gitignore index b89f486..5035770 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ s3transfer-0.1.9.dist-info botocore-1.4.68.dist-info docutils-0.12.dist-info .idea - +DS_Store ### Terraform ### # Local .terraform directories @@ -33,4 +33,5 @@ docutils-0.12.dist-info # Crash log files crash.log +zip_folder *.zip diff --git a/terraform/vars.tf b/terraform/vars.tf index fc66f83..16d3f9b 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -1,20 +1,11 @@ variable "DRYRUN" { - default = false + default = true } -variable "IMAGES_TO_KEEP" { - type = string - default = 1 -} +variable "IMAGES_TO_KEEP" {} -variable "IGNORE_TAGS_REGEX" { - default = "whatever" -} +variable "IGNORE_TAGS_REGEX" {} -variable "NAME_OF_FUNCTION" { - default = "ecr-cleanup-lambda" -} +variable "NAME_OF_FUNCTION" {} -variable "REGION" { - default = "eu-west-1" -} \ No newline at end of file +variable "REGION" {} \ No newline at end of file From c24488ae4b67f1b47de00274330bea1b98cb51e6 Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Tue, 10 Nov 2020 08:23:59 +0100 Subject: [PATCH 03/23] Fix: Add terraform to create Lambda --- .github/workflows/deploy.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index bc48694..ca6c72a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,7 +23,6 @@ jobs: run: | mkdir zip_folder pip install -r requirements.txt -t zip_folder - pip install . -t zip_folder zip --quiet -r ${REPO_NAME}.zip zip_folder - name: 'setup terraform' uses: hashicorp/setup-terraform@v1 From f342fdb7f8992267e6832afee8bb0fc399f841de Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Tue, 10 Nov 2020 08:37:11 +0100 Subject: [PATCH 04/23] Add terraform s3 state --- .github/workflows/deploy.yml | 2 +- terraform/meta.tf | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ca6c72a..43f9faa 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -10,7 +10,7 @@ env: IGNORE_TAGS_REGEX: 'whatever' jobs: - terraform: + : runs-on: ubuntu-latest steps: - name: 'checkout' diff --git a/terraform/meta.tf b/terraform/meta.tf index db0aa7a..c2d1727 100644 --- a/terraform/meta.tf +++ b/terraform/meta.tf @@ -8,12 +8,12 @@ terraform { } } -/* backend "s3" { - bucket = "hopin-terraform-remote-state" - key = "evil/eu-west-1/iam/terraform.tfstate" + backend "s3" { + bucket = "jportasa-terraform-remote-state" + key = "evil/eu-west-1/ecr-cleanup-lambda/terraform.tfstate" region = "eu-west-1" encrypt = true - }*/ + } } provider "aws" { From b773ddfc055655237f03f16df80927b508c64f2c Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Tue, 10 Nov 2020 08:38:12 +0100 Subject: [PATCH 05/23] Typo --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 43f9faa..67d8a27 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -10,7 +10,7 @@ env: IGNORE_TAGS_REGEX: 'whatever' jobs: - : + deploy-lambda: runs-on: ubuntu-latest steps: - name: 'checkout' From 12569ca166ad316d70c43d81e8c70b155ed21f29 Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Tue, 10 Nov 2020 09:03:15 +0100 Subject: [PATCH 06/23] Limit to ecr repo's selected --- .github/workflows/deploy.yml | 20 ++++----- LICENSE | 10 ----- NOTICE | 2 - lambda-cloudformation.yaml | 78 ------------------------------------ main.py | 10 ++--- terraform/vars.tf | 4 +- 6 files changed, 18 insertions(+), 106 deletions(-) delete mode 100644 LICENSE delete mode 100644 NOTICE delete mode 100644 lambda-cloudformation.yaml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 67d8a27..acfb2c2 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -7,34 +7,35 @@ env: AWS_REGION: 'eu-west-1' REPO_NAME: ${{ github.event.repository.name }} IMAGES_TO_KEEP: 1 - IGNORE_TAGS_REGEX: 'whatever' + IGNORE_TAGS_REGEX: 'release|archive' + ECR_REPOS_LIFECYCLE: 'alpine,apache' jobs: deploy-lambda: runs-on: ubuntu-latest steps: - - name: 'checkout' + - name: 'Checkout' uses: actions/checkout@v2 - - name: 'setup python 3.8' + - name: 'Setup python 3.8' uses: actions/setup-python@v1 with: python-version: 3.8 - - name: 'zip it all' + - name: 'Zip it all' run: | mkdir zip_folder pip install -r requirements.txt -t zip_folder zip --quiet -r ${REPO_NAME}.zip zip_folder - - name: 'setup terraform' + - name: 'Setup terraform' uses: hashicorp/setup-terraform@v1 with: terraform_version: '>=0.13.0 <0.14.0' - - name: 'terraform init' + - name: 'Terraform init' working-directory: terraform run: terraform init - - name: 'terraform validate' + - name: 'Terraform validate' working-directory: terraform run: terraform validate - - name: 'terraform plan' + - name: 'Terraform plan' working-directory: terraform run: | terraform plan \ @@ -43,7 +44,8 @@ jobs: -var="IMAGES_TO_KEEP=${IMAGES_TO_KEEP}" \ -var="IGNORE_TAGS_REGEX=${IGNORE_TAGS_REGEX}" \ -var="NAME_OF_FUNCTION=${REPO_NAME}" \ + -var="ECR_REPOS_LIFECYCLE=${ECR_REPOS_LIFECYCLE}" \ -var="REGION=${AWS_REGION}" - - name: 'terraform apply' + - name: 'Terraform apply' working-directory: terraform run: terraform apply -auto-approve "${REPO_NAME}.tfplan" diff --git a/LICENSE b/LICENSE deleted file mode 100644 index ba625fa..0000000 --- a/LICENSE +++ /dev/null @@ -1,10 +0,0 @@ -Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with -the License. A copy of the License is located at - - http://aws.amazon.com/apache2.0/ - -or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file diff --git a/NOTICE b/NOTICE deleted file mode 100644 index 93f0586..0000000 --- a/NOTICE +++ /dev/null @@ -1,2 +0,0 @@ -ECR_CLEANUP -Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. \ No newline at end of file diff --git a/lambda-cloudformation.yaml b/lambda-cloudformation.yaml deleted file mode 100644 index d2c39eb..0000000 --- a/lambda-cloudformation.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with -# the License. A copy of the License is located at -# http://aws.amazon.com/apache2.0/ -# or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -# limitations under the License. - -AWSTemplateFormatVersion: '2010-09-09' -Transform: AWS::Serverless-2016-10-31 -Description: automated cleanup of ECR repositories -Resources: - LambdaFunction: - Type: AWS::Serverless::Function - Properties: - Handler: main.handler - Runtime: python2.7 - Description: ECR Cleanup Lambda - CodeUri: ./ - MemorySize: 128 - Timeout: 300 - Role: !GetAtt LAMBDAROLE.Arn - Events: - Schedule: - Type: Schedule - Properties: - Schedule: cron(0 22 * * ? *) - Environment: - Variables: - REGION: None - DRYRUN: True - IMAGES_TO_KEEP: 100 - LAMBDAROLE: - Type: AWS::IAM::Role - Properties: - #RoleName: !Sub ${AWS::StackName}-Role-${AWS::Region} - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Path: / - LAMBDAPOLICY: - Type: AWS::IAM::Policy - Properties: - PolicyName: !Sub ${AWS::StackName}-Policy-${AWS::Region} - PolicyDocument: - Version: 2012-10-17 - Statement: - - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - Resource: arn:aws:logs:*:*:* - - - Effect: Allow - Action: - - ecr:BatchDeleteImage - - ecr:DescribeRepositories - - ecr:ListImages - - ecr:DescribeImages - - ecs:DescribeClusters - - ecs:DescribeTaskDefinition - - ecs:DescribeTasks - - ecs:ListClusters - - ecs:ListTaskDefinitions - - ecs:ListTasks - Resource: "*" - Roles: - - - !Ref LAMBDAROLE \ No newline at end of file diff --git a/main.py b/main.py index 458a1e4..bb32464 100644 --- a/main.py +++ b/main.py @@ -21,13 +21,14 @@ DRYRUN = None IMAGES_TO_KEEP = None IGNORE_TAGS_REGEX = None - +ECR_REPOS_LIFECYCLE = None def initialize(): global REGION global DRYRUN global IMAGES_TO_KEEP global IGNORE_TAGS_REGEX + global ECR_REPOS_LIFECYCLE REGION = os.environ.get('REGION', "None") DRYRUN = os.environ.get('DRYRUN', "false").lower() @@ -37,6 +38,7 @@ def initialize(): DRYRUN = True IMAGES_TO_KEEP = int(os.environ.get('IMAGES_TO_KEEP', 100)) IGNORE_TAGS_REGEX = os.environ.get('IGNORE_TAGS_REGEX', "^$") + ECR_REPOS_LIFECYCLE = os.environ.get('ECR_REPOS_LIFECYCLE', "None") def handler(event, context): initialize() @@ -53,11 +55,7 @@ def discover_delete_images(regionname): print("Discovering images in " + regionname) ecr_client = boto3.client('ecr', region_name=regionname) - repositories = [] - describe_repo_paginator = ecr_client.get_paginator('describe_repositories') - for response_listrepopaginator in describe_repo_paginator.paginate(): - for repo in response_listrepopaginator['repositories']: - repositories.append(repo) + repositories = ECR_REPOS_LIFECYCLE.split(',') ecs_client = boto3.client('ecs', region_name=regionname) diff --git a/terraform/vars.tf b/terraform/vars.tf index 16d3f9b..b9e7790 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -8,4 +8,6 @@ variable "IGNORE_TAGS_REGEX" {} variable "NAME_OF_FUNCTION" {} -variable "REGION" {} \ No newline at end of file +variable "REGION" {} + +variable "ECR_REPOS_LIFECYCLE" {} \ No newline at end of file From 4b0be1a3eacf2437214d6a36cecdd6042d9cfcc9 Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Tue, 10 Nov 2020 09:05:59 +0100 Subject: [PATCH 07/23] Fix ECR_REPOS_LIFECYCLE var in TF --- terraform/main.tf | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 89f80bd..f04a20b 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -77,9 +77,10 @@ module "lambda_function_existing_package_local" { local_existing_package = "../${var.NAME_OF_FUNCTION}.zip" environment_variables = { - IMAGES_TO_KEEP = tonumber(var.IMAGES_TO_KEEP), - IGNORE_TAGS_REGEX = var.IGNORE_TAGS_REGEX, - REGION = var.REGION + IMAGES_TO_KEEP = tonumber(var.IMAGES_TO_KEEP), + IGNORE_TAGS_REGEX = var.IGNORE_TAGS_REGEX, + REGION = var.REGION + ECR_REPOS_LIFECYCLE = var.ECR_REPOS_LIFECYCLE } // allowed_triggers = { From 517848f9a4e78e0473a3a753271dffae4fd2e3cf Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Tue, 10 Nov 2020 09:16:54 +0100 Subject: [PATCH 08/23] Fix zip --- .github/workflows/deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index acfb2c2..fafe837 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -22,9 +22,9 @@ jobs: python-version: 3.8 - name: 'Zip it all' run: | - mkdir zip_folder - pip install -r requirements.txt -t zip_folder - zip --quiet -r ${REPO_NAME}.zip zip_folder + mkdir -p zip_folder + pip install -r requirements.txt --target=zip_folder + zip -r ${REPO_NAME}.zip ./zip_folder - name: 'Setup terraform' uses: hashicorp/setup-terraform@v1 with: From d6cbc6f174cabbc68a027374b26c52e13d95224a Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Tue, 10 Nov 2020 09:23:56 +0100 Subject: [PATCH 09/23] Fix zip --- .github/workflows/deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fafe837..f169313 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -24,6 +24,7 @@ jobs: run: | mkdir -p zip_folder pip install -r requirements.txt --target=zip_folder + cp -R * ./zip_folder zip -r ${REPO_NAME}.zip ./zip_folder - name: 'Setup terraform' uses: hashicorp/setup-terraform@v1 From 1a90390eb2e6409286a2ca515021d6b06ea5aaac Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Tue, 10 Nov 2020 09:26:20 +0100 Subject: [PATCH 10/23] Fix zip --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f169313..3ded051 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -24,7 +24,7 @@ jobs: run: | mkdir -p zip_folder pip install -r requirements.txt --target=zip_folder - cp -R * ./zip_folder + cp -r !(zip_folder) ./zip_folder zip -r ${REPO_NAME}.zip ./zip_folder - name: 'Setup terraform' uses: hashicorp/setup-terraform@v1 From 275f691f941ebb93fd662eae1831eefef37ac285 Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Tue, 10 Nov 2020 09:27:29 +0100 Subject: [PATCH 11/23] Fix zip --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3ded051..c6439a9 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -24,7 +24,7 @@ jobs: run: | mkdir -p zip_folder pip install -r requirements.txt --target=zip_folder - cp -r !(zip_folder) ./zip_folder + cp * ./zip_folder zip -r ${REPO_NAME}.zip ./zip_folder - name: 'Setup terraform' uses: hashicorp/setup-terraform@v1 From 8ff458180ed89e347c6cd11d31cb6847b8685145 Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Tue, 10 Nov 2020 09:31:05 +0100 Subject: [PATCH 12/23] Fix zip --- .github/workflows/deploy.yml | 2 +- main.py => src/main.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename main.py => src/main.py (100%) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c6439a9..56d4260 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -24,7 +24,7 @@ jobs: run: | mkdir -p zip_folder pip install -r requirements.txt --target=zip_folder - cp * ./zip_folder + cp -R src/* zip_folder zip -r ${REPO_NAME}.zip ./zip_folder - name: 'Setup terraform' uses: hashicorp/setup-terraform@v1 diff --git a/main.py b/src/main.py similarity index 100% rename from main.py rename to src/main.py From a9062c50a1e91d4fecce9371dd95dea9d500cef9 Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Tue, 10 Nov 2020 09:38:43 +0100 Subject: [PATCH 13/23] Fix zip --- .github/workflows/deploy.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 56d4260..7e743c0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,9 +23,10 @@ jobs: - name: 'Zip it all' run: | mkdir -p zip_folder - pip install -r requirements.txt --target=zip_folder cp -R src/* zip_folder - zip -r ${REPO_NAME}.zip ./zip_folder + pip install -r requirements.txt --target=zip_folder + cd zip_folder + zip -r ../${REPO_NAME}.zip . - name: 'Setup terraform' uses: hashicorp/setup-terraform@v1 with: From 04ab6436cc30622f332c11a566cdee307fe9ddd8 Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Tue, 10 Nov 2020 10:56:11 +0100 Subject: [PATCH 14/23] Only consider in Lifecycle repos in var ECR_REPOS_LIFECYCLE --- src/main.py | 8 +++++++- terraform/main.tf | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main.py b/src/main.py index bb32464..c6011e8 100644 --- a/src/main.py +++ b/src/main.py @@ -40,6 +40,7 @@ def initialize(): IGNORE_TAGS_REGEX = os.environ.get('IGNORE_TAGS_REGEX', "^$") ECR_REPOS_LIFECYCLE = os.environ.get('ECR_REPOS_LIFECYCLE', "None") + def handler(event, context): initialize() if REGION == "None": @@ -55,7 +56,12 @@ def discover_delete_images(regionname): print("Discovering images in " + regionname) ecr_client = boto3.client('ecr', region_name=regionname) - repositories = ECR_REPOS_LIFECYCLE.split(',') + repositories = [] + describe_repo_paginator = ecr_client.get_paginator('describe_repositories') + for response_listrepopaginator in describe_repo_paginator.paginate(): + for repo in response_listrepopaginator['repositories']: + if repo['repositoryName'] in ECR_REPOS_LIFECYCLE.split(','): + repositories.append(repo) ecs_client = boto3.client('ecs', region_name=regionname) diff --git a/terraform/main.tf b/terraform/main.tf index f04a20b..3c83d3b 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -89,6 +89,10 @@ module "lambda_function_existing_package_local" { // source_arn = "arn:aws:events:eu-west-1:135367859851:rule/RunDaily" // } // } + tags ={ + Environment = "evil" + Terraform = "true" + } } From 2bff0bdbbfa8b4069aeb2c1a3cba51a68f4773a9 Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Tue, 10 Nov 2020 11:34:16 +0100 Subject: [PATCH 15/23] Refactor README --- README.md | 50 ++++++++++++++++++-------------------------------- src/main.py | 13 +------------ 2 files changed, 19 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 9d22478..026f4a8 100644 --- a/README.md +++ b/README.md @@ -1,65 +1,51 @@ # Automated Image Cleanup for Amazon ECR -The Python script and Lambda function described here help clean up images in [Amazon ECR](https://aws.amazon.com/ecr). The script looks for images that are not used in running [Amazon ECS](https://aws.amazon.com/ecs) tasks that can be deleted. You can configure the script to print the image list first to confirm deletions, specify a region, or specify a number of images to keep for potential rollbacks. +The Python script and Lambda function described here help clean up images in [Amazon ECR](https://aws.amazon.com/ecr). +- The script looks for images that are not used in running [Amazon ECS](https://aws.amazon.com/ecs) tasks that can be deleted. +- Only looks images in repos specified in GitHubActions env var ECR_REPOS_LIFECYCLE. +- You can configure the script to print the image list first to confirm deletions, specify a region, or specify a number of images to keep for potential rollbacks. -## Authenticate with AWS -[Configuring the AWS Command Line Interface.](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html) - -## Use virtualenv for Python execution +## Usage in local To prevent any problems with your system Python version conflicting with the application, we recommend using virtualenv. Install Python: `pip install python 3` +Getting info about local env: + `which python3` + Install virtualenv: $ pip install virtualenv - $ virtualenv -p PATH_TO_YOUR_PYTHON_3 cloudformtion - $ virtualenv ~/.virtualenvs/cloudformtion - $ source ~/.virtualenvs/cloudformtion/bin/activate + $ virtualenv --python=/usr/local/bin/python3 ~/venv-ecr-cleanup-lambda + $ source ~/venv-ecr-cleanup-lambda/bin/activate -## Generate the Lambda package +## Install dependencies -1. CD to the folder that contains main.py. -1. Run the following command: -`pip install -r requirements.txt -t `pwd`` -1. Compress the contents of folder (not the folder). - -## Upload the package to Lambda +`pip install -r requirements.txt` -1. Run the following command: -`aws lambda create-function --function-name {NAME_OF_FUNCTION} --runtime python2.7 ---role {ARN_NUMBER} --handler main.handler --timeout 15 ---zip-file fileb://{ZIP_FILE_PATH}` -## Send the package update to Lambda - -1. Run the following command: - - `aws lambda update-function-code --function-name {NAME_OF_FUNCTION} --zip-file fileb://{ZIP_FILE_PATH}` - - ## Examples -Prints the images that are not used by running tasks and which are older than the last 100 versions, in all regions: +Prints the images that are not used by running tasks and which are older than the last 100 versions, in all regions in ECR repos named alpine and apache: -`python main.py` +`python main.py -ecr_repos_lifecycle 'alpine,apache'` Deletes the images that are not used by running tasks and which are older than the last 100 versions, in all regions: -`python main.py –dryrun False` +`python main.py –dryrun False -ecr_repos_lifecycle 'alpine,apache'` Deletes the images that are not used by running tasks and which are older than the last 20 versions (in each repository), in all regions: -`python main.py –dryrun False –imagestokeep 20` +`python main.py –dryrun False –imagestokeep 20 -ecr_repos_lifecycle 'alpine,apache'` Deletes the images that are not used by running tasks and which are older than the last 20 versions (in each repository), in Oregon only: -`python main.py –dryrun False –imagestokeep 20 –region us-west-2` +`python main.py –dryrun False –imagestokeep 20 –region us-west-2 -ecr_repos_lifecycle 'alpine,apache'` Deletes the images that are not used by running tasks and which are older than the last 20 versions (in each repository), in Oregon only, and ignore image tags that contains `release` or `archive`: -`python main.py –dryrun False –imagestokeep 20 –region us-west-2 -ignoretagsregex release|archive` +`python main.py –dryrun False –imagestokeep 20 –region us-west-2 -ignoretagsregex release|archive -ecr_repos_lifecycle 'alpine,apache'` diff --git a/src/main.py b/src/main.py index c6011e8..8920ef6 100644 --- a/src/main.py +++ b/src/main.py @@ -1,15 +1,3 @@ -''' -Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with -the License. A copy of the License is located at - - http://aws.amazon.com/apache2.0/ - -or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. -''' from __future__ import print_function import argparse @@ -23,6 +11,7 @@ IGNORE_TAGS_REGEX = None ECR_REPOS_LIFECYCLE = None + def initialize(): global REGION global DRYRUN From 9c039adb40fce9daf5699badd9a8fe667d52bddf Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Tue, 10 Nov 2020 12:54:44 +0100 Subject: [PATCH 16/23] Clean --- .github/workflows/deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7e743c0..04f7f31 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -2,13 +2,13 @@ name: Deploy code to Lambda on: push env: + IMAGES_TO_KEEP: 1 + IGNORE_TAGS_REGEX: 'release|archive' + ECR_REPOS_LIFECYCLE: 'alpine,apache' AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: 'eu-west-1' REPO_NAME: ${{ github.event.repository.name }} - IMAGES_TO_KEEP: 1 - IGNORE_TAGS_REGEX: 'release|archive' - ECR_REPOS_LIFECYCLE: 'alpine,apache' jobs: deploy-lambda: From c71f2b9787a8e3044cb16002358753a399915d40 Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Wed, 11 Nov 2020 13:14:45 +0100 Subject: [PATCH 17/23] Fake commit --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 026f4a8..ec6d48b 100644 --- a/README.md +++ b/README.md @@ -49,3 +49,4 @@ Deletes the images that are not used by running tasks and which are older than t `python main.py –dryrun False –imagestokeep 20 –region us-west-2 -ignoretagsregex release|archive -ecr_repos_lifecycle 'alpine,apache'` + From 88b46cf4a67effbb5c932990789ab1bb9b725ab8 Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Thu, 12 Nov 2020 12:16:42 +0100 Subject: [PATCH 18/23] Add terraform linter --- .github/workflows/deploy.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 04f7f31..7f8505c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -37,6 +37,11 @@ jobs: - name: 'Terraform validate' working-directory: terraform run: terraform validate + - name: tflint + uses: reviewdog/action-tflint@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + working-directory: terraform - name: 'Terraform plan' working-directory: terraform run: | From f9478a453c39d093f11e97787856aabbfb68d6f5 Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Thu, 12 Nov 2020 12:18:45 +0100 Subject: [PATCH 19/23] Typo --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7f8505c..9eca935 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -41,7 +41,7 @@ jobs: uses: reviewdog/action-tflint@master with: github_token: ${{ secrets.GITHUB_TOKEN }} - working-directory: terraform + working_directory: terraform - name: 'Terraform plan' working-directory: terraform run: | From bd6a8d869a33b4f5ce37196c86b105eead7fbc4b Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Thu, 12 Nov 2020 12:21:15 +0100 Subject: [PATCH 20/23] Change Name for pipeline step --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 9eca935..9ccfa07 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -37,7 +37,7 @@ jobs: - name: 'Terraform validate' working-directory: terraform run: terraform validate - - name: tflint + - name: Terraform Linter uses: reviewdog/action-tflint@master with: github_token: ${{ secrets.GITHUB_TOKEN }} From 08931585044fdb64ca0c00b40f4c585d5c141267 Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Fri, 13 Nov 2020 09:58:01 +0100 Subject: [PATCH 21/23] Modify readme --- .github/workflows/deploy.yml | 18 +-------------- README.md | 44 +++++++++++++----------------------- terraform/main.tf | 6 ----- terraform/vars.tf | 16 +++++++++---- 4 files changed, 29 insertions(+), 55 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 9ccfa07..f9859c1 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -2,12 +2,8 @@ name: Deploy code to Lambda on: push env: - IMAGES_TO_KEEP: 1 - IGNORE_TAGS_REGEX: 'release|archive' - ECR_REPOS_LIFECYCLE: 'alpine,apache' AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: 'eu-west-1' REPO_NAME: ${{ github.event.repository.name }} jobs: @@ -37,22 +33,10 @@ jobs: - name: 'Terraform validate' working-directory: terraform run: terraform validate - - name: Terraform Linter - uses: reviewdog/action-tflint@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - working_directory: terraform - name: 'Terraform plan' working-directory: terraform run: | - terraform plan \ - -out="${REPO_NAME}.tfplan" \ - -var="DRYRUN=false" \ - -var="IMAGES_TO_KEEP=${IMAGES_TO_KEEP}" \ - -var="IGNORE_TAGS_REGEX=${IGNORE_TAGS_REGEX}" \ - -var="NAME_OF_FUNCTION=${REPO_NAME}" \ - -var="ECR_REPOS_LIFECYCLE=${ECR_REPOS_LIFECYCLE}" \ - -var="REGION=${AWS_REGION}" + terraform plan -out="${REPO_NAME}.tfplan" - name: 'Terraform apply' working-directory: terraform run: terraform apply -auto-approve "${REPO_NAME}.tfplan" diff --git a/README.md b/README.md index ec6d48b..b284c30 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,12 @@ # Automated Image Cleanup for Amazon ECR The Python script and Lambda function described here help clean up images in [Amazon ECR](https://aws.amazon.com/ecr). - The script looks for images that are not used in running [Amazon ECS](https://aws.amazon.com/ecs) tasks that can be deleted. -- Only looks images in repos specified in GitHubActions env var ECR_REPOS_LIFECYCLE. -- You can configure the script to print the image list first to confirm deletions, specify a region, or specify a number of images to keep for potential rollbacks. +- Only looks images in repos specified in GitHubActions env var ECR_REPOS_LIFECYCLE ex: ECR_REPOS_LIFECYCLE='apache, nginx' +- You can configure the script to print the image list first to confirm deletions, specify a region, or specify a number of images to keep for potential rollbacks with DRYRUN=true +- You can skip images followin a REGEX in their tag. ## Usage in local -To prevent any problems with your system Python version conflicting with the application, we recommend using virtualenv. - Install Python: `pip install python 3` @@ -16,37 +15,26 @@ Getting info about local env: Install virtualenv: - $ pip install virtualenv - $ virtualenv --python=/usr/local/bin/python3 ~/venv-ecr-cleanup-lambda - $ source ~/venv-ecr-cleanup-lambda/bin/activate + pip install virtualenv + virtualenv --python=/usr/local/bin/python3 ~/venv-ecr-cleanup-lambda + source ~/venv-ecr-cleanup-lambda/bin/activate ## Install dependencies `pip install -r requirements.txt` - -## Examples -Prints the images that are not used by running tasks and which are older than the last 100 versions, in all regions in ECR repos named alpine and apache: - -`python main.py -ecr_repos_lifecycle 'alpine,apache'` - - -Deletes the images that are not used by running tasks and which are older than the last 100 versions, in all regions: - -`python main.py –dryrun False -ecr_repos_lifecycle 'alpine,apache'` - - -Deletes the images that are not used by running tasks and which are older than the last 20 versions (in each repository), in all regions: - -`python main.py –dryrun False –imagestokeep 20 -ecr_repos_lifecycle 'alpine,apache'` - - -Deletes the images that are not used by running tasks and which are older than the last 20 versions (in each repository), in Oregon only: +## Example -`python main.py –dryrun False –imagestokeep 20 –region us-west-2 -ecr_repos_lifecycle 'alpine,apache'` +Deletes the images that: +- are not used by ECS running tasks . +- are older than the last 20 versions (in each repository). +- in Oregon only. +- ignore image tags that contains `release` or `archive`. +- skip images with tags that contain alpine and apache. -Deletes the images that are not used by running tasks and which are older than the last 20 versions (in each repository), in Oregon only, and ignore image tags that contains `release` or `archive`: +`python main.py –dryrun False –imagestokeep 20 –region us-west-2 -ignoretagsregex 'release|archive' -ecr_repos_lifecycle 'alpine,apache'` -`python main.py –dryrun False –imagestokeep 20 –region us-west-2 -ignoretagsregex release|archive -ecr_repos_lifecycle 'alpine,apache'` +## Credits +Fork of awslabs/ecr-cleanup-lambda adding functionality of ECR_REPOS_LIFECYCLE var. \ No newline at end of file diff --git a/terraform/main.tf b/terraform/main.tf index 3c83d3b..ec303a7 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -83,12 +83,6 @@ module "lambda_function_existing_package_local" { ECR_REPOS_LIFECYCLE = var.ECR_REPOS_LIFECYCLE } - // allowed_triggers = { - // OneRule = { - // principal = "events.amazonaws.com" - // source_arn = "arn:aws:events:eu-west-1:135367859851:rule/RunDaily" - // } - // } tags ={ Environment = "evil" Terraform = "true" diff --git a/terraform/vars.tf b/terraform/vars.tf index b9e7790..577e38d 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -2,12 +2,20 @@ variable "DRYRUN" { default = true } -variable "IMAGES_TO_KEEP" {} +variable "IMAGES_TO_KEEP" { + default = 50 +} -variable "IGNORE_TAGS_REGEX" {} +variable "IGNORE_TAGS_REGEX" { + default = 'release|archive' +} -variable "NAME_OF_FUNCTION" {} +variable "NAME_OF_FUNCTION" { + default = 'ecr-cleanup-lambda' +} variable "REGION" {} -variable "ECR_REPOS_LIFECYCLE" {} \ No newline at end of file +variable "ECR_REPOS_LIFECYCLE" { + default = 'alpine,apache' +} \ No newline at end of file From 4beec05d7cbacc24e7aa4f31e8365fe0e7a7d03d Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Fri, 13 Nov 2020 10:00:00 +0100 Subject: [PATCH 22/23] Fix --- terraform/vars.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/vars.tf b/terraform/vars.tf index 577e38d..5f57a56 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -7,15 +7,15 @@ variable "IMAGES_TO_KEEP" { } variable "IGNORE_TAGS_REGEX" { - default = 'release|archive' + default = "release|archive" } variable "NAME_OF_FUNCTION" { - default = 'ecr-cleanup-lambda' + default = "ecr-cleanup-lambda" } variable "REGION" {} variable "ECR_REPOS_LIFECYCLE" { - default = 'alpine,apache' + default = "alpine,apache" } \ No newline at end of file From 5691649d6a88e7995fa9d05b655b9b99d78a3630 Mon Sep 17 00:00:00 2001 From: Joan Porta Date: Fri, 13 Nov 2020 10:08:29 +0100 Subject: [PATCH 23/23] Fix --- .github/workflows/deploy.yml | 2 +- .github/workflows/push.yml | 38 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/push.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f9859c1..82e6b5d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -39,4 +39,4 @@ jobs: terraform plan -out="${REPO_NAME}.tfplan" - name: 'Terraform apply' working-directory: terraform - run: terraform apply -auto-approve "${REPO_NAME}.tfplan" + run: terraform apply -auto-approve "${REPO_NAME}.tfplan" \ No newline at end of file diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..b4bbd0a --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,38 @@ +name: Deploy code to Lambda +on: push + +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + REPO_NAME: ${{ github.event.repository.name }} + +jobs: + deploy-lambda: + runs-on: ubuntu-latest + steps: + - name: 'Checkout' + uses: actions/checkout@v2 + - name: 'Setup python 3.8' + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: 'Zip it all' + run: | + mkdir -p zip_folder + cp -R src/* zip_folder + pip install -r requirements.txt --target=zip_folder + cd zip_folder + zip -r ../${REPO_NAME}.zip . + - name: 'Setup terraform' + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: '>=0.13.0 <0.14.0' + - name: 'Terraform init' + working-directory: terraform + run: terraform init + - name: 'Terraform validate' + working-directory: terraform + run: terraform validate + - name: Lint Terraform + working-directory: terraform + uses: actionshub/terraform-lint@master