From 978c3ab4f4c135fcc8d6c1c3014416c84a35a3b4 Mon Sep 17 00:00:00 2001 From: Chori <105255517+choridev@users.noreply.github.com> Date: Wed, 10 Sep 2025 22:37:45 +0900 Subject: [PATCH 1/4] Detect ECS Rollback (#111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 💚 Increase wait time for deployment to 15 minutes * 💚 Detect ECS Rollback * 💚 Decrease wait time for deployment to 12 minutes --- .github/workflows/deploy-to-prod.yml | 14 ++++++++++---- .github/workflows/deploy-to-stg.yml | 14 ++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-to-prod.yml b/.github/workflows/deploy-to-prod.yml index a2214313..70787721 100644 --- a/.github/workflows/deploy-to-prod.yml +++ b/.github/workflows/deploy-to-prod.yml @@ -131,16 +131,22 @@ jobs: service: ${{ env.ECS_SERVICE }} cluster: ${{ env.ECS_CLUSTER }} wait-for-service-stability: true - wait-for-minutes: 10 + wait-for-minutes: 12 - name: Check for Rollback id: check-rollback run: | ORIGINAL_TASK_DEF_ARN=$(jq -r '.taskDefinitionArn' task-def-backup.json) - FINAL_TASK_DEF_ARN="${{ steps.deploy.outputs.task-definition-arn }}" + DEPLOYED_TASK_DEF_ARN=$(aws ecs describe-services \ + --cluster ${{ env.ECS_CLUSTER }} \ + --services ${{ env.ECS_SERVICE }} \ + --query "services[0].taskDefinition" \ + --output text) + echo "Original Task Definition ARN: $ORIGINAL_TASK_DEF_ARN" - echo "Final Task Definition ARN: $FINAL_TASK_DEF_ARN" - if [ "$ORIGINAL_TASK_DEF_ARN" == "$FINAL_TASK_DEF_ARN" ]; then + echo "Currently Deployed Task Definition ARN: $DEPLOYED_TASK_DEF_ARN" + + if [ "$ORIGINAL_TASK_DEF_ARN" == "$DEPLOYED_TASK_DEF_ARN" ]; then echo "rollback_occurred=true" >> $GITHUB_OUTPUT else echo "rollback_occurred=false" >> $GITHUB_OUTPUT diff --git a/.github/workflows/deploy-to-stg.yml b/.github/workflows/deploy-to-stg.yml index 37b1c465..e971b6aa 100644 --- a/.github/workflows/deploy-to-stg.yml +++ b/.github/workflows/deploy-to-stg.yml @@ -131,16 +131,22 @@ jobs: service: ${{ env.ECS_SERVICE }} cluster: ${{ env.ECS_CLUSTER }} wait-for-service-stability: true - wait-for-minutes: 8 + wait-for-minutes: 12 - name: Check for Rollback id: check-rollback run: | ORIGINAL_TASK_DEF_ARN=$(jq -r '.taskDefinitionArn' task-def-backup.json) - FINAL_TASK_DEF_ARN="${{ steps.deploy.outputs.task-definition-arn }}" + DEPLOYED_TASK_DEF_ARN=$(aws ecs describe-services \ + --cluster ${{ env.ECS_CLUSTER }} \ + --services ${{ env.ECS_SERVICE }} \ + --query "services[0].taskDefinition" \ + --output text) + echo "Original Task Definition ARN: $ORIGINAL_TASK_DEF_ARN" - echo "Final Task Definition ARN: $FINAL_TASK_DEF_ARN" - if [ "$ORIGINAL_TASK_DEF_ARN" == "$FINAL_TASK_DEF_ARN" ]; then + echo "Currently Deployed Task Definition ARN: $DEPLOYED_TASK_DEF_ARN" + + if [ "$ORIGINAL_TASK_DEF_ARN" == "$DEPLOYED_TASK_DEF_ARN" ]; then echo "rollback_occurred=true" >> $GITHUB_OUTPUT else echo "rollback_occurred=false" >> $GITHUB_OUTPUT From 71261dbd053a206410ce10a1ca04a3441ba60961 Mon Sep 17 00:00:00 2001 From: Chori <105255517+choridev@users.noreply.github.com> Date: Wed, 10 Sep 2025 23:30:28 +0900 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=92=9A=20Detect=20ECS=20Rollback=20(#?= =?UTF-8?q?113)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy-to-prod.yml | 24 +++++------------------- .github/workflows/deploy-to-stg.yml | 24 +++++------------------- 2 files changed, 10 insertions(+), 38 deletions(-) diff --git a/.github/workflows/deploy-to-prod.yml b/.github/workflows/deploy-to-prod.yml index 70787721..9b728634 100644 --- a/.github/workflows/deploy-to-prod.yml +++ b/.github/workflows/deploy-to-prod.yml @@ -100,26 +100,11 @@ jobs: role-to-assume: ${{ env.AWS_IAM_ROLE_TO_ASSUME }} aws-region: ${{ env.AWS_REGION }} - - name: Log Current Service State - run: | - echo "Checking current service state before deployment..." - aws ecs describe-services --cluster ${{ env.ECS_CLUSTER }} --services ${{ env.ECS_SERVICE }} --query 'services[0].deployments' --output table - - - name: Backup Current Task Definition - run: | - echo "Backing up current task definition to task-def-backup.json" - aws ecs describe-task-definition --task-definition ${{ env.ECS_TASK_DEFINITION }} --query taskDefinition > task-def-backup.json - if [ ! -f task-def-backup.json ]; then - echo "::error::Failed to backup task definition." - exit 1 - fi - echo "Backup successful." - - name: Render New Task Definition with New Image id: task-def uses: aws-actions/amazon-ecs-render-task-definition@v1 with: - task-definition: task-def-backup.json + task-definition: ${{ env.ECS_TASK_DEFINITION }} container-name: ${{ env.CONTAINER_NAME }} image: ${{ needs.build.outputs.image_tag }} @@ -135,18 +120,19 @@ jobs: - name: Check for Rollback id: check-rollback + env: + NEW_TASK_DEF_ARN: ${{ steps.deploy.outputs.task-definition-arn }} run: | - ORIGINAL_TASK_DEF_ARN=$(jq -r '.taskDefinitionArn' task-def-backup.json) DEPLOYED_TASK_DEF_ARN=$(aws ecs describe-services \ --cluster ${{ env.ECS_CLUSTER }} \ --services ${{ env.ECS_SERVICE }} \ --query "services[0].taskDefinition" \ --output text) - echo "Original Task Definition ARN: $ORIGINAL_TASK_DEF_ARN" + echo "New Task Definition ARN: $NEW_TASK_DEF_ARN" echo "Currently Deployed Task Definition ARN: $DEPLOYED_TASK_DEF_ARN" - if [ "$ORIGINAL_TASK_DEF_ARN" == "$DEPLOYED_TASK_DEF_ARN" ]; then + if [ "$NEW_TASK_DEF_ARN" != "$DEPLOYED_TASK_DEF_ARN" ]; then echo "rollback_occurred=true" >> $GITHUB_OUTPUT else echo "rollback_occurred=false" >> $GITHUB_OUTPUT diff --git a/.github/workflows/deploy-to-stg.yml b/.github/workflows/deploy-to-stg.yml index e971b6aa..6df1c69e 100644 --- a/.github/workflows/deploy-to-stg.yml +++ b/.github/workflows/deploy-to-stg.yml @@ -100,26 +100,11 @@ jobs: role-to-assume: ${{ env.AWS_IAM_ROLE_TO_ASSUME }} aws-region: ${{ env.AWS_REGION }} - - name: Log Current Service State - run: | - echo "Checking current service state before deployment..." - aws ecs describe-services --cluster ${{ env.ECS_CLUSTER }} --services ${{ env.ECS_SERVICE }} --query 'services[0].deployments' --output table - - - name: Backup Current Task Definition - run: | - echo "Backing up current task definition to task-def-backup.json" - aws ecs describe-task-definition --task-definition ${{ env.ECS_TASK_DEFINITION }} --query taskDefinition > task-def-backup.json - if [ ! -f task-def-backup.json ]; then - echo "::error::Failed to backup task definition." - exit 1 - fi - echo "Backup successful." - - name: Render New Task Definition with New Image id: task-def uses: aws-actions/amazon-ecs-render-task-definition@v1 with: - task-definition: task-def-backup.json + task-definition: ${{ env.ECS_TASK_DEFINITION }} container-name: ${{ env.CONTAINER_NAME }} image: ${{ needs.build.outputs.image_tag }} @@ -135,18 +120,19 @@ jobs: - name: Check for Rollback id: check-rollback + env: + NEW_TASK_DEF_ARN: ${{ steps.deploy.outputs.task-definition-arn }} run: | - ORIGINAL_TASK_DEF_ARN=$(jq -r '.taskDefinitionArn' task-def-backup.json) DEPLOYED_TASK_DEF_ARN=$(aws ecs describe-services \ --cluster ${{ env.ECS_CLUSTER }} \ --services ${{ env.ECS_SERVICE }} \ --query "services[0].taskDefinition" \ --output text) - echo "Original Task Definition ARN: $ORIGINAL_TASK_DEF_ARN" + echo "New Task Definition ARN: $NEW_TASK_DEF_ARN" echo "Currently Deployed Task Definition ARN: $DEPLOYED_TASK_DEF_ARN" - if [ "$ORIGINAL_TASK_DEF_ARN" == "$DEPLOYED_TASK_DEF_ARN" ]; then + if [ "$NEW_TASK_DEF_ARN" != "$DEPLOYED_TASK_DEF_ARN" ]; then echo "rollback_occurred=true" >> $GITHUB_OUTPUT else echo "rollback_occurred=false" >> $GITHUB_OUTPUT From f90ddb5ac1c535241fbae9b22f1187cc22226ada Mon Sep 17 00:00:00 2001 From: Chori <105255517+choridev@users.noreply.github.com> Date: Thu, 11 Sep 2025 00:31:33 +0900 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=92=9A=20Detect=20ECS=20Rollback=20th?= =?UTF-8?q?rough=20image=20tag=20(#115)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy-to-prod.yml | 34 ++++++++++++++++++++-------- .github/workflows/deploy-to-stg.yml | 34 ++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/.github/workflows/deploy-to-prod.yml b/.github/workflows/deploy-to-prod.yml index 9b728634..850cca79 100644 --- a/.github/workflows/deploy-to-prod.yml +++ b/.github/workflows/deploy-to-prod.yml @@ -100,11 +100,21 @@ jobs: role-to-assume: ${{ env.AWS_IAM_ROLE_TO_ASSUME }} aws-region: ${{ env.AWS_REGION }} + - name: Backup Current Task Definition + run: | + echo "Backing up current task definition to task-def-backup.json" + aws ecs describe-task-definition --task-definition ${{ env.ECS_TASK_DEFINITION }} --query taskDefinition > task-def-backup.json + if [ ! -f task-def-backup.json ]; then + echo "::error::Failed to backup task definition." + exit 1 + fi + echo "Backup successful." + - name: Render New Task Definition with New Image id: task-def uses: aws-actions/amazon-ecs-render-task-definition@v1 with: - task-definition: ${{ env.ECS_TASK_DEFINITION }} + task-definition: task-def-backup.json container-name: ${{ env.CONTAINER_NAME }} image: ${{ needs.build.outputs.image_tag }} @@ -120,21 +130,25 @@ jobs: - name: Check for Rollback id: check-rollback - env: - NEW_TASK_DEF_ARN: ${{ steps.deploy.outputs.task-definition-arn }} run: | - DEPLOYED_TASK_DEF_ARN=$(aws ecs describe-services \ - --cluster ${{ env.ECS_CLUSTER }} \ - --services ${{ env.ECS_SERVICE }} \ - --query "services[0].taskDefinition" \ + FINAL_TASK_DEF_ARN="${{ steps.deploy.outputs.task-definition-arn }}" + EXPECTED_IMAGE_TAG="${{ needs.build.outputs.image_tag }}" + + echo "Final stable Task Definition ARN: $FINAL_TASK_DEF_ARN" + echo "Expected image tag: $EXPECTED_IMAGE_TAG" + + # Describe the final task definition and get the actual image tag for the specified container + ACTUAL_IMAGE_TAG=$(aws ecs describe-task-definition --task-definition "$FINAL_TASK_DEF_ARN" \ + --query "taskDefinition.containerDefinitions[?name=='${{ env.CONTAINER_NAME }}'].image" \ --output text) - echo "New Task Definition ARN: $NEW_TASK_DEF_ARN" - echo "Currently Deployed Task Definition ARN: $DEPLOYED_TASK_DEF_ARN" + echo "Actual image tag in final task definition: $ACTUAL_IMAGE_TAG" - if [ "$NEW_TASK_DEF_ARN" != "$DEPLOYED_TASK_DEF_ARN" ]; then + if [ "$ACTUAL_IMAGE_TAG" != "$EXPECTED_IMAGE_TAG" ]; then + echo "::warning::Deployment failed or rolled back. The new image is not running." echo "rollback_occurred=true" >> $GITHUB_OUTPUT else + echo "Deployment successful. The new image is running." echo "rollback_occurred=false" >> $GITHUB_OUTPUT fi diff --git a/.github/workflows/deploy-to-stg.yml b/.github/workflows/deploy-to-stg.yml index 6df1c69e..45ccc4e9 100644 --- a/.github/workflows/deploy-to-stg.yml +++ b/.github/workflows/deploy-to-stg.yml @@ -100,11 +100,21 @@ jobs: role-to-assume: ${{ env.AWS_IAM_ROLE_TO_ASSUME }} aws-region: ${{ env.AWS_REGION }} + - name: Backup Current Task Definition + run: | + echo "Backing up current task definition to task-def-backup.json" + aws ecs describe-task-definition --task-definition ${{ env.ECS_TASK_DEFINITION }} --query taskDefinition > task-def-backup.json + if [ ! -f task-def-backup.json ]; then + echo "::error::Failed to backup task definition." + exit 1 + fi + echo "Backup successful." + - name: Render New Task Definition with New Image id: task-def uses: aws-actions/amazon-ecs-render-task-definition@v1 with: - task-definition: ${{ env.ECS_TASK_DEFINITION }} + task-definition: task-def-backup.json container-name: ${{ env.CONTAINER_NAME }} image: ${{ needs.build.outputs.image_tag }} @@ -120,21 +130,25 @@ jobs: - name: Check for Rollback id: check-rollback - env: - NEW_TASK_DEF_ARN: ${{ steps.deploy.outputs.task-definition-arn }} run: | - DEPLOYED_TASK_DEF_ARN=$(aws ecs describe-services \ - --cluster ${{ env.ECS_CLUSTER }} \ - --services ${{ env.ECS_SERVICE }} \ - --query "services[0].taskDefinition" \ + FINAL_TASK_DEF_ARN="${{ steps.deploy.outputs.task-definition-arn }}" + EXPECTED_IMAGE_TAG="${{ needs.build.outputs.image_tag }}" + + echo "Final stable Task Definition ARN: $FINAL_TASK_DEF_ARN" + echo "Expected image tag: $EXPECTED_IMAGE_TAG" + + # Describe the final task definition and get the actual image tag for the specified container + ACTUAL_IMAGE_TAG=$(aws ecs describe-task-definition --task-definition "$FINAL_TASK_DEF_ARN" \ + --query "taskDefinition.containerDefinitions[?name=='${{ env.CONTAINER_NAME }}'].image" \ --output text) - echo "New Task Definition ARN: $NEW_TASK_DEF_ARN" - echo "Currently Deployed Task Definition ARN: $DEPLOYED_TASK_DEF_ARN" + echo "Actual image tag in final task definition: $ACTUAL_IMAGE_TAG" - if [ "$NEW_TASK_DEF_ARN" != "$DEPLOYED_TASK_DEF_ARN" ]; then + if [ "$ACTUAL_IMAGE_TAG" != "$EXPECTED_IMAGE_TAG" ]; then + echo "::warning::Deployment failed or rolled back. The new image is not running." echo "rollback_occurred=true" >> $GITHUB_OUTPUT else + echo "Deployment successful. The new image is running." echo "rollback_occurred=false" >> $GITHUB_OUTPUT fi From 57539a22de97707fa8c5f26adf398ca4229a804b Mon Sep 17 00:00:00 2001 From: Chori Date: Thu, 11 Sep 2025 00:40:55 +0900 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=92=9A=20Detect=20ECS=20Rollback=20th?= =?UTF-8?q?rough=20image=20tag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy-to-prod.yml | 4 +--- .github/workflows/deploy-to-stg.yml | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-to-prod.yml b/.github/workflows/deploy-to-prod.yml index 269d82e3..850cca79 100644 --- a/.github/workflows/deploy-to-prod.yml +++ b/.github/workflows/deploy-to-prod.yml @@ -114,7 +114,7 @@ jobs: id: task-def uses: aws-actions/amazon-ecs-render-task-definition@v1 with: - task-definition: ${{ env.ECS_TASK_DEFINITION }} + task-definition: task-def-backup.json container-name: ${{ env.CONTAINER_NAME }} image: ${{ needs.build.outputs.image_tag }} @@ -130,8 +130,6 @@ jobs: - name: Check for Rollback id: check-rollback - env: - NEW_TASK_DEF_ARN: ${{ steps.deploy.outputs.task-definition-arn }} run: | FINAL_TASK_DEF_ARN="${{ steps.deploy.outputs.task-definition-arn }}" EXPECTED_IMAGE_TAG="${{ needs.build.outputs.image_tag }}" diff --git a/.github/workflows/deploy-to-stg.yml b/.github/workflows/deploy-to-stg.yml index 186e6dca..45ccc4e9 100644 --- a/.github/workflows/deploy-to-stg.yml +++ b/.github/workflows/deploy-to-stg.yml @@ -114,7 +114,7 @@ jobs: id: task-def uses: aws-actions/amazon-ecs-render-task-definition@v1 with: - task-definition: ${{ env.ECS_TASK_DEFINITION }} + task-definition: task-def-backup.json container-name: ${{ env.CONTAINER_NAME }} image: ${{ needs.build.outputs.image_tag }} @@ -130,8 +130,6 @@ jobs: - name: Check for Rollback id: check-rollback - env: - NEW_TASK_DEF_ARN: ${{ steps.deploy.outputs.task-definition-arn }} run: | FINAL_TASK_DEF_ARN="${{ steps.deploy.outputs.task-definition-arn }}" EXPECTED_IMAGE_TAG="${{ needs.build.outputs.image_tag }}"