update deploy process #6
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy | ||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| environment: | ||
| description: 'Target environment' | ||
| required: true | ||
| default: 'staging' | ||
| type: choice | ||
| options: | ||
| - staging | ||
| - ${{ github.event.inputs.environment }} | ||
| env: | ||
| AWS_REGION: us-east-1 | ||
| PROJECT_NAME: mindsdb | ||
| # # AWS credentials | ||
| # AWS_GIT_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | ||
| # AWS_GIT_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | ||
| # AWS_REGION: us-east-1 | ||
| # # For clonning private composer dependencies | ||
| # COMPOSER_AUTH: '{"github-oauth": { "github.com": "${{ secrets.GIT_COMPOSER_AUTH }}" } }' | ||
| # # Kubernetes | ||
| # PROJECT_NAME: mindsdb | ||
| # KUBE_EKS_CLUSTER: programmatic-ad-prod-cluster | ||
| # # | ||
| # CONTAINER_REPOSITORY_URI: "124082513016.dkr.ecr.us-east-1.amazonaws.com/mindsdb" | ||
| # ENVIRONMENT: prod | ||
| jobs: | ||
| deploy: | ||
| # if: github.ref == 'refs/heads/main' | ||
| name: Deploy to ${{ github.event.inputs.environment }} | ||
| runs-on: ubuntu-latest | ||
| environment: | ||
| name: ${{ github.event.inputs.environment }} | ||
| # Custom environment variables | ||
| # reference: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idenv | ||
| env: | ||
| # Variables need be change on each environment | ||
| ENVIRONMENT: ${{ github.event.inputs.environment }} | ||
| KUBE_EKS_CLUSTER: programmatic-ad-prod-cluster | ||
| K8S_NAMESPACE: mindsdb | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Set Python version | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: "3.12" | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
| - name: Configure AWS credentials | ||
| uses: aws-actions/configure-aws-credentials@v4 | ||
| with: | ||
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | ||
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | ||
| aws-region: ${{ secrets.AWS_REGION }} | ||
| mask-aws-account-id: "true" | ||
| - name: Login to Amazon ECR | ||
| id: login-ecr | ||
| uses: aws-actions/amazon-ecr-login@v2 | ||
| with: | ||
| mask-password: "true" | ||
| # references: | ||
| # https://docs.github.com/en/actions/learn-github-actions/environment-variables#passing-values-between-steps-and-jobs-in-a-workflow | ||
| # https://docs.github.com/en/actions/learn-github-actions/contexts#needs-context | ||
| - name: Generate new APP_VERSION and publish release | ||
| id: generate_app_version | ||
| run: | | ||
| APP_VERSION="$(date +'%Y-%m-%d_%H%M%S')" | ||
| APP_LATEST_IMAGE_VERSION="${APP_VERSION}" | ||
| MIGRATION_JOB_NAME=$(echo $APP_LATEST_IMAGE_VERSION | sed s/_/-/g) | ||
| echo "MIGRATION_JOB_NAME=${MIGRATION_JOB_NAME}" >> $GITHUB_ENV | ||
| echo "APP_VERSION=${APP_VERSION}" >> $GITHUB_ENV | ||
| echo "APP_LATEST_IMAGE_VERSION=${APP_LATEST_IMAGE_VERSION}" >> $GITHUB_ENV | ||
| echo GIT_LAST_COMMIT_MESSAGE=\"$(git log -n 1 --format=%s)\" >> $GITHUB_ENV | ||
| env: | ||
| TZ: America/Sao_Paulo | ||
| - name: Log in K8S | ||
| run: | | ||
| aws eks update-kubeconfig --region ${{ secrets.AWS_REGION }} --name ${{ env.KUBE_EKS_CLUSTER }} | ||
| #| | ||
| #| Update secret APP_VERSION | ||
| #| | ||
| - name: "[${{ env.ENVIRONMENT }}] mindsdb: update Secret 'APP_VERSION'" | ||
| env: | ||
| K8S_NAMESPACE: mindsdb | ||
| run: | | ||
| kubectl patch secret -n mindsdb mindsdb-all -p="{\"data\":{\"APP_VERSION\":\"$(echo -n $APP_VERSION | base64)\"}}" | ||
| # #| | ||
| # #| Build main image | ||
| # #| | ||
| - name: Build and push - mindsdb base | ||
| uses: docker/build-push-action@v6 | ||
| with: | ||
| target: web-base | ||
| context: . | ||
| file: var/docker/Dockerfile | ||
| push: true | ||
| build-args: | | ||
| APP_VERSION=${{ env.APP_VERSION }} | ||
| GH_AUTH=${{ secrets.GIT_COMPOSER_AUTH }} | ||
| tags: | | ||
| ${{ env.CONTAINER_REPOSITORY_URI }}:web-prod-${{ env.APP_LATEST_IMAGE_VERSION }} | ||
| ${{ env.CONTAINER_REPOSITORY_URI }}:web-prod-latest | ||
| cache-from: type=gha | ||
| cache-to: type=gha,mode=max | ||
| # Trivy vulnerability scanner — FS (IaC + dependências) em tabela | ||
| - name: Trivy FS (table) | ||
| uses: aquasecurity/trivy-action@master | ||
| with: | ||
| scan-type: fs | ||
| scan-ref: . | ||
| ignore-unfixed: true | ||
| severity: CRITICAL,HIGH | ||
| format: table | ||
| output: trivy-fs.table.txt | ||
| exit-code: 0 # opcional: não falha o job; remova se quiser falhar em vulnerabilidade | ||
| # Trivy vulnerability scanner — Repo (SBOM/deps) em tabela | ||
| - name: Trivy Repo (table) | ||
| uses: aquasecurity/trivy-action@master | ||
| with: | ||
| scan-type: repo | ||
| scan-ref: . | ||
| ignore-unfixed: true | ||
| severity: CRITICAL,HIGH | ||
| format: table | ||
| output: trivy-repo.table.txt | ||
| exit-code: 0 | ||
| # Trivy vulnerability scanner — FS em JSON (para parsing/CI) | ||
| - name: Trivy FS (json) | ||
| uses: aquasecurity/trivy-action@master | ||
| with: | ||
| scan-type: fs | ||
| scan-ref: . | ||
| ignore-unfixed: true | ||
| severity: CRITICAL,HIGH | ||
| format: json | ||
| output: trivy-fs.json | ||
| exit-code: 0 | ||
| # Trivy vulnerability scanner | ||
| - name: Verificar vulnerabilidades na imagem Docker | ||
| uses: aquasecurity/trivy-action@0.28.0 | ||
| with: | ||
| scan-type: 'image' | ||
| image-ref: '${{ env.CONTAINER_REPOSITORY_URI }}:web-${{ env.ENVIRONMENT }}-${{ env.APP_LATEST_IMAGE_VERSION }}' | ||
| format: 'table' | ||
| exit-code: '0' # Não falhará se encontrar vulnerabilidades | ||
| ignore-unfixed: true | ||
| vuln-type: 'os,library' | ||
| severity: 'CRITICAL,HIGH' | ||
| output: trivy-image.table.txt | ||
| # Escrever um relatório no Job Summary com os relatórios em tabela | ||
| - name: Montar Job Summary | ||
| if: always() | ||
| run: | | ||
| { | ||
| echo "## Trivy — Sumário" | ||
| echo "" | ||
| echo "### FS (table)" | ||
| echo '```' | ||
| sed -n '1,200p' trivy-fs.table.txt | ||
| echo '```' | ||
| echo "" | ||
| echo "### Repo (table)" | ||
| echo '```' | ||
| sed -n '1,200p' trivy-repo.table.txt | ||
| echo '```' | ||
| } >> "$GITHUB_STEP_SUMMARY" | ||
| # Upload de TODOS os relatórios gerados | ||
| - name: Upload dos Relatórios Trivy | ||
| if: always() | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: trivy-reports | ||
| path: | | ||
| trivy-fs.table.txt | ||
| trivy-repo.table.txt | ||
| trivy-fs.json | ||
| trivy-image.table.txt | ||
| retention-days: 1 | ||
| #| Create release | ||
| - run: npm install axios | ||
| - uses: actions/github-script@v6 | ||
| name: "Create release" | ||
| with: | ||
| script: | | ||
| const script = require('./.github/workflows/create_release.js') | ||
| await script({github, context}) | ||
| env: | ||
| # https://app.shortcut.com/talentify/story/40035 | ||
| SHORTCUT_API_TOKEN: ${{ secrets.SHORTCUT_API_TOKEN }} | ||
| #| | ||
| #| Namespace: web / migrations | ||
| #| | ||
| - name: "[prod] migrations: update and run migrations job" | ||
| env: | ||
| K8S_NAMESPACE: mindsdb | ||
| run: | | ||
| sed -i 's/MIGRATION_JOB_NAME/${{ env.MIGRATION_JOB_NAME }}/g' ./var/kubernetes/${{ env.ENVIRONMENT }}/web/migrations.yaml | ||
| sed -i 's/WEB_TAG_TEMPLATE/web-prod-${{ env.APP_LATEST_IMAGE_VERSION }}/g' ./var/kubernetes/${{ env.ENVIRONMENT }}/web/migrations.yaml | ||
| kubectl apply -f ./var/kubernetes/${{ env.ENVIRONMENT }}/web/migrations.yaml -n ${{ env.K8S_NAMESPACE }} | ||
| sleep 5 | ||
| # kubectl wait --for=condition=complete --timeout=3600s job/mindsdb-migrations-${{ env.MIGRATION_JOB_NAME }} -n ${{ env.K8S_NAMESPACE }} | ||
| # please, note that the k8s job also has a timeout defined by "ttlSecondsAfterFinished" | ||
| - name: Wait for Job migrations complete | ||
| id: wait-migrations | ||
| if: ${{ always() }} | ||
| run: | | ||
| while true; do | ||
| STATUS=$(kubectl get job mindsdb-migrations-${{ env.MIGRATION_JOB_NAME }} -n ${{ env.K8S_NAMESPACE }} -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}') | ||
| if [ "$STATUS" == "True" ]; then | ||
| echo "Job succeeded" | ||
| break | ||
| fi | ||
| STATUS=$(kubectl get job mindsdb-migrations-${{ env.MIGRATION_JOB_NAME }} -n ${{ env.K8S_NAMESPACE }} -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}') | ||
| if [ "$STATUS" == "True" ]; then | ||
| echo "Job failed" | ||
| break | ||
| fi | ||
| echo "Waiting for job to complete..." | ||
| sleep 2 | ||
| done | ||
| - name: Get migrations Pod Names | ||
| if: ${{ always() }} | ||
| id: get-pod-names | ||
| run: | | ||
| POD_NAMES=$(kubectl get pods -n ${{ env.K8S_NAMESPACE }} -l job-name=mindsdb-migrations-${{ env.MIGRATION_JOB_NAME }} -o jsonpath='{.items[*].metadata.name}') | ||
| echo "POD_NAMES=${POD_NAMES}" >> $GITHUB_ENV | ||
| - name: Save Migrations Logs | ||
| if: ${{ always() }} | ||
| id: check-status | ||
| run: | | ||
| STATUS=$(kubectl get job mindsdb-migrations-${{ env.MIGRATION_JOB_NAME }} -n ${{ env.K8S_NAMESPACE }} -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}') | ||
| PODS=(${{ env.POD_NAMES }}) | ||
| for POD in "${PODS[@]}"; do | ||
| echo "========== log pod: $POD =============" >> "mindsdb-migrations-${{ env.MIGRATION_JOB_NAME }}.txt" | ||
| kubectl logs pod/$POD -n ${{ env.K8S_NAMESPACE }} >> "mindsdb-migrations-${{ env.MIGRATION_JOB_NAME }}.txt" | ||
| echo "==========================================" >> "mindsdb-migrations-${{ env.MIGRATION_JOB_NAME }}.txt" | ||
| done | ||
| if [ "$STATUS" != "True" ]; then | ||
| echo "Job failed" | ||
| exit 1 | ||
| fi | ||
| echo "Job succeeded" | ||
| - name: Show Migrations Logs | ||
| if: ${{ always() }} | ||
| run: | | ||
| cat mindsdb-migrations-${{ env.MIGRATION_JOB_NAME }}.txt | ||
| - name: Upload Migrations Logs Artifact | ||
| if: ${{ always() }} | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: mindsdb-migrations-${{ env.MIGRATION_JOB_NAME }} | ||
| path: mindsdb-migrations-${{ env.MIGRATION_JOB_NAME }}.txt | ||
| retention-days: 1 | ||
| #| | ||
| #| Namespace: cronjob | ||
| #| | ||
| # ## Nesse método estou dando apply em todo o diretório /cronjob logo se adicionar um arquivo novo ele será adicionado no próximo deploy | ||
| # - name: "[${{ env.ENVIRONMENT }}] cronjob: update configurations for cronjobs" | ||
| # id: deployment | ||
| # env: | ||
| # K8S_NAMESPACE: mindsdb | ||
| # run: | | ||
| # FILES=$(cd var/kubernetes/${{ env.ENVIRONMENT }}/cronjob && ls *.yaml) | ||
| # for pod_name in $FILES; do | ||
| # echo "Updating CronJob $pod_name" | ||
| # sed -i 's/WEB_TAG_TEMPLATE/web-${{ env.ENVIRONMENT }}-${{ env.APP_LATEST_IMAGE_VERSION }}/g' ./var/kubernetes/${{ env.ENVIRONMENT }}/cronjob/$pod_name | ||
| # kubectl apply -f ./var/kubernetes/${{ env.ENVIRONMENT }}/cronjob/$pod_name -n ${{ env.K8S_NAMESPACE }} | ||
| # done | ||
| #| | ||
| #| Deploy web | ||
| #| | ||
| - name: "[${{ env.ENVIRONMENT }}] web: update image for workload 'mindsdb-web', container 'mindsdb-web'." | ||
| id: deployment | ||
| env: | ||
| K8S_NAMESPACE: mindsdb | ||
| run: | | ||
| sed -i 's/WEB_TAG_TEMPLATE/web-${{ env.ENVIRONMENT }}-${{ env.APP_LATEST_IMAGE_VERSION }}/g' ./var/kubernetes/${{ env.ENVIRONMENT }}/web/deployment.yaml | ||
| kubectl config use-context arn:aws:eks:us-east-1:124082513016:cluster/${{ env.KUBE_EKS_CLUSTER }} | ||
| kubectl apply -f ./var/kubernetes/${{ env.ENVIRONMENT }}/web/deployment.yaml -n ${{ env.K8S_NAMESPACE }} | ||
| - uses: actions/checkout@v4 | ||
| - name: Create Sentry release | ||
| uses: getsentry/action-release@v1 | ||
| env: | ||
| SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} | ||
| SENTRY_ORG: ${{ secrets.SENTRY_ORG }} | ||
| SENTRY_PROJECT: ${{ env.PROJECT_NAME }} | ||
| with: | ||
| environment: prod | ||
| version: ${{ env.APP_VERSION }} | ||
| - name: Status | ||
| id: status | ||
| if: ${{ always() }} | ||
| env: | ||
| BLOCK_FAILURE: ff2500 | ||
| BLOCK_SUCCESS: 2b8a3e | ||
| run: | | ||
| if [[ "${{ steps.deployment.outcome }}" != 'success' ]]; then | ||
| echo ${{ steps.deployment.outcome }} | ||
| echo "flag=${{ steps.deployment.outcome }} :red_circle:" >> $GITHUB_OUTPUT | ||
| echo "block=${{ env.BLOCK_FAILURE }}" >> $GITHUB_OUTPUT | ||
| else | ||
| echo ${{ steps.deployment.outcome }} | ||
| echo "flag=${{ steps.deployment.outcome }} :large_green_circle:" >> $GITHUB_OUTPUT | ||
| echo "block=${{ env.BLOCK_SUCCESS }}" >> $GITHUB_OUTPUT | ||
| fi | ||
| #| | ||
| #| Notify | ||
| #| | ||
| # reference: https://github.com/slackapi/slack-github-action#technique-3-slack-incoming-webhook | ||
| # Create your app or use mine: https://api.slack.com/apps/A037R2Q23RQ | ||
| # Copy the "Webhook URL" and then add it in GitHub secret called "SLACK_WEBHOOK_URL". | ||
| - name: Notify about the new deployment | ||
| id: slack | ||
| if: ${{ always() }} | ||
| uses: slackapi/slack-github-action@v2.0.0 | ||
| with: | ||
| webhook: ${{ secrets.SLACK_WEBHOOK_URL }} | ||
| webhook-type: incoming-webhook | ||
| # For posting a rich message using Block Kit | ||
| payload: | | ||
| { | ||
| "attachments": [ | ||
| { | ||
| "color": "${{ steps.status.outputs.block }}", | ||
| "blocks": [ | ||
| { | ||
| "type": "header", | ||
| "text": { | ||
| "type": "plain_text", | ||
| "text": "Deploy - ${{ env.PROJECT_NAME }} - env: ${{ env.ENVIRONMENT }} - Created by: ${{ github.actor }}", | ||
| "emoji": true | ||
| } | ||
| }, | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": "*Status:* ${{ steps.status.outputs.flag }} - *Version:* <${{ env.RELEASE_URL }}|${{ env.APP_VERSION }}> - *Branch:* <${{ github.server_url }}/${{ github.repository }}/tree/${{ github.ref_name }}|${{ github.ref_name }}>" | ||
| } | ||
| } | ||
| ] | ||
| } | ||
| ] | ||
| } | ||
| env: | ||
| RELEASE_URL: ${{ github.server_url }}/${{ github.repository }}/releases/tag/v${{ env.APP_VERSION }} | ||
