Merge pull request #17 from bruno-szdl/leo-teardown-job

implementation of ci_teardown job

.github/workflows/ci_teardown.yml

@@ -0,0 +1,56 @@
+name: CI schema teardown on PR close
+
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  CI_TEARDOWN_job:
+    runs-on: ubuntu-latest
+
+    env:
+      GCLOUD_SERVICE_KEY: ${{ secrets.GCLOUD_SERVICE_KEY }}
+      GCLOUD_STORAGE_PATH_MANIFEST: ${{ secrets.GCLOUD_STORAGE_PATH_MANIFEST }}
+      DBT_ENV_SECRET_TYPE: ${{ secrets.DBT_ENV_SECRET_TYPE }} 
+      DBT_ENV_SECRET_PROJECT_ID: ${{ secrets.DBT_ENV_SECRET_PROJECT_ID }}
+      DBT_ENV_SECRET_PRIVATE_KEY_ID: ${{ secrets.DBT_ENV_SECRET_PRIVATE_KEY_ID }} 
+      DBT_ENV_SECRET_PRIVATE_KEY: ${{ secrets.DBT_ENV_SECRET_PRIVATE_KEY }}  
+      DBT_ENV_SECRET_CLIENT_EMAIL: ${{ secrets.DBT_ENV_SECRET_CLIENT_EMAIL }}
+      DBT_ENV_SECRET_CLIENT_ID: ${{ secrets.DBT_ENV_SECRET_CLIENT_ID }} 
+      DBT_ENV_SECRET_AUTH_URI: ${{ secrets.DBT_ENV_SECRET_AUTH_URI }}
+      DBT_ENV_SECRET_TOKEN_URI: ${{ secrets.DBT_ENV_SECRET_TOKEN_URI }}
+      DBT_ENV_SECRET_AUTH_PROVIDER_X509_CERT_URL: ${{ secrets.DBT_ENV_SECRET_AUTH_PROVIDER_X509_CERT_URL }} 
+      DBT_ENV_SECRET_CLIENT_X509_CERT_URL: ${{ secrets.DBT_ENV_SECRET_CLIENT_X509_CERT_URL }} 
+
+    steps:
+    - name: checkout
+      uses: actions/checkout@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.12'
+
+    - name: Install dependencies
+      run: pip install -r dbt-requirements.txt
+
+    - name: Authenticate to Google Cloud
+      uses: google-github-actions/[email protected]
+      with:
+        credentials_json: ${{ secrets.GCLOUD_SERVICE_KEY }}
+
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v1
+      with:
+        project_id: ${{ secrets.DBT_ENV_SECRET_PROJECT_ID }}
+
+    - name: Get PR NUM
+      id: schema_id
+      run: echo "PR_NUM=${{ github.event.number }}" >> $GITHUB_ENV
+
+    - name: dbt deps
+      run: dbt deps
+
+    - name: drop PR schemas
+      run: dbt run-operation drop_pr_staging_schemas --args "{'PR_number'":" '${PR_NUM}' }" --profiles-dir ./

dbt_project.yml

@@ -41,6 +41,4 @@ data_tests:
 
 seeds:
   dbt_project_evaluator:
-    +enabled: "{{ env_var('ENABLE_DBT_PROJECT_EVALUATOR', 'false') | lower == 'true' | as_bool }}"
-
-on-run-end: "{% if target.name == 'pr' %} drop schema {{ target.schema }} cascade {% endif %}"
+    +enabled: "{{ env_var('ENABLE_DBT_PROJECT_EVALUATOR', 'false') | lower == 'true' | as_bool }}"

macros/drop_pr_staging_schemas.sql

@@ -0,0 +1,29 @@
+{%- macro drop_pr_staging_schemas(PR_number) %}
+
+    {% set pr_cleanup_query %}
+        with pr_staging_schemas as (
+            select schema_name
+            from information_schema.schemata
+            where
+            schema_name like 'pr_'||{{ PR_number }}||'__%'
+        )
+
+        select 
+            'drop schema '||schema_name||';' as drop_command 
+        from pr_staging_schemas
+    {% endset %}
+
+{% do log(pr_cleanup_query, info=TRUE) %}
+
+{% set drop_commands = run_query(pr_cleanup_query).columns[0].values() %}
+
+{% if drop_commands %}
+  {% for drop_command in drop_commands %}
+    {% do log(drop_command, True) %}
+      {% do run_query(drop_command) %}
+  {% endfor %}
+{% else %}
+  {% do log('No schemas to drop.', True) %}
+{% endif %}
+
+{%- endmacro -%}