Add support for bigquery testing in GitHub CI via tox (#203)

* add support for bigquery testing

* add missing var in tox file

* Temporarily only run CI tests for BigQuery

* Prefix the schema for the data source in CI with the name of the target schema

* Store artifacts for logs and target directories for BigQuery

* Set up environment variable for BigQuery credentials (keyfile for service account JSON)

* Set the custom schema in the source definition

* Use the target schema

* Try to align actual vs. expected when the schema name is variable

* Remove extraneous storage of artifacts

* Temporarily disable two failing CI tests

* Revert "Temporarily disable two failing CI tests"

This reverts commit d70d776.

---------

Co-authored-by: Doug Beatty <[email protected]>

Author: emmyoop

Diff for: .github/workflows/ci.yml

@@ -29,6 +29,9 @@ jobs:
         SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
         SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
         SNOWFLAKE_SCHEMA: "integration_tests_snowflake_${{ github.run_number }}"
+        # bigquery
+        BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
+        BIGQUERY_SCHEMA: "integration_tests_bigquery_${{ github.run_number }}"
         # redshift
         REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
         REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
@@ -39,3 +42,4 @@ jobs:
         SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
         DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }}
         DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }}
+        BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }}

Diff for: integration_tests/profiles.yml

@@ -28,10 +28,11 @@ integration_tests:
 
     bigquery:
       type: bigquery
-      method: service-account
-      keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}"
-      project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}"
-      schema: codegen_integration_tests_bigquery
+      method: service-account-json
+      keyfile_json:
+        "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native}}"
+      project: "{{ env_var('BIGQUERY_PROJECT') }}"
+      dataset: "{{ env_var('BIGQUERY_SCHEMA') }}"
       threads: 1
 
     snowflake:

Diff for: supported_adapters.env

@@ -1 +1 @@
-SUPPORTED_ADAPTERS=postgres,snowflake,redshift
+SUPPORTED_ADAPTERS=postgres,snowflake,redshift,bigquery

Diff for: tox.ini

@@ -19,6 +19,10 @@ passenv =
     SNOWFLAKE_DATABASE
     SNOWFLAKE_WAREHOUSE
     SNOWFLAKE_SCHEMA
+    # bigquery env vars
+    BIGQUERY_PROJECT
+    BIGQUERY_SCHEMA
+    BIGQUERY_KEYFILE_JSON
     # redshift env vars
     REDSHIFT_HOST
     REDSHIFT_USER
@@ -57,6 +61,21 @@ commands =
     dbt --warn-error run --target snowflake
     dbt --warn-error test --target snowflake
 
+# bigquery integration tests for centralized dbt testing
+# run dbt commands directly, assumes dbt is already installed in environment
+[testenv:dbt_integration_bigquery]
+changedir = integration_tests
+allowlist_externals = 
+    dbt
+skip_install = true
+commands =
+    dbt --version
+    dbt --warn-error deps --target bigquery
+    dbt --warn-error run-operation create_source_table --target bigquery
+    dbt --warn-error seed --target bigquery --full-refresh
+    dbt --warn-error run --target bigquery
+    dbt --warn-error test --target bigquery
+
 # redshift integration tests for centralized dbt testing
 # run dbt commands directly, assumes dbt is already installed in environment
 [testenv:dbt_integration_redshift]