From 7ce9c368bda09f78e8c9b95cfa61efa7bd10d3c1 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 5 Dec 2024 14:44:42 +0100 Subject: [PATCH 1/7] update connector-tests.yml worfklow to use dev binary of airbyte-ci --- .github/workflows/connector-tests.yml | 34 ++++++++++----------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 0f7033f1f..9fdf7592c 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -86,12 +86,10 @@ jobs: # TODO: Re-enable once fixed # - connector: source-zendesk-support # cdk_extra: n/a - # TODO: These are manifest connectors and won't work as expected until we - # add `--use-local-cdk` support for manifest connectors. - # - connector: source-the-guardian-api - # cdk_extra: n/a - # - connector: source-pokeapi - # cdk_extra: n/a + - connector: source-the-guardian-api + cdk_extra: n/a + - connector: source-pokeapi + cdk_extra: n/a name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs['src'] == 'false' || needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})" steps: @@ -121,23 +119,17 @@ jobs: repository: airbytehq/airbyte ref: master path: airbyte - - name: Test Connector + - name: Test connector if: steps.no_changes.outputs.status != 'cancelled' + id: run-airbyte-ci-connector-test-pr timeout-minutes: 90 - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0" - # TODO: Revert below to use `tools.airbyte-ci-binary.install` after Airbyte CI released: - run: | - cd airbyte - make tools.airbyte-ci-dev.install - airbyte-ci-dev connectors \ - --name ${{matrix.connector}} \ - --use-local-cdk \ - test \ - --fail-fast \ - --skip-step qa_checks \ - --skip-step connector_live_tests + uses: ./.github/actions/run-airbyte-ci + with: + context: "pull_request" + subcommand: "--name ${{matrix.connector}} test --fail-fast --skip-step qa_checks --skip-step connector_live_tests" + # TODO remove me once local cdk is confirmed to work for manifest only connector + # We use a custom binary url to run the airbyte-ci version of this PR: https://github.com/airbytehq/airbyte/pull/48818 + airbyte_ci_binary_url: https://storage.googleapis.com/dev-airbyte-cloud-connector-metadata-service/airbyte-ci/releases/ubuntu/f217461/airbyte-ci - name: Evaluate Test Output if: always() && steps.no_changes.outputs.status != 'cancelled' From bda9cfeca736c181da0ffff476ed0989732a8eb7 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 5 Dec 2024 14:47:29 +0100 Subject: [PATCH 2/7] dummy change to trigger connector test workflow --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9a0e80327..f2efe6c6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -201,3 +201,5 @@ optional_poetry_groups = ["dev"] poetry_extras = ["file-based", "sphinx-docs", "vector-db-based"] poe_tasks = ["check-ci"] mount_docker_socket = true + +#TODO revert me From 956fe71e0338faa030dcd6a0858590fb66d3e6cf Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 5 Dec 2024 14:51:46 +0100 Subject: [PATCH 3/7] fix action path --- .github/workflows/connector-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 9fdf7592c..c52881be2 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -123,7 +123,7 @@ jobs: if: steps.no_changes.outputs.status != 'cancelled' id: run-airbyte-ci-connector-test-pr timeout-minutes: 90 - uses: ./.github/actions/run-airbyte-ci + uses: ./airbyte/.github/actions/run-airbyte-ci with: context: "pull_request" subcommand: "--name ${{matrix.connector}} test --fail-fast --skip-step qa_checks --skip-step connector_live_tests" From 956d7d4ec2c0e19c2f26acf9b35943b361197801 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 5 Dec 2024 18:18:20 +0100 Subject: [PATCH 4/7] use dev branch to install airbyte-ci --- .github/workflows/connector-tests.yml | 36 ++++++++++++++++----------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index c52881be2..97ba9e072 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -86,10 +86,12 @@ jobs: # TODO: Re-enable once fixed # - connector: source-zendesk-support # cdk_extra: n/a - - connector: source-the-guardian-api - cdk_extra: n/a - - connector: source-pokeapi - cdk_extra: n/a + # TODO: These are manifest connectors and won't work as expected until we + # add `--use-local-cdk` support for manifest connectors. + # - connector: source-the-guardian-api + # cdk_extra: n/a + # - connector: source-pokeapi + # cdk_extra: n/a name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs['src'] == 'false' || needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})" steps: @@ -117,19 +119,25 @@ jobs: if: steps.no_changes.outputs.status != 'cancelled' with: repository: airbytehq/airbyte - ref: master + ref: augustin/use-local-cdk-manifest-only path: airbyte - - name: Test connector + - name: Test Connector if: steps.no_changes.outputs.status != 'cancelled' - id: run-airbyte-ci-connector-test-pr timeout-minutes: 90 - uses: ./airbyte/.github/actions/run-airbyte-ci - with: - context: "pull_request" - subcommand: "--name ${{matrix.connector}} test --fail-fast --skip-step qa_checks --skip-step connector_live_tests" - # TODO remove me once local cdk is confirmed to work for manifest only connector - # We use a custom binary url to run the airbyte-ci version of this PR: https://github.com/airbytehq/airbyte/pull/48818 - airbyte_ci_binary_url: https://storage.googleapis.com/dev-airbyte-cloud-connector-metadata-service/airbyte-ci/releases/ubuntu/f217461/airbyte-ci + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0" + # TODO: Revert below to use `tools.airbyte-ci-binary.install` after Airbyte CI released: + run: | + cd airbyte + make tools.airbyte-ci-dev.install + airbyte-ci-dev connectors \ + --name ${{matrix.connector}} \ + --use-local-cdk \ + test \ + --fail-fast \ + --skip-step qa_checks \ + --skip-step connector_live_tests - name: Evaluate Test Output if: always() && steps.no_changes.outputs.status != 'cancelled' From 9e45294d04b06b603ba796f66cd4faf8c1e67bae Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 5 Dec 2024 18:19:06 +0100 Subject: [PATCH 5/7] re-enable manifest only connectors --- .github/workflows/connector-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 97ba9e072..ee7d504f1 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -88,10 +88,10 @@ jobs: # cdk_extra: n/a # TODO: These are manifest connectors and won't work as expected until we # add `--use-local-cdk` support for manifest connectors. - # - connector: source-the-guardian-api - # cdk_extra: n/a - # - connector: source-pokeapi - # cdk_extra: n/a + - connector: source-the-guardian-api + cdk_extra: n/a + - connector: source-pokeapi + cdk_extra: n/a name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs['src'] == 'false' || needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})" steps: From b84e1c2fe3dd7bc1b20a2f07d54408a23dcf8f2e Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 6 Dec 2024 15:02:01 +0100 Subject: [PATCH 6/7] target main monorepo branch --- .github/workflows/connector-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index ee7d504f1..7e5399fc7 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -119,7 +119,7 @@ jobs: if: steps.no_changes.outputs.status != 'cancelled' with: repository: airbytehq/airbyte - ref: augustin/use-local-cdk-manifest-only + ref: master path: airbyte - name: Test Connector if: steps.no_changes.outputs.status != 'cancelled' From f4fe815077443301871b98077ff94a0b175440ca Mon Sep 17 00:00:00 2001 From: Augustin Date: Fri, 6 Dec 2024 15:14:02 +0100 Subject: [PATCH 7/7] revert dummy pyproject.toml change --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 250e3b096..c3267ed70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -198,5 +198,3 @@ optional_poetry_groups = ["dev"] poetry_extras = ["file-based", "vector-db-based"] poe_tasks = ["check-ci"] mount_docker_socket = true - -#TODO revert me