Use labels from arrows (#5)

* Do not concat tables on the driver anymore

* Add integration tests

* Add token permissions

* Add GDS license

* Set environment variables for authentication

* Update neo4j_arrow version

.github/workflows/build.yml

@@ -12,7 +12,9 @@ env:
 jobs:
   build:
     runs-on: ubuntu-latest
-
+    permissions:
+      contents: 'read'
+      id-token: 'write'
     steps:
       - uses: actions/checkout@v3
       - name: set up python
@@ -28,7 +30,20 @@ jobs:
         run: echo "${POETRY_HOME}/bin" >> $GITHUB_PATH
       - name: install dependencies and build
         run: poetry install
-      - name: run tests
-        run: poetry run pytest
+      - name: run unit tests
+        run: poetry run pytest tests/unit
+      - uses: 'google-github-actions/auth@v2'
+        with:
+          project_id: ${{ vars.GCP_PROJECT }}
+          workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          create_credentials_file: true
+          export_environment_variables: true
+      - name: run integration tests
+        env:
+          gds_license: ${{ secrets.GDS_LICENSE }}
+        run: |
+          LICENSE_FILE=${{ runner.temp }}/license.tmp
+          echo "${gds_license}" > $LICENSE_FILE
+          GDS_LICENSE_FILE=$LICENSE_FILE poetry run pytest tests/integration
       - name: build image
         run: docker build . --platform linux/amd64 --file Dockerfile --tag neo4j-bigquery-connector:$(date +%s)

.pre-commit-config.yaml

@@ -23,4 +23,5 @@ repos:
     rev: v1.7.0
     hooks:
       - id: mypy
+        exclude: ^tests/
         args: [ ]

pyproject.toml

@@ -13,9 +13,9 @@ google-cloud-bigquery-storage = { extras = ["pyarrow"], version = "^2.22.0" }
 google-cloud-secret-manager = "^2.16.4"
 antlr4-python3-runtime = "^4.13.1"
 fsspec = { extras = ["gcs"], version = "^2023.10.0" }
-neo4j-arrow = { git = "https://github.com/neo4j-field/neo4j_arrow.git", rev = "main" }
 neo4j = ">=5"
 importlib-metadata = "^6.8.0"
+neo4j-arrow = {git = "https://github.com/neo4j-field/neo4j_arrow.git", rev = "0.6.1"}
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.4.3"
@@ -33,6 +33,8 @@ flake8-spellcheck = "^0.28.0"
 flake8-comprehensions = "^3.14.0"
 flake8-bandit = "^4.1.1"
 types-protobuf = "^4.24.0.4"
+testcontainers = "^3.7.1"
+pandas = "^2.1.4"
 
 [build-system]
 requires = ["poetry-core"]