Merge pull request #14 from b-long/feature/improved-integration-testing

b-long · web-flow · commit a937417d7de3 · 2024-11-20T09:11:55.000-05:00
End-to-end (integration testing) for Python wheels &amp; OpenTDF Platform
diff --git a/.github/workflows/build-golang-macos.yaml b/.github/workflows/build-golang-macos.yaml
@@ -76,32 +76,20 @@ jobs:
         # FIXME: Add more caching
         - name: Add gopy dependencies and build wheel
           run: |
-              # Since we don't have our wheel build / install configured yet we use '--no-root'
-              poetry install --no-root
-
-              source $(poetry env info --path)/bin/activate
-
-              # Add Go bin directory to PATH
-              echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV
-
-              go install golang.org/x/tools/cmd/goimports@latest
-
-              go install github.com/go-python/gopy@latest
-
-              poetry run pip install --upgrade setuptools wheel
-
-              gopy build --output=otdf_python -vm=python3 .
-
-              poetry run python3 setup.py bdist_wheel
+            ./ci-build.sh
 
         - name: Test Python wheel
           run: |
               # Test wheel installation
-              pip install dist/otdf_python-0.0.15-py3-none-any.whl
+              pip install dist/otdf_python-0.1.0-py3-none-any.whl
 
               # Test wheel functionality
               # python3 validate_otdf_python.py
 
+        - uses: ./.github/workflows/platform-integration-test.yaml
+          with:
+            wheel: dist/otdf_python-0.1.0-py3-none-any.whl
+
   # release:
   #   needs: build
   #   runs-on: macos-latest
diff --git a/.github/workflows/build-golang-ubuntu.yaml b/.github/workflows/build-golang-ubuntu.yaml
@@ -57,78 +57,36 @@ jobs:
             # FIXME: Add more caching
             - name: Add gopy dependencies and build wheel
               run: |
-                  # Since we don't have our wheel build / install configured yet we use '--no-root'
-                  poetry install --no-root
-
-                  source $(poetry env info --path)/bin/activate
-
-                  # Add Go bin directory to PATH
-                  echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV
-
-                  go install golang.org/x/tools/cmd/goimports@latest
-
-                  go install github.com/go-python/gopy@latest
-
-                  poetry run pip install --upgrade setuptools wheel
-
-                  gopy build --output=otdf_python -vm=python3 .
-
-                  poetry run python3 setup.py bdist_wheel
-
-    # TODO: Can this be a distinct job?
-    # exercise-ssh:
-    #     needs: build
-
-    #     runs-on: ubuntu-latest
-
-    #     steps:
-    #        - uses: actions/checkout@v4
+                ./ci-build.sh
 
             - name: Check existing workspace content
               run: |
                   ls -la
                   ls -la dist/
 
-            # - name: Set up Docker Buildx
-            #   uses: docker/setup-buildx-action@v3
-
-            # - name: Install Vagrant
-            #   run: sudo apt-get update && sudo apt-get install -y vagrant
-
-            # - name: Create Vagrantfile
-            #   run: |
-            #       cat << EOF > Vagrantfile
-            #       Vagrant.configure("2") do |config|
-            #         config.vm.provider "docker" do |d|
-            #           d.image = "ubuntu"
-            #           d.cmd = [ "/bin/bash", "-c", "echo 'Hello world'" ]
-            #           d.remains_running = false
-            #         end
-            #       end
-            #       EOF
-
-            # - name: Run vagrant up
-            #   run: vagrant up --provider=docker
-
-            # - name: Launch the Docker/Vagrant test machine
-            #   run: |
-            #       # Change directory
-            #       cd ssh-servers/docker-in-vagrant
-
-            #       # List content in the directory
-            #       ls -la
-
-            #       # Launch machine
-            #       vagrant up --provider=docker
-
-            #       # Connect to it with SSH
-            #       vagrant ssh -c "echo 'Hello Ubuntu'"
-
             - name: Test Python wheel
               run: |
                   # Test wheel installation
-                  pip install dist/otdf_python-0.0.15-py3-none-any.whl
+                  pip install dist/otdf_python-0.1.0-py3-none-any.whl
 
                   # DISABLED: Need to figure out Ubuntu nested VM
                   # Test wheel functionality
                   # python3 validate_otdf_python.py
+
+            - uses: actions/cache/restore@v4
+              with:
+                path: dist/otdf_python-0.1.0-py3-none-any.whl
+                key: ${{ runner.os }}-data-${{ github.sha }}
+
+            - uses: actions/cache/save@v4
+              with:
+                path: dist/otdf_python-0.1.0-py3-none-any.whl
+                key: ${{ runner.os }}-data-${{ github.sha }}
+                restore-keys: |
+                  ${{ runner.os }}-data-
+
+    integration-test:
+      needs: build
+      uses: ./.github/workflows/platform-integration-test.yaml
+      with:
+        wheel: dist/otdf_python-0.1.0-py3-none-any.whl
diff --git a/.github/workflows/platform-integration-test.yaml b/.github/workflows/platform-integration-test.yaml
@@ -1,31 +1,43 @@
 # Based on
 #   https://github.com/opentdf/java-sdk/blob/v0.6.1/.github/workflows/checks.yaml
+#
+# Except, that this is a "Composite Action", and specifies 'shell: bash' for
+# each 'run:' step.
 name: "Platform Integration testing"
 
 on:
-  pull_request:
-
-# on:
-#   pull_request:
-#     branches:
-#       - main
-#   push:
-#     branches:
-#       - main
+  workflow_call:
+    inputs:
+      wheel:
+        description: The Python wheel to test
+        required: true
+        type: string
 
 permissions:
   contents: read
 
 jobs:
-
-  platform-integration:
-    runs-on: ubuntu-22.04
+  integration_test:
+    runs-on: ubuntu-latest
     steps:
-      - name: Checkout Java SDK
+      - name: Checkout this repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
-      - uses: bufbuild/buf-setup-action@382440cdb8ec7bc25a68d7b4711163d95f7cc3aa
+
+      - uses: actions/cache/restore@v4
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+          path: dist/otdf_python-0.1.0-py3-none-any.whl
+          key: ${{ runner.os }}-data-${{ github.sha }}
+
+      - name: Prove that the input file is available
+        shell: bash
+        run: |
+          ls -la
+          ls -la "${{ inputs.wheel }}"
+          du -sh "${{ inputs.wheel }}"
+
+      # - uses: bufbuild/buf-setup-action@382440cdb8ec7bc25a68d7b4711163d95f7cc3aa
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Check out platform
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
@@ -44,10 +56,13 @@ jobs:
             platform/protocol/go/go.sum
             platform/sdk/go.sum
       - run: go mod download
+        shell: bash
         working-directory: platform
       - run: go mod verify
+        shell: bash
         working-directory: platform
       - name: Create keys
+        shell: bash
         run: |
           .github/scripts/init-temp-keys.sh
           cp opentdf-dev.yaml opentdf.yaml
@@ -64,12 +79,15 @@ jobs:
       #       -alias localhost-for-tests
       #   working-directory: platform/keys
       - name: Bring the services up
+        shell: bash
         run: docker compose up -d --wait --wait-timeout 240
         working-directory: platform
       - name: Provision keycloak
+        shell: bash
         run: go run ./service provision keycloak
         working-directory: platform
       - name: Provision fixtures
+        shell: bash
         run: go run ./service provision fixtures
         working-directory: platform
       - name: Start server in background
@@ -83,12 +101,37 @@ jobs:
           wait-for: 90s
           working-directory: platform
       - name: Get grpcurl
+        shell: bash
         run: go install github.com/fullstorydev/grpcurl/cmd/grpcurl@v1.8.9
       - name: Make sure that the platform is up
+        shell: bash
         run: |
           grpcurl -plaintext localhost:8080 list && \
           grpcurl -plaintext localhost:8080 kas.AccessService/PublicKey
 
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+            python-version: '3.11'
+
+      - name: Validate the Python SDK
+        env:
+          OPENTDF_CLIENT_ID: "opentdf-sdk"
+          OPENTDF_CLIENT_SECRET: "secret"
+          OPENTDF_HOSTNAME: "localhost:8080"
+          OIDC_TOKEN_ENDPOINT: "http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token"
+          OPENTDF_KAS_URL: "http://localhost:8080/kas"
+        run: |
+          mkdir validation
+          wheel="$(basename ${{ inputs.wheel }} )"
+          cp -v "${{ inputs.wheel }}" validation/
+          cp -v validate_otdf_python.py validation/
+          cd validation
+          python -m venv .venv
+          source .venv/bin/activate
+          pip install ./"$wheel"
+          python validate_otdf_python.py
+
       # - name: Validate the SDK through the command line interface
       #   run: |
       #     printf 'here is some data to encrypt' > data
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -7,7 +7,7 @@ exclude: |
 # See https://pre-commit.com/hooks.html for more hooks#
 repos:
     - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v4.6.0
+      rev: v5.0.0
       hooks:
           - id: check-yaml
           - id: end-of-file-fixer
@@ -40,7 +40,7 @@ repos:
 
     - repo: https://github.com/astral-sh/ruff-pre-commit
       # Ruff version.
-      rev: v0.6.9
+      rev: v0.7.4
       hooks:
         # Run the linter.
         - id: ruff
diff --git a/ci-build.sh b/ci-build.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -eou pipefail
+
+# Since we don't have our wheel build / install configured yet we use '--no-root'
+poetry install --no-root
+
+source $(poetry env info --path)/bin/activate
+
+# Add Go bin directory to PATH
+echo "export PATH=$PATH:~/.local/go/bin" >> $GITHUB_ENV
+
+go install golang.org/x/tools/cmd/goimports@latest
+
+go install github.com/go-python/gopy@latest
+
+poetry run pip install --upgrade setuptools wheel
+
+gopy build --output=otdf_python -vm=python3 .
+
+poetry run python3 setup.py bdist_wheel
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "otdf-python"
 # Should match 'setup.py' version number (used for gopy/pybindgen)
-version = "0.0.15"
+version = "0.1.0"
 description = "Unofficial OpenTDF SDK for Python."
 authors = ["b-long <b-long@users.noreply.github.com>"]
 readme = "README.md"
diff --git a/setup.py b/setup.py
@@ -24,7 +24,7 @@
     url="https://github.com/b-long/opentdf-python-sdk",
     package_data={"otdf_python": ["*.so"]},
     # Should match 'pyproject.toml' version number
-    version="0.0.15",
+    version="0.1.0",
     author_email="b-long@users.noreply.github.com",
     include_package_data=True,
 )
diff --git a/setup_ci.py b/setup_ci.py
@@ -81,7 +81,7 @@ def build_extension(self, ext: Extension):
 
 setuptools.setup(
     name="otdf_python",
-    version="0.0.15",
+    version="0.1.0",
     author="b-long",
     description="Unofficial OpenTDF SDK for Python.",
     long_description_content_type="text/markdown",
diff --git a/validate_otdf_python.py b/validate_otdf_python.py
@@ -5,11 +5,10 @@
 import tempfile
 from pathlib import Path
 from zipfile import is_zipfile
+from os import environ
 
 from otdf_python.gotdf_python import EncryptionConfig
 
-SOME_PLAINTEXT_FILE = Path(__file__).parent / "go.mod"
-
 
 def verify_hello():
     from otdf_python.gotdf_python import Hello
@@ -22,11 +21,14 @@ def _get_configuration() -> EncryptionConfig:
     platformEndpoint = "localhost:8080"
 
     config: EncryptionConfig = EncryptionConfig(
-        ClientId="opentdf-sdk",
-        ClientSecret="secret",
-        PlatformEndpoint=platformEndpoint,
-        TokenEndpoint="http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token",
-        KasUrl=f"http://{platformEndpoint}/kas",
+        ClientId=environ.get("OPENTDF_CLIENT_ID", "opentdf-sdk"),
+        ClientSecret=environ.get("OPENTDF_CLIENT_SECRET", "secret"),
+        PlatformEndpoint=environ.get("OPENTDF_HOSTNAME", platformEndpoint),
+        TokenEndpoint=environ.get(
+            "OIDC_TOKEN_ENDPOINT",
+            "http://localhost:8888/auth/realms/opentdf/protocol/openid-connect/token",
+        ),
+        KasUrl=environ.get("OPENTDF_KAS_URL", f"http://{platformEndpoint}/kas"),
         # FIXME: Be careful with binding the 'DataAttributes' field on this struct.
         #
         # In golang, this is initialized as []string , but passing
@@ -77,6 +79,9 @@ def verify_encrypt_file() -> None:
                     "The output path should not exist before calling 'EncryptFile()'."
                 )
 
+            SOME_PLAINTEXT_FILE = Path(tmpDir) / "new-file.txt"
+            SOME_PLAINTEXT_FILE.write_text("Hello world")
+
             outputFilePath = EncryptFile(
                 inputFilePath=str(SOME_PLAINTEXT_FILE),
                 outputFilePath=str(SOME_ENCRYPTED_FILE),
@@ -87,10 +92,10 @@ def verify_encrypt_file() -> None:
             if not SOME_ENCRYPTED_FILE.exists():
                 raise ValueError("The output file does not exist!")
 
-            if not (
-                SOME_ENCRYPTED_FILE.stat().st_size > 2500
-                and is_zipfile(SOME_ENCRYPTED_FILE)
-            ):
+            encrypted_file_size = SOME_ENCRYPTED_FILE.stat().st_size
+            print(f"The encrypted file size is {encrypted_file_size}")
+
+            if not (encrypted_file_size > 1500 and is_zipfile(SOME_ENCRYPTED_FILE)):
                 raise ValueError("The output file has unexpected content!")
 
             # breakpoint()

Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@`
`24`	`24`	`url="https://github.com/b-long/opentdf-python-sdk",`
`25`	`25`	`package_data={"otdf_python": ["*.so"]},`
`26`	`26`	`# Should match 'pyproject.toml' version number`
`27`		`- version="0.0.15",`
	`27`	`+ version="0.1.0",`
`28`	`28`	`author_email="[email protected]",`
`29`	`29`	`include_package_data=True,`
`30`	`30`	`)`