Generate and test PIE packages (#31)

jduo · web-flow · commit 567f8bda92cc · 2025-09-11T16:33:19.000-07:00
* Support PIE package distribution

Support distributing the extension as a PIE package
on Packagist.

Add a CD pipeline for validating that the package can
be put on a local Composer repo, then built and installed
from the repo using the pie command. The installed
package is then run against the same integration tests
as the dev pipeline.

Rework Makefile.frag so that the extension can
be built with the standard set of commands that
the pie command runs:
- phpize
- ./configure
- make
- make install

This also requires loading the valkey-glide
sub-module, generating protobufs, running the
FFI build internally within the Makefile.

Create shared setup action for running PHP tests.

Signed-off-by: James Duong &lt;duong.james@gmail.com&gt;
diff --git a/.github/workflows/build-php-wrapper/action.yml b/.github/workflows/build-php-wrapper/action.yml
@@ -158,7 +158,7 @@ runs:
     - name: Generate protobuf and bindings
       shell: bash
       run: |
-        make build-modules-pre
+        make include/glide_bindings.h
 
     - name: Debug PHP extension linking
       shell: bash
diff --git a/.github/workflows/ort.yml b/.github/workflows/ort.yml
@@ -367,7 +367,7 @@ jobs:
       - name: Warn of outdated attributions due to the PR
         if: ${{ env.FOUND_DIFF == 'true' && github.event_name == 'pull_request' }}
         run: |
-          ATTRIBUTION_FILES=(                    
+          ATTRIBUTION_FILES=(
             "${{ env.PHP_ATTRIBUTIONS }}"
           )
 
diff --git a/.github/workflows/php.yml b/.github/workflows/php.yml
@@ -155,88 +155,11 @@ jobs:
           echo "Current directory contents:"
           ls -la
 
-      - name: Start Valkey servers
-        working-directory: tests
-        run: |
-          set -e  # Exit on any error
-
-          # Ensure /usr/local/bin is in PATH (where valkey-server is installed)
-          export PATH="/usr/local/bin:$PATH"
-
-          echo "=== DEBUG: Current directory and files ==="
-          pwd
-          ls -la
-
-          echo "=== DEBUG: Current PATH ==="
-          echo "PATH=$PATH"
-
-          echo "=== DEBUG: Checking for installed binaries in /usr/local/bin ==="
-          ls -la /usr/local/bin/ | grep -E "(valkey|redis)" || echo "No valkey/redis binaries in /usr/local/bin"
-
-          echo "=== DEBUG: Checking script permissions ==="
-          ls -la start_valkey_with_replicas.sh create-valkey-cluster.sh
-
-          # Make scripts executable
-          chmod +x start_valkey_with_replicas.sh create-valkey-cluster.sh
-
-          echo "=== DEBUG: After chmod ==="
-          ls -la start_valkey_with_replicas.sh create-valkey-cluster.sh
-
-          # Check if valkey-server is available
-          echo "=== DEBUG: Checking for valkey-server ==="
-          which valkey-server || echo "valkey-server not found in PATH"
-
-          # Check alternative locations and names
-          echo "=== DEBUG: Checking alternative binary names ==="
-          which redis-server || echo "redis-server not found in PATH"
-          which valkey-cli || echo "valkey-cli not found in PATH"
-          which redis-cli || echo "redis-cli not found in PATH"
-
-          # Start standalone servers (6379, 6380, 6381)
-          echo "=== Starting standalone Valkey servers ==="
-          ./start_valkey_with_replicas.sh
-
-          # Wait a moment for servers to start
-          echo "=== Waiting for standalone servers to start ==="
-          sleep 3
-
-          # Check if servers are running
-          echo "=== Checking standalone server status ==="
-          ps aux | grep -E "(valkey|redis)-server" | grep -v grep || echo "No valkey/redis-server processes found"
-
-          # Start cluster servers (7001-7006)
-          echo "=== Starting Valkey cluster ==="
-          ./create-valkey-cluster.sh
-
-          # Final server status check
-          echo "=== Final server status check ==="
-          ps aux | grep -E "(valkey|redis)-server" | grep -v grep || echo "No valkey/redis-server processes found"
-          echo "=== Valkey servers startup completed ==="
-
-      - name: Run PHP extension tests
-        run: |
-          # Run tests using direct extension loading with absolute path
-          php -n -d extension=$(pwd)/modules/valkey_glide.so tests/TestValkeyGlide.php
-
-      - name: Run integration tests
-        run: |
-          # All tests are now run through the main TestValkeyGlide.php runner
-          echo "All 4 test classes are already included in the main test runner"
-          echo "TestValkeyGlide.php now runs all classes by default:"
-          echo "- ValkeyGlide_Test (standalone)"
-          echo "- ValkeyGlide_Cluster_Test (cluster)"
-          echo "- ValkeyGlide_Features_Test (standalone features)"
-          echo "- ValkeyGlide_Cluster_Features_Test (cluster features)"
-
-      - name: Stop Valkey servers
-        if: always()
-        run: |
-          # Stop all Valkey server processes
-          echo "Stopping Valkey servers..."
-          pkill valkey-server || true
-          # Clean up data directories
-          rm -rf $HOME/valkey-cluster || true
-          rm -rf tests/valkey_data || true
+      - name: Run PHP Extension Tests
+        uses: ./.github/workflows/run-php-tests
+        with:
+          extension-path: ${{ github.workspace }}/modules/valkey_glide.so
+          php-version: ${{ matrix.php }}
 
       - uses: ./.github/workflows/test-benchmark
         if: ${{ matrix.engine.version == '8.0' && matrix.host.OS == 'ubuntu' && matrix.host.RUNNER == 'ubuntu-latest' && matrix.php == env.BASE_PHP_VERSION }}
@@ -588,7 +511,7 @@ jobs:
           echo "Test file exists: $(ls -la tests/TestValkeyGlide.php 2>/dev/null || echo 'NOT FOUND')"
           echo "Suppressions file exists: $(ls -la valgrind.supp 2>/dev/null || echo 'NOT FOUND')"
           echo "Valgrind version: $(valgrind --version)"
-                                             
+
           # Step 7: Run Valgrind tests
           echo ""
           echo "=== Step 7: Running Valgrind memory analysis ==="
@@ -619,51 +542,51 @@ jobs:
           if [ -f "./valgrind_logs/valgrind_output.log" ]; then
               LOG_SIZE=$(wc -c < ./valgrind_logs/valgrind_output.log)
               echo "=== Valgrind log file found (${LOG_SIZE} bytes) ==="
-              
+
               # Since we used tee, the output should already be visible above
               # But let's extract key information for summary
               echo ""
               echo "=== Extracting Key Results ==="
-              
+
               # Show test completion status
               if grep -q "All tests passed" ./valgrind_logs/valgrind_output.log; then
                   echo "✅ All tests passed"
               elif grep -q "FAILED" ./valgrind_logs/valgrind_output.log; then
                   echo "❌ Some tests failed"
               fi
-              
+
               # Check for memory leaks
               if grep -q "definitely lost:" ./valgrind_logs/valgrind_output.log; then
                   echo "⚠️  CRITICAL: Definite memory leaks detected"
                   grep -A 3 -B 1 "definitely lost:" ./valgrind_logs/valgrind_output.log || true
               else
                   echo "✅ No definite memory leaks detected"
               fi
-              
+
               # Check for possible leaks
               if grep -q "possibly lost:" ./valgrind_logs/valgrind_output.log; then
                   POSSIBLY_LOST=$(grep "possibly lost:" ./valgrind_logs/valgrind_output.log | head -1)
                   echo "⚠️  Possible leaks: $POSSIBLY_LOST"
               fi
-              
+
               # Show leak summary if present
               if grep -q "LEAK SUMMARY:" ./valgrind_logs/valgrind_output.log; then
                   echo ""
                   echo "=== Memory Leak Summary ==="
                   grep -A 6 "LEAK SUMMARY:" ./valgrind_logs/valgrind_output.log | head -7 || true
               fi
-              
+
               # Show error summary
               if grep -q "ERROR SUMMARY:" ./valgrind_logs/valgrind_output.log; then
                   echo ""
                   echo "=== Error Summary ==="
                   grep "ERROR SUMMARY:" ./valgrind_logs/valgrind_output.log || true
               fi
-              
+
               echo ""
               echo "📝 Full Valgrind output was displayed above during test execution"
               echo "📎 Complete log is also saved as artifact for detailed analysis"
-              
+
           else
               echo "❌ No Valgrind output file found"
               echo "This indicates the Valgrind command may have failed to execute properly"
diff --git a/.github/workflows/run-ort-tools/action.yml b/.github/workflows/run-ort-tools/action.yml
@@ -44,7 +44,7 @@ runs:
             echo "ERROR: Source folder does not exist!"
             exit 1
         fi
-                      
+
         # Create output directory
         echo "Creating output directory: $FOLDER/ort_results"
         mkdir -p "$FOLDER/ort_results"
diff --git a/.github/workflows/run-php-tests/action.yml b/.github/workflows/run-php-tests/action.yml
@@ -0,0 +1,129 @@
+name: Run PHP Extension Tests
+description: Common steps for running PHP extension tests with Valkey servers
+
+inputs:
+  extension-path:
+    description: Path to the built extension (.so file)
+    required: true
+  php-version:
+    description: PHP version to use for testing
+    required: true
+    default: "8.1"
+
+runs:
+  using: composite
+  steps:
+    - name: Start Valkey servers
+      working-directory: tests
+      shell: bash
+      run: |
+        set -e  # Exit on any error
+
+        # Ensure /usr/local/bin is in PATH (where valkey-server is installed)
+        export PATH="/usr/local/bin:$PATH"
+
+        echo "=== DEBUG: Current directory and files ==="
+        pwd
+        ls -la
+
+        echo "=== DEBUG: Environment ==="
+        echo "PATH=$PATH"
+
+        echo "=== DEBUG: Checking for installed binaries in /usr/local/bin ==="
+        ls -la /usr/local/bin/ | grep -E "(valkey|redis)" || echo "No valkey/redis binaries in /usr/local/bin"
+
+        echo "=== DEBUG: Checking script permissions ==="
+        ls -la start_valkey_with_replicas.sh create-valkey-cluster.sh
+
+        # Make scripts executable
+        chmod +x start_valkey_with_replicas.sh create-valkey-cluster.sh
+
+        echo "=== DEBUG: After chmod ==="
+        ls -la start_valkey_with_replicas.sh create-valkey-cluster.sh
+
+        # Check if valkey-server is available
+        echo "=== DEBUG: Checking for valkey-server ==="
+        which valkey-server || echo "valkey-server not found in PATH"
+
+        # Check alternative locations and names
+        echo "=== DEBUG: Checking alternative binary names ==="
+        which redis-server || echo "redis-server not found in PATH"
+        which valkey-cli || echo "valkey-cli not found in PATH"
+        which redis-cli || echo "redis-cli not found in PATH"
+
+        # Start standalone servers (6379, 6380, 6381)
+        echo "=== Starting standalone Valkey servers ==="
+        ./start_valkey_with_replicas.sh
+
+        # Wait a moment for servers to start
+        echo "=== Waiting for standalone servers to start ==="
+        sleep 3
+
+        # Check if servers are running
+        echo "=== Checking standalone server status ==="
+        ps aux | grep -E "(valkey|redis)-server" | grep -v grep || echo "No valkey/redis-server processes found"
+
+        # Start cluster servers (7001-7006)
+        echo "=== Starting Valkey cluster ==="
+        ./create-valkey-cluster.sh
+
+        # Final server status check
+        echo "=== Final server status check ==="
+        ps aux | grep -E "(valkey|redis)-server" | grep -v grep || echo "No valkey/redis-server processes found"
+        echo "=== Valkey servers startup completed ==="
+
+    - name: Run PHP extension tests
+      shell: bash
+      env:
+        EXTENSION_PATH: ${{ inputs.extension-path }}
+      run: |
+        # Debug: Check if extension is loaded and classes exist
+        echo "=== Extension Debug Info ==="
+        echo "Extension path: $EXTENSION_PATH"
+
+        php -n -d "extension=$EXTENSION_PATH" -r "
+          echo 'Extension loaded: ' . (extension_loaded('valkey_glide') ? 'YES' : 'NO') . PHP_EOL;
+          echo 'Extension version: ' . phpversion('valkey_glide') . PHP_EOL;
+          echo 'Extension functions: ' . PHP_EOL;
+          foreach (get_extension_funcs('valkey_glide') as \$func) {
+            echo '  - ' . \$func . PHP_EOL;
+          }
+          echo 'All declared classes: ' . PHP_EOL;
+          foreach (get_declared_classes() as \$class) {
+            if (strpos(\$class, 'Valkey') !== false || strpos(\$class, 'Client') !== false || strpos(\$class, 'Mock') !== false) {
+              echo '  - ' . \$class . PHP_EOL;
+            }
+          }
+          echo 'ClientConstructorMock exists: ' . (class_exists('ClientConstructorMock') ? 'YES' : 'NO') . PHP_EOL;
+          if (class_exists('ClientConstructorMock')) {
+            echo 'ClientConstructorMock methods: ' . PHP_EOL;
+            foreach (get_class_methods('ClientConstructorMock') as \$method) {
+              echo '  - ' . \$method . PHP_EOL;
+            }
+          }
+        "
+
+        # Run tests using extension loading
+        php -n -d "extension=$EXTENSION_PATH" tests/TestValkeyGlide.php
+
+    - name: Run integration tests
+      shell: bash
+      run: |
+        # All tests are now run through the main TestValkeyGlide.php runner
+        echo "All 4 test classes are already included in the main test runner"
+        echo "TestValkeyGlide.php now runs all classes by default:"
+        echo "- ValkeyGlide_Test (standalone)"
+        echo "- ValkeyGlide_Cluster_Test (cluster)"
+        echo "- ValkeyGlide_Features_Test (standalone features)"
+        echo "- ValkeyGlide_Cluster_Features_Test (cluster features)"
+
+    - name: Stop Valkey servers
+      if: always()
+      shell: bash
+      run: |
+        # Stop all Valkey server processes
+        echo "Stopping Valkey servers..."
+        pkill valkey-server || true
+        # Clean up data directories
+        rm -rf $HOME/valkey-cluster || true
+        rm -rf tests/valkey_data || true
diff --git a/.github/workflows/setup-php-extension/action.yml b/.github/workflows/setup-php-extension/action.yml
@@ -0,0 +1,73 @@
+name: Setup PHP Extension Build Environment
+description: Common setup steps for building PHP extension
+
+inputs:
+  php-version:
+    description: PHP version to use
+    required: true
+    default: "8.1"
+  install-composer-deps:
+    description: Whether to install composer dependencies
+    required: false
+    default: "true"
+
+runs:
+  using: composite
+  steps:
+    - name: Install shared dependencies
+      uses: ./.github/workflows/install-shared-dependencies
+      with:
+        target: x86_64-unknown-linux-gnu
+        github-token: ${{ github.token }}
+        engine-version: "7.2.5"
+
+    - name: Setup PHP
+      uses: shivammathur/setup-php@v2
+      with:
+        php-version: ${{ inputs.php-version }}
+        extensions: none
+        tools: pie, jq
+
+    - name: Install system dependencies
+      shell: bash
+      env:
+        PHP_VERSION: ${{ inputs.php-version }}
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+            "php${PHP_VERSION}-dev" \
+            "php${PHP_VERSION}-curl" \
+            "php${PHP_VERSION}-xml" \
+            build-essential \
+            autoconf \
+            automake \
+            libtool \
+            pkg-config \
+            libssl-dev \
+            clang-format \
+            protobuf-c-compiler \
+            libprotobuf-c-dev \
+            libprotobuf-c1
+
+    - name: Install Rust and protoc
+      uses: ./.github/workflows/install-rust-and-protoc
+      with:
+        github-token: ${{ github.token }}
+
+    - name: Install cbindgen
+      shell: bash
+      run: cargo install cbindgen
+
+    - name: Install Composer dependencies
+      if: inputs.install-composer-deps == 'true'
+      shell: bash
+      run: composer install --ignore-platform-reqs
+
+    - name: Generate test protobuf PHP classes
+      if: inputs.install-composer-deps == 'true'
+      shell: bash
+      run: |
+        protoc --proto_path=./valkey-glide/glide-core/src/protobuf --php_out=./tests/ ./valkey-glide/glide-core/src/protobuf/connection_request.proto
+        echo "tests directory contents:"
+        cd tests
+        ls -la
diff --git a/.github/workflows/test-pie.yml b/.github/workflows/test-pie.yml
diff --git a/Makefile.frag b/Makefile.frag
diff --git a/composer.json b/composer.json

Original file line number	Diff line number	Diff line change
`@@ -367,7 +367,7 @@ jobs:`
`367`	`367`	`- name: Warn of outdated attributions due to the PR`
`368`	`368`	`if: ${{ env.FOUND_DIFF == 'true' && github.event_name == 'pull_request' }}`
`369`	`369`	`run: \|`
`370`		`- ATTRIBUTION_FILES=(`
	`370`	`+ ATTRIBUTION_FILES=(`
`371`	`371`	`"${{ env.PHP_ATTRIBUTIONS }}"`
`372`	`372`	`)`
`373`	`373`