Benchmarking Github Actions setup #6
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Benchmarks | |
| permissions: | |
| actions: read | |
| id-token: write | |
| contents: write | |
| deployments: write | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| # TODO: remove pull request | |
| pull_request: | |
| branches: | |
| - main | |
| jobs: | |
| build_ios_app: | |
| runs-on: macos-latest | |
| strategy: | |
| matrix: | |
| engine: [hermes, jsc] | |
| steps: | |
| - name: check Xcode version | |
| run: /usr/bin/xcodebuild -version | |
| - name: checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Use Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 22 | |
| - name: Install the Apple certificate and provisioning profile | |
| env: | |
| BUILD_CERTIFICATE_BASE64: ${{ secrets.BUILD_CERTIFICATE_BASE64 }} | |
| P12_PASSWORD: ${{ secrets.P12_PASSWORD }} | |
| BUILD_PROVISION_PROFILE_BASE64: ${{ secrets.BUILD_PROVISION_PROFILE_BASE64 }} | |
| KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }} | |
| run: | | |
| # create variables | |
| CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12 | |
| PP_PATH=$RUNNER_TEMP/build_pp.mobileprovision | |
| KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db | |
| # import certificate and provisioning profile from secrets | |
| echo -n "$BUILD_CERTIFICATE_BASE64" | base64 --decode -o $CERTIFICATE_PATH | |
| echo -n "$BUILD_PROVISION_PROFILE_BASE64" | base64 --decode -o $PP_PATH | |
| # create temporary keychain | |
| security create-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH | |
| security set-keychain-settings -lut 21600 $KEYCHAIN_PATH | |
| security unlock-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH | |
| # import certificate to keychain | |
| security import $CERTIFICATE_PATH -P "$P12_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH | |
| security list-keychain -d user -s $KEYCHAIN_PATH | |
| # apply provisioning profile | |
| mkdir -p ~/Library/MobileDevice/Provisioning\ Profiles | |
| cp $PP_PATH ~/Library/MobileDevice/Provisioning\ Profiles | |
| - name: build javascript bundle | |
| run: | | |
| npm install | |
| npm run build:ios:release | |
| - uses: actions/cache/restore@v4 | |
| id: cache-xcarchive | |
| with: | |
| path: ./LibrariesBenchmark.xcarchive | |
| key: ${{ runner.os }}-${{ matrix.engine }}-${{ hashFiles('**/Podfile.lock') }} | |
| - name: build archive | |
| if: steps.cache-xcarchive.outputs.cache-hit != 'true' | |
| run: | | |
| npm run install:ios:${{ matrix.engine }} | |
| xcodebuild -scheme "LibrariesBenchmark" \ | |
| -archivePath "LibrariesBenchmark.xcarchive" \ | |
| -workspace ios/LibrariesBenchmark.xcworkspace \ | |
| -configuration Debug \ | |
| -sdk iphoneos \ | |
| -destination generic/platform=iOS \ | |
| -allowProvisioningUpdates \ | |
| DEVELOPMENT_TEAM=${{ secrets.DEVELOPMENT_TEAM_ID }} \ | |
| clean archive | |
| cp dist/main.ios.jsbundle LibrariesBenchmark.xcarchive/Products/Applications/LibrariesBenchmark.app | |
| - uses: actions/cache/save@v4 | |
| with: | |
| path: ./LibrariesBenchmark.xcarchive | |
| key: ${{ steps.cache-xcarchive.outputs.cache-primary-key }} | |
| - name: export ipa | |
| run: | | |
| ls | |
| xcodebuild -exportArchive -archivePath LibrariesBenchmark.xcarchive -exportOptionsPlist LibrariesBenchmark.xcarchive/Info.plist -exportPath ${{ runner.temp }}/build | |
| mv ${{ runner.temp }}/build/LibrariesBenchmark.ipa ${{ runner.temp }}/build/${{matrix.engine}}Benchmarking.ipa | |
| - name: Upload application | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{matrix.engine}}App | |
| path: ${{ runner.temp }}/build/${{matrix.engine}}Benchmarking.ipa | |
| retention-days: 1 | |
| - name: Upload bundle | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{matrix.engine}}Bundle | |
| path: dist/main.ios.jsbundle | |
| retention-days: 1 | |
| benchmark: | |
| runs-on: ubuntu-latest | |
| needs: build_ios_app | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| engine: [hermes, jsc] | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Use Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 22 | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Create Appium Test Zip | |
| run: | | |
| cd packages/benchmark-server | |
| npm ci | |
| npm run bundle | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| role-to-assume: ${{ secrets.DEVICE_FARM_IAM_ARN }} | |
| aws-region: us-west-2 | |
| - name: Download ${{matrix.engine}} artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{matrix.engine}}App | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Run Benchmark with JavaScriptCore Engine | |
| id: run-benchmark | |
| uses: aws-actions/[email protected] | |
| with: | |
| run-settings-json: | | |
| { | |
| "name": "GitHubAction-${{ github.workflow }}_${{ github.run_id }}_${{ github.run_attempt }}", | |
| "projectArn": "TestApp", | |
| "appArn": "${{matrix.engine}}Benchmarking.ipa", | |
| "devicePoolArn": "SmallDevicePool", | |
| "executionConfiguration": { | |
| "jobTimeoutMinutes": 40 | |
| }, | |
| "test": { | |
| "type": "APPIUM_NODE", | |
| "testPackageArn": "packages/benchmark-server/MyTests.zip", | |
| "testSpecArn": "packages/benchmark-server/testSpec.yaml" | |
| } | |
| } | |
| artifact-types: ALL | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: ${{matrix.engine}}OutputFiles | |
| path: ${{ steps.run-benchmark.outputs.artifact-folder }} | |
| parse-benchmarks: | |
| runs-on: ubuntu-latest | |
| needs: benchmark | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Use Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 22 | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Build benchmark parser | |
| run: | | |
| cd packages/benchmark-parser | |
| npm ci | |
| npm run build | |
| - name: Download JSC Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: 'jscOutputFiles' | |
| path: 'jscOutput' | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download Hermes Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: 'hermesOutputFiles' | |
| path: 'hermesOutput' | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract benchmark results from artifacts | |
| run: | | |
| ./scripts/unzipArchive.sh jscOutput benchmarks/jsc | |
| ./scripts/unzipArchive.sh hermesOutput benchmarks/hermes | |
| - name: Download javascript bundle | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: jscBundle | |
| path: bundleDist | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download executable artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: jscApp | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Parse Benchmarks | |
| run: npm run parse-benchmarks | |
| - name: Report Benchmarks | |
| run: cat benchmark.json | |
| - name: Store benchmark result | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| name: 'React Native Benchmarks' | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: benchmark.json | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| auto-push: true |