Benchmarking Github Actions setup

.github/workflows/benchmark.yaml

@@ -0,0 +1,245 @@
+name: Run Benchmarks
+
+permissions:
+  actions: read
+  id-token: write
+  contents: write
+  deployments: write
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  build_ios_app:
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        engine: [hermes, jsc]
+    steps:
+      - name: check Xcode version
+        run: /usr/bin/xcodebuild -version
+
+      - name: checkout repository
+        uses: actions/checkout@v4
+
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Install the Apple certificate and provisioning profile
+        env:
+          BUILD_CERTIFICATE_BASE64: ${{ secrets.BUILD_CERTIFICATE_BASE64 }}
+          P12_PASSWORD: ${{ secrets.P12_PASSWORD }}
+          BUILD_PROVISION_PROFILE_BASE64: ${{ secrets.BUILD_PROVISION_PROFILE_BASE64 }}
+          KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
+        run: |
+          # create variables
+          CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12
+          PP_PATH=$RUNNER_TEMP/build_pp.mobileprovision
+          KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db
+
+          # import certificate and provisioning profile from secrets
+          echo -n "$BUILD_CERTIFICATE_BASE64" | base64 --decode -o $CERTIFICATE_PATH
+          echo -n "$BUILD_PROVISION_PROFILE_BASE64" | base64 --decode -o $PP_PATH
+
+          # create temporary keychain
+          security create-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
+          security set-keychain-settings -lut 21600 $KEYCHAIN_PATH
+          security unlock-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
+
+          # import certificate to keychain
+          security import $CERTIFICATE_PATH -P "$P12_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH
+          security list-keychain -d user -s $KEYCHAIN_PATH
+
+          # apply provisioning profile
+          mkdir -p ~/Library/MobileDevice/Provisioning\ Profiles
+          cp $PP_PATH ~/Library/MobileDevice/Provisioning\ Profiles
+
+      - name: build javascript bundle
+        run: |
+          npm install
+          npm run build:ios:release
+
+      - uses: actions/cache/restore@v4
+        id: cache-xcarchive
+        with:
+          path: ./LibrariesBenchmark.xcarchive
+          key: ${{ runner.os }}-${{ matrix.engine }}-${{ hashFiles('**/Podfile.lock') }}
+
+      - name: build archive
+        if: steps.cache-xcarchive.outputs.cache-hit != 'true'
+        run: |
+          npm run install:ios:${{ matrix.engine }}
+
+          xcodebuild -scheme "LibrariesBenchmark" \
+          -archivePath "LibrariesBenchmark.xcarchive" \
+          -workspace ios/LibrariesBenchmark.xcworkspace \
+          -configuration Debug \
+          -sdk iphoneos \
+          -destination generic/platform=iOS \
+          -allowProvisioningUpdates \
+          DEVELOPMENT_TEAM=${{ secrets.DEVELOPMENT_TEAM_ID }} \
+          clean archive
+
+          cp dist/main.ios.jsbundle LibrariesBenchmark.xcarchive/Products/Applications/LibrariesBenchmark.app
+
+      - uses: actions/cache/save@v4
+        with:
+          path: ./LibrariesBenchmark.xcarchive
+          key: ${{ steps.cache-xcarchive.outputs.cache-primary-key }}
+
+      - name: export ipa
+        run: |
+          ls
+          xcodebuild -exportArchive -archivePath LibrariesBenchmark.xcarchive -exportOptionsPlist LibrariesBenchmark.xcarchive/Info.plist -exportPath ${{ runner.temp }}/build
+          mv ${{ runner.temp }}/build/LibrariesBenchmark.ipa ${{ runner.temp }}/build/${{matrix.engine}}Benchmarking.ipa
+
+      - name: Upload application
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{matrix.engine}}App
+          path: ${{ runner.temp }}/build/${{matrix.engine}}Benchmarking.ipa
+          retention-days: 1
+
+      - name: Upload bundle
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{matrix.engine}}Bundle
+          path: dist/main.ios.jsbundle
+          retention-days: 1
+
+  benchmark:
+    runs-on: ubuntu-latest
+    needs: build_ios_app
+    strategy:
+      max-parallel: 1
+      matrix:
+        engine: [hermes, jsc]
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Create Appium Test Zip
+        run: |
+          cd packages/benchmark-server
+          npm ci
+          npm run bundle
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ secrets.DEVICE_FARM_IAM_ARN }}
+          aws-region: us-west-2
+
+      - name: Download ${{matrix.engine}} artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{matrix.engine}}App
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run Benchmark with JavaScriptCore Engine
+        id: run-benchmark
+        uses: aws-actions/[email protected]
+        with:
+          run-settings-json: |
+            {
+              "name": "GitHubAction-${{ github.workflow }}_${{ github.run_id }}_${{ github.run_attempt }}",
+              "projectArn": "TestApp",
+              "appArn": "${{matrix.engine}}Benchmarking.ipa",
+              "devicePoolArn": "SmallDevicePool",
+              "executionConfiguration": {
+                "jobTimeoutMinutes": 40
+              },
+              "test": {
+                "type": "APPIUM_NODE",
+                "testPackageArn": "packages/benchmark-server/MyTests.zip",
+                "testSpecArn": "packages/benchmark-server/testSpec.yaml"
+              }
+            }
+          artifact-types: ALL
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: ${{matrix.engine}}OutputFiles
+          path: ${{ steps.run-benchmark.outputs.artifact-folder }}
+
+  parse-benchmarks:
+    runs-on: ubuntu-latest
+    needs: benchmark
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build benchmark parser
+        run: |
+          cd packages/benchmark-parser
+          npm ci
+          npm run build
+
+      - name: Download JSC Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: 'jscOutputFiles'
+          path: 'jscOutput'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download Hermes Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: 'hermesOutputFiles'
+          path: 'hermesOutput'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract benchmark results from artifacts
+        run: |
+          ./scripts/unzipArchive.sh jscOutput benchmarks/jsc
+          ./scripts/unzipArchive.sh hermesOutput benchmarks/hermes
+
+      - name: Download javascript bundle
+        uses: actions/download-artifact@v4
+        with:
+          name: jscBundle
+          path: bundleDist
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download executable artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: jscApp
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Parse Benchmarks
+        run: npm run parse-benchmarks
+
+      - name: Report Benchmarks
+        run: cat benchmark.json
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: 'React Native Benchmarks'
+          tool: 'customSmallerIsBetter'
+          output-file-path: benchmark.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true

.gitignore

@@ -39,6 +39,8 @@ local.properties
 node_modules/
 npm-debug.log
 yarn-error.log
+dist/
+MyTests.zip
 
 # fastlane
 #