Benchmark with app harness (reflex-dev#2774)

.github/workflows/benchmarks.yml

@@ -21,11 +21,10 @@ env:
   PYTHONIOENCODING: 'utf8'
   TELEMETRY_ENABLED: false
   NODE_OPTIONS: '--max_old_space_size=4096'
+  DATABASE_URL: ${{ secrets.DATABASE_URL }}
 
 jobs:
   reflex-web:
-    env:
-      DATABASE_URL: ${{ secrets.DATABASE_URL }}
     strategy:
       fail-fast: false
       matrix:
@@ -70,8 +69,58 @@ jobs:
       - name: Run Benchmarks
         # Only run if the database creds are available in this context.
         if: ${{ env.DATABASE_URL }}
-        working-directory: ./integration/benchmarks
-        run: poetry run python benchmarks.py "$GITHUB_SHA" .lighthouseci
+        run: poetry run python scripts/lighthouse_score_upload.py "$GITHUB_SHA" ./integration/benchmarks/.lighthouseci
         env:
           GITHUB_SHA: ${{ github.sha }}
           PR_TITLE: ${{ github.event.pull_request.title }}
+
+  simple-apps-benchmarks:
+    env:
+      OUTPUT_FILE: benchmarks.json
+    timeout-minutes: 50
+    strategy:
+      # Prioritize getting more information out of the workflow (even if something fails)
+      fail-fast: false
+      matrix:
+        # Show OS combos first in GUI
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ['3.8.18', '3.9.18', '3.10.13', '3.11.5', '3.12.0']
+        exclude:
+          - os: windows-latest
+            python-version: '3.10.13'
+          - os: windows-latest
+            python-version: '3.9.18'
+          - os: windows-latest
+            python-version: '3.8.18'
+        include:
+          - os: windows-latest
+            python-version: '3.10.11'
+          - os: windows-latest
+            python-version: '3.9.13'
+          - os: windows-latest
+            python-version: '3.8.10'
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup_build_env
+        with:
+          python-version: ${{ matrix.python-version }}
+          run-poetry-install: true
+          create-venv-at-path: .venv
+      - name: Install additional dependencies for DB access
+        run: poetry run pip install psycopg2-binary
+      - name: Run benchmark tests
+        env:
+          APP_HARNESS_HEADLESS: 1
+          PYTHONUNBUFFERED: 1
+        run: |
+          poetry run pytest -v benchmarks/ --benchmark-json=${{ env.OUTPUT_FILE }} -s
+      - name: Upload benchmark results
+        # Only run if the database creds are available in this context.
+        if: ${{ env.DATABASE_URL }}
+        run: poetry run python scripts/simple_app_benchmark_upload.py --os "${{ matrix.os }}"
+          --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}" 
+          --benchmark-json "${{ env.OUTPUT_FILE }}" --pr-title "${{ github.event.pull_request.title }}"
+          --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}"
+          --event-type "${{ github.event_name }}" --actor "${{ github.actor }}"

.pre-commit-config.yaml

@@ -10,6 +10,7 @@ repos:
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
+        exclude: '^integration/benchmarks/'
 
   - repo: https://github.com/RobertCraigie/pyright-python
     rev: v1.1.313

benchmarks/__init__.py

@@ -0,0 +1,3 @@
+"""Reflex benchmarks."""
+
+WINDOWS_SKIP_REASON = "Takes too much time as a result of npm"

benchmarks/conftest.py

@@ -0,0 +1,20 @@
+"""Shared conftest for all benchmark tests."""
+
+import pytest
+
+from reflex.testing import AppHarness, AppHarnessProd
+
+
+@pytest.fixture(
+    scope="session", params=[AppHarness, AppHarnessProd], ids=["dev", "prod"]
+)
+def app_harness_env(request):
+    """Parametrize the AppHarness class to use for the test, either dev or prod.
+
+    Args:
+        request: The pytest fixture request object.
+
+    Returns:
+        The AppHarness class to use for the test.
+    """
+    return request.param