memory-benchmark.yml

name: Memory benchmark

on:
  pull_request:
    branches: [main]
    types: [opened, synchronize, reopened, ready_for_review]

concurrency:
  group: memory-benchmark-${{ github.event.pull_request.number }}
  cancel-in-progress: true

jobs:
  compare:
    name: Compare base and PR memory
    runs-on: macos-14
    timeout-minutes: 15
    permissions:
      contents: read
    env:
      PI_VERSION: 0.82.1
      BUN_VERSION: 1.3.11
      NODE_VERSION: 22.19.0
      MEMORY_BENCHMARK_RUNS: 6
      MEMORY_BENCHMARK_WARMUP_MS: 2500
      MEMORY_BENCHMARK_SAMPLES: 12
      MEMORY_BENCHMARK_INTERVAL_MS: 100

    steps:
      - name: Check out benchmark implementation
        uses: actions/checkout@v7
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          path: benchmark
          persist-credentials: false

      - name: Check out base revision
        uses: actions/checkout@v7
        with:
          ref: ${{ github.event.pull_request.base.sha }}
          path: base
          persist-credentials: false

      - name: Check out PR revision
        uses: actions/checkout@v7
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          path: head
          persist-credentials: false

      - uses: actions/setup-node@v7
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: npm
          cache-dependency-path: |
            base/package-lock.json
            head/package-lock.json

      - uses: oven-sh/setup-bun@v2.2.0
        with:
          bun-version: ${{ env.BUN_VERSION }}

      - name: Install base dependencies
        working-directory: base
        run: npm ci --ignore-scripts

      - name: Install PR dependencies
        working-directory: head
        run: npm ci --ignore-scripts

      - name: Install pinned pi host
        run: |
          npm install \
            --prefix pi-host \
            --ignore-scripts \
            --no-save \
            "@earendil-works/pi-coding-agent@$PI_VERSION"

      - name: Compare memory usage
        env:
          MEMORY_BENCHMARK_BASE_PATH: ${{ github.workspace }}/base
          MEMORY_BENCHMARK_HEAD_PATH: ${{ github.workspace }}/head
          MEMORY_BENCHMARK_PI_CLI: ${{ github.workspace }}/pi-host/node_modules/@earendil-works/pi-coding-agent/dist/cli.js
          MEMORY_BENCHMARK_BASE_SHA: ${{ github.event.pull_request.base.sha }}
          MEMORY_BENCHMARK_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
          MEMORY_BENCHMARK_PI_VERSION: ${{ env.PI_VERSION }}
          MEMORY_BENCHMARK_OUTPUT: ${{ github.workspace }}/memory-benchmark.md
          MEMORY_BENCHMARK_JSON_OUTPUT: ${{ github.workspace }}/memory-benchmark.json
        run: bun benchmark/.github/scripts/memory-benchmark.mjs

      - name: Add benchmark to job summary
        if: always() && hashFiles('memory-benchmark.md') != ''
        run: cat memory-benchmark.md >> "$GITHUB_STEP_SUMMARY"

      - name: Upload benchmark report
        if: always() && hashFiles('memory-benchmark.md') != ''
        uses: actions/upload-artifact@v7
        with:
          name: memory-benchmark-report
          path: |
            memory-benchmark.md
            memory-benchmark.json
          retention-days: 30

  comment:
    name: Update PR comment
    needs: compare
    if: >-
      always() &&
      needs.compare.result == 'success' &&
      github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-24.04
    permissions:
      actions: read
      contents: read
      pull-requests: write

    steps:
      - name: Download benchmark report
        uses: actions/download-artifact@v8
        with:
          name: memory-benchmark-report

      - name: Update sticky PR comment
        uses: actions/github-script@v9
        env:
          REPORT_PATH: memory-benchmark.md
        with:
          script: |
            const fs = require("node:fs")
            const marker = "<!-- pi-commandcode-memory-benchmark -->"
            const report = fs.readFileSync(process.env.REPORT_PATH, "utf8")
            const body = `${marker}\n${report}`
            const { owner, repo } = context.repo
            const issue_number = context.issue.number
            const comments = await github.paginate(github.rest.issues.listComments, {
              owner,
              repo,
              issue_number,
              per_page: 100,
            })
            const previous = comments.find(
              (comment) => comment.user?.type === "Bot" && comment.body?.includes(marker),
            )

            if (previous) {
              await github.rest.issues.updateComment({
                owner,
                repo,
                comment_id: previous.id,
                body,
              })
            } else {
              await github.rest.issues.createComment({ owner, repo, issue_number, body })
            }