reports.yml

name: Reports

on:
  push:
    branches:
      - master
  pull_request:

jobs:
  build-nargo:
    runs-on: ubuntu-22.04

    steps:
      - name: Checkout Noir repo
        uses: actions/checkout@v4

      - name: Setup toolchain
        uses: dtolnay/rust-toolchain@1.75.0

      - uses: Swatinem/rust-cache@v2
        with:
          key: x86_64-unknown-linux-gnu
          cache-on-failure: true
          save-if: ${{ github.event_name != 'merge_group' }}

      - name: Build Nargo
        run: cargo build --package nargo_cli --release

      - name: Package artifacts
        run: |
          mkdir dist
          cp ./target/release/nargo ./dist/nargo

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: nargo
          path: ./dist/*
          retention-days: 3

  compare_gates_reports:
    name: Circuit sizes
    needs: [build-nargo]
    runs-on: ubuntu-22.04
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Install `bb`
        run: |
          ./scripts/install_bb.sh
          echo "$HOME/.bb/" >> $GITHUB_PATH

      - name: Download nargo binary
        uses: actions/download-artifact@v4
        with:
          name: nargo
          path: ./nargo

      - name: Set nargo on PATH
        run: |
          nargo_binary="${{ github.workspace }}/nargo/nargo"
          chmod +x $nargo_binary
          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
          export PATH="$PATH:$(dirname $nargo_binary)"
          nargo -V

      - name: Generate gates report
        working-directory: ./test_programs
        run: |
          ./rebuild.sh
          ./gates_report.sh
          mv gates_report.json ../gates_report.json
      
      - name: Compare gates reports
        id: gates_diff
        uses: noir-lang/noir-gates-diff@7e4ddaa91c69380f15ccba514eac17bc7432a8cc
        with:
          report: gates_report.json
          summaryQuantile: 0.9 # only display the 10% most significant circuit size diffs in the summary (defaults to 20%)

      - name: Add gates diff to sticky comment
        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          # delete the comment in case changes no longer impact circuit sizes
          delete: ${{ !steps.gates_diff.outputs.markdown }}
          message: ${{ steps.gates_diff.outputs.markdown }}

  compare_brillig_bytecode_size_reports:
    name: Brillig bytecode sizes
    needs: [build-nargo]
    runs-on: ubuntu-22.04
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Download nargo binary
        uses: actions/download-artifact@v4
        with:
          name: nargo
          path: ./nargo

      - name: Set nargo on PATH
        run: |
          nargo_binary="${{ github.workspace }}/nargo/nargo"
          chmod +x $nargo_binary
          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
          export PATH="$PATH:$(dirname $nargo_binary)"
          nargo -V

      - name: Generate Brillig bytecode size report
        working-directory: ./test_programs
        run: |
          ./gates_report_brillig.sh
          mv gates_report_brillig.json ../gates_report_brillig.json

      - name: Compare Brillig bytecode size reports
        id: brillig_bytecode_diff
        uses: noir-lang/noir-gates-diff@7e4ddaa91c69380f15ccba514eac17bc7432a8cc
        with:
          report: gates_report_brillig.json
          header: |
            # Changes to Brillig bytecode sizes
          brillig_report: true
          summaryQuantile: 0.9 # only display the 10% most significant bytecode size diffs in the summary (defaults to 20%)

      - name: Add bytecode size diff to sticky comment
        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: brillig
          # delete the comment in case changes no longer impact brillig bytecode sizes
          delete: ${{ !steps.brillig_bytecode_diff.outputs.markdown }}
          message: ${{ steps.brillig_bytecode_diff.outputs.markdown }}

  compare_brillig_execution_reports:
    name: Brillig execution trace sizes  
    needs: [build-nargo]
    runs-on: ubuntu-22.04
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Download nargo binary
        uses: actions/download-artifact@v4
        with:
          name: nargo
          path: ./nargo

      - name: Set nargo on PATH
        run: |
          nargo_binary="${{ github.workspace }}/nargo/nargo"
          chmod +x $nargo_binary
          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
          export PATH="$PATH:$(dirname $nargo_binary)"
          nargo -V

      - name: Generate Brillig execution report
        working-directory: ./test_programs
        run: |
          ./gates_report_brillig_execution.sh
          mv gates_report_brillig_execution.json ../gates_report_brillig_execution.json

      - name: Compare Brillig execution reports
        id: brillig_execution_diff
        uses: noir-lang/noir-gates-diff@c1503343c3e264925ef67c68a2a5eeadd245a77b
        with:
          report: gates_report_brillig_execution.json
          header: |
            # Changes to number of Brillig opcodes executed
          brillig_report: true
          summaryQuantile: 0.9 # only display the 10% most significant bytecode size diffs in the summary (defaults to 20%)

      - name: Add bytecode size diff to sticky comment
        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: brillig_execution
          # delete the comment in case changes no longer impact brillig bytecode sizes
          delete: ${{ !steps.brillig_execution_diff.outputs.markdown }}
          message: ${{ steps.brillig_execution_diff.outputs.markdown }}

  generate_memory_report:
    name: Peak memory usage  
    needs: [build-nargo]
    runs-on: ubuntu-22.04
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Download nargo binary
        uses: actions/download-artifact@v4
        with:
          name: nargo
          path: ./nargo

      - name: Set nargo on PATH
        run: |
          nargo_binary="${{ github.workspace }}/nargo/nargo"
          chmod +x $nargo_binary
          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
          export PATH="$PATH:$(dirname $nargo_binary)"
          nargo -V

      - name: Generate Memory report
        working-directory: ./test_programs
        run: |
          ./memory_report.sh
          mv memory_report.json ../memory_report.json

      - name: Upload compilation memory report
        uses: actions/upload-artifact@v4
        with:
          name: in_progress_compilation_mem_report
          path: memory_report.json
          retention-days: 3
          overwrite: true

      - name: Generate execution memory report
        working-directory: ./test_programs
        run: |
          ./memory_report.sh 0 1
          mv memory_report.json ../memory_report.json

      - name: Upload execution memory report
        uses: actions/upload-artifact@v4
        with:
          name: in_progress_execution_mem_report
          path: memory_report.json
          retention-days: 3
          overwrite: true

  generate_compilation_and_execution_report:
    name: Compilation and execution time
    needs: [build-nargo]
    runs-on: ubuntu-22.04
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Download nargo binary
        uses: actions/download-artifact@v4
        with:
          name: nargo
          path: ./nargo

      - name: Set nargo on PATH
        run: |
          nargo_binary="${{ github.workspace }}/nargo/nargo"
          chmod +x $nargo_binary
          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
          export PATH="$PATH:$(dirname $nargo_binary)"
          nargo -V

      - name: Generate Compilation report
        working-directory: ./test_programs
        run: |
          ./compilation_report.sh 0 1
          mv compilation_report.json ../compilation_report.json

      - name: Generate Execution report
        working-directory: ./test_programs
        run: |
          ./execution_report.sh 0 1
          mv execution_report.json ../execution_report.json
          
      - name: Upload compilation report
        uses: actions/upload-artifact@v4
        with:
          name: in_progress_compilation_report
          path: compilation_report.json
          retention-days: 3
          overwrite: true
      
      - name: Upload execution report
        uses: actions/upload-artifact@v4
        with:
          name: in_progress_execution_report
          path: execution_report.json
          retention-days: 3
          overwrite: true

  external_repo_compilation_and_execution_report:
    needs: [build-nargo]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    strategy:
      fail-fast: false
      matrix:
        include:
          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts, is_library: true }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner, take_average: true }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail, take_average: true  }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset, take_average: true }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private, take_average: true }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public, take_average: true }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-merge, take_average: true }
          #- project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-root-empty }
          #- project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-single-tx }
          #- project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-root }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-merge, take_average: true }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-root, take_average: true }

    name: External repo compilation and execution reports - ${{ matrix.project.repo }}/${{ matrix.project.path }}
    steps:
      - name: Download nargo binary
        uses: actions/download-artifact@v4
        with:
          name: nargo
          path: ./nargo

      - name: Set nargo on PATH
        run: |
          nargo_binary="${{ github.workspace }}/nargo/nargo"
          chmod +x $nargo_binary
          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
          export PATH="$PATH:$(dirname $nargo_binary)"
          nargo -V

      - uses: actions/checkout@v4
        with:
          path: scripts
          sparse-checkout: |
            test_programs/compilation_report.sh
            test_programs/execution_report.sh
            test_programs/parse_time.sh
          sparse-checkout-cone-mode: false

      - name: Checkout
        uses: actions/checkout@v4
        with:
          repository: ${{ matrix.project.repo }}
          path: test-repo
          ref: ${{ matrix.project.ref }}

      - name: Generate compilation report without averages
        working-directory: ./test-repo/${{ matrix.project.path }}
        if: ${{ !matrix.project.take_average }}
        run: |
          mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh
          touch parse_time.sh
          chmod +x parse_time.sh
          cp /home/runner/work/noir/noir/scripts/test_programs/parse_time.sh ./parse_time.sh
          ./compilation_report.sh 1

      - name: Generate compilation report with averages
        working-directory: ./test-repo/${{ matrix.project.path }}
        if: ${{ matrix.project.take_average }}
        run: |
          mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh
          touch parse_time.sh
          chmod +x parse_time.sh
          cp /home/runner/work/noir/noir/scripts/test_programs/parse_time.sh ./parse_time.sh
          ./compilation_report.sh 1 1
      
      - name: Generate execution report without averages
        working-directory: ./test-repo/${{ matrix.project.path }}
        if: ${{ !matrix.project.is_library && !matrix.project.take_average }}
        run: |
          mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh
          mv /home/runner/work/noir/noir/scripts/test_programs/parse_time.sh ./parse_time.sh
          ./execution_report.sh 1

      - name: Generate execution report with averages
        working-directory: ./test-repo/${{ matrix.project.path }}
        if: ${{ !matrix.project.is_library && matrix.project.take_average }}
        run: |
          mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh
          mv /home/runner/work/noir/noir/scripts/test_programs/parse_time.sh ./parse_time.sh
          ./execution_report.sh 1 1

      - name: Move compilation report
        id: compilation_report
        shell: bash
        run: |
          PACKAGE_NAME=${{ matrix.project.path }}
          PACKAGE_NAME=$(basename $PACKAGE_NAME)
          mv ./test-repo/${{ matrix.project.path }}/compilation_report.json ./compilation_report_$PACKAGE_NAME.json
          echo "compilation_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT

      - name: Move execution report
        id: execution_report
        shell: bash
        if: ${{ !matrix.project.is_library }}
        run: |
          PACKAGE_NAME=${{ matrix.project.path }}
          PACKAGE_NAME=$(basename $PACKAGE_NAME)
          mv ./test-repo/${{ matrix.project.path }}/execution_report.json ./execution_report_$PACKAGE_NAME.json
          echo "execution_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
          
      - name: Upload compilation report
        uses: actions/upload-artifact@v4
        with:
          name: compilation_report_${{ steps.compilation_report.outputs.compilation_report_name }}
          path: compilation_report_${{ steps.compilation_report.outputs.compilation_report_name }}.json
          retention-days: 3
          overwrite: true

      - name: Upload execution report
        uses: actions/upload-artifact@v4
        with:
          name: execution_report_${{ steps.execution_report.outputs.execution_report_name }}
          path: execution_report_${{ steps.execution_report.outputs.execution_report_name }}.json
          retention-days: 3
          overwrite: true

  upload_compilation_report:
    name: Upload compilation report 
    needs: [generate_compilation_and_execution_report, external_repo_compilation_and_execution_report]
    # We want this job to run even if one variation of the matrix in `external_repo_compilation_and_execution_report` fails
    if: always() 
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Download initial compilation report
        uses: actions/download-artifact@v4
        with:
          name: in_progress_compilation_report

      - name: Download matrix compilation reports
        uses: actions/download-artifact@v4
        with:
          pattern: compilation_report_*
          path: ./reports

      - name: Merge compilation reports using jq
        run: |
          mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh
          ./merge-bench-reports.sh compilation_report

      - name: Parse compilation report
        id: compilation_report
        uses: noir-lang/noir-bench-report@6ba151d7795042c4ff51864fbeb13c0a6a79246c
        with:
          report: compilation_report.json
          header: |
            Compilation Report
          memory_report: false

      - name: Add memory report to sticky comment
        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: compilation
          message: ${{ steps.compilation_report.outputs.markdown }}

  external_repo_memory_report:
    needs: [build-nargo]
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        include:
          # TODO: Bring this report back under a flag. The `noir-contracts` report takes just under 30 minutes.
          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public }

    name: External repo memory report - ${{ matrix.project.repo }}/${{ matrix.project.path }}
    steps:
      - name: Download nargo binary
        uses: actions/download-artifact@v4
        with:
          name: nargo
          path: ./nargo

      - name: Set nargo on PATH
        run: |
          nargo_binary="${{ github.workspace }}/nargo/nargo"
          chmod +x $nargo_binary
          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
          export PATH="$PATH:$(dirname $nargo_binary)"
          nargo -V

      - uses: actions/checkout@v4
        with:
          path: scripts
          sparse-checkout: |
            test_programs/memory_report.sh
          sparse-checkout-cone-mode: false

      - name: Checkout
        uses: actions/checkout@v4
        with:
          repository: ${{ matrix.project.repo }}
          path: test-repo
          ref: ${{ matrix.project.ref }}

      - name: Generate compilation memory report
        working-directory: ./test-repo/${{ matrix.project.path }}
        run: |
          mv /home/runner/work/noir/noir/scripts/test_programs/memory_report.sh ./memory_report.sh
          ./memory_report.sh 1
          # Rename the memory report as the execution report is about to write to the same file
          cp memory_report.json compilation_memory_report.json

      - name: Generate execution memory report
        working-directory: ./test-repo/${{ matrix.project.path }}
        run: |
          ./memory_report.sh 1 1

      - name: Move compilation report
        id: compilation_mem_report
        shell: bash
        run: |
          PACKAGE_NAME=${{ matrix.project.path }}
          PACKAGE_NAME=$(basename $PACKAGE_NAME)
          mv ./test-repo/${{ matrix.project.path }}/compilation_memory_report.json ./memory_report_$PACKAGE_NAME.json
          echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT

      - name: Upload compilation memory report
        uses: actions/upload-artifact@v4
        with:
          name: compilation_mem_report_${{ steps.compilation_mem_report.outputs.memory_report_name }}
          path: memory_report_${{ steps.compilation_mem_report.outputs.memory_report_name }}.json
          retention-days: 3
          overwrite: true

      - name: Move execution report
        id: execution_mem_report
        shell: bash
        run: |
          PACKAGE_NAME=${{ matrix.project.path }}
          PACKAGE_NAME=$(basename $PACKAGE_NAME)
          mv ./test-repo/${{ matrix.project.path }}/memory_report.json ./memory_report_$PACKAGE_NAME.json
          echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT

      - name: Upload execution memory report
        uses: actions/upload-artifact@v4
        with:
          name: execution_mem_report_${{ steps.execution_mem_report.outputs.memory_report_name }}
          path: memory_report_${{ steps.execution_mem_report.outputs.memory_report_name }}.json
          retention-days: 3
          overwrite: true

  upload_compilation_memory_report:
    name: Upload compilation memory report 
    needs: [generate_memory_report, external_repo_memory_report]
    # We want this job to run even if one variation of the matrix in `external_repo_memory_report` fails
    if: always() 
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Download initial memory report
        uses: actions/download-artifact@v4
        with:
          name: in_progress_compilation_mem_report

      - name: Download matrix memory reports
        uses: actions/download-artifact@v4
        with:
          pattern: compilation_mem_report_*
          path: ./reports

      - name: Merge memory reports using jq
        run: |
          mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh
          ./merge-bench-reports.sh memory_report

      - name: Parse compilation memory report
        id: compilation_mem_report
        uses: noir-lang/noir-bench-report@6ba151d7795042c4ff51864fbeb13c0a6a79246c
        with:
          report: memory_report.json
          header: |
            Compilation Memory Report
          memory_report: true

      - name: Add memory report to sticky comment
        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: compilation_memory
          message: ${{ steps.compilation_mem_report.outputs.markdown }}

  upload_execution_memory_report:
    name: Upload execution memory report 
    needs: [generate_memory_report, external_repo_memory_report]
    # We want this job to run even if one variation of the matrix in `external_repo_memory_report` fails
    if: always() 
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Download initial memory report
        uses: actions/download-artifact@v4
        with:
          name: in_progress_execution_mem_report

      - name: Download matrix memory reports
        uses: actions/download-artifact@v4
        with:
          pattern: execution_mem_report_*
          path: ./reports

      - name: Merge memory reports using jq
        run: |
          mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh
          ./merge-bench-reports.sh memory_report
          # Rename the memory report as to not clash with the compilation memory report file name
          cp memory_report.json execution_memory_report.json

      - name: Parse execution memory report
        id: execution_mem_report
        uses: noir-lang/noir-bench-report@6ba151d7795042c4ff51864fbeb13c0a6a79246c
        with:
          report: execution_memory_report.json
          header: |
            Execution Memory Report
          memory_report: true

      - name: Add execution memory report to sticky comment
        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: execution_memory
          message: ${{ steps.execution_mem_report.outputs.markdown }}

  upload_execution_report:
    name: Upload execution report 
    needs: [generate_compilation_and_execution_report, external_repo_compilation_and_execution_report]
    # We want this job to run even if one variation of the matrix in `external_repo_compilation_and_execution_report` fails
    if: always() 
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Download initial execution report
        uses: actions/download-artifact@v4
        with:
          name: in_progress_execution_report

      - name: Download matrix execution reports
        uses: actions/download-artifact@v4
        with:
          pattern: execution_report_*
          path: ./reports

      - name: Merge execution reports using jq
        run: |
          mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh
          ./merge-bench-reports.sh execution_report

      - name: Parse execution report
        id: execution_report
        uses: noir-lang/noir-bench-report@6ba151d7795042c4ff51864fbeb13c0a6a79246c
        with:
          report: execution_report.json
          header: |
            Execution Report
          execution_report: true

      - name: Add memory report to sticky comment
        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: execution_time
          message: ${{ steps.execution_report.outputs.markdown }}