chore(ci): Memory reports for execution (#6907)

Co-authored-by: Tom French <[email protected]>
noir-lang · Jan 2, 2025 · 8439eac · 8439eac
1 parent e7600a8
commit 8439eac
Show file tree

Hide file tree

Showing 2 changed files with 113 additions and 25 deletions.
diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml
@@ -217,10 +217,24 @@ jobs:
           ./memory_report.sh
           mv memory_report.json ../memory_report.json
 
-      - name: Upload memory report
+      - name: Upload compilation memory report
         uses: actions/upload-artifact@v4
         with:
-          name: in_progress_memory_report
+          name: in_progress_compilation_mem_report
+          path: memory_report.json
+          retention-days: 3
+          overwrite: true
+
+      - name: Generate execution memory report
+        working-directory: ./test_programs
+        run: |
+          ./memory_report.sh 0 1
+          mv memory_report.json ../memory_report.json
+
+      - name: Upload execution memory report
+        uses: actions/upload-artifact@v4
+        with:
+          name: in_progress_execution_mem_report
           path: memory_report.json
           retention-days: 3
           overwrite: true
@@ -428,11 +442,11 @@ jobs:
 
       - name: Parse compilation report
         id: compilation_report
-        uses: noir-lang/noir-bench-report@e408e131e96c3615b4f820d7d642360fb4d6e2f4
+        uses: noir-lang/noir-bench-report@6ba151d7795042c4ff51864fbeb13c0a6a79246c
         with:
           report: compilation_report.json
           header: |
-            # Compilation Report
+            Compilation Report
           memory_report: false
 
       - name: Add memory report to sticky comment
@@ -489,32 +503,55 @@ jobs:
           path: test-repo
           ref: ${{ matrix.project.ref }}
 
-      - name: Generate compilation report
+      - name: Generate compilation memory report
         working-directory: ./test-repo/${{ matrix.project.path }}
         run: |
           mv /home/runner/work/noir/noir/scripts/test_programs/memory_report.sh ./memory_report.sh
-          chmod +x ./memory_report.sh
           ./memory_report.sh 1
+          # Rename the memory report as the execution report is about to write to the same file
+          cp memory_report.json compilation_memory_report.json
+
+      - name: Generate execution memory report
+        working-directory: ./test-repo/${{ matrix.project.path }}
+        run: |
+          ./memory_report.sh 1 1
 
       - name: Move compilation report
-        id: report
+        id: compilation_mem_report
+        shell: bash
+        run: |
+          PACKAGE_NAME=${{ matrix.project.path }}
+          PACKAGE_NAME=$(basename $PACKAGE_NAME)
+          mv ./test-repo/${{ matrix.project.path }}/compilation_memory_report.json ./memory_report_$PACKAGE_NAME.json
+          echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
+
+      - name: Upload compilation memory report
+        uses: actions/upload-artifact@v4
+        with:
+          name: compilation_mem_report_${{ steps.compilation_mem_report.outputs.memory_report_name }}
+          path: memory_report_${{ steps.compilation_mem_report.outputs.memory_report_name }}.json
+          retention-days: 3
+          overwrite: true
+
+      - name: Move execution report
+        id: execution_mem_report
         shell: bash
         run: |
           PACKAGE_NAME=${{ matrix.project.path }}
           PACKAGE_NAME=$(basename $PACKAGE_NAME)
           mv ./test-repo/${{ matrix.project.path }}/memory_report.json ./memory_report_$PACKAGE_NAME.json
           echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
 
-      - name: Upload memory report
+      - name: Upload execution memory report
         uses: actions/upload-artifact@v4
         with:
-          name: memory_report_${{ steps.report.outputs.memory_report_name }}
-          path: memory_report_${{ steps.report.outputs.memory_report_name }}.json
+          name: execution_mem_report_${{ steps.execution_mem_report.outputs.memory_report_name }}
+          path: memory_report_${{ steps.execution_mem_report.outputs.memory_report_name }}.json
           retention-days: 3
           overwrite: true
 
-  upload_memory_report:
-    name: Upload memory report 
+  upload_compilation_memory_report:
+    name: Upload compilation memory report 
     needs: [generate_memory_report, external_repo_memory_report]
     # We want this job to run even if one variation of the matrix in `external_repo_memory_report` fails
     if: always() 
@@ -528,34 +565,80 @@ jobs:
       - name: Download initial memory report
         uses: actions/download-artifact@v4
         with:
-          name: in_progress_memory_report
+          name: in_progress_compilation_mem_report
 
       - name: Download matrix memory reports
         uses: actions/download-artifact@v4
         with:
-          pattern: memory_report_*
+          pattern: compilation_mem_report_*
           path: ./reports
 
       - name: Merge memory reports using jq
         run: |
           mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh
           ./merge-bench-reports.sh memory_report
 
-      - name: Parse memory report
-        id: memory_report
-        uses: noir-lang/noir-bench-report@e408e131e96c3615b4f820d7d642360fb4d6e2f4
+      - name: Parse compilation memory report
+        id: compilation_mem_report
+        uses: noir-lang/noir-bench-report@6ba151d7795042c4ff51864fbeb13c0a6a79246c
         with:
           report: memory_report.json
           header: |
-            # Memory Report
+            Compilation Memory Report
           memory_report: true
 
       - name: Add memory report to sticky comment
         if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
         uses: marocchino/sticky-pull-request-comment@v2
         with:
-          header: memory
-          message: ${{ steps.memory_report.outputs.markdown }}
+          header: compilation_memory
+          message: ${{ steps.compilation_mem_report.outputs.markdown }}
+
+  upload_execution_memory_report:
+    name: Upload execution memory report 
+    needs: [generate_memory_report, external_repo_memory_report]
+    # We want this job to run even if one variation of the matrix in `external_repo_memory_report` fails
+    if: always() 
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download initial memory report
+        uses: actions/download-artifact@v4
+        with:
+          name: in_progress_execution_mem_report
+
+      - name: Download matrix memory reports
+        uses: actions/download-artifact@v4
+        with:
+          pattern: execution_mem_report_*
+          path: ./reports
+
+      - name: Merge memory reports using jq
+        run: |
+          mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh
+          ./merge-bench-reports.sh memory_report
+          # Rename the memory report as to not clash with the compilation memory report file name
+          cp memory_report.json execution_memory_report.json
+
+      - name: Parse execution memory report
+        id: execution_mem_report
+        uses: noir-lang/noir-bench-report@6ba151d7795042c4ff51864fbeb13c0a6a79246c
+        with:
+          report: execution_memory_report.json
+          header: |
+            Execution Memory Report
+          memory_report: true
+
+      - name: Add execution memory report to sticky comment
+        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: execution_memory
+          message: ${{ steps.execution_mem_report.outputs.markdown }}
 
   upload_execution_report:
     name: Upload execution report 
@@ -587,11 +670,11 @@ jobs:
 
       - name: Parse execution report
         id: execution_report
-        uses: noir-lang/noir-bench-report@e408e131e96c3615b4f820d7d642360fb4d6e2f4
+        uses: noir-lang/noir-bench-report@6ba151d7795042c4ff51864fbeb13c0a6a79246c
         with:
           report: execution_report.json
           header: |
-            # Execution Report
+            Execution Report
           execution_report: true
 
       - name: Add memory report to sticky comment

diff --git a/test_programs/memory_report.sh b/test_programs/memory_report.sh
@@ -12,7 +12,7 @@ current_dir=$(pwd)
 base_path="$current_dir/execution_success"
 
 # If there is an argument that means we want to generate a report for only the current directory
-if [ "$#" -ne 0 ]; then
+if [ "$1" == "1" ]; then
   base_path="$current_dir"
   tests_to_profile=(".")
 fi
@@ -31,11 +31,16 @@ for test_name in ${tests_to_profile[@]}; do
             echo " ," >> $current_dir"/memory_report.json"
         fi
 
-        if [ "$#" -ne 0 ]; then
+        if [ "$1" == "1" ]; then
             test_name=$(basename $current_dir)
         fi
 
-        heaptrack --output $current_dir/$test_name"_heap" $NARGO compile --force
+        COMMAND="compile --force --silence-warnings"
+        if [ "$2" == "1" ]; then
+            COMMAND="execute --silence-warnings"
+        fi
+
+        heaptrack --output $current_dir/$test_name"_heap" $NARGO $COMMAND
         if test -f $current_dir/$test_name"_heap.gz"; 
         then 
             heaptrack --analyze $current_dir/$test_name"_heap.gz" > $current_dir/$test_name"_heap_analysis.txt"