From cc5f0807a8a69843211d43e73b1f8e429c0a7c8c Mon Sep 17 00:00:00 2001
From: Esteve Soler Arderiu <esteve.soler@lambdaclass.com>
Date: Wed, 14 Feb 2024 10:25:39 +0000
Subject: [PATCH 1/6] Add YAS benchmark runner on the CI.

---
 .github/workflows/bench-yas.yml | 78 +++++++++++++++++++++++++++++++++
 scripts/bench-yas.sh            | 34 ++++++++++++++
 2 files changed, 112 insertions(+)
 create mode 100644 .github/workflows/bench-yas.yml
 create mode 100644 scripts/bench-yas.sh

diff --git a/.github/workflows/bench-yas.yml b/.github/workflows/bench-yas.yml
new file mode 100644
index 000000000..eee4b87b8
--- /dev/null
+++ b/.github/workflows/bench-yas.yml
@@ -0,0 +1,78 @@
+name: Bench
+
+on:
+  pull_request:
+    branches: [main]
+  merge_group:
+    types: [checks_requested]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  bench-hyperfine:
+    name: Hyperfine
+    runs-on: ubuntu-latest
+    env:
+      CAIRO_NATIVE_RUNTIME_LIBDIR: ${{ github.workspace }}/target/release
+      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+      MLIR_SYS_170_PREFIX: /usr/lib/llvm-17/
+      TABLEGEN_170_PREFIX: /usr/lib/llvm-17/
+    steps:
+      - uses: actions/checkout@v3
+      - name: check and free hdd space left
+        run: |
+          echo "Listing 20 largest packages"
+          dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 20
+          df -h
+          sudo apt-get update
+          sudo apt-get remove -y '^llvm-.*'
+          sudo apt-get remove -y 'php.*'
+          sudo apt-get remove -y '^dotnet-.*'
+          sudo apt-get remove -y '^temurin-.*'
+          sudo apt-get remove -y azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox powershell mono-devel
+          sudo apt-get autoremove -y
+          sudo apt-get clean
+          df -h
+          echo "Removing large directories"
+          # deleting 15GB
+          sudo rm -rf /usr/share/dotnet/
+          sudo rm -rf /usr/local/lib/android
+          df -h
+      - uses: dtolnay/rust-toolchain@1.72.1
+        with:
+          components: clippy
+      - uses: Swatinem/rust-cache@v2
+      - name: add llvm deb repository
+        uses: myci-actions/add-deb-repo@10
+        with:
+          repo: deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main
+          repo-name: llvm-repo
+          keys-asc: https://apt.llvm.org/llvm-snapshot.gpg.key
+      - name: Install LLVM
+        run: sudo apt-get install llvm-17 llvm-17-dev llvm-17-runtime clang-17 clang-tools-17 lld-17 libpolly-17-dev libmlir-17-dev mlir-17-tools
+      - name: Install deps
+        run: make deps
+      - name: Build project
+        run: make build
+      - name: Run benchmarks
+        run: ./scripts/bench-yas.sh
+
+      - name: Find Bench Comment
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Benchmarking
+      - name: Create or update bench comment
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body-path: bench-yas.md
+          edit-mode: replace
diff --git a/scripts/bench-yas.sh b/scripts/bench-yas.sh
new file mode 100644
index 000000000..256a7f929
--- /dev/null
+++ b/scripts/bench-yas.sh
@@ -0,0 +1,34 @@
+run_bench() {
+    local OUTPUT=$(${*:2})
+    local NUM_EXECS=$(echo $OUTPUT | cut -d' ' -f3)
+    local TOTAL_TIME=$(echo $OUTPUT | cut -d' ' -f6)
+    local EXEC_SPEED=$(echo $OUTPUT | cut -d' ' -f8 | cut -c2-)
+    local EXEC_TIME=$(echo $OUTPUT | cut -d' ' -f11)
+
+    if [[ -z "$NUM_EXECS" || -z "$TOTAL_TIME" || -z "$EXEC_SPEED" || -z "$EXEC_TIME" ]]; then
+        echo "| $1 | CRASHED | CRASHED | CRASHED | CRASHED |"
+    else
+        echo "| $1 | $NUM_EXECS | $TOTAL_TIME | $EXEC_SPEED | $EXEC_TIME |"
+    fi
+}
+
+# Clone cairo native and build the runtime.
+git clone https://github.com/lambdaclass/cairo_native.git cairo-native
+cd cairo-native/
+cargo build --release --package cairo-native-runtime
+cd ..
+
+# Run the benches.
+export CAIRO_NATIVE_RUNTIME_LIBDIR="$(pwd)/cairo-native/target/release"
+BENCH_VM=$(run_bench "VM"         cargo bench --bench yas)
+BENCH_JIT=$(run_bench "Native JIT" cargo bench --bench yas --features=cairo-native -- jit)
+BENCH_AOT=$(run_bench "Native AOT" cargo bench --bench yas --features=cairo-native -- aot)
+
+# Write the results.
+echo "# Benchmarking results" > bench-yas.md
+echo "" >> bench-yas.md
+echo "| Name | Number of runs | Total time (s) | Speed (#/s) | Individual time (s/#) |" >> bench-yas.md
+echo "|------|----------------|----------------|-------------|-----------------------|" >> bench-yas.md
+echo $BENCH_VM >> bench-yas.md
+echo $BENCH_JIT >> bench-yas.md
+echo $BENCH_AOT >> bench-yas.md

From da4d6aeed9468d059547bf1e81ae5476225d0d1c Mon Sep 17 00:00:00 2001
From: Esteve Soler Arderiu <esteve.soler@lambdaclass.com>
Date: Wed, 14 Feb 2024 10:26:49 +0000
Subject: [PATCH 2/6] Fix workflow name.

---
 .github/workflows/bench-yas.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bench-yas.yml b/.github/workflows/bench-yas.yml
index eee4b87b8..670b13e17 100644
--- a/.github/workflows/bench-yas.yml
+++ b/.github/workflows/bench-yas.yml
@@ -14,8 +14,8 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  bench-hyperfine:
-    name: Hyperfine
+  bench-yas:
+    name: YAS benchmark
     runs-on: ubuntu-latest
     env:
       CAIRO_NATIVE_RUNTIME_LIBDIR: ${{ github.workspace }}/target/release

From e7552db8e7c0087a119cc890aea2b954cf24b051 Mon Sep 17 00:00:00 2001
From: Esteve Soler Arderiu <esteve.soler@lambdaclass.com>
Date: Wed, 14 Feb 2024 10:39:27 +0000
Subject: [PATCH 3/6] Fix CI.

---
 .github/workflows/bench-yas.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/bench-yas.yml b/.github/workflows/bench-yas.yml
index 670b13e17..6c2d91619 100644
--- a/.github/workflows/bench-yas.yml
+++ b/.github/workflows/bench-yas.yml
@@ -46,6 +46,11 @@ jobs:
       - uses: dtolnay/rust-toolchain@1.72.1
         with:
           components: clippy
+      - name: Python3 Build
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+          cache: 'pip'
       - uses: Swatinem/rust-cache@v2
       - name: add llvm deb repository
         uses: myci-actions/add-deb-repo@10

From 1e8ac049b91681ce09249d1e8401c0284c3c3fed Mon Sep 17 00:00:00 2001
From: Esteve Soler Arderiu <esteve.soler@lambdaclass.com>
Date: Wed, 14 Feb 2024 10:57:41 +0000
Subject: [PATCH 4/6] Update Rust toolchain.

---
 rust-toolchain.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index baa36b056..e8130aef5 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -1,4 +1,4 @@
 [toolchain]
-channel = "1.72.1"
+channel = "1.74"
 components = ["rustfmt", "clippy"]
 profile = "minimal"

From fbef576d1079b3910df85b18d64e5154126a6395 Mon Sep 17 00:00:00 2001
From: Esteve Soler Arderiu <esteve.soler@lambdaclass.com>
Date: Wed, 14 Feb 2024 12:21:26 +0000
Subject: [PATCH 5/6] Make the script executable.

---
 scripts/bench-yas.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 scripts/bench-yas.sh

diff --git a/scripts/bench-yas.sh b/scripts/bench-yas.sh
old mode 100644
new mode 100755

From 86d69cf71834114192e018347db8cbb1453281ba Mon Sep 17 00:00:00 2001
From: Esteve Soler Arderiu <esteve.soler@lambdaclass.com>
Date: Wed, 14 Feb 2024 12:21:39 +0000
Subject: [PATCH 6/6] Update rust version.

---
 rust-toolchain.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index e8130aef5..4da0ec7b8 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -1,4 +1,4 @@
 [toolchain]
-channel = "1.74"
+channel = "1.74.1"
 components = ["rustfmt", "clippy"]
 profile = "minimal"