Add online benchmarking script

scripts/run_benchmarks.sh

@@ -0,0 +1,303 @@
+#!/bin/bash
+
+echo "Start running experiments"
+
+# =============================================================================
+# HOW TO USE THIS SCRIPT
+#
+# param_sets below is the ONLY thing you need to edit.
+#
+# >>> TO RUN MORE EXPERIMENTS, JUST ADD MORE LINES TO param_sets. <<<
+#
+# ONE LINE = ONE BENCHMARKING RUN.
+#
+# For example, to sweep batch size:
+#
+#   param_sets=(
+#       "bench=aiops ... batch_size=4  ..."
+#       "bench=aiops ... batch_size=8  ..."
+#       "bench=aiops ... batch_size=16 ..."
+#   )
+#
+# Each line runs sequentially: start its own vllm server, run the benchmark
+# against it, kill the server, move to the next line.
+#
+# Results go to results/<auto-generated-name>/, where the name encodes every
+# param below, so different configs never overwrite each other. Re-running the
+# same config overwrites it, unless you pass --no-overwrite (then it creates
+# ..._1, ..._2, ... alongside).
+#
+# PARAMS (all are per-experiment; the last three may be omitted to use defaults)
+#
+#   bench             Which dataset to run. One of the bench_files keys defined
+#                     further down: cics, db2, ims, aiops, tls, all.
+#   num_prompts       Number of prompts to send from that dataset.
+#   batch_size        Server-side max batched sequences (--max-num-seqs).
+#   max_context_len   Server-side max model length (--max-model-len), in tokens.
+#   concurrency       Client-side max in-flight requests (--max-concurrency).
+#                     Usually set equal to batch_size.
+#   chunk_size        Server-side prefill chunk (--max-num-batched-tokens).
+#   prefix_caching    1 = --enable-prefix-caching, 0 = --no-enable-prefix-caching.
+#   shuffle           1 = shuffle the dataset order, 0 = keep dataset order
+#                     (0 passes --disable-shuffle).
+#   ignore_eos        1 = --ignore-eos, so generation always runs to the full
+#                     output length instead of stopping at EOS. 0 = honor EOS.
+#   custom_output_len Force this many output tokens per request. -1 = use the
+#                     output lengths recorded in the dataset. (optional, default -1)
+#   tp_size           Tensor parallel size (--tensor-parallel-size).
+#                     (optional, default $default_tp_size)
+#   num_blocks        KV cache blocks (--num-gpu-blocks-override).
+#                     (optional, default $default_num_blocks)
+#
+# The single line below is the GOLDEN BENCHMARK -- this is the configuration we 
+# use to benchmark spyre-inference
+# =============================================================================
+param_sets=(
+    # GOLDEN BENCHMARK -- leave this line as-is.
+    "bench=aiops num_prompts=200 batch_size=4 max_context_len=8192 concurrency=4 chunk_size=512 prefix_caching=1 shuffle=0 ignore_eos=1 custom_output_len=-1 tp_size=1 num_blocks=2049"
+)
+
+# Parse command line arguments
+NO_OVERWRITE=false
+for arg in "$@"; do
+    if [ "$arg" == "--no-overwrite" ]; then
+        NO_OVERWRITE=true
+        echo "Running with --no-overwrite: will create incremented folders instead of overwriting"
+    fi
+done
+
+# try installing libraries for visualization plots
+uv pip install pandas matplotlib plotly 2>/dev/null
+
+# pick up Spyre hardware libs if they've been installed via
+# scripts/install-pinned-rpms.sh; needed by both the server and the client
+if [ -f ~/spyre-libs/env.sh ]; then
+    source ~/spyre-libs/env.sh
+fi
+
+export SPYRE_NUM_CPUS=8
+export HF_HOME=/models/huggingface_cache
+model=ibm-granite/granite-3.3-8b-instruct
+
+results_folder=results/
+result_filename=result.json
+default_tp_size=1
+default_num_blocks=2049
+timeout=7200 # wait for up to two hours for the server to get ready
+
+# remove trailing slashes
+results_folder="${results_folder%/}"
+
+echo -e "\n"
+echo "model: $model"
+echo "results folder: $results_folder"
+echo -e "\n"
+
+mkdir $results_folder 2>/dev/null
+
+declare -A bench_files
+bench_files["cics"]="/models/online_benchmarking_data_reordered/cics_results_2025.11.03_e2ee1b0_correct_order.jsonl"
+bench_files["db2"]="/models/online_benchmarking_data_reordered/db2_results_2025.11.03_e2ee1b0_correct_order.jsonl"
+bench_files["ims"]="/models/online_benchmarking_data_reordered/ims_results_2025.11.03_e2ee1b0_correct_order.jsonl"
+bench_files["aiops"]="/models/online_benchmarking_data_reordered/aiops_results_2025.11.03_e2ee1b0_correct_order.jsonl"
+bench_files["tls"]="/models/online_benchmarking_data_reordered/tls_results_2025.11.03_e2ee1b0_correct_order.jsonl"
+bench_files["all"]="/models/online_benchmarking_data_reordered/all_sequences_interleaved.jsonl"
+
+# Iterate over keys
+for param_set in "${param_sets[@]}"; do
+
+    # clear optional params so values don't leak across param sets
+    unset tp_size custom_output_len num_blocks
+
+    # retrieve config params
+    eval "$param_set"
+    bench_file=${bench_files[$bench]}
+
+    # Use tp_size from param_set if provided, otherwise fall back to the default
+    if [ -z "$tp_size" ]; then
+        tp_size=$default_tp_size
+    fi
+
+    # Use custom_output_len from param_set if provided, otherwise default to -1
+    if [ -z "$custom_output_len" ]; then
+        custom_output_len="-1"
+    fi
+
+    # Use num_blocks from param_set if provided, otherwise fall back to the default
+    if [ -z "$num_blocks" ]; then
+        num_blocks=$default_num_blocks
+    fi
+
+    experiments_results_base=${results_folder}/${bench}_${num_prompts}_${max_context_len}_bs${batch_size}_conc${concurrency}_chunksize${chunk_size}_pc${prefix_caching}_shuffle${shuffle}_ignoreeos${ignore_eos}_olen${custom_output_len}_tp${tp_size}_nblocks${num_blocks}
+
+    # Handle existing directory based on NO_OVERWRITE flag
+    experiments_results=$experiments_results_base
+    if [ "$NO_OVERWRITE" = true ]; then
+        # Check if directory exists and find next available suffix
+        if [ -d "$experiments_results" ]; then
+            counter=1
+            while [ -d "${experiments_results_base}_${counter}" ]; do
+                counter=$((counter + 1))
+            done
+            experiments_results="${experiments_results_base}_${counter}"
+            echo "Directory $experiments_results_base already exists, using $experiments_results instead"
+        fi
+    else
+        # Delete and recreate the folder if it exists
+        if [ -d "$experiments_results" ]; then
+            echo "Directory $experiments_results already exists, deleting and recreating"
+            rm -rf "$experiments_results"
+        fi
+    fi
+
+    echo -e "\n"
+    echo "benchmark: $bench_file"
+    echo "batch_size: $batch_size"
+    echo "max context length: $max_context_len"
+    echo "max active requests: $concurrency"
+    echo "tensor parallel size: $tp_size"
+    echo "num gpu blocks override: $num_blocks"
+    echo "use prefix caching: $prefix_caching"
+    echo "chunk size: $chunk_size"
+    echo "shuffle: $shuffle"
+    echo "results folder: $experiments_results"
+    echo -e "\n"
+
+    # create result directory
+    mkdir -p $experiments_results
+
+    if [[ "$prefix_caching" -eq 1 ]]; then
+        prefix_caching_arg="--enable-prefix-caching"
+    else
+        prefix_caching_arg="--no-enable-prefix-caching"
+    fi
+
+    max_num_batched_tokens_arg="--max-num-batched-tokens $chunk_size"
+
+    # Set shuffle_arg based on shuffle parameter
+    if [[ "$shuffle" -eq 0 ]]; then
+        shuffle_arg="--disable-shuffle"
+    else
+        shuffle_arg=""
+    fi
+
+    if [[ "$ignore_eos" -eq 1 ]]; then
+        ignore_eos_arg="--ignore-eos"
+    else
+        ignore_eos_arg=""
+    fi
+
+    echo -e "
+    export SPYRE_KERNEL_CACHE=1
+
+    vllm serve $model
+        --max-model-len $max_context_len
+        --max-num-seqs $batch_size
+        --tensor-parallel-size $tp_size
+        $max_num_batched_tokens_arg
+        $prefix_caching_arg > ${experiments_results}/serving_output.txt 2>&1
+    " > ${experiments_results}/serving_output.txt
+
+    export SPYRE_KERNEL_CACHE=1
+    vllm serve $model \
+        --max-model-len $max_context_len \
+        --max-num-seqs $batch_size \
+        --tensor-parallel-size $tp_size \
+        --num-gpu-blocks-override $num_blocks \
+        $max_num_batched_tokens_arg \
+        $prefix_caching_arg >> ${experiments_results}/serving_output.txt 2>&1 &
+
+    PID=$!  # Capture PID
+
+    # Save all bash variables
+    declare -p > ${experiments_results}/experiment_config.log
+
+    # Loop to check periodically that the server is ready
+    elapsed=0
+    while true; do
+        echo "Check server ready"
+        if grep -q "Application startup complete" ${experiments_results}/serving_output.txt; then
+
+            echo "Starting vllm bench serve..."
+
+            echo -e "
+            vllm bench serve
+                --backend vllm
+                --model $model
+                --save-result
+                --save-detailed
+                --result-dir $experiments_results
+                --result-filename $result_filename
+                --endpoint /v1/completions
+                --dataset-name custom
+                --dataset-path $bench_file
+                --num-prompts $num_prompts
+                --skip-chat-template
+                --metric-percentiles 99,100
+                --percentile-metrics ttft,tpot,itl,e2el
+                --custom-output-len $custom_output_len
+                $ignore_eos_arg
+                $shuffle_arg
+                --max-concurrency $concurrency
+            " > ${experiments_results}/bench_output.txt
+
+            vllm bench serve \
+                --backend vllm \
+                --model $model \
+                --save-result \
+                --save-detailed \
+                --result-dir $experiments_results \
+                --result-filename $result_filename \
+                --endpoint /v1/completions \
+                --dataset-name custom \
+                --dataset-path $bench_file \
+                --num-prompts $num_prompts \
+                --skip-chat-template \
+                --metric-percentiles 99,100 \
+                --percentile-metrics ttft,tpot,itl,e2el \
+                --custom-output-len $custom_output_len \
+                $ignore_eos_arg \
+                $shuffle_arg \
+                --max-concurrency $concurrency >> ${experiments_results}/bench_output.txt 2>&1
+
+            EXIT_STATUS=$?
+
+            if [ "$EXIT_STATUS" -ne 0 ]; then
+                echo "[$(date)] Exp failed with status $EXIT_STATUS."
+            fi
+
+            # kill the server
+            echo "kill the server, PID=${PID}"
+            sleep 10
+            kill $PID
+            sleep 10
+            kill -15 $PID 2>/dev/null
+            sleep 10
+            kill -9 $PID 2>/dev/null
+            break
+
+        elif grep -qe "ERROR" -qe "vllm: error" ${experiments_results}/serving_output.txt; then
+            echo "Error detected in server logs. Failing the experiment."
+
+            # We want to wait before killing the server, try ovoiding ghost processes
+            sleep 10
+            kill -15 $(pstree -p $PID | grep -o '([0-9]\+)' | tr -d '()') 2>/dev/null
+            sleep 10
+            kill -9 $(pstree -p $PID | grep -o '([0-9]\+)' | tr -d '()') 2>/dev/null
+            break
+
+        elif [ "$elapsed" -ge "$timeout" ]; then
+            echo "Timeout waiting for server ($timeout sec). Failing the experiment."
+
+            # We need to wait before killing the server otherwise my might get ghost processes
+            sleep 10
+            kill -15 $(pstree -p $PID | grep -o '([0-9]\+)' | tr -d '()') 2>/dev/null
+            sleep 10
+            kill -9 $(pstree -p $PID | grep -o '([0-9]\+)' | tr -d '()') 2>/dev/null
+            sleep 10
+            break
+        fi
+        sleep 5  # Wait before checking again
+        elapsed=$((elapsed + 5))
+    done
+done