Gpt Oss Enablement

.cd/Dockerfile.rhel.tenc.pytorch.vllm

@@ -13,8 +13,9 @@ ARG TORCH_TYPE_SUFFIX
 FROM ${DOCKER_URL}/${VERSION}/${BASE_NAME}/${REPO_TYPE}/pytorch-${TORCH_TYPE_SUFFIX}installer-${PT_VERSION}:${REVISION}
 
 # Parameterize commit/branch for vllm-fork checkout
-ARG VLLM_GAUDI_COMMIT=v0.10.1
-ARG VLLM_PROJECT_COMMIT=v0.10.1
+ARG VLLM_GAUDI_COMMIT=main
+# leave empty to use last-good-commit-for-vllm-gaudi
+ARG VLLM_PROJECT_COMMIT=
 
 ARG BASE_NAME
 ENV BASE_NAME=${BASE_NAME}
@@ -39,23 +40,36 @@ ENV VLLM_PATH=/workspace/vllm-project
 ENV VLLM_PATH2=/workspace/vllm-gaudi
 
 # Clone the vllm-project repository and install inside the container
-RUN mkdir -p $VLLM_PATH && \
+# --- START: COMBINED RUN COMMAND ---
+RUN \
+    # Clone vllm-gaudi and get the commit hash for the vllm-project/vllm
+    set -e && \
+    mkdir -p $VLLM_PATH2 && \
+    git clone https://github.com/vllm-project/vllm-gaudi.git $VLLM_PATH2 && \
+    cd $VLLM_PATH2 && \
+    if [ -z "${VLLM_PROJECT_COMMIT}" ]; then \
+       VLLM_PROJECT_COMMIT=$(git show "origin/vllm/last-good-commit-for-vllm-gaudi:VLLM_STABLE_COMMIT" 2>/dev/null) && \
+       echo "Found vLLM commit hash: ${VLLM_PROJECT_COMMIT}"; \
+    else \
+       echo "Using vLLM commit : ${VLLM_PROJECT_COMMIT}"; \
+    fi && \
+    mkdir -p $VLLM_PATH && \
+    # Clone vllm-project/vllm and use configured or last good commit hash
     git clone https://github.com/vllm-project/vllm.git $VLLM_PATH && \
     cd $VLLM_PATH && \
     git remote add upstream https://github.com/vllm-project/vllm.git && \
     git fetch upstream --tags || true && \
     git checkout ${VLLM_PROJECT_COMMIT} && \
-    bash -c "pip install -r <(sed '/^[torch]/d' requirements/build.txt)" && \
-    VLLM_TARGET_DEVICE=empty pip install --no-build-isolation -e .
-
-# Clone the vllm-gaudi repository and install inside the container
-RUN mkdir -p $VLLM_PATH2 && \
-    git clone https://github.com/vllm-project/vllm-gaudi.git $VLLM_PATH2 && \
+    # Install vllm-project/vllm
+    bash -c "pip install -r <(sed '/^torch/d' requirements/build.txt)" && \
+    VLLM_TARGET_DEVICE=empty pip install --no-build-isolation . && \
+    # Install vllm-gaudi plugin
     cd $VLLM_PATH2 && \
     git checkout ${VLLM_GAUDI_COMMIT} && \
-    VLLM_TARGET_DEVICE=hpu && pip install -v -e $VLLM_PATH2
+    VLLM_TARGET_DEVICE=hpu pip install -v . --no-build-isolation
+# --- END: COMBINED RUN COMMAND ---
 
-    # to be enabled later PWolsza
+# to be enabled later PWolsza
 # RUN pip3 install -v -e $VLLM_PATH/tests/vllm_test_utils
 
 # Install additional Python packages
@@ -70,4 +84,4 @@ COPY benchmark /root/scripts/benchmark/
 WORKDIR /root/scripts
 
 # Set entrypoint script
-ENTRYPOINT ["python3", "-m", "entrypoints.entrypoint_main"]
+ENTRYPOINT ["python3", "-m", "entrypoints.entrypoint_main"]

.cd/Dockerfile.ubuntu.pytorch.vllm

@@ -4,17 +4,17 @@
 # Parameterize base image components
 ARG DOCKER_URL=vault.habana.ai/gaudi-docker
 ARG VERSION=1.22.0
-ARG BASE_NAME=ubuntu22.04
+ARG BASE_NAME=ubuntu24.04
 ARG PT_VERSION=2.7.1
 ARG REVISION=latest
 ARG REPO_TYPE=habanalabs
 
 FROM ${DOCKER_URL}/${VERSION}/${BASE_NAME}/${REPO_TYPE}/pytorch-installer-${PT_VERSION}:${REVISION}
 
 # Parameterize commit/branch for vllm-project & vllm-gaudi checkout
-ARG VLLM_GAUDI_COMMIT=v0.10.2_next
-ARG VLLM_PROJECT_COMMIT=v0.10.2
-
+ARG VLLM_GAUDI_COMMIT=main
+# leave empty to use last-good-commit-for-vllm-gaudi
+ARG VLLM_PROJECT_COMMIT=
 ENV OMPI_MCA_btl_vader_single_copy_mechanism=none
 
 RUN apt update && \
@@ -30,24 +30,34 @@ RUN echo "dash dash/sh boolean false" | debconf-set-selections && \
 ENV ENV=~/.profile
 
 # Clone the vllm-project repository and install inside the container
-
-RUN mkdir -p $VLLM_PATH && \
+# --- START: COMBINED RUN COMMAND ---
+RUN \
+    # Clone vllm-gaudi and get the commit hash for the vllm-project/vllm
+    set -e && \
+    mkdir -p $VLLM_PATH2 && \
+    git clone https://github.com/vllm-project/vllm-gaudi.git $VLLM_PATH2 && \
+    cd $VLLM_PATH2 && \
+    if [ -z "${VLLM_PROJECT_COMMIT}" ]; then \
+       VLLM_PROJECT_COMMIT=$(git show "origin/vllm/last-good-commit-for-vllm-gaudi:VLLM_STABLE_COMMIT" 2>/dev/null) && \
+       echo "Found vLLM commit hash: ${VLLM_PROJECT_COMMIT}"; \
+    else \
+       echo "Using vLLM commit : ${VLLM_PROJECT_COMMIT}"; \
+    fi && \
+    mkdir -p $VLLM_PATH && \
+    # Clone vllm-project/vllm and use configured or last good commit hash
     git clone https://github.com/vllm-project/vllm.git $VLLM_PATH && \
     cd $VLLM_PATH && \
     git remote add upstream https://github.com/vllm-project/vllm.git && \
     git fetch upstream --tags || true && \
     git checkout ${VLLM_PROJECT_COMMIT} && \
-    bash -c "pip install -r <(sed '/^[torch]/d' requirements/build.txt)" && \
-    VLLM_TARGET_DEVICE=empty pip install --no-build-isolation .
-
-# Clone the vllm-gaudi repository and install inside the container
-
-RUN mkdir -p $VLLM_PATH2 && \
-    git clone https://github.com/vllm-project/vllm-gaudi.git $VLLM_PATH2 && \
+    # Install vllm-project/vllm
+    bash -c "pip install -r <(sed '/^torch/d' requirements/build.txt)" && \
+    VLLM_TARGET_DEVICE=empty pip install --no-build-isolation . && \
+    # Install vllm-gaudi plugin
     cd $VLLM_PATH2 && \
-# Comment: enable if vllm-gaudi release version is used otherwise main
-    git checkout ${VLLM_GAUDI_COMMIT} && \ 
-    VLLM_TARGET_DEVICE=hpu && pip install -v $VLLM_PATH2 --no-build-isolation
+    git checkout ${VLLM_GAUDI_COMMIT} && \
+    VLLM_TARGET_DEVICE=hpu pip install -v . --no-build-isolation
+# --- END: COMBINED RUN COMMAND ---
 
 # Install additional Python packages
 RUN pip install datasets && \

.cd/Dockerfile.ubuntu.pytorch.vllm.nixl.latest

@@ -45,7 +45,7 @@ RUN \
     git remote add upstream https://github.com/vllm-project/vllm.git && \
     git fetch upstream --tags || true && \
     git checkout ${VLLM_COMMIT_HASH} && \
-    pip install -r <(sed '/^[torch]/d' requirements/build.txt) && \
+    pip install -r <(sed '/^torch/d' requirements/build.txt) && \
     VLLM_TARGET_DEVICE=empty pip install --no-build-isolation . && \
     \
     # Install vllm-gaudi

.cd/benchmark/benchmark_defaults.yaml

@@ -29,12 +29,10 @@ model_text:
 
 model_vision:
   MODELS:
-    - meta-llama/Llama-3.2-11B-Vision-Instruct
-    - meta-llama/Llama-3.2-90B-Vision-Instruct
     - Qwen/Qwen2.5-VL-7B-Instruct
   DATASET: lmarena-ai/vision-arena-bench-v0.1
   DATASET_NAME: hf
   BACKEND: openai-chat
   ENDPOINT: /v1/chat/completions
   CONCURRENT_REQ: 64
-  NUM_PROMPTS: 500
+  NUM_PROMPTS: 500

.cd/benchmark/benchmark_scenarios_vision.yaml

@@ -1,8 +1,2 @@
-llama32-11B-Vision-Instruct:
-  MODEL: meta-llama/Llama-3.2-11B-Vision-Instruct
-
-llama32-90B-Vision-Instruct:
-  MODEL: meta-llama/Llama-3.2-90B-Vision-Instruct
-
 qwen2.5-vl-7b-instruct:
   MODEL: Qwen/Qwen2.5-VL-7B-Instruct

.cd/entrypoints/entrypoint_main.py

@@ -190,6 +190,7 @@ def run(self):
                 output_script_path="vllm_server.sh",
                 variables=variables,
                 log_dir="logs",
+                varlist_conf_path="server/server_output.env",
             ).create_and_run()
         elif self.mode == "benchmark":
             print("[INFO] Starting container in benchmark mode.")

.cd/entrypoints/script_generator.py

@@ -4,8 +4,9 @@
 
 class ScriptGenerator:
 
-    def __init__(self, template_script_path, output_script_path, variables, log_dir="logs"):
+    def __init__(self, template_script_path, output_script_path, variables, log_dir="logs", varlist_conf_path=None):
         self.template_script_path = template_script_path
+        self.varlist_conf_path = varlist_conf_path
         self.output_script_path = output_script_path
         self.variables = variables
         self.log_dir = log_dir
@@ -19,7 +20,16 @@ def generate_script(self, vars_dict):
         """
         with open(self.template_script_path) as f:
             template = f.read()
-        export_lines = "\n".join([f"export {k}={v}" for k, v in vars_dict.items()])
+        # Create our output list
+        if self.varlist_conf_path:
+            output_dict = {}
+            with open(self.varlist_conf_path) as var_file:
+                for line in var_file:
+                    param = line.strip()
+                    output_dict[param] = vars_dict[param]
+            export_lines = "\n".join([f"export {k}={v}" for k, v in output_dict.items()])
+        else:
+            export_lines = "\n".join([f"export {k}={v}" for k, v in vars_dict.items()])
         script_content = template.replace("#@VARS", export_lines)
         with open(self.output_script_path, 'w') as f:
             f.write(script_content)

.cd/server/server_output.env

@@ -0,0 +1,60 @@
+MODEL
+DTYPE
+DEVICE_NAME
+TENSOR_PARALLEL_SIZE
+MAX_MODEL_LEN
+TOTAL_GPU_MEM
+MODEL_DTYPE
+QUANT_DTYPE
+BLOCK_SIZE
+VLLM_PROMPT_BS_BUCKET_MIN
+VLLM_PROMPT_BS_BUCKET_STEP
+VLLM_DECODE_BS_BUCKET_MIN
+VLLM_DECODE_BS_BUCKET_STEP
+VLLM_PROMPT_SEQ_BUCKET_MIN
+VLLM_PROMPT_SEQ_BUCKET_STEP
+VLLM_DECODE_BLOCK_BUCKET_MIN
+VLLM_DECODE_BLOCK_BUCKET_STEP
+MAX_NUM_PREFILL_SEQS
+NUM_HIDDEN_LAYERS
+HIDDEN_SIZE
+NUM_KEY_VALUE_HEADS
+NUM_ATTENTION_HEADS
+CACHE_DTYPE_BYTES
+LIMIT_MODEL_LEN
+PT_HPU_LAZY_MODE
+VLLM_DELAYED_SAMPLING
+VLLM_SKIP_WARMUP
+EXPERIMENTAL_WEIGHT_SHARING
+VLLM_EXPONENTIAL_BUCKETING
+MAX_NUM_BATCHED_TOKENS
+PT_HPU_ENABLE_LAZY_COLLECTIVES
+DEVICE_HPU_MEM
+MODEL_MEM_IN_GB
+USABLE_MEM
+GPU_MEM_UTILIZATION
+KV_CACHE_PER_SEQ
+EST_MAX_NUM_SEQS
+EST_HPU_BLOCKS
+DECODE_BS_RAMP_GRAPHS
+DECODE_BS_STEP_GRAPHS
+DECODE_BLOCK_RAMP_GRAPHS
+DECODE_BLOCK_STEP_GRAPHS
+NUM_DECODE_GRAPHS
+PROMPT_BS_RAMP_GRAPHS
+PROMPT_BS_STEP_GRAPHS
+PROMPT_SEQ_RAMP_GRAPHS
+PROMPT_SEQ_STEP_GRAPHS
+EST_NUM_PROMPT_GRAPHS
+EST_GRAPH_PROMPT_RATIO
+VLLM_GRAPH_PROMPT_RATIO
+DECODE_GRAPH_TARGET_GB
+EST_GRAPH_RESERVE_MEM
+VLLM_GRAPH_RESERVED_MEM
+KV_CACHE_MEM
+MAX_NUM_SEQS
+VLLM_PROMPT_SEQ_BUCKET_MAX
+VLLM_CONTIGUOUS_PA
+VLLM_DEFRAG
+ASYNC_SCHEDULING
+VLLM_WEIGHT_LOAD_FORCE_SYNC

.cd/server/server_user.env

@@ -10,3 +10,4 @@ MAX_NUM_SEQS
 TENSOR_PARALLEL_SIZE
 VLLM_EXPONENTIAL_BUCKETING
 GPU_MEM_UTILIZATION
+ASYNC_SCHEDULING

.cd/server/settings_vllm.csv

@@ -1,21 +1,19 @@
-MODEL,TENSOR_PARALLEL_SIZE,MAX_MODEL_LEN,TOTAL_GPU_MEM,UNAVAILABLE_MEM_ABS,MODEL_MEM_FROM_CONFIG,MODEL_DTYPE,QUANT_DTYPE,MODEL_MEM,PROFILER_MEM_OVERHEAD,APPROX_MEM_PER_GRAPH_MB,fsdpa,GPU_FREE_MEM_TARGET,BLOCK_SIZE,VLLM_PROMPT_BS_BUCKET_MIN,VLLM_PROMPT_BS_BUCKET_STEP,VLLM_DECODE_BS_BUCKET_MIN,VLLM_DECODE_BS_BUCKET_STEP,VLLM_PROMPT_SEQ_BUCKET_MIN,VLLM_PROMPT_SEQ_BUCKET_STEP,VLLM_DECODE_BLOCK_BUCKET_MIN,VLLM_DECODE_BLOCK_BUCKET_STEP,MAX_NUM_PREFILL_SEQS,NUM_HIDDEN_LAYERS,HIDDEN_SIZE,NUM_KEY_VALUE_HEADS,NUM_ATTENTION_HEADS,CACHE_DTYPE_BYTES,LIMIT_MODEL_LEN,PT_HPU_LAZY_MODE,VLLM_DELAYED_SAMPLING,VLLM_SKIP_WARMUP,EXPERIMENTAL_WEIGHT_SHARING,VLLM_EXPONENTIAL_BUCKETING,MAX_NUM_BATCHED_TOKENS
-meta-llama/Llama-3.1-8B-Instruct,1,4352,128,2,16060522496,2,2,14.95752716,5.5,10,1,1,128,1,32,1,32,128,256,128,256,1,32,4096,8,32,2,131072,1,TRUE,FALSE,0,FALSE,2048
-meta-llama/Llama-3.1-70B-Instruct,4,4352,512,2,1.41107E+11,2,2,131.4165192,5.5,20,1,1,128,1,32,1,32,128,256,128,256,1,80,8192,8,64,2,131072,1,TRUE,FALSE,0,FALSE,2048
-meta-llama/Llama-3.3-70B-Instruct,4,4352,512,2,1.41107E+11,2,2,131.4165192,5.5,20,1,1,128,1,32,1,32,128,256,128,256,1,80,8192,8,64,2,131072,1,TRUE,FALSE,0,FALSE,2048
-meta-llama/Llama-3.2-1B-Instruct,1,4352,128,2,2471645608,2,2,2.301899351,5.5,10,1,1,128,1,32,1,32,128,256,128,256,1,16,2048,8,32,2,131072,1,TRUE,FALSE,0,FALSE,2048
-meta-llama/Llama-3.2-3B-Instruct,1,4352,128,2,6425499648,2,2,5.984212875,5.5,10,1,1,128,1,32,1,32,128,256,128,256,1,28,3072,8,24,2,131072,1,TRUE,FALSE,0,FALSE,2048
-mistralai/Mixtral-8x7B-Instruct-v0.1,2,4352,256,2,93405585408,2,2,86.99073029,5.5,10,1,1,128,1,32,1,32,128,256,128,256,1,32,4096,8,32,2,32768,1,TRUE,FALSE,0,FALSE,2048
-mistralai/Mixtral-8x22B-Instruct-v0.1,4,4352,512,2,2.8126E+11,2,2,261.9439201,5.5,10,1,1,128,1,32,1,32,128,256,128,256,1,56,6144,8,48,2,65536,1,TRUE,FALSE,0,FALSE,2048
-mistralai/Mistral-7B-Instruct-v0.2,1,4352,128,2,14483464192,2,2,13.48877716,5.5,10,1,1,128,1,32,1,32,128,256,128,256,1,32,4096,8,32,2,32768,1,TRUE,FALSE,0,FALSE,2048
-meta-llama/Llama-3.1-405B-Instruct,8,4352,1024,2,8.11707E+11,2,2,755.9608459,5.5,10,1,1,128,1,32,1,32,128,256,128,256,1,126,16384,8,128,2,131072,1,TRUE,FALSE,0,FALSE,2048
-Qwen/Qwen2.5-14B-Instruct,1,4352,128,2,29540067328,2,2,27.51133156,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,48,5120,8,40,2,32768,1,TRUE,FALSE,0,FALSE,2048
-deepseek-ai/DeepSeek-R1-Distill-Llama-70B,4,4352,512,2,1.41107E+11,2,2,131.4165192,5.5,20,1,1,128,1,32,1,32,128,256,128,256,1,80,8192,8,64,2,131072,1,TRUE,FALSE,0,FALSE,2048
-Qwen/Qwen2.5-32B-Instruct,1,4352,128,2,65527752704,2,2,61.02747536,5.5,10,1,1,128,1,32,1,32,128,256,128,256,1,64,5120,8,40,2,32768,1,TRUE,FALSE,0,FALSE,2048
-Qwen/Qwen2.5-72B-Instruct,4,4352,512,2,1.45412E+11,2,2,135.4258575,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,80,8192,8,64,2,32768,1,TRUE,FALSE,0,FALSE,2048
-Qwen/Qwen2.5-7B-Instruct,1,4352,128,2,15231233024,2,2,14.18519115,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,28,3584,4,28,2,32768,1,TRUE,FALSE,0,FALSE,2048
-Qwen/Qwen2.5-32B-Instruct,1,4352,128,2,65527752704,2,2,61.02747536,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,64,5120,8,40,2,32768,1,TRUE,FALSE,0,FALSE,2048
-meta-llama/Llama-3.2-11B-Vision-Instruct,1,8448,128,2,21340441670,2,2,19.87483507,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,40,4096,8,32,2,131072,1,TRUE,FALSE,0,FALSE,2048
-meta-llama/Llama-3.2-90B-Vision-Instruct,4,8448,512,2,177186710646,2,2,165.0179835,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,100,8192,8,64,2,131072,1,TRUE,FALSE,0,FALSE,2048
-ibm-granite/granite-8b-code-instruct-4k,1,2048,128,2,21474836480,2,2,20.00000000,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,64,5120,8,40,2,32768,1,TRUE,FALSE,0,FALSE,2048
-ibm-granite/granite-20b-code-instruct-8k,1,2048,128,2,53687091200,2,2,48.00000000,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,80,8192,16,80,2,65536,1,TRUE,FALSE,0,FALSE,2048
-Qwen/Qwen2.5-VL-7B-Instruct,1,8448,128,2,15231233024,2,2,14.18519115,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,28,3584,4,28,2,32768,1,TRUE,FALSE,0,FALSE,2048
+MODEL,TENSOR_PARALLEL_SIZE,MAX_MODEL_LEN,TOTAL_GPU_MEM,UNAVAILABLE_MEM_ABS,MODEL_MEM_FROM_CONFIG,MODEL_DTYPE,QUANT_DTYPE,MODEL_MEM,PROFILER_MEM_OVERHEAD,APPROX_MEM_PER_GRAPH_MB,fsdpa,GPU_FREE_MEM_TARGET,BLOCK_SIZE,VLLM_PROMPT_BS_BUCKET_MIN,VLLM_PROMPT_BS_BUCKET_STEP,VLLM_DECODE_BS_BUCKET_MIN,VLLM_DECODE_BS_BUCKET_STEP,VLLM_PROMPT_SEQ_BUCKET_MIN,VLLM_PROMPT_SEQ_BUCKET_STEP,VLLM_DECODE_BLOCK_BUCKET_MIN,VLLM_DECODE_BLOCK_BUCKET_STEP,MAX_NUM_PREFILL_SEQS,NUM_HIDDEN_LAYERS,HIDDEN_SIZE,NUM_KEY_VALUE_HEADS,NUM_ATTENTION_HEADS,CACHE_DTYPE_BYTES,LIMIT_MODEL_LEN,PT_HPU_LAZY_MODE,VLLM_DELAYED_SAMPLING,VLLM_SKIP_WARMUP,EXPERIMENTAL_WEIGHT_SHARING,VLLM_EXPONENTIAL_BUCKETING,MAX_NUM_BATCHED_TOKENS,VLLM_CONTIGUOUS_PA,VLLM_DEFRAG,ASYNC_SCHEDULING,VLLM_WEIGHT_LOAD_FORCE_SYNC
+meta-llama/Llama-3.1-8B-Instruct,1,4352,128,2,16060522496,2,2,14.95752716,5.5,10,1,1,128,1,32,1,32,128,256,128,256,1,32,4096,8,32,2,131072,1,TRUE,FALSE,0,FALSE,2048,true,true,0,0
+meta-llama/Llama-3.1-70B-Instruct,4,4352,512,2,1.41107E+11,2,2,131.4165192,5.5,20,1,1,128,1,32,1,32,128,256,128,256,1,80,8192,8,64,2,131072,1,TRUE,FALSE,0,FALSE,2048,true,true,0,0
+meta-llama/Llama-3.3-70B-Instruct,4,4352,512,2,1.41107E+11,2,2,131.4165192,5.5,20,1,1,128,1,32,1,32,128,256,128,256,1,80,8192,8,64,2,131072,1,TRUE,FALSE,0,FALSE,2048,true,true,0,0
+meta-llama/Llama-3.2-1B-Instruct,1,4352,128,2,2471645608,2,2,2.301899351,5.5,10,1,1,128,1,32,1,32,128,256,128,256,1,16,2048,8,32,2,131072,1,TRUE,FALSE,0,FALSE,2048,true,true,0,0
+meta-llama/Llama-3.2-3B-Instruct,1,4352,128,2,6425499648,2,2,5.984212875,5.5,10,1,1,128,1,32,1,32,128,256,128,256,1,28,3072,8,24,2,131072,1,TRUE,FALSE,0,FALSE,2048,true,true,0,0
+mistralai/Mixtral-8x7B-Instruct-v0.1,2,4352,256,2,93405585408,2,2,86.99073029,5.5,10,1,1,128,1,32,1,32,128,256,128,256,1,32,4096,8,32,2,32768,1,TRUE,FALSE,0,FALSE,2048,false,false,0,0
+mistralai/Mixtral-8x22B-Instruct-v0.1,4,4352,512,2,2.8126E+11,2,2,261.9439201,5.5,10,1,1,128,1,32,1,32,128,256,128,256,1,56,6144,8,48,2,65536,1,TRUE,FALSE,0,FALSE,2048,false,false,0,0
+mistralai/Mistral-7B-Instruct-v0.2,1,4352,128,2,14483464192,2,2,13.48877716,5.5,10,1,1,128,1,32,1,32,128,256,128,256,1,32,4096,8,32,2,32768,1,TRUE,FALSE,0,FALSE,2048,false,false,0,0
+meta-llama/Llama-3.1-405B-Instruct,8,4352,1024,2,8.11707E+11,2,2,755.9608459,5.5,20,1,1,128,1,32,1,32,128,256,128,256,1,126,16384,8,128,2,131072,1,TRUE,FALSE,0,FALSE,2048,true,true,0,1
+Qwen/Qwen2.5-14B-Instruct,1,4352,128,2,29540067328,2,2,27.51133156,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,48,5120,8,40,2,32768,1,TRUE,FALSE,0,FALSE,2048,false,false,0,0
+deepseek-ai/DeepSeek-R1-Distill-Llama-70B,4,4352,512,2,1.41107E+11,2,2,131.4165192,5.5,20,1,1,128,1,32,1,32,128,256,128,256,1,80,8192,8,64,2,131072,1,TRUE,FALSE,0,FALSE,2048,true,true,0,0
+Qwen/Qwen2.5-32B-Instruct,1,4352,128,2,65527752704,2,2,61.02747536,5.5,10,1,1,128,1,32,1,32,128,256,128,256,1,64,5120,8,40,2,32768,1,TRUE,FALSE,0,FALSE,2048,false,false,0,0
+Qwen/Qwen2.5-72B-Instruct,4,4352,512,2,1.45412E+11,2,2,135.4258575,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,80,8192,8,64,2,32768,1,TRUE,FALSE,0,FALSE,2048,false,false,0,0
+Qwen/Qwen2.5-7B-Instruct,1,4352,128,2,15231233024,2,2,14.18519115,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,28,3584,4,28,2,32768,1,TRUE,FALSE,0,FALSE,2048,false,false,0,0
+Qwen/Qwen2.5-32B-Instruct,1,4352,128,2,65527752704,2,2,61.02747536,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,64,5120,8,40,2,32768,1,TRUE,FALSE,0,FALSE,2048,false,false,0,0
+ibm-granite/granite-8b-code-instruct-4k,1,4096,128,2,21474836480,2,2,20.00000000,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,36,4096,8,32,2,32768,1,TRUE,FALSE,0,FALSE,2048,true,true,0,0
+ibm-granite/granite-20b-code-instruct-8k,1,4352,128,2,53687091200,2,2,48.00000000,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,52,6144,1,48,2,65536,1,TRUE,FALSE,0,FALSE,2048,true,true,0,0
+Qwen/Qwen2.5-VL-7B-Instruct,1,8448,128,2,15231233024,2,2,14.18519115,5.5,10,0,3,128,1,32,1,32,128,256,128,256,1,28,3584,4,28,2,32768,1,TRUE,FALSE,0,FALSE,2048,false,false,0,0

.cd/server/vllm_autocalc_rules.py

@@ -82,8 +82,12 @@ def calc_DECODE_BLOCK_STEP_GRAPHS(ctx):
 
 def calc_NUM_DECODE_GRAPHS(ctx):
     # 3d update
-    return ((ctx['DECODE_BS_RAMP_GRAPHS'] + ctx['DECODE_BS_STEP_GRAPHS']) *
-            (ctx['DECODE_BLOCK_RAMP_GRAPHS'] + ctx['DECODE_BLOCK_STEP_GRAPHS'])) / 2
+    decode_graphs = ((ctx['DECODE_BS_RAMP_GRAPHS'] + ctx['DECODE_BS_STEP_GRAPHS']) *
+                     (ctx['DECODE_BLOCK_RAMP_GRAPHS'] + ctx['DECODE_BLOCK_STEP_GRAPHS']))
+    if ctx['VLLM_CONTIGUOUS_PA']:
+        return decode_graphs
+    else:
+        return decode_graphs / 2
 
 
 def calc_PROMPT_BS_RAMP_GRAPHS(ctx):

.cd/templates/template_vllm_benchmark.sh

@@ -3,7 +3,7 @@
 #@VARS
 
 # Wait for vLLM server to be ready
-until curl -s http://localhost:8000${ENDPOINT} > /dev/null; do
+until curl -s http://localhost:8000/v1/models > /dev/null; do
     echo "Waiting for vLLM server to be ready..."
     sleep 15
 done
@@ -35,4 +35,4 @@ vllm bench serve \
                 --metric-percentiles 90 \
                 --ignore-eos \
                 --trust-remote-code \
-2>&1 | tee -a logs/perftest_inp${INPUT_TOK}_out${OUTPUT_TOK}_user${CONCURRENT_REQ}.log
+2>&1 | tee -a logs/perftest_inp${INPUT_TOK}_out${OUTPUT_TOK}_user${CONCURRENT_REQ}.log

.cd/templates/template_vllm_server.sh

@@ -2,6 +2,10 @@
 
 #@VARS
 
+if [ $ASYNC_SCHEDULING -gt 0 ]; then # Checks if using async scheduling
+    EXTRA_ARGS+=" --async_scheduling"
+fi
+
 ## Start server
 vllm serve $MODEL \
         --block-size $BLOCK_SIZE \
@@ -11,5 +15,7 @@ vllm serve $MODEL \
         --max-model-len $MAX_MODEL_LEN \
         --gpu-memory-utilization $GPU_MEM_UTILIZATION \
         --max-num-seqs $MAX_NUM_SEQS \
-        --disable-log-requests \
+        --generation-config vllm \
+        --max_num_batched_tokens $MAX_NUM_BATCHED_TOKENS \
+        --disable-log-requests ${EXTRA_ARGS} \
 2>&1 | tee -a  logs/vllm_server.log