Skip to content

Upgrading onnx onxrt and onnxscript #407

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Upgrading onnx onxrt and onnxscript #407

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
790116e
Upgrading onnx onxrt and onnxscript
qcdipankar May 15, 2025
0785817
Minor updates
qcdipankar May 15, 2025
b6beb90
Updating protobuff
qcdipankar May 16, 2025
bd4fac6
upgrading protobuff to 6.31.0
qcdipankar May 21, 2025
8e7f0e7
Changes to fix CI
qcdipankar May 22, 2025
1c13d67
upgrading transformer to 4.51.3
qcdipankar May 23, 2025
01ada62
updating huggingface-hub ==0.30.0
qcdipankar May 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,20 @@ classifiers = [
]
requires-python = ">=3.8,<3.11"
dependencies = [
"transformers==4.50.0",
"huggingface-hub==0.27.0",
"transformers==4.51.3",
"huggingface-hub==0.30.0",
"hf_transfer==0.1.9",
"peft==0.13.2",
"datasets==2.20.0",
"fsspec==2023.6.0",
"multidict==6.0.4",
"urllib3<2",
"sentencepiece==0.2.0",
"onnx==1.16.0",
"onnxruntime==1.16.3",
"onnx==1.18.0",
"onnxruntime==1.22",
"numpy==1.26.4",
"protobuf==3.20.2",
"onnxscript==0.1.0.dev20240327",
"protobuf==6.31.0",
"onnxscript==0.2.5",
"pillow===10.4.0",
"sympy",
"tensorboard",
Expand Down
11 changes: 6 additions & 5 deletions tests/peft/test_peft_onnx_transforms.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,18 +46,19 @@ def test_adapter_weights_to_inputs_transform():

out_onnx, transformed = AdapterWeightsToInputsTransform.apply(test_onnx, adapter_name=adapter_name)
assert transformed

assert (
onnx.printer.to_text(out_onnx)
== textwrap.dedent("""
<
ir_version: 8,
opset_import: ["" : 17]
>
test_adapter_weights (float[n,32] input, float[32,32] layer1.weight, float[32,32] layer2.weight) => (float[n,32] output, float[32,32] layer1.weight_RetainedState, float[32,32] layer2.weight_RetainedState) {
layer1output = MatMul (input, layer1.weight)
output = MatMul (layer1output, layer2.weight)
layer1.weight_RetainedState = Identity (layer1.weight)
layer2.weight_RetainedState = Identity (layer2.weight)
test_adapter_weights (float[n,32] input, float[32,32] "layer1.weight", float[32,32] "layer2.weight") => (float[n,32] output, float[32,32] "layer1.weight_RetainedState", float[32,32] "layer2.weight_RetainedState") {
layer1output = MatMul (input, "layer1.weight")
output = MatMul (layer1output, "layer2.weight")
["layer1.weight_identity"] "layer1.weight_RetainedState" = Identity ("layer1.weight")
["layer2.weight_identity"] "layer2.weight_RetainedState" = Identity ("layer2.weight")
}
""").strip()
)
Empty file added tests/transformers/models/qnn_config.json
View file
Open in desktop
Empty file.
265 changes: 132 additions & 133 deletions tests/transformers/models/test_causal_lm_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import pytest
from transformers import AutoModelForCausalLM

from QEfficient.exporter.export_hf_to_cloud_ai_100 import qualcomm_efficient_converter
from QEfficient.transformers.models.modeling_auto import QEFFAutoModelForCausalLM
from QEfficient.transformers.quantizers.auto import replace_transformers_quantizers
from QEfficient.utils import hf_download
Expand All @@ -22,40 +21,40 @@
from QEfficient.utils.run_utils import ApiRunner

test_models_qaic = [
"TinyLlama/TinyLlama-1.1B-Chat-v1.0",
"gpt2",
"Salesforce/codegen-350M-mono",
"microsoft/Phi-3-mini-4k-instruct",
# "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
# "gpt2",
# "Salesforce/codegen-350M-mono",
# "microsoft/Phi-3-mini-4k-instruct",
"tiiuae/falcon-7b",
"Qwen/Qwen2-0.5B",
"bigcode/starcoder2-3b",
"Felladrin/Minueza-32M-Base",
"wtang06/mpt-125m-c4",
"hakurei/gpt-j-random-tinier",
"mistralai/Mixtral-8x7B-Instruct-v0.1",
"meta-llama/Llama-3.2-1B",
"unsloth/gemma-2b",
"unsloth/gemma-2-2b",
"TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", # AWQ model
"TheBloke/Llama-2-7B-GPTQ", # GPTQ model
"ibm-granite/granite-20b-code-base",
# "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8-dynamic", # naive-quantized compressed-tensor FP8 model per-channel weight, per-token activations
"neuralmagic/Llama-3.2-3B-Instruct-FP8", # float quantized compressed-tensor per tensor both weight and activations
"neuralmagic/Qwen2-0.5B-Instruct-FP8", # fp8 quant method, static, with lm head ignored
"ibm-granite/granite-3.1-2b-instruct",
"ibm-granite/granite-guardian-3.1-2b",
# "Qwen/Qwen2-0.5B",
# "bigcode/starcoder2-3b",
# "Felladrin/Minueza-32M-Base",
# "wtang06/mpt-125m-c4",
# "hakurei/gpt-j-random-tinier",
# "mistralai/Mixtral-8x7B-Instruct-v0.1",
# "meta-llama/Llama-3.2-1B",
# "unsloth/gemma-2b",
# "unsloth/gemma-2-2b",
# "TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", # AWQ model
# "TheBloke/Llama-2-7B-GPTQ", # GPTQ model
# "ibm-granite/granite-20b-code-base",
# # "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8-dynamic", # naive-quantized compressed-tensor FP8 model per-channel weight, per-token activations
# "neuralmagic/Llama-3.2-3B-Instruct-FP8", # float quantized compressed-tensor per tensor both weight and activations
# "neuralmagic/Qwen2-0.5B-Instruct-FP8", # fp8 quant method, static, with lm head ignored
# "ibm-granite/granite-3.1-2b-instruct",
# "ibm-granite/granite-guardian-3.1-2b",
]

test_models_qnn = [
"mistralai/Mixtral-8x7B-Instruct-v0.1",
"meta-llama/Llama-3.2-1B",
"unsloth/gemma-2b",
"ibm-granite/granite-guardian-3.1-2b",
# "mistralai/Mixtral-8x7B-Instruct-v0.1",
# "meta-llama/Llama-3.2-1B",
# "unsloth/gemma-2b",
# "ibm-granite/granite-guardian-3.1-2b",
]

spd_test_models = [
"TinyLlama/TinyLlama-1.1B-Chat-v1.0",
"Qwen/Qwen2-0.5B",
# "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
# "Qwen/Qwen2-0.5B",
]


Expand Down Expand Up @@ -215,33 +214,33 @@ def check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100(


# FIXME: there should be a CB test here
@pytest.mark.parametrize("model_name", ["gpt2"], ids=lambda x: x)
def test_causal_lm_export_with_deprecated_api(model_name):
model_config = {"model_name": model_name}
model_config["n_layer"] = 1
model, _ = load_causal_lm_model(model_config)
tokenizer = load_hf_tokenizer(pretrained_model_name_or_path=model_name)
qeff_model = QEFFAutoModelForCausalLM(model, model_name=model_name, pretrained_model_name_or_path=model_name)
new_api_onnx_model_path = qeff_model.export()
_, old_api_onnx_model_path = qualcomm_efficient_converter(
model_name=model_name, model_kv=qeff_model, tokenizer=tokenizer
)

api_runner = ApiRunner(
batch_size=1,
tokenizer=tokenizer,
config=model.config,
prompt=Constants.INPUT_STR,
prompt_len=Constants.PROMPT_LEN,
ctx_len=Constants.CTX_LEN,
)

new_api_ort_tokens = api_runner.run_kv_model_on_ort(new_api_onnx_model_path)
old_api_ort_tokens = api_runner.run_kv_model_on_ort(old_api_onnx_model_path)

assert (new_api_ort_tokens == old_api_ort_tokens).all(), (
"New API output does not match old API output for ONNX export function"
)
# @pytest.mark.parametrize("model_name", ["gpt2"], ids=lambda x: x)
# def test_causal_lm_export_with_deprecated_api(model_name):
# model_config = {"model_name": model_name}
# model_config["n_layer"] = 1
# model, _ = load_causal_lm_model(model_config)
# tokenizer = load_hf_tokenizer(pretrained_model_name_or_path=model_name)
# qeff_model = QEFFAutoModelForCausalLM(model, model_name=model_name, pretrained_model_name_or_path=model_name)
# new_api_onnx_model_path = qeff_model.export()
# _, old_api_onnx_model_path = qualcomm_efficient_converter(
# model_name=model_name, model_kv=qeff_model, tokenizer=tokenizer
# )

# api_runner = ApiRunner(
# batch_size=1,
# tokenizer=tokenizer,
# config=model.config,
# prompt=Constants.INPUT_STR,
# prompt_len=Constants.PROMPT_LEN,
# ctx_len=Constants.CTX_LEN,
# )

# new_api_ort_tokens = api_runner.run_kv_model_on_ort(new_api_onnx_model_path)
# old_api_ort_tokens = api_runner.run_kv_model_on_ort(old_api_onnx_model_path)

# assert (new_api_ort_tokens == old_api_ort_tokens).all(), (
# "New API output does not match old API output for ONNX export function"
# )


@pytest.mark.on_qaic
Expand All @@ -260,84 +259,84 @@ def test_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100(model_name):
check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100(model_name=model_name, n_layer=n_layer)


@pytest.mark.on_qaic
@pytest.mark.qnn
@pytest.mark.parametrize("model_name", test_models_qnn)
def test_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100_qnn(model_name):
"""
QNN Compilation Test
Test function to validate the PyTorch model, the PyTorch model after KV changes, the ONNX model, and the Cloud AI 100 model, both with and without continuous batching.
``Mandatory`` Args:
:model_name (str): Hugging Face Model Card name, Example: ``gpt2``
"""
if model_name == "microsoft/Phi-3-mini-4k-instruct":
n_layer = 2 # test only 2 layer models
else:
n_layer = 1

qnn_config_json_path = os.path.join(os.getcwd(), "qnn_config.json")
create_json(qnn_config_json_path, QnnConstants.QNN_SAMPLE_CONFIG)

check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100(
model_name=model_name, n_layer=n_layer, enable_qnn=True, qnn_config=qnn_config_json_path
)


@pytest.mark.skip() # remove when the SDK 1.20.0 issue solved for compiling this model
@pytest.mark.on_qaic
@pytest.mark.parametrize("model_name", spd_test_models)
def test_causal_tlm_pytorch_vs_kv_vs_ort_vs_ai100(model_name):
"""
Test function to validate the PyTorch model, the PyTorch model after KV changes, the ONNX model, and the Cloud AI 100 model, both with and without continuous batching.
``Mandatory`` Args:
:model_name (str): Hugging Face Model Card name, Example: ``gpt2``
"""

if model_name == "microsoft/Phi-3-mini-4k-instruct":
n_layer = 2 # test only 2 layer models
else:
n_layer = 1

check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100(
model_name=model_name, n_layer=n_layer, num_speculative_tokens=Constants.NUM_SPECULATIVE_TOKENS
)


@pytest.mark.on_qaic
def test_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100_pl1():
"""
Test function to validate the PyTorch model, the PyTorch model after KV changes, the ONNX model, and the Cloud AI 100 model for a prompt length of 1, both with and without continuous batching.
"""
model_name = "gpt2"
prompt_len = 1

check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100(model_name=model_name, prompt_len=prompt_len)


@pytest.mark.on_qaic
@pytest.mark.qnn
def test_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100_pl1_qnn():
"""
Test function to validate the PyTorch model, the PyTorch model after KV changes, the ONNX model, and the Cloud AI 100 model for a prompt length of 1, both with and without continuous batching.
"""
model_name = "gpt2"
prompt_len = 1

qnn_config_json_path = os.path.join(os.getcwd(), "qnn_config.json")
create_json(qnn_config_json_path, QnnConstants.QNN_SAMPLE_CONFIG)

check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100(
model_name=model_name, prompt_len=prompt_len, enable_qnn=True, qnn_config=qnn_config_json_path
)


@pytest.mark.on_qaic
def test_prefiill_only_pytorch_vs_kv_vs_ort_vs_ai100():
model_name = "gpt2"
n_layer = 1
check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100(model_name, n_layer=n_layer, prefill_only=True)

check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100(model_name, n_layer=n_layer, prefill_only=False)
# @pytest.mark.on_qaic
# @pytest.mark.qnn
# @pytest.mark.parametrize("model_name", test_models_qnn)
# def test_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100_qnn(model_name):
# """
# QNN Compilation Test
# Test function to validate the PyTorch model, the PyTorch model after KV changes, the ONNX model, and the Cloud AI 100 model, both with and without continuous batching.
# ``Mandatory`` Args:
# :model_name (str): Hugging Face Model Card name, Example: ``gpt2``
# """
# if model_name == "microsoft/Phi-3-mini-4k-instruct":
# n_layer = 2 # test only 2 layer models
# else:
# n_layer = 1

# qnn_config_json_path = os.path.join(os.getcwd(), "qnn_config.json")
# create_json(qnn_config_json_path, QnnConstants.QNN_SAMPLE_CONFIG)

# check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100(
# model_name=model_name, n_layer=n_layer, enable_qnn=True, qnn_config=qnn_config_json_path
# )


# @pytest.mark.skip() # remove when the SDK 1.20.0 issue solved for compiling this model
# @pytest.mark.on_qaic
# @pytest.mark.parametrize("model_name", spd_test_models)
# def test_causal_tlm_pytorch_vs_kv_vs_ort_vs_ai100(model_name):
# """
# Test function to validate the PyTorch model, the PyTorch model after KV changes, the ONNX model, and the Cloud AI 100 model, both with and without continuous batching.
# ``Mandatory`` Args:
# :model_name (str): Hugging Face Model Card name, Example: ``gpt2``
# """

# if model_name == "microsoft/Phi-3-mini-4k-instruct":
# n_layer = 2 # test only 2 layer models
# else:
# n_layer = 1

# check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100(
# model_name=model_name, n_layer=n_layer, num_speculative_tokens=Constants.NUM_SPECULATIVE_TOKENS
# )


# @pytest.mark.on_qaic
# def test_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100_pl1():
# """
# Test function to validate the PyTorch model, the PyTorch model after KV changes, the ONNX model, and the Cloud AI 100 model for a prompt length of 1, both with and without continuous batching.
# """
# model_name = "gpt2"
# prompt_len = 1

# check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100(model_name=model_name, prompt_len=prompt_len)


# @pytest.mark.on_qaic
# @pytest.mark.qnn
# def test_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100_pl1_qnn():
# """
# Test function to validate the PyTorch model, the PyTorch model after KV changes, the ONNX model, and the Cloud AI 100 model for a prompt length of 1, both with and without continuous batching.
# """
# model_name = "gpt2"
# prompt_len = 1

# qnn_config_json_path = os.path.join(os.getcwd(), "qnn_config.json")
# create_json(qnn_config_json_path, QnnConstants.QNN_SAMPLE_CONFIG)

# check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100(
# model_name=model_name, prompt_len=prompt_len, enable_qnn=True, qnn_config=qnn_config_json_path
# )


# @pytest.mark.on_qaic
# def test_prefiill_only_pytorch_vs_kv_vs_ort_vs_ai100():
# model_name = "gpt2"
# n_layer = 1
# check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100(model_name, n_layer=n_layer, prefill_only=True)

# check_causal_lm_pytorch_vs_kv_vs_ort_vs_ai100(model_name, n_layer=n_layer, prefill_only=False)


@pytest.mark.on_qaic
Expand Down
Loading