|
| 1 | +# ----------------------------------------------------------------------------- |
| 2 | +# |
| 3 | +# Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. |
| 4 | +# SPDX-License-Identifier: BSD-3-Clause |
| 5 | +# |
| 6 | +# ----------------------------------------------------------------------------- |
| 7 | + |
| 8 | +import torch |
| 9 | +import transformers |
| 10 | +from transformers import AutoConfig, AutoModelForImageTextToText, AutoProcessor, TextStreamer |
| 11 | + |
| 12 | +from QEfficient import QEFFAutoModelForImageTextToText |
| 13 | + |
| 14 | +model_id = "meta-llama/Llama-4-Scout-17B-16E-Instruct" |
| 15 | +config = AutoConfig.from_pretrained(model_id) |
| 16 | +# For Testing Purpose Only |
| 17 | +# config.text_config.num_hidden_layers = 1 |
| 18 | +# config.vision_config.num_hidden_layers = 2 |
| 19 | + |
| 20 | +model = AutoModelForImageTextToText.from_pretrained(model_id, attn_implementation="eager", config=config) |
| 21 | +model.eval() |
| 22 | + |
| 23 | +qeff_model = QEFFAutoModelForImageTextToText(model, kv_offload=True) |
| 24 | + |
| 25 | +# TODO: Map the Vision Encoder to FP16 Only and Disable MXFP6 For Better Accuracy. |
| 26 | +qeff_model.compile( |
| 27 | + prefill_seq_len=128, |
| 28 | + ctx_len=3072, |
| 29 | + img_size=336, |
| 30 | + num_cores=16, |
| 31 | + num_devices=8, |
| 32 | + batch_size_times_num_tiles=17, |
| 33 | + mxfp6_matmul=True, |
| 34 | + mxint8_kv_cache=True, |
| 35 | + aic_enable_depth_first=True, |
| 36 | + mos=1, |
| 37 | +) |
| 38 | + |
| 39 | +image_url = ( |
| 40 | + "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/cat_style_layout.png" |
| 41 | +) |
| 42 | + |
| 43 | +messages = [ |
| 44 | + { |
| 45 | + "role": "user", |
| 46 | + "content": [ |
| 47 | + {"type": "image", "url": image_url}, |
| 48 | + {"type": "text", "text": "Can you describe the image in detail."}, |
| 49 | + ], |
| 50 | + }, |
| 51 | +] |
| 52 | + |
| 53 | +tokenizer = transformers.AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) |
| 54 | +processor = AutoProcessor.from_pretrained(model_id) |
| 55 | +inputs = processor.apply_chat_template( |
| 56 | + messages, |
| 57 | + add_generation_prompt=True, |
| 58 | + tokenize=True, |
| 59 | + return_dict=True, |
| 60 | + return_tensors="pt", |
| 61 | +) |
| 62 | +inputs["pixel_values"] = inputs["pixel_values"].to(torch.float32) |
| 63 | +streamer = TextStreamer(tokenizer) |
| 64 | +output = qeff_model.generate(inputs=inputs, device_ids=[0, 1, 2, 3, 4, 5, 6, 7], generation_len=100) |
| 65 | +print(output.generated_ids) |
| 66 | +print(tokenizer.batch_decode(output.generated_ids)) |
| 67 | +print(output) |
0 commit comments