decapoda-research_llama-13b-hf-a100-sxm-80gb-w16a16e16-tp1-pp1-dp1-sp1-fe0.7-hbme0.9-summary-readable.txt

----------------------------------------------Summary-----------------------------------------------
batch_size_per_gpu: 1
seq_len: 512
tp_size: 1
pp_size: 1
num_tokens_to_generate: 32
use_kv_cache: True
kv_cache_memory_per_gpu: 445.64 MB
kv_cache_latency: 485.69 us
layernorm_dtype_bytes: 2
embedding_memory_per_gpu: 327.68 MB
weight_memory_per_gpu: 25.49 GB
prefill_max_batch_size_per_gpu: 10
prefill_activation_memory_per_gpu: 5.03 GB
prefill_num_flops_fwd_total: 13.27 T
decode_max_batch_size_per_gpu: 120
decode_activation_memory_per_gpu: 6.56 MB
decode_num_flops_fwd_total: 25.49 G
prefill_latency: 61.38 ms
prefill_latency_fwd_attn: 20.65 ms
prefill_latency_fwd_mlp: 39.33 ms
prefill_latency_fwd_layernorm: 457.12 us
prefill_latency_fwd_tp_comm: 0.0 us
prefill_latency_fwd_input_embedding: 178.56 us
prefill_latency_fwd_output_embedding_loss: 768.19 us
decode_latency: 14.38 ms
decode_latency_fwd_attn: 4.57 ms
decode_latency_fwd_mlp: 9.14 ms
decode_latency_fwd_layernorm: 0.89 us
decode_latency_fwd_tp_comm: 0.0 us
decode_latency_fwd_input_embedding: 178.56 us
decode_latency_fwd_output_embedding_loss: 1.5 us
total_decode_latency: 460.27 ms
total_latency: 521.65 ms
total_per_token_latency: 16.3 ms
----------------------------------------------------------------------------------------------------