decapoda-research_llama-30b-hf-a100-sxm-80gb-w16a16e16-tp1-pp1-dp1-sp1-fe0.7-hbme0.9-summary.json

{
    "batch_size_per_gpu": 1,
    "seq_len": 512,
    "tp_size": 1,
    "pp_size": 1,
    "num_tokens_to_generate": 32,
    "use_kv_cache": true,
    "kv_cache_memory_per_gpu": 869007360.0,
    "kv_cache_latency": 0.0009470953735491252,
    "layernorm_dtype_bytes": 2,
    "embedding_memory_per_gpu": 425984000.0,
    "weight_memory_per_gpu": 64221347840.0,
    "prefill_max_batch_size_per_gpu": 1,
    "prefill_activation_memory_per_gpu": 9814671360.0,
    "prefill_num_flops_fwd_total": 33300089405440,
    "decode_max_batch_size_per_gpu": 17,
    "decode_activation_memory_per_gpu": 12792000.0,
    "decode_num_flops_fwd_total": 64222945280,
    "prefill_latency": 0.1535964519380898,
    "prefill_latency_fwd_attn": 0.051769695085714294,
    "prefill_latency_fwd_mlp": 0.09970459794285716,
    "prefill_latency_fwd_layernorm": 0.000891383880987412,
    "prefill_latency_fwd_tp_comm": 0,
    "prefill_latency_fwd_input_embedding": 0.00023213121900713855,
    "prefill_latency_fwd_output_embedding_loss": 0.0009986438095238098,
    "decode_latency": 0.0359529896566166,
    "decode_latency_fwd_attn": 0.011590173483733854,
    "decode_latency_fwd_mlp": 0.023179898119993457,
    "decode_latency_fwd_layernorm": 1.740984142553539e-06,
    "decode_latency_fwd_tp_comm": 0,
    "decode_latency_fwd_input_embedding": 0.00023213121900713855,
    "decode_latency_fwd_output_embedding_loss": 1.950476190476191e-06,
    "total_decode_latency": 1.1504956690117312,
    "total_latency": 1.304092120949821,
    "total_per_token_latency": 0.04075287877968191
}