From 09d9093b94d15ddac29fd54fc55928d1eead0948 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 5 Mar 2025 12:56:44 -0800 Subject: [PATCH 1/3] Fix wrong benchmark data associated with commits --- .../components/benchmark/llms/components/LLMsGraphPanel.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/torchci/components/benchmark/llms/components/LLMsGraphPanel.tsx b/torchci/components/benchmark/llms/components/LLMsGraphPanel.tsx index 614d820723..471e64ab70 100644 --- a/torchci/components/benchmark/llms/components/LLMsGraphPanel.tsx +++ b/torchci/components/benchmark/llms/components/LLMsGraphPanel.tsx @@ -289,7 +289,9 @@ export default function LLMsGraphPanel({ .filter((metric) => chartData[metric].length !== 0) .map((metric: string) => ( - {chartData[metric][index] !== undefined + {chartData[metric][index] !== undefined && + chartData[metric][index].workflow_id === + entry.workflow_id ? chartData[metric][index].actual : ""} From be7c91390f621eba80c811e1cfd5278b8f015ff5 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 5 Mar 2025 16:12:17 -0800 Subject: [PATCH 2/3] Clean up some TODO --- .../llms/components/LLMsGraphPanel.tsx | 69 ++++++++++--------- torchci/lib/benchmark/llms/utils/llmUtils.ts | 30 -------- 2 files changed, 36 insertions(+), 63 deletions(-) diff --git a/torchci/components/benchmark/llms/components/LLMsGraphPanel.tsx b/torchci/components/benchmark/llms/components/LLMsGraphPanel.tsx index 471e64ab70..dae52729e7 100644 --- a/torchci/components/benchmark/llms/components/LLMsGraphPanel.tsx +++ b/torchci/components/benchmark/llms/components/LLMsGraphPanel.tsx @@ -4,6 +4,7 @@ import { WORKFLOW_ID_TO_COMMIT, } from "components/benchmark/BranchAndCommitPicker"; import { TIME_FIELD_NAME } from "components/benchmark/common"; +import _ from "lodash"; import { Granularity, @@ -151,26 +152,7 @@ export default function LLMsGraphPanel({ if (repoName === "vllm-project/vllm") { let requestRate = record.extra!["request_rate"]; - // TODO (huydhn): Fix the invalid JSON on vLLM side - if ( - metric.includes("itl") || - metric.includes("tpot") || - metric.includes("ttft") - ) { - requestRate = requestRate !== "" ? requestRate : "Inf"; - } - let tensorParallel = record.extra!["tensor_parallel_size"]; - // TODO (huydhn): Fix the passing of tensor_parallel_size to the benchmark - // script on vLLM side - if (model.includes("8B")) { - tensorParallel = tensorParallel !== "" ? tensorParallel : "1"; - } else if (model.includes("70B")) { - tensorParallel = tensorParallel !== "" ? tensorParallel : "4"; - } else if (model.includes("8x7B")) { - tensorParallel = tensorParallel !== "" ? tensorParallel : "2"; - } - if (requestRate !== "") { record.display = `${model} / tp${tensorParallel} / qps_${requestRate}`; } else { @@ -207,9 +189,23 @@ export default function LLMsGraphPanel({ ); }); - const availableMetric = - metricNames.find((metric) => chartData[metric].length !== 0) ?? - metricNames[0]; + // This is the detail table below the charts + const tableData: { [k: string]: any } = {}; + Object.keys(chartData).forEach((metric: string) => { + chartData[metric].forEach((data: LLMsBenchmarkData) => { + const commit = WORKFLOW_ID_TO_COMMIT[data.workflow_id]; + if (!(commit in tableData)) { + tableData[commit] = []; + } + tableData[commit].push(data); + }); + }); + console.log(chartData); + //if ("6946094f6fe9ff77ca02736567ddf2fc031d5fd300ce7c60d832cf9c301c4c48" in tableData) { + // console.log( tableData["6946094f6fe9ff77ca02736567ddf2fc031d5fd300ce7c60d832cf9c301c4c48"].filter( + // (data: LLMsBenchmarkData) => data.metric === "autoquant_vs_compile_speedup" + // )); + //} return ( <> @@ -262,19 +258,19 @@ export default function LLMsGraphPanel({ {chartData[metric].length !== 0 ? metric in METRIC_DISPLAY_SHORT_HEADERS - ? METRIC_DISPLAY_SHORT_HEADERS[metric] - : metric + ? `${METRIC_DISPLAY_SHORT_HEADERS[metric]} |` + : `${metric} |` : ""} ))} - {chartData[availableMetric].map((entry: any, index: number) => { - let commit = WORKFLOW_ID_TO_COMMIT[entry.workflow_id]; + {Object.keys(tableData).map((commit: string) => { + const commitData = tableData[commit]; return ( - - {entry.granularity_bucket} + + {commitData[0].granularity_bucket} chartData[metric].length !== 0) .map((metric: string) => ( - - {chartData[metric][index] !== undefined && - chartData[metric][index].workflow_id === - entry.workflow_id - ? chartData[metric][index].actual + + {commitData.filter( + (data: LLMsBenchmarkData) => data.metric === metric + ).length !== 0 + ? _.mean( + commitData + .filter( + (data: LLMsBenchmarkData) => + data.metric === metric + ) + .map((data: LLMsBenchmarkData) => data.actual) + ).toFixed(2) : ""} ))} diff --git a/torchci/lib/benchmark/llms/utils/llmUtils.ts b/torchci/lib/benchmark/llms/utils/llmUtils.ts index b8ebc04b47..74ae7664b7 100644 --- a/torchci/lib/benchmark/llms/utils/llmUtils.ts +++ b/torchci/lib/benchmark/llms/utils/llmUtils.ts @@ -238,36 +238,6 @@ export function combineLeftAndRight( if (repoName === "vllm-project/vllm") { // These fields are only available on vLLM benchmark const extraInfo = JSON.parse(extra); - // TODO (huydhn): Fix the invalid JSON on vLLM side - if ( - metric.includes("itl") || - metric.includes("tpot") || - metric.includes("ttft") - ) { - extraInfo["request_rate"] = - extraInfo["request_rate"] !== "" - ? extraInfo["request_rate"] - : "Inf"; - } - // TODO (huydhn): Fix the passing of tensor_parallel_size to the benchmark - // script on vLLM side - if (model.includes("8B")) { - extraInfo["tensor_parallel_size"] = - extraInfo["tensor_parallel_size"] !== "" - ? extraInfo["tensor_parallel_size"] - : 1; - } else if (model.includes("70B")) { - extraInfo["tensor_parallel_size"] = - extraInfo["tensor_parallel_size"] !== "" - ? extraInfo["tensor_parallel_size"] - : 4; - } else if (model.includes("8x7B")) { - extraInfo["tensor_parallel_size"] = - extraInfo["tensor_parallel_size"] !== "" - ? extraInfo["tensor_parallel_size"] - : 2; - } - row["extra"] = extraInfo; row["tensor_parallel_size"] = extraInfo["tensor_parallel_size"]; row["request_rate"] = extraInfo["request_rate"]; From 34b071a8947aac9bd8a77f5bb5c111333aded3d1 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 5 Mar 2025 16:26:50 -0800 Subject: [PATCH 3/3] Remove debug message --- .../components/benchmark/llms/components/LLMsGraphPanel.tsx | 6 ------ 1 file changed, 6 deletions(-) diff --git a/torchci/components/benchmark/llms/components/LLMsGraphPanel.tsx b/torchci/components/benchmark/llms/components/LLMsGraphPanel.tsx index dae52729e7..28180a3e3e 100644 --- a/torchci/components/benchmark/llms/components/LLMsGraphPanel.tsx +++ b/torchci/components/benchmark/llms/components/LLMsGraphPanel.tsx @@ -200,12 +200,6 @@ export default function LLMsGraphPanel({ tableData[commit].push(data); }); }); - console.log(chartData); - //if ("6946094f6fe9ff77ca02736567ddf2fc031d5fd300ce7c60d832cf9c301c4c48" in tableData) { - // console.log( tableData["6946094f6fe9ff77ca02736567ddf2fc031d5fd300ce7c60d832cf9c301c4c48"].filter( - // (data: LLMsBenchmarkData) => data.metric === "autoquant_vs_compile_speedup" - // )); - //} return ( <>