Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix wrong benchmark data associated with commits #6366

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Fix wrong benchmark data associated with commits #6366

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
09d9093
Fix wrong benchmark data associated with commits
huydhn Mar 5, 2025
f31b6c3
Merge branch 'main' into fix-wrong-data-commit
huydhn Mar 5, 2025
be7c913
Clean up some TODO
huydhn Mar 6, 2025
34b071a
Remove debug message
huydhn Mar 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 30 additions & 31 deletions torchci/components/benchmark/llms/components/LLMsGraphPanel.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
WORKFLOW_ID_TO_COMMIT,
} from "components/benchmark/BranchAndCommitPicker";
import { TIME_FIELD_NAME } from "components/benchmark/common";
import _ from "lodash";

import {
Granularity,
Expand Down Expand Up @@ -151,26 +152,7 @@ export default function LLMsGraphPanel({

if (repoName === "vllm-project/vllm") {
let requestRate = record.extra!["request_rate"];
// TODO (huydhn): Fix the invalid JSON on vLLM side
if (
metric.includes("itl") ||
metric.includes("tpot") ||
metric.includes("ttft")
) {
requestRate = requestRate !== "" ? requestRate : "Inf";
}

let tensorParallel = record.extra!["tensor_parallel_size"];
// TODO (huydhn): Fix the passing of tensor_parallel_size to the benchmark
// script on vLLM side
if (model.includes("8B")) {
tensorParallel = tensorParallel !== "" ? tensorParallel : "1";
} else if (model.includes("70B")) {
tensorParallel = tensorParallel !== "" ? tensorParallel : "4";
} else if (model.includes("8x7B")) {
tensorParallel = tensorParallel !== "" ? tensorParallel : "2";
}

if (requestRate !== "") {
record.display = `${model} / tp${tensorParallel} / qps_${requestRate}`;
} else {
Expand Down Expand Up @@ -207,9 +189,17 @@ export default function LLMsGraphPanel({
);
});

const availableMetric =
metricNames.find((metric) => chartData[metric].length !== 0) ??
metricNames[0];
// This is the detail table below the charts
const tableData: { [k: string]: any } = {};
Object.keys(chartData).forEach((metric: string) => {
chartData[metric].forEach((data: LLMsBenchmarkData) => {
const commit = WORKFLOW_ID_TO_COMMIT[data.workflow_id];
if (!(commit in tableData)) {
tableData[commit] = [];
}
tableData[commit].push(data);
});
});

return (
<>
Expand Down Expand Up @@ -262,19 +252,19 @@ export default function LLMsGraphPanel({
<th key={metric}>
{chartData[metric].length !== 0
? metric in METRIC_DISPLAY_SHORT_HEADERS
? METRIC_DISPLAY_SHORT_HEADERS[metric]
: metric
? `${METRIC_DISPLAY_SHORT_HEADERS[metric]} |`
: `${metric} |`
: ""}
</th>
))}
</tr>
</thead>
<tbody>
{chartData[availableMetric].map((entry: any, index: number) => {
let commit = WORKFLOW_ID_TO_COMMIT[entry.workflow_id];
{Object.keys(tableData).map((commit: string) => {
const commitData = tableData[commit];
return (
<tr key={index}>
<td>{entry.granularity_bucket}</td>
<tr key={commit}>
<td>{commitData[0].granularity_bucket}</td>
<td>
<code>
<a
Expand All @@ -288,9 +278,18 @@ export default function LLMsGraphPanel({
{metricNames
.filter((metric) => chartData[metric].length !== 0)
.map((metric: string) => (
<td key={`${metric}-${index}`}>
{chartData[metric][index] !== undefined
? chartData[metric][index].actual
<td key={`${metric}-${commit}`}>
{commitData.filter(
(data: LLMsBenchmarkData) => data.metric === metric
).length !== 0
? _.mean(
commitData
.filter(
(data: LLMsBenchmarkData) =>
data.metric === metric
)
.map((data: LLMsBenchmarkData) => data.actual)
).toFixed(2)
: ""}
</td>
))}
Expand Down
30 changes: 0 additions & 30 deletions torchci/lib/benchmark/llms/utils/llmUtils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,36 +238,6 @@ export function combineLeftAndRight(
if (repoName === "vllm-project/vllm") {
// These fields are only available on vLLM benchmark
const extraInfo = JSON.parse(extra);
// TODO (huydhn): Fix the invalid JSON on vLLM side
if (
metric.includes("itl") ||
metric.includes("tpot") ||
metric.includes("ttft")
) {
extraInfo["request_rate"] =
extraInfo["request_rate"] !== ""
? extraInfo["request_rate"]
: "Inf";
}
// TODO (huydhn): Fix the passing of tensor_parallel_size to the benchmark
// script on vLLM side
if (model.includes("8B")) {
extraInfo["tensor_parallel_size"] =
extraInfo["tensor_parallel_size"] !== ""
? extraInfo["tensor_parallel_size"]
: 1;
} else if (model.includes("70B")) {
extraInfo["tensor_parallel_size"] =
extraInfo["tensor_parallel_size"] !== ""
? extraInfo["tensor_parallel_size"]
: 4;
} else if (model.includes("8x7B")) {
extraInfo["tensor_parallel_size"] =
extraInfo["tensor_parallel_size"] !== ""
? extraInfo["tensor_parallel_size"]
: 2;
}

row["extra"] = extraInfo;
row["tensor_parallel_size"] = extraInfo["tensor_parallel_size"];
row["request_rate"] = extraInfo["request_rate"];
Expand Down