From 3e45dc1d4259c43525eab0349ea134abf54f34a3 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 11 Sep 2025 12:22:22 -0400 Subject: [PATCH 1/3] improve api --- .../query.sql | 41 +- .../compilers_benchmark_api_query/params.json | 3 +- .../compilers_benchmark_api_query/query.sql | 4 + .../compilers/SummaryGraphPanelV2.tsx | 433 ++++++++++++++++++ .../components/benchmark/compilers/common.tsx | 6 +- .../uiModules/UMDateRangePicker.tsx | 34 +- .../UtilizationReportPage.tsx | 8 +- .../compilers/get_compiler_benchmark_data.ts | 64 ++- .../api_helper/compilers/helpers/general.ts | 49 +- .../compilers/helpers/precompute.ts | 52 ++- .../benchmark/api_helper/compilers/type.ts | 108 +++++ torchci/lib/benchmark/api_helper/utils.ts | 10 +- .../pages/api/benchmark/get_time_series.ts | 18 +- torchci/pages/api/benchmark/list_commits.ts | 48 ++ .../pages/benchmark/compilers_regression.tsx | 193 ++++++++ 15 files changed, 965 insertions(+), 106 deletions(-) create mode 100644 torchci/components/benchmark/compilers/SummaryGraphPanelV2.tsx create mode 100644 torchci/pages/api/benchmark/list_commits.ts create mode 100644 torchci/pages/benchmark/compilers_regression.tsx diff --git a/torchci/clickhouse_queries/compilers_benchmark_api_commit_query/query.sql b/torchci/clickhouse_queries/compilers_benchmark_api_commit_query/query.sql index 02b47df11d..803144104f 100644 --- a/torchci/clickhouse_queries/compilers_benchmark_api_commit_query/query.sql +++ b/torchci/clickhouse_queries/compilers_benchmark_api_commit_query/query.sql @@ -1,27 +1,42 @@ SELECT DISTINCT replaceOne(head_branch, 'refs/heads/', '') AS branch, head_sha AS commit, - workflow_id AS id, - timestamp + workflow_id, + toDate(fromUnixTimestamp(timestamp), 'UTC') AS date FROM benchmark.oss_ci_benchmark_torchinductor PREWHERE timestamp >= toUnixTimestamp({startTime: DateTime64(3)}) - AND timestamp < toUnixTimestamp({stopTime: DateTime64(3)}) + AND timestamp < toUnixTimestamp({stopTime: DateTime64(3)}) WHERE + -- optional branches ( - has( - {branches: Array(String)}, - replaceOne(head_branch, 'refs/heads/', '') - ) + has({branches: Array(String)}, replaceOne(head_branch, 'refs/heads/', '')) OR empty({branches: Array(String)}) ) + -- optional suites AND ( - has({suites: Array(String) }, suite) - OR empty({suites: Array(String) }) + has({suites: Array(String)}, suite) + OR empty({suites: Array(String)}) + ) + -- optional dtype + AND ( + benchmark_dtype = {dtype: String} + OR empty({dtype: String}) + ) + -- optional mode + AND ( + benchmark_mode = {mode: String} + OR empty({mode: String}) + ) + -- optional device + AND ( + device = {device: String} + OR empty({device: String}) + ) + -- optional arch (array param); if empty array, skip filter + AND ( + multiSearchAnyCaseInsensitive(arch, {arch: Array(String)}) + OR empty({arch: Array(String)}) ) - AND benchmark_dtype = {dtype: String} - AND benchmark_mode = {mode: String} - AND device = {device: String} - AND multiSearchAnyCaseInsensitive(arch, {arch: Array(String)}) ORDER BY timestamp SETTINGS session_timezone = 'UTC'; diff --git a/torchci/clickhouse_queries/compilers_benchmark_api_query/params.json b/torchci/clickhouse_queries/compilers_benchmark_api_query/params.json index fe5be8efcf..5f56edb418 100644 --- a/torchci/clickhouse_queries/compilers_benchmark_api_query/params.json +++ b/torchci/clickhouse_queries/compilers_benchmark_api_query/params.json @@ -8,7 +8,8 @@ "dtype": "String", "granularity": "String", "mode": "String", - "suites": "Array(String)" + "suites": "Array(String)", + "models": "Array(String)" }, "tests": [] } diff --git a/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql b/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql index ca1edec659..6aa36faf8e 100644 --- a/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql +++ b/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql @@ -30,6 +30,10 @@ WHERE has({suites: Array(String) }, suite) OR empty({suites: Array(String) }) ) + AND ( + has({models: Array(String)}, model_name) + OR empty({models: Array(String) }) + ) AND benchmark_dtype = {dtype: String} AND benchmark_mode = {mode: String} AND device = {device: String} diff --git a/torchci/components/benchmark/compilers/SummaryGraphPanelV2.tsx b/torchci/components/benchmark/compilers/SummaryGraphPanelV2.tsx new file mode 100644 index 0000000000..3faab6f3cd --- /dev/null +++ b/torchci/components/benchmark/compilers/SummaryGraphPanelV2.tsx @@ -0,0 +1,433 @@ +import { Grid, Skeleton } from "@mui/material"; +import { COMMIT_TO_WORKFLOW_ID } from "components/benchmark/BranchAndCommitPicker"; +import { TIME_FIELD_NAME } from "components/benchmark/common"; +import { SUITES } from "components/benchmark/compilers/SuitePicker"; +import { + Granularity, + seriesWithInterpolatedTimes, + TimeSeriesPanelWithData, +} from "components/metrics/panels/TimeSeriesPanel"; +import dayjs from "dayjs"; +import { + computeCompilationTime, + computeExecutionTime, + computeGeomean, + computeMemoryCompressionRatio, + computePassrate, + computePeakMemoryUsage, + convertToCompilerPerformanceData, + getPassingModels, +} from "lib/benchmark/compilerUtils"; +import { fetcher } from "lib/GeneralUtils"; +import useSWR from "swr"; + +const GRAPH_ROW_HEIGHT = 245; + +export function GraphPanel({ + queryName, + queryParams, + granularity, + suite, + branch, + lCommit, + rCommit, +}: { + queryName: string; + queryParams: { [key: string]: any }; + granularity: Granularity; + suite: string; + branch: string; + lCommit: string; + rCommit: string; +}) { + // NB: I need to do multiple queries here for different suites to keep the response + // from the database small enough (<6MB) to fit into Vercel lambda limit + return ( + + ); +} + +function SuiteGraphPanel({ + queryName, + queryParams, + granularity, + suite, + branch, + lCommit, + rCommit, +}: { + queryName: string; + queryParams: { [key: string]: any }; + granularity: Granularity; + suite: string; + branch: string; + lCommit: string; + rCommit: string; +}) { + const queryParamsWithSuite: { [key: string]: any } = { + ...queryParams, + branches: [branch], + suites: [suite], + }; + // NB: Querying data for all the suites blows up the response from the database + // over the lambda reponse body limit of 6MB. So I need to split up the query + // here into multiple smaller ones to keep them under the limit + // + // See more: + // * https://nextjs.org/docs/messages/api-routes-body-size-limit + // * https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-limits.html + const url = `/api/clickhouse/${queryName}?parameters=${encodeURIComponent( + JSON.stringify(queryParamsWithSuite) + )}`; + + let { data, error } = useSWR(url, fetcher, { + refreshInterval: 60 * 60 * 1000, // refresh every hour + }); + + if (error !== undefined) { + return ( +
+ An error occurred while fetching data, perhaps there are too many + results with your choice of time range and granularity? +
+ ); + } + + if (data === undefined || data.length === 0) { + return ; + } + + data = convertToCompilerPerformanceData(data); + + // Clamp to the nearest granularity (e.g. nearest hour) so that the times will + // align with the data we get from the database + const startTime = dayjs(queryParams["startTime"]).startOf(granularity); + const stopTime = dayjs(queryParams["stopTime"]).startOf(granularity); + + // Compute the metrics for all passing models + const models = getPassingModels(data); + const groupByFieldName = "compiler"; + + // Only show records between these twos + const lWorkflowId = COMMIT_TO_WORKFLOW_ID[lCommit]; + const rWorkflowId = COMMIT_TO_WORKFLOW_ID[rCommit]; + + // Accuracy + const passrate = computePassrate(data, models).filter((r: any) => { + const id = r.workflow_id; + return ( + (id >= lWorkflowId && id <= rWorkflowId) || + (id <= lWorkflowId && id >= rWorkflowId) + ); + }); + const passrateSeries = seriesWithInterpolatedTimes( + passrate, + startTime, + stopTime, + granularity, + groupByFieldName, + TIME_FIELD_NAME, + "passrate", + false + ); + const totalModelCountSeries = seriesWithInterpolatedTimes( + passrate, + startTime, + stopTime, + granularity, + groupByFieldName, + TIME_FIELD_NAME, + "total_count", + false + ); + + // Geomean speedup + const geomean = computeGeomean(data, models).filter((r: any) => { + const id = r.workflow_id; + return ( + (id >= lWorkflowId && id <= rWorkflowId) || + (id <= lWorkflowId && id >= rWorkflowId) + ); + }); + const geomeanSeries = seriesWithInterpolatedTimes( + geomean, + startTime, + stopTime, + granularity, + groupByFieldName, + TIME_FIELD_NAME, + "geomean", + false + ); + + // Compilation time + const compTime = computeCompilationTime(data, models).filter((r: any) => { + const id = r.workflow_id; + return ( + (id >= lWorkflowId && id <= rWorkflowId) || + (id <= lWorkflowId && id >= rWorkflowId) + ); + }); + const compTimeSeries = seriesWithInterpolatedTimes( + compTime, + startTime, + stopTime, + granularity, + groupByFieldName, + TIME_FIELD_NAME, + "compilation_latency", + false + ); + + // Execution time + const executionTime = computeExecutionTime(data, models).filter((r: any) => { + const id = r.workflow_id; + return ( + (id >= lWorkflowId && id <= rWorkflowId) || + (id <= lWorkflowId && id >= rWorkflowId) + ); + }); + const executionTimeSeries = seriesWithInterpolatedTimes( + executionTime, + startTime, + stopTime, + granularity, + groupByFieldName, + TIME_FIELD_NAME, + "abs_latency", + false + ); + + // Memory compression ratio + const memory = computeMemoryCompressionRatio(data, models).filter( + (r: any) => { + const id = r.workflow_id; + return ( + (id >= lWorkflowId && id <= rWorkflowId) || + (id <= lWorkflowId && id >= rWorkflowId) + ); + } + ); + const memorySeries = seriesWithInterpolatedTimes( + memory, + startTime, + stopTime, + granularity, + groupByFieldName, + TIME_FIELD_NAME, + "compression_ratio", + false + ); + + // Dynamo peak memory usage + const peakMemory = computePeakMemoryUsage(data, models).filter((r: any) => { + const id = r.workflow_id; + return ( + (id >= lWorkflowId && id <= rWorkflowId) || + (id <= lWorkflowId && id >= rWorkflowId) + ); + }); + + const peakMemorySeries = seriesWithInterpolatedTimes( + peakMemory, + startTime, + stopTime, + granularity, + groupByFieldName, + TIME_FIELD_NAME, + "dynamo_peak_mem", + false + ); + + return ( + + + { + return `${(unit * 100).toFixed(0)} %`; + }} + additionalOptions={{ + yAxis: { + scale: true, + }, + label: { + show: true, + align: "left", + formatter: (r: any) => { + return (r.value[1] * 100).toFixed(0); + }, + }, + }} + legendPadding={310} + /> + + + + { + return `${unit}`; + }} + additionalOptions={{ + yAxis: { + scale: true, + }, + label: { + show: true, + align: "left", + formatter: (r: any) => { + return r.value[1]; + }, + }, + }} + legendPadding={310} + /> + + + + { + return `${unit}`; + }} + additionalOptions={{ + yAxis: { + scale: true, + min: 1.0, + }, + label: { + show: true, + align: "left", + formatter: (r: any) => { + return r.value[1]; + }, + }, + }} + legendPadding={310} + /> + + + + { + return `${unit}`; + }} + additionalOptions={{ + yAxis: { + scale: true, + }, + label: { + show: true, + align: "left", + formatter: (r: any) => { + return Number(r.value[1]).toFixed(0); + }, + }, + }} + legendPadding={310} + /> + + + + { + return `${unit}`; + }} + additionalOptions={{ + yAxis: { + scale: true, + }, + label: { + show: true, + align: "left", + formatter: (r: any) => { + return r.value[1]; + }, + }, + }} + legendPadding={310} + /> + + + + { + return `${unit}`; + }} + additionalOptions={{ + yAxis: { + scale: true, + }, + label: { + show: true, + align: "left", + formatter: (r: any) => { + return r.value[1]; + }, + }, + }} + legendPadding={310} + /> + + + + { + return `${unit}`; + }} + additionalOptions={{ + yAxis: { + scale: true, + }, + label: { + show: true, + align: "left", + formatter: (r: any) => { + return Number(r.value[1]).toFixed(0); + }, + }, + }} + legendPadding={310} + /> + + + ); +} diff --git a/torchci/components/benchmark/compilers/common.tsx b/torchci/components/benchmark/compilers/common.tsx index e5f0126f8e..a516f0a2a5 100644 --- a/torchci/components/benchmark/compilers/common.tsx +++ b/torchci/components/benchmark/compilers/common.tsx @@ -57,6 +57,9 @@ export const HELP_LINK = export const DTYPES = ["amp", "float16", "bfloat16", "quant", "notset"]; +export const DTYPES_V2 = ["amp", "float16", "bfloat16", "notset"]; +export const MODES_V2= ["training", "inference", "notset"]; + export const DEFAULT_DEVICE_NAME = "cuda (h100)"; export const DISPLAY_NAMES_TO_DEVICE_NAMES: { [k: string]: string } = { "cuda (a100)": "cuda", @@ -75,8 +78,7 @@ export const DISPLAY_NAMES_TO_ARCH_NAMES: { [k: string]: string } = { "cpu (x86_64)": "x86_64", "cpu (x86_zen)": "x86_zen", "cpu (aarch64)": "aarch64", - // TODO (huydhn): Figure out a way to get the GPU name for ROCm - "rocm (mi300x)": "", + "rocm (mi300x)": "mi300x", mps: "", }; export const DISPLAY_NAMES_TO_WORKFLOW_NAMES: { [k: string]: string } = { diff --git a/torchci/components/uiModules/UMDateRangePicker.tsx b/torchci/components/uiModules/UMDateRangePicker.tsx index bfe8db5b42..7c9a0c0ed9 100644 --- a/torchci/components/uiModules/UMDateRangePicker.tsx +++ b/torchci/components/uiModules/UMDateRangePicker.tsx @@ -16,22 +16,24 @@ const presets = [ interface PresetDateRangeSelectorProps { setTimeRange?: (startDate: Dayjs, endDate: Dayjs) => void; - start?: string; - end?: string; + start?: dayjs.Dayjs; + end?: dayjs.Dayjs; + gap?: number; } export function UMDateRangePicker({ - start = dayjs().utc().startOf("day").subtract(6, "day").format("YYYY-MM-DD"), - end = dayjs().utc().startOf("day").format("YYYY-MM-DD"), + start = dayjs().utc().startOf("day").subtract(6, "day"), + end = dayjs().utc().endOf("day"), + gap = 1, setTimeRange = () => {}, }: PresetDateRangeSelectorProps) { - const [startDate, setStartDate] = React.useState(dayjs(start).utc()); - const [endDate, setEndDate] = React.useState(dayjs(end).utc()); + const [startDate, setStartDate] = React.useState(dayjs.utc(start)); + const [endDate, setEndDate] = React.useState(dayjs.utc(end)); const [activePreset, setActivePreset] = React.useState(""); const setRange = (days: number, key: string) => { - const now = dayjs().utc(); - const start = now.startOf("day").subtract(days - 1, "day"); + const now = dayjs().utc().startOf("hour"); + const start = now.startOf("day").subtract(days - gap, "day"); setStartDate(start); setEndDate(now); setActivePreset(key); @@ -40,17 +42,19 @@ export function UMDateRangePicker({ const handleManualStart = (newValue: any) => { if (newValue) { - setStartDate(newValue); + const newStart = dayjs.utc(newValue).startOf("day"); + setStartDate(newStart); setActivePreset(null); - setTimeRange(newValue, dayjs().utc()); + setTimeRange(newValue, endDate); } }; const handleManualEnd = (newValue: any) => { if (newValue) { - setEndDate(newValue); + let newEnd = dayjs.utc(newValue).endOf("day"); + setEndDate(newEnd); setActivePreset(null); - setTimeRange(startDate, newValue); + setTimeRange(startDate, newEnd); } }; @@ -88,8 +92,8 @@ export function UMDateRangePicker({ } export function UMDateButtonPicker({ - start = dayjs().utc().startOf("day").subtract(6, "day").format("YYYY-MM-DD"), - end = dayjs().utc().startOf("day").format("YYYY-MM-DD"), + start = dayjs().utc().startOf("day").subtract(6, "day"), + end = dayjs().utc().endOf("day"), setTimeRange = () => {}, }: PresetDateRangeSelectorProps) { const [open, setOpen] = React.useState(false); @@ -117,7 +121,7 @@ export function UMDateButtonPicker({ justifyContent: "space-between", }} > - {start} - {end} + {start.format("YYYY-MM-DD")} - {end.format("YYYY-MM-DD")} { const router = useRouter(); useEffect(() => { const { - start_time = dayjs.utc().format("YYYY-MM-DD"), - end_time = dayjs.utc().format("YYYY-MM-DD"), + start_time = dayjs.utc(), + end_time = dayjs.utc(), } = router.query; const newprops: any = { start_time, @@ -71,8 +71,8 @@ const InnerUtilizationContent = ({ setTimeRange={(start: dayjs.Dayjs, end: dayjs.Dayjs) => { const newprops: any = { ...timeRange, - start_time: start.format("YYYY-MM-DD"), - end_time: end.format("YYYY-MM-DD"), + start_time: start, + end_time: end, }; dispatch({ type: "UPDATE_FIELDS", diff --git a/torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts b/torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts index 639951022a..d644aae04a 100644 --- a/torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts +++ b/torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts @@ -7,7 +7,7 @@ import { } from "./helpers/common"; import { toGeneralCompilerData } from "./helpers/general"; import { toPrecomputeCompilerData } from "./helpers/precompute"; -import { CompilerQueryType } from "./type"; +import { CompilerQueryType, defaultGetTimeSeriesInputs, defaultListCommitsInputs } from "./type"; //["x86_64","NVIDIA A10G","NVIDIA H100 80GB HBM3"] const COMPILER_BENCHMARK_TABLE_NAME = "compilers_benchmark_api_query"; const COMPILER_BENCHMARK_COMMITS_TABLE_NAME = @@ -16,7 +16,7 @@ const COMPILER_BENCHMARK_COMMITS_TABLE_NAME = export async function getCompilerBenchmarkData( inputparams: any, type: CompilerQueryType = CompilerQueryType.PRECOMPUTE, - format: string = "time_series" + format: string[] = ["time_series"] ) { const rows = await getCompilerDataFromClickhouse(inputparams); @@ -34,47 +34,81 @@ export async function getCompilerBenchmarkData( } } +export async function getCompilerCommits(inputparams: any): Promise { + if (!inputparams.startTime || !inputparams.stopTime) { + throw new Error("no start/end time provided in request"); + } + const queryParams = { + ...defaultListCommitsInputs, // base defaults + ...inputparams, // override with caller's values +}; + + if (queryParams.arch && queryParams.device){ + const arch_list = toQueryArch(inputparams.device, inputparams.arch); + queryParams["arch"] = arch_list; + } + + const commit_results = await queryClickhouseSaved( + COMPILER_BENCHMARK_COMMITS_TABLE_NAME, + queryParams + ); + return commit_results; +} + async function getCompilerDataFromClickhouse(inputparams: any): Promise { const start = Date.now(); - const arch_list = toQueryArch(inputparams.device, inputparams.arch); - inputparams["arch"] = arch_list; + + const queryParams = { + ...defaultGetTimeSeriesInputs, // base defaults + ...inputparams, // override with caller's values + }; + + if (queryParams.arch && queryParams.device){ + const arch_list = toQueryArch(queryParams.device, queryParams.arch); + queryParams["arch"] = arch_list; + } + // use the startTime and endTime to fetch commits from clickhouse if commits field is not provided - if (!inputparams.commits || inputparams.commits.length == 0) { - if (!inputparams.startTime || !inputparams.stopTime) { + if (!queryParams.commits || queryParams.commits.length == 0) { + if (!queryParams.startTime || !queryParams.stopTime) { console.log("no commits or start/end time provided in request"); return []; } + + console.log("fetch commits"); // get commits from clickhouse const commit_results = await queryClickhouseSaved( COMPILER_BENCHMARK_COMMITS_TABLE_NAME, - inputparams - ); + queryParams + ); // get unique commits const unique_commits = [...new Set(commit_results.map((c) => c.commit))]; if (unique_commits.length === 0) { - console.log("no commits found in clickhouse using", inputparams); + console.log("no commits found in clickhouse using", queryParams); return []; } console.log( - "no commits provided in request, found unqiue commits", - unique_commits + `no commits provided in request, searched unqiue commits based on + start/end time unique_commits: ${unique_commits.length}`, ); if (commit_results.length > 0) { - inputparams["commits"] = unique_commits; + queryParams["commits"] = unique_commits; } else { - console.log(`no commits found in clickhouse using ${inputparams}`); + console.log(`no commits found in clickhouse using ${queryParams}`); return []; } } else { - console.log("commits provided in request", inputparams.commits); + console.log("commits provided in request", queryParams.commits); } + console.log("query params", queryParams) + let rows = await queryClickhouseSaved( COMPILER_BENCHMARK_TABLE_NAME, - inputparams + queryParams ); const end = Date.now(); console.log("time to get compiler timeseris data", end - start); diff --git a/torchci/lib/benchmark/api_helper/compilers/helpers/general.ts b/torchci/lib/benchmark/api_helper/compilers/helpers/general.ts index 001a7df824..2a98687da7 100644 --- a/torchci/lib/benchmark/api_helper/compilers/helpers/general.ts +++ b/torchci/lib/benchmark/api_helper/compilers/helpers/general.ts @@ -35,31 +35,38 @@ const COMPILER_GENERAL_TABLE_SUB_GROUP_KEY = ["metric"]; */ export function toGeneralCompilerData( rawData: any[], - type: string = "time_series" + formats: string[] = ["time_series"] ) { const start_ts = new Date(rawData[0].granularity_bucket).getTime(); const end_ts = new Date( rawData[rawData.length - 1].granularity_bucket ).getTime(); - let res: any[] = []; - switch (type) { - case "time_series": - res = to_time_series_data( - rawData, - COMPILER_GENERAL_TS_GROUP_KEY, - COMPILER_GENERAL_TS_SUB_GROUP_KEY - ); - break; - case "table": - res = groupByBenchmarkData( - rawData, - COMPILER_GENERAL_TABLE_GROUP_KEY, - COMPILER_GENERAL_TABLE_SUB_GROUP_KEY - ); - break; - default: - throw new Error("Invalid type"); - } - return toTimeSeriesResponse(res, rawData.length, start_ts, end_ts); + let formats_result:any = {} + + formats.forEach((format) => { + const data = getformat(rawData,format); + formats_result[format] = data; + }); + return toTimeSeriesResponse(formats_result, rawData.length, start_ts, end_ts); } + +function getformat(data: any,format:string) { + switch (format) { + case "time_series": + return to_time_series_data( + data, + COMPILER_GENERAL_TS_GROUP_KEY, + COMPILER_GENERAL_TS_SUB_GROUP_KEY + ); + case "table": + return groupByBenchmarkData( + data, + COMPILER_GENERAL_TABLE_GROUP_KEY, + COMPILER_GENERAL_TABLE_SUB_GROUP_KEY + ); + break; + default: + throw new Error("Invalid type"); + } + } diff --git a/torchci/lib/benchmark/api_helper/compilers/helpers/precompute.ts b/torchci/lib/benchmark/api_helper/compilers/helpers/precompute.ts index e1f13d9626..d34c670456 100644 --- a/torchci/lib/benchmark/api_helper/compilers/helpers/precompute.ts +++ b/torchci/lib/benchmark/api_helper/compilers/helpers/precompute.ts @@ -15,6 +15,7 @@ import { toWorkflowIdMap, } from "../../utils"; import { toApiArch } from "./common"; +import { all } from "axios"; const COMPILER_PRECOMPUTE_TS_GROUP_KEY = [ "dtype", @@ -39,7 +40,7 @@ const COMPILER_PRECOMPUTE_TABLE_SUB_GROUP_KEY = ["compiler"]; export function toPrecomputeCompilerData( rawData: any[], - type: string = "time_series" + formats:string[]=["time_series"] ) { const metadata = { dtype: rawData[0].dtype, @@ -67,7 +68,7 @@ export function toPrecomputeCompilerData( peakMemory, compilationTime, executionTime, - peakMemoryUsage, + peakMemoryUsage, ].flat(); all_data = [...all_data].sort( @@ -77,25 +78,12 @@ export function toPrecomputeCompilerData( // post process data to get start_ts and end_ts, and add commit metadata const { start_ts, end_ts } = postFetchProcess(all_data, commit_map, metadata); - let res: any[] = []; - switch (type) { - case "time_series": - res = to_time_series_data( - all_data, - COMPILER_PRECOMPUTE_TS_GROUP_KEY, - COMPILER_PRECOMPUTE_TS_SUB_GROUP_KEY - ); - break; - case "table": - res = groupByBenchmarkData( - all_data, - COMPILER_PRECOMPUTE_TABLE_GROUP_KEY, - COMPILER_PRECOMPUTE_TABLE_SUB_GROUP_KEY - ); - break; - default: - throw new Error("Invalid type"); - } + + let res: any ={} + formats.forEach((format) => { + const f = getFormat(all_data,format) + res[format] = f + }); return toTimeSeriesResponse(res, rawData.length, start_ts, end_ts); } @@ -121,3 +109,25 @@ function postFetchProcess( end_ts, }; } + + +function getFormat(data: any,format:string) { + switch (format) { + case "time_series": + return to_time_series_data( + data, + COMPILER_PRECOMPUTE_TS_GROUP_KEY, + COMPILER_PRECOMPUTE_TS_SUB_GROUP_KEY + ); + break; + case "table": + return groupByBenchmarkData( + data, + COMPILER_PRECOMPUTE_TABLE_GROUP_KEY, + COMPILER_PRECOMPUTE_TABLE_SUB_GROUP_KEY + ); + break; + default: + throw new Error("Invalid type"); + } + } diff --git a/torchci/lib/benchmark/api_helper/compilers/type.ts b/torchci/lib/benchmark/api_helper/compilers/type.ts index 81b0a50fa1..caee340303 100644 --- a/torchci/lib/benchmark/api_helper/compilers/type.ts +++ b/torchci/lib/benchmark/api_helper/compilers/type.ts @@ -1,3 +1,6 @@ +import { table } from "console"; +import useSWR, { SWRResponse } from "swr"; + export enum CompilerQueryType { PRECOMPUTE = "precompute", GENERAL = "general", @@ -24,3 +27,108 @@ export function getExtremeTs( return Number.isFinite(extreme) ? extreme : null; } + + +export interface CompilerPrecomputeRequest { + name: "compiler_precompute"; + query_params: Record; +} + +// generic response type +export interface ApiResponse { + success: boolean; + data?: T; + error?: string; +} + +// --- Fetcher --- + +export async function postBenchmarkTimeSeriesFetcher(name:string, formats:string[], queryParams: Record): Promise { + const body = { + name: name, + query_params: queryParams, + response_formats: formats, + }; + const url = "/api/benchmark/get_time_series"; + const res = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + if (!res.ok) { + const payload = res.json() + throw new Error(`Failed to fetch data" ${res.status} ,${payload}`); + } + return res.json(); +} + +export async function listBenchmarkCommits(name:string, queryParams: Record): Promise { + const body = { + name: name, + query_params: queryParams, + + }; + const url = "/api/benchmark/list_commits"; + const res = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + return res.json(); +} +export interface CompilerBundleResult{ + commits: any; + timeSeries: any; +} + + +// --- Hook wrapper --- +export function useCompilerData( + name: string, + queryParams: Record | null +): SWRResponse { + const shouldFetch = !!queryParams; + return useSWR( + shouldFetch ? [name, queryParams] : null, + async ([n, qp]) => { + return postBenchmarkTimeSeriesFetcher( + n as string, + ["time_series","table"], + qp as Record, + ); + }, + { + revalidateOnFocus: false, + keepPreviousData: true, + dedupingInterval: 10_000, + } + ); +} + +export const defaultGetTimeSeriesInputs: any ={ + models: [], + commits: [], + compilers: [], + branches: [], + device: "", + arch: "", + dtype: "", + mode: "", + granularity: "hour", + startTime: "", + stopTime: "", + suites: [], +} + +export const defaultListCommitsInputs: any = { + branches: [], + device: "", + arch: [], + dtype: "", + mode: "", + startTime: "", + stopTime: "", + suites: [], + workflowIds: [], +}; diff --git a/torchci/lib/benchmark/api_helper/utils.ts b/torchci/lib/benchmark/api_helper/utils.ts index 7e7c301c0a..c0a46027c4 100644 --- a/torchci/lib/benchmark/api_helper/utils.ts +++ b/torchci/lib/benchmark/api_helper/utils.ts @@ -44,6 +44,7 @@ type Params = Record; // it accepts both ?parameters= and POST with JSON body export function readApiGetParams(req: NextApiRequest): Params { + // 1) If POST with parsed JSON body if (req.method === "POST" && req.body && typeof req.body === "object") { return req.body as Params; @@ -136,8 +137,7 @@ export function getNestedField(obj: any, path: string): any { } export type BenchmarkTimeSeriesResponse = { - total_rows: number; - time_series: any[]; + data: any; time_range: { start: string; end: string }; total_raw_rows?: number; }; @@ -195,19 +195,18 @@ export function toJobIdMap(data: any[]) { } export function toTimeSeriesResponse( - res: any[], + res: any, rawDataLength: number, start_ts: number, end_ts: number ) { const response: BenchmarkTimeSeriesResponse = { - total_rows: res.length, total_raw_rows: rawDataLength, time_range: { start: new Date(start_ts).toISOString(), end: new Date(end_ts).toISOString(), }, - time_series: res, + data: res, }; return response; } @@ -216,6 +215,7 @@ export function emptyTimeSeriesResponse() { return { total_rows: 0, time_series: [], + table: [], time_range: { start: new Date().toISOString(), end: new Date().toISOString(), diff --git a/torchci/pages/api/benchmark/get_time_series.ts b/torchci/pages/api/benchmark/get_time_series.ts index 1ed856b830..8342f2692c 100644 --- a/torchci/pages/api/benchmark/get_time_series.ts +++ b/torchci/pages/api/benchmark/get_time_series.ts @@ -43,13 +43,13 @@ export default async function handler( // get time series data try { - const { name, response_format, query_params } = params; - const format = - response_format && response_format.length > 0 - ? response_format - : "time_series"; + const { name, response_formats, query_params } = params; + const formats = + response_formats && response_formats.length > 0 + ? response_formats + : ["time_series"]; - const data = await getBenmarkTimeSeriesData(name, query_params, format); + const data = await getBenmarkTimeSeriesData(name, query_params, formats); return res.status(200).json({ data }); } catch (err: any) { console.error("API error:", err.message); @@ -60,20 +60,20 @@ export default async function handler( async function getBenmarkTimeSeriesData( request_name: string, query_params: any, - response_format: string = "time_series" + formats: string[]= ["time_series"] ) { switch (request_name) { case "compiler_precompute": return await getCompilerBenchmarkData( query_params, CompilerQueryType.PRECOMPUTE, - response_format + formats ); case "compiler": return await getCompilerBenchmarkData( query_params, CompilerQueryType.GENERAL, - response_format + formats ); default: throw new Error(`Unsupported request_name: ${request_name}`); diff --git a/torchci/pages/api/benchmark/list_commits.ts b/torchci/pages/api/benchmark/list_commits.ts new file mode 100644 index 0000000000..8f7c34ded7 --- /dev/null +++ b/torchci/pages/api/benchmark/list_commits.ts @@ -0,0 +1,48 @@ +import { getCompilerCommits } from "lib/benchmark/api_helper/compilers/get_compiler_benchmark_data"; +import { readApiGetParams } from "lib/benchmark/api_helper/utils"; +import { NextApiRequest, NextApiResponse } from "next"; + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse +) { + if (req.method !== "GET" && req.method !== "POST") { + res.setHeader("Allow", "GET, POST"); + return res.status(405).json({ error: "Only GET and POST allowed" }); + } + + const params = readApiGetParams(req); + console.log("[API]list commits, received request:", params); + + // validate params + if ( + !params || + !params.query_params || + Object.keys(params.query_params).length == 0 || + Object.keys(params).length === 0 + ) { + return res.status(400).json({ error: "Missing parameters" }); + } + + // get time series data + try { + const { name, query_params } = params; + const data = await getBenmarkCommits(name, query_params); + return res.status(200).json({ data }); + } catch (err: any) { + console.error("API error:", err.message); + return res.status(400).json({ error: err.message }); + } +} + +async function getBenmarkCommits( + request_name: string, + query_params: any, +) { + switch (request_name) { + case "compiler": + return await getCompilerCommits(query_params) + default: + throw new Error(`Unsupported request_name: ${request_name}`); + } +} diff --git a/torchci/pages/benchmark/compilers_regression.tsx b/torchci/pages/benchmark/compilers_regression.tsx new file mode 100644 index 0000000000..b85209707a --- /dev/null +++ b/torchci/pages/benchmark/compilers_regression.tsx @@ -0,0 +1,193 @@ +import { Divider, Skeleton, Stack, Typography } from "@mui/material"; +import utc from "dayjs/plugin/utc"; +import { + MAIN_BRANCH, +} from "components/benchmark/common"; +import { + DEFAULT_DEVICE_NAME, + DISPLAY_NAMES_TO_ARCH_NAMES, + DISPLAY_NAMES_TO_DEVICE_NAMES, + DTYPES, + DTYPES_V2, + MODES_V2, +} from "components/benchmark/compilers/common"; +import { SUITES } from "components/benchmark/compilers/SuitePicker"; +import { + DEFAULT_MODE, + DTypePicker, + ModePicker, + MODES, +} from "components/benchmark/ModeAndDTypePicker"; +import dayjs from "dayjs"; +import { useRouter } from "next/router"; +import { useEffect, useReducer, useState } from "react"; +import { UMDateButtonPicker } from "components/uiModules/UMDateRangePicker"; +import { UMPropReducer } from "components/uiModules/UMPropReducer"; +import { useCompilerData } from "lib/benchmark/api_helper/compilers/type"; +import LoadingPage from "components/common/LoadingPage"; +dayjs.extend(utc); + + +export default function Page() { + const initialDropdownFields = { + mode: DEFAULT_MODE, + dtype: MODES[DEFAULT_MODE], + lBranch: MAIN_BRANCH, + lCommit: "", + rBranch: MAIN_BRANCH, + rCommit: "", + deviceName: DEFAULT_DEVICE_NAME, + device: "cuda", + arch: "h100" + } + + const router = useRouter(); + + const [timeRange, dispatchTimeRange] = useReducer(UMPropReducer, { + start_time : dayjs.utc().startOf("day").subtract(7, "day"), + end_time : dayjs.utc().endOf("day"), + }); + const [dropdowns, dispatchDropdowns] = useReducer(UMPropReducer, initialDropdownFields); + + useEffect(() => { + const { + startTime, + stopTime, + mode, + dtype, + deviceName, + lBranch, + lCommit, + rBranch, + rCommit, + } = router.query; + + if (startTime && stopTime) { + // update time range + dispatchTimeRange({ + type: "UPDATE_FIELDS", + payload: { + start_time: dayjs.utc(startTime as string), + end_time: dayjs.utc(stopTime as string), + }, + }); + } + + // collect dropdown updates only if they exist + const newDropdowns = { + ...dropdowns, + }; + if (mode) newDropdowns.mode = mode as string; + if (dtype) newDropdowns.dtype = dtype as string; + if (deviceName) newDropdowns.deviceName = deviceName as string; + if (lBranch) newDropdowns.lBranch = lBranch as string; + if (lCommit) newDropdowns.lCommit = lCommit as string; + if (rBranch) newDropdowns.rBranch = rBranch as string; + if (rCommit) newDropdowns.rCommit = rCommit as string; + + if (Object.keys(newDropdowns).length > 0) { + dispatchDropdowns({ type: "UPDATE_FIELDS", payload: newDropdowns }); + } +}, [router.query]); + + const granularity = "hour" // hardcoded for now + + const queryParams: { [key: string]: any } = { + commits: [], + branches:["main"], + compilers: [], + arch: DISPLAY_NAMES_TO_ARCH_NAMES[dropdowns.deviceName], + device: DISPLAY_NAMES_TO_DEVICE_NAMES[dropdowns.deviceName], + dtype: dropdowns.dtype, + granularity: granularity, + mode: dropdowns.mode, + startTime: dayjs.utc(timeRange.start_time).format("YYYY-MM-DDTHH:mm:ss"), + stopTime: dayjs.utc(timeRange.end_time).format("YYYY-MM-DDTHH:mm:ss"), + suites: Object.keys(SUITES), + }; + return ( +
+ + + TorchInductor Performance DashBoard + + + + { + dispatchTimeRange({ + type: "UPDATE_FIELDS", + payload: { + start_time: start, + end_time: end, + }, + }); + }} + start={timeRange.start_time} + end={timeRange.end_time} + /> + + + +
+ +
+ + ); +} + +function DataRender(props: any) { + const { data, isLoading, error } = useCompilerData("compiler_precompute",props.queryParams); + if (isLoading) { + return ; + } + if (error) { + return
Error: {error.message}
; + } + return 
{JSON.stringify(data, null, 2)}
; +} + +type DropdownsProps = { + dropdowns: any; + dispatchDropdowns: React.Dispatch; +}; + +function Dropdowns({ + dropdowns, + dispatchDropdowns, +}: DropdownsProps) { + + const setDropdownField = (key: string, value: string) => { + dispatchDropdowns({ type: "UPDATE_FIELDS", payload: { [key]: value } }); + }; + return ( + <> + setDropdownField("mode", val)} + setDType={(val: string) => setDropdownField("dtype", val)} + /> + { + if (val === "notset"){ + setDropdownField("dtype", '') + } else{ + setDropdownField("dtype", val) + } + }} + dtypes={DTYPES_V2} + label="Precision" + /> + setDropdownField("deviceName", val)} + dtypes={Object.keys(DISPLAY_NAMES_TO_DEVICE_NAMES)} + label="Device" + /> + + ); +} From a85ab421da07a5bee01128c1e93fda10aba445a1 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 11 Sep 2025 12:34:37 -0400 Subject: [PATCH 2/3] improve api --- .../UtilizationReportPage.tsx | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/torchci/components/utilization/UtilizationReportPage/UtilizationReportPage.tsx b/torchci/components/utilization/UtilizationReportPage/UtilizationReportPage.tsx index 424d13ff71..16b3b00e4c 100644 --- a/torchci/components/utilization/UtilizationReportPage/UtilizationReportPage.tsx +++ b/torchci/components/utilization/UtilizationReportPage/UtilizationReportPage.tsx @@ -15,19 +15,25 @@ import { UtilReportPageSyncParamsToUrl } from "./UtilReportPageSyncParamsToUrl"; dayjs.extend(utc); const UtilizationReportPage = () => { - const [timeRange, dispatch] = useReducer(UMPropReducer, {}); + const [timeRange, dispatch] = useReducer(UMPropReducer, { + start_time: dayjs.utc(), + end_time: dayjs.utc(), + }); const router = useRouter(); useEffect(() => { const { - start_time = dayjs.utc(), - end_time = dayjs.utc(), - } = router.query; - const newprops: any = { start_time, end_time, - }; - dispatch({ type: "UPDATE_FIELDS", payload: newprops }); + } = router.query; + + if (start_time && end_time) { + const newprops: any = { + start_time: dayjs.utc(start_time as string) || dayjs.utc(), + end_time: dayjs.utc(end_time as string) || dayjs.utc(), + }; + dispatch({ type: "UPDATE_FIELDS", payload: newprops }); + } }, [router.query]); return ( @@ -83,8 +89,8 @@ const InnerUtilizationContent = ({ end={timeRange.end_time} /> ); From 39f58b4fd7be1b8c13522cd5dc093bb47f033946 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Fri, 12 Sep 2025 00:45:35 -0400 Subject: [PATCH 3/3] add old way --- .../components/benchmark/compilers/common.tsx | 2 +- .../umcompilers/CommitChoiceSection.tsx | 127 +++++++++++ .../umcompilers/MainOptionSideBar.tsx | 97 +++++++++ .../umcompilers/RegressionTrackingPage.tsx | 93 +++++++++ .../uiModules/UMDateRangePicker.tsx | 2 + .../uiModules/UMDenseComponents.tsx | 1 + .../components/uiModules/UMDenseRenderers.tsx | 197 ++++++++++++++++++ .../UtilizationReportPage.tsx | 5 +- .../compilers/get_compiler_benchmark_data.ts | 28 +-- .../api_helper/compilers/helpers/general.ts | 40 ++-- .../compilers/helpers/precompute.ts | 22 +- .../benchmark/api_helper/compilers/type.ts | 73 ++++--- torchci/lib/benchmark/api_helper/utils.ts | 1 - .../store/benchmark_dashboard_provider.tsx | 48 +++++ .../store/benchmark_regression_store.ts | 90 ++++++++ torchci/package.json | 3 +- .../pages/api/benchmark/get_time_series.ts | 2 +- torchci/pages/api/benchmark/list_commits.ts | 7 +- .../pages/benchmark/compilers_regression.tsx | 192 +---------------- torchci/tsconfig.json | 2 +- torchci/yarn.lock | 5 + 21 files changed, 763 insertions(+), 274 deletions(-) create mode 100644 torchci/components/benchmark/umcompilers/CommitChoiceSection.tsx create mode 100644 torchci/components/benchmark/umcompilers/MainOptionSideBar.tsx create mode 100644 torchci/components/benchmark/umcompilers/RegressionTrackingPage.tsx create mode 100644 torchci/components/uiModules/UMDenseRenderers.tsx create mode 100644 torchci/lib/benchmark/store/benchmark_dashboard_provider.tsx create mode 100644 torchci/lib/benchmark/store/benchmark_regression_store.ts diff --git a/torchci/components/benchmark/compilers/common.tsx b/torchci/components/benchmark/compilers/common.tsx index a516f0a2a5..1bf40f77aa 100644 --- a/torchci/components/benchmark/compilers/common.tsx +++ b/torchci/components/benchmark/compilers/common.tsx @@ -58,7 +58,7 @@ export const HELP_LINK = export const DTYPES = ["amp", "float16", "bfloat16", "quant", "notset"]; export const DTYPES_V2 = ["amp", "float16", "bfloat16", "notset"]; -export const MODES_V2= ["training", "inference", "notset"]; +export const MODES_V2 = ["training", "inference", "notset"]; export const DEFAULT_DEVICE_NAME = "cuda (h100)"; export const DISPLAY_NAMES_TO_DEVICE_NAMES: { [k: string]: string } = { diff --git a/torchci/components/benchmark/umcompilers/CommitChoiceSection.tsx b/torchci/components/benchmark/umcompilers/CommitChoiceSection.tsx new file mode 100644 index 0000000000..db3e66372a --- /dev/null +++ b/torchci/components/benchmark/umcompilers/CommitChoiceSection.tsx @@ -0,0 +1,127 @@ +import { Button, FormControl, InputLabel, MenuItem, Select, Typography } from "@mui/material"; +import { Box, Stack } from "@mui/system"; +import dayjs from "dayjs"; +import { useBenchmarkCommitsData } from "lib/benchmark/api_helper/compilers/type"; +import { useDashboardStore } from "lib/benchmark/store/benchmark_dashboard_provider"; +import { CommitMeta } from "lib/benchmark/store/benchmark_regression_store"; +import { useEffect, useState } from "react"; +import { DISPLAY_NAMES_TO_ARCH_NAMES, DISPLAY_NAMES_TO_DEVICE_NAMES } from "../compilers/common"; +import { UMDenseCommitDropdown } from "components/uiModules/UMDenseRenderers"; + +export const REQUIRED_KEYS = ["mode", "dtype", "deviceName"] as const; + +export function CommitChoiceSection({ benchmarkId }: { benchmarkId?: string }) { + const useStore= useDashboardStore(); + + const committedTime = useStore(s => s.committedTime); + const committedFilters = useStore(s => s.committedFilters); + const lcommit = useStore(s => s.lcommit); + const rcommit = useStore(s => s.rcommit); + const setLCommit = useStore(s => s.setLCommit); + const setRCommit = useStore(s => s.setRCommit); + + const [leftList, setLeftList] = useState([]); + const [rightList, setRightList] = useState([]); + const [autoLeftSha, setAutoLeftSha] = useState(null); + const [autoRightSha, setAutoRightSha] = useState(null); + + const ready = + !!committedTime?.start && + !!committedTime?.end && + REQUIRED_KEYS.every((k) => !!committedFilters[k]); + + useEffect(() => { + if (!ready) { + setLeftList([]); + setRightList([]); + setAutoLeftSha(null); + setAutoRightSha(null); + setLCommit(null); + setRCommit(null); + } + }, [ready, setLCommit, setRCommit]); + + const baseParams: any | null = ready + ? { + benchmarkId, + startTime: dayjs.utc(committedTime.start).format("YYYY-MM-DDTHH:mm:ss"), + stopTime: dayjs.utc(committedTime.end).format("YYYY-MM-DDTHH:mm:ss"), + arch: DISPLAY_NAMES_TO_ARCH_NAMES[committedFilters.deviceName], + device: DISPLAY_NAMES_TO_DEVICE_NAMES[committedFilters.deviceName], + dtype: committedFilters.dtype, + mode: committedFilters.mode, + branch: ["main"], + } + : null; + + // Fetch data + const { data, isLoading, error } = useBenchmarkCommitsData("compiler", baseParams); + + // Helper + const inList = (list: CommitMeta[], workflow_id?: string | null) => + !!workflow_id && list.some((c) => c.workflow_id === workflow_id); + + // Sync lists & auto-picks when data / selection changes + useEffect(() => { + const L: CommitMeta[] | undefined = data?.left ?? data?.data ?? []; + const R: CommitMeta[] | undefined = data?.right ?? data?.data ?? []; + if (!L || !R) return; + + setLeftList(L); + setRightList(R); + + const nextAutoL = inList(L, lcommit?.workflow_id) ? lcommit!.workflow_id : L[0]?.workflow_id ?? null; + const nextAutoR = inList(R, rcommit?.workflow_id) ? rcommit!.workflow_id : R[R.length - 1]?.workflow_id ?? null; + + setAutoLeftSha(nextAutoL); + setAutoRightSha(nextAutoR); + + console.log("dataaaaa", data); + + if (!inList(L, lcommit?.workflow_id)) { + setLCommit(nextAutoL ? L.find((c) => c.workflow_id === nextAutoL) ?? null : null); + } + if (!inList(R, rcommit?.workflow_id)) { + setRCommit(nextAutoR ? R.find((c) => c.workflow_id === nextAutoR) ?? null : null); + } + }, [data, lcommit?.workflow_id, rcommit?.workflow_id, setLCommit, setRCommit]); // ✅ correct deps + + if (error) return
Error: {error.message}
; + if (isLoading || !data) return null; + + const leftChangedByUser = !!lcommit?.workflow_id && autoLeftSha != null && lcommit.workflow_id !== autoLeftSha; + const rightChangedByUser = !!rcommit?.workflow_id && autoRightSha != null && rcommit.workflow_id !== autoRightSha; + + return ( + + Commits + + + + {(leftChangedByUser || rightChangedByUser) && ( + + )} + + ); +} diff --git a/torchci/components/benchmark/umcompilers/MainOptionSideBar.tsx b/torchci/components/benchmark/umcompilers/MainOptionSideBar.tsx new file mode 100644 index 0000000000..e2da643e30 --- /dev/null +++ b/torchci/components/benchmark/umcompilers/MainOptionSideBar.tsx @@ -0,0 +1,97 @@ +// components/Sidebar.tsx +"use client"; +import Button from "@mui/material/Button"; +import Divider from "@mui/material/Divider"; +import Stack from "@mui/material/Stack"; +import Typography from "@mui/material/Typography"; +import { UMDateButtonPicker } from "components/uiModules/UMDateRangePicker"; +import dayjs from "dayjs"; +import { useDashboardStore } from "lib/benchmark/store/benchmark_dashboard_provider"; +import { DISPLAY_NAMES_TO_DEVICE_NAMES, DTYPES_V2 } from "../compilers/common"; +import { UMDenseDropdown, UMDenseModePicker } from "components/uiModules/UMDenseRenderers"; + +export function Sidebar() { + const useStore = useDashboardStore(); + + const stagedTime = useStore((s) => s.stagedTime); + const stagedFilters = useStore((s) => s.stagedFilters); + const setStagedTime = useStore((s) => s.setStagedTime); + const committedTime = useStore((s) => s.committedTime); + const committedFilters = useStore((s) => s.committedFilters); + const commitMainOptions = useStore((s) => s.commitMainOptions); + const revertMainOptions = useStore((s) => s.revertMainOptions); + + const dirty = + stagedTime.start.valueOf() !== committedTime.start.valueOf() || + stagedTime.end.valueOf() !== committedTime.end.valueOf() || + JSON.stringify(stagedFilters) !== JSON.stringify(committedFilters); + + return ( + + Search + + setStagedTime({ start, end }) + } + start={stagedTime.start} + end={stagedTime.end} + /> + + {/* Dropdown filters */} + Filters + + + + + {/* Apply / Revert */} + + + + + + ); +} + +export function Dropdowns() { + const useStore = useDashboardStore(); + const stagedFilters = useStore((s) => s.stagedFilters); + const setStagedFilter = useStore((s) => s.setStagedFilter); + + return ( + <> + setStagedFilter("mode", val)} + setDType={(val: string) => setStagedFilter("dtype", val)} + /> + + val === "notset" + ? setStagedFilter("dtype", "") + : setStagedFilter("dtype", val) + } + dtypes={DTYPES_V2} + label="Precision" + /> + setStagedFilter("deviceName", val)} + dtypes={Object.keys(DISPLAY_NAMES_TO_DEVICE_NAMES)} + label="Device" + /> + + ); +} diff --git a/torchci/components/benchmark/umcompilers/RegressionTrackingPage.tsx b/torchci/components/benchmark/umcompilers/RegressionTrackingPage.tsx new file mode 100644 index 0000000000..8bdbc766f5 --- /dev/null +++ b/torchci/components/benchmark/umcompilers/RegressionTrackingPage.tsx @@ -0,0 +1,93 @@ + +import { + DEFAULT_DEVICE_NAME, + DISPLAY_NAMES_TO_ARCH_NAMES, + DISPLAY_NAMES_TO_DEVICE_NAMES, +} from "components/benchmark/compilers/common"; +import { DEFAULT_MODE, MODES } from "components/benchmark/ModeAndDTypePicker"; +import LoadingPage from "components/common/LoadingPage"; +import dayjs from "dayjs"; +import utc from "dayjs/plugin/utc"; +import { + useCompilerData, +} from "lib/benchmark/api_helper/compilers/type"; +import { + DashboardStoreProvider, + useDashboardStore, +} from "lib/benchmark/store/benchmark_dashboard_provider"; +import { SUITES } from "../compilers/SuitePicker"; +import { Sidebar } from "./MainOptionSideBar"; +import { CommitChoiceSection, REQUIRED_KEYS } from "./CommitChoiceSection"; +dayjs.extend(utc); + +export default function BenchmarkRegressionPage() { + // Defaults for this benchmark + const initial = { + time: { + start: dayjs.utc().startOf("day").subtract(7, "day"), + end: dayjs.utc().endOf("day"), + }, + filters: { + mode: DEFAULT_MODE, + dtype: MODES[DEFAULT_MODE], + deviceName: DEFAULT_DEVICE_NAME, + device: "cuda", + arch: "h100", + }, + }; + + return ( + +
+ +
+ +
+
+ + ); +} + +function DataRender({ benchmarkId }: { benchmarkId?: string }) { + const useStore = useDashboardStore(); + const committedTime = useStore((s) => s.committedTime); + const committedFilters = useStore((s) => s.committedFilters); + const lcommit = useStore((s) => s.lcommit); + const rcommit = useStore((s) => s.rcommit); + + const ready = + !!committedTime?.start && + !!committedTime?.end && + REQUIRED_KEYS.every((k) => !!committedFilters[k]); + + const queryParams: any | null = ready + ? { + commits: [], + branches: ["main"], + compilers: [], + arch: DISPLAY_NAMES_TO_ARCH_NAMES[committedFilters.deviceName], + device: DISPLAY_NAMES_TO_DEVICE_NAMES[committedFilters.deviceName], + dtype: committedFilters.dtype, + granularity: "hour", + mode: committedFilters.mode, + startTime: dayjs.utc(committedTime.start).format("YYYY-MM-DDTHH:mm:ss"), + stopTime: dayjs.utc(committedTime.end).format("YYYY-MM-DDTHH:mm:ss"), + suites: Object.keys(SUITES), + } + : null; + + const { data, isLoading, error } = useCompilerData( + "compiler_precompute", + queryParams + ); + if (isLoading) { + return ; + } + if (error) { + return
Error: {error.message}
; + } + return 
{JSON.stringify(data, null, 2)}
; +} diff --git a/torchci/components/uiModules/UMDateRangePicker.tsx b/torchci/components/uiModules/UMDateRangePicker.tsx index 7c9a0c0ed9..fc3c32d610 100644 --- a/torchci/components/uiModules/UMDateRangePicker.tsx +++ b/torchci/components/uiModules/UMDateRangePicker.tsx @@ -9,7 +9,9 @@ import { UMDenseDatePicker } from "./UMDenseDatePicker"; const presets = [ { key: "today", label: "Today", days: 1 }, { key: "last2", label: "Last 2 Days", days: 2 }, + { key: "last3", label: "Last 3 Days", days: 3 }, { key: "last7", label: "Last 7 Days", days: 7 }, + { key: "last10", label: "Last 10 Days", days: 10 }, { key: "last14", label: "Last 14 Days", days: 14 }, { key: "last30", label: "Last 30 Days", days: 30 }, ]; diff --git a/torchci/components/uiModules/UMDenseComponents.tsx b/torchci/components/uiModules/UMDenseComponents.tsx index dbd7ef9395..929733bc90 100644 --- a/torchci/components/uiModules/UMDenseComponents.tsx +++ b/torchci/components/uiModules/UMDenseComponents.tsx @@ -10,3 +10,4 @@ export const UMDenseButton = styled(Button)(({ theme }) => ({ borderRadius: 0, textTransform: "none", // optional: avoids uppercase })); + diff --git a/torchci/components/uiModules/UMDenseRenderers.tsx b/torchci/components/uiModules/UMDenseRenderers.tsx new file mode 100644 index 0000000000..4f13647382 --- /dev/null +++ b/torchci/components/uiModules/UMDenseRenderers.tsx @@ -0,0 +1,197 @@ +import React from "react"; +import { + FormControl, + InputLabel, + MenuItem, + Select, + Typography, + type SelectChangeEvent, +} from "@mui/material"; +import { Box } from "@mui/system"; + +// Reusable dense menu style (affects the dropdown list items) +export const DENSE_MENU_STYLE = { + // shrink the whole list + "& .MuiList-root": { + paddingTop: 0, + paddingBottom: 0, + + }, + // make each item short & tight + "& .MuiMenuItem-root": { + minHeight: 18, // default ~48 + paddingTop: 1, // 2px + paddingBottom: 0, + paddingLeft: 8, + paddingRight: 8, + }, + // smaller text + tight line height + "& .MuiTypography-root": { + fontSize: 12, + lineHeight: 1.2, + }, +}; + +// Optional: compact display for the Select trigger itself +const DENSE_SELECT_SX = { + "& .MuiSelect-select": { + minHeight: 24, + paddingTop: 0.25, + paddingBottom: 0.25, + fontSize: "0.75rem", + }, +}; + +type Props = { + dtype: string; + setDType: (v: string) => void; + dtypes: string[]; + label: string; +}; + + +export const DEFAULT_MODE = "inference"; +// The value is the default dtype for that mode +export const MODES: { [k: string]: string } = { + training: "amp", + inference: "bfloat16", +}; + +export const UMDenseDropdown: React.FC = ({ + dtype, + setDType, + dtypes, + label, +}) => { + const labelId = "dtype-picker-label"; + const selectId = "dtype-picker-select"; + + const handleChange = (e: SelectChangeEvent) => { + setDType(e.target.value); + }; + + return ( + + {label} + + + ); +}; + + +export function UMDenseModePicker({ + mode, + setMode, + setDType, +}: { + mode: string; + setMode: any; + setDType: any; +}) { + function handleChange(e: SelectChangeEvent) { + const selectedMode = e.target.value; + setMode(selectedMode); + setDType(selectedMode in MODES ? MODES[selectedMode] : "amp"); + } + return ( + <> + + Mode + + + + ); +} + + +export type CommitMeta = { + commit: string; + workflow_id: string; + date: string; +}; + +type UMDenseCommitDropdownProps = { + label: string; + disable: boolean; + commitList: CommitMeta[]; + selectedCommit: CommitMeta | null; + setCommit: (commit: CommitMeta | null) => void; +}; + +export const UMDenseCommitDropdown: React.FC = ({ + label, + disable, + commitList, + selectedCommit, + setCommit, +}) => { + function handleChange(e: SelectChangeEvent) { + const val = e.target.value; + setCommit(commitList.find((c) => c.commit === val) ?? null); + } + + return ( + + {label} + + + ); +}; diff --git a/torchci/components/utilization/UtilizationReportPage/UtilizationReportPage.tsx b/torchci/components/utilization/UtilizationReportPage/UtilizationReportPage.tsx index 16b3b00e4c..c82605506b 100644 --- a/torchci/components/utilization/UtilizationReportPage/UtilizationReportPage.tsx +++ b/torchci/components/utilization/UtilizationReportPage/UtilizationReportPage.tsx @@ -22,10 +22,7 @@ const UtilizationReportPage = () => { const router = useRouter(); useEffect(() => { - const { - start_time, - end_time, - } = router.query; + const { start_time, end_time } = router.query; if (start_time && end_time) { const newprops: any = { diff --git a/torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts b/torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts index d644aae04a..276b5d572c 100644 --- a/torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts +++ b/torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts @@ -7,7 +7,11 @@ import { } from "./helpers/common"; import { toGeneralCompilerData } from "./helpers/general"; import { toPrecomputeCompilerData } from "./helpers/precompute"; -import { CompilerQueryType, defaultGetTimeSeriesInputs, defaultListCommitsInputs } from "./type"; +import { + CompilerQueryType, + defaultGetTimeSeriesInputs, + defaultListCommitsInputs, +} from "./type"; //["x86_64","NVIDIA A10G","NVIDIA H100 80GB HBM3"] const COMPILER_BENCHMARK_TABLE_NAME = "compilers_benchmark_api_query"; const COMPILER_BENCHMARK_COMMITS_TABLE_NAME = @@ -40,17 +44,17 @@ export async function getCompilerCommits(inputparams: any): Promise { } const queryParams = { ...defaultListCommitsInputs, // base defaults - ...inputparams, // override with caller's values -}; + ...inputparams, // override with caller's values + }; - if (queryParams.arch && queryParams.device){ + if (queryParams.arch && queryParams.device) { const arch_list = toQueryArch(inputparams.device, inputparams.arch); queryParams["arch"] = arch_list; } const commit_results = await queryClickhouseSaved( - COMPILER_BENCHMARK_COMMITS_TABLE_NAME, - queryParams + COMPILER_BENCHMARK_COMMITS_TABLE_NAME, + queryParams ); return commit_results; } @@ -60,15 +64,14 @@ async function getCompilerDataFromClickhouse(inputparams: any): Promise { const queryParams = { ...defaultGetTimeSeriesInputs, // base defaults - ...inputparams, // override with caller's values + ...inputparams, // override with caller's values }; - if (queryParams.arch && queryParams.device){ + if (queryParams.arch && queryParams.device) { const arch_list = toQueryArch(queryParams.device, queryParams.arch); queryParams["arch"] = arch_list; } - // use the startTime and endTime to fetch commits from clickhouse if commits field is not provided if (!queryParams.commits || queryParams.commits.length == 0) { if (!queryParams.startTime || !queryParams.stopTime) { @@ -76,12 +79,11 @@ async function getCompilerDataFromClickhouse(inputparams: any): Promise { return []; } - console.log("fetch commits"); // get commits from clickhouse const commit_results = await queryClickhouseSaved( COMPILER_BENCHMARK_COMMITS_TABLE_NAME, queryParams - ); + ); // get unique commits const unique_commits = [...new Set(commit_results.map((c) => c.commit))]; if (unique_commits.length === 0) { @@ -91,7 +93,7 @@ async function getCompilerDataFromClickhouse(inputparams: any): Promise { console.log( `no commits provided in request, searched unqiue commits based on - start/end time unique_commits: ${unique_commits.length}`, + start/end time unique_commits: ${unique_commits.length}` ); if (commit_results.length > 0) { @@ -104,8 +106,6 @@ async function getCompilerDataFromClickhouse(inputparams: any): Promise { console.log("commits provided in request", queryParams.commits); } - console.log("query params", queryParams) - let rows = await queryClickhouseSaved( COMPILER_BENCHMARK_TABLE_NAME, queryParams diff --git a/torchci/lib/benchmark/api_helper/compilers/helpers/general.ts b/torchci/lib/benchmark/api_helper/compilers/helpers/general.ts index 2a98687da7..275680e218 100644 --- a/torchci/lib/benchmark/api_helper/compilers/helpers/general.ts +++ b/torchci/lib/benchmark/api_helper/compilers/helpers/general.ts @@ -42,31 +42,31 @@ export function toGeneralCompilerData( rawData[rawData.length - 1].granularity_bucket ).getTime(); - let formats_result:any = {} + let formats_result: any = {}; formats.forEach((format) => { - const data = getformat(rawData,format); + const data = getformat(rawData, format); formats_result[format] = data; }); return toTimeSeriesResponse(formats_result, rawData.length, start_ts, end_ts); } -function getformat(data: any,format:string) { - switch (format) { - case "time_series": - return to_time_series_data( - data, - COMPILER_GENERAL_TS_GROUP_KEY, - COMPILER_GENERAL_TS_SUB_GROUP_KEY - ); - case "table": - return groupByBenchmarkData( - data, - COMPILER_GENERAL_TABLE_GROUP_KEY, - COMPILER_GENERAL_TABLE_SUB_GROUP_KEY - ); - break; - default: - throw new Error("Invalid type"); - } +function getformat(data: any, format: string) { + switch (format) { + case "time_series": + return to_time_series_data( + data, + COMPILER_GENERAL_TS_GROUP_KEY, + COMPILER_GENERAL_TS_SUB_GROUP_KEY + ); + case "table": + return groupByBenchmarkData( + data, + COMPILER_GENERAL_TABLE_GROUP_KEY, + COMPILER_GENERAL_TABLE_SUB_GROUP_KEY + ); + break; + default: + throw new Error("Invalid type"); } +} diff --git a/torchci/lib/benchmark/api_helper/compilers/helpers/precompute.ts b/torchci/lib/benchmark/api_helper/compilers/helpers/precompute.ts index d34c670456..6eac552688 100644 --- a/torchci/lib/benchmark/api_helper/compilers/helpers/precompute.ts +++ b/torchci/lib/benchmark/api_helper/compilers/helpers/precompute.ts @@ -15,7 +15,6 @@ import { toWorkflowIdMap, } from "../../utils"; import { toApiArch } from "./common"; -import { all } from "axios"; const COMPILER_PRECOMPUTE_TS_GROUP_KEY = [ "dtype", @@ -40,7 +39,7 @@ const COMPILER_PRECOMPUTE_TABLE_SUB_GROUP_KEY = ["compiler"]; export function toPrecomputeCompilerData( rawData: any[], - formats:string[]=["time_series"] + formats: string[] = ["time_series"] ) { const metadata = { dtype: rawData[0].dtype, @@ -68,7 +67,7 @@ export function toPrecomputeCompilerData( peakMemory, compilationTime, executionTime, - peakMemoryUsage, + peakMemoryUsage, ].flat(); all_data = [...all_data].sort( @@ -79,11 +78,11 @@ export function toPrecomputeCompilerData( // post process data to get start_ts and end_ts, and add commit metadata const { start_ts, end_ts } = postFetchProcess(all_data, commit_map, metadata); - let res: any ={} - formats.forEach((format) => { - const f = getFormat(all_data,format) - res[format] = f - }); + let res: any = {}; + formats.forEach((format) => { + const f = getFormat(all_data, format); + res[format] = f; + }); return toTimeSeriesResponse(res, rawData.length, start_ts, end_ts); } @@ -110,9 +109,8 @@ function postFetchProcess( }; } - -function getFormat(data: any,format:string) { - switch (format) { +function getFormat(data: any, format: string) { + switch (format) { case "time_series": return to_time_series_data( data, @@ -130,4 +128,4 @@ function getFormat(data: any,format:string) { default: throw new Error("Invalid type"); } - } +} diff --git a/torchci/lib/benchmark/api_helper/compilers/type.ts b/torchci/lib/benchmark/api_helper/compilers/type.ts index caee340303..aa7c596eb8 100644 --- a/torchci/lib/benchmark/api_helper/compilers/type.ts +++ b/torchci/lib/benchmark/api_helper/compilers/type.ts @@ -1,4 +1,3 @@ -import { table } from "console"; import useSWR, { SWRResponse } from "swr"; export enum CompilerQueryType { @@ -28,7 +27,6 @@ export function getExtremeTs( return Number.isFinite(extreme) ? extreme : null; } - export interface CompilerPrecomputeRequest { name: "compiler_precompute"; query_params: Record; @@ -43,7 +41,11 @@ export interface ApiResponse { // --- Fetcher --- -export async function postBenchmarkTimeSeriesFetcher(name:string, formats:string[], queryParams: Record): Promise { +export async function postBenchmarkTimeSeriesFetcher( + name: string, + formats: string[], + queryParams: Record +): Promise { const body = { name: name, query_params: queryParams, @@ -57,17 +59,19 @@ export async function postBenchmarkTimeSeriesFetcher(name:string, formats:str }); if (!res.ok) { - const payload = res.json() + const payload = res.json(); throw new Error(`Failed to fetch data" ${res.status} ,${payload}`); } return res.json(); } -export async function listBenchmarkCommits(name:string, queryParams: Record): Promise { +export async function listBenchmarkCommits( + name: string, + queryParams: Record +): Promise { const body = { name: name, query_params: queryParams, - }; const url = "/api/benchmark/list_commits"; const res = await fetch(url, { @@ -77,11 +81,32 @@ export async function listBenchmarkCommits(name:string, queryParams: Record | null +): SWRResponse { + const shouldFetch = !!queryParams; + + return useSWR( + shouldFetch ? [name, queryParams] : null, + async ([n, qp]) => { + return listBenchmarkCommits( + n as string, + qp as Record + ); + }, + { + revalidateOnFocus: false, + keepPreviousData: true, + dedupingInterval: 10_000, + } + ); +} // --- Hook wrapper --- export function useCompilerData( @@ -94,8 +119,8 @@ export function useCompilerData( async ([n, qp]) => { return postBenchmarkTimeSeriesFetcher( n as string, - ["time_series","table"], - qp as Record, + ["time_series", "table"], + qp as Record ); }, { @@ -106,20 +131,20 @@ export function useCompilerData( ); } -export const defaultGetTimeSeriesInputs: any ={ - models: [], - commits: [], - compilers: [], - branches: [], - device: "", - arch: "", - dtype: "", - mode: "", - granularity: "hour", - startTime: "", - stopTime: "", - suites: [], -} +export const defaultGetTimeSeriesInputs: any = { + models: [], + commits: [], + compilers: [], + branches: [], + device: "", + arch: "", + dtype: "", + mode: "", + granularity: "hour", + startTime: "", + stopTime: "", + suites: [], +}; export const defaultListCommitsInputs: any = { branches: [], diff --git a/torchci/lib/benchmark/api_helper/utils.ts b/torchci/lib/benchmark/api_helper/utils.ts index c0a46027c4..1d90dd13be 100644 --- a/torchci/lib/benchmark/api_helper/utils.ts +++ b/torchci/lib/benchmark/api_helper/utils.ts @@ -44,7 +44,6 @@ type Params = Record; // it accepts both ?parameters= and POST with JSON body export function readApiGetParams(req: NextApiRequest): Params { - // 1) If POST with parsed JSON body if (req.method === "POST" && req.body && typeof req.body === "object") { return req.body as Params; diff --git a/torchci/lib/benchmark/store/benchmark_dashboard_provider.tsx b/torchci/lib/benchmark/store/benchmark_dashboard_provider.tsx new file mode 100644 index 0000000000..d99faeea01 --- /dev/null +++ b/torchci/lib/benchmark/store/benchmark_dashboard_provider.tsx @@ -0,0 +1,48 @@ +import { createContext, useContext, useMemo } from "react"; +import type { StoreApi, UseBoundStore } from "zustand"; +import type { + BenchmarkDashboardState, + CommitMeta, + TimeRange, +} from "./benchmark_regression_store"; +import { createDashboardStore } from "./benchmark_regression_store"; + +// The context holds the Zustand *hook* returned by createDashboardStore +type DashboardStoreHook = UseBoundStore>; +const DashboardContext = createContext(null); + +export function DashboardStoreProvider({ + children, + initial, +}: { + children: React.ReactNode; + initial: { + time: TimeRange; + filters: Record; + lcommit?: CommitMeta; + rcommit?: CommitMeta; + }; +}) { + const store = useMemo( + () => createDashboardStore(initial), + [initial.time, initial.filters] + ); + return ( + + {children} + + ); +} + +export function useDashboardStore(): DashboardStoreHook { + const ctx = useContext(DashboardContext); + if (!ctx) throw new Error("DashboardStoreProvider missing"); + return ctx; +} + +export function useDashboardSelector( + selector: (s: BenchmarkDashboardState) => T +): T { + const useStore = useDashboardStore(); + return useStore(selector); +} diff --git a/torchci/lib/benchmark/store/benchmark_regression_store.ts b/torchci/lib/benchmark/store/benchmark_regression_store.ts new file mode 100644 index 0000000000..e79615bcd1 --- /dev/null +++ b/torchci/lib/benchmark/store/benchmark_regression_store.ts @@ -0,0 +1,90 @@ +// benchmark_regression_store.ts +import type { Dayjs } from "dayjs"; +import { create } from 'zustand'; + +export type TimeRange = { start: Dayjs; end: Dayjs }; +type KV = Record; + +export type CommitMeta = { + commit: string; + date: string; + branch?: string; + workflow_id: string; + index?: number; +}; + +export interface BenchmarkDashboardState { + stagedTime: TimeRange; + stagedFilters: Record; + committedTime: TimeRange; + committedFilters: Record; + + lcommit: CommitMeta | null; + rcommit: CommitMeta | null; + + setStagedTime: (t: TimeRange) => void; + setStagedFilter: (k: string, v: string) => void; + setStagedFilters: (filters: Record) => void; + + commitMainOptions: () => void; + revertMainOptions: () => void; + setLCommit: (c: CommitMeta | null) => void; + setRCommit: (c: CommitMeta | null) => void; + + reset: (initial: { + time: TimeRange; + filters: Record; + lcommit?: CommitMeta | null; + rcommit?: CommitMeta | null; + }) => void; +} + +export function createDashboardStore(initial: { + time: TimeRange; + filters: Record; + lcommit?: CommitMeta; + rcommit?: CommitMeta; +}) { + return create((set, get) => ({ + stagedTime: initial.time, + stagedFilters: initial.filters, + committedTime: initial.time, + committedFilters: initial.filters, + + lcommit: initial.lcommit ?? null, + rcommit: initial.rcommit ?? null, + + setStagedTime: (t) => set({ stagedTime: t }), + setStagedFilter: (k, v) => + set((s) => ({ stagedFilters: { ...s.stagedFilters, [k]: v } })), + setStagedFilters: (filters) => + set((state) => ({ + stagedFilters: { ...state.stagedFilters, ...filters }, + })), + + commitMainOptions: () => + set({ + committedTime: get().stagedTime, + committedFilters: get().stagedFilters, + }), + + revertMainOptions: () => + set({ + stagedTime: get().committedTime, + stagedFilters: get().committedFilters, + }), + + setLCommit: (c) => set({ lcommit: c }), + setRCommit: (c) => set({ rcommit: c }), + + reset: (next) => + set({ + stagedTime: next.time, + committedTime: next.time, + stagedFilters: next.filters, + committedFilters: next.filters, + lcommit: next.lcommit ?? null, + rcommit: next.rcommit ?? null, + }), + })); +} diff --git a/torchci/package.json b/torchci/package.json index c2f4249f4d..891d35c0d7 100644 --- a/torchci/package.json +++ b/torchci/package.json @@ -69,7 +69,8 @@ "typed-rest-client": "^1.8.9", "urllib": "2.44.0", "uuid": "^8.3.2", - "zod": "^3.25.64" + "zod": "^3.25.64", + "zustand": "^5.0.8" }, "devDependencies": { "@types/argparse": "^2.0.10", diff --git a/torchci/pages/api/benchmark/get_time_series.ts b/torchci/pages/api/benchmark/get_time_series.ts index 8342f2692c..d48262fcee 100644 --- a/torchci/pages/api/benchmark/get_time_series.ts +++ b/torchci/pages/api/benchmark/get_time_series.ts @@ -60,7 +60,7 @@ export default async function handler( async function getBenmarkTimeSeriesData( request_name: string, query_params: any, - formats: string[]= ["time_series"] + formats: string[] = ["time_series"] ) { switch (request_name) { case "compiler_precompute": diff --git a/torchci/pages/api/benchmark/list_commits.ts b/torchci/pages/api/benchmark/list_commits.ts index 8f7c34ded7..9ac7647f2f 100644 --- a/torchci/pages/api/benchmark/list_commits.ts +++ b/torchci/pages/api/benchmark/list_commits.ts @@ -35,13 +35,10 @@ export default async function handler( } } -async function getBenmarkCommits( - request_name: string, - query_params: any, -) { +async function getBenmarkCommits(request_name: string, query_params: any) { switch (request_name) { case "compiler": - return await getCompilerCommits(query_params) + return await getCompilerCommits(query_params); default: throw new Error(`Unsupported request_name: ${request_name}`); } diff --git a/torchci/pages/benchmark/compilers_regression.tsx b/torchci/pages/benchmark/compilers_regression.tsx index b85209707a..43f875fad7 100644 --- a/torchci/pages/benchmark/compilers_regression.tsx +++ b/torchci/pages/benchmark/compilers_regression.tsx @@ -1,193 +1,5 @@ -import { Divider, Skeleton, Stack, Typography } from "@mui/material"; -import utc from "dayjs/plugin/utc"; -import { - MAIN_BRANCH, -} from "components/benchmark/common"; -import { - DEFAULT_DEVICE_NAME, - DISPLAY_NAMES_TO_ARCH_NAMES, - DISPLAY_NAMES_TO_DEVICE_NAMES, - DTYPES, - DTYPES_V2, - MODES_V2, -} from "components/benchmark/compilers/common"; -import { SUITES } from "components/benchmark/compilers/SuitePicker"; -import { - DEFAULT_MODE, - DTypePicker, - ModePicker, - MODES, -} from "components/benchmark/ModeAndDTypePicker"; -import dayjs from "dayjs"; -import { useRouter } from "next/router"; -import { useEffect, useReducer, useState } from "react"; -import { UMDateButtonPicker } from "components/uiModules/UMDateRangePicker"; -import { UMPropReducer } from "components/uiModules/UMPropReducer"; -import { useCompilerData } from "lib/benchmark/api_helper/compilers/type"; -import LoadingPage from "components/common/LoadingPage"; -dayjs.extend(utc); - +import BenchmarkRegressionPage from "components/benchmark/umcompilers/RegressionTrackingPage"; export default function Page() { - const initialDropdownFields = { - mode: DEFAULT_MODE, - dtype: MODES[DEFAULT_MODE], - lBranch: MAIN_BRANCH, - lCommit: "", - rBranch: MAIN_BRANCH, - rCommit: "", - deviceName: DEFAULT_DEVICE_NAME, - device: "cuda", - arch: "h100" - } - - const router = useRouter(); - - const [timeRange, dispatchTimeRange] = useReducer(UMPropReducer, { - start_time : dayjs.utc().startOf("day").subtract(7, "day"), - end_time : dayjs.utc().endOf("day"), - }); - const [dropdowns, dispatchDropdowns] = useReducer(UMPropReducer, initialDropdownFields); - - useEffect(() => { - const { - startTime, - stopTime, - mode, - dtype, - deviceName, - lBranch, - lCommit, - rBranch, - rCommit, - } = router.query; - - if (startTime && stopTime) { - // update time range - dispatchTimeRange({ - type: "UPDATE_FIELDS", - payload: { - start_time: dayjs.utc(startTime as string), - end_time: dayjs.utc(stopTime as string), - }, - }); - } - - // collect dropdown updates only if they exist - const newDropdowns = { - ...dropdowns, - }; - if (mode) newDropdowns.mode = mode as string; - if (dtype) newDropdowns.dtype = dtype as string; - if (deviceName) newDropdowns.deviceName = deviceName as string; - if (lBranch) newDropdowns.lBranch = lBranch as string; - if (lCommit) newDropdowns.lCommit = lCommit as string; - if (rBranch) newDropdowns.rBranch = rBranch as string; - if (rCommit) newDropdowns.rCommit = rCommit as string; - - if (Object.keys(newDropdowns).length > 0) { - dispatchDropdowns({ type: "UPDATE_FIELDS", payload: newDropdowns }); - } -}, [router.query]); - - const granularity = "hour" // hardcoded for now - - const queryParams: { [key: string]: any } = { - commits: [], - branches:["main"], - compilers: [], - arch: DISPLAY_NAMES_TO_ARCH_NAMES[dropdowns.deviceName], - device: DISPLAY_NAMES_TO_DEVICE_NAMES[dropdowns.deviceName], - dtype: dropdowns.dtype, - granularity: granularity, - mode: dropdowns.mode, - startTime: dayjs.utc(timeRange.start_time).format("YYYY-MM-DDTHH:mm:ss"), - stopTime: dayjs.utc(timeRange.end_time).format("YYYY-MM-DDTHH:mm:ss"), - suites: Object.keys(SUITES), - }; - return ( -
- - - TorchInductor Performance DashBoard - - - - { - dispatchTimeRange({ - type: "UPDATE_FIELDS", - payload: { - start_time: start, - end_time: end, - }, - }); - }} - start={timeRange.start_time} - end={timeRange.end_time} - /> - - - -
- -
- - ); -} - -function DataRender(props: any) { - const { data, isLoading, error } = useCompilerData("compiler_precompute",props.queryParams); - if (isLoading) { - return ; - } - if (error) { - return
Error: {error.message}
; - } - return 
{JSON.stringify(data, null, 2)}
; -} - -type DropdownsProps = { - dropdowns: any; - dispatchDropdowns: React.Dispatch; -}; - -function Dropdowns({ - dropdowns, - dispatchDropdowns, -}: DropdownsProps) { - - const setDropdownField = (key: string, value: string) => { - dispatchDropdowns({ type: "UPDATE_FIELDS", payload: { [key]: value } }); - }; - return ( - <> - setDropdownField("mode", val)} - setDType={(val: string) => setDropdownField("dtype", val)} - /> - { - if (val === "notset"){ - setDropdownField("dtype", '') - } else{ - setDropdownField("dtype", val) - } - }} - dtypes={DTYPES_V2} - label="Precision" - /> - setDropdownField("deviceName", val)} - dtypes={Object.keys(DISPLAY_NAMES_TO_DEVICE_NAMES)} - label="Device" - /> - - ); + return ; } diff --git a/torchci/tsconfig.json b/torchci/tsconfig.json index a2dd4661ea..dc94ac2e6e 100644 --- a/torchci/tsconfig.json +++ b/torchci/tsconfig.json @@ -30,7 +30,7 @@ "**/*.ts", "**/*.tsx", ".next/types/**/*.ts" - ], +, "components/uiModules/UMDenseRenderers.tsx" ], "exclude": [ "node_modules" ] diff --git a/torchci/yarn.lock b/torchci/yarn.lock index bf3b9a35e7..8356314535 100644 --- a/torchci/yarn.lock +++ b/torchci/yarn.lock @@ -9451,3 +9451,8 @@ zrender@5.6.1: integrity sha512-OFXkDJKcrlx5su2XbzJvj/34Q3m6PvyCZkVPHGYpcCJ52ek4U/ymZyfuV1nKE23AyBJ51E/6Yr0mhZ7xGTO4ag== dependencies: tslib "2.3.0" + +zustand@^5.0.8: + version "5.0.8" + resolved "https://registry.yarnpkg.com/zustand/-/zustand-5.0.8.tgz#b998a0c088c7027a20f2709141a91cb07ac57f8a" + integrity sha512-gyPKpIaxY9XcO2vSMrLbiER7QMAMGOQZVRdJ6Zi782jkbzZygq5GI9nG8g+sMgitRtndwaBSl7uiqC49o1SSiw==