Skip to content

Commit 2b1319b

Browse files
Runloop Agentclaude
andcommitted
perf(bmj): parallelize scenario run fetching and name resolution in logs
Replace serial per-agent listBenchmarkRunScenarioRuns loop with Promise.allSettled so all agents fetch in parallel. Also parallelize resolveScenarioName calls with Promise.all. This makes loading time proportional to the slowest single agent rather than the sum of all agents. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 8f4145a commit 2b1319b

File tree

1 file changed

+46
-28
lines changed

1 file changed

+46
-28
lines changed

src/commands/benchmark-job/logs.ts

Lines changed: 46 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -302,41 +302,59 @@ export async function downloadBenchmarkJobLogs(
302302
// Build scenario outcome lookup from completed outcomes
303303
const outcomeMap = buildScenarioOutcomeMap(job);
304304

305-
// Gather all scenario log targets across benchmark runs
306-
const targets: ScenarioLogTarget[] = [];
307-
308-
for (const run of runs) {
309-
const agentLabel = run.modelName
310-
? `${run.agentName}:${run.modelName}`
311-
: run.agentName;
312-
console.log(
313-
chalk.dim(`Fetching scenario runs for agent "${agentLabel}"...`),
314-
);
315-
let scenarioRuns = await listBenchmarkRunScenarioRuns(run.benchmarkRunId);
316-
317-
// Apply --scenario filter
305+
// Gather all scenario log targets across benchmark runs, fetching all
306+
// agents' scenario run lists in parallel for speed.
307+
console.log(
308+
chalk.dim(
309+
`Fetching scenario runs for ${runs.length} agent(s) in parallel...`,
310+
),
311+
);
312+
313+
const agentScenarioRuns = await Promise.allSettled(
314+
runs.map((run) => listBenchmarkRunScenarioRuns(run.benchmarkRunId)),
315+
);
316+
317+
// Collect (run, scenarioRun) pairs, applying --scenario filter
318+
const pairs: { run: BenchmarkRunInfo; sr: ScenarioRun }[] = [];
319+
for (let i = 0; i < runs.length; i++) {
320+
const result = agentScenarioRuns[i];
321+
if (result.status === "rejected") {
322+
const agentLabel = runs[i].modelName
323+
? `${runs[i].agentName}:${runs[i].modelName}`
324+
: runs[i].agentName;
325+
console.error(
326+
chalk.yellow(
327+
` Warning: failed to fetch scenario runs for agent "${agentLabel}": ${result.reason}`,
328+
),
329+
);
330+
continue;
331+
}
332+
let scenarioRuns = result.value;
318333
if (options.scenario) {
319334
scenarioRuns = scenarioRuns.filter((sr) => sr.id === options.scenario);
320335
}
321-
322336
for (const sr of scenarioRuns) {
323-
const scenarioName = await resolveScenarioName(
324-
sr.id,
325-
sr.scenario_id,
326-
outcomeMap,
327-
);
328-
targets.push({
329-
agentName: run.agentName,
330-
modelName: run.modelName,
331-
scenarioName,
332-
scenarioRunId: sr.id,
333-
scenarioRun: sr,
334-
outcome: outcomeMap.get(sr.id),
335-
destDir: "", // assigned below
336-
});
337+
pairs.push({ run: runs[i], sr });
337338
}
338339
}
339340

341+
// Resolve all scenario names in parallel (may hit the API for in-progress runs)
342+
const resolvedNames = await Promise.all(
343+
pairs.map(({ sr }) =>
344+
resolveScenarioName(sr.id, sr.scenario_id, outcomeMap),
345+
),
346+
);
347+
348+
const targets: ScenarioLogTarget[] = pairs.map(({ run, sr }, i) => ({
349+
agentName: run.agentName,
350+
modelName: run.modelName,
351+
scenarioName: resolvedNames[i],
352+
scenarioRunId: sr.id,
353+
scenarioRun: sr,
354+
outcome: outcomeMap.get(sr.id),
355+
destDir: "", // assigned below
356+
}));
357+
340358
if (targets.length === 0) {
341359
console.log(chalk.yellow("No scenario runs found to download logs for."));
342360
return;

0 commit comments

Comments
 (0)