From 253de53f0fe5b5e57ca78ba03336870392a11afd Mon Sep 17 00:00:00 2001
From: Catherine Lee <csl@fb.com>
Date: Fri, 7 Mar 2025 11:35:18 -0800
Subject: [PATCH 1/5] tc
---
torchci/pages/api/drci/drci.ts | 118 ++++++++++++++++-----------------
1 file changed, 57 insertions(+), 61 deletions(-)
diff --git a/torchci/pages/api/drci/drci.ts b/torchci/pages/api/drci/drci.ts
index 81e2abc3f9..9841f69d8b 100644
--- a/torchci/pages/api/drci/drci.ts
+++ b/torchci/pages/api/drci/drci.ts
@@ -127,13 +127,13 @@ export async function updateDrciComments(
NUM_MINUTES + ""
),
]);
-
const workflowsByPR = await reorganizeWorkflows(
OWNER,
repo,
recentWorkflows.concat(workflowsFromPendingComments),
octokit
);
+
const head = get_head_branch(repo);
await addMergeBaseCommits(octokit, repo, head, workflowsByPR);
const sevs = getActiveSEVs(await fetchIssuesByLabel("ci: sev"));
@@ -166,7 +166,6 @@ export async function updateDrciComments(
pr_info.repo,
pr_info.pr_number
);
-
const {
pending,
failedJobs,
@@ -1110,69 +1109,63 @@ export async function reorganizeWorkflows(
workflow.name,
])
);
- const workflowsByPR: Map<number, PRandJobs> = new Map();
- const headShaTimestamps: Map<string, string> = new Map();
-
- for (const workflow of dedupedRecentWorkflows) {
- const prNumber = workflow.pr_number;
- if (!workflowsByPR.has(prNumber)) {
- let headShaTimestamp = workflow.head_sha_timestamp;
- // NB: The head SHA timestamp is currently used as the end date when
- // searching for similar failures. However, it's not available on CH for
- // commits from forked PRs before a ciflow ref is pushed. In such case,
- // the head SHA timestamp will be undefined and we will make an additional
- // query to GitHub to get the value
- if (octokit && isTime0(headShaTimestamp)) {
- headShaTimestamp = await fetchCommitTimestamp(
- octokit,
- owner,
- repo,
- workflow.head_sha
- );
- headShaTimestamps.set(workflow.head_sha, headShaTimestamp);
- }
-
- let prTitle = "";
- let prBody = "";
- let prShas: { sha: string; title: string }[] = [];
- // Gate this to PyTorch as disabled tests feature is only available there
- if (octokit && repo === "pytorch") {
- const prData = await fetchPR(owner, repo, `${prNumber}`, octokit);
- prTitle = prData.title;
- prBody = prData.body;
- prShas = prData.shas;
- }
-
- workflowsByPR.set(prNumber, {
- pr_number: prNumber,
- head_sha: workflow.head_sha,
- head_sha_timestamp: headShaTimestamp,
- jobs: [],
- merge_base: "",
- merge_base_date: "",
- owner: owner,
- repo: repo,
- title: prTitle,
- body: prBody,
- shas: prShas,
- });
- }
+ const workflowsByPR: PRandJobs[] = await Promise.all(
+ _(dedupedRecentWorkflows)
+ .groupBy("pr_number")
+ .map(async (workflows, prNumber) => {
+ // NB: The head SHA timestamp is currently used as the end date when
+ // searching for similar failures. However, it's not available on CH for
+ // commits from forked PRs before a ciflow ref is pushed. In such case,
+ // the head SHA timestamp will be undefined and we will make an additional
+ // query to GitHub to get the value
+ let headShaTimestamp = workflows.find(
+ (workflow) => !isTime0(workflow.head_sha_timestamp)
+ )?.head_sha_timestamp;
+ if (octokit && headShaTimestamp === undefined) {
+ headShaTimestamp = await fetchCommitTimestamp(
+ octokit,
+ owner,
+ repo,
+ workflows[0].head_sha
+ );
+ }
- const headShaTimestamp = headShaTimestamps.get(workflow.head_sha);
- if (
- isTime0(workflow.head_sha_timestamp) &&
- headShaTimestamp &&
- !isTime0(headShaTimestamp)
- ) {
- workflow.head_sha_timestamp = headShaTimestamp;
- }
+ workflows.forEach((workflow) => {
+ if (isTime0(workflow.head_sha_timestamp) && headShaTimestamp) {
+ workflow.head_sha_timestamp = headShaTimestamp;
+ }
+ });
- workflowsByPR.get(prNumber)!.jobs.push(workflow);
- }
+ let prTitle = "";
+ let prBody = "";
+ let prShas: { sha: string; title: string }[] = [];
+ // Gate this to PyTorch as disabled tests feature is only available there
+ if (octokit && repo === "pytorch") {
+ const prData = await fetchPR(owner, repo, `${prNumber}`, octokit);
+ prTitle = prData.title;
+ prBody = prData.body;
+ prShas = prData.shas;
+ }
+ return {
+ pr_number: parseInt(prNumber),
+ head_sha: workflows[0].head_sha,
+ head_sha_timestamp: headShaTimestamp ?? "",
+ jobs: workflows,
+ merge_base: "",
+ merge_base_date: "",
+ owner: owner,
+ repo: repo,
+ title: prTitle,
+ body: prBody,
+ shas: prShas,
+ };
+ })
+ .value()
+ );
// clean up the workflows - remove retries, remove workflows that have jobs,
// remove cancelled jobs with weird names
- for (const [, prInfo] of workflowsByPR) {
+ for (const prInfo of workflowsByPR) {
const [workflows, jobs] = _.partition(
prInfo.jobs,
(job) => job.workflowId === 0
@@ -1220,5 +1213,8 @@ export async function reorganizeWorkflows(
// Remove cancelled jobs with weird names
prInfo.jobs = removeCancelledJobAfterRetry<RecentWorkflowsData>(allJobs);
}
- return workflowsByPR;
+ return workflowsByPR.reduce((acc, prInfo) => {
+ acc.set(prInfo.pr_number, prInfo);
+ return acc;
+ }, new Map<number, PRandJobs>());
}
From 9ceeb3ccaa335e49cb0b6a60ec4f4d149d2a34f7 Mon Sep 17 00:00:00 2001
From: Catherine Lee <csl@fb.com>
Date: Fri, 7 Mar 2025 11:36:33 -0800
Subject: [PATCH 2/5] tc
---
torchci/pages/api/drci/drci.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/torchci/pages/api/drci/drci.ts b/torchci/pages/api/drci/drci.ts
index 9841f69d8b..af3eec15bc 100644
--- a/torchci/pages/api/drci/drci.ts
+++ b/torchci/pages/api/drci/drci.ts
@@ -127,13 +127,13 @@ export async function updateDrciComments(
NUM_MINUTES + ""
),
]);
+
const workflowsByPR = await reorganizeWorkflows(
OWNER,
repo,
recentWorkflows.concat(workflowsFromPendingComments),
octokit
);
-
const head = get_head_branch(repo);
await addMergeBaseCommits(octokit, repo, head, workflowsByPR);
const sevs = getActiveSEVs(await fetchIssuesByLabel("ci: sev"));
From b365094f00f6d3aa0b4811444f6ea12a020326bd Mon Sep 17 00:00:00 2001
From: Catherine Lee <csl@fb.com>
Date: Fri, 7 Mar 2025 11:36:42 -0800
Subject: [PATCH 3/5] tc
---
torchci/pages/api/drci/drci.ts | 1 +
1 file changed, 1 insertion(+)
diff --git a/torchci/pages/api/drci/drci.ts b/torchci/pages/api/drci/drci.ts
index af3eec15bc..a0222c7975 100644
--- a/torchci/pages/api/drci/drci.ts
+++ b/torchci/pages/api/drci/drci.ts
@@ -166,6 +166,7 @@ export async function updateDrciComments(
pr_info.repo,
pr_info.pr_number
);
+
const {
pending,
failedJobs,
From 10d9f3ae8fe6801400a2115412807bfdd805809e Mon Sep 17 00:00:00 2001
From: Catherine Lee <csl@fb.com>
Date: Fri, 7 Mar 2025 11:36:54 -0800
Subject: [PATCH 4/5] tc
---
torchci/pages/api/drci/drci.ts | 1 -
1 file changed, 1 deletion(-)
diff --git a/torchci/pages/api/drci/drci.ts b/torchci/pages/api/drci/drci.ts
index a0222c7975..efdedc87a3 100644
--- a/torchci/pages/api/drci/drci.ts
+++ b/torchci/pages/api/drci/drci.ts
@@ -1130,7 +1130,6 @@ export async function reorganizeWorkflows(
workflows[0].head_sha
);
}
-
workflows.forEach((workflow) => {
if (isTime0(workflow.head_sha_timestamp) && headShaTimestamp) {
workflow.head_sha_timestamp = headShaTimestamp;
From 6588e3656d9c9d4bf8d58c6f912fcbbb3bec963d Mon Sep 17 00:00:00 2001
From: Catherine Lee <csl@fb.com>
Date: Fri, 7 Mar 2025 11:37:01 -0800
Subject: [PATCH 5/5] tc
---
torchci/pages/api/drci/drci.ts | 1 +
1 file changed, 1 insertion(+)
diff --git a/torchci/pages/api/drci/drci.ts b/torchci/pages/api/drci/drci.ts
index efdedc87a3..afdb0a2a8c 100644
--- a/torchci/pages/api/drci/drci.ts
+++ b/torchci/pages/api/drci/drci.ts
@@ -1146,6 +1146,7 @@ export async function reorganizeWorkflows(
prBody = prData.body;
prShas = prData.shas;
}
+
return {
pr_number: parseInt(prNumber),
head_sha: workflows[0].head_sha,