[queue time] Add Job Queue Time hook

.github/workflows/update-job-queue-times.yml

@@ -0,0 +1,26 @@
+name: Update job queue times dataset
+
+on:
+  schedule:
+    # Run every 15 minutes
+    - cron: "*/15 * * * *"
+  workflow_dispatch:
+
+defaults:
+  run:
+    working-directory: torchci
+jobs:
+  update-queue-times:
+    runs-on: ubuntu-24.04
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - run: yarn install --frozen-lockfile
+      - name: configure aws credentials
+        id: aws_creds
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_update_queue_times
+          aws-region: us-east-1
+      - run: yarn node scripts/updatJobQueueTimes.mjs

torchci/clickhouse_queries/queued_jobs/query.sql

@@ -16,7 +16,8 @@ SELECT
         job.created_at,
         CURRENT_TIMESTAMP()
     ) AS queue_s,
-    CONCAT(workflow.name, ' / ', job.name) AS name,
+    workflow.name AS workflow_name,
+    job.name AS job_name,
     job.html_url,
     IF(
         LENGTH(job.labels) = 0,

torchci/components/metrics/panels/TablePanel.tsx

@@ -21,6 +21,8 @@ export default function TablePanel({
   helpLink,
   // An optional flag to show the table footer
   showFooter,
+  // A custom function to modify the data before it is passed to table rendering
+  dataModifier,
 }: {
   title: string;
   queryName: string;
@@ -29,6 +31,7 @@ export default function TablePanel({
   dataGridProps: any;
   helpLink?: string;
   showFooter?: boolean;
+  dataModifier?: (data: any) => any;
 }) {
   const url = `/api/clickhouse/${queryName}?parameters=${encodeURIComponent(
     JSON.stringify(queryParams)
@@ -38,10 +41,15 @@ export default function TablePanel({
     refreshInterval: 5 * 60 * 1000, // refresh every 5 minutes
   });
 
+  let inputData = data;
+  if (dataModifier !== undefined && data != undefined) {
+    inputData = dataModifier(data);
+  }
+
   return (
     <TablePanelWithData
       title={title}
-      data={data}
+      data={inputData}
       columns={columns}
       dataGridProps={dataGridProps}
       helpLink={helpLink}

torchci/pages/metrics.tsx

@@ -778,6 +778,12 @@ export default function Page() {
               },
               getRowId: (el: any) => el.html_url,
             }}
+            dataModifier={(data: any[]) => {
+              return data.map((el) => {
+                el.name = el.workflow_name + "/" + el.job_name;
+                return el;
+              });
+            }}
           />
         </Grid2>
 

torchci/scripts/updateJobQueueTimes.mjs

@@ -0,0 +1,30 @@
+// We compute queue times by looking at a snapshot of jobs in CI that are
+// currently queued and seeing how long they've existed. This approach doesn't
+// give us historical data, so write our snapshot regularly to s3 so we can get
+// a view of the queue over time.
+// this script is used to update the job queue times in s3 bucket for each job.
+import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+
+export function getS3Client() {
+  return new S3Client({
+    region: "us-east-1",
+  });
+}
+
+const s3client = getS3Client();
+
+// %7B%7D = encoded {}
+const response = await fetch(
+  "https://hud.pytorch.org/api/clickhouse/queued_jobs?parameters=%7B%7D"
+).then((r) => r.json());
+
+const unixTime = Math.floor(Date.now() / 1000);
+const json_records = response.map((item) => JSON.stringify(item)).join("\n");
+
+s3client.send(
+  new PutObjectCommand({
+    Bucket: "ossci-raw-job-status",
+    Key: `job_queue_times_historical/${unixTime}.txt`,
+    Body: json_records,
+  })
+);