Try workflow dispatch (#76)

* Fix the bug of evaluation is not triggered

* Try workflow dispatch

Author: jiarong0907

.github/workflows/pr-evaluation-run.yml

@@ -1,35 +1,46 @@
 name: Router Submission Evaluation
 
 on:
-  pull_request_target:
-    types: [labeled, synchronize, reopened]
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: Pull request number to evaluate
+        required: true
+        type: string
+      base_ref:
+        description: Base branch ref for evaluation scripts checkout
+        required: true
+        type: string
+      base_sha:
+        description: Base commit SHA for PR diff/evaluation
+        required: true
+        type: string
 
 jobs:
   evaluate-router:
-    if: >-
-      contains(github.event.pull_request.labels.*.name, 'evaluate-requested') &&
-      (
-        github.event.action != 'labeled' ||
-        github.event.label.name == 'evaluate-requested'
-      )
     runs-on: self-hosted
     permissions:
       contents: read
       issues: write
       checks: write
       pull-requests: write
+    env:
+      PR_NUMBER: ${{ inputs.pr_number }}
+      BASE_REF: ${{ inputs.base_ref }}
+      BASE_SHA: ${{ inputs.base_sha }}
+      PR_CHECKOUT_REF: ${{ format('refs/pull/{0}/head', inputs.pr_number) }}
     steps:
       - name: Checkout base repository (for evaluation scripts)
         uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.pull_request.base.ref }}
+          ref: ${{ env.BASE_REF }}
           path: base
           fetch-depth: 0
 
       - name: Checkout PR branch (for prediction file only)
         uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ env.PR_CHECKOUT_REF }}
           path: pr
           fetch-depth: 0
 
@@ -39,8 +50,8 @@ jobs:
         working-directory: pr
         run: |
           set -euo pipefail
-          BASE_REF="${{ github.event.pull_request.base.ref }}"
-          BASE_SHA="${{ github.event.pull_request.base.sha }}"
+          BASE_REF="${{ env.BASE_REF }}"
+          BASE_SHA="${{ env.BASE_SHA }}"
 
           if [[ -z "$BASE_SHA" ]]; then
             echo "Error: Could not determine PR base SHA" >&2
@@ -154,9 +165,9 @@ jobs:
           ROUTERARENA_DATASET_DIR: ${{ github.workspace }}/dataset
         run: |
           set -euo pipefail; trap 'cat evaluation_output.txt' EXIT
-          BASE_SHA="${{ github.event.pull_request.base.sha }}"
+          BASE_SHA="${{ env.BASE_SHA }}"
           uv run python automation/process_pr_submission.py \
-            --pr "${{ github.event.pull_request.number }}" \
+            --pr "${{ env.PR_NUMBER }}" \
             --router "${{ steps.detect.outputs.router }}" \
             --split "${{ steps.detect.outputs.split }}" \
             --base-ref "$BASE_SHA" > evaluation_output.txt 2>&1
@@ -206,28 +217,9 @@ jobs:
             comment += '*Evaluation completed by RouterArena automated workflow*';
 
             await github.rest.issues.createComment({
-              issue_number: context.payload.pull_request.number,
+              issue_number: Number('${{ env.PR_NUMBER }}'),
               owner: context.repo.owner,
               repo: context.repo.repo,
               body: comment
             });
             console.log('Successfully posted evaluation results as PR comment');
-
-      - name: Remove evaluation label
-        if: ${{ always() }}
-        uses: actions/github-script@v7
-        with:
-          script: |
-            try {
-              await github.rest.issues.removeLabel({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: context.payload.pull_request.number,
-                name: 'evaluate-requested'
-              });
-            } catch (error) {
-              // Ignore if label is already removed/missing.
-              if (error.status !== 404) {
-                throw error;
-              }
-            }

.github/workflows/pr-evaluation.yml

@@ -17,6 +17,7 @@ jobs:
       )
     runs-on: self-hosted
     permissions:
+      actions: write
       issues: write
       pull-requests: write
       checks: write
@@ -43,42 +44,32 @@ jobs:
               }
             }
 
-      - name: Add evaluation label to PR
+      - name: Fetch PR details
+        id: pr
         uses: actions/github-script@v7
-        env:
-          HAS_ROUTER_EVAL_BOT_TOKEN: ${{ secrets.ROUTER_EVAL_BOT_TOKEN != '' }}
         with:
-          github-token: ${{ secrets.ROUTER_EVAL_BOT_TOKEN || github.token }}
           script: |
-            const labelName = 'evaluate-requested';
-            const issue_number = context.payload.issue.number;
-            if (process.env.HAS_ROUTER_EVAL_BOT_TOKEN !== 'true') {
-              core.warning('ROUTER_EVAL_BOT_TOKEN is not configured. Labeling may not trigger the evaluation workflow due to GITHUB_TOKEN recursion guard.');
-            }
+            const pr = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.payload.issue.number
+            });
+            core.setOutput('number', String(pr.data.number));
+            core.setOutput('base_ref', pr.data.base.ref);
+            core.setOutput('base_sha', pr.data.base.sha);
 
-            try {
-              await github.rest.issues.addLabels({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number,
-                labels: [labelName]
-              });
-            } catch (error) {
-              // If the label does not exist yet, create it once then retry.
-              if (error.status !== 422) {
-                throw error;
+      - name: Dispatch evaluation workflow
+        uses: actions/github-script@v7
+        with:
+          script: |
+            await github.rest.actions.createWorkflowDispatch({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: 'pr-evaluation-run.yml',
+              ref: '${{ steps.pr.outputs.base_ref }}',
+              inputs: {
+                pr_number: '${{ steps.pr.outputs.number }}',
+                base_ref: '${{ steps.pr.outputs.base_ref }}',
+                base_sha: '${{ steps.pr.outputs.base_sha }}'
               }
-              await github.rest.issues.createLabel({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                name: labelName,
-                color: '0e8a16',
-                description: 'Triggers Router Submission Evaluation workflow'
-              });
-              await github.rest.issues.addLabels({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number,
-                labels: [labelName]
-              });
-            }
+            });