ci(github): add workflow to check ci stability of release branches (#…

…12256)

## Motivation

We decided to run daily tests for release branches to monitor stability.
This change adds a new workflow in the master branch that triggers CI on
release branches using the `workflow_dispatch` event via matrix with
`max-parallel` set to `1` to make sure that these runs don't run in
parallel but one after the other.

Each branch's `build-test-distribute` workflow was already updated
earlier to support `workflow_dispatch`. The new master workflow uses the
GitHub API to trigger these workflows. This ensures we are able to check
branch stability with minimal changes and no interruptions to ongoing
work.

## Test run

https://github.com/kumahq/kuma/actions/runs/12355975391

## Supporting documentation

<!-- Is there a MADR? An Issue? A related PR? -->

Part of:
- #12164
- #12163

<!--
> Changelog: skip
-->
<!--
Uncomment the above section to explicitly set a [`> Changelog:` entry
here](https://github.com/kumahq/kuma/blob/master/CONTRIBUTING.md#submitting-a-patch)?
-->

---------

Signed-off-by: Bart Smykla <[email protected]>

.github/workflows/ci-stability-release-branches.yaml

@@ -0,0 +1,183 @@
+name: ci-stability-release-branches
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 19 * * *" # Once a day at 19:00 UTC
+
+permissions: {}
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
+
+jobs:
+  get-active-release-branches:
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    outputs:
+      branches: ${{ steps.get-branches.outputs.result }}
+    steps:
+      - name: "Get active branches"
+        id: get-branches
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.1.0
+        with:
+          script: |
+            const defaultBranch = context.payload.repository.default_branch;
+
+            const { data } = await github.rest.repos.getContent({
+              ...context.repo,
+              path: 'active-branches.json',
+            });
+
+            const content = Buffer.from(data.content, 'base64').toString('utf-8');
+            const branches = JSON.parse(content).filter(b => b !== defaultBranch);
+
+            if (!Array.isArray(branches) || branches.length === 0) {
+              throw new Error('No active branches to process');
+            }
+            
+            await core.summary
+              .addRaw('Active release branches:', true)
+              .addCodeBlock(JSON.stringify(branches), 'json')
+              .write();
+            
+            return branches;
+
+  trigger-build-test-distribute:
+    needs: get-active-release-branches
+    runs-on: ubuntu-24.04
+    permissions:
+      actions: write # required to trigger workflows
+      checks: read # required to list workflow runs
+    continue-on-error: true
+    strategy:
+      matrix:
+        branch: ${{ fromJSON(needs.get-active-release-branches.outputs.branches) }}
+      fail-fast: false
+      max-parallel: 1
+    env:
+      BRANCH: ${{ matrix.branch }}
+    steps:
+      - name: "Trigger the workflow"
+        id: trigger-workflow
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.1.0
+        with:
+          script: |
+            core.setOutput('started', new Date().toISOString());
+
+            await github.rest.actions.createWorkflowDispatch({
+              ...context.repo,
+              workflow_id: 'build-test-distribute.yaml',
+              ref: process.env.BRANCH,
+            });
+
+            core.setOutput('finished', new Date(Date.now() + 10_000).toISOString());
+
+      - name: "Retrieve workflow run ID"
+        id: get-run-id
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.1.0
+        env:
+          STARTED: ${{ steps.trigger-workflow.outputs.started }}
+          FINISHED: ${{ steps.trigger-workflow.outputs.finished }}
+        with:
+          script: |
+            const maxRetries = 5;
+            let retryCount = 0;
+            let run = null;
+
+            while (retryCount < maxRetries && !run) {
+              console.log(`Checking for workflow run (attempt ${retryCount + 1})...`);
+
+              const response = await github.rest.actions.listWorkflowRuns({
+                ...context.repo,
+                workflow_id: 'build-test-distribute.yaml',
+                branch: process.env.BRANCH,
+                created: `${process.env.STARTED}..${process.env.FINISHED}`,
+                per_page: 1,
+              });
+              
+              if (response.data.workflow_runs.length > 0) {
+                run = response.data.workflow_runs[0];
+              }
+
+              if (!run) {
+                retryCount++;
+                await new Promise(resolve => setTimeout(resolve, 5000));
+              }
+            }
+
+            if (!run) {
+              throw new Error(`Unable to retrieve run ID after ${maxRetries} retries.`);
+            }
+
+            console.log(`Retrieved run ID: ${run.id}`);
+            
+            await core.summary
+              .addHeading(`Run <a href="${run.html_url}">${run.id}</a>`, 4)
+              .write();
+            
+            return run.id
+
+      - name: "Monitor triggered workflow"
+        id: monitor-triggered-workflow
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.1.0
+        timeout-minutes: 120
+        env:
+          RUN_ID: ${{ steps.get-run-id.outputs.result }}
+        with:
+          retries: '5'
+          script: |
+            const interval = 120_000; // Check every 2 minutes
+
+            while (true) {
+              const { data: { jobs } } = await github.rest.actions.listJobsForWorkflowRun({
+                ...context.repo,
+                run_id: process.env.RUN_ID,
+              });
+              
+              if (!jobs.length) {
+                console.log('No jobs in run, which suggests that the run is pending or waiting');
+              } else if (jobs.every(job => job.conclusion !== null)) {
+                console.log('All jobs completed');
+                
+                // Prepare table data for summary
+                const headers = [
+                  { data: 'Job', header: true },
+                  { data: 'Steps', header: true },
+                  { data: 'Conclusion', header: true },
+                ];
+                
+                const rows = jobs.map(job => {
+                  const finishedSteps = job.steps.filter(s => s.conclusion).length;
+                  
+                  return [
+                    { data: `<a href="${job.html_url}">${job.name}</a>` },
+                    { data: `${finishedSteps}/${job.steps.length}` },
+                    { data: job.conclusion || 'in_progress' },
+                  ];
+                });
+                
+                return core.summary.addTable([headers, ...rows]).write();
+              } else {
+                // Log jobs in progress
+                console.table(jobs
+                  .filter(job => job.status !== 'completed')
+                  .map(job => {
+                    const finishedSteps = job.steps.filter(s => s.conclusion).length;
+                    
+                    return {
+                      Job: job.name,
+                      Status: job.status,
+                      Steps: `${finishedSteps}/${job.steps.length}`,
+                      Conclusion: job.conclusion || 'in_progress',
+                    };
+                  }));  
+              }
+              
+              // Wait before checking again
+              const nextCheck = new Date(Date.now() + interval).toLocaleTimeString();
+              console.log(`Next check will be in ${interval / 1_000} seconds, at: ${nextCheck} (UTC)`);
+              await new Promise(resolve => setTimeout(resolve, interval));
+            }