[CICD] flagos user tests management

.github/CODEOWNERS

@@ -0,0 +1,42 @@
+# FlagOS DevOps - Code Owners
+
+# Default owners for everything
+* @flagos-ai/devops-team
+
+# CI/CD workflows
+.github/ @flagos-ai/devops-team
+
+# Shared actions
+actions/ @flagos-ai/devops-team
+
+# === User Tests ===
+
+# FlagScale test cases
+flagos-user-tests/tests/flagscale/ @flagos-ai/flagscale-team
+
+# FlagGems test cases
+flagos-user-tests/tests/flaggems/ @flagos-ai/flaggems-team
+
+# FlagCX test cases
+flagos-user-tests/tests/flagcx/ @flagos-ai/flagcx-team
+
+# FlagTree test cases
+flagos-user-tests/tests/flagtree/ @flagos-ai/flagtree-team
+
+# vLLM-FL test cases
+flagos-user-tests/tests/vllm-fl/ @flagos-ai/vllm-team
+
+# vLLM-plugin-FL test cases
+flagos-user-tests/tests/vllm-plugin-fl/ @flagos-ai/vllm-team
+
+# TE-FL test cases
+flagos-user-tests/tests/te-fl/ @flagos-ai/te-team
+
+# Megatron-LM-FL test cases
+flagos-user-tests/tests/megatron-lm-fl/ @flagos-ai/megatron-team
+
+# Experimental test cases
+flagos-user-tests/tests/experimental/ @flagos-ai/devops-team
+
+# Validation tools
+flagos-user-tests/tools/ @flagos-ai/devops-team

.github/ISSUE_TEMPLATE/new_test_case.yml

@@ -0,0 +1,94 @@
+name: New Test Case Submission
+description: Submit a new test case for FlagOS repositories
+title: "[Test Case] "
+labels: ["new-test-case"]
+body:
+  - type: dropdown
+    id: target-repo
+    attributes:
+      label: Target Repository
+      description: Which FlagOS repository is this test case for?
+      options:
+        - FlagScale
+        - FlagGems
+        - FlagCX
+        - FlagTree
+        - vLLM-FL
+        - vLLM-plugin-FL
+        - TE-FL
+        - Megatron-LM-FL
+    validations:
+      required: true
+
+  - type: dropdown
+    id: test-type
+    attributes:
+      label: Test Type
+      description: What type of test is this?
+      options:
+        - train
+        - inference
+        - hetero_train
+        - unit
+        - integration
+        - benchmark
+    validations:
+      required: true
+
+  - type: input
+    id: model-name
+    attributes:
+      label: Model Name
+      description: Name of the model being tested (if applicable)
+      placeholder: e.g., llama2, mixtral, deepseek
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Test Case Description
+      description: Describe what this test case validates
+      placeholder: |
+        This test case validates ...
+    validations:
+      required: true
+
+  - type: textarea
+    id: config
+    attributes:
+      label: Configuration
+      description: Paste the YAML configuration for the test case
+      render: yaml
+    validations:
+      required: true
+
+  - type: textarea
+    id: gold-values
+    attributes:
+      label: Gold Values
+      description: Paste the expected gold values (JSON format)
+      render: json
+
+  - type: textarea
+    id: environment
+    attributes:
+      label: Environment Requirements
+      description: Describe the hardware/software requirements
+      placeholder: |
+        - GPU: 8x A100 80GB
+        - CUDA: 12.1
+        - Python: 3.10
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: checklist
+    attributes:
+      label: Submission Checklist
+      options:
+        - label: I have tested this test case locally
+          required: true
+        - label: I have included gold values (if applicable)
+        - label: I have added a README.md with test description
+          required: true
+        - label: My YAML configuration follows the schema specification
+          required: true

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,37 @@
+## Test Case PR
+
+### Target Repository
+<!-- Which FlagOS repo does this test case target? -->
+- [ ] FlagScale
+- [ ] FlagGems
+- [ ] FlagCX
+- [ ] FlagTree
+- [ ] vLLM-FL
+- [ ] vLLM-plugin-FL
+- [ ] TE-FL
+- [ ] Megatron-LM-FL
+
+### Test Type
+<!-- What kind of test is this? -->
+- [ ] train
+- [ ] inference
+- [ ] hetero_train
+- [ ] unit
+- [ ] integration
+
+### Description
+<!-- Briefly describe what this test case validates -->
+
+
+### Environment Requirements
+<!-- What hardware/software is needed to run this test? -->
+- GPU:
+- CUDA:
+- Python:
+
+### Checklist
+- [ ] YAML configuration passes schema validation
+- [ ] Gold values are included (if applicable)
+- [ ] README.md is present for each test case
+- [ ] Test case has been verified locally
+- [ ] No sensitive data (tokens, passwords, private paths) in configs

.github/scripts/detect_changed_repos.js

@@ -0,0 +1,65 @@
+// Detect which repos have changed test cases.
+//
+// Outputs (via core.setOutput):
+//   changed_cases      — JSON array of case paths (manual single-case dispatch)
+//   changed_repos      — JSON object {repo, task, model} (manual repo dispatch or _none_)
+//   changed_repos_list — JSON array of repo names (auto-detected from PR/push)
+//
+// Called from workflow via:
+//   uses: actions/github-script@v7
+//   with:
+//     script: |
+//       const run = require('./.github/scripts/detect_changed_repos.js');
+//       await run({ github, context, core });
+
+module.exports = async ({ github, context, core }) => {
+  const inputCase = process.env.INPUT_CASE || '';
+  const inputRepo = process.env.INPUT_REPO || '';
+  const inputTask = process.env.INPUT_TASK || '';
+  const inputModel = process.env.INPUT_MODEL || '';
+
+  // Manual dispatch — single case
+  if (inputCase) {
+    core.setOutput('changed_cases', JSON.stringify([inputCase]));
+    return;
+  }
+
+  // Manual dispatch — by repo
+  if (inputRepo) {
+    core.setOutput('changed_repos', JSON.stringify({
+      repo: inputRepo,
+      task: inputTask,
+      model: inputModel,
+    }));
+    return;
+  }
+
+  // Auto-detect from changed files
+  let files = [];
+  if (context.eventName === 'pull_request') {
+    const resp = await github.paginate(
+      github.rest.pulls.listFiles,
+      { owner: context.repo.owner, repo: context.repo.repo, pull_number: context.issue.number }
+    );
+    files = resp.map(f => f.filename);
+  } else {
+    const resp = await github.rest.repos.compareCommits({
+      owner: context.repo.owner, repo: context.repo.repo,
+      base: context.payload.before, head: context.payload.after,
+    });
+    files = resp.data.files.map(f => f.filename);
+  }
+
+  // Extract unique repos from changed paths
+  const repos = new Set();
+  for (const f of files) {
+    const m = f.match(/^flagos-user-tests\/tests\/([^/]+)\//);
+    if (m && m[1] !== 'experimental') repos.add(m[1]);
+  }
+
+  if (repos.size === 0) {
+    core.setOutput('changed_repos', JSON.stringify({ repo: '_none_' }));
+  } else {
+    core.setOutput('changed_repos_list', JSON.stringify([...repos]));
+  }
+};

.github/workflows/nightly_integration.yml

@@ -0,0 +1,94 @@
+name: Nightly Integration Test - User Tests
+
+on:
+  schedule:
+    - cron: "0 2 * * *"
+  workflow_dispatch:
+
+defaults:
+  run:
+    working-directory: flagos-user-tests
+
+jobs:
+  discover-cases:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.resolve.outputs.matrix }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: pip install pyyaml
+
+      - name: Discover all test cases and resolve runner labels
+        id: resolve
+        working-directory: flagos-user-tests
+        run: |
+          python3 -c "
+          import json, os, sys
+          sys.path.insert(0, 'tools')
+          from run_user_tests import list_test_resources
+          from pathlib import Path
+
+          root = Path('.')
+          resources_list = list_test_resources(root)
+
+          matrix_entries = []
+          for entry in resources_list:
+              matrix_entries.append({
+                  'case_path': entry['case_path'],
+                  'runner_labels': json.dumps(entry['runner_labels']),
+              })
+
+          if not matrix_entries:
+              matrix_entries.append({
+                  'case_path': '_none_',
+                  'runner_labels': json.dumps(['ubuntu-latest']),
+              })
+
+          matrix = {'include': matrix_entries}
+          output = json.dumps(matrix)
+          print(f'Matrix: {output}')
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
+              f.write(f'matrix={output}\n')
+          "
+
+  run-tests:
+    needs: discover-cases
+    if: ${{ !contains(needs.discover-cases.outputs.matrix, '_none_') }}
+    runs-on: ${{ fromJson(matrix.runner_labels) }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.discover-cases.outputs.matrix) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install runner dependencies
+        run: pip install pyyaml
+
+      - name: Run test case
+        run: python tools/run_user_tests.py --case ${{ matrix.case_path }}
+
+  notify:
+    needs: run-tests
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Generate summary
+        run: |
+          echo "## Nightly Integration Test Summary" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Run: ${{ github.run_number }}" >> $GITHUB_STEP_SUMMARY
+          echo "Date: $(date -u '+%Y-%m-%d %H:%M UTC')" >> $GITHUB_STEP_SUMMARY

.github/workflows/post_test_cases.yml

@@ -0,0 +1,49 @@
+name: Post Test Cases Report
+
+on:
+  pull_request:
+    branches: [main]
+    types: [closed]
+  workflow_dispatch:
+
+defaults:
+  run:
+    working-directory: flagos-user-tests
+
+jobs:
+  post-report:
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: pip install pyyaml
+
+      - name: Collect test cases
+        run: python tools/collect_test_cases.py --output test_cases_report.json
+
+      - name: Post report
+        uses: ./actions/post-benchmark-report
+        with:
+          backend_url: ${{ secrets.FLAGOPS_BACKEND_URL }}
+          api_token: ${{ secrets.FLAGOPS_API_TOKEN }}
+          report_path: flagos-user-tests/test_cases_report.json
+          list_code: flagops-user-test-cases
+          list_name: FlagOps User Test Cases
+          header_config: >-
+            [
+              {"field": "case_id",    "name": "用例ID",     "required": true,  "sortable": true,  "type": "string"},
+              {"field": "case_name",  "name": "用例名称",   "required": true,  "sortable": false, "type": "string"},
+              {"field": "repo",       "name": "所属子仓库", "required": true,  "sortable": true,  "type": "string"},
+              {"field": "updated_at", "name": "更新时间",   "required": true,  "sortable": true,  "type": "string"}
+            ]
+          fail_on_error: "false"