E2E Test Orchestrator #148
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) Microsoft Corporation. | |
| # Licensed under the MIT License. | |
| name: E2E Test Orchestrator | |
| on: | |
| schedule: | |
| # Run every 6 hours | |
| - cron: '0 */6 * * *' | |
| push: | |
| branches: [main] | |
| # No paths filter - E2E tests should always run to catch regressions | |
| pull_request: | |
| branches: [main] | |
| # No paths filter - E2E tests should always run to catch regressions | |
| workflow_dispatch: | |
| inputs: | |
| sample: | |
| description: 'Sample to test (leave empty for all)' | |
| required: false | |
| type: choice | |
| options: | |
| - '' | |
| - 'python-openai' | |
| - 'python-af' | |
| - 'nodejs-openai' | |
| - 'nodejs-langchain' | |
| - 'dotnet-sk' | |
| - 'dotnet-af' | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| python-openai: | |
| name: Python OpenAI E2E | |
| if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'python-openai' }} | |
| uses: ./.github/workflows/e2e-python-openai.yml | |
| secrets: inherit | |
| python-af: | |
| name: Python Agent Framework E2E | |
| if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'python-af' }} | |
| uses: ./.github/workflows/e2e-python-agent-framework.yml | |
| secrets: inherit | |
| nodejs-openai: | |
| name: Node.js OpenAI E2E | |
| if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'nodejs-openai' }} | |
| uses: ./.github/workflows/e2e-nodejs-openai.yml | |
| secrets: inherit | |
| nodejs-langchain: | |
| name: Node.js LangChain E2E | |
| if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'nodejs-langchain' }} | |
| uses: ./.github/workflows/e2e-nodejs-langchain.yml | |
| secrets: inherit | |
| dotnet-sk: | |
| name: .NET Semantic Kernel E2E | |
| if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'dotnet-sk' }} | |
| uses: ./.github/workflows/e2e-dotnet-semantic-kernel.yml | |
| secrets: inherit | |
| dotnet-af: | |
| name: .NET Agent Framework E2E | |
| if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'dotnet-af' }} | |
| uses: ./.github/workflows/e2e-dotnet-agent-framework.yml | |
| secrets: inherit | |
| # =========================================================================== | |
| # Summary - Posts test results to PR | |
| # This is the ONLY check you need to mark as "Required" in branch protection | |
| # =========================================================================== | |
| e2e-status: | |
| name: E2E Status | |
| runs-on: ubuntu-latest | |
| needs: [python-openai, python-af, nodejs-openai, nodejs-langchain, dotnet-sk, dotnet-af] | |
| if: always() | |
| steps: | |
| - name: Check E2E Status | |
| run: | | |
| echo "Checking E2E test results..." | |
| # Get all job results (skipped jobs are fine, failed jobs are not) | |
| results="${{ needs.python-openai.result }} ${{ needs.python-af.result }} ${{ needs.nodejs-openai.result }} ${{ needs.nodejs-langchain.result }} ${{ needs.dotnet-sk.result }} ${{ needs.dotnet-af.result }}" | |
| echo "Job results: $results" | |
| # Check if any job failed or was cancelled | |
| if echo "$results" | grep -qE "failure|cancelled"; then | |
| echo "❌ One or more E2E tests failed or were cancelled" | |
| exit 1 | |
| fi | |
| echo "✅ All E2E tests passed (or were skipped)" | |
| - name: Generate Summary | |
| if: always() | |
| id: summary | |
| run: | | |
| # Determine overall status | |
| OVERALL_STATUS="✅ All Tests Passed" | |
| if [[ "${{ needs.python-openai.result }}" != "success" ]] || \ | |
| [[ "${{ needs.python-af.result }}" != "success" ]] || \ | |
| [[ "${{ needs.nodejs-openai.result }}" != "success" ]] || \ | |
| [[ "${{ needs.nodejs-langchain.result }}" != "success" ]] || \ | |
| [[ "${{ needs.dotnet-sk.result }}" != "success" ]] || \ | |
| [[ "${{ needs.dotnet-af.result }}" != "success" ]]; then | |
| OVERALL_STATUS="⚠️ Some Tests Failed" | |
| fi | |
| # Generate status icons using if-else for reliability | |
| if [[ "${{ needs.python-openai.result }}" == "success" ]]; then PYTHON_OPENAI_ICON="✅"; else PYTHON_OPENAI_ICON="❌"; fi | |
| if [[ "${{ needs.python-af.result }}" == "success" ]]; then PYTHON_AF_ICON="✅"; else PYTHON_AF_ICON="❌"; fi | |
| if [[ "${{ needs.nodejs-openai.result }}" == "success" ]]; then NODEJS_OPENAI_ICON="✅"; else NODEJS_OPENAI_ICON="❌"; fi | |
| if [[ "${{ needs.nodejs-langchain.result }}" == "success" ]]; then NODEJS_LANGCHAIN_ICON="✅"; else NODEJS_LANGCHAIN_ICON="❌"; fi | |
| if [[ "${{ needs.dotnet-sk.result }}" == "success" ]]; then DOTNET_SK_ICON="✅"; else DOTNET_SK_ICON="❌"; fi | |
| if [[ "${{ needs.dotnet-af.result }}" == "success" ]]; then DOTNET_AF_ICON="✅"; else DOTNET_AF_ICON="❌"; fi | |
| echo "## E2E Test Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Status:** $OVERALL_STATUS" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Sample | Status | Result |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|--------|--------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Python OpenAI | $PYTHON_OPENAI_ICON | ${{ needs.python-openai.result }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Python Agent Framework | $PYTHON_AF_ICON | ${{ needs.python-af.result }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Node.js OpenAI | $NODEJS_OPENAI_ICON | ${{ needs.nodejs-openai.result }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Node.js LangChain | $NODEJS_LANGCHAIN_ICON | ${{ needs.nodejs-langchain.result }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| .NET Semantic Kernel | $DOTNET_SK_ICON | ${{ needs.dotnet-sk.result }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| .NET Agent Framework | $DOTNET_AF_ICON | ${{ needs.dotnet-af.result }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "> 📦 **SDK Versions**: Click on each sample workflow above to view detailed SDK version information in the step summary." >> $GITHUB_STEP_SUMMARY | |
| - name: Post PR Comment | |
| if: always() && github.event_name == 'pull_request' && github.event.pull_request.number | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const pythonOpenaiIcon = '${{ needs.python-openai.result }}' === 'success' ? '✅' : '❌'; | |
| const pythonAfIcon = '${{ needs.python-af.result }}' === 'success' ? '✅' : '❌'; | |
| const nodejsOpenaiIcon = '${{ needs.nodejs-openai.result }}' === 'success' ? '✅' : '❌'; | |
| const nodejsLangchainIcon = '${{ needs.nodejs-langchain.result }}' === 'success' ? '✅' : '❌'; | |
| const dotnetSkIcon = '${{ needs.dotnet-sk.result }}' === 'success' ? '✅' : '❌'; | |
| const dotnetAfIcon = '${{ needs.dotnet-af.result }}' === 'success' ? '✅' : '❌'; | |
| const allPassed = '${{ needs.python-openai.result }}' === 'success' && | |
| '${{ needs.python-af.result }}' === 'success' && | |
| '${{ needs.nodejs-openai.result }}' === 'success' && | |
| '${{ needs.nodejs-langchain.result }}' === 'success' && | |
| '${{ needs.dotnet-sk.result }}' === 'success' && | |
| '${{ needs.dotnet-af.result }}' === 'success'; | |
| const overallStatus = allPassed ? '✅ All E2E Tests Passed' : '⚠️ Some E2E Tests Failed'; | |
| const body = [ | |
| `## ${overallStatus}`, | |
| '', | |
| '| Sample | Status | Result |', | |
| '|--------|--------|--------|', | |
| `| Python OpenAI | ${pythonOpenaiIcon} | ${{ needs.python-openai.result }} |`, | |
| `| Python Agent Framework | ${pythonAfIcon} | ${{ needs.python-af.result }} |`, | |
| `| Node.js OpenAI | ${nodejsOpenaiIcon} | ${{ needs.nodejs-openai.result }} |`, | |
| `| Node.js LangChain | ${nodejsLangchainIcon} | ${{ needs.nodejs-langchain.result }} |`, | |
| `| .NET Semantic Kernel | ${dotnetSkIcon} | ${{ needs.dotnet-sk.result }} |`, | |
| `| .NET Agent Framework | ${dotnetAfIcon} | ${{ needs.dotnet-af.result }} |`, | |
| '', | |
| `> 📦 **SDK Versions**: View the [workflow summary](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}) for detailed SDK version information.`, | |
| '', | |
| `[View full test details](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})` | |
| ].join('\n'); | |
| // Get PR number from event context | |
| const prNumber = ${{ github.event.pull_request.number }}; | |
| console.log(`PR number: ${prNumber}`); | |
| // Find existing comment | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNumber, | |
| }); | |
| console.log(`Found ${comments.length} comments on PR #${prNumber}`); | |
| const botComment = comments.find(comment => | |
| comment.user.login === 'github-actions[bot]' && | |
| comment.body.includes('E2E') | |
| ); | |
| console.log(`Bot comment found: ${!!botComment}, ID: ${botComment?.id}`); | |
| if (botComment) { | |
| // Update existing comment | |
| console.log(`Updating existing comment ${botComment.id}...`); | |
| const result = await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: botComment.id, | |
| body: body | |
| }); | |
| console.log(`Update result: ${result.status}`); | |
| } else { | |
| // Create new comment | |
| console.log('Creating new comment...'); | |
| const result = await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNumber, | |
| body: body | |
| }); | |
| console.log(`Create result: ${result.status}, comment ID: ${result.data.id}`); | |
| } |