From 10ff9f292efe06882a400e36a13a919510d0d994 Mon Sep 17 00:00:00 2001 From: abdulanu0 Date: Wed, 28 Jan 2026 09:29:03 -0800 Subject: [PATCH 01/15] Add Claude and LangChain samples to E2E workflow - Add nodejs-claude, python-claude, and nodejs-langchain jobs - Add SDK version logging for all samples (Get-SDKVersions.ps1) - Add prettified test conversation output (Emit-TestConversations.ps1) - Add PR comment with test summary (skips manual/scheduled runs) - Update summary job to include all 7 samples with status icons - Add pull-requests: write permission for PR comments --- .github/workflows/e2e-agent-samples.yml | 664 +++++++++++++++++++++++- scripts/e2e/Emit-TestConversations.ps1 | 215 ++++++++ scripts/e2e/Get-SDKVersions.ps1 | 228 ++++++++ 3 files changed, 1093 insertions(+), 14 deletions(-) create mode 100644 scripts/e2e/Emit-TestConversations.ps1 create mode 100644 scripts/e2e/Get-SDKVersions.ps1 diff --git a/.github/workflows/e2e-agent-samples.yml b/.github/workflows/e2e-agent-samples.yml index 7a80f456..2ec73b5d 100644 --- a/.github/workflows/e2e-agent-samples.yml +++ b/.github/workflows/e2e-agent-samples.yml @@ -48,6 +48,7 @@ on: permissions: contents: read + pull-requests: write env: # MCP Authentication - Required for all samples @@ -97,6 +98,13 @@ jobs: working-directory: ${{ env.SAMPLE_PATH }} run: uv sync + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "python" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + - name: Acquire Bearer Token (ROPC) id: token shell: pwsh @@ -210,14 +218,12 @@ jobs: path: ${{ runner.temp }}/TestResults/ retention-days: 30 - - name: Upload Test Conversations + - name: Emit Test Conversations if: always() - uses: actions/upload-artifact@v4 - with: - name: test-conversations-python-openai - path: ${{ runner.temp }}/TestConversations/ - retention-days: 30 - continue-on-error: true + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" - name: Capture Agent Logs if: always() @@ -272,6 +278,13 @@ jobs: } shell: pwsh + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "nodejs" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + - name: Acquire Bearer Token (ROPC) id: token shell: pwsh @@ -366,6 +379,13 @@ jobs: path: ${{ runner.temp }}/TestResults/ retention-days: 30 + - name: Emit Test Conversations + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" + - name: Capture Agent Logs if: always() shell: pwsh @@ -409,6 +429,13 @@ jobs: working-directory: ${{ env.SAMPLE_PATH }} run: dotnet build --no-restore + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "dotnet" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + - name: Acquire Bearer Token (ROPC) id: token shell: pwsh @@ -498,6 +525,13 @@ jobs: path: ${{ runner.temp }}/TestResults/ retention-days: 30 + - name: Emit Test Conversations + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" + - name: Capture Logs if: always() shell: pwsh @@ -541,6 +575,13 @@ jobs: working-directory: ${{ env.SAMPLE_PATH }} run: dotnet build --no-restore + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "dotnet" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + - name: Acquire Bearer Token (ROPC) id: token shell: pwsh @@ -630,6 +671,13 @@ jobs: path: ${{ runner.temp }}/TestResults/ retention-days: 30 + - name: Emit Test Conversations + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" + - name: Capture Logs if: always() shell: pwsh @@ -644,23 +692,611 @@ jobs: -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` -Port ${{ env.AGENT_PORT }} + # =========================================================================== + # Node.js Claude Sample + # =========================================================================== + nodejs-claude: + name: Node.js Claude Agent + runs-on: windows-latest + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'nodejs-claude' }} + + env: + SAMPLE_PATH: nodejs/claude/sample-agent + AGENT_PORT: 3979 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: | + if (Test-Path "package-lock.json") { + npm ci + } else { + npm install + } + if (Test-Path "tsconfig.json") { + npm run build + } + shell: pwsh + + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "nodejs" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate .env configuration + shell: pwsh + run: | + $configMappings = @{ + "NODE_ENV" = "development" + "ANTHROPIC_API_KEY" = "${{ secrets.ANTHROPIC_API_KEY }}" + "connections__service_connection__settings__authType" = "ClientSecret" + "connections__service_connection__settings__clientId" = "${{ secrets.NODEJS_CLAUDE_AGENT_ID }}" + "connections__service_connection__settings__clientSecret" = "${{ secrets.NODEJS_CLAUDE_CLIENT_SECRET }}" + "connections__service_connection__settings__tenantId" = "${{ secrets.TENANT_ID }}" + "connectionsMap__0__serviceUrl" = "*" + "connectionsMap__0__connection" = "service_connection" + } + & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Port ${{ env.AGENT_PORT }} ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "node dist/index.js" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "nodejs" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + throw "Agent process has stopped" + } + } + $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" + $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --filter "Category=HTTP" ` + --logger "trx;LogFileName=test-results.trx" ` + --results-directory "${{ runner.temp }}/TestResults" + env: + AGENT_PORT: ${{ env.AGENT_PORT }} + AGENT_URL: http://localhost:${{ env.AGENT_PORT }} + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-nodejs-claude + path: ${{ runner.temp }}/TestResults/ + retention-days: 30 + + - name: Emit Test Conversations + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" + + - name: Capture Agent Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Cleanup + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` + -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` + -Port ${{ env.AGENT_PORT }} + + # =========================================================================== + # Node.js LangChain Sample + # =========================================================================== + nodejs-langchain: + name: Node.js LangChain Agent + runs-on: windows-latest + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'nodejs-langchain' }} + + env: + SAMPLE_PATH: nodejs/langchain/sample-agent + AGENT_PORT: 3979 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: | + if (Test-Path "package-lock.json") { + npm ci + } else { + npm install + } + if (Test-Path "tsconfig.json") { + npm run build + } + shell: pwsh + + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "nodejs" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate .env configuration + shell: pwsh + run: | + $configMappings = @{ + "NODE_ENV" = "development" + "AZURE_OPENAI_API_KEY" = "${{ secrets.NODEJS_LANGCHAIN_AZURE_OPENAI_API_KEY }}" + "AZURE_OPENAI_ENDPOINT" = "${{ secrets.NODEJS_LANGCHAIN_AZURE_OPENAI_ENDPOINT }}" + "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.NODEJS_LANGCHAIN_AZURE_OPENAI_DEPLOYMENT }}" + "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" + "connections__service_connection__settings__authType" = "ClientSecret" + "connections__service_connection__settings__clientId" = "${{ secrets.NODEJS_LANGCHAIN_AGENT_ID }}" + "connections__service_connection__settings__clientSecret" = "${{ secrets.NODEJS_LANGCHAIN_CLIENT_SECRET }}" + "connections__service_connection__settings__tenantId" = "${{ secrets.TENANT_ID }}" + "connectionsMap__0__serviceUrl" = "*" + "connectionsMap__0__connection" = "service_connection" + } + & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Port ${{ env.AGENT_PORT }} ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "node dist/index.js" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "nodejs" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + throw "Agent process has stopped" + } + } + $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" + $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --filter "Category=HTTP" ` + --logger "trx;LogFileName=test-results.trx" ` + --results-directory "${{ runner.temp }}/TestResults" + env: + AGENT_PORT: ${{ env.AGENT_PORT }} + AGENT_URL: http://localhost:${{ env.AGENT_PORT }} + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-nodejs-langchain + path: ${{ runner.temp }}/TestResults/ + retention-days: 30 + + - name: Emit Test Conversations + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" + + - name: Capture Agent Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Cleanup + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` + -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` + -Port ${{ env.AGENT_PORT }} + + # =========================================================================== + # Python Claude Sample + # =========================================================================== + python-claude: + name: Python Claude Agent + runs-on: windows-latest + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'python-claude' }} + + env: + SAMPLE_PATH: python/claude/sample-agent + AGENT_PORT: 3979 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install uv package manager + run: pip install uv + + - name: Install dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: uv sync + + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "python" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate .env configuration + shell: pwsh + run: | + $configMappings = @{ + "ANTHROPIC_API_KEY" = "${{ secrets.ANTHROPIC_API_KEY }}" + "AGENT_ID" = "${{ secrets.PYTHON_CLAUDE_AGENT_ID }}" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__AUTHTYPE" = "ClientSecret" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID" = "${{ secrets.PYTHON_CLAUDE_AGENT_ID }}" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET" = "${{ secrets.PYTHON_CLAUDE_CLIENT_SECRET }}" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID" = "${{ secrets.TENANT_ID }}" + "CONNECTIONSMAP__0__SERVICEURL" = "*" + "CONNECTIONSMAP__0__CONNECTION" = "SERVICE_CONNECTION" + "ENABLE_OBSERVABILITY" = "true" + } + & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Port ${{ env.AGENT_PORT }} ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "uv run python start_with_generic_host.py" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "python" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + Write-Host "=== Verifying Agent ===" -ForegroundColor Cyan + + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + Write-Host "Checking agent process (PID: $agentPid)..." -ForegroundColor Gray + + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + + $logFile = "${{ env.SAMPLE_PATH }}/agent.log" + if (Test-Path $logFile) { + Write-Host "Agent logs:" -ForegroundColor Yellow + Get-Content $logFile + } + throw "Agent process has stopped" + } + } + + $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" + Write-Host "Verifying agent at $agentUrl..." -ForegroundColor Gray + $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + shell: pwsh + run: | + dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --filter "Category=HTTP" ` + --logger "trx;LogFileName=test-results.trx" ` + --results-directory "${{ runner.temp }}/TestResults" + env: + AGENT_PORT: ${{ env.AGENT_PORT }} + AGENT_URL: http://localhost:${{ env.AGENT_PORT }} + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-python-claude + path: ${{ runner.temp }}/TestResults/ + retention-days: 30 + + - name: Emit Test Conversations + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" + + - name: Capture Agent Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Cleanup Agent Process + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` + -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` + -Port ${{ env.AGENT_PORT }} + # =========================================================================== # Summary # =========================================================================== summary: name: Test Summary runs-on: ubuntu-latest - needs: [python-openai, nodejs-openai, dotnet-semantic-kernel, dotnet-agent-framework] + needs: [python-openai, nodejs-openai, dotnet-semantic-kernel, dotnet-agent-framework, nodejs-claude, nodejs-langchain, python-claude] if: always() steps: - name: Generate Summary + id: summary run: | + # Determine overall status + OVERALL_STATUS="✅ All Tests Passed" + if [[ "${{ needs.python-openai.result }}" != "success" ]] || \ + [[ "${{ needs.nodejs-openai.result }}" != "success" ]] || \ + [[ "${{ needs.dotnet-semantic-kernel.result }}" != "success" ]] || \ + [[ "${{ needs.dotnet-agent-framework.result }}" != "success" ]] || \ + [[ "${{ needs.nodejs-claude.result }}" != "success" ]] || \ + [[ "${{ needs.nodejs-langchain.result }}" != "success" ]] || \ + [[ "${{ needs.python-claude.result }}" != "success" ]]; then + OVERALL_STATUS="⚠️ Some Tests Failed" + fi + + # Generate status icons + PYTHON_OPENAI_ICON=$([[ "${{ needs.python-openai.result }}" == "success" ]] && echo "✅" || echo "❌") + NODEJS_OPENAI_ICON=$([[ "${{ needs.nodejs-openai.result }}" == "success" ]] && echo "✅" || echo "❌") + DOTNET_SK_ICON=$([[ "${{ needs.dotnet-semantic-kernel.result }}" == "success" ]] && echo "✅" || echo "❌") + DOTNET_AF_ICON=$([[ "${{ needs.dotnet-agent-framework.result }}" == "success" ]] && echo "✅" || echo "❌") + NODEJS_CLAUDE_ICON=$([[ "${{ needs.nodejs-claude.result }}" == "success" ]] && echo "✅" || echo "❌") + NODEJS_LANGCHAIN_ICON=$([[ "${{ needs.nodejs-langchain.result }}" == "success" ]] && echo "✅" || echo "❌") + PYTHON_CLAUDE_ICON=$([[ "${{ needs.python-claude.result }}" == "success" ]] && echo "✅" || echo "❌") + echo "## E2E Test Results" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "| Sample | Status |" >> $GITHUB_STEP_SUMMARY - echo "|--------|--------|" >> $GITHUB_STEP_SUMMARY - echo "| Python OpenAI | ${{ needs.python-openai.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| Node.js OpenAI | ${{ needs.nodejs-openai.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| .NET Semantic Kernel | ${{ needs.dotnet-semantic-kernel.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| .NET Agent Framework | ${{ needs.dotnet-agent-framework.result }} |" >> $GITHUB_STEP_SUMMARY + echo "**Status:** $OVERALL_STATUS" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Sample | Status | Result |" >> $GITHUB_STEP_SUMMARY + echo "|--------|--------|--------|" >> $GITHUB_STEP_SUMMARY + echo "| Python OpenAI | $PYTHON_OPENAI_ICON | ${{ needs.python-openai.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| Python Claude | $PYTHON_CLAUDE_ICON | ${{ needs.python-claude.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| Node.js OpenAI | $NODEJS_OPENAI_ICON | ${{ needs.nodejs-openai.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| Node.js Claude | $NODEJS_CLAUDE_ICON | ${{ needs.nodejs-claude.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| Node.js LangChain | $NODEJS_LANGCHAIN_ICON | ${{ needs.nodejs-langchain.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| .NET Semantic Kernel | $DOTNET_SK_ICON | ${{ needs.dotnet-semantic-kernel.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| .NET Agent Framework | $DOTNET_AF_ICON | ${{ needs.dotnet-agent-framework.result }} |" >> $GITHUB_STEP_SUMMARY + + # Save summary for PR comment + echo "OVERALL_STATUS=$OVERALL_STATUS" >> $GITHUB_OUTPUT + echo "PYTHON_OPENAI_RESULT=${{ needs.python-openai.result }}" >> $GITHUB_OUTPUT + echo "PYTHON_CLAUDE_RESULT=${{ needs.python-claude.result }}" >> $GITHUB_OUTPUT + echo "NODEJS_OPENAI_RESULT=${{ needs.nodejs-openai.result }}" >> $GITHUB_OUTPUT + echo "NODEJS_CLAUDE_RESULT=${{ needs.nodejs-claude.result }}" >> $GITHUB_OUTPUT + echo "NODEJS_LANGCHAIN_RESULT=${{ needs.nodejs-langchain.result }}" >> $GITHUB_OUTPUT + echo "DOTNET_SK_RESULT=${{ needs.dotnet-semantic-kernel.result }}" >> $GITHUB_OUTPUT + echo "DOTNET_AF_RESULT=${{ needs.dotnet-agent-framework.result }}" >> $GITHUB_OUTPUT + + - name: Post PR Comment + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const pythonOpenaiIcon = '${{ needs.python-openai.result }}' === 'success' ? '✅' : '❌'; + const pythonClaudeIcon = '${{ needs.python-claude.result }}' === 'success' ? '✅' : '❌'; + const nodejsOpenaiIcon = '${{ needs.nodejs-openai.result }}' === 'success' ? '✅' : '❌'; + const nodejsClaudeIcon = '${{ needs.nodejs-claude.result }}' === 'success' ? '✅' : '❌'; + const nodejsLangchainIcon = '${{ needs.nodejs-langchain.result }}' === 'success' ? '✅' : '❌'; + const dotnetSkIcon = '${{ needs.dotnet-semantic-kernel.result }}' === 'success' ? '✅' : '❌'; + const dotnetAfIcon = '${{ needs.dotnet-agent-framework.result }}' === 'success' ? '✅' : '❌'; + + const allPassed = '${{ needs.python-openai.result }}' === 'success' && + '${{ needs.python-claude.result }}' === 'success' && + '${{ needs.nodejs-openai.result }}' === 'success' && + '${{ needs.nodejs-claude.result }}' === 'success' && + '${{ needs.nodejs-langchain.result }}' === 'success' && + '${{ needs.dotnet-semantic-kernel.result }}' === 'success' && + '${{ needs.dotnet-agent-framework.result }}' === 'success'; + + const overallStatus = allPassed ? '✅ All E2E Tests Passed' : '⚠️ Some E2E Tests Failed'; + + const body = `## ${overallStatus} + + | Sample | Status | Result | + |--------|--------|--------| + | Python OpenAI | ${pythonOpenaiIcon} | ${{ needs.python-openai.result }} | + | Python Claude | ${pythonClaudeIcon} | ${{ needs.python-claude.result }} | + | Node.js OpenAI | ${nodejsOpenaiIcon} | ${{ needs.nodejs-openai.result }} | + | Node.js Claude | ${nodejsClaudeIcon} | ${{ needs.nodejs-claude.result }} | + | Node.js LangChain | ${nodejsLangchainIcon} | ${{ needs.nodejs-langchain.result }} | + | .NET Semantic Kernel | ${dotnetSkIcon} | ${{ needs.dotnet-semantic-kernel.result }} | + | .NET Agent Framework | ${dotnetAfIcon} | ${{ needs.dotnet-agent-framework.result }} | + + [View full test details](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}) + `; + + // Find existing comment + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const botComment = comments.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('E2E Tests') + ); + + if (botComment) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: body + }); + } else { + // Create new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body + }); + } diff --git a/scripts/e2e/Emit-TestConversations.ps1 b/scripts/e2e/Emit-TestConversations.ps1 new file mode 100644 index 00000000..9f5bb92d --- /dev/null +++ b/scripts/e2e/Emit-TestConversations.ps1 @@ -0,0 +1,215 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Emits test conversation results in a prettified format to GitHub Actions. + +.DESCRIPTION + This script reads the test-conversations.json file generated by the E2E tests + and outputs a formatted table showing prompts, responses, pass/fail status, + and any error messages. The output is written to both console and GitHub + Actions step summary. + +.PARAMETER TestResultsDir + The directory containing the test-conversations.json file + +.EXAMPLE + .\Emit-TestConversations.ps1 -TestResultsDir "$env:RUNNER_TEMP/TestConversations" +#> + +param( + [Parameter(Mandatory = $true)] + [string]$TestResultsDir +) + +$ErrorActionPreference = 'Continue' + +Write-Host "=== Test Conversations Report ===" -ForegroundColor Cyan +Write-Host "Results Directory: $TestResultsDir" -ForegroundColor Gray +Write-Host "" + +$jsonPath = Join-Path $TestResultsDir "test-conversations.json" + +if (-not (Test-Path $jsonPath)) { + Write-Host "No test-conversations.json found at $jsonPath" -ForegroundColor Yellow + Write-Host "This may indicate tests did not run or did not record conversations." -ForegroundColor Yellow + exit 0 +} + +try { + $results = Get-Content $jsonPath -Raw | ConvertFrom-Json +} +catch { + Write-Host "Failed to parse test-conversations.json: $_" -ForegroundColor Red + exit 1 +} + +# Console output +Write-Host "" +Write-Host "Summary:" -ForegroundColor Green +Write-Host " Total Tests: $($results.TotalTests)" +Write-Host " Passed: $($results.PassedTests)" -ForegroundColor Green +Write-Host " Failed: $($results.FailedTests)" -ForegroundColor $(if ($results.FailedTests -gt 0) { 'Red' } else { 'Green' }) +Write-Host " Generated At: $($results.GeneratedAt)" +Write-Host "" + +# Detailed console output +Write-Host "Test Details:" -ForegroundColor Yellow +Write-Host ("-" * 80) + +foreach ($conv in $results.Conversations) { + $statusIcon = if ($conv.Passed) { "✅" } else { "❌" } + $statusColor = if ($conv.Passed) { "Green" } else { "Red" } + + Write-Host "" + Write-Host "$statusIcon $($conv.TestName)" -ForegroundColor $statusColor + Write-Host " Duration: $($conv.Duration)" + + foreach ($turn in $conv.Turns) { + $promptPreview = if ($turn.UserMessage.Length -gt 60) { + $turn.UserMessage.Substring(0, 60) + "..." + } else { + $turn.UserMessage + } + + $responsePreview = if ($turn.AgentResponse) { + if ($turn.AgentResponse.Length -gt 80) { + $turn.AgentResponse.Substring(0, 80) + "..." + } else { + $turn.AgentResponse + } + } else { + "(no response)" + } + + Write-Host " Prompt: $promptPreview" -ForegroundColor Cyan + Write-Host " Response: $responsePreview" -ForegroundColor Gray + } + + if (-not $conv.Passed -and $conv.ErrorMessage) { + Write-Host " Error: $($conv.ErrorMessage)" -ForegroundColor Red + } +} + +Write-Host "" +Write-Host ("-" * 80) + +# GitHub Actions Step Summary +if ($env:GITHUB_STEP_SUMMARY) { + Write-Host "Writing to GitHub Step Summary..." -ForegroundColor Gray + + $passedIcon = if ($results.FailedTests -eq 0) { "✅" } else { "⚠️" } + + $summary = @" +## $passedIcon Test Conversations Report + +**Summary:** $($results.PassedTests)/$($results.TotalTests) tests passed + +| Test | Status | Prompt | Response | Duration | Error | +|------|--------|--------|----------|----------|-------| +"@ + + foreach ($conv in $results.Conversations) { + $statusIcon = if ($conv.Passed) { "✅ Pass" } else { "❌ Fail" } + + foreach ($turn in $conv.Turns) { + # Escape pipe characters and limit length for markdown table + $promptDisplay = $turn.UserMessage -replace '\|', '\|' -replace '\n', ' ' + if ($promptDisplay.Length -gt 50) { + $promptDisplay = $promptDisplay.Substring(0, 47) + "..." + } + + $responseDisplay = if ($turn.AgentResponse) { + $resp = $turn.AgentResponse -replace '\|', '\|' -replace '\n', ' ' + if ($resp.Length -gt 60) { + $resp.Substring(0, 57) + "..." + } else { + $resp + } + } else { + "_(no response)_" + } + + $durationDisplay = if ($conv.Duration) { + # Parse duration string and format + $conv.Duration -replace '00:00:', '' -replace '(\d+\.\d{2})\d+', '$1s' + } else { + "-" + } + + $errorDisplay = if ($conv.ErrorMessage) { + $err = $conv.ErrorMessage -replace '\|', '\|' -replace '\n', ' ' + if ($err.Length -gt 40) { + $err.Substring(0, 37) + "..." + } else { + $err + } + } else { + "-" + } + + $summary += "`n| $($conv.TestName) | $statusIcon | $promptDisplay | $responseDisplay | $durationDisplay | $errorDisplay |" + } + } + + # Add expandable details section for full responses + $summary += @" + +
+📝 Full Test Details (click to expand) + +"@ + + foreach ($conv in $results.Conversations) { + $statusIcon = if ($conv.Passed) { "✅" } else { "❌" } + + $summary += @" + +### $statusIcon $($conv.TestName) + +- **Status:** $(if ($conv.Passed) { 'Passed' } else { 'Failed' }) +- **Duration:** $($conv.Duration) +- **Timestamp:** $($conv.Timestamp) + +"@ + + $turnNum = 1 + foreach ($turn in $conv.Turns) { + $summary += @" +**Turn $turnNum:** +- **Prompt:** ``$($turn.UserMessage)`` +- **Response:** +`````` +$($turn.AgentResponse ?? '(no response)') +`````` + +"@ + $turnNum++ + } + + if (-not $conv.Passed -and $conv.ErrorMessage) { + $summary += @" +> ⚠️ **Error:** $($conv.ErrorMessage) + +"@ + } + } + + $summary += @" +
+"@ + + $summary | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8 + Write-Host "Step summary written successfully" -ForegroundColor Green +} + +# Return exit code based on test results +if ($results.FailedTests -gt 0) { + Write-Host "" + Write-Host "⚠️ $($results.FailedTests) test(s) failed" -ForegroundColor Yellow +} +else { + Write-Host "" + Write-Host "✅ All tests passed!" -ForegroundColor Green +} diff --git a/scripts/e2e/Get-SDKVersions.ps1 b/scripts/e2e/Get-SDKVersions.ps1 new file mode 100644 index 00000000..0c1049a6 --- /dev/null +++ b/scripts/e2e/Get-SDKVersions.ps1 @@ -0,0 +1,228 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Gets the installed SDK versions for Agent 365 and Microsoft Agents packages. + +.DESCRIPTION + This script retrieves and logs the version numbers of installed SDK packages + for Python, Node.js, or .NET runtimes. It outputs the versions to the console + and to GitHub Actions step summary if running in CI. + +.PARAMETER Runtime + The runtime to check: 'python', 'nodejs', or 'dotnet' + +.PARAMETER WorkingDirectory + The working directory containing the project files + +.EXAMPLE + .\Get-SDKVersions.ps1 -Runtime python -WorkingDirectory "python/openai/sample-agent" +#> + +param( + [Parameter(Mandatory = $true)] + [ValidateSet('python', 'nodejs', 'dotnet')] + [string]$Runtime, + + [Parameter(Mandatory = $true)] + [string]$WorkingDirectory +) + +$ErrorActionPreference = 'Continue' + +Write-Host "=== SDK Version Information ===" -ForegroundColor Cyan +Write-Host "Runtime: $Runtime" -ForegroundColor Gray +Write-Host "Directory: $WorkingDirectory" -ForegroundColor Gray +Write-Host "" + +$versions = @() + +switch ($Runtime) { + 'python' { + Write-Host "Checking Python package versions..." -ForegroundColor Yellow + + Push-Location $WorkingDirectory + try { + # Get installed packages using uv + $pipList = uv pip list --format=json 2>$null | ConvertFrom-Json + + if ($pipList) { + # Microsoft Agents SDK packages + $agentsSdkPackages = $pipList | Where-Object { + $_.name -like 'microsoft-agents-*' + } + + # Agent 365 SDK packages + $a365Packages = $pipList | Where-Object { + $_.name -like 'microsoft-agents-a365-*' -or + $_.name -like 'microsoft_agents_a365_*' + } + + Write-Host "" + Write-Host "Microsoft Agents SDK Packages:" -ForegroundColor Green + foreach ($pkg in $agentsSdkPackages) { + Write-Host " $($pkg.name): $($pkg.version)" + $versions += [PSCustomObject]@{ + Category = "Microsoft Agents SDK" + Package = $pkg.name + Version = $pkg.version + } + } + + Write-Host "" + Write-Host "Microsoft Agent 365 SDK Packages:" -ForegroundColor Green + foreach ($pkg in $a365Packages) { + Write-Host " $($pkg.name): $($pkg.version)" + $versions += [PSCustomObject]@{ + Category = "Microsoft Agent 365 SDK" + Package = $pkg.name + Version = $pkg.version + } + } + } + else { + Write-Host "Could not retrieve package list" -ForegroundColor Red + } + } + finally { + Pop-Location + } + } + + 'nodejs' { + Write-Host "Checking Node.js package versions..." -ForegroundColor Yellow + + Push-Location $WorkingDirectory + try { + # Read package.json + $packageJson = Get-Content "package.json" -Raw | ConvertFrom-Json + + # Get installed versions from node_modules + $nodeModulesPath = Join-Path $WorkingDirectory "node_modules" + + # Microsoft Agents SDK packages + $agentsPackages = @( + '@microsoft/agents-hosting', + '@microsoft/agents-activity' + ) + + # Agent 365 SDK packages + $a365Packages = @( + '@microsoft/agents-a365-notifications', + '@microsoft/agents-a365-observability', + '@microsoft/agents-a365-observability-hosting', + '@microsoft/agents-a365-runtime', + '@microsoft/agents-a365-tooling', + '@microsoft/agents-a365-tooling-extensions-claude', + '@microsoft/agents-a365-tooling-extensions-langchain', + '@microsoft/agents-a365-tooling-extensions-openai' + ) + + Write-Host "" + Write-Host "Microsoft Agents SDK Packages:" -ForegroundColor Green + foreach ($pkg in $agentsPackages) { + $pkgPath = Join-Path $nodeModulesPath ($pkg -replace '/', '\') "package.json" + if (Test-Path $pkgPath) { + $pkgJson = Get-Content $pkgPath -Raw | ConvertFrom-Json + Write-Host " $($pkg): $($pkgJson.version)" + $versions += [PSCustomObject]@{ + Category = "Microsoft Agents SDK" + Package = $pkg + Version = $pkgJson.version + } + } + } + + Write-Host "" + Write-Host "Microsoft Agent 365 SDK Packages:" -ForegroundColor Green + foreach ($pkg in $a365Packages) { + $pkgPath = Join-Path $nodeModulesPath ($pkg -replace '/', '\') "package.json" + if (Test-Path $pkgPath) { + $pkgJson = Get-Content $pkgPath -Raw | ConvertFrom-Json + Write-Host " $($pkg): $($pkgJson.version)" + $versions += [PSCustomObject]@{ + Category = "Microsoft Agent 365 SDK" + Package = $pkg + Version = $pkgJson.version + } + } + } + } + finally { + Pop-Location + } + } + + 'dotnet' { + Write-Host "Checking .NET package versions..." -ForegroundColor Yellow + + Push-Location $WorkingDirectory + try { + # Find .csproj file + $csproj = Get-ChildItem -Filter "*.csproj" | Select-Object -First 1 + + if ($csproj) { + # Get package references from project + $packages = dotnet list package --format json 2>$null | ConvertFrom-Json + + if ($packages -and $packages.projects) { + foreach ($project in $packages.projects) { + foreach ($framework in $project.frameworks) { + foreach ($pkg in $framework.topLevelPackages) { + $isAgentsSdk = $pkg.id -like 'Microsoft.Agents.*' + $isA365Sdk = $pkg.id -like 'Microsoft.Agents.A365.*' + + if ($isAgentsSdk -and -not $isA365Sdk) { + Write-Host " $($pkg.id): $($pkg.resolvedVersion)" -ForegroundColor Gray + $versions += [PSCustomObject]@{ + Category = "Microsoft Agents SDK" + Package = $pkg.id + Version = $pkg.resolvedVersion + } + } + elseif ($isA365Sdk) { + Write-Host " $($pkg.id): $($pkg.resolvedVersion)" -ForegroundColor Gray + $versions += [PSCustomObject]@{ + Category = "Microsoft Agent 365 SDK" + Package = $pkg.id + Version = $pkg.resolvedVersion + } + } + } + } + } + } + } + } + finally { + Pop-Location + } + } +} + +# Output to GitHub Actions step summary if available +if ($env:GITHUB_STEP_SUMMARY) { + Write-Host "" + Write-Host "Writing to GitHub Step Summary..." -ForegroundColor Gray + + $summary = @" +## SDK Versions ($Runtime) + +| Category | Package | Version | +|----------|---------|---------| +"@ + + foreach ($v in $versions) { + $summary += "`n| $($v.Category) | ``$($v.Package)`` | ``$($v.Version)`` |" + } + + $summary | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8 +} + +# Output versions as JSON for potential downstream use +$versionsJson = $versions | ConvertTo-Json -Compress +Write-Host "" +Write-Host "SDK_VERSIONS_JSON=$versionsJson" -ForegroundColor Gray + +return $versions From c4620889e1e8aaa83d3427493fa8c254b63b05ab Mon Sep 17 00:00:00 2001 From: abdulanu0 Date: Wed, 28 Jan 2026 09:47:31 -0800 Subject: [PATCH 02/15] fix(e2e): Fix PowerShell variable parsing in Emit-TestConversations.ps1 Use \ instead of \ --- scripts/e2e/Emit-TestConversations.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/e2e/Emit-TestConversations.ps1 b/scripts/e2e/Emit-TestConversations.ps1 index 9f5bb92d..56dbcbdf 100644 --- a/scripts/e2e/Emit-TestConversations.ps1 +++ b/scripts/e2e/Emit-TestConversations.ps1 @@ -177,7 +177,7 @@ if ($env:GITHUB_STEP_SUMMARY) { $turnNum = 1 foreach ($turn in $conv.Turns) { $summary += @" -**Turn $turnNum:** +**Turn ${turnNum}:** - **Prompt:** ``$($turn.UserMessage)`` - **Response:** `````` From b5dc342fcf9cf6da07b447992bb45b75a1e8eac6 Mon Sep 17 00:00:00 2001 From: abdulanu0 Date: Wed, 28 Jan 2026 10:17:17 -0800 Subject: [PATCH 03/15] fix(e2e): Add agentic_scopes to nodejs-claude and nodejs-langchain jobs Required for AgenticAuthorization handler --- .github/workflows/e2e-agent-samples.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e2e-agent-samples.yml b/.github/workflows/e2e-agent-samples.yml index 2ec73b5d..e0f8faa4 100644 --- a/.github/workflows/e2e-agent-samples.yml +++ b/.github/workflows/e2e-agent-samples.yml @@ -761,6 +761,7 @@ jobs: $configMappings = @{ "NODE_ENV" = "development" "ANTHROPIC_API_KEY" = "${{ secrets.ANTHROPIC_API_KEY }}" + "agentic_scopes" = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1/.default" "connections__service_connection__settings__authType" = "ClientSecret" "connections__service_connection__settings__clientId" = "${{ secrets.NODEJS_CLAUDE_AGENT_ID }}" "connections__service_connection__settings__clientSecret" = "${{ secrets.NODEJS_CLAUDE_CLIENT_SECRET }}" @@ -921,6 +922,7 @@ jobs: "AZURE_OPENAI_ENDPOINT" = "${{ secrets.NODEJS_LANGCHAIN_AZURE_OPENAI_ENDPOINT }}" "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.NODEJS_LANGCHAIN_AZURE_OPENAI_DEPLOYMENT }}" "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" + "agentic_scopes" = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1/.default" "connections__service_connection__settings__authType" = "ClientSecret" "connections__service_connection__settings__clientId" = "${{ secrets.NODEJS_LANGCHAIN_AGENT_ID }}" "connections__service_connection__settings__clientSecret" = "${{ secrets.NODEJS_LANGCHAIN_CLIENT_SECRET }}" From db8887bc3bd7c7cf81707ba74aa009d0b9d0cb1b Mon Sep 17 00:00:00 2001 From: abdulanu0 Date: Wed, 28 Jan 2026 10:45:51 -0800 Subject: [PATCH 04/15] Address PR review comments - Fix pipe character escaping in Emit-TestConversations.ps1 (use \\| instead of \|) - Fix Windows-specific path construction in Get-SDKVersions.ps1 (use nested Join-Path) - Fix package filtering to exclude A365 packages from Agents SDK list - Fix bash status icon syntax (use if-else instead of command substitution) --- .github/workflows/e2e-agent-samples.yml | 16 ++++++------- scripts/e2e/Emit-TestConversations.ps1 | 6 ++--- scripts/e2e/Get-SDKVersions.ps1 | 30 ++++++++++++++++++------- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/.github/workflows/e2e-agent-samples.yml b/.github/workflows/e2e-agent-samples.yml index e0f8faa4..c6c31b48 100644 --- a/.github/workflows/e2e-agent-samples.yml +++ b/.github/workflows/e2e-agent-samples.yml @@ -1202,14 +1202,14 @@ jobs: OVERALL_STATUS="⚠️ Some Tests Failed" fi - # Generate status icons - PYTHON_OPENAI_ICON=$([[ "${{ needs.python-openai.result }}" == "success" ]] && echo "✅" || echo "❌") - NODEJS_OPENAI_ICON=$([[ "${{ needs.nodejs-openai.result }}" == "success" ]] && echo "✅" || echo "❌") - DOTNET_SK_ICON=$([[ "${{ needs.dotnet-semantic-kernel.result }}" == "success" ]] && echo "✅" || echo "❌") - DOTNET_AF_ICON=$([[ "${{ needs.dotnet-agent-framework.result }}" == "success" ]] && echo "✅" || echo "❌") - NODEJS_CLAUDE_ICON=$([[ "${{ needs.nodejs-claude.result }}" == "success" ]] && echo "✅" || echo "❌") - NODEJS_LANGCHAIN_ICON=$([[ "${{ needs.nodejs-langchain.result }}" == "success" ]] && echo "✅" || echo "❌") - PYTHON_CLAUDE_ICON=$([[ "${{ needs.python-claude.result }}" == "success" ]] && echo "✅" || echo "❌") + # Generate status icons using if-else for reliability + if [[ "${{ needs.python-openai.result }}" == "success" ]]; then PYTHON_OPENAI_ICON="✅"; else PYTHON_OPENAI_ICON="❌"; fi + if [[ "${{ needs.nodejs-openai.result }}" == "success" ]]; then NODEJS_OPENAI_ICON="✅"; else NODEJS_OPENAI_ICON="❌"; fi + if [[ "${{ needs.dotnet-semantic-kernel.result }}" == "success" ]]; then DOTNET_SK_ICON="✅"; else DOTNET_SK_ICON="❌"; fi + if [[ "${{ needs.dotnet-agent-framework.result }}" == "success" ]]; then DOTNET_AF_ICON="✅"; else DOTNET_AF_ICON="❌"; fi + if [[ "${{ needs.nodejs-claude.result }}" == "success" ]]; then NODEJS_CLAUDE_ICON="✅"; else NODEJS_CLAUDE_ICON="❌"; fi + if [[ "${{ needs.nodejs-langchain.result }}" == "success" ]]; then NODEJS_LANGCHAIN_ICON="✅"; else NODEJS_LANGCHAIN_ICON="❌"; fi + if [[ "${{ needs.python-claude.result }}" == "success" ]]; then PYTHON_CLAUDE_ICON="✅"; else PYTHON_CLAUDE_ICON="❌"; fi echo "## E2E Test Results" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY diff --git a/scripts/e2e/Emit-TestConversations.ps1 b/scripts/e2e/Emit-TestConversations.ps1 index 56dbcbdf..4673d283 100644 --- a/scripts/e2e/Emit-TestConversations.ps1 +++ b/scripts/e2e/Emit-TestConversations.ps1 @@ -115,13 +115,13 @@ if ($env:GITHUB_STEP_SUMMARY) { foreach ($turn in $conv.Turns) { # Escape pipe characters and limit length for markdown table - $promptDisplay = $turn.UserMessage -replace '\|', '\|' -replace '\n', ' ' + $promptDisplay = $turn.UserMessage -replace '\|', '\\|' -replace '\n', ' ' if ($promptDisplay.Length -gt 50) { $promptDisplay = $promptDisplay.Substring(0, 47) + "..." } $responseDisplay = if ($turn.AgentResponse) { - $resp = $turn.AgentResponse -replace '\|', '\|' -replace '\n', ' ' + $resp = $turn.AgentResponse -replace '\|', '\\|' -replace '\n', ' ' if ($resp.Length -gt 60) { $resp.Substring(0, 57) + "..." } else { @@ -139,7 +139,7 @@ if ($env:GITHUB_STEP_SUMMARY) { } $errorDisplay = if ($conv.ErrorMessage) { - $err = $conv.ErrorMessage -replace '\|', '\|' -replace '\n', ' ' + $err = $conv.ErrorMessage -replace '\|', '\\|' -replace '\n', ' ' if ($err.Length -gt 40) { $err.Substring(0, 37) + "..." } else { diff --git a/scripts/e2e/Get-SDKVersions.ps1 b/scripts/e2e/Get-SDKVersions.ps1 index 0c1049a6..a6017314 100644 --- a/scripts/e2e/Get-SDKVersions.ps1 +++ b/scripts/e2e/Get-SDKVersions.ps1 @@ -48,17 +48,19 @@ switch ($Runtime) { $pipList = uv pip list --format=json 2>$null | ConvertFrom-Json if ($pipList) { - # Microsoft Agents SDK packages - $agentsSdkPackages = $pipList | Where-Object { - $_.name -like 'microsoft-agents-*' - } - - # Agent 365 SDK packages + # Agent 365 SDK packages (check first for exclusion from Agents SDK list) $a365Packages = $pipList | Where-Object { $_.name -like 'microsoft-agents-a365-*' -or $_.name -like 'microsoft_agents_a365_*' } + # Microsoft Agents SDK packages (excluding A365 packages) + $agentsSdkPackages = $pipList | Where-Object { + $_.name -like 'microsoft-agents-*' -and + $_.name -notlike 'microsoft-agents-a365-*' -and + $_.name -notlike 'microsoft_agents_a365_*' + } + Write-Host "" Write-Host "Microsoft Agents SDK Packages:" -ForegroundColor Green foreach ($pkg in $agentsSdkPackages) { @@ -122,7 +124,13 @@ switch ($Runtime) { Write-Host "" Write-Host "Microsoft Agents SDK Packages:" -ForegroundColor Green foreach ($pkg in $agentsPackages) { - $pkgPath = Join-Path $nodeModulesPath ($pkg -replace '/', '\') "package.json" + # Handle scoped packages (e.g., @microsoft/agents-sdk) + $pkgPathParts = $pkg -split '/' + $pkgPath = $nodeModulesPath + foreach ($part in $pkgPathParts) { + $pkgPath = Join-Path $pkgPath $part + } + $pkgPath = Join-Path $pkgPath "package.json" if (Test-Path $pkgPath) { $pkgJson = Get-Content $pkgPath -Raw | ConvertFrom-Json Write-Host " $($pkg): $($pkgJson.version)" @@ -137,7 +145,13 @@ switch ($Runtime) { Write-Host "" Write-Host "Microsoft Agent 365 SDK Packages:" -ForegroundColor Green foreach ($pkg in $a365Packages) { - $pkgPath = Join-Path $nodeModulesPath ($pkg -replace '/', '\') "package.json" + # Handle scoped packages (e.g., @microsoft/agents-a365-runtime) + $pkgPathParts = $pkg -split '/' + $pkgPath = $nodeModulesPath + foreach ($part in $pkgPathParts) { + $pkgPath = Join-Path $pkgPath $part + } + $pkgPath = Join-Path $pkgPath "package.json" if (Test-Path $pkgPath) { $pkgJson = Get-Content $pkgPath -Raw | ConvertFrom-Json Write-Host " $($pkg): $($pkgJson.version)" From 5aaab1d0140ac962de31128297fd959ee265f83e Mon Sep 17 00:00:00 2001 From: abdulanu0 Date: Wed, 28 Jan 2026 11:15:50 -0800 Subject: [PATCH 05/15] Remove Claude samples from E2E workflow --- .github/workflows/e2e-agent-samples.yml | 342 +----------------------- 1 file changed, 2 insertions(+), 340 deletions(-) diff --git a/.github/workflows/e2e-agent-samples.yml b/.github/workflows/e2e-agent-samples.yml index c6c31b48..6a87d858 100644 --- a/.github/workflows/e2e-agent-samples.yml +++ b/.github/workflows/e2e-agent-samples.yml @@ -692,164 +692,6 @@ jobs: -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` -Port ${{ env.AGENT_PORT }} - # =========================================================================== - # Node.js Claude Sample - # =========================================================================== - nodejs-claude: - name: Node.js Claude Agent - runs-on: windows-latest - if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'nodejs-claude' }} - - env: - SAMPLE_PATH: nodejs/claude/sample-agent - AGENT_PORT: 3979 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup .NET SDK - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '8.0.x' - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - - - name: Install dependencies - working-directory: ${{ env.SAMPLE_PATH }} - run: | - if (Test-Path "package-lock.json") { - npm ci - } else { - npm install - } - if (Test-Path "tsconfig.json") { - npm run build - } - shell: pwsh - - - name: Log SDK Versions - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` - -Runtime "nodejs" ` - -WorkingDirectory "${{ env.SAMPLE_PATH }}" - - - name: Acquire Bearer Token (ROPC) - id: token - shell: pwsh - run: | - $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` - -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` - -TenantId "${{ secrets.MCP_TENANT_ID }}" ` - -Username "${{ secrets.MCP_TEST_USERNAME }}" ` - -Password "${{ secrets.MCP_TEST_PASSWORD }}" - echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT - echo "::add-mask::$token" - - - name: Copy ToolingManifest.json - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" - - - name: Generate .env configuration - shell: pwsh - run: | - $configMappings = @{ - "NODE_ENV" = "development" - "ANTHROPIC_API_KEY" = "${{ secrets.ANTHROPIC_API_KEY }}" - "agentic_scopes" = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1/.default" - "connections__service_connection__settings__authType" = "ClientSecret" - "connections__service_connection__settings__clientId" = "${{ secrets.NODEJS_CLAUDE_AGENT_ID }}" - "connections__service_connection__settings__clientSecret" = "${{ secrets.NODEJS_CLAUDE_CLIENT_SECRET }}" - "connections__service_connection__settings__tenantId" = "${{ secrets.TENANT_ID }}" - "connectionsMap__0__serviceUrl" = "*" - "connectionsMap__0__connection" = "service_connection" - } - & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` - -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Port ${{ env.AGENT_PORT }} ` - -ConfigMappings $configMappings - - - name: Start Agent - id: start-agent - shell: pwsh - run: | - $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` - -AgentPath "${{ env.SAMPLE_PATH }}" ` - -StartCommand "node dist/index.js" ` - -Port ${{ env.AGENT_PORT }} ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Environment "Development" ` - -Runtime "nodejs" - echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT - - - name: Verify Agent Running - shell: pwsh - run: | - $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" - if ($agentPid) { - try { - $proc = Get-Process -Id $agentPid -ErrorAction Stop - Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green - } catch { - Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red - throw "Agent process has stopped" - } - } - $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" - $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue - Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green - - - name: Restore E2E Test Dependencies - run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" - - - name: Run .NET E2E Tests - shell: pwsh - run: | - dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` - --no-restore ` - --filter "Category=HTTP" ` - --logger "trx;LogFileName=test-results.trx" ` - --results-directory "${{ runner.temp }}/TestResults" - env: - AGENT_PORT: ${{ env.AGENT_PORT }} - AGENT_URL: http://localhost:${{ env.AGENT_PORT }} - TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-results-nodejs-claude - path: ${{ runner.temp }}/TestResults/ - retention-days: 30 - - - name: Emit Test Conversations - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` - -TestResultsDir "${{ runner.temp }}/TestConversations" - - - name: Capture Agent Logs - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" - - - name: Cleanup - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` - -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` - -Port ${{ env.AGENT_PORT }} - # =========================================================================== # Node.js LangChain Sample # =========================================================================== @@ -1011,179 +853,13 @@ jobs: -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` -Port ${{ env.AGENT_PORT }} - # =========================================================================== - # Python Claude Sample - # =========================================================================== - python-claude: - name: Python Claude Agent - runs-on: windows-latest - if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'python-claude' }} - - env: - SAMPLE_PATH: python/claude/sample-agent - AGENT_PORT: 3979 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup .NET SDK - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '8.0.x' - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install uv package manager - run: pip install uv - - - name: Install dependencies - working-directory: ${{ env.SAMPLE_PATH }} - run: uv sync - - - name: Log SDK Versions - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` - -Runtime "python" ` - -WorkingDirectory "${{ env.SAMPLE_PATH }}" - - - name: Acquire Bearer Token (ROPC) - id: token - shell: pwsh - run: | - $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` - -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` - -TenantId "${{ secrets.MCP_TENANT_ID }}" ` - -Username "${{ secrets.MCP_TEST_USERNAME }}" ` - -Password "${{ secrets.MCP_TEST_PASSWORD }}" - echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT - echo "::add-mask::$token" - - - name: Copy ToolingManifest.json - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" - - - name: Generate .env configuration - shell: pwsh - run: | - $configMappings = @{ - "ANTHROPIC_API_KEY" = "${{ secrets.ANTHROPIC_API_KEY }}" - "AGENT_ID" = "${{ secrets.PYTHON_CLAUDE_AGENT_ID }}" - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__AUTHTYPE" = "ClientSecret" - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID" = "${{ secrets.PYTHON_CLAUDE_AGENT_ID }}" - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET" = "${{ secrets.PYTHON_CLAUDE_CLIENT_SECRET }}" - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID" = "${{ secrets.TENANT_ID }}" - "CONNECTIONSMAP__0__SERVICEURL" = "*" - "CONNECTIONSMAP__0__CONNECTION" = "SERVICE_CONNECTION" - "ENABLE_OBSERVABILITY" = "true" - } - & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` - -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Port ${{ env.AGENT_PORT }} ` - -ConfigMappings $configMappings - - - name: Start Agent - id: start-agent - shell: pwsh - run: | - $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` - -AgentPath "${{ env.SAMPLE_PATH }}" ` - -StartCommand "uv run python start_with_generic_host.py" ` - -Port ${{ env.AGENT_PORT }} ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Environment "Development" ` - -Runtime "python" - echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT - - - name: Verify Agent Running - shell: pwsh - run: | - Write-Host "=== Verifying Agent ===" -ForegroundColor Cyan - - $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" - Write-Host "Checking agent process (PID: $agentPid)..." -ForegroundColor Gray - - if ($agentPid) { - try { - $proc = Get-Process -Id $agentPid -ErrorAction Stop - Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green - } catch { - Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red - - $logFile = "${{ env.SAMPLE_PATH }}/agent.log" - if (Test-Path $logFile) { - Write-Host "Agent logs:" -ForegroundColor Yellow - Get-Content $logFile - } - throw "Agent process has stopped" - } - } - - $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" - Write-Host "Verifying agent at $agentUrl..." -ForegroundColor Gray - $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue - Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green - - - name: Restore E2E Test Dependencies - shell: pwsh - run: | - dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" - - - name: Run .NET E2E Tests - shell: pwsh - run: | - dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` - --no-restore ` - --filter "Category=HTTP" ` - --logger "trx;LogFileName=test-results.trx" ` - --results-directory "${{ runner.temp }}/TestResults" - env: - AGENT_PORT: ${{ env.AGENT_PORT }} - AGENT_URL: http://localhost:${{ env.AGENT_PORT }} - TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-results-python-claude - path: ${{ runner.temp }}/TestResults/ - retention-days: 30 - - - name: Emit Test Conversations - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` - -TestResultsDir "${{ runner.temp }}/TestConversations" - - - name: Capture Agent Logs - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" - - - name: Cleanup Agent Process - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` - -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` - -Port ${{ env.AGENT_PORT }} - # =========================================================================== # Summary # =========================================================================== summary: name: Test Summary runs-on: ubuntu-latest - needs: [python-openai, nodejs-openai, dotnet-semantic-kernel, dotnet-agent-framework, nodejs-claude, nodejs-langchain, python-claude] + needs: [python-openai, nodejs-openai, dotnet-semantic-kernel, dotnet-agent-framework, nodejs-langchain] if: always() steps: @@ -1196,9 +872,7 @@ jobs: [[ "${{ needs.nodejs-openai.result }}" != "success" ]] || \ [[ "${{ needs.dotnet-semantic-kernel.result }}" != "success" ]] || \ [[ "${{ needs.dotnet-agent-framework.result }}" != "success" ]] || \ - [[ "${{ needs.nodejs-claude.result }}" != "success" ]] || \ - [[ "${{ needs.nodejs-langchain.result }}" != "success" ]] || \ - [[ "${{ needs.python-claude.result }}" != "success" ]]; then + [[ "${{ needs.nodejs-langchain.result }}" != "success" ]]; then OVERALL_STATUS="⚠️ Some Tests Failed" fi @@ -1207,9 +881,7 @@ jobs: if [[ "${{ needs.nodejs-openai.result }}" == "success" ]]; then NODEJS_OPENAI_ICON="✅"; else NODEJS_OPENAI_ICON="❌"; fi if [[ "${{ needs.dotnet-semantic-kernel.result }}" == "success" ]]; then DOTNET_SK_ICON="✅"; else DOTNET_SK_ICON="❌"; fi if [[ "${{ needs.dotnet-agent-framework.result }}" == "success" ]]; then DOTNET_AF_ICON="✅"; else DOTNET_AF_ICON="❌"; fi - if [[ "${{ needs.nodejs-claude.result }}" == "success" ]]; then NODEJS_CLAUDE_ICON="✅"; else NODEJS_CLAUDE_ICON="❌"; fi if [[ "${{ needs.nodejs-langchain.result }}" == "success" ]]; then NODEJS_LANGCHAIN_ICON="✅"; else NODEJS_LANGCHAIN_ICON="❌"; fi - if [[ "${{ needs.python-claude.result }}" == "success" ]]; then PYTHON_CLAUDE_ICON="✅"; else PYTHON_CLAUDE_ICON="❌"; fi echo "## E2E Test Results" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY @@ -1218,9 +890,7 @@ jobs: echo "| Sample | Status | Result |" >> $GITHUB_STEP_SUMMARY echo "|--------|--------|--------|" >> $GITHUB_STEP_SUMMARY echo "| Python OpenAI | $PYTHON_OPENAI_ICON | ${{ needs.python-openai.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| Python Claude | $PYTHON_CLAUDE_ICON | ${{ needs.python-claude.result }} |" >> $GITHUB_STEP_SUMMARY echo "| Node.js OpenAI | $NODEJS_OPENAI_ICON | ${{ needs.nodejs-openai.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| Node.js Claude | $NODEJS_CLAUDE_ICON | ${{ needs.nodejs-claude.result }} |" >> $GITHUB_STEP_SUMMARY echo "| Node.js LangChain | $NODEJS_LANGCHAIN_ICON | ${{ needs.nodejs-langchain.result }} |" >> $GITHUB_STEP_SUMMARY echo "| .NET Semantic Kernel | $DOTNET_SK_ICON | ${{ needs.dotnet-semantic-kernel.result }} |" >> $GITHUB_STEP_SUMMARY echo "| .NET Agent Framework | $DOTNET_AF_ICON | ${{ needs.dotnet-agent-framework.result }} |" >> $GITHUB_STEP_SUMMARY @@ -1228,9 +898,7 @@ jobs: # Save summary for PR comment echo "OVERALL_STATUS=$OVERALL_STATUS" >> $GITHUB_OUTPUT echo "PYTHON_OPENAI_RESULT=${{ needs.python-openai.result }}" >> $GITHUB_OUTPUT - echo "PYTHON_CLAUDE_RESULT=${{ needs.python-claude.result }}" >> $GITHUB_OUTPUT echo "NODEJS_OPENAI_RESULT=${{ needs.nodejs-openai.result }}" >> $GITHUB_OUTPUT - echo "NODEJS_CLAUDE_RESULT=${{ needs.nodejs-claude.result }}" >> $GITHUB_OUTPUT echo "NODEJS_LANGCHAIN_RESULT=${{ needs.nodejs-langchain.result }}" >> $GITHUB_OUTPUT echo "DOTNET_SK_RESULT=${{ needs.dotnet-semantic-kernel.result }}" >> $GITHUB_OUTPUT echo "DOTNET_AF_RESULT=${{ needs.dotnet-agent-framework.result }}" >> $GITHUB_OUTPUT @@ -1241,17 +909,13 @@ jobs: with: script: | const pythonOpenaiIcon = '${{ needs.python-openai.result }}' === 'success' ? '✅' : '❌'; - const pythonClaudeIcon = '${{ needs.python-claude.result }}' === 'success' ? '✅' : '❌'; const nodejsOpenaiIcon = '${{ needs.nodejs-openai.result }}' === 'success' ? '✅' : '❌'; - const nodejsClaudeIcon = '${{ needs.nodejs-claude.result }}' === 'success' ? '✅' : '❌'; const nodejsLangchainIcon = '${{ needs.nodejs-langchain.result }}' === 'success' ? '✅' : '❌'; const dotnetSkIcon = '${{ needs.dotnet-semantic-kernel.result }}' === 'success' ? '✅' : '❌'; const dotnetAfIcon = '${{ needs.dotnet-agent-framework.result }}' === 'success' ? '✅' : '❌'; const allPassed = '${{ needs.python-openai.result }}' === 'success' && - '${{ needs.python-claude.result }}' === 'success' && '${{ needs.nodejs-openai.result }}' === 'success' && - '${{ needs.nodejs-claude.result }}' === 'success' && '${{ needs.nodejs-langchain.result }}' === 'success' && '${{ needs.dotnet-semantic-kernel.result }}' === 'success' && '${{ needs.dotnet-agent-framework.result }}' === 'success'; @@ -1263,9 +927,7 @@ jobs: | Sample | Status | Result | |--------|--------|--------| | Python OpenAI | ${pythonOpenaiIcon} | ${{ needs.python-openai.result }} | - | Python Claude | ${pythonClaudeIcon} | ${{ needs.python-claude.result }} | | Node.js OpenAI | ${nodejsOpenaiIcon} | ${{ needs.nodejs-openai.result }} | - | Node.js Claude | ${nodejsClaudeIcon} | ${{ needs.nodejs-claude.result }} | | Node.js LangChain | ${nodejsLangchainIcon} | ${{ needs.nodejs-langchain.result }} | | .NET Semantic Kernel | ${dotnetSkIcon} | ${{ needs.dotnet-semantic-kernel.result }} | | .NET Agent Framework | ${dotnetAfIcon} | ${{ needs.dotnet-agent-framework.result }} | From c78abe3ec2b7c06021bbe7d8a77ed33a51261f05 Mon Sep 17 00:00:00 2001 From: abdulanu0 Date: Wed, 28 Jan 2026 11:27:39 -0800 Subject: [PATCH 06/15] Use Node.js OpenAI secrets for LangChain sample --- .github/workflows/e2e-agent-samples.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e-agent-samples.yml b/.github/workflows/e2e-agent-samples.yml index 6a87d858..0f225248 100644 --- a/.github/workflows/e2e-agent-samples.yml +++ b/.github/workflows/e2e-agent-samples.yml @@ -760,14 +760,14 @@ jobs: run: | $configMappings = @{ "NODE_ENV" = "development" - "AZURE_OPENAI_API_KEY" = "${{ secrets.NODEJS_LANGCHAIN_AZURE_OPENAI_API_KEY }}" - "AZURE_OPENAI_ENDPOINT" = "${{ secrets.NODEJS_LANGCHAIN_AZURE_OPENAI_ENDPOINT }}" - "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.NODEJS_LANGCHAIN_AZURE_OPENAI_DEPLOYMENT }}" + "AZURE_OPENAI_API_KEY" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_API_KEY }}" + "AZURE_OPENAI_ENDPOINT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_ENDPOINT }}" + "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_DEPLOYMENT }}" "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" "agentic_scopes" = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1/.default" "connections__service_connection__settings__authType" = "ClientSecret" - "connections__service_connection__settings__clientId" = "${{ secrets.NODEJS_LANGCHAIN_AGENT_ID }}" - "connections__service_connection__settings__clientSecret" = "${{ secrets.NODEJS_LANGCHAIN_CLIENT_SECRET }}" + "connections__service_connection__settings__clientId" = "${{ secrets.NODEJS_OPENAI_AGENT_ID }}" + "connections__service_connection__settings__clientSecret" = "${{ secrets.NODEJS_OPENAI_CLIENT_SECRET }}" "connections__service_connection__settings__tenantId" = "${{ secrets.TENANT_ID }}" "connectionsMap__0__serviceUrl" = "*" "connectionsMap__0__connection" = "service_connection" From 4420d44204c50e651bde13d5b8c826d4e2507762 Mon Sep 17 00:00:00 2001 From: abdulanu0 Date: Wed, 28 Jan 2026 11:40:51 -0800 Subject: [PATCH 07/15] Fix LangChain sample: add health endpoint and read PORT from env --- nodejs/langchain/sample-agent/src/index.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/nodejs/langchain/sample-agent/src/index.ts b/nodejs/langchain/sample-agent/src/index.ts index 584cb028..2e1930d7 100644 --- a/nodejs/langchain/sample-agent/src/index.ts +++ b/nodejs/langchain/sample-agent/src/index.ts @@ -13,6 +13,15 @@ const authConfig: AuthConfiguration = isProduction ? loadAuthConfigFromEnv() : { const server: Express = express() server.use(express.json()) + +// Health endpoint - placed BEFORE auth middleware so it doesn't require authentication +server.get('/api/health', (req, res: Response) => { + res.status(200).json({ + status: 'healthy', + timestamp: new Date().toISOString() + }); +}); + server.use(authorizeJWT(authConfig)) server.post('/api/messages', (req: Request, res: Response) => { @@ -22,7 +31,7 @@ server.post('/api/messages', (req: Request, res: Response) => { }) }) -const port = 3978 +const port = Number(process.env.PORT) || 3978 const host = isProduction ? '0.0.0.0' : '127.0.0.1'; server.listen(port, host, async () => { console.log(`\nServer listening on http://${host}:${port} for appId ${authConfig.clientId} debug ${process.env.DEBUG}`) From 51d59fb626a482707cfa17dae04ddde49e53f5f0 Mon Sep 17 00:00:00 2001 From: abdulanu0 Date: Wed, 28 Jan 2026 11:49:44 -0800 Subject: [PATCH 08/15] Update LangChain sample to support both Azure OpenAI and OpenAI - Add flexible model configuration that checks for Azure OpenAI first - Fall back to regular OpenAI if Azure credentials not present - Update .env.example with both options documented - Provides clear error message if no credentials found --- nodejs/langchain/sample-agent/.env.example | 11 +++++- nodejs/langchain/sample-agent/src/client.ts | 38 +++++++++++++++++++-- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/nodejs/langchain/sample-agent/.env.example b/nodejs/langchain/sample-agent/.env.example index cbb85229..4345c0f2 100644 --- a/nodejs/langchain/sample-agent/.env.example +++ b/nodejs/langchain/sample-agent/.env.example @@ -1,5 +1,14 @@ -# OpenAI Configuration +# LLM Configuration (choose one option) + +# Option 1: Azure OpenAI (preferred for enterprise) +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_ENDPOINT= +AZURE_OPENAI_DEPLOYMENT= +AZURE_OPENAI_API_VERSION=2024-12-01-preview + +# Option 2: OpenAI (if Azure OpenAI not configured) OPENAI_API_KEY= +OPENAI_MODEL=gpt-4o # MCP Tooling Configuration BEARER_TOKEN= diff --git a/nodejs/langchain/sample-agent/src/client.ts b/nodejs/langchain/sample-agent/src/client.ts index a6ef930f..3f0c6251 100644 --- a/nodejs/langchain/sample-agent/src/client.ts +++ b/nodejs/langchain/sample-agent/src/client.ts @@ -2,7 +2,8 @@ // Licensed under the MIT License. import { createAgent, ReactAgent } from "langchain"; -import { ChatOpenAI } from "@langchain/openai"; +import { AzureChatOpenAI, ChatOpenAI } from "@langchain/openai"; +import { BaseChatModel } from "@langchain/core/language_models/chat_models"; // Tooling Imports import { McpToolRegistrationService } from '@microsoft/agents-a365-tooling-extensions-langchain'; @@ -48,8 +49,41 @@ a365Observability.start(); const toolService = new McpToolRegistrationService(); const agentName = "LangChainA365Agent"; + +/** + * Creates the appropriate chat model based on available environment variables. + * Supports both Azure OpenAI and regular OpenAI. + */ +function createChatModel(): BaseChatModel { + // Check for Azure OpenAI configuration first + if (process.env.AZURE_OPENAI_API_KEY && process.env.AZURE_OPENAI_ENDPOINT && process.env.AZURE_OPENAI_DEPLOYMENT) { + console.log('Using Azure OpenAI'); + return new AzureChatOpenAI({ + azureOpenAIApiKey: process.env.AZURE_OPENAI_API_KEY, + azureOpenAIApiInstanceName: process.env.AZURE_OPENAI_ENDPOINT?.replace('https://', '').replace('.openai.azure.com/', '').replace('.openai.azure.com', ''), + azureOpenAIApiDeploymentName: process.env.AZURE_OPENAI_DEPLOYMENT, + azureOpenAIApiVersion: process.env.AZURE_OPENAI_API_VERSION || "2024-12-01-preview", + temperature: 0, + }); + } + + // Fall back to regular OpenAI + if (process.env.OPENAI_API_KEY) { + console.log('Using OpenAI'); + return new ChatOpenAI({ + openAIApiKey: process.env.OPENAI_API_KEY, + modelName: process.env.OPENAI_MODEL || "gpt-4o", + temperature: 0, + }); + } + + throw new Error('No OpenAI credentials found. Please set either AZURE_OPENAI_API_KEY + AZURE_OPENAI_ENDPOINT + AZURE_OPENAI_DEPLOYMENT, or OPENAI_API_KEY.'); +} + +const model = createChatModel(); + const agent = createAgent({ - model: new ChatOpenAI({ temperature: 0 }), + model, name: agentName, systemPrompt: `You are a helpful assistant with access to tools. From 767a7529c3765c2d923015f4b5a5a0899dddc348 Mon Sep 17 00:00:00 2001 From: Rahul Devikar Date: Thu, 29 Jan 2026 16:34:00 -0800 Subject: [PATCH 09/15] Split E2E in its own workflow for better management (#198) * fix: Correct YAML syntax in update-e2e-status workflow * fix: Apply Copilot review suggestions - proper URL encoding, empty string handling, textwrap.dedent, and job count tracking * Split into multiple workflows * refactor: Use workflow_call instead of workflow_dispatch for orchestrator - removes need for actions:write permission * fix: Correct PowerShell parameter names - use -AgentPID instead of -ProcessId, remove invalid -OutputPath and -TestName parameters * fix: Remove invalid test filter - test names BasicConversation and Notification don't exist * chore: Remove deprecated monolithic E2E workflow * chore: Remove deprecated monolithic E2E workflow --- .github/workflows/e2e-agent-samples.yml | 966 ------------------ .../workflows/e2e-dotnet-agent-framework.yml | 153 +++ .../workflows/e2e-dotnet-semantic-kernel.yml | 153 +++ .github/workflows/e2e-nodejs-openai.yml | 169 +++ .github/workflows/e2e-orchestrator.yml | 67 ++ .github/workflows/e2e-python-openai.yml | 187 ++++ .github/workflows/update-e2e-status.yml | 118 --- README.md | 10 +- 8 files changed, 734 insertions(+), 1089 deletions(-) delete mode 100644 .github/workflows/e2e-agent-samples.yml create mode 100644 .github/workflows/e2e-dotnet-agent-framework.yml create mode 100644 .github/workflows/e2e-dotnet-semantic-kernel.yml create mode 100644 .github/workflows/e2e-nodejs-openai.yml create mode 100644 .github/workflows/e2e-orchestrator.yml create mode 100644 .github/workflows/e2e-python-openai.yml delete mode 100644 .github/workflows/update-e2e-status.yml diff --git a/.github/workflows/e2e-agent-samples.yml b/.github/workflows/e2e-agent-samples.yml deleted file mode 100644 index 0f225248..00000000 --- a/.github/workflows/e2e-agent-samples.yml +++ /dev/null @@ -1,966 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -# ============================================================================= -# E2E Tests: Agent Samples -# ============================================================================= -# Runs E2E integration tests for all agent samples -# Uses external PowerShell scripts for maintainability -# ============================================================================= - -name: E2E Agent Samples - -on: - # Run nightly at 6 AM UTC - schedule: - - cron: '0 6 * * *' - push: - branches: - - main - - 'feature/*' - paths: - - 'python/**' - - 'nodejs/**' - - 'dotnet/**' - - 'tests/e2e/**' - - '.github/workflows/e2e-agent-samples.yml' - - 'scripts/e2e/**' - pull_request: - branches: - - main - paths: - - 'python/**' - - 'nodejs/**' - - 'dotnet/**' - - 'tests/e2e/**' - - '.github/workflows/e2e-agent-samples.yml' - - 'scripts/e2e/**' - workflow_dispatch: - inputs: - sample: - description: 'Specific sample to test (leave empty for all)' - required: false - default: '' - debug: - description: 'Enable debug logging' - required: false - default: 'false' - -permissions: - contents: read - pull-requests: write - -env: - # MCP Authentication - Required for all samples - MCP_CLIENT_ID: ${{ secrets.MCP_CLIENT_ID }} - MCP_TENANT_ID: ${{ secrets.MCP_TENANT_ID }} - MCP_TEST_USERNAME: ${{ secrets.MCP_TEST_USERNAME }} - MCP_TEST_PASSWORD: ${{ secrets.MCP_TEST_PASSWORD }} - MCP_ENVIRONMENT: 'Development' - - # Common settings - SCRIPTS_PATH: scripts/e2e - - # E2E Integration Tests - located in this repository - E2E_TESTS_PATH: 'tests/e2e' - -jobs: - # =========================================================================== - # Python OpenAI Sample - # =========================================================================== - python-openai: - name: Python OpenAI Agent - runs-on: windows-latest - if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'python-openai' }} - - env: - SAMPLE_PATH: python/openai/sample-agent - AGENT_PORT: 3979 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup .NET SDK - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '8.0.x' - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install uv package manager - run: pip install uv - - - name: Install dependencies - working-directory: ${{ env.SAMPLE_PATH }} - run: uv sync - - - name: Log SDK Versions - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` - -Runtime "python" ` - -WorkingDirectory "${{ env.SAMPLE_PATH }}" - - - name: Acquire Bearer Token (ROPC) - id: token - shell: pwsh - run: | - $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` - -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` - -TenantId "${{ secrets.MCP_TENANT_ID }}" ` - -Username "${{ secrets.MCP_TEST_USERNAME }}" ` - -Password "${{ secrets.MCP_TEST_PASSWORD }}" - echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT - echo "::add-mask::$token" - - - name: Copy ToolingManifest.json - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" - - - name: Generate .env configuration - shell: pwsh - run: | - $configMappings = @{ - "AZURE_OPENAI_API_KEY" = "${{ secrets.PYTHON_OPENAI_AZURE_OPENAI_API_KEY }}" - "AZURE_OPENAI_ENDPOINT" = "${{ secrets.PYTHON_OPENAI_AZURE_OPENAI_ENDPOINT }}" - "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.PYTHON_OPENAI_AZURE_OPENAI_DEPLOYMENT }}" - "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" - "USE_AGENTIC_AUTH" = "false" - "MCP_PLATFORM_ENDPOINT" = "https://agent365.svc.cloud.microsoft" - "AGENT_ID" = "${{ secrets.PYTHON_OPENAI_AGENT_ID }}" - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__AUTHTYPE" = "ClientSecret" - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID" = "${{ secrets.PYTHON_OPENAI_AGENT_ID }}" - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET" = "${{ secrets.PYTHON_OPENAI_CLIENT_SECRET }}" - "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID" = "${{ secrets.TENANT_ID }}" - "CONNECTIONSMAP__0__SERVICEURL" = "*" - "CONNECTIONSMAP__0__CONNECTION" = "SERVICE_CONNECTION" - "ENABLE_OBSERVABILITY" = "true" - } - & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` - -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Port ${{ env.AGENT_PORT }} ` - -ConfigMappings $configMappings - - - name: Start Agent - id: start-agent - shell: pwsh - run: | - $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` - -AgentPath "${{ env.SAMPLE_PATH }}" ` - -StartCommand "uv run python start_with_generic_host.py" ` - -Port ${{ env.AGENT_PORT }} ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Environment "Development" ` - -Runtime "python" - echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT - - - name: Verify Agent Running - shell: pwsh - run: | - Write-Host "=== Verifying Agent ===" -ForegroundColor Cyan - - # Verify agent process is still running - $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" - Write-Host "Checking agent process (PID: $agentPid)..." -ForegroundColor Gray - - if ($agentPid) { - try { - $proc = Get-Process -Id $agentPid -ErrorAction Stop - Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green - } catch { - Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red - - # Show logs - $logFile = "${{ env.SAMPLE_PATH }}/agent.log" - if (Test-Path $logFile) { - Write-Host "Agent logs:" -ForegroundColor Yellow - Get-Content $logFile - } - throw "Agent process has stopped" - } - } - - # Quick health check before running tests - $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" - Write-Host "Verifying agent at $agentUrl..." -ForegroundColor Gray - $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue - Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green - - - name: Restore E2E Test Dependencies - shell: pwsh - run: | - dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" - - - name: Run .NET E2E Tests - shell: pwsh - run: | - dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` - --no-restore ` - --filter "Category=HTTP" ` - --logger "trx;LogFileName=test-results.trx" ` - --results-directory "${{ runner.temp }}/TestResults" - env: - AGENT_PORT: ${{ env.AGENT_PORT }} - AGENT_URL: http://localhost:${{ env.AGENT_PORT }} - TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-results-python-openai - path: ${{ runner.temp }}/TestResults/ - retention-days: 30 - - - name: Emit Test Conversations - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` - -TestResultsDir "${{ runner.temp }}/TestConversations" - - - name: Capture Agent Logs - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" - - - name: Cleanup Agent Process - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` - -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` - -Port ${{ env.AGENT_PORT }} - - # =========================================================================== - # Node.js OpenAI Sample - # =========================================================================== - nodejs-openai: - name: Node.js OpenAI Agent - runs-on: windows-latest - if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'nodejs-openai' }} - - env: - SAMPLE_PATH: nodejs/openai/sample-agent - AGENT_PORT: 3979 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup .NET SDK - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '8.0.x' - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - - - name: Install dependencies - working-directory: ${{ env.SAMPLE_PATH }} - run: | - if (Test-Path "package-lock.json") { - npm ci - } else { - npm install - } - if (Test-Path "tsconfig.json") { - npm run build - } - shell: pwsh - - - name: Log SDK Versions - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` - -Runtime "nodejs" ` - -WorkingDirectory "${{ env.SAMPLE_PATH }}" - - - name: Acquire Bearer Token (ROPC) - id: token - shell: pwsh - run: | - $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` - -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` - -TenantId "${{ secrets.MCP_TENANT_ID }}" ` - -Username "${{ secrets.MCP_TEST_USERNAME }}" ` - -Password "${{ secrets.MCP_TEST_PASSWORD }}" - echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT - echo "::add-mask::$token" - - - name: Copy ToolingManifest.json - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" - - - name: Generate .env configuration - shell: pwsh - run: | - $configMappings = @{ - "NODE_ENV" = "development" - "AZURE_OPENAI_API_KEY" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_API_KEY }}" - "AZURE_OPENAI_ENDPOINT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_ENDPOINT }}" - "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_DEPLOYMENT }}" - "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" - "connections__service_connection__settings__authType" = "ClientSecret" - "connections__service_connection__settings__clientId" = "${{ secrets.NODEJS_OPENAI_AGENT_ID }}" - "connections__service_connection__settings__clientSecret" = "${{ secrets.NODEJS_OPENAI_CLIENT_SECRET }}" - "connections__service_connection__settings__tenantId" = "${{ secrets.TENANT_ID }}" - "connectionsMap__0__serviceUrl" = "*" - "connectionsMap__0__connection" = "service_connection" - "agentic_scopes" = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1/.default" - } - & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` - -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Port ${{ env.AGENT_PORT }} ` - -ConfigMappings $configMappings - - - name: Start Agent - id: start-agent - shell: pwsh - run: | - $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` - -AgentPath "${{ env.SAMPLE_PATH }}" ` - -StartCommand "node dist/index.js" ` - -Port ${{ env.AGENT_PORT }} ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Environment "Development" ` - -Runtime "nodejs" - echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT - - - name: Verify Agent Running - shell: pwsh - run: | - $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" - if ($agentPid) { - try { - $proc = Get-Process -Id $agentPid -ErrorAction Stop - Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green - } catch { - Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red - throw "Agent process has stopped" - } - } - $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" - $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue - Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green - - - name: Restore E2E Test Dependencies - run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" - - - name: Run .NET E2E Tests - shell: pwsh - run: | - dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` - --no-restore ` - --filter "Category=HTTP" ` - --logger "trx;LogFileName=test-results.trx" ` - --results-directory "${{ runner.temp }}/TestResults" - env: - AGENT_PORT: ${{ env.AGENT_PORT }} - AGENT_URL: http://localhost:${{ env.AGENT_PORT }} - TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-results-nodejs-openai - path: ${{ runner.temp }}/TestResults/ - retention-days: 30 - - - name: Emit Test Conversations - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` - -TestResultsDir "${{ runner.temp }}/TestConversations" - - - name: Capture Agent Logs - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" - - - name: Cleanup - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` - -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` - -Port ${{ env.AGENT_PORT }} - - # =========================================================================== - # .NET Semantic Kernel Sample - # =========================================================================== - dotnet-semantic-kernel: - name: .NET Semantic Kernel Agent - runs-on: windows-latest - if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'dotnet-semantic-kernel' }} - - env: - SAMPLE_PATH: dotnet/semantic-kernel/sample-agent - AGENT_PORT: 3978 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup .NET - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '8.0.x' - - - name: Restore dependencies - working-directory: ${{ env.SAMPLE_PATH }} - run: dotnet restore - - - name: Build - working-directory: ${{ env.SAMPLE_PATH }} - run: dotnet build --no-restore - - - name: Log SDK Versions - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` - -Runtime "dotnet" ` - -WorkingDirectory "${{ env.SAMPLE_PATH }}" - - - name: Acquire Bearer Token (ROPC) - id: token - shell: pwsh - run: | - $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` - -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` - -TenantId "${{ secrets.MCP_TENANT_ID }}" ` - -Username "${{ secrets.MCP_TEST_USERNAME }}" ` - -Password "${{ secrets.MCP_TEST_PASSWORD }}" - echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT - echo "::add-mask::$token" - - - name: Copy ToolingManifest.json - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" - - - name: Generate appsettings.json - shell: pwsh - run: | - $configMappings = @{ - "AIServices:UseAzureOpenAI" = "true" - "AIServices:AzureOpenAI:Endpoint" = "${{ secrets.DOTNET_SK_AZURE_OPENAI_ENDPOINT }}" - "AIServices:AzureOpenAI:ApiKey" = "${{ secrets.DOTNET_SK_AZURE_OPENAI_API_KEY }}" - "AIServices:AzureOpenAI:DeploymentName" = "${{ secrets.DOTNET_SK_AZURE_OPENAI_DEPLOYMENT }}" - "TokenValidation:Enabled" = "false" - "TokenValidation:TenantId" = "${{ secrets.TENANT_ID }}" - "Connections:ServiceConnection:Settings:AuthType" = "ClientSecret" - "Connections:ServiceConnection:Settings:ClientId" = "${{ secrets.DOTNET_SK_CLIENT_ID }}" - "Connections:ServiceConnection:Settings:ClientSecret" = "${{ secrets.DOTNET_SK_CLIENT_SECRET }}" - "Connections:ServiceConnection:Settings:TenantId" = "${{ secrets.TENANT_ID }}" - } - & "${{ env.SCRIPTS_PATH }}/Generate-AppSettings.ps1" ` - -OutputPath "${{ env.SAMPLE_PATH }}/appsettings.json" ` - -ConfigMappings $configMappings - - - name: Start Agent - id: start-agent - shell: pwsh - run: | - $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` - -AgentPath "${{ env.SAMPLE_PATH }}" ` - -StartCommand "dotnet run --no-build" ` - -Port ${{ env.AGENT_PORT }} ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Environment "Development" ` - -Runtime "dotnet" - echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT - - - name: Verify Agent Running - shell: pwsh - run: | - $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" - if ($agentPid) { - try { - $proc = Get-Process -Id $agentPid -ErrorAction Stop - Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green - } catch { - Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red - throw "Agent process has stopped" - } - } - $response = Invoke-WebRequest -Uri "http://localhost:${{ env.AGENT_PORT }}/api/health" -UseBasicParsing -ErrorAction SilentlyContinue - Write-Host "Health: $($response.StatusCode)" -ForegroundColor Green - - - name: Restore E2E Test Dependencies - run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" - - - name: Run .NET E2E Tests - shell: pwsh - run: | - dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` - --no-restore ` - --filter "Category=HTTP" ` - --logger "trx;LogFileName=test-results.trx" ` - --results-directory "${{ runner.temp }}/TestResults" - env: - AGENT_PORT: ${{ env.AGENT_PORT }} - AGENT_URL: http://localhost:${{ env.AGENT_PORT }} - TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-results-dotnet-sk - path: ${{ runner.temp }}/TestResults/ - retention-days: 30 - - - name: Emit Test Conversations - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` - -TestResultsDir "${{ runner.temp }}/TestConversations" - - - name: Capture Logs - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" - - - name: Cleanup - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` - -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` - -Port ${{ env.AGENT_PORT }} - - # =========================================================================== - # .NET Agent Framework Sample - # =========================================================================== - dotnet-agent-framework: - name: .NET Agent Framework Agent - runs-on: windows-latest - if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'dotnet-agent-framework' }} - - env: - SAMPLE_PATH: dotnet/agent-framework/sample-agent - AGENT_PORT: 3978 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup .NET - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '8.0.x' - - - name: Restore dependencies - working-directory: ${{ env.SAMPLE_PATH }} - run: dotnet restore - - - name: Build - working-directory: ${{ env.SAMPLE_PATH }} - run: dotnet build --no-restore - - - name: Log SDK Versions - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` - -Runtime "dotnet" ` - -WorkingDirectory "${{ env.SAMPLE_PATH }}" - - - name: Acquire Bearer Token (ROPC) - id: token - shell: pwsh - run: | - $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` - -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` - -TenantId "${{ secrets.MCP_TENANT_ID }}" ` - -Username "${{ secrets.MCP_TEST_USERNAME }}" ` - -Password "${{ secrets.MCP_TEST_PASSWORD }}" - echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT - echo "::add-mask::$token" - - - name: Copy ToolingManifest.json - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" - - - name: Generate appsettings.json - shell: pwsh - run: | - $configMappings = @{ - "AIServices:AzureOpenAI:Endpoint" = "${{ secrets.DOTNET_AF_AZURE_OPENAI_ENDPOINT }}" - "AIServices:AzureOpenAI:ApiKey" = "${{ secrets.DOTNET_AF_AZURE_OPENAI_API_KEY }}" - "AIServices:AzureOpenAI:DeploymentName" = "${{ secrets.DOTNET_AF_AZURE_OPENAI_DEPLOYMENT }}" - "TokenValidation:Enabled" = "false" - "Connections:ServiceConnection:Settings:AuthType" = "ClientSecret" - "Connections:ServiceConnection:Settings:ClientId" = "${{ secrets.DOTNET_AF_CLIENT_ID }}" - "Connections:ServiceConnection:Settings:ClientSecret" = "${{ secrets.DOTNET_AF_CLIENT_SECRET }}" - "Connections:ServiceConnection:Settings:TenantId" = "${{ secrets.TENANT_ID }}" - "Connections:ServiceConnection:Settings:AuthorityEndpoint" = "https://login.microsoftonline.com/${{ secrets.TENANT_ID }}" - "Connections:ServiceConnection:Settings:Scopes:0" = "5a807f24-c9de-44ee-a3a7-329e88a00ffc/.default" - } - & "${{ env.SCRIPTS_PATH }}/Generate-AppSettings.ps1" ` - -OutputPath "${{ env.SAMPLE_PATH }}/appsettings.json" ` - -ConfigMappings $configMappings - - - name: Start Agent - id: start-agent - shell: pwsh - run: | - $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` - -AgentPath "${{ env.SAMPLE_PATH }}" ` - -StartCommand "dotnet run --no-build" ` - -Port ${{ env.AGENT_PORT }} ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Environment "Development" ` - -Runtime "dotnet" - echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT - - - name: Verify Agent Running - shell: pwsh - run: | - $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" - if ($agentPid) { - try { - $proc = Get-Process -Id $agentPid -ErrorAction Stop - Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green - } catch { - Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red - throw "Agent process has stopped" - } - } - $response = Invoke-WebRequest -Uri "http://localhost:${{ env.AGENT_PORT }}/api/health" -UseBasicParsing -ErrorAction SilentlyContinue - Write-Host "Health: $($response.StatusCode)" -ForegroundColor Green - - - name: Restore E2E Test Dependencies - run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" - - - name: Run .NET E2E Tests - shell: pwsh - run: | - dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` - --no-restore ` - --filter "Category=HTTP" ` - --logger "trx;LogFileName=test-results.trx" ` - --results-directory "${{ runner.temp }}/TestResults" - env: - AGENT_PORT: ${{ env.AGENT_PORT }} - AGENT_URL: http://localhost:${{ env.AGENT_PORT }} - TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-results-dotnet-af - path: ${{ runner.temp }}/TestResults/ - retention-days: 30 - - - name: Emit Test Conversations - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` - -TestResultsDir "${{ runner.temp }}/TestConversations" - - - name: Capture Logs - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" - - - name: Cleanup - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` - -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` - -Port ${{ env.AGENT_PORT }} - - # =========================================================================== - # Node.js LangChain Sample - # =========================================================================== - nodejs-langchain: - name: Node.js LangChain Agent - runs-on: windows-latest - if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'nodejs-langchain' }} - - env: - SAMPLE_PATH: nodejs/langchain/sample-agent - AGENT_PORT: 3979 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup .NET SDK - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '8.0.x' - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - - - name: Install dependencies - working-directory: ${{ env.SAMPLE_PATH }} - run: | - if (Test-Path "package-lock.json") { - npm ci - } else { - npm install - } - if (Test-Path "tsconfig.json") { - npm run build - } - shell: pwsh - - - name: Log SDK Versions - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` - -Runtime "nodejs" ` - -WorkingDirectory "${{ env.SAMPLE_PATH }}" - - - name: Acquire Bearer Token (ROPC) - id: token - shell: pwsh - run: | - $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` - -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` - -TenantId "${{ secrets.MCP_TENANT_ID }}" ` - -Username "${{ secrets.MCP_TEST_USERNAME }}" ` - -Password "${{ secrets.MCP_TEST_PASSWORD }}" - echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT - echo "::add-mask::$token" - - - name: Copy ToolingManifest.json - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" - - - name: Generate .env configuration - shell: pwsh - run: | - $configMappings = @{ - "NODE_ENV" = "development" - "AZURE_OPENAI_API_KEY" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_API_KEY }}" - "AZURE_OPENAI_ENDPOINT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_ENDPOINT }}" - "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_DEPLOYMENT }}" - "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" - "agentic_scopes" = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1/.default" - "connections__service_connection__settings__authType" = "ClientSecret" - "connections__service_connection__settings__clientId" = "${{ secrets.NODEJS_OPENAI_AGENT_ID }}" - "connections__service_connection__settings__clientSecret" = "${{ secrets.NODEJS_OPENAI_CLIENT_SECRET }}" - "connections__service_connection__settings__tenantId" = "${{ secrets.TENANT_ID }}" - "connectionsMap__0__serviceUrl" = "*" - "connectionsMap__0__connection" = "service_connection" - } - & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` - -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Port ${{ env.AGENT_PORT }} ` - -ConfigMappings $configMappings - - - name: Start Agent - id: start-agent - shell: pwsh - run: | - $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` - -AgentPath "${{ env.SAMPLE_PATH }}" ` - -StartCommand "node dist/index.js" ` - -Port ${{ env.AGENT_PORT }} ` - -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` - -Environment "Development" ` - -Runtime "nodejs" - echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT - - - name: Verify Agent Running - shell: pwsh - run: | - $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" - if ($agentPid) { - try { - $proc = Get-Process -Id $agentPid -ErrorAction Stop - Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green - } catch { - Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red - throw "Agent process has stopped" - } - } - $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" - $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue - Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green - - - name: Restore E2E Test Dependencies - run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" - - - name: Run .NET E2E Tests - shell: pwsh - run: | - dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` - --no-restore ` - --filter "Category=HTTP" ` - --logger "trx;LogFileName=test-results.trx" ` - --results-directory "${{ runner.temp }}/TestResults" - env: - AGENT_PORT: ${{ env.AGENT_PORT }} - AGENT_URL: http://localhost:${{ env.AGENT_PORT }} - TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-results-nodejs-langchain - path: ${{ runner.temp }}/TestResults/ - retention-days: 30 - - - name: Emit Test Conversations - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` - -TestResultsDir "${{ runner.temp }}/TestConversations" - - - name: Capture Agent Logs - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" - - - name: Cleanup - if: always() - shell: pwsh - run: | - & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` - -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` - -Port ${{ env.AGENT_PORT }} - - # =========================================================================== - # Summary - # =========================================================================== - summary: - name: Test Summary - runs-on: ubuntu-latest - needs: [python-openai, nodejs-openai, dotnet-semantic-kernel, dotnet-agent-framework, nodejs-langchain] - if: always() - - steps: - - name: Generate Summary - id: summary - run: | - # Determine overall status - OVERALL_STATUS="✅ All Tests Passed" - if [[ "${{ needs.python-openai.result }}" != "success" ]] || \ - [[ "${{ needs.nodejs-openai.result }}" != "success" ]] || \ - [[ "${{ needs.dotnet-semantic-kernel.result }}" != "success" ]] || \ - [[ "${{ needs.dotnet-agent-framework.result }}" != "success" ]] || \ - [[ "${{ needs.nodejs-langchain.result }}" != "success" ]]; then - OVERALL_STATUS="⚠️ Some Tests Failed" - fi - - # Generate status icons using if-else for reliability - if [[ "${{ needs.python-openai.result }}" == "success" ]]; then PYTHON_OPENAI_ICON="✅"; else PYTHON_OPENAI_ICON="❌"; fi - if [[ "${{ needs.nodejs-openai.result }}" == "success" ]]; then NODEJS_OPENAI_ICON="✅"; else NODEJS_OPENAI_ICON="❌"; fi - if [[ "${{ needs.dotnet-semantic-kernel.result }}" == "success" ]]; then DOTNET_SK_ICON="✅"; else DOTNET_SK_ICON="❌"; fi - if [[ "${{ needs.dotnet-agent-framework.result }}" == "success" ]]; then DOTNET_AF_ICON="✅"; else DOTNET_AF_ICON="❌"; fi - if [[ "${{ needs.nodejs-langchain.result }}" == "success" ]]; then NODEJS_LANGCHAIN_ICON="✅"; else NODEJS_LANGCHAIN_ICON="❌"; fi - - echo "## E2E Test Results" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Status:** $OVERALL_STATUS" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "| Sample | Status | Result |" >> $GITHUB_STEP_SUMMARY - echo "|--------|--------|--------|" >> $GITHUB_STEP_SUMMARY - echo "| Python OpenAI | $PYTHON_OPENAI_ICON | ${{ needs.python-openai.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| Node.js OpenAI | $NODEJS_OPENAI_ICON | ${{ needs.nodejs-openai.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| Node.js LangChain | $NODEJS_LANGCHAIN_ICON | ${{ needs.nodejs-langchain.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| .NET Semantic Kernel | $DOTNET_SK_ICON | ${{ needs.dotnet-semantic-kernel.result }} |" >> $GITHUB_STEP_SUMMARY - echo "| .NET Agent Framework | $DOTNET_AF_ICON | ${{ needs.dotnet-agent-framework.result }} |" >> $GITHUB_STEP_SUMMARY - - # Save summary for PR comment - echo "OVERALL_STATUS=$OVERALL_STATUS" >> $GITHUB_OUTPUT - echo "PYTHON_OPENAI_RESULT=${{ needs.python-openai.result }}" >> $GITHUB_OUTPUT - echo "NODEJS_OPENAI_RESULT=${{ needs.nodejs-openai.result }}" >> $GITHUB_OUTPUT - echo "NODEJS_LANGCHAIN_RESULT=${{ needs.nodejs-langchain.result }}" >> $GITHUB_OUTPUT - echo "DOTNET_SK_RESULT=${{ needs.dotnet-semantic-kernel.result }}" >> $GITHUB_OUTPUT - echo "DOTNET_AF_RESULT=${{ needs.dotnet-agent-framework.result }}" >> $GITHUB_OUTPUT - - - name: Post PR Comment - if: github.event_name == 'pull_request' - uses: actions/github-script@v7 - with: - script: | - const pythonOpenaiIcon = '${{ needs.python-openai.result }}' === 'success' ? '✅' : '❌'; - const nodejsOpenaiIcon = '${{ needs.nodejs-openai.result }}' === 'success' ? '✅' : '❌'; - const nodejsLangchainIcon = '${{ needs.nodejs-langchain.result }}' === 'success' ? '✅' : '❌'; - const dotnetSkIcon = '${{ needs.dotnet-semantic-kernel.result }}' === 'success' ? '✅' : '❌'; - const dotnetAfIcon = '${{ needs.dotnet-agent-framework.result }}' === 'success' ? '✅' : '❌'; - - const allPassed = '${{ needs.python-openai.result }}' === 'success' && - '${{ needs.nodejs-openai.result }}' === 'success' && - '${{ needs.nodejs-langchain.result }}' === 'success' && - '${{ needs.dotnet-semantic-kernel.result }}' === 'success' && - '${{ needs.dotnet-agent-framework.result }}' === 'success'; - - const overallStatus = allPassed ? '✅ All E2E Tests Passed' : '⚠️ Some E2E Tests Failed'; - - const body = `## ${overallStatus} - - | Sample | Status | Result | - |--------|--------|--------| - | Python OpenAI | ${pythonOpenaiIcon} | ${{ needs.python-openai.result }} | - | Node.js OpenAI | ${nodejsOpenaiIcon} | ${{ needs.nodejs-openai.result }} | - | Node.js LangChain | ${nodejsLangchainIcon} | ${{ needs.nodejs-langchain.result }} | - | .NET Semantic Kernel | ${dotnetSkIcon} | ${{ needs.dotnet-semantic-kernel.result }} | - | .NET Agent Framework | ${dotnetAfIcon} | ${{ needs.dotnet-agent-framework.result }} | - - [View full test details](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}) - `; - - // Find existing comment - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - - const botComment = comments.find(comment => - comment.user.type === 'Bot' && - comment.body.includes('E2E Tests') - ); - - if (botComment) { - // Update existing comment - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: botComment.id, - body: body - }); - } else { - // Create new comment - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: body - }); - } diff --git a/.github/workflows/e2e-dotnet-agent-framework.yml b/.github/workflows/e2e-dotnet-agent-framework.yml new file mode 100644 index 00000000..8e5b9379 --- /dev/null +++ b/.github/workflows/e2e-dotnet-agent-framework.yml @@ -0,0 +1,153 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: E2E - .NET Agent Framework + +on: + workflow_call: + workflow_dispatch: + push: + branches: [main] + paths: + - 'dotnet/agent-framework/**' + - 'scripts/e2e/**' + - '.github/workflows/e2e-dotnet-agent-framework.yml' + pull_request: + branches: [main] + paths: + - 'dotnet/agent-framework/**' + - 'scripts/e2e/**' + +permissions: + contents: read + +env: + SAMPLE_PATH: dotnet/agent-framework/sample-agent + AGENT_PORT: 3978 + SCRIPTS_PATH: scripts/e2e + E2E_TESTS_PATH: tests/e2e + +jobs: + dotnet-agent-framework: + name: .NET Agent Framework Agent + runs-on: windows-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Restore dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: dotnet restore + + - name: Build + working-directory: ${{ env.SAMPLE_PATH }} + run: dotnet build --no-restore + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate appsettings.json + shell: pwsh + run: | + $configMappings = @{ + "AIServices:AzureOpenAI:Endpoint" = "${{ secrets.DOTNET_AF_AZURE_OPENAI_ENDPOINT }}" + "AIServices:AzureOpenAI:ApiKey" = "${{ secrets.DOTNET_AF_AZURE_OPENAI_API_KEY }}" + "AIServices:AzureOpenAI:DeploymentName" = "${{ secrets.DOTNET_AF_AZURE_OPENAI_DEPLOYMENT }}" + "TokenValidation:Enabled" = "false" + "Connections:ServiceConnection:Settings:AuthType" = "ClientSecret" + "Connections:ServiceConnection:Settings:ClientId" = "${{ secrets.DOTNET_AF_CLIENT_ID }}" + "Connections:ServiceConnection:Settings:ClientSecret" = "${{ secrets.DOTNET_AF_CLIENT_SECRET }}" + "Connections:ServiceConnection:Settings:TenantId" = "${{ secrets.TENANT_ID }}" + "Connections:ServiceConnection:Settings:AuthorityEndpoint" = "https://login.microsoftonline.com/${{ secrets.TENANT_ID }}" + "Connections:ServiceConnection:Settings:Scopes:0" = "5a807f24-c9de-44ee-a3a7-329e88a00ffc/.default" + } + & "${{ env.SCRIPTS_PATH }}/Generate-AppSettings.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/appsettings.json" ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "dotnet run --no-build" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "dotnet" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + throw "Agent process has stopped" + } + } + $response = Invoke-WebRequest -Uri "http://localhost:${{ env.AGENT_PORT }}/api/health" -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health: $($response.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --verbosity normal ` + --logger "console;verbosity=detailed" ` + --logger "trx;LogFileName=test-results-dotnet-af.trx" + + - name: Capture Agent Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Stop Agent Process + if: always() + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" -AgentPID $agentPid + } + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-dotnet-af + path: | + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*.trx + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*-logs.txt + retention-days: 7 diff --git a/.github/workflows/e2e-dotnet-semantic-kernel.yml b/.github/workflows/e2e-dotnet-semantic-kernel.yml new file mode 100644 index 00000000..583ed034 --- /dev/null +++ b/.github/workflows/e2e-dotnet-semantic-kernel.yml @@ -0,0 +1,153 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: E2E - .NET Semantic Kernel + +on: + workflow_call: + workflow_dispatch: + push: + branches: [main] + paths: + - 'dotnet/semantic-kernel/**' + - 'scripts/e2e/**' + - '.github/workflows/e2e-dotnet-semantic-kernel.yml' + pull_request: + branches: [main] + paths: + - 'dotnet/semantic-kernel/**' + - 'scripts/e2e/**' + +permissions: + contents: read + +env: + SAMPLE_PATH: dotnet/semantic-kernel/sample-agent + AGENT_PORT: 3978 + SCRIPTS_PATH: scripts/e2e + E2E_TESTS_PATH: tests/e2e + +jobs: + dotnet-semantic-kernel: + name: .NET Semantic Kernel Agent + runs-on: windows-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Restore dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: dotnet restore + + - name: Build + working-directory: ${{ env.SAMPLE_PATH }} + run: dotnet build --no-restore + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate appsettings.json + shell: pwsh + run: | + $configMappings = @{ + "AIServices:UseAzureOpenAI" = "true" + "AIServices:AzureOpenAI:Endpoint" = "${{ secrets.DOTNET_SK_AZURE_OPENAI_ENDPOINT }}" + "AIServices:AzureOpenAI:ApiKey" = "${{ secrets.DOTNET_SK_AZURE_OPENAI_API_KEY }}" + "AIServices:AzureOpenAI:DeploymentName" = "${{ secrets.DOTNET_SK_AZURE_OPENAI_DEPLOYMENT }}" + "TokenValidation:Enabled" = "false" + "TokenValidation:TenantId" = "${{ secrets.TENANT_ID }}" + "Connections:ServiceConnection:Settings:AuthType" = "ClientSecret" + "Connections:ServiceConnection:Settings:ClientId" = "${{ secrets.DOTNET_SK_CLIENT_ID }}" + "Connections:ServiceConnection:Settings:ClientSecret" = "${{ secrets.DOTNET_SK_CLIENT_SECRET }}" + "Connections:ServiceConnection:Settings:TenantId" = "${{ secrets.TENANT_ID }}" + } + & "${{ env.SCRIPTS_PATH }}/Generate-AppSettings.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/appsettings.json" ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "dotnet run --no-build" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "dotnet" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + throw "Agent process has stopped" + } + } + $response = Invoke-WebRequest -Uri "http://localhost:${{ env.AGENT_PORT }}/api/health" -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health: $($response.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --verbosity normal ` + --logger "console;verbosity=detailed" ` + --logger "trx;LogFileName=test-results-dotnet-sk.trx" + + - name: Capture Agent Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Stop Agent Process + if: always() + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" -AgentPID $agentPid + } + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-dotnet-sk + path: | + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*.trx + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*-logs.txt + retention-days: 7 diff --git a/.github/workflows/e2e-nodejs-openai.yml b/.github/workflows/e2e-nodejs-openai.yml new file mode 100644 index 00000000..8c207dd4 --- /dev/null +++ b/.github/workflows/e2e-nodejs-openai.yml @@ -0,0 +1,169 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: E2E - Node.js OpenAI + +on: + workflow_call: # Allow orchestrator to call this workflow + workflow_dispatch: # Allow manual triggering + push: + branches: [main] + paths: + - 'nodejs/openai/**' + - 'scripts/e2e/**' + - '.github/workflows/e2e-nodejs-openai.yml' + pull_request: + branches: [main] + paths: + - 'nodejs/openai/**' + - 'scripts/e2e/**' + +permissions: + contents: read + +env: + SAMPLE_PATH: nodejs/openai/sample-agent + AGENT_PORT: 3979 + SCRIPTS_PATH: scripts/e2e + E2E_TESTS_PATH: tests/e2e + +jobs: + nodejs-openai: + name: Node.js OpenAI Agent + runs-on: windows-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: | + if (Test-Path "package-lock.json") { + npm ci + } else { + npm install + } + if (Test-Path "tsconfig.json") { + npm run build + } + shell: pwsh + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate .env configuration + shell: pwsh + run: | + $configMappings = @{ + "NODE_ENV" = "development" + "AZURE_OPENAI_API_KEY" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_API_KEY }}" + "AZURE_OPENAI_ENDPOINT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_ENDPOINT }}" + "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_DEPLOYMENT }}" + "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" + "connections__service_connection__settings__authType" = "ClientSecret" + "connections__service_connection__settings__clientId" = "${{ secrets.NODEJS_OPENAI_AGENT_ID }}" + "connections__service_connection__settings__clientSecret" = "${{ secrets.NODEJS_OPENAI_CLIENT_SECRET }}" + "connections__service_connection__settings__tenantId" = "${{ secrets.TENANT_ID }}" + "connectionsMap__0__serviceUrl" = "*" + "connectionsMap__0__connection" = "service_connection" + "agentic_scopes" = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1/.default" + } + & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Port ${{ env.AGENT_PORT }} ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "node dist/index.js" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "nodejs" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + throw "Agent process has stopped" + } + } + $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" + $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --verbosity normal ` + --logger "console;verbosity=detailed" ` + --logger "trx;LogFileName=test-results-nodejs-openai.trx" ` + --filter "FullyQualifiedName~BasicConversation|FullyQualifiedName~Notification" + + - name: Capture Agent Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Stop Agent Process + if: always() + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" -AgentPID $agentPid + } + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-nodejs-openai + path: | + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*.trx + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*-logs.txt + retention-days: 7 diff --git a/.github/workflows/e2e-orchestrator.yml b/.github/workflows/e2e-orchestrator.yml new file mode 100644 index 00000000..1cee986c --- /dev/null +++ b/.github/workflows/e2e-orchestrator.yml @@ -0,0 +1,67 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: E2E Test Orchestrator + +on: + schedule: + # Run every 6 hours + - cron: '0 */6 * * *' + push: + branches: [main] + paths: + - 'python/openai/**' + - 'nodejs/openai/**' + - 'dotnet/semantic-kernel/**' + - 'dotnet/agent-framework/**' + - '.github/workflows/e2e-*.yml' + - 'scripts/e2e/**' + pull_request: + branches: [main] + paths: + - 'python/openai/**' + - 'nodejs/openai/**' + - 'dotnet/semantic-kernel/**' + - 'dotnet/agent-framework/**' + - '.github/workflows/e2e-*.yml' + - 'scripts/e2e/**' + workflow_dispatch: + inputs: + sample: + description: 'Sample to test (leave empty for all)' + required: false + type: choice + options: + - '' + - 'python-openai' + - 'nodejs-openai' + - 'dotnet-sk' + - 'dotnet-af' + +permissions: + contents: read + +jobs: + python-openai: + name: Python OpenAI E2E + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'python-openai' }} + uses: ./.github/workflows/e2e-python-openai.yml + secrets: inherit + + nodejs-openai: + name: Node.js OpenAI E2E + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'nodejs-openai' }} + uses: ./.github/workflows/e2e-nodejs-openai.yml + secrets: inherit + + dotnet-sk: + name: .NET Semantic Kernel E2E + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'dotnet-sk' }} + uses: ./.github/workflows/e2e-dotnet-semantic-kernel.yml + secrets: inherit + + dotnet-af: + name: .NET Agent Framework E2E + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'dotnet-af' }} + uses: ./.github/workflows/e2e-dotnet-agent-framework.yml + secrets: inherit diff --git a/.github/workflows/e2e-python-openai.yml b/.github/workflows/e2e-python-openai.yml new file mode 100644 index 00000000..668f4e8d --- /dev/null +++ b/.github/workflows/e2e-python-openai.yml @@ -0,0 +1,187 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: E2E - Python OpenAI + +on: + workflow_call: # Allow orchestrator to call this workflow + workflow_dispatch: # Allow manual triggering + push: + branches: [main] + paths: + - 'python/openai/**' + - 'scripts/e2e/**' + - '.github/workflows/e2e-python-openai.yml' + pull_request: + branches: [main] + paths: + - 'python/openai/**' + - 'scripts/e2e/**' + +permissions: + contents: read + +env: + SAMPLE_PATH: python/openai/sample-agent + AGENT_PORT: 3979 + SCRIPTS_PATH: scripts/e2e + E2E_TESTS_PATH: tests/e2e + +jobs: + python-openai: + name: Python OpenAI Agent + runs-on: windows-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install uv package manager + run: pip install uv + + - name: Install dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: uv sync + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate .env configuration + shell: pwsh + run: | + $configMappings = @{ + "AZURE_OPENAI_API_KEY" = "${{ secrets.PYTHON_OPENAI_AZURE_OPENAI_API_KEY }}" + "AZURE_OPENAI_ENDPOINT" = "${{ secrets.PYTHON_OPENAI_AZURE_OPENAI_ENDPOINT }}" + "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.PYTHON_OPENAI_AZURE_OPENAI_DEPLOYMENT }}" + "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" + "USE_AGENTIC_AUTH" = "false" + "MCP_PLATFORM_ENDPOINT" = "https://agent365.svc.cloud.microsoft" + "AGENT_ID" = "${{ secrets.PYTHON_OPENAI_AGENT_ID }}" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__AUTHTYPE" = "ClientSecret" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID" = "${{ secrets.PYTHON_OPENAI_AGENT_ID }}" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET" = "${{ secrets.PYTHON_OPENAI_CLIENT_SECRET }}" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID" = "${{ secrets.TENANT_ID }}" + "CONNECTIONSMAP__0__SERVICEURL" = "*" + "CONNECTIONSMAP__0__CONNECTION" = "SERVICE_CONNECTION" + "ENABLE_OBSERVABILITY" = "true" + } + & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Port ${{ env.AGENT_PORT }} ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "uv run python start_with_generic_host.py" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "python" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + Write-Host "=== Verifying Agent ===" -ForegroundColor Cyan + + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + Write-Host "Checking agent process (PID: $agentPid)..." -ForegroundColor Gray + + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + + $logFile = "${{ env.SAMPLE_PATH }}/agent.log" + if (Test-Path $logFile) { + Write-Host "Agent logs:" -ForegroundColor Yellow + Get-Content $logFile + } + throw "Agent process has stopped" + } + } + + $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" + Write-Host "Verifying agent at $agentUrl..." -ForegroundColor Gray + $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + shell: pwsh + run: | + dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --filter "Category=HTTP" ` + --logger "trx;LogFileName=test-results.trx" ` + --results-directory "${{ runner.temp }}/TestResults" + env: + AGENT_PORT: ${{ env.AGENT_PORT }} + AGENT_URL: http://localhost:${{ env.AGENT_PORT }} + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-python-openai + path: ${{ runner.temp }}/TestResults/ + retention-days: 30 + + - name: Upload Test Conversations + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-conversations-python-openai + path: ${{ runner.temp }}/TestConversations/ + retention-days: 30 + continue-on-error: true + + - name: Capture Agent Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Cleanup Agent Process + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` + -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` + -Port ${{ env.AGENT_PORT }} diff --git a/.github/workflows/update-e2e-status.yml b/.github/workflows/update-e2e-status.yml deleted file mode 100644 index d20dac05..00000000 --- a/.github/workflows/update-e2e-status.yml +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -name: Update E2E Status - -on: - workflow_run: - workflows: ["E2E Agent Samples"] - types: - - completed - workflow_dispatch: - -permissions: - contents: write - -jobs: - update-status: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: main - - - name: Get job statuses and update README - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - - // Get the triggering workflow run or latest run - let runId; - if (context.payload.workflow_run) { - runId = context.payload.workflow_run.id; - } else { - // Manual trigger - get latest run - const runs = await github.rest.actions.listWorkflowRuns({ - owner: context.repo.owner, - repo: context.repo.repo, - workflow_id: 'e2e-agent-samples.yml', - branch: 'main', - per_page: 1 - }); - if (runs.data.workflow_runs.length === 0) { - console.log('No workflow runs found'); - return; - } - runId = runs.data.workflow_runs[0].id; - } - - console.log(`Processing workflow run: ${runId}`); - - // Get jobs for this run - const jobs = await github.rest.actions.listJobsForWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: runId - }); - - // Map job names to their status badges - const jobStatusMap = { - 'Python OpenAI Agent': { key: 'python-openai', label: 'Python OpenAI' }, - 'Node.js OpenAI Agent': { key: 'nodejs-openai', label: 'Node.js OpenAI' }, - '.NET Semantic Kernel Agent': { key: 'dotnet-sk', label: '.NET Semantic Kernel' }, - '.NET Agent Framework Agent': { key: 'dotnet-af', label: '.NET Agent Framework' } - }; - - const statuses = {}; - for (const job of jobs.data.jobs) { - const mapping = jobStatusMap[job.name]; - if (mapping) { - const conclusion = job.conclusion || job.status; - let badge; - if (conclusion === 'success') { - badge = `![${mapping.label}](https://img.shields.io/badge/${encodeURIComponent(mapping.label)}-passing-brightgreen)`; - } else if (conclusion === 'failure') { - badge = `![${mapping.label}](https://img.shields.io/badge/${encodeURIComponent(mapping.label)}-failing-red)`; - } else if (conclusion === 'in_progress' || job.status === 'in_progress') { - badge = `![${mapping.label}](https://img.shields.io/badge/${encodeURIComponent(mapping.label)}-running-yellow)`; - } else { - badge = `![${mapping.label}](https://img.shields.io/badge/${encodeURIComponent(mapping.label)}-pending-lightgrey)`; - } - statuses[mapping.key] = badge; - console.log(`${job.name}: ${conclusion} -> ${badge}`); - } - } - - // Read current README - let readme = fs.readFileSync('README.md', 'utf8'); - - // Generate new status table - const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`; - const newTable = `## E2E Test Status - -| Sample | Status | -|--------|--------| -| Python OpenAI | ${statuses['python-openai'] || '![Python OpenAI](https://img.shields.io/badge/Python%20OpenAI-unknown-lightgrey)'} | -| Node.js OpenAI | ${statuses['nodejs-openai'] || '![Node.js OpenAI](https://img.shields.io/badge/Node.js%20OpenAI-unknown-lightgrey)'} | -| .NET Semantic Kernel | ${statuses['dotnet-sk'] || '![.NET SK](https://img.shields.io/badge/.NET%20SK-unknown-lightgrey)'} | -| .NET Agent Framework | ${statuses['dotnet-af'] || '![.NET AF](https://img.shields.io/badge/.NET%20AF-unknown-lightgrey)'} | - -*Last updated: ${new Date().toISOString().split('T')[0]} | [View Run](${runUrl})*`; - - // Replace the status section - const statusRegex = /## E2E Test Status[\s\S]*?\n(?=\n>|$)/; - if (statusRegex.test(readme)) { - readme = readme.replace(statusRegex, newTable + '\n'); - } - - fs.writeFileSync('README.md', readme); - console.log('README updated successfully'); - - - name: Commit and push changes - run: | - git config --local user.email "github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - git add README.md - git diff --staged --quiet || (git commit -m "chore: Update E2E test status [skip ci]" && git push) diff --git a/README.md b/README.md index b6281d03..72eaca34 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Microsoft Agent 365 SDK Samples and Prompts -[![E2E Tests](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-agent-samples.yml?branch=main&label=E2E%20Agent%20Samples)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml) +[![E2E Tests](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-orchestrator.yml?branch=main&label=E2E%20All%20Samples)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-orchestrator.yml) This repository contains sample agents and prompts for building with the Microsoft Agent 365 SDK. The Microsoft Agent 365 SDK extends the Microsoft 365 Agents SDK with enterprise-grade capabilities for building sophisticated agents. It provides comprehensive tooling for observability, notifications, runtime utilities, and development tools that help developers create production-ready agents for platforms including M365, Teams, Copilot Studio, and Webchat. @@ -11,10 +11,10 @@ This repository contains sample agents and prompts for building with the Microso | Sample | Status | |--------|--------| -| Python OpenAI | [![Python OpenAI](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-agent-samples.yml?branch=main&label=E2E&job=python-openai)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml) | -| Node.js OpenAI | [![Node.js OpenAI](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-agent-samples.yml?branch=main&label=E2E&job=nodejs-openai)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml) | -| .NET Semantic Kernel | [![.NET SK](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-agent-samples.yml?branch=main&label=E2E&job=dotnet-semantic-kernel)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml) | -| .NET Agent Framework | [![.NET AF](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-agent-samples.yml?branch=main&label=E2E&job=dotnet-agent-framework)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml) | +| Python OpenAI | [![Python OpenAI](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-python-openai.yml?branch=main&label=E2E)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-python-openai.yml) | +| Node.js OpenAI | [![Node.js OpenAI](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-nodejs-openai.yml?branch=main&label=E2E)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-nodejs-openai.yml) | +| .NET Semantic Kernel | [![.NET SK](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-dotnet-semantic-kernel.yml?branch=main&label=E2E)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-dotnet-semantic-kernel.yml) | +| .NET Agent Framework | [![.NET AF](https://img.shields.io/github/actions/workflow/status/microsoft/Agent365-Samples/e2e-dotnet-agent-framework.yml?branch=main&label=E2E)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-dotnet-agent-framework.yml) | > #### Note: > Use the information in this README to contribute to this open-source project. To learn about using this SDK in your projects, refer to the [Microsoft Agent 365 Developer documentation](https://learn.microsoft.com/en-us/microsoft-agent-365/developer/). From 5249b20a0dcadc28be08aacfcb7ecdb37035b3a6 Mon Sep 17 00:00:00 2001 From: abdulanu0 Date: Thu, 29 Jan 2026 16:49:30 -0800 Subject: [PATCH 10/15] Add E2E workflow enhancements - Add LangChain E2E workflow (e2e-nodejs-langchain.yml) - Add SDK version logging step to all E2E workflows - Add test conversation emission step to all E2E workflows - Add PR test summary comment to orchestrator (only on PR events) - Update orchestrator to include LangChain and add pull-requests: write permission --- .../workflows/e2e-dotnet-agent-framework.yml | 16 ++ .../workflows/e2e-dotnet-semantic-kernel.yml | 16 ++ .github/workflows/e2e-nodejs-langchain.yml | 185 ++++++++++++++++++ .github/workflows/e2e-nodejs-openai.yml | 16 ++ .github/workflows/e2e-orchestrator.yml | 114 +++++++++++ .github/workflows/e2e-python-openai.yml | 19 +- 6 files changed, 359 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/e2e-nodejs-langchain.yml diff --git a/.github/workflows/e2e-dotnet-agent-framework.yml b/.github/workflows/e2e-dotnet-agent-framework.yml index 8e5b9379..9a7c3349 100644 --- a/.github/workflows/e2e-dotnet-agent-framework.yml +++ b/.github/workflows/e2e-dotnet-agent-framework.yml @@ -49,6 +49,13 @@ jobs: working-directory: ${{ env.SAMPLE_PATH }} run: dotnet build --no-restore + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "dotnet" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + - name: Acquire Bearer Token (ROPC) id: token shell: pwsh @@ -125,6 +132,15 @@ jobs: --verbosity normal ` --logger "console;verbosity=detailed" ` --logger "trx;LogFileName=test-results-dotnet-af.trx" + env: + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Emit Test Conversations + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" - name: Capture Agent Logs if: always() diff --git a/.github/workflows/e2e-dotnet-semantic-kernel.yml b/.github/workflows/e2e-dotnet-semantic-kernel.yml index 583ed034..1736e848 100644 --- a/.github/workflows/e2e-dotnet-semantic-kernel.yml +++ b/.github/workflows/e2e-dotnet-semantic-kernel.yml @@ -49,6 +49,13 @@ jobs: working-directory: ${{ env.SAMPLE_PATH }} run: dotnet build --no-restore + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "dotnet" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + - name: Acquire Bearer Token (ROPC) id: token shell: pwsh @@ -125,6 +132,15 @@ jobs: --verbosity normal ` --logger "console;verbosity=detailed" ` --logger "trx;LogFileName=test-results-dotnet-sk.trx" + env: + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Emit Test Conversations + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" - name: Capture Agent Logs if: always() diff --git a/.github/workflows/e2e-nodejs-langchain.yml b/.github/workflows/e2e-nodejs-langchain.yml new file mode 100644 index 00000000..14a904d7 --- /dev/null +++ b/.github/workflows/e2e-nodejs-langchain.yml @@ -0,0 +1,185 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: E2E - Node.js LangChain + +on: + workflow_call: # Allow orchestrator to call this workflow + workflow_dispatch: # Allow manual triggering + push: + branches: [main] + paths: + - 'nodejs/langchain/**' + - 'scripts/e2e/**' + - '.github/workflows/e2e-nodejs-langchain.yml' + pull_request: + branches: [main] + paths: + - 'nodejs/langchain/**' + - 'scripts/e2e/**' + +permissions: + contents: read + +env: + SAMPLE_PATH: nodejs/langchain/sample-agent + AGENT_PORT: 3979 + SCRIPTS_PATH: scripts/e2e + E2E_TESTS_PATH: tests/e2e + +jobs: + nodejs-langchain: + name: Node.js LangChain Agent + runs-on: windows-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: | + if (Test-Path "package-lock.json") { + npm ci + } else { + npm install + } + if (Test-Path "tsconfig.json") { + npm run build + } + shell: pwsh + + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "nodejs" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate .env configuration + shell: pwsh + run: | + $configMappings = @{ + "NODE_ENV" = "development" + "AZURE_OPENAI_API_KEY" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_API_KEY }}" + "AZURE_OPENAI_ENDPOINT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_ENDPOINT }}" + "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_DEPLOYMENT }}" + "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" + "connections__service_connection__settings__authType" = "ClientSecret" + "connections__service_connection__settings__clientId" = "${{ secrets.NODEJS_OPENAI_AGENT_ID }}" + "connections__service_connection__settings__clientSecret" = "${{ secrets.NODEJS_OPENAI_CLIENT_SECRET }}" + "connections__service_connection__settings__tenantId" = "${{ secrets.TENANT_ID }}" + "connectionsMap__0__serviceUrl" = "*" + "connectionsMap__0__connection" = "service_connection" + "agentic_scopes" = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1/.default" + } + & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Port ${{ env.AGENT_PORT }} ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "node dist/index.js" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "nodejs" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + throw "Agent process has stopped" + } + } + $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" + $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --verbosity normal ` + --logger "console;verbosity=detailed" ` + --logger "trx;LogFileName=test-results-nodejs-langchain.trx" ` + --filter "FullyQualifiedName~BasicConversation|FullyQualifiedName~Notification" + env: + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Emit Test Conversations + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" + + - name: Capture Agent Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Stop Agent Process + if: always() + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" -AgentPID $agentPid + } + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-nodejs-langchain + path: | + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*.trx + ${{ env.E2E_TESTS_PATH }}/TestResults/**/*-logs.txt + retention-days: 7 diff --git a/.github/workflows/e2e-nodejs-openai.yml b/.github/workflows/e2e-nodejs-openai.yml index 8c207dd4..2f7555cd 100644 --- a/.github/workflows/e2e-nodejs-openai.yml +++ b/.github/workflows/e2e-nodejs-openai.yml @@ -59,6 +59,13 @@ jobs: } shell: pwsh + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "nodejs" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + - name: Acquire Bearer Token (ROPC) id: token shell: pwsh @@ -141,6 +148,15 @@ jobs: --logger "console;verbosity=detailed" ` --logger "trx;LogFileName=test-results-nodejs-openai.trx" ` --filter "FullyQualifiedName~BasicConversation|FullyQualifiedName~Notification" + env: + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Emit Test Conversations + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" - name: Capture Agent Logs if: always() diff --git a/.github/workflows/e2e-orchestrator.yml b/.github/workflows/e2e-orchestrator.yml index 1cee986c..4b7bc70f 100644 --- a/.github/workflows/e2e-orchestrator.yml +++ b/.github/workflows/e2e-orchestrator.yml @@ -12,6 +12,7 @@ on: paths: - 'python/openai/**' - 'nodejs/openai/**' + - 'nodejs/langchain/**' - 'dotnet/semantic-kernel/**' - 'dotnet/agent-framework/**' - '.github/workflows/e2e-*.yml' @@ -21,6 +22,7 @@ on: paths: - 'python/openai/**' - 'nodejs/openai/**' + - 'nodejs/langchain/**' - 'dotnet/semantic-kernel/**' - 'dotnet/agent-framework/**' - '.github/workflows/e2e-*.yml' @@ -35,11 +37,13 @@ on: - '' - 'python-openai' - 'nodejs-openai' + - 'nodejs-langchain' - 'dotnet-sk' - 'dotnet-af' permissions: contents: read + pull-requests: write jobs: python-openai: @@ -53,6 +57,12 @@ jobs: if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'nodejs-openai' }} uses: ./.github/workflows/e2e-nodejs-openai.yml secrets: inherit + + nodejs-langchain: + name: Node.js LangChain E2E + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'nodejs-langchain' }} + uses: ./.github/workflows/e2e-nodejs-langchain.yml + secrets: inherit dotnet-sk: name: .NET Semantic Kernel E2E @@ -65,3 +75,107 @@ jobs: if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'dotnet-af' }} uses: ./.github/workflows/e2e-dotnet-agent-framework.yml secrets: inherit + + # =========================================================================== + # Summary - Posts test results to PR + # =========================================================================== + summary: + name: Test Summary + runs-on: ubuntu-latest + needs: [python-openai, nodejs-openai, nodejs-langchain, dotnet-sk, dotnet-af] + if: always() + + steps: + - name: Generate Summary + id: summary + run: | + # Determine overall status + OVERALL_STATUS="✅ All Tests Passed" + if [[ "${{ needs.python-openai.result }}" != "success" ]] || \ + [[ "${{ needs.nodejs-openai.result }}" != "success" ]] || \ + [[ "${{ needs.nodejs-langchain.result }}" != "success" ]] || \ + [[ "${{ needs.dotnet-sk.result }}" != "success" ]] || \ + [[ "${{ needs.dotnet-af.result }}" != "success" ]]; then + OVERALL_STATUS="⚠️ Some Tests Failed" + fi + + # Generate status icons using if-else for reliability + if [[ "${{ needs.python-openai.result }}" == "success" ]]; then PYTHON_OPENAI_ICON="✅"; else PYTHON_OPENAI_ICON="❌"; fi + if [[ "${{ needs.nodejs-openai.result }}" == "success" ]]; then NODEJS_OPENAI_ICON="✅"; else NODEJS_OPENAI_ICON="❌"; fi + if [[ "${{ needs.nodejs-langchain.result }}" == "success" ]]; then NODEJS_LANGCHAIN_ICON="✅"; else NODEJS_LANGCHAIN_ICON="❌"; fi + if [[ "${{ needs.dotnet-sk.result }}" == "success" ]]; then DOTNET_SK_ICON="✅"; else DOTNET_SK_ICON="❌"; fi + if [[ "${{ needs.dotnet-af.result }}" == "success" ]]; then DOTNET_AF_ICON="✅"; else DOTNET_AF_ICON="❌"; fi + + echo "## E2E Test Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Status:** $OVERALL_STATUS" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Sample | Status | Result |" >> $GITHUB_STEP_SUMMARY + echo "|--------|--------|--------|" >> $GITHUB_STEP_SUMMARY + echo "| Python OpenAI | $PYTHON_OPENAI_ICON | ${{ needs.python-openai.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| Node.js OpenAI | $NODEJS_OPENAI_ICON | ${{ needs.nodejs-openai.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| Node.js LangChain | $NODEJS_LANGCHAIN_ICON | ${{ needs.nodejs-langchain.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| .NET Semantic Kernel | $DOTNET_SK_ICON | ${{ needs.dotnet-sk.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| .NET Agent Framework | $DOTNET_AF_ICON | ${{ needs.dotnet-af.result }} |" >> $GITHUB_STEP_SUMMARY + + - name: Post PR Comment + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const pythonOpenaiIcon = '${{ needs.python-openai.result }}' === 'success' ? '✅' : '❌'; + const nodejsOpenaiIcon = '${{ needs.nodejs-openai.result }}' === 'success' ? '✅' : '❌'; + const nodejsLangchainIcon = '${{ needs.nodejs-langchain.result }}' === 'success' ? '✅' : '❌'; + const dotnetSkIcon = '${{ needs.dotnet-sk.result }}' === 'success' ? '✅' : '❌'; + const dotnetAfIcon = '${{ needs.dotnet-af.result }}' === 'success' ? '✅' : '❌'; + + const allPassed = '${{ needs.python-openai.result }}' === 'success' && + '${{ needs.nodejs-openai.result }}' === 'success' && + '${{ needs.nodejs-langchain.result }}' === 'success' && + '${{ needs.dotnet-sk.result }}' === 'success' && + '${{ needs.dotnet-af.result }}' === 'success'; + + const overallStatus = allPassed ? '✅ All E2E Tests Passed' : '⚠️ Some E2E Tests Failed'; + + const body = `## ${overallStatus} + +| Sample | Status | Result | +|--------|--------|--------| +| Python OpenAI | ${pythonOpenaiIcon} | ${{ needs.python-openai.result }} | +| Node.js OpenAI | ${nodejsOpenaiIcon} | ${{ needs.nodejs-openai.result }} | +| Node.js LangChain | ${nodejsLangchainIcon} | ${{ needs.nodejs-langchain.result }} | +| .NET Semantic Kernel | ${dotnetSkIcon} | ${{ needs.dotnet-sk.result }} | +| .NET Agent Framework | ${dotnetAfIcon} | ${{ needs.dotnet-af.result }} | + +[View full test details](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}) +`; + + // Find existing comment + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const botComment = comments.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('E2E Tests') + ); + + if (botComment) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: body + }); + } else { + // Create new comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body + }); + } diff --git a/.github/workflows/e2e-python-openai.yml b/.github/workflows/e2e-python-openai.yml index 668f4e8d..e666053c 100644 --- a/.github/workflows/e2e-python-openai.yml +++ b/.github/workflows/e2e-python-openai.yml @@ -53,6 +53,13 @@ jobs: working-directory: ${{ env.SAMPLE_PATH }} run: uv sync + - name: Log SDK Versions + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Get-SDKVersions.ps1" ` + -Runtime "python" ` + -WorkingDirectory "${{ env.SAMPLE_PATH }}" + - name: Acquire Bearer Token (ROPC) id: token shell: pwsh @@ -163,14 +170,12 @@ jobs: path: ${{ runner.temp }}/TestResults/ retention-days: 30 - - name: Upload Test Conversations + - name: Emit Test Conversations if: always() - uses: actions/upload-artifact@v4 - with: - name: test-conversations-python-openai - path: ${{ runner.temp }}/TestConversations/ - retention-days: 30 - continue-on-error: true + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Emit-TestConversations.ps1" ` + -TestResultsDir "${{ runner.temp }}/TestConversations" - name: Capture Agent Logs if: always() From 1cea2248a7bf4d814c6a08fec99040d4bf5ec071 Mon Sep 17 00:00:00 2001 From: abdulanu0 Date: Thu, 29 Jan 2026 17:43:35 -0800 Subject: [PATCH 11/15] Fix YAML syntax error in e2e-orchestrator.yml - use array join for PR comment body --- .github/workflows/e2e-orchestrator.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/e2e-orchestrator.yml b/.github/workflows/e2e-orchestrator.yml index 4b7bc70f..e94d6ccd 100644 --- a/.github/workflows/e2e-orchestrator.yml +++ b/.github/workflows/e2e-orchestrator.yml @@ -137,18 +137,19 @@ jobs: const overallStatus = allPassed ? '✅ All E2E Tests Passed' : '⚠️ Some E2E Tests Failed'; - const body = `## ${overallStatus} - -| Sample | Status | Result | -|--------|--------|--------| -| Python OpenAI | ${pythonOpenaiIcon} | ${{ needs.python-openai.result }} | -| Node.js OpenAI | ${nodejsOpenaiIcon} | ${{ needs.nodejs-openai.result }} | -| Node.js LangChain | ${nodejsLangchainIcon} | ${{ needs.nodejs-langchain.result }} | -| .NET Semantic Kernel | ${dotnetSkIcon} | ${{ needs.dotnet-sk.result }} | -| .NET Agent Framework | ${dotnetAfIcon} | ${{ needs.dotnet-af.result }} | - -[View full test details](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}) -`; + const body = [ + `## ${overallStatus}`, + '', + '| Sample | Status | Result |', + '|--------|--------|--------|', + `| Python OpenAI | ${pythonOpenaiIcon} | ${{ needs.python-openai.result }} |`, + `| Node.js OpenAI | ${nodejsOpenaiIcon} | ${{ needs.nodejs-openai.result }} |`, + `| Node.js LangChain | ${nodejsLangchainIcon} | ${{ needs.nodejs-langchain.result }} |`, + `| .NET Semantic Kernel | ${dotnetSkIcon} | ${{ needs.dotnet-sk.result }} |`, + `| .NET Agent Framework | ${dotnetAfIcon} | ${{ needs.dotnet-af.result }} |`, + '', + `[View full test details](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})` + ].join('\n'); // Find existing comment const { data: comments } = await github.rest.issues.listComments({ From e79210ce74ba47704e2fa313ab655dfcdf8ad52f Mon Sep 17 00:00:00 2001 From: abdulanu0 Date: Thu, 29 Jan 2026 18:06:04 -0800 Subject: [PATCH 12/15] Add debug logging to PR comment step --- .github/workflows/e2e-orchestrator.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/e2e-orchestrator.yml b/.github/workflows/e2e-orchestrator.yml index e94d6ccd..e8c2cf6a 100644 --- a/.github/workflows/e2e-orchestrator.yml +++ b/.github/workflows/e2e-orchestrator.yml @@ -151,6 +151,10 @@ jobs: `[View full test details](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})` ].join('\n'); + console.log('PR Number:', context.issue.number); + console.log('Event:', context.eventName); + console.log('Body length:', body.length); + // Find existing comment const { data: comments } = await github.rest.issues.listComments({ owner: context.repo.owner, @@ -158,13 +162,18 @@ jobs: issue_number: context.issue.number, }); + console.log('Found', comments.length, 'comments'); + const botComment = comments.find(comment => comment.user.type === 'Bot' && comment.body.includes('E2E Tests') ); + console.log('Bot comment found:', !!botComment, botComment?.id); + if (botComment) { // Update existing comment + console.log('Updating comment', botComment.id); await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, From 81b246ad24b289f4caca35e8b8b6ebc8e568255f Mon Sep 17 00:00:00 2001 From: abdulanu0 Date: Thu, 29 Jan 2026 18:12:12 -0800 Subject: [PATCH 13/15] Fix bot comment detection - use login name instead of type --- .github/workflows/e2e-orchestrator.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-orchestrator.yml b/.github/workflows/e2e-orchestrator.yml index e8c2cf6a..4e31014d 100644 --- a/.github/workflows/e2e-orchestrator.yml +++ b/.github/workflows/e2e-orchestrator.yml @@ -164,9 +164,14 @@ jobs: console.log('Found', comments.length, 'comments'); + // Debug: log all comments + comments.forEach((c, i) => { + console.log(`Comment ${i}: user=${c.user.login}, type=${c.user.type}, includes E2E=${c.body.includes('E2E')}`); + }); + const botComment = comments.find(comment => - comment.user.type === 'Bot' && - comment.body.includes('E2E Tests') + comment.user.login === 'github-actions[bot]' && + comment.body.includes('E2E') ); console.log('Bot comment found:', !!botComment, botComment?.id); From 31ce13f4979de10926341075b336f770781ff09b Mon Sep 17 00:00:00 2001 From: abdulanu0 Date: Thu, 29 Jan 2026 18:23:24 -0800 Subject: [PATCH 14/15] Remove debug logging from PR comment step --- .github/workflows/e2e-orchestrator.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/e2e-orchestrator.yml b/.github/workflows/e2e-orchestrator.yml index 4e31014d..ae4ceb91 100644 --- a/.github/workflows/e2e-orchestrator.yml +++ b/.github/workflows/e2e-orchestrator.yml @@ -151,10 +151,6 @@ jobs: `[View full test details](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})` ].join('\n'); - console.log('PR Number:', context.issue.number); - console.log('Event:', context.eventName); - console.log('Body length:', body.length); - // Find existing comment const { data: comments } = await github.rest.issues.listComments({ owner: context.repo.owner, @@ -162,23 +158,13 @@ jobs: issue_number: context.issue.number, }); - console.log('Found', comments.length, 'comments'); - - // Debug: log all comments - comments.forEach((c, i) => { - console.log(`Comment ${i}: user=${c.user.login}, type=${c.user.type}, includes E2E=${c.body.includes('E2E')}`); - }); - const botComment = comments.find(comment => comment.user.login === 'github-actions[bot]' && comment.body.includes('E2E') ); - console.log('Bot comment found:', !!botComment, botComment?.id); - if (botComment) { // Update existing comment - console.log('Updating comment', botComment.id); await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, From 4f3f9693a08d9cd152b4355c539d79d1985973de Mon Sep 17 00:00:00 2001 From: abdulanu0 Date: Thu, 29 Jan 2026 18:38:58 -0800 Subject: [PATCH 15/15] Add logging to diagnose PR comment update issue --- .github/workflows/e2e-orchestrator.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-orchestrator.yml b/.github/workflows/e2e-orchestrator.yml index ae4ceb91..e84be697 100644 --- a/.github/workflows/e2e-orchestrator.yml +++ b/.github/workflows/e2e-orchestrator.yml @@ -158,25 +158,33 @@ jobs: issue_number: context.issue.number, }); + console.log(`Found ${comments.length} comments on PR #${context.issue.number}`); + const botComment = comments.find(comment => comment.user.login === 'github-actions[bot]' && comment.body.includes('E2E') ); + console.log(`Bot comment found: ${!!botComment}, ID: ${botComment?.id}`); + if (botComment) { // Update existing comment - await github.rest.issues.updateComment({ + console.log(`Updating existing comment ${botComment.id}...`); + const result = await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: botComment.id, body: body }); + console.log(`Update result: ${result.status}`); } else { // Create new comment - await github.rest.issues.createComment({ + console.log('Creating new comment...'); + const result = await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, body: body }); + console.log(`Create result: ${result.status}, comment ID: ${result.data.id}`); }