diff --git a/.github/workflows/e2e-agent-samples.yml b/.github/workflows/e2e-agent-samples.yml new file mode 100644 index 00000000..7a80f456 --- /dev/null +++ b/.github/workflows/e2e-agent-samples.yml @@ -0,0 +1,666 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# ============================================================================= +# E2E Tests: Agent Samples +# ============================================================================= +# Runs E2E integration tests for all agent samples +# Uses external PowerShell scripts for maintainability +# ============================================================================= + +name: E2E Agent Samples + +on: + # Run nightly at 6 AM UTC + schedule: + - cron: '0 6 * * *' + push: + branches: + - main + - 'feature/*' + paths: + - 'python/**' + - 'nodejs/**' + - 'dotnet/**' + - 'tests/e2e/**' + - '.github/workflows/e2e-agent-samples.yml' + - 'scripts/e2e/**' + pull_request: + branches: + - main + paths: + - 'python/**' + - 'nodejs/**' + - 'dotnet/**' + - 'tests/e2e/**' + - '.github/workflows/e2e-agent-samples.yml' + - 'scripts/e2e/**' + workflow_dispatch: + inputs: + sample: + description: 'Specific sample to test (leave empty for all)' + required: false + default: '' + debug: + description: 'Enable debug logging' + required: false + default: 'false' + +permissions: + contents: read + +env: + # MCP Authentication - Required for all samples + MCP_CLIENT_ID: ${{ secrets.MCP_CLIENT_ID }} + MCP_TENANT_ID: ${{ secrets.MCP_TENANT_ID }} + MCP_TEST_USERNAME: ${{ secrets.MCP_TEST_USERNAME }} + MCP_TEST_PASSWORD: ${{ secrets.MCP_TEST_PASSWORD }} + MCP_ENVIRONMENT: 'Development' + + # Common settings + SCRIPTS_PATH: scripts/e2e + + # E2E Integration Tests - located in this repository + E2E_TESTS_PATH: 'tests/e2e' + +jobs: + # =========================================================================== + # Python OpenAI Sample + # =========================================================================== + python-openai: + name: Python OpenAI Agent + runs-on: windows-latest + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'python-openai' }} + + env: + SAMPLE_PATH: python/openai/sample-agent + AGENT_PORT: 3979 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install uv package manager + run: pip install uv + + - name: Install dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: uv sync + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate .env configuration + shell: pwsh + run: | + $configMappings = @{ + "AZURE_OPENAI_API_KEY" = "${{ secrets.PYTHON_OPENAI_AZURE_OPENAI_API_KEY }}" + "AZURE_OPENAI_ENDPOINT" = "${{ secrets.PYTHON_OPENAI_AZURE_OPENAI_ENDPOINT }}" + "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.PYTHON_OPENAI_AZURE_OPENAI_DEPLOYMENT }}" + "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" + "USE_AGENTIC_AUTH" = "false" + "MCP_PLATFORM_ENDPOINT" = "https://agent365.svc.cloud.microsoft" + "AGENT_ID" = "${{ secrets.PYTHON_OPENAI_AGENT_ID }}" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__AUTHTYPE" = "ClientSecret" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTID" = "${{ secrets.PYTHON_OPENAI_AGENT_ID }}" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__CLIENTSECRET" = "${{ secrets.PYTHON_OPENAI_CLIENT_SECRET }}" + "CONNECTIONS__SERVICE_CONNECTION__SETTINGS__TENANTID" = "${{ secrets.TENANT_ID }}" + "CONNECTIONSMAP__0__SERVICEURL" = "*" + "CONNECTIONSMAP__0__CONNECTION" = "SERVICE_CONNECTION" + "ENABLE_OBSERVABILITY" = "true" + } + & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Port ${{ env.AGENT_PORT }} ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "uv run python start_with_generic_host.py" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "python" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + Write-Host "=== Verifying Agent ===" -ForegroundColor Cyan + + # Verify agent process is still running + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + Write-Host "Checking agent process (PID: $agentPid)..." -ForegroundColor Gray + + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + + # Show logs + $logFile = "${{ env.SAMPLE_PATH }}/agent.log" + if (Test-Path $logFile) { + Write-Host "Agent logs:" -ForegroundColor Yellow + Get-Content $logFile + } + throw "Agent process has stopped" + } + } + + # Quick health check before running tests + $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" + Write-Host "Verifying agent at $agentUrl..." -ForegroundColor Gray + $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + shell: pwsh + run: | + dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --filter "Category=HTTP" ` + --logger "trx;LogFileName=test-results.trx" ` + --results-directory "${{ runner.temp }}/TestResults" + env: + AGENT_PORT: ${{ env.AGENT_PORT }} + AGENT_URL: http://localhost:${{ env.AGENT_PORT }} + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-python-openai + path: ${{ runner.temp }}/TestResults/ + retention-days: 30 + + - name: Upload Test Conversations + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-conversations-python-openai + path: ${{ runner.temp }}/TestConversations/ + retention-days: 30 + continue-on-error: true + + - name: Capture Agent Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Cleanup Agent Process + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` + -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` + -Port ${{ env.AGENT_PORT }} + + # =========================================================================== + # Node.js OpenAI Sample + # =========================================================================== + nodejs-openai: + name: Node.js OpenAI Agent + runs-on: windows-latest + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'nodejs-openai' }} + + env: + SAMPLE_PATH: nodejs/openai/sample-agent + AGENT_PORT: 3979 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: | + if (Test-Path "package-lock.json") { + npm ci + } else { + npm install + } + if (Test-Path "tsconfig.json") { + npm run build + } + shell: pwsh + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate .env configuration + shell: pwsh + run: | + $configMappings = @{ + "NODE_ENV" = "development" + "AZURE_OPENAI_API_KEY" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_API_KEY }}" + "AZURE_OPENAI_ENDPOINT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_ENDPOINT }}" + "AZURE_OPENAI_DEPLOYMENT" = "${{ secrets.NODEJS_OPENAI_AZURE_OPENAI_DEPLOYMENT }}" + "AZURE_OPENAI_API_VERSION" = "2024-12-01-preview" + "connections__service_connection__settings__authType" = "ClientSecret" + "connections__service_connection__settings__clientId" = "${{ secrets.NODEJS_OPENAI_AGENT_ID }}" + "connections__service_connection__settings__clientSecret" = "${{ secrets.NODEJS_OPENAI_CLIENT_SECRET }}" + "connections__service_connection__settings__tenantId" = "${{ secrets.TENANT_ID }}" + "connectionsMap__0__serviceUrl" = "*" + "connectionsMap__0__connection" = "service_connection" + "agentic_scopes" = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1/.default" + } + & "${{ env.SCRIPTS_PATH }}/Generate-EnvConfig.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/.env" ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Port ${{ env.AGENT_PORT }} ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "node dist/index.js" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "nodejs" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + throw "Agent process has stopped" + } + } + $agentUrl = "http://localhost:${{ env.AGENT_PORT }}" + $healthResponse = Invoke-WebRequest -Uri "$agentUrl/api/health" -Method GET -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health check: $($healthResponse.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --filter "Category=HTTP" ` + --logger "trx;LogFileName=test-results.trx" ` + --results-directory "${{ runner.temp }}/TestResults" + env: + AGENT_PORT: ${{ env.AGENT_PORT }} + AGENT_URL: http://localhost:${{ env.AGENT_PORT }} + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-nodejs-openai + path: ${{ runner.temp }}/TestResults/ + retention-days: 30 + + - name: Capture Agent Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Cleanup + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` + -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` + -Port ${{ env.AGENT_PORT }} + + # =========================================================================== + # .NET Semantic Kernel Sample + # =========================================================================== + dotnet-semantic-kernel: + name: .NET Semantic Kernel Agent + runs-on: windows-latest + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'dotnet-semantic-kernel' }} + + env: + SAMPLE_PATH: dotnet/semantic-kernel/sample-agent + AGENT_PORT: 3978 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Restore dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: dotnet restore + + - name: Build + working-directory: ${{ env.SAMPLE_PATH }} + run: dotnet build --no-restore + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate appsettings.json + shell: pwsh + run: | + $configMappings = @{ + "AIServices:UseAzureOpenAI" = "true" + "AIServices:AzureOpenAI:Endpoint" = "${{ secrets.DOTNET_SK_AZURE_OPENAI_ENDPOINT }}" + "AIServices:AzureOpenAI:ApiKey" = "${{ secrets.DOTNET_SK_AZURE_OPENAI_API_KEY }}" + "AIServices:AzureOpenAI:DeploymentName" = "${{ secrets.DOTNET_SK_AZURE_OPENAI_DEPLOYMENT }}" + "TokenValidation:Enabled" = "false" + "TokenValidation:TenantId" = "${{ secrets.TENANT_ID }}" + "Connections:ServiceConnection:Settings:AuthType" = "ClientSecret" + "Connections:ServiceConnection:Settings:ClientId" = "${{ secrets.DOTNET_SK_CLIENT_ID }}" + "Connections:ServiceConnection:Settings:ClientSecret" = "${{ secrets.DOTNET_SK_CLIENT_SECRET }}" + "Connections:ServiceConnection:Settings:TenantId" = "${{ secrets.TENANT_ID }}" + } + & "${{ env.SCRIPTS_PATH }}/Generate-AppSettings.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/appsettings.json" ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "dotnet run --no-build" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "dotnet" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + throw "Agent process has stopped" + } + } + $response = Invoke-WebRequest -Uri "http://localhost:${{ env.AGENT_PORT }}/api/health" -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health: $($response.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --filter "Category=HTTP" ` + --logger "trx;LogFileName=test-results.trx" ` + --results-directory "${{ runner.temp }}/TestResults" + env: + AGENT_PORT: ${{ env.AGENT_PORT }} + AGENT_URL: http://localhost:${{ env.AGENT_PORT }} + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-dotnet-sk + path: ${{ runner.temp }}/TestResults/ + retention-days: 30 + + - name: Capture Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Cleanup + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` + -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` + -Port ${{ env.AGENT_PORT }} + + # =========================================================================== + # .NET Agent Framework Sample + # =========================================================================== + dotnet-agent-framework: + name: .NET Agent Framework Agent + runs-on: windows-latest + if: ${{ github.event.inputs.sample == '' || github.event.inputs.sample == 'dotnet-agent-framework' }} + + env: + SAMPLE_PATH: dotnet/agent-framework/sample-agent + AGENT_PORT: 3978 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Restore dependencies + working-directory: ${{ env.SAMPLE_PATH }} + run: dotnet restore + + - name: Build + working-directory: ${{ env.SAMPLE_PATH }} + run: dotnet build --no-restore + + - name: Acquire Bearer Token (ROPC) + id: token + shell: pwsh + run: | + $token = & "${{ env.SCRIPTS_PATH }}/Acquire-BearerToken.ps1" ` + -ClientId "${{ secrets.MCP_CLIENT_ID }}" ` + -TenantId "${{ secrets.MCP_TENANT_ID }}" ` + -Username "${{ secrets.MCP_TEST_USERNAME }}" ` + -Password "${{ secrets.MCP_TEST_PASSWORD }}" + echo "BEARER_TOKEN=$token" >> $env:GITHUB_OUTPUT + echo "::add-mask::$token" + + - name: Copy ToolingManifest.json + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Copy-ToolingManifest.ps1" -TargetPath "${{ env.SAMPLE_PATH }}" + + - name: Generate appsettings.json + shell: pwsh + run: | + $configMappings = @{ + "AIServices:AzureOpenAI:Endpoint" = "${{ secrets.DOTNET_AF_AZURE_OPENAI_ENDPOINT }}" + "AIServices:AzureOpenAI:ApiKey" = "${{ secrets.DOTNET_AF_AZURE_OPENAI_API_KEY }}" + "AIServices:AzureOpenAI:DeploymentName" = "${{ secrets.DOTNET_AF_AZURE_OPENAI_DEPLOYMENT }}" + "TokenValidation:Enabled" = "false" + "Connections:ServiceConnection:Settings:AuthType" = "ClientSecret" + "Connections:ServiceConnection:Settings:ClientId" = "${{ secrets.DOTNET_AF_CLIENT_ID }}" + "Connections:ServiceConnection:Settings:ClientSecret" = "${{ secrets.DOTNET_AF_CLIENT_SECRET }}" + "Connections:ServiceConnection:Settings:TenantId" = "${{ secrets.TENANT_ID }}" + "Connections:ServiceConnection:Settings:AuthorityEndpoint" = "https://login.microsoftonline.com/${{ secrets.TENANT_ID }}" + "Connections:ServiceConnection:Settings:Scopes:0" = "5a807f24-c9de-44ee-a3a7-329e88a00ffc/.default" + } + & "${{ env.SCRIPTS_PATH }}/Generate-AppSettings.ps1" ` + -OutputPath "${{ env.SAMPLE_PATH }}/appsettings.json" ` + -ConfigMappings $configMappings + + - name: Start Agent + id: start-agent + shell: pwsh + run: | + $agentPid = & "${{ env.SCRIPTS_PATH }}/Start-Agent.ps1" ` + -AgentPath "${{ env.SAMPLE_PATH }}" ` + -StartCommand "dotnet run --no-build" ` + -Port ${{ env.AGENT_PORT }} ` + -BearerToken "${{ steps.token.outputs.BEARER_TOKEN }}" ` + -Environment "Development" ` + -Runtime "dotnet" + echo "AGENT_PID=$agentPid" >> $env:GITHUB_OUTPUT + + - name: Verify Agent Running + shell: pwsh + run: | + $agentPid = "${{ steps.start-agent.outputs.AGENT_PID }}" + if ($agentPid) { + try { + $proc = Get-Process -Id $agentPid -ErrorAction Stop + Write-Host "Agent process (PID: $agentPid) is running: $($proc.ProcessName)" -ForegroundColor Green + } catch { + Write-Host "ERROR: Agent process (PID: $agentPid) is NOT running!" -ForegroundColor Red + throw "Agent process has stopped" + } + } + $response = Invoke-WebRequest -Uri "http://localhost:${{ env.AGENT_PORT }}/api/health" -UseBasicParsing -ErrorAction SilentlyContinue + Write-Host "Health: $($response.StatusCode)" -ForegroundColor Green + + - name: Restore E2E Test Dependencies + run: dotnet restore "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" + + - name: Run .NET E2E Tests + shell: pwsh + run: | + dotnet test "${{ env.E2E_TESTS_PATH }}/Agent365.E2E.Tests.csproj" ` + --no-restore ` + --filter "Category=HTTP" ` + --logger "trx;LogFileName=test-results.trx" ` + --results-directory "${{ runner.temp }}/TestResults" + env: + AGENT_PORT: ${{ env.AGENT_PORT }} + AGENT_URL: http://localhost:${{ env.AGENT_PORT }} + TEST_RESULTS_DIR: ${{ runner.temp }}/TestConversations + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-dotnet-af + path: ${{ runner.temp }}/TestResults/ + retention-days: 30 + + - name: Capture Logs + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Capture-AgentLogs.ps1" -AgentPath "${{ env.SAMPLE_PATH }}" + + - name: Cleanup + if: always() + shell: pwsh + run: | + & "${{ env.SCRIPTS_PATH }}/Stop-AgentProcess.ps1" ` + -AgentPID "${{ steps.start-agent.outputs.AGENT_PID }}" ` + -Port ${{ env.AGENT_PORT }} + + # =========================================================================== + # Summary + # =========================================================================== + summary: + name: Test Summary + runs-on: ubuntu-latest + needs: [python-openai, nodejs-openai, dotnet-semantic-kernel, dotnet-agent-framework] + if: always() + + steps: + - name: Generate Summary + run: | + echo "## E2E Test Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Sample | Status |" >> $GITHUB_STEP_SUMMARY + echo "|--------|--------|" >> $GITHUB_STEP_SUMMARY + echo "| Python OpenAI | ${{ needs.python-openai.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| Node.js OpenAI | ${{ needs.nodejs-openai.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| .NET Semantic Kernel | ${{ needs.dotnet-semantic-kernel.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| .NET Agent Framework | ${{ needs.dotnet-agent-framework.result }} |" >> $GITHUB_STEP_SUMMARY diff --git a/README.md b/README.md index 76a2f6de..e160d912 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,21 @@ # Microsoft Agent 365 SDK Samples and Prompts +[![E2E Tests](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml/badge.svg)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml) + This repository contains sample agents and prompts for building with the Microsoft Agent 365 SDK. The Microsoft Agent 365 SDK extends the Microsoft 365 Agents SDK with enterprise-grade capabilities for building sophisticated agents. It provides comprehensive tooling for observability, notifications, runtime utilities, and development tools that help developers create production-ready agents for platforms including M365, Teams, Copilot Studio, and Webchat. - **Sample agents** are available in C# (.NET), Python, and Node.js/TypeScript - **Prompts** to help you get started with AI-powered development tools like Cursor IDE +## E2E Test Status + +| Sample | Status | +|--------|--------| +| Python OpenAI | [![Python OpenAI](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml/badge.svg?job=python-openai)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml) | +| Node.js OpenAI | [![Node.js OpenAI](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml/badge.svg?job=nodejs-openai)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml) | +| .NET Semantic Kernel | [![.NET SK](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml/badge.svg?job=dotnet-semantic-kernel)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml) | +| .NET Agent Framework | [![.NET AF](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml/badge.svg?job=dotnet-agent-framework)](https://github.com/microsoft/Agent365-Samples/actions/workflows/e2e-agent-samples.yml) | + > #### Note: > Use the information in this README to contribute to this open-source project. To learn about using this SDK in your projects, refer to the [Microsoft Agent 365 Developer documentation](https://learn.microsoft.com/en-us/microsoft-agent-365/developer/). diff --git a/dotnet/agent-framework/sample-agent/Program.cs b/dotnet/agent-framework/sample-agent/Program.cs index 00d58233..8651aae4 100644 --- a/dotnet/agent-framework/sample-agent/Program.cs +++ b/dotnet/agent-framework/sample-agent/Program.cs @@ -118,6 +118,8 @@ await AgentMetrics.InvokeObservedHttpOperation("agent.process_message", async () }).ConfigureAwait(false); }); +// Health check endpoint for CI/CD pipelines and monitoring +app.MapGet("/api/health", () => Results.Ok(new { status = "healthy", timestamp = System.DateTime.UtcNow })); if (app.Environment.IsDevelopment() || app.Environment.EnvironmentName == "Playground") { diff --git a/dotnet/semantic-kernel/sample-agent/Program.cs b/dotnet/semantic-kernel/sample-agent/Program.cs index 6cdc2c98..35c0ccb4 100644 --- a/dotnet/semantic-kernel/sample-agent/Program.cs +++ b/dotnet/semantic-kernel/sample-agent/Program.cs @@ -61,9 +61,9 @@ builder.Services.AddAgenticTracingExporter(); // Add A365 tracing with Semantic Kernel integration -builder.AddA365Tracing(config => +builder.AddA365Tracing(config => { - config.WithSemanticKernel(); + config.WithSemanticKernel(); }); @@ -98,21 +98,24 @@ // This receives incoming messages from Azure Bot Service or other SDK Agents var incomingRoute = app.MapPost("/api/messages", async (HttpRequest request, HttpResponse response, IAgentHttpAdapter adapter, IAgent agent, CancellationToken cancellationToken) => -{ - await AgentMetrics.InvokeObservedHttpOperation("agent.process_message", async () => - { - await adapter.ProcessAsync(request, response, agent, cancellationToken); - }).ConfigureAwait(false); +{ + await AgentMetrics.InvokeObservedHttpOperation("agent.process_message", async () => + { + await adapter.ProcessAsync(request, response, agent, cancellationToken); + }).ConfigureAwait(false); }); +// Health check endpoint for CI/CD pipelines and monitoring +app.MapGet("/api/health", () => Results.Ok(new { status = "healthy", timestamp = System.DateTime.UtcNow })); + if (app.Environment.IsDevelopment() || app.Environment.EnvironmentName == "Playground") { app.MapGet("/", () => "Agent 365 Semantic Kernel Example Agent"); app.UseDeveloperExceptionPage(); - app.MapControllers().AllowAnonymous(); - - // Hard coded for brevity and ease of testing. - // In production, this should be set in configuration. + app.MapControllers().AllowAnonymous(); + + // Hard coded for brevity and ease of testing. + // In production, this should be set in configuration. app.Urls.Add($"http://localhost:3978"); } else diff --git a/scripts/e2e/Acquire-BearerToken.ps1 b/scripts/e2e/Acquire-BearerToken.ps1 new file mode 100644 index 00000000..9ee6d361 --- /dev/null +++ b/scripts/e2e/Acquire-BearerToken.ps1 @@ -0,0 +1,92 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Acquires a bearer token using Resource Owner Password Credentials (ROPC) flow. + +.DESCRIPTION + MCP servers only support delegated (user) permissions, not application permissions. + This script uses ROPC flow with a test service account to acquire user-delegated + tokens in the CI/CD pipeline. + +.PARAMETER ClientId + The application (client) ID with MCP permissions. + +.PARAMETER TenantId + The Azure AD tenant ID. + +.PARAMETER Username + The test service account UPN (should be excluded from MFA). + +.PARAMETER Password + The test service account password. + +.PARAMETER Scope + The scope to request. Defaults to MCP scope. + +.OUTPUTS + Returns the access token as a string. + +.EXAMPLE + $token = ./Acquire-BearerToken.ps1 -ClientId $clientId -TenantId $tenantId -Username $user -Password $pass +#> + +param( + [Parameter(Mandatory = $true)] + [string]$ClientId, + + [Parameter(Mandatory = $true)] + [string]$TenantId, + + [Parameter(Mandatory = $true)] + [string]$Username, + + [Parameter(Mandatory = $true)] + [string]$Password, + + [Parameter(Mandatory = $false)] + [string]$Scope = "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1/.default" +) + +$ErrorActionPreference = "Stop" + +Write-Host "Acquiring bearer token for MCP servers using ROPC flow..." -ForegroundColor Cyan +Write-Host "MCP servers require delegated (user) tokens, not service principal tokens" -ForegroundColor Gray + +$body = @{ + grant_type = "password" + client_id = $ClientId + scope = $Scope + username = $Username + password = $Password +} + +$tokenEndpoint = "https://login.microsoftonline.com/$TenantId/oauth2/v2.0/token" + +try { + $response = Invoke-RestMethod -Uri $tokenEndpoint -Method POST -Body $body -ContentType "application/x-www-form-urlencoded" + $token = $response.access_token + + if ([string]::IsNullOrEmpty($token)) { + throw "Token response did not contain access_token" + } + + $tokenLength = $token.Length + $expiresIn = $response.expires_in + Write-Host "Bearer token acquired successfully" -ForegroundColor Green + Write-Host " Token length: $tokenLength characters" -ForegroundColor Gray + Write-Host " Expires in: $expiresIn seconds" -ForegroundColor Gray + + # Return the token + return $token +} +catch { + Write-Error "Failed to acquire bearer token via ROPC: $_" + Write-Host "Ensure the following variables are set:" -ForegroundColor Yellow + Write-Host " - ClientId: App registration with MCP permissions" -ForegroundColor Yellow + Write-Host " - TenantId: Azure AD tenant ID" -ForegroundColor Yellow + Write-Host " - Username: Service account UPN (excluded from MFA)" -ForegroundColor Yellow + Write-Host " - Password: Service account password" -ForegroundColor Yellow + throw +} diff --git a/scripts/e2e/Capture-AgentLogs.ps1 b/scripts/e2e/Capture-AgentLogs.ps1 new file mode 100644 index 00000000..a406aa12 --- /dev/null +++ b/scripts/e2e/Capture-AgentLogs.ps1 @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Captures and displays agent logs for debugging. + +.DESCRIPTION + Reads agent log files and displays them with sensitive values redacted. + +.PARAMETER AgentPath + The path to the agent directory. + +.EXAMPLE + ./Capture-AgentLogs.ps1 -AgentPath "./sample-agent" +#> + +param( + [Parameter(Mandatory = $true)] + [string]$AgentPath +) + +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "AGENT LOGS (transcript)" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan + +$logFile = Join-Path $AgentPath "agent.log" +if (Test-Path $logFile) { + Write-Host "Agent transcript file found at: $logFile" -ForegroundColor Green + Write-Host "----------------------------------------" + Get-Content $logFile + Write-Host "----------------------------------------" + Write-Host "End of agent transcript" -ForegroundColor Green +} +else { + Write-Host "No agent.log file found at: $logFile" -ForegroundColor Yellow +} + +# Check command output log +$outputLogFile = Join-Path $AgentPath "agent-output.log" +if (Test-Path $outputLogFile) { + $outputContent = Get-Content $outputLogFile -Raw + if (-not [string]::IsNullOrWhiteSpace($outputContent)) { + Write-Host "" + Write-Host "========================================" -ForegroundColor Cyan + Write-Host "AGENT COMMAND OUTPUT" -ForegroundColor Cyan + Write-Host "========================================" -ForegroundColor Cyan + Write-Host $outputContent + } +} + +# Show .env file contents (redacted) +$envFile = Join-Path $AgentPath ".env" +if (Test-Path $envFile) { + Write-Host "" + Write-Host "========================================" -ForegroundColor Cyan + Write-Host ".ENV FILE (secrets redacted)" -ForegroundColor Cyan + Write-Host "========================================" -ForegroundColor Cyan + Get-Content $envFile | ForEach-Object { + if ($_ -match '^(BEARER_TOKEN|.*SECRET|.*PASSWORD|.*KEY)=') { + $name = $_ -replace '=.*', '' + $value = $_ -replace '^[^=]+=', '' + Write-Host "$name=***REDACTED*** (length: $($value.Length))" + } + else { + Write-Host $_ + } + } +} + +# Show wrapper script (redacted) +$wrapperScript = Join-Path $AgentPath "run-agent.ps1" +if (Test-Path $wrapperScript) { + Write-Host "" + Write-Host "========================================" -ForegroundColor Cyan + Write-Host "WRAPPER SCRIPT (tokens redacted)" -ForegroundColor Cyan + Write-Host "========================================" -ForegroundColor Cyan + Get-Content $wrapperScript | ForEach-Object { + if ($_ -match "BEARER_TOKEN\s*=\s*'") { + Write-Host "`$env:BEARER_TOKEN = '***REDACTED***'" + } + else { + Write-Host $_ + } + } +} diff --git a/scripts/e2e/Copy-ToolingManifest.ps1 b/scripts/e2e/Copy-ToolingManifest.ps1 new file mode 100644 index 00000000..d375a0d4 --- /dev/null +++ b/scripts/e2e/Copy-ToolingManifest.ps1 @@ -0,0 +1,42 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Copies the centralized ToolingManifest.json to a sample agent directory. + +.DESCRIPTION + Creates a standardized ToolingManifest.json for E2E testing with MCP servers. + +.PARAMETER TargetPath + The path to the sample agent directory. + +.EXAMPLE + ./Copy-ToolingManifest.ps1 -TargetPath "./python/openai/sample-agent" +#> + +param( + [Parameter(Mandatory = $true)] + [string]$TargetPath +) + +$ErrorActionPreference = "Stop" + +Write-Host "Copying ToolingManifest.json to: $TargetPath" -ForegroundColor Cyan + +# Standard ToolingManifest.json for E2E testing +$manifest = @{ + mcpServers = @( + @{ + mcpServerName = "mcp_MailTools" + mcpServerUniqueName = "mcp_MailTools" + } + ) +} + +$manifestPath = Join-Path $TargetPath "ToolingManifest.json" +$manifest | ConvertTo-Json -Depth 5 | Out-File -FilePath $manifestPath -Encoding utf8 + +Write-Host "ToolingManifest.json created at: $manifestPath" -ForegroundColor Green +Write-Host "Contents:" -ForegroundColor Gray +Get-Content $manifestPath | Write-Host diff --git a/scripts/e2e/Generate-AppSettings.ps1 b/scripts/e2e/Generate-AppSettings.ps1 new file mode 100644 index 00000000..8b94a9d0 --- /dev/null +++ b/scripts/e2e/Generate-AppSettings.ps1 @@ -0,0 +1,109 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Updates appsettings.json configuration for .NET agents. + +.DESCRIPTION + Reads an existing appsettings.json file and applies configuration mappings. + Supports nested properties using colon (:) or double underscore (__) notation. + +.PARAMETER OutputPath + The path to the appsettings.json file. + +.PARAMETER ConfigMappings + A hashtable of configuration key-value pairs. + Keys use colon notation for nested properties (e.g., "AzureOpenAI:Endpoint"). + +.EXAMPLE + ./Generate-AppSettings.ps1 -OutputPath "./sample-agent/appsettings.json" -ConfigMappings @{ "AzureOpenAI:ApiKey" = "xxx" } +#> + +param( + [Parameter(Mandatory = $true)] + [string]$OutputPath, + + [Parameter(Mandatory = $false)] + [hashtable]$ConfigMappings = @{} +) + +$ErrorActionPreference = "Stop" + +Write-Host "Updating appsettings.json at: $OutputPath" -ForegroundColor Cyan + +# Helper function to set nested value using colon-separated path +function Set-NestedValue { + param($obj, $path, $value) + $parts = $path -split ':' + $current = $obj + for ($i = 0; $i -lt $parts.Length - 1; $i++) { + $key = $parts[$i] + if (-not $current.ContainsKey($key)) { + $current[$key] = @{} + } + elseif ($current[$key] -isnot [hashtable]) { + $current[$key] = @{} + } + $current = $current[$key] + } + # Handle boolean and numeric values + $finalKey = $parts[-1] + if ($value -eq 'true') { + $current[$finalKey] = $true + } + elseif ($value -eq 'false') { + $current[$finalKey] = $false + } + elseif ($value -match '^\d+$') { + $current[$finalKey] = [int]$value + } + else { + $current[$finalKey] = $value + } +} + +# Load existing appsettings.json +if (Test-Path $OutputPath) { + # Read the file and strip JSON comments (// style) + $lines = Get-Content $OutputPath + $cleanLines = @() + foreach ($line in $lines) { + if ($line -match '^\s*//') { + continue + } + $cleanLine = $line -replace '(? + +param( + [Parameter(Mandatory = $true)] + [string]$OutputPath, + + [Parameter(Mandatory = $true)] + [string]$BearerToken, + + [Parameter(Mandatory = $false)] + [int]$Port = 3979, + + [Parameter(Mandatory = $false)] + [hashtable]$ConfigMappings = @{} +) + +$ErrorActionPreference = "Stop" + +Write-Host "Generating .env at: $OutputPath" -ForegroundColor Cyan + +# Validate BEARER_TOKEN is available +if ([string]::IsNullOrEmpty($BearerToken)) { + Write-Host "ERROR: BEARER_TOKEN is empty or not set!" -ForegroundColor Red + throw "BEARER_TOKEN is required" +} +Write-Host "BEARER_TOKEN available (length: $($BearerToken.Length) chars)" -ForegroundColor Green + +$envContent = @() +$envContent += "# Auto-generated configuration" +$envContent += "# Generated at: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')" +$envContent += "" + +# Add BEARER_TOKEN +$envContent += "BEARER_TOKEN=$BearerToken" + +# Add PORT +$envContent += "PORT=$Port" + +# Add all config mappings +foreach ($key in $ConfigMappings.Keys) { + $value = $ConfigMappings[$key] + $envContent += "$key=$value" +} + +# Write file +$envContent | Out-File -FilePath $OutputPath -Encoding utf8 + +Write-Host "Configuration generated:" -ForegroundColor Green +Get-Content $OutputPath | ForEach-Object { + if ($_ -match '^([^=]+)=') { + $key = $Matches[1] + if ($key -match 'KEY|SECRET|TOKEN|PASSWORD') { + $value = $_.Split('=', 2)[1] + Write-Host " $key=*** (length: $($value.Length))" -ForegroundColor Gray + } + else { + Write-Host " $_" -ForegroundColor Gray + } + } +} + +Write-Host ".env file generated successfully" -ForegroundColor Green diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md new file mode 100644 index 00000000..238a89b7 --- /dev/null +++ b/scripts/e2e/README.md @@ -0,0 +1,87 @@ +# E2E Test Scripts + +This folder contains PowerShell scripts used by the GitHub Actions E2E workflow to test agent samples. + +## Scripts + +| Script | Description | +|--------|-------------| +| [Acquire-BearerToken.ps1](Acquire-BearerToken.ps1) | Acquires a bearer token using ROPC flow for MCP authentication | +| [Generate-EnvConfig.ps1](Generate-EnvConfig.ps1) | Generates `.env` configuration files for Python/Node.js agents | +| [Generate-AppSettings.ps1](Generate-AppSettings.ps1) | Updates `appsettings.json` for .NET agents | +| [Start-Agent.ps1](Start-Agent.ps1) | Starts an agent and waits for health check | +| [Stop-AgentProcess.ps1](Stop-AgentProcess.ps1) | Stops agent processes and cleans up ports | +| [Capture-AgentLogs.ps1](Capture-AgentLogs.ps1) | Captures agent logs with secrets redacted | +| [Copy-ToolingManifest.ps1](Copy-ToolingManifest.ps1) | Creates ToolingManifest.json for MCP server configuration | + +## MCP Authentication + +MCP servers require **delegated (user) permissions**, not application permissions. This means: + +- Client credentials flow (service principal) does **not** work +- ROPC (Resource Owner Password Credentials) flow is used with a test service account +- The test account should be excluded from MFA/Conditional Access + +### Required Secrets + +| Secret | Description | +|--------|-------------| +| `GH_PAT` | GitHub Personal Access Token with repo access to checkout E2E tests repository | +| `MCP_CLIENT_ID` | App registration client ID with MCP permissions (`0de5c94a-9929-45e1-a436-ffe5b3415df3`) | +| `MCP_TENANT_ID` | Azure AD tenant ID | +| `MCP_TEST_USERNAME` | Test service account UPN | +| `MCP_TEST_PASSWORD` | Test service account password | +| `TENANT_ID` | Common tenant ID for agent connections | + +### Repository Variables (Optional) + +| Variable | Description | +|----------|-------------| +| `E2E_TESTS_REPO` | Override the E2E tests repository (default: `microsoft/Agent365-E2EIntegration`) | + +### Per-Sample Secrets + +Each sample requires its own set of secrets: + +**Python OpenAI (`PYTHON_OPENAI_*`):** +- `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_DEPLOYMENT` +- `AGENT_ID`, `CLIENT_SECRET` + +**Node.js OpenAI (`NODEJS_OPENAI_*`):** +- `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_DEPLOYMENT` +- `AGENT_ID`, `CLIENT_SECRET` + +**.NET Semantic Kernel (`DOTNET_SK_*`):** +- `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_DEPLOYMENT` +- `CLIENT_ID`, `CLIENT_SECRET` + +**.NET Agent Framework (`DOTNET_AF_*`):** +- `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_DEPLOYMENT` +- `CLIENT_ID`, `CLIENT_SECRET` + +## Environment Variable: ENVIRONMENT + +Setting `ENVIRONMENT=Development` is **critical** for E2E tests. This tells the SDK to: + +1. Read from local `ToolingManifest.json` instead of calling the cloud gateway +2. Use the `BEARER_TOKEN` environment variable for MCP authentication + +Without this, the agent will try to call the cloud MCP gateway and fail. + +## Usage + +These scripts are designed to be called from GitHub Actions but can also be run locally for testing: + +```powershell +# Acquire token +$token = ./Acquire-BearerToken.ps1 -ClientId $clientId -TenantId $tenantId -Username $user -Password $pass + +# Generate .env +./Generate-EnvConfig.ps1 -OutputPath "./sample/.env" -BearerToken $token -Port 3979 -ConfigMappings @{ "API_KEY" = "xxx" } + +# Start agent +$pid = ./Start-Agent.ps1 -AgentPath "./sample" -StartCommand "npm start" -Port 3979 -BearerToken $token -Runtime "nodejs" + +# Stop agent +./Stop-AgentProcess.ps1 -AgentPID $pid -Port 3979 +``` diff --git a/scripts/e2e/Start-Agent.ps1 b/scripts/e2e/Start-Agent.ps1 new file mode 100644 index 00000000..d0af3863 --- /dev/null +++ b/scripts/e2e/Start-Agent.ps1 @@ -0,0 +1,417 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Starts an agent and waits for it to become ready. + +.DESCRIPTION + Creates a PowerShell wrapper script to set environment variables and start the agent. + Waits for the agent to respond on the health endpoint or messages endpoint. + +.PARAMETER AgentPath + The path to the agent directory. + +.PARAMETER StartCommand + The command to start the agent (e.g., "npm start", "dotnet run"). + +.PARAMETER Port + The port the agent will listen on. + +.PARAMETER BearerToken + The bearer token for MCP authentication. + +.PARAMETER HealthEndpoint + The health check endpoint path. Defaults to "/api/health". + +.PARAMETER Environment + The MCP environment mode. Defaults to "Development". + +.PARAMETER TimeoutSeconds + Maximum time to wait for agent to become ready. Defaults to 90 seconds. + +.PARAMETER Runtime + The runtime type (python, nodejs, dotnet). + +.OUTPUTS + Returns the process ID of the started agent. + +.EXAMPLE + $pid = ./Start-Agent.ps1 -AgentPath "./sample-agent" -StartCommand "npm start" -Port 3979 -BearerToken $token +#> + +param( + [Parameter(Mandatory = $true)] + [string]$AgentPath, + + [Parameter(Mandatory = $true)] + [string]$StartCommand, + + [Parameter(Mandatory = $true)] + [int]$Port, + + [Parameter(Mandatory = $true)] + [string]$BearerToken, + + [Parameter(Mandatory = $false)] + [string]$HealthEndpoint = "/api/health", + + [Parameter(Mandatory = $false)] + [string]$Environment = "Development", + + [Parameter(Mandatory = $false)] + [int]$TimeoutSeconds = 90, + + [Parameter(Mandatory = $false)] + [ValidateSet("python", "nodejs", "dotnet")] + [string]$Runtime = "nodejs" +) + +$ErrorActionPreference = "Stop" + +# Resolve to absolute path +$AgentPath = Resolve-Path $AgentPath | Select-Object -ExpandProperty Path + +# Handle empty Environment parameter - default to Development for local ToolingManifest.json +if ([string]::IsNullOrEmpty($Environment)) { + Write-Host "Environment not set, defaulting to: Development" -ForegroundColor Yellow + $Environment = "Development" +} + +Write-Host "Starting agent on port $Port..." -ForegroundColor Cyan +Write-Host " Command: $StartCommand" -ForegroundColor Gray +Write-Host " Path: $AgentPath" -ForegroundColor Gray +Write-Host " Runtime: $Runtime" -ForegroundColor Gray +Write-Host " Environment: $Environment" -ForegroundColor Gray + +# Verify config file exists based on runtime +if ($Runtime -eq "dotnet") { + $configPath = Join-Path $AgentPath "appsettings.json" + $configType = "appsettings.json" +} +else { + $configPath = Join-Path $AgentPath ".env" + $configType = ".env" +} + +if (-not (Test-Path $configPath)) { + Write-Host " ERROR: $configType file not found at $configPath" -ForegroundColor Red + throw "$configType file not found" +} + +Write-Host " $configType file exists at: $configPath" -ForegroundColor Green + +# Show config with secrets masked +if ($Runtime -eq "dotnet") { + Write-Host " $configType contents (secrets masked):" -ForegroundColor Gray + $config = Get-Content $configPath -Raw | ConvertFrom-Json + $configJson = $config | ConvertTo-Json -Depth 5 + $configJson = $configJson -replace '("(?:ApiKey|ClientSecret|Password|Secret|Token|Key)":\s*")[^"]+(")', '$1***$2' + Write-Host $configJson -ForegroundColor DarkGray +} +else { + Write-Host " $configType contents (masked):" -ForegroundColor Gray + Get-Content $configPath | ForEach-Object { + if ($_ -match '^([^=]+)=') { + $key = $Matches[1] + if ($key -match 'KEY|SECRET|TOKEN|PASSWORD') { + Write-Host " $key=***" -ForegroundColor Gray + } + else { + Write-Host " $_" -ForegroundColor Gray + } + } + } +} + +# Log file paths - the wrapper script creates these via Start-Transcript and Tee-Object +$logFile = Join-Path $AgentPath "agent.log" # PowerShell transcript +$outputLogFile = Join-Path $AgentPath "agent-output.log" # Command output + +# Validate bearer token +if ([string]::IsNullOrEmpty($BearerToken)) { + Write-Host " ERROR: Cannot start agent - BEARER_TOKEN is empty!" -ForegroundColor Red + throw "BEARER_TOKEN is required" +} +Write-Host " BEARER_TOKEN length: $($BearerToken.Length)" -ForegroundColor Green + +# Create PowerShell wrapper script in temp directory (not in repo working directory) +# This prevents plaintext secrets from leaking if workspace is reused or artifacts are collected +$tempDir = [System.IO.Path]::GetTempPath() +$wrapperScript = Join-Path $tempDir "run-agent-$([Guid]::NewGuid().ToString('N').Substring(0,8)).ps1" + +# Build wrapper script with error handling and diagnostics +# Key change: The wrapper script handles its own logging via Start-Transcript +$scriptLines = @( + "`$ErrorActionPreference = 'Continue'" + "" + "# Set up logging via transcript (writes to agent directory)" + "`$logPath = Join-Path '$AgentPath' 'agent.log'" + "Start-Transcript -Path `$logPath -Force" + "" + "Write-Host '=== Agent Wrapper Script Started ==='" + "Write-Host 'Working Directory:' (Get-Location)" + "Write-Host 'PowerShell Version:' `$PSVersionTable.PSVersion" + "Write-Host 'Log file:' `$logPath" + "" + "# Set environment variables (token passed via env var, not in script)" + "`$env:PORT = '$Port'" + "`$env:ASPNETCORE_URLS = 'http://localhost:$Port'" + "`$env:PYTHONIOENCODING = 'utf-8'" + "`$env:PYTHONUNBUFFERED = '1'" + "`$env:ENVIRONMENT = '$Environment'" + "`$env:NODE_ENV = 'development'" + "" + "Write-Host 'Environment configured:'" + "Write-Host ' PORT=' `$env:PORT" + "Write-Host ' BEARER_TOKEN length:' `$env:BEARER_TOKEN.Length" + "Write-Host ' ENVIRONMENT:' `$env:ENVIRONMENT" + "" + "Write-Host '=== Pre-flight Checks ==='" +) + +# Add runtime-specific pre-flight checks +if ($Runtime -eq "python") { + $scriptLines += @( + "Write-Host 'Checking Python environment...'" + "uv run python --version" + "Write-Host 'Python packages:'" + "uv run pip list | Select-Object -First 20" + "Write-Host ''" + "Write-Host 'Testing Python can import main script...'" + "uv run python -c `"import start_with_generic_host; print('Import OK')`"" + "if (`$LASTEXITCODE -ne 0) {" + " Write-Host 'ERROR: Failed to import start_with_generic_host.py' -ForegroundColor Red" + " exit 1" + "}" + ) +} +elseif ($Runtime -eq "nodejs") { + $scriptLines += @( + "Write-Host 'Checking Node.js environment...'" + "node --version" + "Write-Host 'Checking if entry point exists...'" + "if (Test-Path 'dist/index.js') { Write-Host 'dist/index.js exists' } else { Write-Host 'ERROR: dist/index.js not found!' -ForegroundColor Red; exit 1 }" + ) +} +elseif ($Runtime -eq "dotnet") { + $scriptLines += @( + "Write-Host 'Checking .NET environment...'" + "dotnet --version" + ) +} + +$scriptLines += @( + "" + "Write-Host '=== Starting Agent Command ==='" + "Write-Host 'Command: $StartCommand'" + "Write-Host ''" + "" + "# Run the command - this should be a long-running server process" + "# Pipe both stdout and stderr to Tee-Object to capture in log" + "try {" + " $StartCommand *>&1 | Tee-Object -FilePath (Join-Path (Get-Location) 'agent-output.log')" + "} catch {" + " Write-Host 'Exception running command:' `$_.Exception.Message -ForegroundColor Red" + "}" + "" + "# If we get here, the server exited" + "`$exitCode = `$LASTEXITCODE" + "Write-Host ''" + "Write-Host '=== Agent Process Exited ===' -ForegroundColor Yellow" + "Write-Host 'Exit code:' `$exitCode" + "if (`$exitCode -ne 0) {" + " Write-Host 'ERROR: Server exited with non-zero code' -ForegroundColor Red" + "}" + "Stop-Transcript -ErrorAction SilentlyContinue" + "exit `$exitCode" +) +$scriptContent = $scriptLines -join "`r`n" +[System.IO.File]::WriteAllText($wrapperScript, $scriptContent) + +Write-Host "PowerShell wrapper script created at: $wrapperScript" -ForegroundColor Gray + +# Start the agent process +# Set BEARER_TOKEN in current process environment so child inherits it +# This avoids writing the token to the script file on disk +$env:BEARER_TOKEN = $BearerToken +Push-Location $AgentPath +try { + # Start the wrapper script directly without output redirection + # The wrapper uses Start-Transcript and Tee-Object for logging + # BEARER_TOKEN is inherited from parent process environment + $process = Start-Process -FilePath "pwsh" ` + -ArgumentList "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $wrapperScript ` + -WorkingDirectory $AgentPath -PassThru -WindowStyle Hidden + + Write-Host "Agent process started (PID: $($process.Id))" -ForegroundColor Green + + # Delete the wrapper script now that process has started (token not in file anyway) + Start-Sleep -Seconds 1 + if (Test-Path $wrapperScript) { + Remove-Item $wrapperScript -Force -ErrorAction SilentlyContinue + Write-Host "Wrapper script cleaned up" -ForegroundColor Gray + } + + # Give the process a moment to start and initialize transcript + Start-Sleep -Seconds 2 + + # Wait for agent to be ready + $elapsed = 2 + $ready = $false + + $healthUrl = "http://localhost:$Port$HealthEndpoint" + $messagesUrl = "http://localhost:$Port/api/messages" + + Write-Host "Waiting for agent to be ready (timeout: $TimeoutSeconds seconds)..." -ForegroundColor Gray + Write-Host " Checking: $healthUrl and $messagesUrl" -ForegroundColor Gray + + $healthError = "" + + while ($elapsed -lt $TimeoutSeconds) { + # Check if process died + if ($process.HasExited) { + Write-Host "Agent process exited prematurely!" -ForegroundColor Red + Write-Host "Exit code: $($process.ExitCode)" -ForegroundColor Red + + # Wait a moment for file handles to be released + Start-Sleep -Milliseconds 500 + + # Show transcript log + if (Test-Path $logFile) { + $logContent = Get-Content $logFile -Raw -ErrorAction SilentlyContinue + if ($logContent) { + Write-Host "Agent transcript log:" -ForegroundColor Yellow + Write-Host $logContent + } else { + Write-Host "Agent transcript file exists but is empty" -ForegroundColor Yellow + } + } else { + Write-Host "No agent transcript file found at: $logFile" -ForegroundColor Yellow + } + + # Show command output log + if (Test-Path $outputLogFile) { + $outputContent = Get-Content $outputLogFile -Raw -ErrorAction SilentlyContinue + if ($outputContent) { + Write-Host "Agent command output:" -ForegroundColor Yellow + Write-Host $outputContent + } + } + + throw "Agent process exited with code $($process.ExitCode)" + } + + # Try health endpoint first + try { + $response = Invoke-WebRequest -Uri $healthUrl -Method GET -UseBasicParsing -TimeoutSec 5 -ErrorAction Stop + if ($response.StatusCode -eq 200) { + $ready = $true + Write-Host "Agent is ready! (health endpoint returned 200)" -ForegroundColor Green + break + } + } + catch { + $healthError = $_.Exception.Message + if ($elapsed % 15 -eq 0) { + Write-Host " Health check error: $healthError" -ForegroundColor DarkYellow + } + # Try messages endpoint - 405 means it's running + try { + $response = Invoke-WebRequest -Uri $messagesUrl -Method GET -UseBasicParsing -TimeoutSec 5 -ErrorAction Stop + } + catch { + $exResponse = $_.Exception.Response + if ($exResponse -and $exResponse.StatusCode -eq 405) { + $ready = $true + Write-Host "Agent is ready! (messages endpoint returned 405)" -ForegroundColor Green + break + } + } + } + + Start-Sleep -Seconds 3 + $elapsed += 3 + Write-Host " Waiting... ($elapsed/$TimeoutSeconds seconds)" -ForegroundColor Gray + + # Show recent log output every 15 seconds + if ($elapsed % 15 -eq 0 -and (Test-Path $logFile)) { + Write-Host " Recent agent output:" -ForegroundColor Gray + Get-Content $logFile -Tail 5 | ForEach-Object { Write-Host " $_" -ForegroundColor DarkGray } + } + } + + if (-not $ready) { + Write-Host "`nAgent did not become ready within $TimeoutSeconds seconds" -ForegroundColor Red + Write-Host "Last health check error: $healthError" -ForegroundColor Red + + # Diagnostics + Write-Host "`nFinal diagnostic check:" -ForegroundColor Yellow + try { + $diag = Invoke-WebRequest -Uri $healthUrl -Method GET -UseBasicParsing -TimeoutSec 10 -ErrorAction Stop + Write-Host " Unexpected success: StatusCode=$($diag.StatusCode)" -ForegroundColor Green + } + catch { + Write-Host " Error: $($_.Exception.Message)" -ForegroundColor Red + } + + # Check port + $netstat = netstat -ano | Select-String ":$Port" + if ($netstat) { + Write-Host "`nPort $Port listeners:" -ForegroundColor Yellow + $netstat | ForEach-Object { Write-Host " $_" -ForegroundColor Gray } + } + else { + Write-Host "`nNo process listening on port $Port!" -ForegroundColor Red + } + + if (Test-Path $logFile) { + Write-Host "`nFull agent logs:" -ForegroundColor Yellow + Get-Content $logFile + } + + throw "Agent did not become ready within $TimeoutSeconds seconds" + } + + # Verify process is still running before returning + # Wait a bit longer to ensure the agent is stable + Write-Host "Verifying agent stability..." -ForegroundColor Gray + for ($i = 1; $i -le 5; $i++) { + Start-Sleep -Seconds 1 + if ($process.HasExited) { + Write-Host "WARNING: Agent process exited during stability check (iteration $i)!" -ForegroundColor Red + Write-Host "Exit code: $($process.ExitCode)" -ForegroundColor Red + + Start-Sleep -Milliseconds 500 + if (Test-Path $logFile) { + Write-Host "Agent transcript log:" -ForegroundColor Yellow + Get-Content $logFile + } + if (Test-Path $outputLogFile) { + $outContent = Get-Content $outputLogFile -Raw + if ($outContent) { + Write-Host "Agent command output:" -ForegroundColor Yellow + Write-Host $outContent + } + } + throw "Agent process exited unexpectedly after health check passed" + } + + # Also verify health endpoint still works + try { + $check = Invoke-WebRequest -Uri $healthUrl -Method GET -UseBasicParsing -TimeoutSec 3 -ErrorAction Stop + Write-Host " Stability check $i/5: OK (health: $($check.StatusCode))" -ForegroundColor Gray + } + catch { + Write-Host " Stability check $i/5: Health check failed - $($_.Exception.Message)" -ForegroundColor Yellow + } + } + + Write-Host "Agent process (PID: $($process.Id)) is running and stable" -ForegroundColor Green + + # Return the process ID + return $process.Id +} +finally { + Pop-Location +} diff --git a/scripts/e2e/Stop-AgentProcess.ps1 b/scripts/e2e/Stop-AgentProcess.ps1 new file mode 100644 index 00000000..f34b35f6 --- /dev/null +++ b/scripts/e2e/Stop-AgentProcess.ps1 @@ -0,0 +1,45 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Stops the agent process and cleans up. + +.DESCRIPTION + Stops the agent process by PID and/or by killing any process on the specified port. + +.PARAMETER AgentPID + The process ID of the agent to stop. + +.PARAMETER Port + The port to clean up. Any process listening on this port will be terminated. + +.EXAMPLE + ./Stop-AgentProcess.ps1 -AgentPID 1234 -Port 3979 +#> + +param( + [Parameter(Mandatory = $false)] + [string]$AgentPID, + + [Parameter(Mandatory = $false)] + [int]$Port = 3979 +) + +Write-Host "Cleaning up agent process..." -ForegroundColor Yellow + +# Stop the agent process if PID was provided +if ($AgentPID -and $AgentPID -match '^\d+$') { + Write-Host "Stopping agent process (PID: $AgentPID)..." -ForegroundColor Gray + Stop-Process -Id $AgentPID -Force -ErrorAction SilentlyContinue +} + +# Kill any process on the port as backup +$connections = Get-NetTCPConnection -LocalPort $Port -ErrorAction SilentlyContinue +if ($connections) { + Write-Host "Cleaning up processes on port $Port..." -ForegroundColor Gray + $connections | Select-Object -ExpandProperty OwningProcess | + ForEach-Object { Stop-Process -Id $_ -Force -ErrorAction SilentlyContinue } +} + +Write-Host "Cleanup complete" -ForegroundColor Green diff --git a/tests/e2e/Agent365.E2E.Tests.csproj b/tests/e2e/Agent365.E2E.Tests.csproj new file mode 100644 index 00000000..0614ce56 --- /dev/null +++ b/tests/e2e/Agent365.E2E.Tests.csproj @@ -0,0 +1,27 @@ + + + + net8.0 + enable + enable + false + true + Agent365.E2E.Tests + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + diff --git a/tests/e2e/HttpIntegrationTests.cs b/tests/e2e/HttpIntegrationTests.cs new file mode 100644 index 00000000..f449704b --- /dev/null +++ b/tests/e2e/HttpIntegrationTests.cs @@ -0,0 +1,595 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Net; +using System.Text; +using System.Text.Json; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +namespace Agent365.E2E.Tests; + +/// +/// HTTP integration tests for local agent testing. +/// Tests basic agent connectivity and message processing. +/// +/// NOTE: This test verifies the agent processes messages but may not capture +/// responses when running locally without Bot Framework infrastructure. +/// Uses MockBotFrameworkServer to capture agent responses. +/// +/// For full request/response testing: +/// - Use Bot Framework Emulator (connects via Direct Line) +/// - Deploy to Azure and test via Agent 365 Playground +/// +public class HttpIntegrationTests : IAsyncLifetime +{ + private readonly ITestOutputHelper _output; + private readonly LocalAgentTestClient _client; + private readonly string _agentUrl; + + public HttpIntegrationTests(ITestOutputHelper output) + { + _output = output; + _agentUrl = Environment.GetEnvironmentVariable("AGENT_URL") ?? "http://localhost:3978"; + _client = new LocalAgentTestClient(_agentUrl, mockPort: 3980); + } + + public async Task InitializeAsync() + { + await _client.StartMockServerAsync(); + _output.WriteLine($"Mock server started on {_client.MockServerServiceUrl}"); + } + + public async Task DisposeAsync() + { + await _client.StopMockServerAsync(); + _output.WriteLine("Mock server stopped"); + } + + [Fact] + [Trait("Category", "HTTP")] + public async Task TestAgentHealth_WhenCalledOnMessagesEndpoint_AgentIsAccessible() + { + // Arrange & Act + _output.WriteLine("Checking if agent is accessible..."); + var startTime = DateTime.UtcNow; + bool passed = false; + string? errorMessage = null; + string? responseInfo = null; + + using var httpClient = new HttpClient { Timeout = TimeSpan.FromSeconds(3) }; + + try + { + var response = await httpClient.GetAsync($"{_agentUrl}/api/messages"); + + // Assert - We expect 405 (Method Not Allowed) for GET on messages endpoint + // This means the agent is running + response.StatusCode.Should().BeOneOf( + HttpStatusCode.MethodNotAllowed, + HttpStatusCode.OK, + HttpStatusCode.NotFound, + HttpStatusCode.Unauthorized + ); + + responseInfo = $"Agent is running (Status: {response.StatusCode})"; + _output.WriteLine($"[OK] Agent is running on {_agentUrl} (Status: {response.StatusCode})"); + passed = true; + } + catch (HttpRequestException ex) + { + errorMessage = $"Cannot connect to agent on {_agentUrl}. Error: {ex.Message}"; + Assert.Fail($"Cannot connect to agent on {_agentUrl}. Make sure agent is running. Error: {ex.Message}"); + } + catch (TaskCanceledException) + { + responseInfo = "Agent responding slowly, but may be running"; + _output.WriteLine("[WARNING] Agent responding slowly, but may be running"); + passed = true; // Still consider it a pass if it times out but doesn't error + } + finally + { + // Record the health check for reporting + TestResultsCollector.RecordConversation( + testName: "TestAgentHealth", + userMessage: $"GET {_agentUrl}/api/messages", + agentResponse: responseInfo ?? "(no response)", + passed: passed, + errorMessage: errorMessage, + duration: DateTime.UtcNow - startTime); + } + } + + [Fact] + [Trait("Category", "HTTP")] + public async Task TestAgentHttpEndpoint_WhenSendingMessage_ProcessesSuccessfully() + { + // Arrange + _output.WriteLine("Testing agent message processing..."); + _output.WriteLine($"Mock server on {_client.MockServerServiceUrl} to capture responses"); + + var conversationId = Guid.NewGuid().ToString(); + var userMessage = "Hello! Can you tell me a joke?"; + var startTime = DateTime.UtcNow; + string? responseText = null; + string? errorMessage = null; + bool passed = false; + + try + { + // Act + _output.WriteLine("\nSending test message..."); + var result = await _client.SendMessageAsync(userMessage, conversationId); + + _output.WriteLine($"Initial status: {result.Status}"); + + // Assert - Agent must at least accept the message + result.Status.Should().BeOneOf("accepted", "success", "processed_no_channel", + $"Agent not responding: {result.Message}. Make sure agent is running."); + + _output.WriteLine("[OK] Agent accepted the message, waiting for response..."); + + // Wait for actual response from mock server + _output.WriteLine("\nWaiting for response via mock server (timeout: 30s)..."); + var responses = await _client.GetResponsesAsync(conversationId, timeout: TimeSpan.FromSeconds(30)); + + // Validate we got a response + responses.Should().NotBeEmpty( + "No response received from agent via mock server. Agent may have failed to process the message."); + + _output.WriteLine($"[OK] Received {responses.Count} response(s) from agent"); + + // Find a response with actual text content + foreach (var resp in responses) + { + var text = MockBotFrameworkServer.GetResponseText(resp); + if (!string.IsNullOrEmpty(text)) + { + responseText = text; + var preview = text.Length > 200 ? text[..200] + "..." : text; + _output.WriteLine($"\nResponse: {preview}"); + break; + } + } + + // Validate response has content + responseText.Should().NotBeNullOrEmpty( + $"Response received but text is empty. Raw responses: {JsonSerializer.Serialize(responses)}"); + + _output.WriteLine("\n[PASSED] Agent processed message and returned valid response"); + passed = true; + } + catch (Exception ex) + { + errorMessage = ex.Message; + throw; + } + finally + { + // Record the conversation for reporting + TestResultsCollector.RecordConversation( + testName: "TestAgentHttpEndpoint", + userMessage: userMessage, + agentResponse: responseText, + passed: passed, + errorMessage: errorMessage, + duration: DateTime.UtcNow - startTime); + } + } + + [Fact] + [Trait("Category", "HTTP")] + public async Task TestMultipleMessages_WhenSendingSequentialMessages_AllProcessedSuccessfully() + { + // Arrange + _output.WriteLine("Testing multiple message processing..."); + _output.WriteLine("Starting mock server to capture responses..."); + + var conversationId = Guid.NewGuid().ToString(); + var messages = new[] { "Hello!", "Can you help me?", "Tell me a joke" }; + var turns = new List<(string UserMessage, string? AgentResponse)>(); + var startTime = DateTime.UtcNow; + string? errorMessage = null; + bool passed = false; + + try + { + // Act & Assert + foreach (var msg in messages) + { + _output.WriteLine($"\nSending: {msg}"); + + // Clear previous responses + _client.ClearResponses(conversationId); + + // Send message + var result = await _client.SendMessageAsync(msg, conversationId); + _output.WriteLine($"Initial status: {result.Status}"); + + // First check - agent must accept the message + result.Status.Should().BeOneOf("accepted", "success", "processed_no_channel", + $"Agent not responding: {result.Message}. Make sure agent is running on port 3979"); + + // Wait for actual response from mock server + _output.WriteLine(" Waiting for response (timeout: 30s)..."); + var responses = await _client.GetResponsesAsync(conversationId, timeout: TimeSpan.FromSeconds(30)); + + // Validate we got a response + responses.Should().NotBeEmpty( + $"No response received for message '{msg}'. Agent may have failed to process."); + + // Find response with text content + string? responseText = null; + foreach (var resp in responses) + { + var text = MockBotFrameworkServer.GetResponseText(resp); + if (!string.IsNullOrEmpty(text)) + { + responseText = text; + break; + } + } + + responseText.Should().NotBeNullOrEmpty( + $"Response for '{msg}' has no text content. Raw: {JsonSerializer.Serialize(responses)}"); + + // Record this turn + turns.Add((msg, responseText)); + + var preview = responseText!.Length > 100 ? responseText[..100] + "..." : responseText; + _output.WriteLine($" [OK] Response: {preview}"); + + await Task.Delay(500); + } + + _output.WriteLine($"\n[PASSED] All {messages.Length}/{messages.Length} messages processed with valid responses"); + passed = true; + } + catch (Exception ex) + { + errorMessage = ex.Message; + throw; + } + finally + { + // Record the multi-turn conversation for reporting + TestResultsCollector.RecordMultiTurnConversation( + testName: "TestMultipleMessages", + turns: turns, + passed: passed, + errorMessage: errorMessage, + duration: DateTime.UtcNow - startTime); + } + } + + [Fact] + [Trait("Category", "HTTP")] + public async Task TestMcpEmailTools_WhenAskingForTools_ReturnsEmailCapabilities() + { + // Arrange + _output.WriteLine("TEST: MCP Email Tools Configuration"); + _output.WriteLine(new string('-', 60)); + + var conversationId = Guid.NewGuid().ToString(); + var userMessage = "What tools and MCP servers do you have configured? List all available email tools."; + var startTime = DateTime.UtcNow; + string? responseText = null; + string? errorMessage = null; + bool passed = false; + + try + { + // Act - Ask agent to list its tools and MCP servers + _output.WriteLine("\n[Query] What tools and MCP servers do you have configured?"); + + var result = await _client.SendMessageAsync(userMessage, conversationId); + + // Assert - First check if agent accepted the message + result.Should().NotBeNull("Failed to send message to agent"); + result.Status.Should().BeOneOf("accepted", "success", "processed_no_channel", + $"Agent not responding: {result.Message}. Make sure agent is running on port 3979"); + + _output.WriteLine("[OK] Agent accepted tool listing query, waiting for response..."); + + // Wait for response from agent (longer timeout for tool listing) + _output.WriteLine("[Waiting] Allowing up to 30 seconds for agent to list tools..."); + var responses = await _client.GetResponsesAsync(conversationId, timeout: TimeSpan.FromSeconds(30)); + + // Validate we got a response + responses.Should().NotBeEmpty( + "No response received from agent via mock server"); + + // Get the actual response text + for (int idx = 0; idx < responses.Count; idx++) + { + var text = MockBotFrameworkServer.GetResponseText(responses[idx]); + if (!string.IsNullOrEmpty(text)) + { + responseText = text; + _output.WriteLine($"[OK] Received response from agent (response #{idx + 1})"); + break; + } + } + + responseText.Should().NotBeNullOrEmpty( + $"Response received but all text fields are empty. Raw responses: {JsonSerializer.Serialize(responses)}"); + + var responseTextLower = responseText!.ToLowerInvariant(); + var preview = responseText.Length > 500 ? responseText[..500] + "..." : responseText; + _output.WriteLine($"\n[Response Preview]\n{new string('-', 60)}\n{preview}\n{new string('-', 60)}"); + + // Expected email tool capabilities from MCP Mail server + var expectedTools = new Dictionary + { + { "draft", new[] { "draft", "create draft" } }, + { "send", new[] { "send email", "send directly" } }, + { "search", new[] { "search email", "search" } }, + { "reply", new[] { "reply" } }, + { "forward", new[] { "forward" } }, + { "attachment", new[] { "attachment", "attach" } }, + { "update", new[] { "update email" } }, + { "delete", new[] { "delete email" } } + }; + + var foundTools = expectedTools + .Where(kvp => kvp.Value.Any(keyword => responseTextLower.Contains(keyword))) + .Select(kvp => kvp.Key) + .ToList(); + + _output.WriteLine($"\n[Analysis] Found {foundTools.Count}/{expectedTools.Count} expected email tool categories:"); + foreach (var tool in foundTools) + { + _output.WriteLine($" [OK] {tool}"); + } + + // Check for negative indicators + var negativeIndicators = new[] + { + "i don't have any tools", + "no tools configured", + "i don't have access to", + "i cannot", + "i'm unable to" + }; + var noTools = negativeIndicators.Any(phrase => responseTextLower.Contains(phrase)); + + noTools.Should().BeFalse("Agent indicates it has no tools or cannot access MCP servers"); + + foundTools.Should().NotBeEmpty( + $"No email tool categories found in response. Expected at least some of: {string.Join(", ", expectedTools.Keys)}"); + + if (foundTools.Count >= 3) + { + _output.WriteLine($"\n[PASSED] Agent has MCP Mail tools configured"); + _output.WriteLine($" Found tools: {string.Join(", ", foundTools)}"); + } + else + { + _output.WriteLine($"\n[WARNING] Only found {foundTools.Count} email tool categories, but test passes"); + _output.WriteLine($" Found tools: {string.Join(", ", foundTools)}"); + } + passed = true; + } + catch (Exception ex) + { + errorMessage = ex.Message; + throw; + } + finally + { + // Record the conversation for reporting + TestResultsCollector.RecordConversation( + testName: "TestMcpEmailTools", + userMessage: userMessage, + agentResponse: responseText, + passed: passed, + errorMessage: errorMessage, + duration: DateTime.UtcNow - startTime); + } + } + + [Fact] + [Trait("Category", "HTTP")] + public async Task TestEmptyPayload_WhenSendingEmptyMessage_AgentHandlesGracefully() + { + // Arrange + var conversationId = Guid.NewGuid().ToString(); + var startTime = DateTime.UtcNow; + bool passed = false; + string? errorMessage = null; + + try + { + // Act + var result = await _client.SendMessageAsync("", conversationId); + + // Assert - Agent should either process or return an error gracefully + _output.WriteLine($"Status: {result.Status}"); + + // The agent should not crash - any valid status is acceptable + result.Status.Should().NotBeNull(); + passed = true; + } + catch (Exception ex) + { + errorMessage = ex.Message; + throw; + } + finally + { + // Record the conversation for reporting + TestResultsCollector.RecordConversation( + testName: "TestEmptyPayload", + userMessage: "(empty message)", + agentResponse: "(graceful handling verified)", + passed: passed, + errorMessage: errorMessage, + duration: DateTime.UtcNow - startTime); + } + } +} + +/// +/// Test client that simulates Bot Framework Emulator behavior. +/// Uses MockBotFrameworkServer to capture agent responses. +/// +public class LocalAgentTestClient +{ + private readonly string _agentUrl; + private readonly MockBotFrameworkServer _mockServer; + private readonly HttpClient _httpClient; + + public string MockServerServiceUrl => _mockServer.ServiceUrl; + + public LocalAgentTestClient(string agentUrl = "http://localhost:3979", int mockPort = 3980) + { + _agentUrl = agentUrl; + _mockServer = new MockBotFrameworkServer(port: mockPort); + _httpClient = new HttpClient { Timeout = TimeSpan.FromSeconds(30) }; + } + + public Task StartMockServerAsync() => _mockServer.StartAsync(); + + public Task StopMockServerAsync() => _mockServer.StopAsync(); + + public Task> GetResponsesAsync( + string conversationId, TimeSpan? timeout = null) + { + return _mockServer.WaitForResponsesAsync(conversationId, timeout) + .ContinueWith(t => t.Result.Select(d => + JsonSerializer.Deserialize(JsonSerializer.Serialize(d))).ToList()); + } + + public async Task>> GetResponsesAsync( + string conversationId, + TimeSpan timeout) + { + return await _mockServer.WaitForResponsesAsync(conversationId, timeout); + } + + public void ClearResponses(string conversationId) => _mockServer.ClearResponses(conversationId); + + /// + /// Send message to agent and capture response. + /// This simulates what Bot Framework Emulator does. + /// + public async Task SendMessageAsync( + string text, + string? conversationId = null, + string userId = "test-user", + string userName = "Test User") + { + // Create conversation in mock server + conversationId ??= Guid.NewGuid().ToString(); + _mockServer.CreateConversation(conversationId); + + // Create activity + var activity = new + { + type = "message", + id = Guid.NewGuid().ToString(), + timestamp = DateTime.UtcNow.ToString("o"), + serviceUrl = _mockServer.ServiceUrl, + channelId = "emulator", + from = new + { + id = userId, + name = userName + }, + conversation = new + { + id = conversationId + }, + recipient = new + { + id = "bot", + name = "Agent" + }, + text + }; + + var json = JsonSerializer.Serialize(activity); + var content = new StringContent(json, Encoding.UTF8, "application/json"); + + try + { + var response = await _httpClient.PostAsync($"{_agentUrl}/api/messages", content); + + if (response.StatusCode == HttpStatusCode.Accepted) + { + // Agent accepted message (standard Bot Framework response) + return new SendMessageResult + { + Status = "accepted", + ConversationId = conversationId, + Message = "Message processed. Response would be sent via Bot Framework channel." + }; + } + else if (response.StatusCode == HttpStatusCode.OK) + { + // Some agents return 200 with response + var responseData = await response.Content.ReadAsStringAsync(); + return new SendMessageResult + { + Status = "success", + ConversationId = conversationId, + ResponseData = responseData + }; + } + else if (response.StatusCode == HttpStatusCode.InternalServerError) + { + // 500 is expected when agent processes message but can't send response + return new SendMessageResult + { + Status = "processed_no_channel", + ConversationId = conversationId, + Message = "Agent processed message and generated response, but couldn't send it back (no Bot Framework channel). This is EXPECTED in local testing without Emulator/Direct Line." + }; + } + else + { + var errorText = await response.Content.ReadAsStringAsync(); + return new SendMessageResult + { + Status = "error", + StatusCode = (int)response.StatusCode, + Message = errorText + }; + } + } + catch (TaskCanceledException) + { + return new SendMessageResult + { + Status = "timeout", + Message = "Agent did not respond in time" + }; + } + catch (HttpRequestException ex) + { + return new SendMessageResult + { + Status = "error", + Message = $"Cannot connect to host {_agentUrl} - Agent may not be running. Error: {ex.Message}" + }; + } + catch (Exception ex) + { + return new SendMessageResult + { + Status = "error", + Message = ex.Message + }; + } + } +} + +public class SendMessageResult +{ + public string Status { get; set; } = string.Empty; + public string? ConversationId { get; set; } + public string? Message { get; set; } + public string? ResponseData { get; set; } + public int? StatusCode { get; set; } +} diff --git a/tests/e2e/MockBotFrameworkServer.cs b/tests/e2e/MockBotFrameworkServer.cs new file mode 100644 index 00000000..4c0d468c --- /dev/null +++ b/tests/e2e/MockBotFrameworkServer.cs @@ -0,0 +1,335 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Net; +using System.Text; +using System.Text.Json; + +namespace Agent365.E2E.Tests; + +/// +/// Mock Bot Framework Connector server that captures agent responses. +/// +/// When an agent receives a message via /api/messages, it sends responses +/// back via the Bot Framework Connector API (POST to serviceUrl). +/// This mock server captures those responses for testing. +/// +/// How it works: +/// 1. Start mock server on a local port (e.g., http://localhost:3980) +/// 2. Send message to agent with serviceUrl pointing to mock server +/// 3. Agent processes message and calls mock server to send response +/// 4. Mock server captures response for test validation +/// +public class MockBotFrameworkServer : IAsyncDisposable +{ + private readonly int _port; + private HttpListener? _listener; + private CancellationTokenSource? _cts; + private Task? _listenTask; + + // Store responses by conversation ID + private readonly Dictionary>> _responses = new(); + private readonly object _lock = new(); + + public string ServiceUrl => $"http://localhost:{_port}"; + + public MockBotFrameworkServer(int port = 3980) + { + _port = port; + } + + /// + /// Start the mock server to capture agent responses. + /// + public async Task StartAsync() + { + _listener = new HttpListener(); + _listener.Prefixes.Add($"http://localhost:{_port}/"); + _cts = new CancellationTokenSource(); + + try + { + _listener.Start(); + _listenTask = ListenAsync(_cts.Token); + + // Give it a moment to start + await Task.Delay(100); + } + catch (HttpListenerException ex) when (ex.ErrorCode == 5) + { + // Access denied - port may be in use or need admin rights + throw new InvalidOperationException( + $"Cannot start mock server on port {_port}. Port may be in use or requires admin rights.", ex); + } + } + + /// + /// Stop the mock server. + /// + public async Task StopAsync() + { + if (_cts != null) + { + _cts.Cancel(); + + if (_listenTask != null) + { + try + { + await _listenTask; + } + catch (OperationCanceledException) + { + // Expected + } + } + + _cts.Dispose(); + _cts = null; + } + + if (_listener != null) + { + _listener.Stop(); + _listener.Close(); + _listener = null; + } + } + + public async ValueTask DisposeAsync() + { + await StopAsync(); + } + + /// + /// Create a conversation to track responses. + /// + public void CreateConversation(string conversationId) + { + lock (_lock) + { + _responses[conversationId] = new List>(); + } + } + + /// + /// Clear responses for a conversation. + /// + public void ClearResponses(string conversationId) + { + lock (_lock) + { + if (_responses.TryGetValue(conversationId, out var list)) + { + list.Clear(); + } + } + } + + /// + /// Wait for responses from the agent. + /// + public async Task>> WaitForResponsesAsync( + string conversationId, + TimeSpan? timeout = null) + { + var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(10)); + + while (DateTime.UtcNow < deadline) + { + lock (_lock) + { + if (_responses.TryGetValue(conversationId, out var responses) && responses.Count > 0) + { + return new List>(responses); + } + } + + await Task.Delay(100); + } + + // Return empty list if no responses + lock (_lock) + { + if (_responses.TryGetValue(conversationId, out var responses)) + { + return new List>(responses); + } + } + + return new List>(); + } + + /// + /// Extract response text from activity JSON. + /// + public static string? GetResponseText(Dictionary activity) + { + if (activity.TryGetValue("text", out var textElement)) + { + return textElement.GetString(); + } + return null; + } + + /// + /// Extract response text from JsonElement activity. + /// + public static string? GetResponseText(JsonElement activity) + { + if (activity.TryGetProperty("text", out var textElement)) + { + return textElement.GetString(); + } + return null; + } + + private async Task ListenAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested && _listener?.IsListening == true) + { + try + { + var contextTask = _listener.GetContextAsync(); + + // Wait with cancellation support + using var registration = cancellationToken.Register(() => + { + try { _listener?.Stop(); } catch { } + }); + + HttpListenerContext context; + try + { + context = await contextTask; + } + catch (HttpListenerException) + { + break; + } + catch (ObjectDisposedException) + { + break; + } + + // Process request in background + _ = ProcessRequestAsync(context); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception) + { + // Log error but continue listening + } + } + } + + private async Task ProcessRequestAsync(HttpListenerContext context) + { + var request = context.Request; + var response = context.Response; + + try + { + // Bot Framework Connector API endpoints + var path = request.Url?.AbsolutePath ?? ""; + + // POST /v3/conversations/{conversationId}/activities + if (request.HttpMethod == "POST" && path.Contains("/v3/conversations/") && path.Contains("/activities")) + { + await HandleSendActivityAsync(request, response); + return; + } + + // POST /v3/conversations (create conversation) + if (request.HttpMethod == "POST" && path == "/v3/conversations") + { + await HandleCreateConversationAsync(response); + return; + } + + // Default response + response.StatusCode = 200; + var bytes = Encoding.UTF8.GetBytes("{\"status\":\"ok\"}"); + await response.OutputStream.WriteAsync(bytes); + } + catch (Exception) + { + response.StatusCode = 500; + } + finally + { + response.Close(); + } + } + + private async Task HandleSendActivityAsync(HttpListenerRequest request, HttpListenerResponse response) + { + // Read activity from request body + using var reader = new StreamReader(request.InputStream); + var body = await reader.ReadToEndAsync(); + + try + { + var activity = JsonSerializer.Deserialize>(body); + + if (activity != null) + { + // Extract conversation ID from path or activity + string? conversationId = null; + + var path = request.Url?.AbsolutePath ?? ""; + var match = System.Text.RegularExpressions.Regex.Match( + path, @"/v3/conversations/([^/]+)/activities"); + if (match.Success) + { + conversationId = match.Groups[1].Value; + } + else if (activity.TryGetValue("conversation", out var conv) && + conv.TryGetProperty("id", out var convId)) + { + conversationId = convId.GetString(); + } + + if (!string.IsNullOrEmpty(conversationId)) + { + lock (_lock) + { + if (!_responses.ContainsKey(conversationId)) + { + _responses[conversationId] = new List>(); + } + _responses[conversationId].Add(activity); + } + } + } + + // Return success (Bot Framework expects 200 or 201) + response.StatusCode = 200; + response.ContentType = "application/json"; + var result = new { id = Guid.NewGuid().ToString() }; + var bytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(result)); + await response.OutputStream.WriteAsync(bytes); + } + catch (JsonException) + { + response.StatusCode = 400; + } + } + + private async Task HandleCreateConversationAsync(HttpListenerResponse response) + { + // Return a fake conversation ID + response.StatusCode = 200; + response.ContentType = "application/json"; + var result = new + { + id = Guid.NewGuid().ToString(), + serviceUrl = ServiceUrl + }; + var bytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(result)); + await response.OutputStream.WriteAsync(bytes); + } +} diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 00000000..10cbba39 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,91 @@ +# E2E Integration Tests + +This directory contains end-to-end integration tests for Agent365 samples. + +## Test Project + +- **Framework**: .NET 8.0 +- **Test Framework**: xUnit 2.9.0 +- **Assertions**: FluentAssertions 6.12.0 + +## Test Classes + +### HttpIntegrationTests + +HTTP-based tests that verify agent connectivity and message processing. + +| Test | Description | +|------|-------------| +| `TestAgentHealth` | Verifies agent is running and accessible | +| `TestAgentHttpEndpoint` | Tests basic message sending and response | +| `TestMultipleMessages` | Tests sequential multi-turn conversations | +| `TestMcpEmailTools` | Verifies MCP email tools are configured | +| `TestEmptyPayload` | Ensures graceful handling of empty messages | + +All HTTP tests are tagged with `[Trait("Category", "HTTP")]`. + +### MockBotFrameworkServer + +A lightweight HTTP server that simulates the Bot Framework Connector API. This allows tests to capture agent responses without requiring a real Bot Framework connection. + +- Runs on port 3980 by default +- Captures POST requests to `/v3/conversations/{id}/activities` +- Stores responses by conversation ID for test validation + +### TestResultsCollector + +Collects test results and saves them to a JSON file for reporting. + +- Output path: `TEST_RESULTS_DIR` environment variable or `./TestResults` +- Records single and multi-turn conversations +- Tracks pass/fail status and error messages + +## Running Tests Locally + +```bash +# Set environment variables +$env:AGENT_URL = "http://localhost:3979" +$env:TEST_RESULTS_DIR = "./TestResults" + +# Run all HTTP tests +dotnet test --filter "Category=HTTP" + +# Run a specific test +dotnet test --filter "FullyQualifiedName~TestAgentHealth" +``` + +## Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `AGENT_URL` | URL of the running agent | `http://localhost:3978` | +| `AGENT_PORT` | Port of the running agent | `3978` | +| `TEST_RESULTS_DIR` | Directory for test result files | `./TestResults` | + +## Architecture + +``` +┌─────────────┐ POST /api/messages ┌─────────────┐ +│ Test │ ──────────────────────────► │ Agent │ +│ Client │ (serviceUrl=mock:3980) │ │ +└─────────────┘ └──────┬──────┘ + ▲ │ + │ │ POST /v3/conversations/{id}/activities + │ ▼ + │ WaitForResponsesAsync() ┌─────────────┐ + └─────────────────────────────────── │ Mock Server │ + │ (port 3980) │ + └─────────────┘ +``` + +1. Test sends message to agent with `serviceUrl` pointing to mock server +2. Agent processes message and calls mock server to send response +3. Mock server captures response +4. Test retrieves response and validates + +## CI/CD Integration + +These tests are run automatically by the GitHub Actions workflow: +- Workflow: `.github/workflows/e2e-agent-samples.yml` +- Filter: `--filter "Category=HTTP"` +- Results uploaded as artifacts diff --git a/tests/e2e/TestResultsCollector.cs b/tests/e2e/TestResultsCollector.cs new file mode 100644 index 00000000..2054e2ae --- /dev/null +++ b/tests/e2e/TestResultsCollector.cs @@ -0,0 +1,171 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Text.Json; + +namespace Agent365.E2E.Tests; + +/// +/// Collects and saves test results to a JSON file for reporting. +/// This enables integration with CI/CD pipelines and test result analysis. +/// +public static class TestResultsCollector +{ + private static readonly object _lock = new(); + private static readonly List _conversations = new(); + private static bool _initialized = false; + + private static string GetOutputPath() + { + var dir = Environment.GetEnvironmentVariable("TEST_RESULTS_DIR") + ?? Path.Combine(Directory.GetCurrentDirectory(), "TestResults"); + + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + return Path.Combine(dir, "test-conversations.json"); + } + + /// + /// Record a single-turn conversation test result. + /// + public static void RecordConversation( + string testName, + string userMessage, + string? agentResponse, + bool passed, + string? errorMessage = null, + TimeSpan? duration = null) + { + var conversation = new TestConversation + { + TestName = testName, + Timestamp = DateTime.UtcNow, + Duration = duration, + Passed = passed, + ErrorMessage = errorMessage, + Turns = new List + { + new() + { + UserMessage = userMessage, + AgentResponse = agentResponse + } + } + }; + + AddConversation(conversation); + } + + /// + /// Record a multi-turn conversation test result. + /// + public static void RecordMultiTurnConversation( + string testName, + IEnumerable<(string UserMessage, string? AgentResponse)> turns, + bool passed, + string? errorMessage = null, + TimeSpan? duration = null) + { + var conversation = new TestConversation + { + TestName = testName, + Timestamp = DateTime.UtcNow, + Duration = duration, + Passed = passed, + ErrorMessage = errorMessage, + Turns = turns.Select(t => new ConversationTurn + { + UserMessage = t.UserMessage, + AgentResponse = t.AgentResponse + }).ToList() + }; + + AddConversation(conversation); + } + + private static void AddConversation(TestConversation conversation) + { + lock (_lock) + { + // Load existing results on first use + if (!_initialized) + { + LoadExisting(); + _initialized = true; + } + + _conversations.Add(conversation); + SaveToFile(); + } + } + + private static void LoadExisting() + { + var path = GetOutputPath(); + if (File.Exists(path)) + { + try + { + var json = File.ReadAllText(path); + var results = JsonSerializer.Deserialize(json); + if (results?.Conversations != null) + { + _conversations.AddRange(results.Conversations); + } + } + catch + { + // Ignore errors loading existing file + } + } + } + + private static void SaveToFile() + { + var path = GetOutputPath(); + var results = new TestResultsFile + { + GeneratedAt = DateTime.UtcNow, + TotalTests = _conversations.Count, + PassedTests = _conversations.Count(c => c.Passed), + FailedTests = _conversations.Count(c => !c.Passed), + Conversations = _conversations + }; + + var options = new JsonSerializerOptions + { + WriteIndented = true + }; + + var json = JsonSerializer.Serialize(results, options); + File.WriteAllText(path, json); + } +} + +public class TestResultsFile +{ + public DateTime GeneratedAt { get; set; } + public int TotalTests { get; set; } + public int PassedTests { get; set; } + public int FailedTests { get; set; } + public List Conversations { get; set; } = new(); +} + +public class TestConversation +{ + public string TestName { get; set; } = string.Empty; + public DateTime Timestamp { get; set; } + public TimeSpan? Duration { get; set; } + public bool Passed { get; set; } + public string? ErrorMessage { get; set; } + public List Turns { get; set; } = new(); +} + +public class ConversationTurn +{ + public string UserMessage { get; set; } = string.Empty; + public string? AgentResponse { get; set; } +}