diff --git a/CHANGELOG.md b/CHANGELOG.md index e8573d66..81ed4bec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,28 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Upgrade Notes + +#### Existing agents: grant Observability API permissions + +Agents provisioned before this release need `Agent365.Observability.OtelWrite` granted as both a **delegated** and an **application** permission on the blueprint app. Requires Global Administrator. + +**Option A — Entra portal** (no config files required): + +1. [Entra portal](https://entra.microsoft.com) > **App registrations** > select your **Blueprint** app > **API permissions** +2. **Add a permission** > **APIs my organization uses** > search `9b975845-388f-4429-889e-eab1ef63949c` +3. **Delegated permissions** > select `Agent365.Observability.OtelWrite` > **Add permissions** +4. Repeat step 2 > **Application permissions** > select `Agent365.Observability.OtelWrite` > **Add permissions** +5. **Grant admin consent for \** > confirm + +**Option B — CLI** (requires `a365.config.json` and `a365.generated.config.json` in the config directory): + +```bash +a365 setup admin --config-dir "" +``` + ### Added +- `a365 setup` now writes the `Agent365Observability` placeholder section (`AgentId`, `AgentBlueprintId`, `TenantId`, `AgentName`, `AgentDescription`) and `EnableAgent365Exporter: false` to `appsettings.json` (.NET) and `ENABLE_A365_OBSERVABILITY_EXPORTER=false` to `.env` (Python/Node.js), so observability configuration is pre-populated for all three platforms after running setup - Re-enabled `a365 create-instance` command (previously deprecated) — creates agent identity, agent user, and assigns licenses in a single command. The custom client app now requires the `User.ReadWrite.All` delegated permission for user creation and license assignment; existing users may need to update admin consent on their client app. - `Agent365.Observability.OtelWrite` granted to all provisioned agent identities on the Observability API as both a **delegated** permission (OAuth2 grant) and an **application** permission (S2S app role assignment), enabling agents to write OpenTelemetry data to the Agent 365 observability service - S2S app role assignment support in `a365 setup permissions` and `a365 setup admin` — the CLI now automatically grants application-type (`appRoleAssignments`) permissions on the blueprint service principal when a `ResourcePermissionSpec` defines `AppRoleScopes`. Global Administrator is required for S2S grants; non-admin users receive actionable PowerShell fallback instructions diff --git a/docs/agent365-guided-setup/a365-observability-instructions.md b/docs/agent365-guided-setup/a365-observability-instructions.md new file mode 100644 index 00000000..d842044a --- /dev/null +++ b/docs/agent365-guided-setup/a365-observability-instructions.md @@ -0,0 +1,55 @@ +# Add Agent 365 Observability + +Add Agent 365 observability to your agent at any point after `a365 setup all` has completed. + +> **Implementation reference:** [Agent observability — Microsoft Learn](https://learn.microsoft.com/en-us/microsoft-agent-365/developer/observability) +> +> **Prerequisite:** `a365 setup all` must have completed successfully so `AgentId` and `TenantId` values are present in `appsettings.json` (written by setup) or `a365.generated.config.json`. If neither source has these values, run `a365 setup all` first. + +--- + +## Task A — Add the observability SDK to the project + +Ask your coding agent (Claude Code, GitHub Copilot, or similar): + +> "Using #file:a365-observability-instructions.md, add observability to this project" + +The agent will follow the MS Learn reference above to: +1. Install the observability SDK packages for your project type (.NET, Python, or Node.js) +2. Register the exporter and tracing in startup code +3. Wire up the token resolver in the agent's turn handler +4. Add the exporter configuration setting (`EnableAgent365Exporter` / `ENABLE_A365_OBSERVABILITY_EXPORTER`) and leave it disabled by default + +After completing the above steps, the agent **must** ask the user: + +> "Setup is complete. Would you like me to scan your code and add instrumentation automatically? I'll find LLM calls, tool dispatches, agent-to-agent calls, and output operations and wrap each with the appropriate tracing scope." + +- If **yes**: scan all agent source files, identify operations matching the scope types in Task B, present a summary of planned changes, confirm with the user, then apply — adding the correct scope wrapper and required usings to each. +- If **no**: skip — instrumentation can be added later via Task B. + +After setup, set `EnableAgent365Exporter` to `true` in `appsettings.json` (or `ENABLE_A365_OBSERVABILITY_EXPORTER=true` in `.env`) to start exporting traces. + +--- + +## Task B — Instrument individual code blocks + +Select a code block in your editor (an LLM call, a tool dispatch, or an agent-to-agent call), then ask: + +> "Using #file:a365-observability-instructions.md, add observability to the selected code" + +The agent will: +1. Read the selected code and identify the operation type +2. Infer the relevant parameters (model name, provider, tool name, etc.) — it will not ask for values it can read from the code +3. Present its interpretation and ask for confirmation before making any changes +4. Wrap the code block with the correct Agent365 tracing scope + +### Supported scope types (auto-detected from code) + +| What the code does | Scope applied | +|-------------------|---------------| +| Calls an LLM/model API (`gpt-4o`, `claude-3`, etc.) | `InferenceScope` | +| Dispatches a tool or plugin function | `ExecuteToolScope` | +| Calls another agent (A2A) | `InvokeAgentScope` | +| Sends final response back to user | `OutputScope` | + +For Python and Node.js, equivalent OpenTelemetry spans are used with the same Agent365 attribute names. See the [MS Learn reference](https://learn.microsoft.com/en-us/microsoft-agent-365/developer/observability) for attribute names and patterns. diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/SetupSubcommands/PermissionsSubcommand.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/SetupSubcommands/PermissionsSubcommand.cs index bc0ba826..1196eb18 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/SetupSubcommands/PermissionsSubcommand.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/SetupSubcommands/PermissionsSubcommand.cs @@ -211,7 +211,7 @@ private static Command CreateBotSubcommand( logger.LogInformation("Would configure Bot API permissions:"); logger.LogInformation(" - Blueprint: {BlueprintId}", setupConfig.AgentBlueprintId); logger.LogInformation(" - Messaging Bot API: Authorization.ReadWrite, user_impersonation"); - logger.LogInformation(" - Observability API: user_impersonation, {OtelScope}", ConfigConstants.ObservabilityApiOtelWriteScope); + logger.LogInformation(" - Observability API: {OtelScope} (delegated + application)", ConfigConstants.ObservabilityApiOtelWriteScope); logger.LogInformation(" - Power Platform API: Connectivity.Connections.Read"); return; } diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/SetupSubcommands/SetupHelpers.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/SetupSubcommands/SetupHelpers.cs index b9e8e14f..fc69182a 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Commands/SetupSubcommands/SetupHelpers.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Commands/SetupSubcommands/SetupHelpers.cs @@ -171,7 +171,16 @@ public static void DisplaySetupSummary(SetupResults results, ILogger logger) if (pendingS2SAction) { itemNum++; - logger.LogInformation(" {N}. Observability API S2S app role — run as Global Administrator (PowerShell):", itemNum); + logger.LogInformation(" {N}. Observability API permissions — Global Administrator action required:", itemNum); + logger.LogInformation(""); + logger.LogInformation(" Option A — Entra portal (covers delegated + application in one step):"); + logger.LogInformation(" 1. Entra portal > App registrations > Blueprint app > API permissions"); + logger.LogInformation(" 2. Add a permission > APIs my organization uses > search {ObsApiAppId}", ConfigConstants.ObservabilityApiAppId); + logger.LogInformation(" 3. Delegated permissions > select {ObsScope} > Add permissions", ConfigConstants.ObservabilityApiOtelWriteScope); + logger.LogInformation(" 4. Repeat step 2, Application permissions > select {ObsScope} > Add permissions", ConfigConstants.ObservabilityApiOtelWriteScope); + logger.LogInformation(" 5. Grant admin consent for "); + logger.LogInformation(""); + logger.LogInformation(" Option B — PowerShell (application permission only; also complete Option A steps 2-5 for delegated):"); logger.LogInformation(" Connect-MgGraph -Scopes 'AppRoleAssignment.ReadWrite.All'"); logger.LogInformation(" $bp = Get-MgServicePrincipal -Filter \"appId eq '{BlueprintAppId}'\"", blueprintAppId); logger.LogInformation(" $obs = Get-MgServicePrincipal -Filter \"appId eq '{ObsApiAppId}'\"", ConfigConstants.ObservabilityApiAppId); diff --git a/src/Microsoft.Agents.A365.DevTools.Cli/Helpers/ProjectSettingsSyncHelper.cs b/src/Microsoft.Agents.A365.DevTools.Cli/Helpers/ProjectSettingsSyncHelper.cs index 6c3ad3b8..eae1f687 100644 --- a/src/Microsoft.Agents.A365.DevTools.Cli/Helpers/ProjectSettingsSyncHelper.cs +++ b/src/Microsoft.Agents.A365.DevTools.Cli/Helpers/ProjectSettingsSyncHelper.cs @@ -459,6 +459,18 @@ static JsonObject RequireObj(JsonObject parent, string prop) }; root["ConnectionsMap"] = connectionsMap; + // -- Agent365Observability -- + root["EnableAgent365Exporter"] ??= false; + var obsSection = RequireObj(root, "Agent365Observability"); + if (!string.IsNullOrWhiteSpace(pkgConfig.AgentBlueprintId)) + { + obsSection["AgentBlueprintId"] = pkgConfig.AgentBlueprintId; + } + if (!string.IsNullOrWhiteSpace(pkgConfig.TenantId)) + obsSection["TenantId"] = pkgConfig.TenantId; + obsSection["AgentName"] ??= ""; + obsSection["AgentDescription"] ??= ""; + var updated = root.ToJsonString(new JsonSerializerOptions { WriteIndented = true }); await File.WriteAllTextAsync(appsettingsPath, updated, new UTF8Encoding(false)); } @@ -512,6 +524,9 @@ void Set(string key, string? value) Set("CONNECTIONSMAP__0__SERVICEURL", "*"); Set("CONNECTIONSMAP__0__CONNECTION", "SERVICE_CONNECTION"); + // --- Agent365 Observability --- + Set("ENABLE_A365_OBSERVABILITY_EXPORTER", "false"); + await File.WriteAllLinesAsync(envPath, lines, new UTF8Encoding(false)); } @@ -564,6 +579,9 @@ void Set(string key, string? value) Set("agentic_scopes", DEFAULT_USER_AUTHORIZATION_SCOPE); Set("agentic_connectionName", "AgenticAuthConnection"); + // --- Agent365 Observability --- + Set("ENABLE_A365_OBSERVABILITY_EXPORTER", "false"); + await File.WriteAllLinesAsync(envPath, lines, new UTF8Encoding(false)); } diff --git a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Helpers/ProjectSettingsSyncHelperTests.cs b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Helpers/ProjectSettingsSyncHelperTests.cs index 9b5b4518..44845143 100644 --- a/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Helpers/ProjectSettingsSyncHelperTests.cs +++ b/src/Tests/Microsoft.Agents.A365.DevTools.Cli.Tests/Helpers/ProjectSettingsSyncHelperTests.cs @@ -185,6 +185,9 @@ void AssertHas(string key, string value) AssertHas("CONNECTIONSMAP__0__SERVICEURL", "*"); AssertHas("CONNECTIONSMAP__0__CONNECTION", "SERVICE_CONNECTION"); + + // Observability + AssertHas("ENABLE_A365_OBSERVABILITY_EXPORTER", "false"); } [Fact] @@ -239,6 +242,9 @@ void AssertHas(string key, string value) AssertHas("agentic_altBlueprintConnectionName", "service_connection"); AssertHas("agentic_scopes", "https://graph.microsoft.com/.default"); AssertHas("agentic_connectionName", "AgenticAuthConnection"); + + // Observability + AssertHas("ENABLE_A365_OBSERVABILITY_EXPORTER", "false"); } [Fact] @@ -555,4 +561,47 @@ public async Task ExecuteAsync_DotNet_UnprotectedSecret_WritesAsIs() Assert.Equal(plaintextSecret, clientSecret); } -} \ No newline at end of file + + [Fact] + public async Task ExecuteAsync_DotNet_WritesAgent365ObservabilitySection() + { + // Arrange + var projectDir = Path.Combine(_tempRoot, "dotnet_obs"); + Directory.CreateDirectory(projectDir); + + WriteFile(projectDir, "MyAgent.csproj", ""); + var appsettingsPath = WriteFile(projectDir, "appsettings.json", "{}"); + + var genPath = WriteFile(_tempRoot, "a365.generated.config.json", "{}"); + var cfgPath = WriteFile(_tempRoot, "a365.config.json", "{}"); + + var cfg = new Agent365Config + { + DeploymentProjectPath = projectDir, + TenantId = "5369a35c-46a5-4677-8ff9-2e65587654e7", + AgentBlueprintId = "73cfe0a9-87bb-4cfd-bfe1-4309c487d56c", + AgentBlueprintClientSecret = "secret" + }; + + var configService = MockConfigService(cfg).Object; + var platformDetector = CreatePlatformDetector(); + var logger = CreateLogger(); + + // Act + await ProjectSettingsSyncHelper.ExecuteAsync(cfgPath, genPath, configService, platformDetector, logger); + + // Assert + var j = ReadJson(appsettingsPath); + + // EnableAgent365Exporter written at root level, defaulting to false + Assert.False(j["EnableAgent365Exporter"]!.GetValue()); + + // Agent365Observability section present with correct values + var obs = j["Agent365Observability"]!.AsObject(); + Assert.Null(obs["AgentId"]); + Assert.Equal(cfg.AgentBlueprintId, obs["AgentBlueprintId"]!.GetValue()); + Assert.Equal(cfg.TenantId, obs["TenantId"]!.GetValue()); + Assert.Equal("", obs["AgentName"]!.GetValue()); + Assert.Equal("", obs["AgentDescription"]!.GetValue()); + } +}