From f8ac3a9f3fb1d887ec805dfae1b890e121870072 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 10 Nov 2025 22:30:47 -0800 Subject: [PATCH 01/13] Add MacOS job for Helix tests in pipeline Add a new job for MacOS with Helix testing steps. --- azure-pipelines-pr.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index 4f2a75102bf..e83766aed28 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -208,6 +208,35 @@ stages: SYSTEM_ACCESSTOKEN: $(System.AccessToken) HelixAccessToken: '' + - job: MacOs + timeoutInMinutes: 90 + pool: + vmimage: macOS-13 + strategy: + matrix: + Build_Debug: + _BuildConfig: Debug + variables: + - _Testing: Helix + preSteps: + - checkout: self + clean: true + steps: + - script: eng/common/build.sh + --configuration $(_BuildConfig) + --prepareMachine + --ci + --restore + --test + --warnAsError false + --projects $(System.DefaultWorkingDirectory)/tests/UnitTests.proj + /bl:$(System.DefaultWorkingDirectory)/artifacts/log/$(_BuildConfig)/Helix.binlog + /p:RestoreUsingNuGetTargets=false + displayName: Run Helix Tests + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HelixAccessToken: '' + - stage: Test_XHarness displayName: Test XHarness SDK dependsOn: build From 1615f794cff52fd733376bf8e1e94a0d6e0a5e72 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 10 Nov 2025 22:32:44 -0800 Subject: [PATCH 02/13] Add OSX target queue to UnitTests project --- tests/UnitTests.proj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/UnitTests.proj b/tests/UnitTests.proj index baaa1b68e4f..42c36109cf1 100644 --- a/tests/UnitTests.proj +++ b/tests/UnitTests.proj @@ -63,6 +63,7 @@ + @@ -75,6 +76,7 @@ + From ccb6642355f22de2345caaf67e5df47b4d5930dc Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 10 Nov 2025 22:50:31 -0800 Subject: [PATCH 03/13] Add MacOS build job to Azure pipeline --- azure-pipelines-pr.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index e83766aed28..a517cca4aa7 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -124,6 +124,23 @@ stages: /p:Test=false displayName: Unix Build / Publish + - job: MacOS + pool: + vmImage: macOS-13 + strategy: + matrix: + Build_Debug: + _BuildConfig: Debug + preSteps: + - checkout: self + clean: true + steps: + - script: eng/common/cibuild.sh + --configuration $(_BuildConfig) + --prepareMachine + /p:Test=false + displayName: MacOS Build / Publish + - stage: Test dependsOn: build jobs: From 22682a2a36922c9399c08a869705374218a4558d Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 11 Nov 2025 08:57:30 -0800 Subject: [PATCH 04/13] Add some diagnostics, and a timeout --- .../Sdk/AzureDevOpsTask.cs | 62 +++++++++++++++++++ .../AzurePipelines.MultiQueue.targets | 7 ++- .../azure-pipelines/AzurePipelines.props | 3 +- 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs b/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs index a102acb64dc..8f818716794 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Net; using System.Net.Http; @@ -21,6 +23,11 @@ namespace Microsoft.DotNet.Helix.AzureDevOps { public abstract class AzureDevOpsTask : BaseTask { + /// + /// Timeout in seconds for HTTP requests. Default is 0 seconds (no timeout). + /// + public int TimeoutInSeconds { get; set; } = 0; + private bool InAzurePipeline => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_BUILDNUMBER")); protected string GetEnvironmentVariable(string name) @@ -44,6 +51,57 @@ protected string GetEnvironmentVariable(string name) protected abstract Task ExecuteCoreAsync(HttpClient client); + private ActivityListener CreateHttpActivityListener() + { + var listener = new ActivityListener + { + ShouldListenTo = source => source.Name == "System.Net.Http", + Sample = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllDataAndRecorded, + ActivityStarted = activity => + { + Log.LogMessage(MessageImportance.Low, + $"[System.Net.Http] {activity.OperationName} started - {activity.DisplayName}"); + + // Log tags which contain request details + foreach (var tag in activity.Tags) + { + Log.LogMessage(MessageImportance.Low, $" {tag.Key}: {tag.Value}"); + } + }, + ActivityStopped = activity => + { + var status = activity.Status == ActivityStatusCode.Error ? "ERROR" : "OK"; + Log.LogMessage(MessageImportance.Low, + $"[System.Net.Http] {activity.OperationName} stopped - Duration: {activity.Duration.TotalMilliseconds:F2}ms, Status: {status}"); + + if (activity.Status == ActivityStatusCode.Error && !string.IsNullOrEmpty(activity.StatusDescription)) + { + Log.LogMessage(MessageImportance.Normal, + $"[System.Net.Http] Error: {activity.StatusDescription}"); + } + + // Log response tags (status code, etc.) + foreach (var tag in activity.Tags) + { + Log.LogMessage(MessageImportance.Low, $" {tag.Key}: {tag.Value}"); + } + + // Log events (connection established, request/response headers sent/received, etc.) + foreach (var evt in activity.Events) + { + var tagsStr = string.Join(", ", evt.Tags); + Log.LogMessage(MessageImportance.Low, + $" Event: {evt.Name} at {evt.Timestamp:O}" + + (string.IsNullOrEmpty(tagsStr) ? "" : $" ({tagsStr})")); + } + } + }; + + ActivitySource.AddActivityListener(listener); + Log.LogMessage(MessageImportance.Low, "System.Net.Http ActivityListener registered"); + return listener; + } + public override bool Execute() => ExecuteAsync().GetAwaiter().GetResult(); @@ -57,6 +115,9 @@ private async Task ExecuteAsync() } else { + // Set up HTTP activity listener for diagnostics - only active during this task execution + using var httpActivityListener = CreateHttpActivityListener(); + // Configure the cert revocation check in a fail-open state to avoid intermittent failures // on Mac if the endpoint is not available. This is only available on .NET Core, but has only been // observed on Mac anyway. @@ -96,6 +157,7 @@ private async Task ExecuteAsync() }) #endif { + Timeout = TimeoutInSeconds > 0 ? TimeSpan.FromSeconds(TimeoutInSeconds) : System.Threading.Timeout.InfiniteTimeSpan, DefaultRequestHeaders = { Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes("unused:" + AccessToken))), diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.MultiQueue.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.MultiQueue.targets index ea35a0158f3..27f6ac3a394 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.MultiQueue.targets +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.MultiQueue.targets @@ -15,7 +15,7 @@ <_CurrentTestRunName Condition="'$(_CurrentTestRunName)' == ''">$(TestRunNamePrefix)$(_CurrentTargetQueue)$(TestRunNameSuffix) - + @@ -29,13 +29,13 @@ - + - + diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.props b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.props index e25086445d4..12375f18758 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.props +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.props @@ -4,6 +4,7 @@ true false + 180 @@ -13,7 +14,7 @@ - false + false true From 5d3d896fab6aaf75bd046b607e218930cb826733 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 11 Nov 2025 09:01:46 -0800 Subject: [PATCH 05/13] Update VM image to macOS-Latest --- azure-pipelines-pr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index a517cca4aa7..c47a6f3909e 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -126,7 +126,7 @@ stages: - job: MacOS pool: - vmImage: macOS-13 + vmImage: macOS-latest strategy: matrix: Build_Debug: @@ -228,7 +228,7 @@ stages: - job: MacOs timeoutInMinutes: 90 pool: - vmimage: macOS-13 + vmimage: macOS-latest strategy: matrix: Build_Debug: From 6296d25bc524c24f72867605c39228a82640ae1a Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Tue, 11 Nov 2025 19:36:18 +0100 Subject: [PATCH 06/13] Rename TimeoutInSeconds property to Timeout --- src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs b/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs index 8f818716794..5fb763e862b 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs @@ -26,7 +26,7 @@ public abstract class AzureDevOpsTask : BaseTask /// /// Timeout in seconds for HTTP requests. Default is 0 seconds (no timeout). /// - public int TimeoutInSeconds { get; set; } = 0; + public int Timeout { get; set; } = 0; private bool InAzurePipeline => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_BUILDNUMBER")); From c934540c955427a478afe8a0902e8abc18815495 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Tue, 11 Nov 2025 19:56:11 +0100 Subject: [PATCH 07/13] Fix timeout configuration in AzureDevOpsTask --- src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs b/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs index 5fb763e862b..67a1d8bd23f 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs @@ -157,7 +157,7 @@ private async Task ExecuteAsync() }) #endif { - Timeout = TimeoutInSeconds > 0 ? TimeSpan.FromSeconds(TimeoutInSeconds) : System.Threading.Timeout.InfiniteTimeSpan, + Timeout = Timeout > 0 ? TimeSpan.FromSeconds(Timeout) : System.Threading.Timeout.InfiniteTimeSpan, DefaultRequestHeaders = { Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes("unused:" + AccessToken))), From f96933507e17e33610afb40a0e203f298514265f Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 11 Nov 2025 13:12:45 -0800 Subject: [PATCH 08/13] Add MSBuild detailed logging to see where hang is occuring --- azure-pipelines-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index c47a6f3909e..e67b86cbe48 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -247,6 +247,7 @@ stages: --test --warnAsError false --projects $(System.DefaultWorkingDirectory)/tests/UnitTests.proj + /v:detailed /bl:$(System.DefaultWorkingDirectory)/artifacts/log/$(_BuildConfig)/Helix.binlog /p:RestoreUsingNuGetTargets=false displayName: Run Helix Tests From c21d7dfbe8860dcc1b1cb7c09ab2a4685dcc1eda Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 11 Nov 2025 14:18:10 -0800 Subject: [PATCH 09/13] Too much logging --- azure-pipelines-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index e67b86cbe48..76ad4297fea 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -247,7 +247,7 @@ stages: --test --warnAsError false --projects $(System.DefaultWorkingDirectory)/tests/UnitTests.proj - /v:detailed + /v:normal /bl:$(System.DefaultWorkingDirectory)/artifacts/log/$(_BuildConfig)/Helix.binlog /p:RestoreUsingNuGetTargets=false displayName: Run Helix Tests From c8c7306185a8f0aaa7ffbbe7bfc34c7adfa6eee6 Mon Sep 17 00:00:00 2001 From: "Eric St. John" Date: Tue, 11 Nov 2025 14:38:35 -0800 Subject: [PATCH 10/13] Update default timeout for AzDo AzureDevOpsTask --- src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs b/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs index 67a1d8bd23f..612b7a83857 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs @@ -24,9 +24,9 @@ namespace Microsoft.DotNet.Helix.AzureDevOps public abstract class AzureDevOpsTask : BaseTask { /// - /// Timeout in seconds for HTTP requests. Default is 0 seconds (no timeout). + /// Timeout in seconds for HTTP requests. Default is 180 seconds. /// - public int Timeout { get; set; } = 0; + public int Timeout { get; set; } = 180; private bool InAzurePipeline => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_BUILDNUMBER")); From 72795bccf360643755c8c74ef38cfa06e7279eb4 Mon Sep 17 00:00:00 2001 From: chcosta Date: Tue, 11 Nov 2025 21:42:02 -0800 Subject: [PATCH 11/13] just zip one queue --- tests/UnitTests.proj | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/UnitTests.proj b/tests/UnitTests.proj index 42c36109cf1..d9994c2c5d8 100644 --- a/tests/UnitTests.proj +++ b/tests/UnitTests.proj @@ -60,9 +60,6 @@ - - - @@ -73,9 +70,6 @@ - - - From 8b0c0246032719bbd449cd008817bcbf35635c16 Mon Sep 17 00:00:00 2001 From: chcosta Date: Tue, 11 Nov 2025 23:22:52 -0800 Subject: [PATCH 12/13] update GetAsync exception logging --- src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs b/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs index 671215d884a..25b1c7a0b49 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs @@ -245,7 +245,7 @@ await _retry.RunAsync(async attempt => } catch (Exception toLog) { - Log.LogMessage(MessageImportance.Low, $"Hit exception in GetAsync(); will retry up to 10 times ({SanitizeString(toLog.Message)})"); + Log.LogMessage(MessageImportance.Low, $"Hit exception in GetAsync(); will retry up to 10 times ({SanitizeString(toLog.ToString())})"); return false; } }); From 63d0e7b59bca1fbce9688a282e1adcbd2107db75 Mon Sep 17 00:00:00 2001 From: chcosta Date: Tue, 11 Nov 2025 23:36:55 -0800 Subject: [PATCH 13/13] disable crl on osx --- src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs b/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs index 25b1c7a0b49..b26333da139 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs @@ -59,7 +59,11 @@ public class FindDotNetCliPackage : MSBuildTaskBase public override void ConfigureServices(IServiceCollection collection) { - _httpMessageHandler = new HttpClientHandler { CheckCertificateRevocationList = true }; + bool isOSX = System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.OSX); + _httpMessageHandler = new HttpClientHandler + { + CheckCertificateRevocationList = !isOSX // Disable on macOS + }; collection.TryAddSingleton(_httpMessageHandler); collection.TryAddSingleton(Log); }