[Debugger Default-On] DEBUG-3322 Debugger in-product enablement (#7366)

## Summary of changes
This PR adds remote enablement support for debugger products.
depends on #7441

## Reason for change
Currently, the only option to enable DI (for example) is by setting an
environment variable. We want to give customers the ability to enable
products on demand.

## Implementation details
Gets RC events with new configuration and update the products based on
defined rules.

## Test coverage
All existing tests
New system tests (DataDog/system-tests#4991)
DebuggerManagerTests
DebuggerManagerDynamicTests

---------

Co-authored-by: Andrew Lock <[email protected]>

Author: dudikeleti

tracer/src/Datadog.Trace.Trimming/build/Datadog.Trace.Trimming.xml

@@ -1039,7 +1039,6 @@
       <type fullname="System.Threading.Monitor" />
       <type fullname="System.Threading.Mutex" />
       <type fullname="System.Threading.ReaderWriterLockSlim" />
-      <type fullname="System.Threading.SemaphoreFullException" />
       <type fullname="System.Threading.SemaphoreSlim" />
       <type fullname="System.Threading.SpinLock" />
       <type fullname="System.Threading.SpinWait" />

tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs

@@ -496,26 +496,28 @@ private static void StartDiagnosticManager()
         private static void InitializeDebugger(TracerSettings tracerSettings)
         {
             var manager = DebuggerManager.Instance;
-            if (manager.DebuggerSettings.CodeOriginForSpansEnabled
-                || manager.DebuggerSettings.DynamicInstrumentationEnabled
-                || manager.ExceptionReplaySettings.Enabled)
+            var debuggerSettings = manager.DebuggerSettings;
+
+            if (!debuggerSettings.DynamicInstrumentationEnabled)
             {
-                _ = Task.Run(
-                    async () =>
-                    {
-                        try
-                        {
-                            await DebuggerManager.Instance.UpdateConfiguration(tracerSettings).ConfigureAwait(false);
-                        }
-                        catch (Exception ex)
-                        {
-                            Log.Error(ex, "Error initializing debugger");
-                        }
-                    });
+                // we need this line for tests
+                Log.Information("Dynamic Instrumentation is disabled. To enable it, please set DD_DYNAMIC_INSTRUMENTATION_ENABLED environment variable to 'true'.");
+            }
+
+            if (!debuggerSettings.DynamicInstrumentationEnabled
+             && !debuggerSettings.CodeOriginForSpansEnabled
+             && !manager.ExceptionReplaySettings.Enabled)
+            {
+                Log.Debug("Debugger products are not enabled");
             }
             else
             {
-                Log.Information($"Dynamic Instrumentation is disabled. To enable it, please set {Datadog.Trace.Configuration.ConfigurationKeys.Debugger.DynamicInstrumentationEnabled} environment variable to 'true'.");
+                _ = manager.UpdateConfiguration(tracerSettings)
+                           .ContinueWith(
+                                t => Log.Error(t?.Exception, "Error initializing debugger"),
+                                CancellationToken.None,
+                                TaskContinuationOptions.OnlyOnFaulted,
+                                TaskScheduler.Default);
             }
         }
 

tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs

@@ -10,11 +10,15 @@
 using System.Collections.ObjectModel;
 using System.Text;
 using System.Threading;
+using System.Threading.Tasks;
 using Datadog.Trace.Configuration.ConfigurationSources;
 using Datadog.Trace.Configuration.Telemetry;
+using Datadog.Trace.Debugger;
+using Datadog.Trace.Debugger.Configurations;
 using Datadog.Trace.Logging;
 using Datadog.Trace.RemoteConfigurationManagement;
 using Datadog.Trace.Telemetry;
+using Datadog.Trace.Vendors.Serilog.Events;
 
 namespace Datadog.Trace.Configuration
 {
@@ -41,12 +45,16 @@ public void Start()
             {
                 _subscriptionManager.SubscribeToChanges(_subscription!);
 
-                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingCustomTags, true);     // 15
-                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingHttpHeaderTags, true); // 14
-                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingLogsInjection, true);  // 13
-                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingSampleRate, true);     // 12
-                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingTracingEnabled, true); // 19
-                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingSampleRules, true);    // 29
+                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingCustomTags, true);                          // 15
+                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingHttpHeaderTags, true);                      // 14
+                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingLogsInjection, true);                       // 13
+                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingSampleRate, true);                          // 12
+                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingTracingEnabled, true);                      // 19
+                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingSampleRules, true);                         // 29
+                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingEnabledDynamicInstrumentation, true);       // 38
+                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingEnableExceptionReplay, true);               // 39
+                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingEnableCodeOrigin, true);                    // 40
+                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingEnableLiveDebugging, true);                 // 41
             }
         }
 
@@ -91,27 +99,60 @@ private static void OnConfigurationChanged(ConfigurationBuilder settings)
             // Needs to be done before returning, to feed the value to the telemetry
             // var debugLogsEnabled = settings.WithKeys(ConfigurationKeys.DebugEnabled).AsBool();
 
+            TracerSettings newSettings;
             if (dynamicSettings.Equals(oldSettings.DynamicSettings))
             {
                 Log.Debug("No changes detected in the new dynamic configuration");
-                return;
+                newSettings = oldSettings;
             }
+            else
+            {
+                Log.Information("Applying new dynamic configuration");
+
+                newSettings = oldSettings with { DynamicSettings = dynamicSettings };
+
+                /*
+                if (debugLogsEnabled != null && debugLogsEnabled.Value != GlobalSettings.Instance.DebugEnabled)
+                {
+                    GlobalSettings.SetDebugEnabled(debugLogsEnabled.Value);
+                    Security.Instance.SetDebugEnabled(debugLogsEnabled.Value);
 
-            Log.Information("Applying new dynamic configuration");
+                    NativeMethods.UpdateSettings(new[] { ConfigurationKeys.DebugEnabled }, new[] { debugLogsEnabled.Value ? "1" : "0" });
+                }
+                */
 
-            var newSettings = oldSettings with { DynamicSettings = dynamicSettings };
+                Tracer.Configure(newSettings);
+            }
 
-            /*
-            if (debugLogsEnabled != null && debugLogsEnabled.Value != GlobalSettings.Instance.DebugEnabled)
+            var dynamicDebuggerSettings = new ImmutableDynamicDebuggerSettings
             {
-                GlobalSettings.SetDebugEnabled(debugLogsEnabled.Value);
-                Security.Instance.SetDebugEnabled(debugLogsEnabled.Value);
+                DynamicInstrumentationEnabled = settings.WithKeys(ConfigurationKeys.Debugger.DynamicInstrumentationEnabled).AsBool(),
+                ExceptionReplayEnabled = settings.WithKeys(ConfigurationKeys.Debugger.ExceptionReplayEnabled).AsBool(),
+                CodeOriginEnabled = settings.WithKeys(ConfigurationKeys.Debugger.CodeOriginForSpansEnabled).AsBool(),
+            };
 
-                NativeMethods.UpdateSettings(new[] { ConfigurationKeys.DebugEnabled }, new[] { debugLogsEnabled.Value ? "1" : "0" });
+            var oldDebuggerSettings = DebuggerManager.Instance.DebuggerSettings;
+
+            if (dynamicDebuggerSettings.Equals(oldDebuggerSettings.DynamicSettings))
+            {
+                Log.Debug("No changes detected in the new dynamic debugger configuration");
+                return;
             }
-            */
 
-            Tracer.Configure(newSettings);
+            Log.Information("Applying new dynamic debugger configuration");
+            if (Log.IsEnabled(LogEventLevel.Debug))
+            {
+                Log.Debug(
+                    "DynamicInstrumentationEnabled={DynamicInstrumentationEnabled}, ExceptionReplayEnabled={ExceptionReplayEnabled}, CodeOriginEnabled={CodeOriginEnabled}",
+                    dynamicDebuggerSettings.DynamicInstrumentationEnabled,
+                    dynamicDebuggerSettings.ExceptionReplayEnabled,
+                    dynamicDebuggerSettings.CodeOriginEnabled);
+            }
+
+            var newDebuggerSettings = oldDebuggerSettings with { DynamicSettings = dynamicDebuggerSettings };
+
+            DebuggerManager.Instance.UpdateConfiguration(newSettings, newDebuggerSettings)
+                           .ContinueWith(t => Log.Error(t?.Exception, "Error updating dynamic configuration for debugger"), TaskContinuationOptions.OnlyOnFaulted);
         }
 
         private ApplyDetails[] ConfigurationUpdated(Dictionary<string, List<RemoteConfiguration>> configByProduct, Dictionary<string, List<RemoteConfigurationPath>>? removedConfigByProduct)

tracer/src/Datadog.Trace/Debugger/Configurations/ImmutableDynamicDebuggerSettings.cs

@@ -0,0 +1,18 @@
+// <copyright file="ImmutableDynamicDebuggerSettings.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+
+namespace Datadog.Trace.Debugger.Configurations
+{
+    internal record ImmutableDynamicDebuggerSettings
+    {
+        public bool? DynamicInstrumentationEnabled { get; init; }
+
+        public bool? ExceptionReplayEnabled { get; init; }
+
+        public bool? CodeOriginEnabled { get; init; }
+    }
+}