diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/Agent.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/Agent.cs index 2a4bbe232a8..d0b2aae354f 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/Agent.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/Agent.cs @@ -24,12 +24,12 @@ public class Agent readonly IPlanner planner; readonly IPlanRunner planRunner; + readonly ISuspendManager suspendManager; readonly IReporter reporter; readonly IConfigSource configSource; readonly IModuleIdentityLifecycleManager moduleIdentityLifecycleManager; readonly IEntityStore configStore; readonly IEnvironmentProvider environmentProvider; - readonly AsyncLock reconcileLock = new AsyncLock(); readonly ISerde deploymentConfigInfoSerde; readonly IEncryptionProvider encryptionProvider; readonly IDeploymentMetrics deploymentMetrics; @@ -43,6 +43,7 @@ public Agent( IEnvironmentProvider environmentProvider, IPlanner planner, IPlanRunner planRunner, + ISuspendManager suspendManager, IReporter reporter, IModuleIdentityLifecycleManager moduleIdentityLifecycleManager, IEntityStore configStore, @@ -55,6 +56,7 @@ public Agent( this.configSource = Preconditions.CheckNotNull(configSource, nameof(configSource)); this.planner = Preconditions.CheckNotNull(planner, nameof(planner)); this.planRunner = Preconditions.CheckNotNull(planRunner, nameof(planRunner)); + this.suspendManager = Preconditions.CheckNotNull(suspendManager, nameof(suspendManager)); this.reporter = Preconditions.CheckNotNull(reporter, nameof(reporter)); this.moduleIdentityLifecycleManager = Preconditions.CheckNotNull(moduleIdentityLifecycleManager, nameof(moduleIdentityLifecycleManager)); this.configStore = Preconditions.CheckNotNull(configStore, nameof(configStore)); @@ -73,6 +75,7 @@ public static async Task Create( IConfigSource configSource, IPlanner planner, IPlanRunner planRunner, + ISuspendManager suspendManager, IReporter reporter, IModuleIdentityLifecycleManager moduleIdentityLifecycleManager, IEnvironmentProvider environmentProvider, @@ -106,6 +109,7 @@ await deploymentConfigInfoJson.ForEachAsync( environmentProvider, planner, planRunner, + suspendManager, reporter, moduleIdentityLifecycleManager, configStore, @@ -120,8 +124,13 @@ await deploymentConfigInfoJson.ForEachAsync( public async Task ReconcileAsync(CancellationToken token) { ModuleSet moduleSetToReport = null; - using (await this.reconcileLock.LockAsync(token)) + using (await this.suspendManager.BeginUpdateCycleAsync(token)) { + if (this.suspendManager.IsSuspended()) + { + return; + } + try { Events.StartingReconcile(); diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/ISuspendManager.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/ISuspendManager.cs new file mode 100644 index 00000000000..d9368c73ea0 --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/ISuspendManager.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Core +{ + using System; + using System.Threading; + using System.Threading.Tasks; + + public interface ISuspendManager + { + Task BeginUpdateCycleAsync(CancellationToken token); + + bool IsSuspended(); + + Task SuspendUpdatesAsync(CancellationToken token); + + Task ResumeUpdatesAsync(CancellationToken token); + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/SuspendManager.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/SuspendManager.cs new file mode 100644 index 00000000000..b314d7ba61a --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/SuspendManager.cs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Core +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Devices.Edge.Util; + using Microsoft.Azure.Devices.Edge.Util.Concurrency; + using Microsoft.Extensions.Logging; + + public class SuspendManager : ISuspendManager + { + static readonly TimeSpan maxSuspendPeriod = TimeSpan.FromMinutes(15); + readonly ISystemTime systemTime; + readonly AsyncLock reconcileLock = new AsyncLock(); + DateTime suspendTime = DateTime.MaxValue; + + public SuspendManager(ISystemTime systemTime) + { + this.systemTime = Preconditions.CheckNotNull(systemTime, nameof(systemTime)); + } + + public async Task BeginUpdateCycleAsync(CancellationToken token) + => await this.reconcileLock.LockAsync(token); + + public bool IsSuspended() + { + if (this.suspendTime == DateTime.MaxValue) + { + return false; + } + + TimeSpan elapsedTime = this.systemTime.UtcNow - this.suspendTime; + if (elapsedTime >= maxSuspendPeriod) + { + this.suspendTime = DateTime.MaxValue; + return false; + } + + Events.UpdatesSuspended(elapsedTime, maxSuspendPeriod - elapsedTime); + return true; + } + + public async Task SuspendUpdatesAsync(CancellationToken token) + { + // wait for update cycle to complete before returning + using var cycle = await this.reconcileLock.LockAsync(token); + + // allow extending the suspend timeout each call + this.suspendTime = this.systemTime.UtcNow; + } + + public Task ResumeUpdatesAsync(CancellationToken token) + { + this.suspendTime = DateTime.MaxValue; + + // do not wait for update cycle + return Task.CompletedTask; + } + + static class Events + { + const int IdStart = AgentEventIds.RestartManager + 50; + static readonly ILogger Log = Logger.Factory.CreateLogger(); + + enum EventIds + { + UpdatesSuspended = IdStart + 1 + } + + public static void UpdatesSuspended(TimeSpan elapsedTime, TimeSpan remainingTime) + { + Log.LogInformation( + (int)EventIds.UpdatesSuspended, + $"Updates have been suspended for {elapsedTime.Humanize()} (auto-resume in {remainingTime.Humanize()})."); + } + } + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/ResumeUpdatesRequestHandler.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/ResumeUpdatesRequestHandler.cs new file mode 100644 index 00000000000..57edd2f48d5 --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/ResumeUpdatesRequestHandler.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Core.Requests +{ + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Devices.Edge.Util; + + public class ResumeUpdatesRequestHandler : RequestHandlerBase + { + readonly ISuspendManager suspendManager; + + public override string RequestName => "ResumeUpdates"; + + public ResumeUpdatesRequestHandler(ISuspendManager suspendManager) + { + this.suspendManager = Preconditions.CheckNotNull(suspendManager, nameof(suspendManager)); + } + + protected override async Task> HandleRequestInternal(Option payload, CancellationToken token) + { + await this.suspendManager.ResumeUpdatesAsync(token); + + return Option.None(); + } + + protected override Option ParsePayload(Option payloadJson) + => Option.None(); + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/SuspendUpdatesRequestHandler.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/SuspendUpdatesRequestHandler.cs new file mode 100644 index 00000000000..6f301513bf8 --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/SuspendUpdatesRequestHandler.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Core.Requests +{ + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Devices.Edge.Util; + + public class SuspendUpdatesRequestHandler : RequestHandlerBase + { + readonly ISuspendManager suspendManager; + + public override string RequestName => "SuspendUpdates"; + + public SuspendUpdatesRequestHandler(ISuspendManager suspendManager) + { + this.suspendManager = Preconditions.CheckNotNull(suspendManager, nameof(suspendManager)); + } + + protected override async Task> HandleRequestInternal(Option payload, CancellationToken token) + { + await this.suspendManager.SuspendUpdatesAsync(token); + + return Option.None(); + } + + protected override Option ParsePayload(Option payloadJson) + => Option.None(); + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/AgentModule.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/AgentModule.cs index 02f5849a609..e9601b41b1e 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/AgentModule.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/AgentModule.cs @@ -269,6 +269,11 @@ protected override void Load(ContainerBuilder builder) .As>>() .SingleInstance(); + // ISuspendManager + builder.Register(c => new SuspendManager(SystemTime.Instance)) + .As() + .SingleInstance(); + // IRestartManager builder.Register(c => new RestartPolicyManager(this.maxRestartCount, this.coolOffTimeUnitInSeconds)) .As() @@ -328,6 +333,7 @@ protected override void Load(ContainerBuilder builder) var environmentProvider = c.Resolve>(); var planner = c.Resolve>(); var planRunner = c.Resolve(); + var suspendManager = c.Resolve(); var reporter = c.Resolve(); var moduleIdentityLifecycleManager = c.Resolve(); var deploymentConfigInfoSerde = c.Resolve>(); @@ -338,6 +344,7 @@ protected override void Load(ContainerBuilder builder) await configSource, await planner, planRunner, + suspendManager, reporter, moduleIdentityLifecycleManager, await environmentProvider, diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/TwinConfigSourceModule.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/TwinConfigSourceModule.cs index db20d9ad58d..93b454099ac 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/TwinConfigSourceModule.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/TwinConfigSourceModule.cs @@ -144,6 +144,26 @@ protected override void Load(ContainerBuilder builder) .As>() .SingleInstance(); + // Task - SuspendUpdatesRequestHandler + builder.Register( + c => + { + IRequestHandler handler = new SuspendUpdatesRequestHandler(c.Resolve()); + return Task.FromResult(handler); + }) + .As>() + .SingleInstance(); + + // Task - ResumeUpdatesRequestHandler + builder.Register( + c => + { + IRequestHandler handler = new ResumeUpdatesRequestHandler(c.Resolve()); + return Task.FromResult(handler); + }) + .As>() + .SingleInstance(); + // ISdkModuleClientProvider builder.Register(c => new SdkModuleClientProvider()) .As() diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/AgentTests.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/AgentTests.cs index 0bffac5ede4..28fc225ffd8 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/AgentTests.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/AgentTests.cs @@ -49,6 +49,7 @@ public void AgentConstructorInvalidArgs() var mockEnvironmentProvider = new Mock(); var mockPlanner = new Mock(); var mockPlanRunner = new Mock(); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); var mockModuleLifecycleManager = new Mock(); var configStore = Mock.Of>(); @@ -56,17 +57,18 @@ public void AgentConstructorInvalidArgs() var encryptionDecryptionProvider = Mock.Of(); var availabilityMetric = Mock.Of(); - Assert.Throws(() => new Agent(null, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); - Assert.Throws(() => new Agent(mockConfigSource.Object, null, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); - Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, null, mockPlanRunner.Object, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); - Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, null, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); - Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, null, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); - Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, null, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); - Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleLifecycleManager.Object, null, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); - Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, null, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); - Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, null, encryptionDecryptionProvider, availabilityMetric, Option.None())); - Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, null, availabilityMetric, Option.None())); - Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, null, Option.None())); + Assert.Throws(() => new Agent(null, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); + Assert.Throws(() => new Agent(mockConfigSource.Object, null, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); + Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, null, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); + Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, null, mockSuspendManager.Object, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); + Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, null, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); + Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, null, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); + Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, null, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); + Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleLifecycleManager.Object, null, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); + Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, null, serde, encryptionDecryptionProvider, availabilityMetric, Option.None())); + Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, null, encryptionDecryptionProvider, availabilityMetric, Option.None())); + Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, null, availabilityMetric, Option.None())); + Assert.Throws(() => new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, null, Option.None())); } [Fact] @@ -76,6 +78,7 @@ public async void AgentCreateSuccessWhenDecryptFails() var mockEnvironmentProvider = new Mock(); var mockPlanner = new Mock(); var mockPlanRunner = new Mock(); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); var mockModuleLifecycleManager = new Mock(); var configStore = new Mock>(); @@ -87,7 +90,7 @@ public async void AgentCreateSuccessWhenDecryptFails() encryptionDecryptionProvider.Setup(ep => ep.DecryptAsync(It.IsAny())) .ThrowsAsync(new WorkloadCommunicationException("failed", 404)); - Agent agent = await Agent.Create(mockConfigSource.Object, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleLifecycleManager.Object, mockEnvironmentProvider.Object, configStore.Object, serde, encryptionDecryptionProvider.Object, availabilityMetric, Option.None()); + Agent agent = await Agent.Create(mockConfigSource.Object, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleLifecycleManager.Object, mockEnvironmentProvider.Object, configStore.Object, serde, encryptionDecryptionProvider.Object, availabilityMetric, Option.None()); Assert.NotNull(agent); encryptionDecryptionProvider.Verify(ep => ep.DecryptAsync(It.IsAny()), Times.Once); @@ -103,6 +106,7 @@ public async void ReconcileAsyncOnEmptyPlan() var mockEnvironmentProvider = new Mock(); var mockPlanner = new Mock(); var mockPlanRunner = new Mock(); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); var mockModuleIdentityLifecycleManager = new Mock(); var runtimeInfo = Mock.Of(); @@ -136,7 +140,7 @@ public async void ReconcileAsyncOnEmptyPlan() mockReporter.Setup(r => r.ReportAsync(token, It.IsAny(), It.IsAny(), It.IsAny(), DeploymentStatus.Success)) .Returns(Task.CompletedTask); - var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); + var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); await agent.ReconcileAsync(token); @@ -154,6 +158,7 @@ public async void ReconcileAsyncAbortsWhenConfigSourceThrows() var mockEnvironment = new Mock(); var mockPlanner = new Mock(); var mockPlanRunner = new Mock(); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); var token = default(CancellationToken); var currentSet = ModuleSet.Empty; @@ -170,7 +175,7 @@ public async void ReconcileAsyncAbortsWhenConfigSourceThrows() mockReporter.Setup(r => r.ReportAsync(token, It.IsAny(), It.IsAny(), It.IsAny(), It.Is(s => s.Code == DeploymentStatusCode.Failed))) .Returns(Task.CompletedTask); - var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); + var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); // Act // Assert @@ -191,6 +196,7 @@ public async void ReconcileAsyncAbortsWhenConfigSourceReturnsKnownExceptions( var mockEnvironment = new Mock(); var mockPlanner = new Mock(); var mockPlanRunner = new Mock(); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); var token = default(CancellationToken); var currentSet = ModuleSet.Empty; @@ -209,7 +215,7 @@ public async void ReconcileAsyncAbortsWhenConfigSourceReturnsKnownExceptions( mockReporter.Setup(r => r.ReportAsync(token, It.IsAny(), It.IsAny(), It.IsAny(), It.Is(s => s.Code == statusCode))) .Returns(Task.CompletedTask); - var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); + var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); // Act // Assert @@ -227,6 +233,7 @@ public async void ReconcileAsyncAbortsWhenEnvironmentSourceThrows() var mockEnvironment = new Mock(); var mockPlanner = new Mock(); var mockPlanRunner = new Mock(); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); var token = default(CancellationToken); var mockModuleIdentityLifecycleManager = new Mock(); @@ -244,7 +251,7 @@ public async void ReconcileAsyncAbortsWhenEnvironmentSourceThrows() mockReporter.Setup(r => r.ReportAsync(token, null, It.IsAny(), It.IsAny(), It.Is(s => s.Code == DeploymentStatusCode.Failed))) .Returns(Task.CompletedTask); - var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); + var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); // Act // Assert @@ -262,6 +269,7 @@ public async void ReconcileAsyncAbortsWhenModuleIdentityLifecycleManagerThrows() var mockEnvironment = new Mock(); var mockPlanner = new Mock(); var mockPlanRunner = new Mock(); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); var token = default(CancellationToken); var mockModuleIdentityLifecycleManager = new Mock(); @@ -291,7 +299,7 @@ public async void ReconcileAsyncAbortsWhenModuleIdentityLifecycleManagerThrows() mockReporter.Setup(r => r.ReportAsync(token, It.IsAny(), It.IsAny(), It.IsAny(), It.Is(s => s.Code == DeploymentStatusCode.Failed))) .Returns(Task.CompletedTask); - var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); + var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); // Act // Assert @@ -311,6 +319,7 @@ public async void ReconcileAsyncReportsFailedWhenEncryptProviderThrows() var mockEnvironmentProvider = new Mock(); var mockPlanner = new Mock(); var mockPlanRunner = new Mock(); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); var mockModuleIdentityLifecycleManager = new Mock(); var runtimeInfo = Mock.Of(); @@ -352,7 +361,7 @@ public async void ReconcileAsyncReportsFailedWhenEncryptProviderThrows() encryptionDecryptionProvider.Setup(ep => ep.EncryptAsync(It.IsAny())) .ThrowsAsync(new WorkloadCommunicationException("failed", 404)); - var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider.Object, availabilityMetric, Option.None()); + var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider.Object, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider.Object, availabilityMetric, Option.None()); await agent.ReconcileAsync(token); @@ -391,6 +400,7 @@ public async void ReconcileAsyncOnSetPlan() var mockEnvironment = new Mock(); var mockPlanner = new Mock(); var planRunner = new OrderedRetryPlanRunner(MaxRunCount, CoolOffTimeInSeconds, new SystemTime()); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); var mockModuleIdentityLifecycleManager = new Mock(); var configStore = Mock.Of>(); @@ -412,7 +422,7 @@ public async void ReconcileAsyncOnSetPlan() mockReporter.Setup(r => r.ReportAsync(token, It.IsAny(), It.IsAny(), It.IsAny(), DeploymentStatus.Success)) .Returns(Task.CompletedTask); - var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, planRunner, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); + var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, planRunner, mockSuspendManager.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); await agent.ReconcileAsync(token); @@ -440,6 +450,7 @@ public async void ReconcileAsyncWithNoDeploymentChange() var mockEnvironment = new Mock(); var mockPlanner = new Mock(); var planRunner = new OrderedRetryPlanRunner(MaxRunCount, CoolOffTimeInSeconds, new SystemTime()); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); var mockModuleIdentityLifecycleManager = new Mock(); var configStore = Mock.Of>(); @@ -459,7 +470,7 @@ public async void ReconcileAsyncWithNoDeploymentChange() mockReporter.Setup(r => r.ReportAsync(token, It.IsAny(), It.IsAny(), It.IsAny(), DeploymentStatus.Success)) .Returns(Task.CompletedTask); - var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, planRunner, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); + var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, planRunner, mockSuspendManager.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); await agent.ReconcileAsync(token); @@ -476,6 +487,7 @@ public async Task DesiredIsNotNullBecauseCurrentThrew() var mockEnvironment = new Mock(); var mockPlanner = new Mock(); var mockPlanRunner = new Mock(); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); var runtimeInfo = new Mock(); var mockModuleIdentityLifecycleManager = new Mock(); @@ -492,7 +504,7 @@ public async Task DesiredIsNotNullBecauseCurrentThrew() .Throws(); // Act - var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); + var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); await agent.ReconcileAsync(token); // Assert @@ -508,6 +520,7 @@ public async Task CurrentIsNotNullBecauseDesiredThrew() var mockEnvironment = new Mock(); var mockPlanner = new Mock(); var mockPlanRunner = new Mock(); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); var mockModuleIdentityLifecycleManager = new Mock(); var token = default(CancellationToken); @@ -523,7 +536,7 @@ public async Task CurrentIsNotNullBecauseDesiredThrew() .ReturnsAsync(ModuleSet.Empty); // Act - var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); + var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); await agent.ReconcileAsync(token); // Assert @@ -553,6 +566,7 @@ public async void ReconcileAsyncExecuteAsyncIncompleteDefaulsUnknown() var mockEnvironment = new Mock(); var mockPlanner = new Mock(); var mockPlanRunner = new Mock(); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); var mockModuleIdentityLifecycleManager = new Mock(); var configStore = Mock.Of>(); @@ -575,7 +589,7 @@ public async void ReconcileAsyncExecuteAsyncIncompleteDefaulsUnknown() .Returns(Task.CompletedTask); mockPlanRunner.SetupSequence(m => m.ExecuteAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(false); - var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, deploymentMetrics, Option.None()); + var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, deploymentMetrics, Option.None()); await agent.ReconcileAsync(token); @@ -608,6 +622,7 @@ public async void ReconcileAsyncExecuteAsyncIncompleteReportsLastState() var mockEnvironment = new Mock(); var mockPlanner = new Mock(); var mockPlanRunner = new Mock(); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); var mockModuleIdentityLifecycleManager = new Mock(); var configStore = Mock.Of>(); @@ -633,7 +648,7 @@ public async void ReconcileAsyncExecuteAsyncIncompleteReportsLastState() mockPlanRunner.SetupSequence(m => m.ExecuteAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ThrowsAsync(new Exception("generic exception")) .ReturnsAsync(false); - var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, deploymentMetrics, Option.None()); + var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, deploymentMetrics, Option.None()); await agent.ReconcileAsync(token); await agent.ReconcileAsync(token); @@ -654,6 +669,7 @@ public async Task ReportShutdownAsyncConfigTest() var mockEnvironment = new Mock(); var mockPlanner = new Mock(); var mockPlanRunner = new Mock(); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); var mockModuleIdentityLifecycleManager = new Mock(); var configStore = Mock.Of>(); @@ -678,7 +694,7 @@ public async Task ReportShutdownAsyncConfigTest() mockEnvironment.Setup(e => e.GetModulesAsync(token)) .ReturnsAsync(ModuleSet.Empty); - var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); + var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); await agent.ReportShutdownAsync(token); // Assert @@ -715,6 +731,7 @@ public async Task HandleShutdownTest() await Task.Delay(TimeSpan.FromSeconds(5)); return true; }); + var mockSuspendManager = new Mock(); var mockReporter = new Mock(); mockReporter.Setup( @@ -741,7 +758,7 @@ public async Task HandleShutdownTest() .ReturnsAsync(deploymentConfigInfo); // Act - var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); + var agent = new Agent(mockConfigSource.Object, mockEnvironmentProvider, mockPlanner.Object, mockPlanRunner.Object, mockSuspendManager.Object, mockReporter.Object, mockModuleIdentityLifecycleManager.Object, configStore, DeploymentConfigInfo.Empty, serde, encryptionDecryptionProvider, availabilityMetric, Option.None()); var shutdownTask = agent.HandleShutdown(token); var waitTask = Task.Delay(TimeSpan.FromSeconds(6)); diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/SuspendManagerTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/SuspendManagerTest.cs new file mode 100644 index 00000000000..4508b20f6f8 --- /dev/null +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/SuspendManagerTest.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Core.Test +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Devices.Edge.Util; + using Microsoft.Azure.Devices.Edge.Util.Test.Common; + using Moq; + using Xunit; + + [Unit] + public class SuspendManagerTest + { + [Fact] + public void TestCreate() + { + Assert.Throws(() => new SuspendManager(null)); + Assert.NotNull(new SuspendManager(SystemTime.Instance)); + } + + [Fact] + public void IsNotSuspendedByDefault() + { + var manager = new SuspendManager(SystemTime.Instance); + + Assert.False(manager.IsSuspended()); + } + + [Fact] + public async Task IsSuspendedWhenSuspended() + { + var cts = new CancellationTokenSource(); + var manager = new SuspendManager(SystemTime.Instance); + + await manager.SuspendUpdatesAsync(cts.Token); + + Assert.True(manager.IsSuspended()); + } + + [Fact] + public async Task IsNotSuspendedWhenResumed() + { + var cts = new CancellationTokenSource(); + var manager = new SuspendManager(SystemTime.Instance); + + await manager.SuspendUpdatesAsync(cts.Token); + await manager.ResumeUpdatesAsync(cts.Token); + + Assert.False(manager.IsSuspended()); + } + + [Fact] + public async Task SuspendTimeout() + { + var systemTime = new Mock(); + var cts = new CancellationTokenSource(); + var manager = new SuspendManager(systemTime.Object); + + DateTime callTime = DateTime.UtcNow; + systemTime.SetupGet(s => s.UtcNow) + .Returns(() => callTime) + .Callback(() => callTime = callTime.AddMinutes(10)); + + await manager.SuspendUpdatesAsync(cts.Token); + + Assert.True(manager.IsSuspended()); + Assert.False(manager.IsSuspended()); + } + + [Fact] + public async Task SuspendBlocksUntilCycleComplete() + { + var manager = new SuspendManager(SystemTime.Instance); + + using var cycle = await manager.BeginUpdateCycleAsync(CancellationToken.None); + var suspendTask = manager.SuspendUpdatesAsync(CancellationToken.None); + + Assert.False(suspendTask.IsCompleted); + Assert.False(manager.IsSuspended()); + + cycle.Dispose(); + await suspendTask.TimeoutAfter(TimeSpan.FromSeconds(2)); + + Assert.True(manager.IsSuspended()); + } + } +} diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/ResumeUpdatesRequestHandlerTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/ResumeUpdatesRequestHandlerTest.cs new file mode 100644 index 00000000000..92c1277853f --- /dev/null +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/ResumeUpdatesRequestHandlerTest.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Core.Test.Requests +{ + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Devices.Edge.Agent.Core.Requests; + using Microsoft.Azure.Devices.Edge.Util; + using Microsoft.Azure.Devices.Edge.Util.Test.Common; + using Moq; + using Xunit; + + [Unit] + public class ResumeUpdatesRequestHandlerTest + { + [Fact] + public async Task ResumeUpdatesTest() + { + var cts = new CancellationTokenSource(); + var suspendManager = Mock.Of(MockBehavior.Strict); + Mock.Get(suspendManager).Setup(m => m.ResumeUpdatesAsync(cts.Token)).Returns(Task.CompletedTask); + + var handler = new ResumeUpdatesRequestHandler(suspendManager); + + Option response = await handler.HandleRequest(Option.None(), cts.Token); + + Assert.False(response.HasValue); + Mock.Get(suspendManager).VerifyAll(); + } + } +} diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/SuspendUpdatesRequestHandlerTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/SuspendUpdatesRequestHandlerTest.cs new file mode 100644 index 00000000000..9a59c6256c9 --- /dev/null +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/SuspendUpdatesRequestHandlerTest.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Core.Test.Requests +{ + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Devices.Edge.Agent.Core.Requests; + using Microsoft.Azure.Devices.Edge.Util; + using Microsoft.Azure.Devices.Edge.Util.Test.Common; + using Moq; + using Xunit; + + [Unit] + public class SuspendUpdatesRequestHandlerTest + { + [Fact] + public async Task SuspendUpdatesTest() + { + var cts = new CancellationTokenSource(); + var suspendManager = Mock.Of(MockBehavior.Strict); + Mock.Get(suspendManager).Setup(m => m.SuspendUpdatesAsync(cts.Token)).Returns(Task.CompletedTask); + + var handler = new SuspendUpdatesRequestHandler(suspendManager); + + Option response = await handler.HandleRequest(Option.None(), cts.Token); + + Assert.False(response.HasValue); + Mock.Get(suspendManager).VerifyAll(); + } + } +} diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Integration.Test/AgentTestsBase.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Integration.Test/AgentTestsBase.cs index 5eb816591b9..cd9f4d903f6 100644 --- a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Integration.Test/AgentTestsBase.cs +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Integration.Test/AgentTestsBase.cs @@ -165,7 +165,8 @@ protected async Task AgentExecutionTestAsync(TestConfig testConfig) Agent agent = await Agent.Create( configSource.Object, restartPlanner, - new OrderedRetryPlanRunner(20, 10, new SystemTime()), + new OrderedRetryPlanRunner(20, 10, SystemTime.Instance), + new SuspendManager(SystemTime.Instance), reporter, moduleIdentityLifecycleManager.Object, environmentProvider.Object,