diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs index 724f5269f..0826c821e 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs @@ -2,6 +2,7 @@ using FluentAssertions; using k8s; using Newtonsoft.Json; +using NSubstitute; using Octopus.Diagnostics; using Octopus.Tentacle.Diagnostics; using Octopus.Tentacle.Kubernetes.Diagnostics; @@ -26,19 +27,20 @@ public KubernetesClientConfiguration Get() return KubernetesClientConfiguration.BuildConfigFromConfigFile(filename); } } - + [Test] public async Task FetchingTimestampFromEmptyConfigMapEntryShouldBeMinValue() { //Arrange + var config = Substitute.For(); var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath); - var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); - var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", configMapService); + var configMapService = new Support.TestSupportConfigMapService(config, kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); + var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", config, configMapService); var metrics = new KubernetesAgentMetrics(persistenceProvider, ConfigMapNames.AgentMetricsConfigMapKey, systemLog); //Act var result = await metrics.GetLatestEventTimestamp(CancellationToken.None); - + //Assert result.Should().Be(DateTimeOffset.MinValue); } @@ -47,14 +49,15 @@ public async Task FetchingTimestampFromEmptyConfigMapEntryShouldBeMinValue() public async Task FetchingLatestEventTimestampFromNonexistentConfigMapThrowsException() { //Arrange + var config = Substitute.For(); var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath); - var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); - var persistenceProvider = new PersistenceProvider("nonexistent-config-map", configMapService); + var configMapService = new Support.TestSupportConfigMapService(config, kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); + var persistenceProvider = new PersistenceProvider("nonexistent-config-map", config, configMapService); var metrics = new KubernetesAgentMetrics(persistenceProvider, "metrics", systemLog); //Act Func func = async () => await metrics.GetLatestEventTimestamp(CancellationToken.None); - + //Assert await func.Should().ThrowAsync(); } @@ -63,9 +66,10 @@ public async Task FetchingLatestEventTimestampFromNonexistentConfigMapThrowsExce public async Task WritingEventToNonExistentConfigMapShouldFailSilently() { //Arrange + var config = Substitute.For(); var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath); - var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); - var persistenceProvider = new PersistenceProvider("nonexistent-config-map", configMapService); + var configMapService = new Support.TestSupportConfigMapService(config, kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); + var persistenceProvider = new PersistenceProvider("nonexistent-config-map", config, configMapService); var metrics = new KubernetesAgentMetrics(persistenceProvider, ConfigMapNames.AgentMetricsConfigMapKey, systemLog); //Act @@ -79,15 +83,16 @@ public async Task WritingEventToNonExistentConfigMapShouldFailSilently() public async Task WritingEventToExistingConfigMapShouldPersistJsonEntry() { //Arrange + var config = Substitute.For(); var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath); - var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); - var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", configMapService); + var configMapService = new Support.TestSupportConfigMapService(config, kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); + var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", config, configMapService); var metrics = new KubernetesAgentMetrics(persistenceProvider, ConfigMapNames.AgentMetricsConfigMapKey, systemLog); //Act var eventTimestamp = DateTimeOffset.Now; await metrics.TrackEvent("reason", "source", eventTimestamp, CancellationToken.None); - + //Assert var persistedDictionary = await persistenceProvider.ReadValues(CancellationToken.None); var metricsData = persistedDictionary[ConfigMapNames.AgentMetricsConfigMapKey]; diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj index 412854765..f2e545276 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj @@ -18,6 +18,7 @@ + diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Support/TestSupportConfigMapService.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Support/TestSupportConfigMapService.cs index af9b58fda..e6d99b2b4 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Support/TestSupportConfigMapService.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Support/TestSupportConfigMapService.cs @@ -7,15 +7,14 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration.Support { - // This is a copy of the production ConfigMapService, but allows the namespace to be explicitly // defined. public class TestSupportConfigMapService : KubernetesService, IKubernetesConfigMapService { readonly string targetNamespace; - public TestSupportConfigMapService(IKubernetesClientConfigProvider configProvider, ISystemLog log, string targetNamespace) - : base(configProvider, log) + public TestSupportConfigMapService(IKubernetesConfiguration kubernetesConfiguration, IKubernetesClientConfigProvider configProvider, ISystemLog log, string targetNamespace) + : base(configProvider, kubernetesConfiguration, log) { this.targetNamespace = targetNamespace; } diff --git a/source/Octopus.Tentacle.Tests/Capabilities/CapabilitiesServiceV2Fixture.cs b/source/Octopus.Tentacle.Tests/Capabilities/CapabilitiesServiceV2Fixture.cs index 0c6326a05..de4e8ea2a 100644 --- a/source/Octopus.Tentacle.Tests/Capabilities/CapabilitiesServiceV2Fixture.cs +++ b/source/Octopus.Tentacle.Tests/Capabilities/CapabilitiesServiceV2Fixture.cs @@ -2,6 +2,7 @@ using System.Threading; using System.Threading.Tasks; using FluentAssertions; +using NSubstitute; using NUnit.Framework; using Octopus.Tentacle.Contracts; using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1; @@ -16,7 +17,10 @@ public class CapabilitiesServiceV2Fixture [Test] public async Task CapabilitiesAreReturned() { - var capabilities = (await new CapabilitiesServiceV2() + var k8sDetection = Substitute.For(); + k8sDetection.IsRunningAsKubernetesAgent.Returns(false); + + var capabilities = (await new CapabilitiesServiceV2(k8sDetection) .GetCapabilitiesAsync(CancellationToken.None)) .SupportedCapabilities; @@ -29,9 +33,10 @@ public async Task CapabilitiesAreReturned() [Test] public async Task OnlyKubernetesScriptServicesAreReturnedWhenRunningAsKubernetesAgent() { - Environment.SetEnvironmentVariable(KubernetesConfig.NamespaceVariableName, "ABC"); + var k8sDetection = Substitute.For(); + k8sDetection.IsRunningAsKubernetesAgent.Returns(true); - var capabilities = (await new CapabilitiesServiceV2() + var capabilities = (await new CapabilitiesServiceV2(k8sDetection) .GetCapabilitiesAsync(CancellationToken.None)) .SupportedCapabilities; @@ -39,8 +44,6 @@ public async Task OnlyKubernetesScriptServicesAreReturnedWhenRunningAsKubernetes capabilities.Count.Should().Be(2); capabilities.Should().NotContainMatch("IScriptService*"); - - Environment.SetEnvironmentVariable(KubernetesConfig.NamespaceVariableName, null); } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests/Configuration/ApplicationInstanceSelectorFixture.cs b/source/Octopus.Tentacle.Tests/Configuration/ApplicationInstanceSelectorFixture.cs index e34be43b1..cdf1e856b 100644 --- a/source/Octopus.Tentacle.Tests/Configuration/ApplicationInstanceSelectorFixture.cs +++ b/source/Octopus.Tentacle.Tests/Configuration/ApplicationInstanceSelectorFixture.cs @@ -188,13 +188,16 @@ ApplicationInstanceSelector CreateApplicationInstanceSelector(StartUpInstanceReq IApplicationConfigurationContributor[]? additionalConfigurations = null) { var log = Substitute.For(); + var kubernetesConfig = Substitute.For(); + var k8sDetection = Substitute.For(); return new ApplicationInstanceSelector(ApplicationName.Tentacle, applicationInstanceStore, instanceRequest ?? new StartUpDynamicInstanceRequest(), - additionalConfigurations ?? new IApplicationConfigurationContributor[0], - new Lazy(() => new ConfigMapKeyValueStore(Substitute.For(), Substitute.For())), + additionalConfigurations ?? Array.Empty(), + new Lazy(() => new ConfigMapKeyValueStore(kubernetesConfig,Substitute.For(), Substitute.For())), octopusFileSystem, - log); + log, + k8sDetection); } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesDirectoryInformationProviderFixture.cs b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesDirectoryInformationProviderFixture.cs index 5f89b8126..558c4cfbc 100644 --- a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesDirectoryInformationProviderFixture.cs +++ b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesDirectoryInformationProviderFixture.cs @@ -18,7 +18,7 @@ public class KubernetesDirectoryInformationProviderFixture { // Sizes const ulong Megabyte = 1000 * 1000; - + [Test] public void DuOutputParses() { @@ -30,10 +30,10 @@ public void DuOutputParses() x.ArgAt>(3).Invoke($"{usedSize}\t/octopus"); }); var memoryCache = new MemoryCache(new MemoryCacheOptions()); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(usedSize); } - + [Test] public void DuOutputParsesWithMultipleLines() { @@ -44,10 +44,10 @@ public void DuOutputParsesWithMultipleLines() { x.ArgAt>(3).Invoke($"500\t/octopus/extradir"); x.ArgAt>(3).Invoke($"{usedSize}\t/octopus"); - x.ArgAt>(3).Invoke($"{usedSize+1000}\tTotal"); + x.ArgAt>(3).Invoke($"{usedSize + 1000}\tTotal"); }); var memoryCache = new MemoryCache(new MemoryCacheOptions()); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(usedSize); } @@ -64,10 +64,10 @@ public void IfDuFailsWeStillGetData() }); spr.ReturnsForAll(1); var memoryCache = new MemoryCache(new MemoryCacheOptions()); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(usedSize); } - + [Test] public void IfDuFailsWeLogCorrectly() { @@ -81,16 +81,16 @@ public void IfDuFailsWeLogCorrectly() // stdout x.ArgAt>(3).Invoke("500\t/octopus"); x.ArgAt>(3).Invoke($"{usedSize}\t/octopus"); - + // stderr x.ArgAt>(4).Invoke("no permission for foo"); x.ArgAt>(4).Invoke("also no permission for bar"); }); spr.ReturnsForAll(1); var memoryCache = new MemoryCache(new MemoryCacheOptions()); - var sut = new KubernetesDirectoryInformationProvider(systemLog, spr, memoryCache); + var sut = new KubernetesDirectoryInformationProvider(systemLog, Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(usedSize); - + systemLog.GetLogsForCategory(LogCategory.Warning).Should().Contain("Could not reliably get disk space using du. Getting best approximation..."); systemLog.GetLogsForCategory(LogCategory.Info).Should().Contain($"Du stdout returned 500\t/octopus, {usedSize}\t/octopus"); systemLog.GetLogsForCategory(LogCategory.Info).Should().Contain("Du stderr returned no permission for foo, also no permission for bar"); @@ -102,10 +102,10 @@ public void IfDuFailsCompletelyReturnNull() var spr = Substitute.For(); spr.ReturnsForAll(1); var memoryCache = new MemoryCache(new MemoryCacheOptions()); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(null); } - + [Test] public void ReturnedValueShouldBeCached() { @@ -113,10 +113,10 @@ public void ReturnedValueShouldBeCached() spr.ReturnsForAll(1); var baseTime = DateTimeOffset.UtcNow; var clock = new TestClock(baseTime); - var memoryCache = new MemoryCache(new MemoryCacheOptions(){ Clock = clock}); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var memoryCache = new MemoryCache(new MemoryCacheOptions() { Clock = clock }); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(null); - + const ulong usedSize = 500 * Megabyte; spr.When(x => x.ExecuteCommand("du", "-s -B 1 /octopus", "/", Arg.Any>(), Arg.Any>())) .Do(x => @@ -128,7 +128,7 @@ public void ReturnedValueShouldBeCached() sut.GetPathUsedBytes("/octopus").Should().Be(null); } - + [Test] public void DuCacheExpiresAfter30Seconds() { @@ -136,10 +136,10 @@ public void DuCacheExpiresAfter30Seconds() spr.ReturnsForAll(1); var baseTime = DateTimeOffset.UtcNow; var clock = new TestClock(baseTime); - var memoryCache = new MemoryCache(new MemoryCacheOptions(){ Clock = clock}); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var memoryCache = new MemoryCache(new MemoryCacheOptions() { Clock = clock }); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(null); - + const ulong usedSize = 500 * Megabyte; spr.When(x => x.ExecuteCommand("du", "-s -B 1 /octopus", "/", Arg.Any>(), Arg.Any>())) .Do(x => @@ -149,10 +149,9 @@ public void DuCacheExpiresAfter30Seconds() }); clock.UtcNow = baseTime + TimeSpan.FromSeconds(30); - + sut.GetPathUsedBytes("/octopus").Should().Be(usedSize); } - } public class TestClock : ISystemClock diff --git a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesEventMonitorFixture.cs b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesEventMonitorFixture.cs index 8c980203a..5840cdaa5 100644 --- a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesEventMonitorFixture.cs +++ b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesEventMonitorFixture.cs @@ -51,6 +51,7 @@ public class KubernetesEventMonitorFixture readonly CancellationTokenSource tokenSource = new(); readonly DateTimeOffset testEpoch = DateTimeOffset.Now; readonly ISystemLog log = Substitute.For(); + readonly IKubernetesConfiguration kubernetesConfiguration = Substitute.For(); [Test] public async Task NoEntriesAreSentToMetricsWhenEventListIsEmpty() @@ -58,7 +59,7 @@ public async Task NoEntriesAreSentToMetricsWhenEventListIsEmpty() var agentMetrics = Substitute.For(); agentMetrics.GetLatestEventTimestamp(Arg.Any()).ReturnsForAnyArgs(testEpoch); var eventService = Substitute.For(); - var sut = new KubernetesEventMonitor(agentMetrics, eventService, "arbitraryNamespace", new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); + var sut = new KubernetesEventMonitor(kubernetesConfiguration, agentMetrics, eventService, new IEventMapper[] { new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper() }, log); await sut.CacheNewEvents(tokenSource.Token); @@ -71,7 +72,7 @@ public async Task NfsPodKillingEventsAreTrackedInMetrics() //Arrange var agentMetrics = new StubbedAgentMetrics(testEpoch); var eventService = Substitute.For(); - eventService.FetchAllEventsAsync(Arg.Any(), Arg.Any()).ReturnsForAnyArgs( + eventService.FetchAllEventsAsync(Arg.Any()).ReturnsForAnyArgs( new Corev1EventList(new List { new() @@ -85,7 +86,7 @@ public async Task NfsPodKillingEventsAreTrackedInMetrics() LastTimestamp = testEpoch.DateTime.AddMinutes(1) } })); - var sut = new KubernetesEventMonitor(agentMetrics, eventService, "arbitraryNamespace", new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); + var sut = new KubernetesEventMonitor(kubernetesConfiguration, agentMetrics, eventService, new IEventMapper[] { new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper() }, log); //Act await sut.CacheNewEvents(tokenSource.Token); @@ -104,7 +105,7 @@ public async Task NfsWatchDogEventsAreTrackedInMetrics() var podName = "octopus-script-123412341234.123412341234"; var agentMetrics = new StubbedAgentMetrics(testEpoch); var eventService = Substitute.For(); - eventService.FetchAllEventsAsync(Arg.Any(), Arg.Any()).ReturnsForAnyArgs( + eventService.FetchAllEventsAsync(Arg.Any()).ReturnsForAnyArgs( new Corev1EventList(new List { new() @@ -119,7 +120,7 @@ public async Task NfsWatchDogEventsAreTrackedInMetrics() } })); - var sut = new KubernetesEventMonitor(agentMetrics, eventService, "arbitraryNamespace", new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); + var sut = new KubernetesEventMonitor(kubernetesConfiguration, agentMetrics, eventService, new IEventMapper[] { new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper() }, log); //Act await sut.CacheNewEvents(tokenSource.Token); @@ -138,7 +139,7 @@ public async Task EventsOlderThanOrEqualToMetricsTimestampCursorAreNotAddedToMet var podName = "octopus-script-123412341234.123412341234"; var agentMetrics = new StubbedAgentMetrics(testEpoch); var eventService = Substitute.For(); - eventService.FetchAllEventsAsync(Arg.Any(), Arg.Any()).ReturnsForAnyArgs( + eventService.FetchAllEventsAsync(Arg.Any()).ReturnsForAnyArgs( new Corev1EventList(new List { new() @@ -152,11 +153,11 @@ public async Task EventsOlderThanOrEqualToMetricsTimestampCursorAreNotAddedToMet LastTimestamp = testEpoch.DateTime } })); - - var sut = new KubernetesEventMonitor(agentMetrics, eventService, "arbitraryNamespace", new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); + + var sut = new KubernetesEventMonitor(kubernetesConfiguration, agentMetrics, eventService, new IEventMapper[] { new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper() }, log); //Act await sut.CacheNewEvents(tokenSource.Token); - + //Assert agentMetrics.Events.Should().BeEquivalentTo(new Dictionary>>()); } @@ -168,7 +169,7 @@ public async Task NewestTimeStampInEventIsUsedToDetermineAgeAndAsMetricsValue() var podName = "octopus-script-123412341234.123412341234"; var agentMetrics = new StubbedAgentMetrics(testEpoch); var eventService = Substitute.For(); - eventService.FetchAllEventsAsync(Arg.Any(), Arg.Any()).ReturnsForAnyArgs( + eventService.FetchAllEventsAsync(Arg.Any()).ReturnsForAnyArgs( new Corev1EventList(new List { new() @@ -183,11 +184,11 @@ public async Task NewestTimeStampInEventIsUsedToDetermineAgeAndAsMetricsValue() EventTime = testEpoch.DateTime.AddMinutes(1) } })); - - var sut = new KubernetesEventMonitor(agentMetrics, eventService, "arbitraryNamespace", new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); + + var sut = new KubernetesEventMonitor(kubernetesConfiguration, agentMetrics, eventService, new IEventMapper[] { new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper() }, log); //Act await sut.CacheNewEvents(tokenSource.Token); - + //Assert // The event.EventTime is newest event Time stamp, and is larger than the last metric date (TestEpoch) as such // the event should factored into the metrics, and should report this latest time value. diff --git a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs index bc026e045..ef9410ab8 100644 --- a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs +++ b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs @@ -28,6 +28,7 @@ public class KubernetesOrphanedPodCleanerTests ITentacleScriptLogProvider scriptLogProvider; IScriptPodSinceTimeStore scriptPodSinceTimeStore; IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider; + IKubernetesConfiguration config; [SetUp] public void Setup() @@ -42,8 +43,12 @@ public void Setup() scriptPodLogEncryptionKeyProvider = Substitute.For(); monitor = Substitute.For(); scriptTicket = new ScriptTicket(Guid.NewGuid().ToString()); + + config = Substitute.For(); + config.ScriptPodConsideredOrphanedAfterTimeSpan.Returns(TimeSpan.FromMinutes(10)); + config.DisableAutomaticPodCleanup.Returns(false); - cleaner = new KubernetesOrphanedPodCleaner(monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore, scriptPodLogEncryptionKeyProvider); + cleaner = new KubernetesOrphanedPodCleaner(config, monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore, scriptPodLogEncryptionKeyProvider); overCutoff = cleaner.CompletedPodConsideredOrphanedAfterTimeSpan + 1.Minutes(); underCutoff = cleaner.CompletedPodConsideredOrphanedAfterTimeSpan - 1.Minutes(); @@ -149,7 +154,7 @@ public async Task OrphanedPodNotCleanedUpIfOnly9MinutesHavePassed() public async Task OrphanedPodNotCleanedUpIfPodCleanupIsDisabled() { //Arrange - Environment.SetEnvironmentVariable("OCTOPUS__K8STENTACLE__DISABLEAUTOPODCLEANUP", "true"); + config.DisableAutomaticPodCleanup.Returns(true); var pods = new List { CreatePod(TrackedScriptPodState.Succeeded(0, startTime)) @@ -172,10 +177,10 @@ public async Task OrphanedPodNotCleanedUpIfPodCleanupIsDisabled() public async Task EnvironmentVariableDictatesWhenPodsAreConsideredOrphaned(int checkAfterMinutes, bool shouldDelete) { //Arrange - Environment.SetEnvironmentVariable("OCTOPUS__K8STENTACLE__PODSCONSIDEREDORPHANEDAFTERMINUTES", "2"); + config.ScriptPodConsideredOrphanedAfterTimeSpan.Returns(TimeSpan.FromMinutes(2)); // We need to reinitialise the sut after changing the env var value - cleaner = new KubernetesOrphanedPodCleaner(monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore, scriptPodLogEncryptionKeyProvider); + cleaner = new KubernetesOrphanedPodCleaner(config, monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore, scriptPodLogEncryptionKeyProvider); var pods = new List { CreatePod(TrackedScriptPodState.Succeeded(0, startTime)) diff --git a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesPodContainerResolverTests.cs b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesPodContainerResolverTests.cs index 216938fdc..97b3a92bc 100644 --- a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesPodContainerResolverTests.cs +++ b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesPodContainerResolverTests.cs @@ -26,11 +26,13 @@ public class KubernetesPodContainerResolverTests }); readonly IToolsImageVersionMetadataProvider mockToolsImageVersionMetadataProvider = Substitute.For(); + IKubernetesConfiguration kubernetesConfig; [SetUp] public void Init() { mockToolsImageVersionMetadataProvider.TryGetVersionMetadata().Returns(testVersionMetadata); + kubernetesConfig = Substitute.For(); } [TestCase(30)] @@ -42,7 +44,7 @@ public async Task GetContainerImageForCluster_VersionMetadataExists_ClusterVersi var clusterService = Substitute.For(); clusterService.GetClusterVersion().Returns(new ClusterVersion(1, clusterMinorVersion)); - var podContainerResolver = new KubernetesPodContainerResolver(clusterService, mockToolsImageVersionMetadataProvider); + var podContainerResolver = new KubernetesPodContainerResolver(kubernetesConfig, clusterService, mockToolsImageVersionMetadataProvider); // Act var result = await podContainerResolver.GetContainerImageForCluster(); @@ -59,7 +61,7 @@ public async Task GetContainerImageForCluster_VersionMetadataExists_ClusterVersi var clusterService = Substitute.For(); clusterService.GetClusterVersion().Returns(new ClusterVersion(1, clusterMinorVersion)); - var podContainerResolver = new KubernetesPodContainerResolver(clusterService, mockToolsImageVersionMetadataProvider); + var podContainerResolver = new KubernetesPodContainerResolver(kubernetesConfig, clusterService, mockToolsImageVersionMetadataProvider); // Act var result = await podContainerResolver.GetContainerImageForCluster(); @@ -75,7 +77,7 @@ public async Task GetContainerImageForCluster_VersionMetadataExists_ClusterVersi var clusterService = Substitute.For(); clusterService.GetClusterVersion().Returns(new ClusterVersion(1, 31)); - var podContainerResolver = new KubernetesPodContainerResolver(clusterService, mockToolsImageVersionMetadataProvider); + var podContainerResolver = new KubernetesPodContainerResolver(kubernetesConfig, clusterService, mockToolsImageVersionMetadataProvider); // Act var result = await podContainerResolver.GetContainerImageForCluster(); @@ -91,7 +93,7 @@ public async Task GetContainerImageForCluster_VersionMetadataExists_ClusterVersi var clusterService = Substitute.For(); clusterService.GetClusterVersion().Returns(new ClusterVersion(1, 40)); - var podContainerResolver = new KubernetesPodContainerResolver(clusterService, mockToolsImageVersionMetadataProvider); + var podContainerResolver = new KubernetesPodContainerResolver(kubernetesConfig, clusterService, mockToolsImageVersionMetadataProvider); // Act var result = await podContainerResolver.GetContainerImageForCluster(); @@ -114,7 +116,7 @@ public async Task GetContainerImageForCluster_VersionMetadataNotFound_FallBackTo clusterService.GetClusterVersion().Returns(new ClusterVersion(1, clusterMinorVersion)); mockToolsImageVersionMetadataProvider.TryGetVersionMetadata().ReturnsNull(); - var podContainerResolver = new KubernetesPodContainerResolver(clusterService, mockToolsImageVersionMetadataProvider); + var podContainerResolver = new KubernetesPodContainerResolver(kubernetesConfig, clusterService, mockToolsImageVersionMetadataProvider); // Act var result = await podContainerResolver.GetContainerImageForCluster(); diff --git a/source/Octopus.Tentacle/Configuration/Crypto/LinuxGeneratedMachineKey.cs b/source/Octopus.Tentacle/Configuration/Crypto/LinuxGeneratedMachineKey.cs index b5324d983..6088e8ca4 100644 --- a/source/Octopus.Tentacle/Configuration/Crypto/LinuxGeneratedMachineKey.cs +++ b/source/Octopus.Tentacle/Configuration/Crypto/LinuxGeneratedMachineKey.cs @@ -2,6 +2,7 @@ using System.IO; using System.Security.Cryptography; using Octopus.Diagnostics; +using Octopus.Tentacle.Kubernetes; using Octopus.Tentacle.Util; using Octopus.Tentacle.Variables; @@ -13,7 +14,7 @@ public class LinuxGeneratedMachineKey: ICryptoKeyNixSource readonly IOctopusFileSystem fileSystem; static string FileName => - PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent + KubernetesAgentDetection.IsRunningAsKubernetesAgent //if we are running in K8S, we want to save the machine key to the home directory, which is likely on a network drives ? Path.Combine(Environment.GetEnvironmentVariable(EnvironmentVariables.TentacleHome)!, "machinekey") : "/etc/octopus/machinekey"; diff --git a/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceManager.cs b/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceManager.cs index ae5482e0f..a9f1a8aa6 100644 --- a/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceManager.cs +++ b/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceManager.cs @@ -11,6 +11,7 @@ class ApplicationInstanceManager : IApplicationInstanceManager readonly IOctopusFileSystem fileSystem; readonly ISystemLog log; readonly IApplicationInstanceStore instanceStore; + readonly IKubernetesAgentDetection kubernetesAgentDetection; readonly Lazy homeConfiguration; readonly ApplicationName applicationName; readonly StartUpInstanceRequest startUpInstanceRequest; @@ -21,6 +22,7 @@ public ApplicationInstanceManager( IOctopusFileSystem fileSystem, ISystemLog log, IApplicationInstanceStore instanceStore, + IKubernetesAgentDetection kubernetesAgentDetection, Lazy homeConfiguration) { this.applicationName = applicationName; @@ -28,6 +30,7 @@ public ApplicationInstanceManager( this.fileSystem = fileSystem; this.log = log; this.instanceStore = instanceStore; + this.kubernetesAgentDetection = kubernetesAgentDetection; this.homeConfiguration = homeConfiguration; } @@ -67,11 +70,11 @@ void WriteHomeDirectory(string homeDirectory) void EnsureConfigurationFileExists(string configurationFile, string homeDirectory) { - //Skip this step if we're running on Kubernetes - if (PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) return; + //Skip this step if we're running as a Kubernetes Agent + if (kubernetesAgentDetection.IsRunningAsKubernetesAgent) return; // Ensure we can write configuration file - string configurationDirectory = Path.GetDirectoryName(configurationFile) ?? homeDirectory; + var configurationDirectory = Path.GetDirectoryName(configurationFile) ?? homeDirectory; fileSystem.EnsureDirectoryExists(configurationDirectory); if (!fileSystem.FileExists(configurationFile)) { diff --git a/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceSelector.cs b/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceSelector.cs index 5a3765ae7..56f9724fe 100644 --- a/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceSelector.cs +++ b/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceSelector.cs @@ -15,6 +15,7 @@ class ApplicationInstanceSelector : IApplicationInstanceSelector readonly IApplicationConfigurationContributor[] instanceStrategies; readonly IOctopusFileSystem fileSystem; readonly ISystemLog log; + readonly IKubernetesAgentDetection kubernetesAgentDetection; readonly object @lock = new object(); ApplicationInstanceConfiguration? current; Lazy configMapStoreFactory; @@ -26,13 +27,15 @@ public ApplicationInstanceSelector( IApplicationConfigurationContributor[] instanceStrategies, Lazy configMapStoreFactory, IOctopusFileSystem fileSystem, - ISystemLog log) + ISystemLog log, + IKubernetesAgentDetection kubernetesAgentDetection) { this.applicationInstanceStore = applicationInstanceStore; this.startUpInstanceRequest = startUpInstanceRequest; this.instanceStrategies = instanceStrategies; this.fileSystem = fileSystem; this.log = log; + this.kubernetesAgentDetection = kubernetesAgentDetection; ApplicationName = applicationName; this.configMapStoreFactory = configMapStoreFactory; } @@ -78,10 +81,9 @@ ApplicationInstanceConfiguration LoadInstance() (IKeyValueStore, IWritableKeyValueStore) LoadConfigurationStore((string? instanceName, string? configurationpath) appInstance) { - if (appInstance is { instanceName: not null, configurationpath: null } && - PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) + if (appInstance is { instanceName: not null, configurationpath: null } && kubernetesAgentDetection.IsRunningAsKubernetesAgent) { - log.Verbose($"Loading configuration from ConfigMap for namespace {KubernetesConfig.Namespace}"); + log.Verbose($"Loading configuration from ConfigMap for namespace {kubernetesAgentDetection.Namespace}"); var configMapWritableStore = configMapStoreFactory.Value; return (ContributeAdditionalConfiguration(configMapWritableStore), configMapWritableStore); } diff --git a/source/Octopus.Tentacle/Kubernetes/Configuration/ConfigMapKeyValueStore.cs b/source/Octopus.Tentacle/Kubernetes/Configuration/ConfigMapKeyValueStore.cs index ab5c4a172..ac3348596 100644 --- a/source/Octopus.Tentacle/Kubernetes/Configuration/ConfigMapKeyValueStore.cs +++ b/source/Octopus.Tentacle/Kubernetes/Configuration/ConfigMapKeyValueStore.cs @@ -19,12 +19,12 @@ class ConfigMapKeyValueStore : IWritableKeyValueStore, IAggregatableKeyValueStor readonly Lazy configMap; IDictionary ConfigMapData => configMap.Value.Data ??= new Dictionary(); - public ConfigMapKeyValueStore(IKubernetesConfigMapService configMapService, IKubernetesMachineKeyEncryptor encryptor) + public ConfigMapKeyValueStore(IKubernetesConfiguration kubernetesConfiguration, IKubernetesConfigMapService configMapService, IKubernetesMachineKeyEncryptor encryptor) { this.configMapService = configMapService; this.encryptor = encryptor; configMap = new Lazy(() => configMapService.TryGet(Name, CancellationToken.None).GetAwaiter().GetResult() - ?? throw new InvalidOperationException($"Unable to retrieve Tentacle Configuration from config map for namespace {KubernetesConfig.Namespace}")); + ?? throw new InvalidOperationException($"Unable to retrieve Tentacle Configuration from config map for namespace {kubernetesConfiguration.Namespace}")); } public string? Get(string name, ProtectionLevel protectionLevel = ProtectionLevel.None) diff --git a/source/Octopus.Tentacle/Kubernetes/Crypto/KubernetesMachineEncryptionKeyProvider.cs b/source/Octopus.Tentacle/Kubernetes/Crypto/KubernetesMachineEncryptionKeyProvider.cs index 09d8c1a7f..e4ed25f02 100644 --- a/source/Octopus.Tentacle/Kubernetes/Crypto/KubernetesMachineEncryptionKeyProvider.cs +++ b/source/Octopus.Tentacle/Kubernetes/Crypto/KubernetesMachineEncryptionKeyProvider.cs @@ -20,13 +20,15 @@ public class KubernetesMachineEncryptionKeyProvider : IKubernetesMachineEncrypti const string MachineKeyName = "machine-key"; const string MachineIvName = "machine-iv"; + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesSecretService secretService; readonly ISystemLog log; V1Secret? secret; readonly SemaphoreSlim accessSemaphore; - public KubernetesMachineEncryptionKeyProvider(IKubernetesSecretService secretService, ISystemLog log) + public KubernetesMachineEncryptionKeyProvider(IKubernetesConfiguration kubernetesConfiguration, IKubernetesSecretService secretService, ISystemLog log) { + this.kubernetesConfiguration = kubernetesConfiguration; this.secretService = secretService; this.log = log; accessSemaphore = new SemaphoreSlim(1, 1); @@ -42,7 +44,7 @@ public KubernetesMachineEncryptionKeyProvider(IKubernetesSecretService secretSer secret ??= await secretService.TryGetSecretAsync(SecretName, CancellationToken.None); if (secret is null) - throw new InvalidOperationException($"Unable to retrieve MachineKey from secret for namespace {KubernetesConfig.Namespace}"); + throw new InvalidOperationException($"Unable to retrieve MachineKey from secret for namespace {kubernetesConfiguration.Namespace}"); var data = secret.Data; if (data is null || diff --git a/source/Octopus.Tentacle/Kubernetes/Diagnostics/PersistenceProvider.cs b/source/Octopus.Tentacle/Kubernetes/Diagnostics/PersistenceProvider.cs index 5784d06f0..47c7f4403 100644 --- a/source/Octopus.Tentacle/Kubernetes/Diagnostics/PersistenceProvider.cs +++ b/source/Octopus.Tentacle/Kubernetes/Diagnostics/PersistenceProvider.cs @@ -18,12 +18,14 @@ public class PersistenceProvider : IPersistenceProvider public delegate PersistenceProvider Factory(string configMapName); readonly string configMapName; + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesConfigMapService configMapService; - public PersistenceProvider(string configMapName, IKubernetesConfigMapService configMapService) + public PersistenceProvider(string configMapName, IKubernetesConfiguration kubernetesConfiguration, IKubernetesConfigMapService configMapService) { this.configMapService = configMapService; this.configMapName = configMapName; + this.kubernetesConfiguration = kubernetesConfiguration; } public async Task GetValue(string key, CancellationToken cancellationToken) @@ -35,7 +37,7 @@ public PersistenceProvider(string configMapName, IKubernetesConfigMapService con public async Task PersistValue(string key, string value, CancellationToken cancellationToken) { var configMapData = await LoadConfigMapData(cancellationToken); - if (configMapData is null) throw new InvalidOperationException($"Unable to retrieve Tentacle Configuration from config map for namespace {KubernetesConfig.Namespace}"); + if (configMapData is null) throw new InvalidOperationException($"Unable to retrieve Tentacle Configuration from config map for namespace {kubernetesConfiguration.Namespace}"); configMapData[key] = value; await configMapService.Patch(configMapName, configMapData, cancellationToken); diff --git a/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs b/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs new file mode 100644 index 000000000..4ab85cb96 --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Octopus.Tentacle.Util; + +namespace Octopus.Tentacle.Kubernetes +{ + public class EnvironmentKubernetesConfiguration : IKubernetesConfiguration + { + public static class VariableNames + { + const string EnvVarPrefix = "OCTOPUS__K8STENTACLE"; + + public static readonly string Namespace = $"{EnvVarPrefix}__NAMESPACE"; + + public static readonly string BootstrapRunnerExecutablePath = "BOOTSTRAPRUNNEREXECUTABLEPATH"; + + public static readonly string PersistentVolumeSizeBytes = $"{EnvVarPrefix}__PERSISTENTVOLUMETOTALBYTES"; + public static readonly string PersistentVolumeFreeBytes = $"{EnvVarPrefix}__PERSISTENTVOLUMEFREEBYTES"; + + public static readonly string HelmReleaseName = $"{EnvVarPrefix}__HELMRELEASENAME"; + public static readonly string HelmChartVersion = $"{EnvVarPrefix}__HELMCHARTVERSION"; + + public static readonly string ServerCommsAddress = "ServerCommsAddress"; + public static readonly string ServerCommsAddresses = "ServerCommsAddresses"; + + public static readonly string ScriptPodServiceAccountName = $"{EnvVarPrefix}__PODSERVICEACCOUNTNAME"; + public static readonly string ScriptPodImagePullSecretNames = $"{EnvVarPrefix}__PODIMAGEPULLSECRETNAMES"; + public static readonly string ScriptPodVolumeClaimName = $"{EnvVarPrefix}__PODVOLUMECLAIMNAME"; + + public static readonly string ScriptPodResourceJson = $"{EnvVarPrefix}__PODRESOURCEJSON"; + public static readonly string ScriptPodAffinityJson = $"{EnvVarPrefix}__PODAFFINITYJSON"; + public static readonly string ScriptPodTolerationsJson = $"{EnvVarPrefix}__PODTOLERATIONSJSON"; + public static readonly string ScriptPodSecurityContextJson = $"{EnvVarPrefix}__PODSECURITYCONTEXTJSON"; + + + public static readonly string ScriptPodContainerImage = $"{EnvVarPrefix}__SCRIPTPODIMAGE"; + public static readonly string ScriptPodContainerImageTag = $"{EnvVarPrefix}__SCRIPTPODIMAGETAG"; + public static readonly string ScriptPodPullPolicy = $"{EnvVarPrefix}__SCRIPTPODPULLPOLICY"; + + public static readonly string ScriptPodProxiesSecretName = $"{EnvVarPrefix}__PODPROXIESSECRETNAME"; + + public static readonly string NfsWatchdogImage = $"{EnvVarPrefix}__NFSWATCHDOGIMAGE"; + public static readonly string ScriptPodMonitorTimeout = $"{EnvVarPrefix}__PODMONITORTIMEOUT"; + public static readonly string ScriptPodConsideredOrphanedAfterTimeSpan = $"{EnvVarPrefix}__PODSCONSIDEREDORPHANEDAFTERMINUTES"; + + public static readonly string DisableAutomaticPodCleanup = $"{EnvVarPrefix}__DISABLEAUTOPODCLEANUP"; + public static readonly string DisablePodEventsInTaskLog = $"{EnvVarPrefix}__DISABLEPODEVENTSINTASKLOG"; + + public static readonly string PersistentVolumeSize = $"{EnvVarPrefix}__PERSISTENTVOLUMESIZE"; + + public static readonly string IsMetricsEnabled = $"{EnvVarPrefix}__ENABLEMETRICSCAPTURE"; + } + + public string Namespace => GetRequiredEnvVar(VariableNames.Namespace, "Unable to determine Kubernetes namespace."); + public string BootstrapRunnerExecutablePath => GetRequiredEnvVar(VariableNames.BootstrapRunnerExecutablePath, "Unable to determine Bootstrap Runner Executable Path"); + public string ScriptPodServiceAccountName => GetRequiredEnvVar(VariableNames.ScriptPodServiceAccountName, "Unable to determine Kubernetes Pod service account name."); + + public IEnumerable ScriptPodImagePullSecretNames => Environment.GetEnvironmentVariable(VariableNames.ScriptPodImagePullSecretNames) + ?.Split(',') + .Select(str => str.Trim()) + .WhereNotNullOrWhiteSpace() + .ToArray() ?? Array.Empty(); + + public string ScriptPodVolumeClaimName => GetRequiredEnvVar(VariableNames.ScriptPodVolumeClaimName, "Unable to determine Kubernetes Pod persistent volume claim name."); + public string? ScriptPodResourceJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodResourceJson); + + public string? ScriptPodContainerImage => Environment.GetEnvironmentVariable(VariableNames.ScriptPodContainerImage); + public string ScriptPodContainerImageTag => Environment.GetEnvironmentVariable(VariableNames.ScriptPodContainerImageTag) ?? "latest"; + public string? ScriptPodPullPolicy => Environment.GetEnvironmentVariable(VariableNames.ScriptPodPullPolicy); + public string? NfsWatchdogImage => Environment.GetEnvironmentVariable(VariableNames.NfsWatchdogImage); + + public string HelmReleaseName => GetRequiredEnvVar(VariableNames.HelmReleaseName, "Unable to determine Helm release name."); + public string HelmChartVersion => GetRequiredEnvVar(VariableNames.HelmChartVersion, "Unable to determine Helm chart version."); + + public string[] ServerCommsAddresses + { + get + { + var addresses = new List(); + if (Environment.GetEnvironmentVariable(VariableNames.ServerCommsAddress) is { Length: > 0 } addressString) + { + addresses.Add(addressString); + } + + if (Environment.GetEnvironmentVariable(VariableNames.ServerCommsAddresses) is { } addressesString) + { + addresses.AddRange(addressesString.Split(',').Where(a => !a.IsNullOrEmpty())); + } + + return addresses.ToArray(); + } + } + + public int? ScriptPodMonitorTimeoutSeconds => int.TryParse(Environment.GetEnvironmentVariable(VariableNames.ScriptPodMonitorTimeout), out var podMonitorTimeout) ? podMonitorTimeout : 10 * 60; //10min + public TimeSpan ScriptPodConsideredOrphanedAfterTimeSpan => TimeSpan.FromMinutes(int.TryParse(Environment.GetEnvironmentVariable(VariableNames.ScriptPodConsideredOrphanedAfterTimeSpan), out var podsConsideredOrphanedAfterTimeSpan) ? podsConsideredOrphanedAfterTimeSpan : 10); + public bool DisableAutomaticPodCleanup => bool.TryParse(Environment.GetEnvironmentVariable(VariableNames.DisableAutomaticPodCleanup), out var disableAutoCleanup) && disableAutoCleanup; + public bool DisablePodEventsInTaskLog => bool.TryParse(Environment.GetEnvironmentVariable(VariableNames.DisablePodEventsInTaskLog), out var disable) && disable; + public string PersistentVolumeSize => GetRequiredEnvVar(VariableNames.PersistentVolumeSize, "Unable to determine Persistent Volume Size"); + + public bool IsMetricsEnabled => !bool.TryParse(Environment.GetEnvironmentVariable(VariableNames.IsMetricsEnabled), out var enableMetrics) || enableMetrics; + public string? ScriptPodAffinityJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodAffinityJson); + public string? ScriptPodTolerationsJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodTolerationsJson); + public string? ScriptPodSecurityContextJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodSecurityContextJson); + public string? ScriptPodProxiesSecretName => Environment.GetEnvironmentVariable(VariableNames.ScriptPodProxiesSecretName); + + static string GetRequiredEnvVar(string variable, string errorMessage) + => Environment.GetEnvironmentVariable(variable) + ?? throw new InvalidOperationException($"{errorMessage} The environment variable '{variable}' must be defined with a non-null value."); + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs new file mode 100644 index 000000000..996c86ea8 --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs @@ -0,0 +1,43 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Octopus.Tentacle.Kubernetes +{ + public interface IKubernetesAgentDetection + { + /// + /// Indicates if the Tentacle is running inside a Kubernetes cluster as the Kubernetes Agent + /// + [MemberNotNullWhen(true, nameof(Namespace))] + bool IsRunningAsKubernetesAgent { get; } + + /// + /// The Kubernetes namespace the agent is running under, null if not running as a Kubernetes agent + /// + string? Namespace { get; } + } + + /// + /// Used for detection if the tentacle is running as part of the Kubernetes agent helm chart + /// Can be used with dependency injection via or statically (in scenarios where dependency injection isn't available) + /// + public class KubernetesAgentDetection : IKubernetesAgentDetection + { + /// + /// Indicates if the Tentacle is running inside a Kubernetes cluster as the Kubernetes Agent + /// + [MemberNotNullWhen(true, nameof(Namespace))] + public static bool IsRunningAsKubernetesAgent => !string.IsNullOrWhiteSpace(Namespace); + + /// + /// The Kubernetes namespace the Kubernetes Agent is running under, null if not running as a Kubernetes agent + /// + public static string? Namespace => Environment.GetEnvironmentVariable(EnvironmentKubernetesConfiguration.VariableNames.Namespace); + + /// + bool IKubernetesAgentDetection.IsRunningAsKubernetesAgent => IsRunningAsKubernetesAgent; + + /// + string? IKubernetesAgentDetection.Namespace => Namespace; + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs new file mode 100644 index 000000000..879d52043 --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; + +namespace Octopus.Tentacle.Kubernetes +{ + public interface IKubernetesConfiguration + { + string Namespace { get; } + string BootstrapRunnerExecutablePath { get; } + string ScriptPodServiceAccountName { get; } + IEnumerable ScriptPodImagePullSecretNames { get; } + string ScriptPodVolumeClaimName { get; } + string? ScriptPodResourceJson { get; } + string? ScriptPodAffinityJson { get; } + string? ScriptPodTolerationsJson { get; } + string? ScriptPodSecurityContextJson { get; } + string? ScriptPodProxiesSecretName { get; } + string? ScriptPodContainerImage { get; } + string ScriptPodContainerImageTag { get; } + string? ScriptPodPullPolicy { get; } + string? NfsWatchdogImage { get; } + string HelmReleaseName { get; } + string HelmChartVersion { get; } + string[] ServerCommsAddresses { get; } + int? ScriptPodMonitorTimeoutSeconds { get; } + TimeSpan ScriptPodConsideredOrphanedAfterTimeSpan { get; } + bool DisableAutomaticPodCleanup { get; } + bool DisablePodEventsInTaskLog { get; } + string PersistentVolumeSize { get; } + bool IsMetricsEnabled { get; } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs index af2435813..3b31fd752 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs @@ -14,8 +14,8 @@ public interface IKubernetesClusterService public class KubernetesClusterService : KubernetesService, IKubernetesClusterService { readonly AsyncLazy lazyVersion; - public KubernetesClusterService(IKubernetesClientConfigProvider configProvider, ISystemLog log) - : base(configProvider, log) + public KubernetesClusterService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) + : base(configProvider, kubernetesConfiguration, log) { //As the cluster version isn't going to change without restarting, we just cache the version in an AsyncLazy lazyVersion = new AsyncLazy(async () => diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs deleted file mode 100644 index 8c2586cd0..000000000 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs +++ /dev/null @@ -1,104 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Octopus.Tentacle.Util; - -namespace Octopus.Tentacle.Kubernetes -{ - public static class KubernetesConfig - { - const string ServerCommsAddressVariableName = "ServerCommsAddress"; - const string EnvVarPrefix = "OCTOPUS__K8STENTACLE"; - - public static string NamespaceVariableName => $"{EnvVarPrefix}__NAMESPACE"; - public static string Namespace => GetRequiredEnvVar(NamespaceVariableName, "Unable to determine Kubernetes namespace."); - public static string PodServiceAccountName => GetRequiredEnvVar($"{EnvVarPrefix}__PODSERVICEACCOUNTNAME", "Unable to determine Kubernetes Pod service account name."); - public static string PodVolumeClaimName => GetRequiredEnvVar($"{EnvVarPrefix}__PODVOLUMECLAIMNAME", "Unable to determine Kubernetes Pod persistent volume claim name."); - - public static int PodMonitorTimeoutSeconds => int.TryParse(Environment.GetEnvironmentVariable($"{EnvVarPrefix}__PODMONITORTIMEOUT"), out var podMonitorTimeout) ? podMonitorTimeout : 10 * 60; //10min - public static string NfsWatchdogImageVariableName => $"{EnvVarPrefix}__NFSWATCHDOGIMAGE"; - public static string? NfsWatchdogImage => Environment.GetEnvironmentVariable(NfsWatchdogImageVariableName); - - public static TimeSpan PodsConsideredOrphanedAfterTimeSpan => TimeSpan.FromMinutes(int.TryParse(Environment.GetEnvironmentVariable($"{EnvVarPrefix}__PODSCONSIDEREDORPHANEDAFTERMINUTES"), out var podsConsideredOrphanedAfterTimeSpan) ? podsConsideredOrphanedAfterTimeSpan : 10); - public static bool DisableAutomaticPodCleanup => bool.TryParse(Environment.GetEnvironmentVariable($"{EnvVarPrefix}__DISABLEAUTOPODCLEANUP"), out var disableAutoCleanup) && disableAutoCleanup; - public static bool DisablePodEventsInTaskLog => bool.TryParse(Environment.GetEnvironmentVariable($"{EnvVarPrefix}__DISABLEPODEVENTSINTASKLOG"), out var disable) && disable; - - public static string HelmReleaseNameVariableName => $"{EnvVarPrefix}__HELMRELEASENAME"; - public static string HelmReleaseName => GetRequiredEnvVar(HelmReleaseNameVariableName, "Unable to determine Helm release name."); - - public static string HelmChartVersionVariableName => $"{EnvVarPrefix}__HELMCHARTVERSION"; - public static string HelmChartVersion => GetRequiredEnvVar(HelmChartVersionVariableName, "Unable to determine Helm chart version."); - - public static string BootstrapRunnerExecutablePath => GetRequiredEnvVar("BOOTSTRAPRUNNEREXECUTABLEPATH", "Unable to determine Bootstrap Runner Executable Path"); - - public static string PersistentVolumeSizeVariableName => $"{EnvVarPrefix}__PERSISTENTVOLUMESIZE"; - public static string PersistentVolumeSize => GetRequiredEnvVar(PersistentVolumeSizeVariableName, "Unable to determine Persistent Volume Size"); - - public static string PersistentVolumeSizeBytesVariableName => $"{EnvVarPrefix}__PERSISTENTVOLUMETOTALBYTES"; - public static string PersistentVolumeFreeBytesVariableName => $"{EnvVarPrefix}__PERSISTENTVOLUMEFREEBYTES"; - - public const string ServerCommsAddressesVariableName = "ServerCommsAddresses"; - - public static string? ScriptPodContainerImage => Environment.GetEnvironmentVariable($"{EnvVarPrefix}__SCRIPTPODIMAGE"); - public static string ScriptPodContainerImageTag => Environment.GetEnvironmentVariable($"{EnvVarPrefix}__SCRIPTPODIMAGETAG") ?? "latest"; - public static string? ScriptPodPullPolicy => Environment.GetEnvironmentVariable($"{EnvVarPrefix}__SCRIPTPODPULLPOLICY"); - public static string? ScriptPodProxiesSecretName => Environment.GetEnvironmentVariable($"{EnvVarPrefix}__PODPROXIESSECRETNAME"); - - public static IEnumerable PodImagePullSecretNames => Environment.GetEnvironmentVariable($"{EnvVarPrefix}__PODIMAGEPULLSECRETNAMES") - ?.Split(',') - .Select(str => str.Trim()) - .WhereNotNullOrWhiteSpace() - .ToArray() ?? Array.Empty(); - - public static readonly string PodResourceJsonVariableName = $"{EnvVarPrefix}__PODRESOURCEJSON"; - public static string? PodResourceJson => Environment.GetEnvironmentVariable(PodResourceJsonVariableName); - - public static readonly string PodAffinityJsonVariableName = $"{EnvVarPrefix}__PODAFFINITYJSON"; - public static string? PodAffinityJson => Environment.GetEnvironmentVariable(PodAffinityJsonVariableName); - - public static readonly string PodTolerationsJsonVariableName = $"{EnvVarPrefix}__PODTOLERATIONSJSON"; - public static string? PodTolerationsJson => Environment.GetEnvironmentVariable(PodTolerationsJsonVariableName); - - public static readonly string PodSecurityContextJsonVariableName = $"{EnvVarPrefix}__PODSECURITYCONTEXTJSON"; - public static string? PodSecurityContextJson => Environment.GetEnvironmentVariable(PodSecurityContextJsonVariableName); - - public static string MetricsEnableVariableName => $"{EnvVarPrefix}__ENABLEMETRICSCAPTURE"; - - public static bool MetricsIsEnabled - { - get - { - var envContent = Environment.GetEnvironmentVariable(MetricsEnableVariableName); - if (bool.TryParse(envContent, out var result)) - { - return result; - } - - return true; - } - } - - public static string[] ServerCommsAddresses - { - get - { - var addresses = new List(); - if (Environment.GetEnvironmentVariable(ServerCommsAddressVariableName) is { Length: > 0 } addressString) - { - addresses.Add(addressString); - } - - if (Environment.GetEnvironmentVariable(ServerCommsAddressesVariableName) is { } addressesString) - { - addresses.AddRange(addressesString.Split(',').Where(a => !a.IsNullOrEmpty())); - } - - return addresses.ToArray(); - } - } - - static string GetRequiredEnvVar(string variable, string errorMessage) - => Environment.GetEnvironmentVariable(variable) - ?? throw new InvalidOperationException($"{errorMessage} The environment variable '{variable}' must be defined with a non-null value."); - } -} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesConfigMapService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesConfigMapService.cs index 19292a205..a7fbbeb29 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesConfigMapService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesConfigMapService.cs @@ -18,8 +18,8 @@ public interface IKubernetesConfigMapService public class KubernetesConfigMapService : KubernetesService, IKubernetesConfigMapService { - public KubernetesConfigMapService(IKubernetesClientConfigProvider configProvider, ISystemLog log) - : base(configProvider, log) + public KubernetesConfigMapService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) + : base(configProvider, kubernetesConfiguration, log) { } @@ -29,7 +29,7 @@ public KubernetesConfigMapService(IKubernetesClientConfigProvider configProvider { try { - return await Client.CoreV1.ReadNamespacedConfigMapAsync(name, KubernetesConfig.Namespace, cancellationToken: cancellationToken); + return await Client.CoreV1.ReadNamespacedConfigMapAsync(name, Namespace, cancellationToken: cancellationToken); } catch (HttpOperationException opException) when (opException.Response.StatusCode == HttpStatusCode.NotFound) @@ -49,7 +49,7 @@ public async Task Patch(string name, IDictionary da var configMapJson = KubernetesJson.Serialize(configMap); return await RetryPolicy.ExecuteAsync(async () => - await Client.CoreV1.PatchNamespacedConfigMapAsync(new V1Patch(configMapJson, V1Patch.PatchType.MergePatch), name, KubernetesConfig.Namespace, cancellationToken: cancellationToken)); + await Client.CoreV1.PatchNamespacedConfigMapAsync(new V1Patch(configMapJson, V1Patch.PatchType.MergePatch), name, Namespace, cancellationToken: cancellationToken)); } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesDirectoryInformationProvider.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesDirectoryInformationProvider.cs index 2a1a5be53..dac211ae5 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesDirectoryInformationProvider.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesDirectoryInformationProvider.cs @@ -17,6 +17,7 @@ public interface IKubernetesDirectoryInformationProvider public class KubernetesDirectoryInformationProvider : IKubernetesDirectoryInformationProvider { readonly ISystemLog log; + readonly IKubernetesConfiguration kubernetesConfiguration; readonly ISilentProcessRunner silentProcessRunner; readonly IMemoryCache directoryInformationCache; @@ -29,9 +30,10 @@ public class KubernetesDirectoryInformationProvider : IKubernetesDirectoryInform //No calls to `du` at all: 8min ea. static readonly TimeSpan CacheExpiry = TimeSpan.FromSeconds(30); - public KubernetesDirectoryInformationProvider(ISystemLog log, ISilentProcessRunner silentProcessRunner, IMemoryCache directoryInformationCache) + public KubernetesDirectoryInformationProvider(ISystemLog log, IKubernetesConfiguration kubernetesConfiguration, ISilentProcessRunner silentProcessRunner, IMemoryCache directoryInformationCache) { this.log = log; + this.kubernetesConfiguration = kubernetesConfiguration; this.silentProcessRunner = silentProcessRunner; this.directoryInformationCache = directoryInformationCache; } @@ -47,7 +49,7 @@ public KubernetesDirectoryInformationProvider(ISystemLog log, ISilentProcessRunn public ulong? GetPathTotalBytes() { - return KubernetesUtilities.GetResourceBytes(KubernetesConfig.PersistentVolumeSize); + return KubernetesUtilities.GetResourceBytes(kubernetesConfiguration.PersistentVolumeSize); } diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitor.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitor.cs index 7342416b8..7a3eff08f 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitor.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitor.cs @@ -18,27 +18,25 @@ public interface IKubernetesEventMonitor public class KubernetesEventMonitor : IKubernetesEventMonitor { - public delegate KubernetesEventMonitor Factory(string kubernetesNamespace); - + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesAgentMetrics agentMetrics; readonly IKubernetesEventService eventService; - readonly string kubernetesNamespace; readonly IEventMapper[] eventMappers; readonly ISystemLog log; - public KubernetesEventMonitor(IKubernetesAgentMetrics agentMetrics, IKubernetesEventService eventService, string kubernetesNamespace, IEventMapper[] eventMappers, ISystemLog log) + public KubernetesEventMonitor(IKubernetesConfiguration kubernetesConfiguration, IKubernetesAgentMetrics agentMetrics, IKubernetesEventService eventService, IEventMapper[] eventMappers, ISystemLog log) { + this.kubernetesConfiguration = kubernetesConfiguration; this.agentMetrics = agentMetrics; this.eventService = eventService; - this.kubernetesNamespace = kubernetesNamespace; this.eventMappers = eventMappers; this.log = log; } public async Task CacheNewEvents(CancellationToken cancellationToken) { - log.Info($"Parsing kubernetes event list for namespace {kubernetesNamespace}."); - var allEvents = await eventService.FetchAllEventsAsync(kubernetesNamespace, cancellationToken) ?? new Corev1EventList(new List()); + log.Info($"Parsing kubernetes event list for namespace {kubernetesConfiguration.Namespace}."); + var allEvents = await eventService.FetchAllEventsAsync(cancellationToken) ?? new Corev1EventList(new List()); var lastCachedEventTimeStamp = await agentMetrics.GetLatestEventTimestamp(cancellationToken); diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs index fe2d910a6..2c7454720 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs @@ -9,20 +9,20 @@ namespace Octopus.Tentacle.Kubernetes { public class KubernetesEventMonitorTask : BackgroundTask { - public delegate KubernetesEventMonitorTask Factory(IKubernetesEventMonitor eventMonitor); - readonly IKubernetesEventMonitor eventMonitor; readonly ISystemLog log; + readonly IKubernetesConfiguration kubernetesConfiguration; readonly TimeSpan taskInterval = TimeSpan.FromMinutes(10); - public KubernetesEventMonitorTask(ISystemLog log, IKubernetesEventMonitor eventMonitor) : base(log, TimeSpan.FromSeconds(30)) + public KubernetesEventMonitorTask(ISystemLog log, IKubernetesConfiguration kubernetesConfiguration, IKubernetesEventMonitor eventMonitor) : base(log, TimeSpan.FromSeconds(30)) { this.log = log; + this.kubernetesConfiguration = kubernetesConfiguration; this.eventMonitor = eventMonitor; } protected override async Task RunTask(CancellationToken cancellationToken) { - if (!KubernetesConfig.MetricsIsEnabled) + if (!kubernetesConfiguration.IsMetricsEnabled) { log.Info("Event monitoring for agent metrics is not enabled."); return; diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs index d42819530..fc1cdcf5c 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs @@ -1,4 +1,3 @@ -using System.Collections.Generic; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -6,29 +5,29 @@ using k8s.Autorest; using k8s.Models; using Octopus.Diagnostics; -using Octopus.Tentacle.Contracts; namespace Octopus.Tentacle.Kubernetes { public interface IKubernetesEventService { - Task FetchAllEventsAsync(string kubernetesNamespace, CancellationToken cancellationToken); - Task FetchAllEventsAsync(string kubernetesNamespace, string podName, CancellationToken cancellationToken); + Task FetchAllEventsAsync(CancellationToken cancellationToken); + Task FetchAllEventsAsync(string podName, CancellationToken cancellationToken); } public class KubernetesEventService : KubernetesService, IKubernetesEventService { - public KubernetesEventService(IKubernetesClientConfigProvider configProvider, ISystemLog log) : base(configProvider, log) + public KubernetesEventService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) + : base(configProvider, kubernetesConfiguration, log) { } - public async Task FetchAllEventsAsync(string kubernetesNamespace, CancellationToken cancellationToken) + public async Task FetchAllEventsAsync(CancellationToken cancellationToken) { return await RetryPolicy.ExecuteAsync(async () => { try { - return await Client.CoreV1.ListNamespacedEventAsync(kubernetesNamespace, cancellationToken: cancellationToken); + return await Client.CoreV1.ListNamespacedEventAsync(Namespace, cancellationToken: cancellationToken); } catch (HttpOperationException opException) when (opException.Response.StatusCode == HttpStatusCode.NotFound) @@ -38,7 +37,7 @@ public KubernetesEventService(IKubernetesClientConfigProvider configProvider, IS }); } - public async Task FetchAllEventsAsync(string kubernetesNamespace, string podName, CancellationToken cancellationToken) + public async Task FetchAllEventsAsync(string podName, CancellationToken cancellationToken) { return await RetryPolicy.ExecuteAsync(async () => { @@ -46,7 +45,7 @@ public KubernetesEventService(IKubernetesClientConfigProvider configProvider, IS { //get all the events for a specific script pod return await Client.CoreV1.ListNamespacedEventAsync( - kubernetesNamespace, + Namespace, fieldSelector: $"involvedObject.name={podName}", cancellationToken: cancellationToken); } diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs index da64cba9d..f1e55ed78 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs @@ -59,8 +59,7 @@ protected override void Load(ContainerBuilder builder) .As(); builder.RegisterType(); - builder.Register(ctx => ctx.Resolve().Invoke(KubernetesConfig.Namespace)) - .As(); + builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As(); diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs index 738c9558a..192ba4c13 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs @@ -17,6 +17,7 @@ public interface IKubernetesOrphanedPodCleaner public class KubernetesOrphanedPodCleaner : IKubernetesOrphanedPodCleaner { + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesPodStatusProvider podStatusProvider; readonly IKubernetesPodService podService; readonly ISystemLog log; @@ -26,9 +27,10 @@ public class KubernetesOrphanedPodCleaner : IKubernetesOrphanedPodCleaner readonly IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider; readonly TimeSpan initialDelay = TimeSpan.FromMinutes(1); - internal readonly TimeSpan CompletedPodConsideredOrphanedAfterTimeSpan = KubernetesConfig.PodsConsideredOrphanedAfterTimeSpan; + internal TimeSpan CompletedPodConsideredOrphanedAfterTimeSpan => kubernetesConfiguration.ScriptPodConsideredOrphanedAfterTimeSpan; public KubernetesOrphanedPodCleaner( + IKubernetesConfiguration kubernetesConfiguration, IKubernetesPodStatusProvider podStatusProvider, IKubernetesPodService podService, ISystemLog log, @@ -37,6 +39,7 @@ public KubernetesOrphanedPodCleaner( IScriptPodSinceTimeStore scriptPodSinceTimeStore, IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider) { + this.kubernetesConfiguration = kubernetesConfiguration; this.podStatusProvider = podStatusProvider; this.podService = podService; this.log = log; @@ -104,7 +107,7 @@ state.FinishedAt is not null && scriptPodSinceTimeStore.Delete(pod.ScriptTicket); scriptPodLogEncryptionKeyProvider.Delete(pod.ScriptTicket); - if (KubernetesConfig.DisableAutomaticPodCleanup) + if (kubernetesConfiguration.DisableAutomaticPodCleanup) { log.Verbose($"OrphanedPodCleaner: Not deleting orphaned pod {pod.ScriptTicket} as automatic cleanup is disabled"); continue; diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesPodContainerResolver.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesPodContainerResolver.cs index eab1790bd..08f88eebb 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesPodContainerResolver.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesPodContainerResolver.cs @@ -13,11 +13,13 @@ public interface IKubernetesPodContainerResolver public class KubernetesPodContainerResolver : IKubernetesPodContainerResolver { + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesClusterService clusterService; readonly IToolsImageVersionMetadataProvider imageVersionMetadataProvider; - public KubernetesPodContainerResolver(IKubernetesClusterService clusterService, IToolsImageVersionMetadataProvider imageVersionMetadataProvider) + public KubernetesPodContainerResolver(IKubernetesConfiguration kubernetesConfiguration, IKubernetesClusterService clusterService, IToolsImageVersionMetadataProvider imageVersionMetadataProvider) { + this.kubernetesConfiguration = kubernetesConfiguration; this.clusterService = clusterService; this.imageVersionMetadataProvider = imageVersionMetadataProvider; } @@ -36,13 +38,13 @@ public KubernetesPodContainerResolver(IKubernetesClusterService clusterService, public async Task GetContainerImageForCluster() { - var imageRepository = KubernetesConfig.ScriptPodContainerImage; + var imageRepository = kubernetesConfiguration.ScriptPodContainerImage; if (imageRepository.IsNullOrEmpty()) { return await GetAgentToolsContainerImage(); } - var imageTag = KubernetesConfig.ScriptPodContainerImageTag; + var imageTag = kubernetesConfiguration.ScriptPodContainerImageTag; return $"{imageRepository}:{imageTag}"; } diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesPodLogService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesPodLogService.cs index e8d331344..7d8ac165b 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesPodLogService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesPodLogService.cs @@ -28,13 +28,14 @@ class KubernetesPodLogService : KubernetesService, IKubernetesPodLogService public KubernetesPodLogService( IKubernetesClientConfigProvider configProvider, + IKubernetesConfiguration kubernetesConfiguration, IKubernetesPodMonitor podMonitor, ITentacleScriptLogProvider scriptLogProvider, IScriptPodSinceTimeStore scriptPodSinceTimeStore, IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider, IKubernetesEventService eventService, ISystemLog log) - : base(configProvider, log) + : base(configProvider, kubernetesConfiguration,log) { this.podMonitor = podMonitor; this.scriptLogProvider = scriptLogProvider; @@ -130,14 +131,14 @@ public KubernetesPodLogService( async Task> GetPodEvents(ScriptTicket scriptTicket, string podName, CancellationToken cancellationToken) { //if we don't want to write pod events to the task log, don't do anything - if (KubernetesConfig.DisablePodEventsInTaskLog) + if (KubernetesConfiguration.DisablePodEventsInTaskLog) { return Array.Empty(); } var sinceTime = scriptPodSinceTimeStore.GetPodEventsSinceTime(scriptTicket); - var allEvents = await eventService.FetchAllEventsAsync(KubernetesConfig.Namespace, podName, cancellationToken); + var allEvents = await eventService.FetchAllEventsAsync(podName, cancellationToken); if (allEvents is null) { return Array.Empty(); @@ -188,7 +189,7 @@ async Task> GetPodEvents(ScriptTicket scriptTicket, s { try { - return await Client.GetNamespacedPodLogsAsync(podName, KubernetesConfig.Namespace, podName, sinceTime, cancellationToken: cancellationToken); + return await Client.GetNamespacedPodLogsAsync(podName, Namespace, podName, sinceTime, cancellationToken: cancellationToken); } catch (HttpOperationException ex) { diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs index c1f3962b5..508552df1 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs @@ -20,14 +20,14 @@ public interface IKubernetesPodService public class KubernetesPodService : KubernetesService, IKubernetesPodService { - public KubernetesPodService(IKubernetesClientConfigProvider configProvider, ISystemLog log) - : base(configProvider, log) + public KubernetesPodService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) + : base(configProvider, kubernetesConfiguration,log) { } public async Task ListAllPods(CancellationToken cancellationToken) { - return await Client.ListNamespacedPodAsync(KubernetesConfig.Namespace, + return await Client.ListNamespacedPodAsync(Namespace, labelSelector: OctopusLabels.ScriptTicketId, cancellationToken: cancellationToken); } @@ -35,11 +35,11 @@ public async Task ListAllPods(CancellationToken cancellationToken) public async Task WatchAllPods(string initialResourceVersion, Func onChange, Action onError, CancellationToken cancellationToken) { using var response = Client.CoreV1.ListNamespacedPodWithHttpMessagesAsync( - KubernetesConfig.Namespace, + Namespace, labelSelector: OctopusLabels.ScriptTicketId, resourceVersion: initialResourceVersion, watch: true, - timeoutSeconds: KubernetesConfig.PodMonitorTimeoutSeconds, + timeoutSeconds: KubernetesConfiguration.ScriptPodMonitorTimeoutSeconds, cancellationToken: cancellationToken); var watchErrorCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); @@ -78,7 +78,7 @@ public async Task WatchAllPods(string initialResourceVersion, Func Create(V1Pod pod, CancellationToken cancellationToken) { AddStandardMetadata(pod); - return await Client.CreateNamespacedPodAsync(pod, KubernetesConfig.Namespace, cancellationToken: cancellationToken); + return await Client.CreateNamespacedPodAsync(pod, Namespace, cancellationToken: cancellationToken); } public async Task DeleteIfExists(ScriptTicket scriptTicket, CancellationToken cancellationToken) @@ -88,6 +88,6 @@ public async Task DeleteIfExists(ScriptTicket scriptTicket, TimeSpan gracePeriod => await DeleteIfExistsInternal(scriptTicket, (long)Math.Floor(gracePeriod.TotalSeconds), cancellationToken); async Task DeleteIfExistsInternal(ScriptTicket scriptTicket, long? gracePeriodSeconds, CancellationToken cancellationToken) - => await TryExecuteAsync(async () => await Client.DeleteNamespacedPodAsync(scriptTicket.ToKubernetesScriptPodName(), KubernetesConfig.Namespace, new V1DeleteOptions(gracePeriodSeconds: gracePeriodSeconds), cancellationToken: cancellationToken)); + => await TryExecuteAsync(async () => await Client.DeleteNamespacedPodAsync(scriptTicket.ToKubernetesScriptPodName(), Namespace, new V1DeleteOptions(gracePeriodSeconds: gracePeriodSeconds), cancellationToken: cancellationToken)); } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs index 2ed5ea06e..f41405506 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs @@ -21,6 +21,7 @@ public class KubernetesRawScriptPodCreator : KubernetesScriptPodCreator, IKubern readonly IKubernetesPodContainerResolver containerResolver; public KubernetesRawScriptPodCreator( + IKubernetesConfiguration kubernetesConfiguration, IKubernetesPodService podService, IKubernetesPodMonitor podMonitor, IKubernetesSecretService secretService, @@ -31,7 +32,7 @@ public KubernetesRawScriptPodCreator( IHomeConfiguration homeConfiguration, KubernetesPhysicalFileSystem kubernetesPhysicalFileSystem, IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider) - : base(podService, podMonitor, secretService, containerResolver, appInstanceSelector, log, scriptLogProvider, homeConfiguration, kubernetesPhysicalFileSystem, scriptPodLogEncryptionKeyProvider) + : base(kubernetesConfiguration, podService, podMonitor, secretService, containerResolver, appInstanceSelector, log, scriptLogProvider, homeConfiguration, kubernetesPhysicalFileSystem, scriptPodLogEncryptionKeyProvider) { this.containerResolver = containerResolver; } @@ -42,7 +43,7 @@ protected override async Task> CreateInitContainers(StartKube { Name = $"{podName}-init", Image = command.PodImageConfiguration?.Image ?? await containerResolver.GetContainerImageForCluster(), - ImagePullPolicy = KubernetesConfig.ScriptPodPullPolicy, + ImagePullPolicy = KubernetesConfiguration.ScriptPodPullPolicy, Command = new List { "sh", "-c", GetInitExecutionScript("/nfs-mount", homeDir, workspacePath) }, VolumeMounts = new List { new("/nfs-mount", "init-nfs-volume"), new(homeDir, "tentacle-home") }, Resources = new V1ResourceRequirements @@ -80,7 +81,7 @@ protected override IList CreateVolumes(StartKubernetesScriptCommandV1 Name = "init-nfs-volume", PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource { - ClaimName = KubernetesConfig.PodVolumeClaimName + ClaimName = KubernetesConfiguration.ScriptPodVolumeClaimName } }, CreateAgentUpgradeSecretVolume(), diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs index 761c713db..d8867b3d5 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs @@ -38,8 +38,11 @@ public class KubernetesScriptPodCreator : IKubernetesScriptPodCreator readonly IHomeConfiguration homeConfiguration; readonly KubernetesPhysicalFileSystem kubernetesPhysicalFileSystem; readonly IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider; + + protected IKubernetesConfiguration KubernetesConfiguration { get; } public KubernetesScriptPodCreator( + IKubernetesConfiguration kubernetesConfiguration, IKubernetesPodService podService, IKubernetesPodMonitor podMonitor, IKubernetesSecretService secretService, @@ -51,6 +54,7 @@ public KubernetesScriptPodCreator( KubernetesPhysicalFileSystem kubernetesPhysicalFileSystem, IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider) { + KubernetesConfiguration = kubernetesConfiguration; this.podService = podService; this.podMonitor = podMonitor; this.secretService = secretService; @@ -128,7 +132,7 @@ public async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorks Metadata = new V1ObjectMeta { Name = secretName, - NamespaceProperty = KubernetesConfig.Namespace + NamespaceProperty = KubernetesConfiguration.Namespace }, Data = new Dictionary { @@ -172,19 +176,19 @@ async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorkspace wo LogVerboseToBothLogs($"Creating Kubernetes Pod '{podName}'.", tentacleScriptLog); - workspace.CopyFile(KubernetesConfig.BootstrapRunnerExecutablePath, "bootstrapRunner", true); + workspace.CopyFile(KubernetesConfiguration.BootstrapRunnerExecutablePath, "bootstrapRunner", true); var scriptName = Path.GetFileName(workspace.BootstrapScriptFilePath); var workspacePath = Path.Combine("Work", workspace.ScriptTicket.TaskId); var serviceAccountName = !string.IsNullOrWhiteSpace(command.ScriptPodServiceAccountName) ? command.ScriptPodServiceAccountName - : KubernetesConfig.PodServiceAccountName; + : KubernetesConfiguration.ScriptPodServiceAccountName; // image pull secrets may have been defined in the helm chart (e.g. to avoid docker hub rate limiting) // we put any specified secret name first so it's resolved first var imagePullSecretNames = new[] { imagePullSecretName } - .Concat(KubernetesConfig.PodImagePullSecretNames) + .Concat(KubernetesConfiguration.ScriptPodImagePullSecretNames) .WhereNotNull() .Select(secretName => new V1LocalObjectReference(secretName)) .ToList(); @@ -194,7 +198,7 @@ async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorkspace wo Metadata = new V1ObjectMeta { Name = podName, - NamespaceProperty = KubernetesConfig.Namespace, + NamespaceProperty = KubernetesConfiguration.Namespace, Labels = new Dictionary { ["octopus.com/serverTaskId"] = command.TaskId, @@ -245,7 +249,7 @@ protected virtual IList CreateVolumes(StartKubernetesScriptCommandV1 c Name = "tentacle-home", PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource { - ClaimName = KubernetesConfig.PodVolumeClaimName + ClaimName = KubernetesConfiguration.ScriptPodVolumeClaimName } }, CreateAgentUpgradeSecretVolume(), @@ -295,14 +299,14 @@ protected async Task CreateScriptContainer(StartKubernetesScriptCom var envFrom = new List(); //if there is a scrip pod proxies defined - if (!string.IsNullOrWhiteSpace(KubernetesConfig.ScriptPodProxiesSecretName)) + if (!string.IsNullOrWhiteSpace(KubernetesConfiguration.ScriptPodProxiesSecretName)) { //add sourcing environment variables from the secret envFrom.Add(new V1EnvFromSource { SecretRef = new V1SecretEnvSource { - Name = KubernetesConfig.ScriptPodProxiesSecretName + Name = KubernetesConfiguration.ScriptPodProxiesSecretName } }); } @@ -311,7 +315,7 @@ protected async Task CreateScriptContainer(StartKubernetesScriptCom { Name = podName, Image = command.PodImageConfiguration?.Image ?? await containerResolver.GetContainerImageForCluster(), - ImagePullPolicy = KubernetesConfig.ScriptPodPullPolicy, + ImagePullPolicy = KubernetesConfiguration.ScriptPodPullPolicy, Command = new List { "sh" }, Args = new List { @@ -326,12 +330,12 @@ protected async Task CreateScriptContainer(StartKubernetesScriptCom }, Env = new List { - new(KubernetesConfig.NamespaceVariableName, KubernetesConfig.Namespace), - new(KubernetesConfig.HelmReleaseNameVariableName, KubernetesConfig.HelmReleaseName), - new(KubernetesConfig.HelmChartVersionVariableName, KubernetesConfig.HelmChartVersion), - new(KubernetesConfig.ServerCommsAddressesVariableName, string.Join(",", KubernetesConfig.ServerCommsAddresses)), - new(KubernetesConfig.PersistentVolumeFreeBytesVariableName, spaceInformation?.freeSpaceBytes.ToString()), - new(KubernetesConfig.PersistentVolumeSizeBytesVariableName, spaceInformation?.totalSpaceBytes.ToString()), + new(EnvironmentKubernetesConfiguration.VariableNames.Namespace, KubernetesConfiguration.Namespace), + new(EnvironmentKubernetesConfiguration.VariableNames.HelmReleaseName, KubernetesConfiguration.HelmReleaseName), + new(EnvironmentKubernetesConfiguration.VariableNames.HelmChartVersion, KubernetesConfiguration.HelmChartVersion), + new(EnvironmentKubernetesConfiguration.VariableNames.ServerCommsAddresses, string.Join(",", KubernetesConfiguration.ServerCommsAddresses)), + new(EnvironmentKubernetesConfiguration.VariableNames.PersistentVolumeFreeBytes, spaceInformation?.freeSpaceBytes.ToString()), + new(EnvironmentKubernetesConfiguration.VariableNames.PersistentVolumeSizeBytes, spaceInformation?.totalSpaceBytes.ToString()), new(EnvironmentVariables.TentacleHome, homeDir), new(EnvironmentVariables.TentacleInstanceName, appInstanceSelector.Current.InstanceName), new(EnvironmentVariables.TentacleVersion, Environment.GetEnvironmentVariable(EnvironmentVariables.TentacleVersion)), @@ -347,7 +351,7 @@ protected async Task CreateScriptContainer(StartKubernetesScriptCom V1ResourceRequirements GetScriptPodResourceRequirements(InMemoryTentacleScriptLog tentacleScriptLog) { - var json = KubernetesConfig.PodResourceJson; + var json = KubernetesConfiguration.ScriptPodResourceJson; if (!string.IsNullOrWhiteSpace(json)) { try @@ -356,7 +360,7 @@ V1ResourceRequirements GetScriptPodResourceRequirements(InMemoryTentacleScriptLo } catch (Exception e) { - var message = $"Failed to deserialize env.{KubernetesConfig.PodResourceJsonVariableName} into valid pod resource requirements.{Environment.NewLine}JSON value: {json}{Environment.NewLine}Using default resource requests for script pod."; + var message = $"Failed to deserialize env.{EnvironmentKubernetesConfiguration.VariableNames.ScriptPodResourceJson} into valid pod resource requirements.{Environment.NewLine}JSON value: {json}{Environment.NewLine}Using default resource requests for script pod."; //if we can't parse the JSON, fall back to the defaults below and warn the user log.WarnFormat(e, message); //write a verbose message to the script log. @@ -378,8 +382,8 @@ V1ResourceRequirements GetScriptPodResourceRequirements(InMemoryTentacleScriptLo V1Affinity ParseScriptPodAffinity(InMemoryTentacleScriptLog tentacleScriptLog) => ParseScriptPodJson( tentacleScriptLog, - KubernetesConfig.PodAffinityJson, - KubernetesConfig.PodAffinityJsonVariableName, + KubernetesConfiguration.ScriptPodAffinityJson, + EnvironmentKubernetesConfiguration.VariableNames.ScriptPodAffinityJson, "pod affinity", //we default to running on linux/arm64 and linux/amd64 nodes new V1Affinity(new V1NodeAffinity(requiredDuringSchedulingIgnoredDuringExecution: new V1NodeSelector(new List @@ -394,15 +398,15 @@ V1Affinity ParseScriptPodAffinity(InMemoryTentacleScriptLog tentacleScriptLog) List? ParseScriptPodTolerations(InMemoryTentacleScriptLog tentacleScriptLog) => ParseScriptPodJson>( tentacleScriptLog, - KubernetesConfig.PodTolerationsJson, - KubernetesConfig.PodTolerationsJsonVariableName, + KubernetesConfiguration.ScriptPodTolerationsJson, + EnvironmentKubernetesConfiguration.VariableNames.ScriptPodTolerationsJson, "pod tolerations"); V1PodSecurityContext? ParseScriptPodSecurityContext(InMemoryTentacleScriptLog tentacleScriptLog) => ParseScriptPodJson( tentacleScriptLog, - KubernetesConfig.PodSecurityContextJson, - KubernetesConfig.PodSecurityContextJsonVariableName, + KubernetesConfiguration.ScriptPodSecurityContextJson, + EnvironmentKubernetesConfiguration.VariableNames.ScriptPodSecurityContextJson, "pod security context"); [return: NotNullIfNotNull("defaultValue")] @@ -430,9 +434,9 @@ V1Affinity ParseScriptPodAffinity(InMemoryTentacleScriptLog tentacleScriptLog) return defaultValue; } - static V1Container? CreateWatchdogContainer(string homeDir) + V1Container? CreateWatchdogContainer(string homeDir) { - if (KubernetesConfig.NfsWatchdogImage is null) + if (KubernetesConfiguration.NfsWatchdogImage is null) { return null; } @@ -440,7 +444,7 @@ V1Affinity ParseScriptPodAffinity(InMemoryTentacleScriptLog tentacleScriptLog) return new V1Container { Name = "nfs-watchdog", - Image = KubernetesConfig.NfsWatchdogImage, + Image = KubernetesConfiguration.NfsWatchdogImage, VolumeMounts = new List { new(homeDir, "tentacle-home"), diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesSecretService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesSecretService.cs index 866c77f40..7574916a8 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesSecretService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesSecretService.cs @@ -19,8 +19,8 @@ public interface IKubernetesSecretService public class KubernetesSecretService : KubernetesService, IKubernetesSecretService { - public KubernetesSecretService(IKubernetesClientConfigProvider configProvider, ISystemLog log) - : base(configProvider, log) + public KubernetesSecretService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) + : base(configProvider,kubernetesConfiguration, log) { } @@ -30,7 +30,7 @@ public KubernetesSecretService(IKubernetesClientConfigProvider configProvider, I { try { - return await Client.ReadNamespacedSecretAsync(name, KubernetesConfig.Namespace, cancellationToken: cancellationToken); + return await Client.ReadNamespacedSecretAsync(name, Namespace, cancellationToken: cancellationToken); } catch (HttpOperationException opException) when (opException.Response.StatusCode == HttpStatusCode.NotFound) @@ -45,7 +45,7 @@ public async Task CreateSecretAsync(V1Secret secret, CancellationToken cancellat AddStandardMetadata(secret); //We only want to retry read/modify operations for now (since they are idempotent) - await Client.CreateNamespacedSecretAsync(secret, KubernetesConfig.Namespace, cancellationToken: cancellationToken); + await Client.CreateNamespacedSecretAsync(secret, Namespace, cancellationToken: cancellationToken); } public async Task UpdateSecretDataAsync(string secretName, IDictionary secretData, CancellationToken cancellationToken) @@ -58,7 +58,7 @@ public async Task UpdateSecretDataAsync(string secretName, IDictionary var patchYaml = KubernetesJson.Serialize(patchSecret); return await RetryPolicy.ExecuteAsync(async () => - await Client.PatchNamespacedSecretAsync(new V1Patch(patchYaml, V1Patch.PatchType.MergePatch), secretName, KubernetesConfig.Namespace, cancellationToken: cancellationToken)); + await Client.PatchNamespacedSecretAsync(new V1Patch(patchYaml, V1Patch.PatchType.MergePatch), secretName, Namespace, cancellationToken: cancellationToken)); } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesService.cs index 27614adee..8a67e0119 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesService.cs @@ -20,9 +20,14 @@ public abstract class KubernetesService protected ISystemLog Log { get; } protected AsyncRetryPolicy RetryPolicy { get; } protected k8sClient Client { get; } + + protected IKubernetesConfiguration KubernetesConfiguration { get; } + + protected string Namespace => KubernetesConfiguration.Namespace; - protected KubernetesService(IKubernetesClientConfigProvider configProvider, ISystemLog log) + protected KubernetesService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) { + KubernetesConfiguration = kubernetesConfiguration; Log = log; Client = new k8sClient(configProvider.Get()); RetryPolicy = Policy.Handle().WaitAndRetryAsync(5, @@ -37,12 +42,12 @@ protected KubernetesService(IKubernetesClientConfigProvider configProvider, ISys protected void AddStandardMetadata(IKubernetesObject k8sObject) { //Everything should be in the main namespace - k8sObject.Metadata.NamespaceProperty = KubernetesConfig.Namespace; + k8sObject.Metadata.NamespaceProperty = Namespace; - //Add helm specific metadata so it's removed if the helm release is uninstalled + //Add helm specific metadata, so it's removed if the helm release is uninstalled k8sObject.Metadata.Annotations ??= new Dictionary(); - k8sObject.Metadata.Annotations["meta.helm.sh/release-name"] = KubernetesConfig.HelmReleaseName; - k8sObject.Metadata.Annotations["meta.helm.sh/release-namespace"] = KubernetesConfig.Namespace; + k8sObject.Metadata.Annotations["meta.helm.sh/release-name"] = KubernetesConfiguration.HelmReleaseName; + k8sObject.Metadata.Annotations["meta.helm.sh/release-namespace"] = Namespace; k8sObject.Metadata.Labels ??= new Dictionary(); k8sObject.Metadata.Labels["app.kubernetes.io/managed-by"] = "Helm"; diff --git a/source/Octopus.Tentacle/NullableReferenceTypeAttributes.cs b/source/Octopus.Tentacle/NullableReferenceTypeAttributes.cs index 004ec67b0..73d8279c6 100644 --- a/source/Octopus.Tentacle/NullableReferenceTypeAttributes.cs +++ b/source/Octopus.Tentacle/NullableReferenceTypeAttributes.cs @@ -5,7 +5,7 @@ /// These attributes replicate the ones from System.Diagnostics.CodeAnalysis, and are here so we can still compile against the older frameworks. /// -namespace Octopus.Tentacle +namespace System.Diagnostics.CodeAnalysis { [AttributeUsage(AttributeTargets.Property | AttributeTargets.Parameter | AttributeTargets.ReturnValue, AllowMultiple = true)] public sealed class NotNullIfNotNullAttribute : Attribute @@ -63,5 +63,12 @@ public sealed class MemberNotNullAttribute : Attribute /// Gets field or property member names. public string[] Members { get; } } + + [AttributeUsage(AttributeTargets.Method | AttributeTargets.Property, AllowMultiple = true)] + public sealed class MemberNotNullWhenAttribute : Attribute + { + public MemberNotNullWhenAttribute(bool returnValue, params string[] members) { } + public MemberNotNullWhenAttribute(bool returnValue, string member) { } + } } #endif \ No newline at end of file diff --git a/source/Octopus.Tentacle/Program.cs b/source/Octopus.Tentacle/Program.cs index a9a05a74f..ef470cf96 100644 --- a/source/Octopus.Tentacle/Program.cs +++ b/source/Octopus.Tentacle/Program.cs @@ -58,7 +58,10 @@ public override IContainer BuildContainer(StartUpInstanceRequest startUpInstance builder.RegisterModule(new VersioningModule(GetType().Assembly)); builder.RegisterModule(new MaintenanceModule()); - if (PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) + //register kubernetes agent detection + builder.RegisterType().As().SingleInstance(); + builder.RegisterType().As().SingleInstance(); + if (KubernetesAgentDetection.IsRunningAsKubernetesAgent) { builder.RegisterModule(); } diff --git a/source/Octopus.Tentacle/Services/Capabilities/CapabilitiesServiceV2.cs b/source/Octopus.Tentacle/Services/Capabilities/CapabilitiesServiceV2.cs index 6ab67d9b3..d138ab40d 100644 --- a/source/Octopus.Tentacle/Services/Capabilities/CapabilitiesServiceV2.cs +++ b/source/Octopus.Tentacle/Services/Capabilities/CapabilitiesServiceV2.cs @@ -5,6 +5,7 @@ using Octopus.Tentacle.Contracts.Capabilities; using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1; using Octopus.Tentacle.Contracts.ScriptServiceV2; +using Octopus.Tentacle.Kubernetes; using Octopus.Tentacle.Util; namespace Octopus.Tentacle.Services.Capabilities @@ -12,12 +13,19 @@ namespace Octopus.Tentacle.Services.Capabilities [Service(typeof(ICapabilitiesServiceV2))] public class CapabilitiesServiceV2 : IAsyncCapabilitiesServiceV2 { + readonly IKubernetesAgentDetection kubernetesAgentDetection; + + public CapabilitiesServiceV2(IKubernetesAgentDetection kubernetesAgentDetection) + { + this.kubernetesAgentDetection = kubernetesAgentDetection; + } + public async Task GetCapabilitiesAsync(CancellationToken cancellationToken) { await Task.CompletedTask; //the kubernetes agent only supports the kubernetes script services - if (PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) + if (kubernetesAgentDetection.IsRunningAsKubernetesAgent) { return new CapabilitiesResponseV2(new List { nameof(IFileTransferService), nameof(IKubernetesScriptServiceV1) }); } diff --git a/source/Octopus.Tentacle/Services/Scripts/Kubernetes/KubernetesScriptServiceV1.cs b/source/Octopus.Tentacle/Services/Scripts/Kubernetes/KubernetesScriptServiceV1.cs index e6ed60700..c472427dd 100644 --- a/source/Octopus.Tentacle/Services/Scripts/Kubernetes/KubernetesScriptServiceV1.cs +++ b/source/Octopus.Tentacle/Services/Scripts/Kubernetes/KubernetesScriptServiceV1.cs @@ -16,6 +16,7 @@ namespace Octopus.Tentacle.Services.Scripts.Kubernetes [KubernetesService(typeof(IKubernetesScriptServiceV1))] public class KubernetesScriptServiceV1 : IAsyncKubernetesScriptServiceV1, IRunningScriptReporter { + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesPodService podService; readonly IScriptWorkspaceFactory workspaceFactory; readonly IKubernetesPodStatusProvider podStatusProvider; @@ -28,6 +29,7 @@ public class KubernetesScriptServiceV1 : IAsyncKubernetesScriptServiceV1, IRunni readonly IKeyedSemaphore keyedSemaphore; public KubernetesScriptServiceV1( + IKubernetesConfiguration kubernetesConfiguration, IKubernetesPodService podService, IScriptWorkspaceFactory workspaceFactory, IKubernetesPodStatusProvider podStatusProvider, @@ -39,6 +41,7 @@ public KubernetesScriptServiceV1( IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider, IKeyedSemaphore keyedSemaphore) { + this.kubernetesConfiguration = kubernetesConfiguration; this.podService = podService; this.workspaceFactory = workspaceFactory; this.podStatusProvider = podStatusProvider; @@ -129,7 +132,7 @@ public async Task CompleteScriptAsync(CompleteKubernetesScriptCommandV1 command, scriptPodSinceTimeStore.Delete(command.ScriptTicket); scriptPodLogEncryptionKeyProvider.Delete(command.ScriptTicket); - if (!KubernetesConfig.DisableAutomaticPodCleanup) + if (!kubernetesConfiguration.DisableAutomaticPodCleanup) await podService.DeleteIfExists(command.ScriptTicket, cancellationToken); } diff --git a/source/Octopus.Tentacle/Services/ServicesModule.cs b/source/Octopus.Tentacle/Services/ServicesModule.cs index 4e24c14e4..e4157130f 100644 --- a/source/Octopus.Tentacle/Services/ServicesModule.cs +++ b/source/Octopus.Tentacle/Services/ServicesModule.cs @@ -4,6 +4,7 @@ using System.Reflection; using Autofac; using Octopus.Tentacle.Communications; +using Octopus.Tentacle.Kubernetes; using Octopus.Tentacle.Packages; using Octopus.Tentacle.Scripts; using Octopus.Tentacle.Util; @@ -27,7 +28,7 @@ protected override void Load(ContainerBuilder builder) RegisterHalibutServices(builder, allTypes); //only register kubernetes services when - if (PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) + if (KubernetesAgentDetection.IsRunningAsKubernetesAgent) { RegisterHalibutServices(builder, allTypes); } diff --git a/source/Octopus.Tentacle/Startup/OctopusProgram.cs b/source/Octopus.Tentacle/Startup/OctopusProgram.cs index bc7b6687f..5c4f5c5d4 100644 --- a/source/Octopus.Tentacle/Startup/OctopusProgram.cs +++ b/source/Octopus.Tentacle/Startup/OctopusProgram.cs @@ -308,7 +308,7 @@ void InitializeLogging() Target.Register("EventLog"); #endif #if REQUIRES_EXPLICIT_LOG_CONFIG - var nLogFileExtension = !PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent + var nLogFileExtension = !KubernetesAgentDetection.IsRunningAsKubernetesAgent ? "exe.nlog" : "exe.k8s.nlog"; @@ -384,7 +384,7 @@ StartUpInstanceRequest TryLoadInstanceNameFromCommandLineArguments(string[] comm if (!string.IsNullOrWhiteSpace(instanceName)) { - return PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent + return KubernetesAgentDetection.IsRunningAsKubernetesAgent ? new StartUpKubernetesConfigMapInstanceRequest(instanceName) : new StartUpRegistryInstanceRequest(instanceName); } diff --git a/source/Octopus.Tentacle/Util/OctopusFileSystemModule.cs b/source/Octopus.Tentacle/Util/OctopusFileSystemModule.cs index 53ab25598..d98451dfd 100644 --- a/source/Octopus.Tentacle/Util/OctopusFileSystemModule.cs +++ b/source/Octopus.Tentacle/Util/OctopusFileSystemModule.cs @@ -9,7 +9,7 @@ public class OctopusFileSystemModule : Module protected override void Load(ContainerBuilder builder) { base.Load(builder); - if (PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) + if (KubernetesAgentDetection.IsRunningAsKubernetesAgent) { builder.RegisterType().AsSelf().As(); } diff --git a/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs b/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs index fe5f93a6b..5d37b9470 100644 --- a/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs +++ b/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs @@ -265,10 +265,6 @@ public virtual void EnsureDiskHasEnoughFreeSpace(string directoryPath, long requ if (!Path.IsPathRooted(directoryPath)) return; - //We can't perform this check in Kubernetes due to how drives are mounted and reported (always returns 0 byte sized drives) - if(PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) - return; - var driveInfo = SafelyGetDriveInfo(directoryPath); var required = requiredSpaceInBytes < 0 ? 0 : (ulong)requiredSpaceInBytes; diff --git a/source/Octopus.Tentacle/Util/PlatformDetection.cs b/source/Octopus.Tentacle/Util/PlatformDetection.cs index 801e10bfe..3d7b219d7 100644 --- a/source/Octopus.Tentacle/Util/PlatformDetection.cs +++ b/source/Octopus.Tentacle/Util/PlatformDetection.cs @@ -1,6 +1,5 @@ using System; using System.Runtime.InteropServices; -using Octopus.Tentacle.Kubernetes; namespace Octopus.Tentacle.Util { @@ -9,13 +8,5 @@ public static class PlatformDetection public static bool IsRunningOnNix => RuntimeInformation.IsOSPlatform(OSPlatform.Linux); public static bool IsRunningOnWindows => RuntimeInformation.IsOSPlatform(OSPlatform.Windows); public static bool IsRunningOnMac => RuntimeInformation.IsOSPlatform(OSPlatform.OSX); - - public static class Kubernetes - { - /// - /// Indicates if the Tentacle is running inside a Kubernetes cluster as the Kubernetes Agent. This is done by checking if the namespace environment variable is set - /// - public static bool IsRunningAsKubernetesAgent => !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(KubernetesConfig.NamespaceVariableName)); - } } } \ No newline at end of file diff --git a/source/Tentacle.sln.DotSettings b/source/Tentacle.sln.DotSettings index 0e7802a6d..2c88b318a 100644 --- a/source/Tentacle.sln.DotSettings +++ b/source/Tentacle.sln.DotSettings @@ -183,6 +183,7 @@ True True True + True True True True