From 774dcd8ef8e58f3d07b99eb7cb20df0a1c07ca51 Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Thu, 19 Dec 2024 16:30:48 +1100 Subject: [PATCH 1/9] Change to use non-static IKubernetesConfiguration class --- .../Configuration/ConfigMapKeyValueStore.cs | 4 +- .../KubernetesMachineEncryptionKeyProvider.cs | 6 +- .../Diagnostics/PersistenceProvider.cs | 6 +- .../EnvironmentKubernetesConfiguration.cs | 86 +++++++++++++++++++ .../Kubernetes/IKubernetesConfiguration.cs | 36 ++++++++ .../Kubernetes/KubernetesClusterService.cs | 4 +- .../Kubernetes/KubernetesConfig.cs | 2 +- .../Kubernetes/KubernetesConfigMapService.cs | 8 +- .../KubernetesDirectoryInformationProvider.cs | 6 +- .../Kubernetes/KubernetesEventMonitorTask.cs | 6 +- .../Kubernetes/KubernetesEventService.cs | 3 +- .../Kubernetes/KubernetesModule.cs | 3 +- .../KubernetesOrphanedPodCleaner.cs | 7 +- .../KubernetesPodContainerResolver.cs | 8 +- .../Kubernetes/KubernetesPodLogService.cs | 9 +- .../Kubernetes/KubernetesPodService.cs | 14 +-- .../KubernetesRawScriptPodCreator.cs | 7 +- .../Kubernetes/KubernetesScriptPodCreator.cs | 56 ++++++------ .../Kubernetes/KubernetesSecretService.cs | 10 +-- .../Kubernetes/KubernetesService.cs | 15 ++-- .../Octopus.Tentacle/Octopus.Tentacle.csproj | 2 +- source/Tentacle.sln.DotSettings | 1 + 22 files changed, 223 insertions(+), 76 deletions(-) create mode 100644 source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs create mode 100644 source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs diff --git a/source/Octopus.Tentacle/Kubernetes/Configuration/ConfigMapKeyValueStore.cs b/source/Octopus.Tentacle/Kubernetes/Configuration/ConfigMapKeyValueStore.cs index ab5c4a172..ac3348596 100644 --- a/source/Octopus.Tentacle/Kubernetes/Configuration/ConfigMapKeyValueStore.cs +++ b/source/Octopus.Tentacle/Kubernetes/Configuration/ConfigMapKeyValueStore.cs @@ -19,12 +19,12 @@ class ConfigMapKeyValueStore : IWritableKeyValueStore, IAggregatableKeyValueStor readonly Lazy configMap; IDictionary ConfigMapData => configMap.Value.Data ??= new Dictionary(); - public ConfigMapKeyValueStore(IKubernetesConfigMapService configMapService, IKubernetesMachineKeyEncryptor encryptor) + public ConfigMapKeyValueStore(IKubernetesConfiguration kubernetesConfiguration, IKubernetesConfigMapService configMapService, IKubernetesMachineKeyEncryptor encryptor) { this.configMapService = configMapService; this.encryptor = encryptor; configMap = new Lazy(() => configMapService.TryGet(Name, CancellationToken.None).GetAwaiter().GetResult() - ?? throw new InvalidOperationException($"Unable to retrieve Tentacle Configuration from config map for namespace {KubernetesConfig.Namespace}")); + ?? throw new InvalidOperationException($"Unable to retrieve Tentacle Configuration from config map for namespace {kubernetesConfiguration.Namespace}")); } public string? Get(string name, ProtectionLevel protectionLevel = ProtectionLevel.None) diff --git a/source/Octopus.Tentacle/Kubernetes/Crypto/KubernetesMachineEncryptionKeyProvider.cs b/source/Octopus.Tentacle/Kubernetes/Crypto/KubernetesMachineEncryptionKeyProvider.cs index 09d8c1a7f..e4ed25f02 100644 --- a/source/Octopus.Tentacle/Kubernetes/Crypto/KubernetesMachineEncryptionKeyProvider.cs +++ b/source/Octopus.Tentacle/Kubernetes/Crypto/KubernetesMachineEncryptionKeyProvider.cs @@ -20,13 +20,15 @@ public class KubernetesMachineEncryptionKeyProvider : IKubernetesMachineEncrypti const string MachineKeyName = "machine-key"; const string MachineIvName = "machine-iv"; + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesSecretService secretService; readonly ISystemLog log; V1Secret? secret; readonly SemaphoreSlim accessSemaphore; - public KubernetesMachineEncryptionKeyProvider(IKubernetesSecretService secretService, ISystemLog log) + public KubernetesMachineEncryptionKeyProvider(IKubernetesConfiguration kubernetesConfiguration, IKubernetesSecretService secretService, ISystemLog log) { + this.kubernetesConfiguration = kubernetesConfiguration; this.secretService = secretService; this.log = log; accessSemaphore = new SemaphoreSlim(1, 1); @@ -42,7 +44,7 @@ public KubernetesMachineEncryptionKeyProvider(IKubernetesSecretService secretSer secret ??= await secretService.TryGetSecretAsync(SecretName, CancellationToken.None); if (secret is null) - throw new InvalidOperationException($"Unable to retrieve MachineKey from secret for namespace {KubernetesConfig.Namespace}"); + throw new InvalidOperationException($"Unable to retrieve MachineKey from secret for namespace {kubernetesConfiguration.Namespace}"); var data = secret.Data; if (data is null || diff --git a/source/Octopus.Tentacle/Kubernetes/Diagnostics/PersistenceProvider.cs b/source/Octopus.Tentacle/Kubernetes/Diagnostics/PersistenceProvider.cs index 5784d06f0..47c7f4403 100644 --- a/source/Octopus.Tentacle/Kubernetes/Diagnostics/PersistenceProvider.cs +++ b/source/Octopus.Tentacle/Kubernetes/Diagnostics/PersistenceProvider.cs @@ -18,12 +18,14 @@ public class PersistenceProvider : IPersistenceProvider public delegate PersistenceProvider Factory(string configMapName); readonly string configMapName; + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesConfigMapService configMapService; - public PersistenceProvider(string configMapName, IKubernetesConfigMapService configMapService) + public PersistenceProvider(string configMapName, IKubernetesConfiguration kubernetesConfiguration, IKubernetesConfigMapService configMapService) { this.configMapService = configMapService; this.configMapName = configMapName; + this.kubernetesConfiguration = kubernetesConfiguration; } public async Task GetValue(string key, CancellationToken cancellationToken) @@ -35,7 +37,7 @@ public PersistenceProvider(string configMapName, IKubernetesConfigMapService con public async Task PersistValue(string key, string value, CancellationToken cancellationToken) { var configMapData = await LoadConfigMapData(cancellationToken); - if (configMapData is null) throw new InvalidOperationException($"Unable to retrieve Tentacle Configuration from config map for namespace {KubernetesConfig.Namespace}"); + if (configMapData is null) throw new InvalidOperationException($"Unable to retrieve Tentacle Configuration from config map for namespace {kubernetesConfiguration.Namespace}"); configMapData[key] = value; await configMapService.Patch(configMapName, configMapData, cancellationToken); diff --git a/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs b/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs new file mode 100644 index 000000000..bb1c74c0e --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Octopus.Tentacle.Util; + +namespace Octopus.Tentacle.Kubernetes +{ + public class EnvironmentKubernetesConfiguration : IKubernetesConfiguration + { + public static class VariableNames + { + public const string EnvVarPrefix = "OCTOPUS__K8STENTACLE"; + + public const string PersistentVolumeSizeBytesVariableName = $"{EnvVarPrefix}__PERSISTENTVOLUMETOTALBYTES"; + public const string PersistentVolumeFreeBytesVariableName = $"{EnvVarPrefix}__PERSISTENTVOLUMEFREEBYTES"; + public const string NamespaceVariableName = $"{EnvVarPrefix}__NAMESPACE"; + + public const string HelmReleaseNameVariableName = $"{EnvVarPrefix}__HELMRELEASENAME"; + public const string HelmChartVersionVariableName = $"{EnvVarPrefix}__HELMCHARTVERSION"; + public const string ServerCommsAddressesVariableName = "ServerCommsAddresses"; + + public const string PodAffinityJsonVariableName = $"{EnvVarPrefix}__PODAFFINITYJSON"; + public const string PodTolerationsJsonVariableName = $"{EnvVarPrefix}__PODTOLERATIONSJSON"; + public const string PodSecurityContextJsonVariableName = $"{EnvVarPrefix}__PODSECURITYCONTEXTJSON"; + public const string PodResourceJsonVariableName = $"{EnvVarPrefix}__PODRESOURCEJSON"; + } + + public string Namespace => GetRequiredEnvVar(VariableNames.NamespaceVariableName, "Unable to determine Kubernetes namespace."); + public string BootstrapRunnerExecutablePath => GetRequiredEnvVar("BOOTSTRAPRUNNEREXECUTABLEPATH", "Unable to determine Bootstrap Runner Executable Path"); + public string ScriptPodServiceAccountName => GetRequiredEnvVar($"{VariableNames.EnvVarPrefix}__SCRIPTPODSERVICEACCOUNTNAME", "Unable to determine Kubernetes Pod service account name."); + + public IEnumerable ScriptPodImagePullSecretNames => Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__PODIMAGEPULLSECRETNAMES") + ?.Split(',') + .Select(str => str.Trim()) + .WhereNotNullOrWhiteSpace() + .ToArray() ?? []; + + public string ScriptPodVolumeClaimName => GetRequiredEnvVar($"{VariableNames.EnvVarPrefix}__PODVOLUMECLAIMNAME", "Unable to determine Kubernetes Pod persistent volume claim name."); + public string? ScriptPodResourceJson => Environment.GetEnvironmentVariable(ScriptPodResourceJsonVariableName); + public string ScriptPodResourceJsonVariableName => $"{VariableNames.EnvVarPrefix}__PODRESOURCEJSON"; + + public static string NfsWatchdogImageVariableName => $"{VariableNames.EnvVarPrefix}__NFSWATCHDOGIMAGE"; + public string? NfsWatchdogImage => Environment.GetEnvironmentVariable(NfsWatchdogImageVariableName); + + public static string HelmReleaseNameVariableName => $"{VariableNames.EnvVarPrefix}__HELMRELEASENAME"; + public static string HelmChartVersionVariableName => $"{VariableNames.EnvVarPrefix}__HELMCHARTVERSION"; + const string ServerCommsAddressVariableName = "ServerCommsAddress"; + public const string ServerCommsAddressesVariableName = "ServerCommsAddresses"; + + public string HelmReleaseName => GetRequiredEnvVar(HelmReleaseNameVariableName, "Unable to determine Helm release name."); + public string HelmChartVersion => GetRequiredEnvVar(HelmChartVersionVariableName, "Unable to determine Helm chart version."); + + public string[] ServerCommsAddresses + { + get + { + var addresses = new List(); + if (Environment.GetEnvironmentVariable(ServerCommsAddressVariableName) is { Length: > 0 } addressString) + { + addresses.Add(addressString); + } + + if (Environment.GetEnvironmentVariable(ServerCommsAddressesVariableName) is { } addressesString) + { + addresses.AddRange(addressesString.Split(',').Where(a => !a.IsNullOrEmpty())); + } + + return addresses.ToArray(); + } + } + + public string PodVolumeClaimName => GetRequiredEnvVar($"{VariableNames.EnvVarPrefix}__PODVOLUMECLAIMNAME", "Unable to determine Kubernetes Pod persistent volume claim name."); + public int? PodMonitorTimeoutSeconds => int.TryParse(Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__PODMONITORTIMEOUT"), out var podMonitorTimeout) ? podMonitorTimeout : 10 * 60; //10min + public TimeSpan PodsConsideredOrphanedAfterTimeSpan => TimeSpan.FromMinutes(int.TryParse(Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__PODSCONSIDEREDORPHANEDAFTERMINUTES"), out var podsConsideredOrphanedAfterTimeSpan) ? podsConsideredOrphanedAfterTimeSpan : 10); + public bool DisableAutomaticPodCleanup => bool.TryParse(Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__DISABLEAUTOPODCLEANUP"), out var disableAutoCleanup) && disableAutoCleanup; + + public static string PersistentVolumeSizeVariableName => $"{VariableNames.EnvVarPrefix}__PERSISTENTVOLUMESIZE"; + public string PersistentVolumeSize => GetRequiredEnvVar(PersistentVolumeSizeVariableName, "Unable to determine Persistent Volume Size"); + + public bool IsMetricsEnabled => !bool.TryParse(Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__ENABLEMETRICSCAPTURE"), out var enableMetrics) || enableMetrics; + + static string GetRequiredEnvVar(string variable, string errorMessage) + => Environment.GetEnvironmentVariable(variable) + ?? throw new InvalidOperationException($"{errorMessage} The environment variable '{variable}' must be defined with a non-null value."); + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs new file mode 100644 index 000000000..e63c71e81 --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; + +namespace Octopus.Tentacle.Kubernetes +{ + public interface IKubernetesConfiguration + { + string Namespace { get;} + string BootstrapRunnerExecutablePath { get; } + + string ScriptPodServiceAccountName { get; } + IEnumerable ScriptPodImagePullSecretNames { get; } + string ScriptPodVolumeClaimName { get; } + string? ScriptPodResourceJson { get; } + string ScriptPodResourceJsonVariableName { get; } + string ScriptPodContainerImage { get; } + string ScriptPodContainerImageTag { get;} + string ScriptPodPullPolicy { get; set; } + string? NfsWatchdogImage { get; } + string HelmReleaseName { get; } + string HelmChartVersion { get; } + string[] ServerCommsAddresses { get; } + string PodVolumeClaimName { get; } + int? PodMonitorTimeoutSeconds { get; } + TimeSpan PodsConsideredOrphanedAfterTimeSpan { get; } + bool DisableAutomaticPodCleanup { get; } + bool DisablePodEventsInTaskLog { get; } + string PersistentVolumeSize { get; } + bool IsMetricsEnabled { get; } + string? PodResourceJson { get; set; } + string? PodAffinityJson { get; set; } + string? PodTolerationsJson { get; set; } + string? PodSecurityContextJson { get; set; } + string? ScriptPodProxiesSecretName { get; set; } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs index af2435813..3b31fd752 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs @@ -14,8 +14,8 @@ public interface IKubernetesClusterService public class KubernetesClusterService : KubernetesService, IKubernetesClusterService { readonly AsyncLazy lazyVersion; - public KubernetesClusterService(IKubernetesClientConfigProvider configProvider, ISystemLog log) - : base(configProvider, log) + public KubernetesClusterService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) + : base(configProvider, kubernetesConfiguration, log) { //As the cluster version isn't going to change without restarting, we just cache the version in an AsyncLazy lazyVersion = new AsyncLazy(async () => diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs index 8c2586cd0..03aee6f70 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs @@ -5,7 +5,7 @@ namespace Octopus.Tentacle.Kubernetes { - public static class KubernetesConfig + public static class KubernetesConfigz { const string ServerCommsAddressVariableName = "ServerCommsAddress"; const string EnvVarPrefix = "OCTOPUS__K8STENTACLE"; diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesConfigMapService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesConfigMapService.cs index 19292a205..a7fbbeb29 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesConfigMapService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesConfigMapService.cs @@ -18,8 +18,8 @@ public interface IKubernetesConfigMapService public class KubernetesConfigMapService : KubernetesService, IKubernetesConfigMapService { - public KubernetesConfigMapService(IKubernetesClientConfigProvider configProvider, ISystemLog log) - : base(configProvider, log) + public KubernetesConfigMapService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) + : base(configProvider, kubernetesConfiguration, log) { } @@ -29,7 +29,7 @@ public KubernetesConfigMapService(IKubernetesClientConfigProvider configProvider { try { - return await Client.CoreV1.ReadNamespacedConfigMapAsync(name, KubernetesConfig.Namespace, cancellationToken: cancellationToken); + return await Client.CoreV1.ReadNamespacedConfigMapAsync(name, Namespace, cancellationToken: cancellationToken); } catch (HttpOperationException opException) when (opException.Response.StatusCode == HttpStatusCode.NotFound) @@ -49,7 +49,7 @@ public async Task Patch(string name, IDictionary da var configMapJson = KubernetesJson.Serialize(configMap); return await RetryPolicy.ExecuteAsync(async () => - await Client.CoreV1.PatchNamespacedConfigMapAsync(new V1Patch(configMapJson, V1Patch.PatchType.MergePatch), name, KubernetesConfig.Namespace, cancellationToken: cancellationToken)); + await Client.CoreV1.PatchNamespacedConfigMapAsync(new V1Patch(configMapJson, V1Patch.PatchType.MergePatch), name, Namespace, cancellationToken: cancellationToken)); } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesDirectoryInformationProvider.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesDirectoryInformationProvider.cs index 2a1a5be53..dac211ae5 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesDirectoryInformationProvider.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesDirectoryInformationProvider.cs @@ -17,6 +17,7 @@ public interface IKubernetesDirectoryInformationProvider public class KubernetesDirectoryInformationProvider : IKubernetesDirectoryInformationProvider { readonly ISystemLog log; + readonly IKubernetesConfiguration kubernetesConfiguration; readonly ISilentProcessRunner silentProcessRunner; readonly IMemoryCache directoryInformationCache; @@ -29,9 +30,10 @@ public class KubernetesDirectoryInformationProvider : IKubernetesDirectoryInform //No calls to `du` at all: 8min ea. static readonly TimeSpan CacheExpiry = TimeSpan.FromSeconds(30); - public KubernetesDirectoryInformationProvider(ISystemLog log, ISilentProcessRunner silentProcessRunner, IMemoryCache directoryInformationCache) + public KubernetesDirectoryInformationProvider(ISystemLog log, IKubernetesConfiguration kubernetesConfiguration, ISilentProcessRunner silentProcessRunner, IMemoryCache directoryInformationCache) { this.log = log; + this.kubernetesConfiguration = kubernetesConfiguration; this.silentProcessRunner = silentProcessRunner; this.directoryInformationCache = directoryInformationCache; } @@ -47,7 +49,7 @@ public KubernetesDirectoryInformationProvider(ISystemLog log, ISilentProcessRunn public ulong? GetPathTotalBytes() { - return KubernetesUtilities.GetResourceBytes(KubernetesConfig.PersistentVolumeSize); + return KubernetesUtilities.GetResourceBytes(kubernetesConfiguration.PersistentVolumeSize); } diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs index fe2d910a6..289fa04e4 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs @@ -13,16 +13,18 @@ public class KubernetesEventMonitorTask : BackgroundTask readonly IKubernetesEventMonitor eventMonitor; readonly ISystemLog log; + readonly IKubernetesConfiguration kubernetesConfiguration; readonly TimeSpan taskInterval = TimeSpan.FromMinutes(10); - public KubernetesEventMonitorTask(ISystemLog log, IKubernetesEventMonitor eventMonitor) : base(log, TimeSpan.FromSeconds(30)) + public KubernetesEventMonitorTask(ISystemLog log, IKubernetesConfiguration kubernetesConfiguration, IKubernetesEventMonitor eventMonitor) : base(log, TimeSpan.FromSeconds(30)) { this.log = log; + this.kubernetesConfiguration = kubernetesConfiguration; this.eventMonitor = eventMonitor; } protected override async Task RunTask(CancellationToken cancellationToken) { - if (!KubernetesConfig.MetricsIsEnabled) + if (!kubernetesConfiguration.IsMetricsEnabled) { log.Info("Event monitoring for agent metrics is not enabled."); return; diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs index d42819530..861378294 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs @@ -18,7 +18,8 @@ public interface IKubernetesEventService public class KubernetesEventService : KubernetesService, IKubernetesEventService { - public KubernetesEventService(IKubernetesClientConfigProvider configProvider, ISystemLog log) : base(configProvider, log) + public KubernetesEventService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) + : base(configProvider, kubernetesConfiguration, log) { } diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs index da64cba9d..f1e55ed78 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs @@ -59,8 +59,7 @@ protected override void Load(ContainerBuilder builder) .As(); builder.RegisterType(); - builder.Register(ctx => ctx.Resolve().Invoke(KubernetesConfig.Namespace)) - .As(); + builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As(); diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs index 738c9558a..060c929ee 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs @@ -17,6 +17,7 @@ public interface IKubernetesOrphanedPodCleaner public class KubernetesOrphanedPodCleaner : IKubernetesOrphanedPodCleaner { + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesPodStatusProvider podStatusProvider; readonly IKubernetesPodService podService; readonly ISystemLog log; @@ -26,9 +27,10 @@ public class KubernetesOrphanedPodCleaner : IKubernetesOrphanedPodCleaner readonly IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider; readonly TimeSpan initialDelay = TimeSpan.FromMinutes(1); - internal readonly TimeSpan CompletedPodConsideredOrphanedAfterTimeSpan = KubernetesConfig.PodsConsideredOrphanedAfterTimeSpan; + internal TimeSpan CompletedPodConsideredOrphanedAfterTimeSpan => kubernetesConfiguration.PodsConsideredOrphanedAfterTimeSpan; public KubernetesOrphanedPodCleaner( + IKubernetesConfiguration kubernetesConfiguration, IKubernetesPodStatusProvider podStatusProvider, IKubernetesPodService podService, ISystemLog log, @@ -37,6 +39,7 @@ public KubernetesOrphanedPodCleaner( IScriptPodSinceTimeStore scriptPodSinceTimeStore, IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider) { + this.kubernetesConfiguration = kubernetesConfiguration; this.podStatusProvider = podStatusProvider; this.podService = podService; this.log = log; @@ -104,7 +107,7 @@ state.FinishedAt is not null && scriptPodSinceTimeStore.Delete(pod.ScriptTicket); scriptPodLogEncryptionKeyProvider.Delete(pod.ScriptTicket); - if (KubernetesConfig.DisableAutomaticPodCleanup) + if (kubernetesConfiguration.DisableAutomaticPodCleanup) { log.Verbose($"OrphanedPodCleaner: Not deleting orphaned pod {pod.ScriptTicket} as automatic cleanup is disabled"); continue; diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesPodContainerResolver.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesPodContainerResolver.cs index eab1790bd..08f88eebb 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesPodContainerResolver.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesPodContainerResolver.cs @@ -13,11 +13,13 @@ public interface IKubernetesPodContainerResolver public class KubernetesPodContainerResolver : IKubernetesPodContainerResolver { + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesClusterService clusterService; readonly IToolsImageVersionMetadataProvider imageVersionMetadataProvider; - public KubernetesPodContainerResolver(IKubernetesClusterService clusterService, IToolsImageVersionMetadataProvider imageVersionMetadataProvider) + public KubernetesPodContainerResolver(IKubernetesConfiguration kubernetesConfiguration, IKubernetesClusterService clusterService, IToolsImageVersionMetadataProvider imageVersionMetadataProvider) { + this.kubernetesConfiguration = kubernetesConfiguration; this.clusterService = clusterService; this.imageVersionMetadataProvider = imageVersionMetadataProvider; } @@ -36,13 +38,13 @@ public KubernetesPodContainerResolver(IKubernetesClusterService clusterService, public async Task GetContainerImageForCluster() { - var imageRepository = KubernetesConfig.ScriptPodContainerImage; + var imageRepository = kubernetesConfiguration.ScriptPodContainerImage; if (imageRepository.IsNullOrEmpty()) { return await GetAgentToolsContainerImage(); } - var imageTag = KubernetesConfig.ScriptPodContainerImageTag; + var imageTag = kubernetesConfiguration.ScriptPodContainerImageTag; return $"{imageRepository}:{imageTag}"; } diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesPodLogService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesPodLogService.cs index e8d331344..aa9b49203 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesPodLogService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesPodLogService.cs @@ -28,13 +28,14 @@ class KubernetesPodLogService : KubernetesService, IKubernetesPodLogService public KubernetesPodLogService( IKubernetesClientConfigProvider configProvider, + IKubernetesConfiguration kubernetesConfiguration, IKubernetesPodMonitor podMonitor, ITentacleScriptLogProvider scriptLogProvider, IScriptPodSinceTimeStore scriptPodSinceTimeStore, IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider, IKubernetesEventService eventService, ISystemLog log) - : base(configProvider, log) + : base(configProvider, kubernetesConfiguration,log) { this.podMonitor = podMonitor; this.scriptLogProvider = scriptLogProvider; @@ -130,14 +131,14 @@ public KubernetesPodLogService( async Task> GetPodEvents(ScriptTicket scriptTicket, string podName, CancellationToken cancellationToken) { //if we don't want to write pod events to the task log, don't do anything - if (KubernetesConfig.DisablePodEventsInTaskLog) + if (KubernetesConfiguration.DisablePodEventsInTaskLog) { return Array.Empty(); } var sinceTime = scriptPodSinceTimeStore.GetPodEventsSinceTime(scriptTicket); - var allEvents = await eventService.FetchAllEventsAsync(KubernetesConfig.Namespace, podName, cancellationToken); + var allEvents = await eventService.FetchAllEventsAsync(Namespace, podName, cancellationToken); if (allEvents is null) { return Array.Empty(); @@ -188,7 +189,7 @@ async Task> GetPodEvents(ScriptTicket scriptTicket, s { try { - return await Client.GetNamespacedPodLogsAsync(podName, KubernetesConfig.Namespace, podName, sinceTime, cancellationToken: cancellationToken); + return await Client.GetNamespacedPodLogsAsync(podName, Namespace, podName, sinceTime, cancellationToken: cancellationToken); } catch (HttpOperationException ex) { diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs index c1f3962b5..81bb8f0be 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs @@ -20,14 +20,14 @@ public interface IKubernetesPodService public class KubernetesPodService : KubernetesService, IKubernetesPodService { - public KubernetesPodService(IKubernetesClientConfigProvider configProvider, ISystemLog log) - : base(configProvider, log) + public KubernetesPodService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) + : base(configProvider, kubernetesConfiguration,log) { } public async Task ListAllPods(CancellationToken cancellationToken) { - return await Client.ListNamespacedPodAsync(KubernetesConfig.Namespace, + return await Client.ListNamespacedPodAsync(Namespace, labelSelector: OctopusLabels.ScriptTicketId, cancellationToken: cancellationToken); } @@ -35,11 +35,11 @@ public async Task ListAllPods(CancellationToken cancellationToken) public async Task WatchAllPods(string initialResourceVersion, Func onChange, Action onError, CancellationToken cancellationToken) { using var response = Client.CoreV1.ListNamespacedPodWithHttpMessagesAsync( - KubernetesConfig.Namespace, + Namespace, labelSelector: OctopusLabels.ScriptTicketId, resourceVersion: initialResourceVersion, watch: true, - timeoutSeconds: KubernetesConfig.PodMonitorTimeoutSeconds, + timeoutSeconds: KubernetesConfiguration.PodMonitorTimeoutSeconds, cancellationToken: cancellationToken); var watchErrorCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); @@ -78,7 +78,7 @@ public async Task WatchAllPods(string initialResourceVersion, Func Create(V1Pod pod, CancellationToken cancellationToken) { AddStandardMetadata(pod); - return await Client.CreateNamespacedPodAsync(pod, KubernetesConfig.Namespace, cancellationToken: cancellationToken); + return await Client.CreateNamespacedPodAsync(pod, Namespace, cancellationToken: cancellationToken); } public async Task DeleteIfExists(ScriptTicket scriptTicket, CancellationToken cancellationToken) @@ -88,6 +88,6 @@ public async Task DeleteIfExists(ScriptTicket scriptTicket, TimeSpan gracePeriod => await DeleteIfExistsInternal(scriptTicket, (long)Math.Floor(gracePeriod.TotalSeconds), cancellationToken); async Task DeleteIfExistsInternal(ScriptTicket scriptTicket, long? gracePeriodSeconds, CancellationToken cancellationToken) - => await TryExecuteAsync(async () => await Client.DeleteNamespacedPodAsync(scriptTicket.ToKubernetesScriptPodName(), KubernetesConfig.Namespace, new V1DeleteOptions(gracePeriodSeconds: gracePeriodSeconds), cancellationToken: cancellationToken)); + => await TryExecuteAsync(async () => await Client.DeleteNamespacedPodAsync(scriptTicket.ToKubernetesScriptPodName(), Namespace, new V1DeleteOptions(gracePeriodSeconds: gracePeriodSeconds), cancellationToken: cancellationToken)); } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs index 2ed5ea06e..34cb80ec2 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs @@ -21,6 +21,7 @@ public class KubernetesRawScriptPodCreator : KubernetesScriptPodCreator, IKubern readonly IKubernetesPodContainerResolver containerResolver; public KubernetesRawScriptPodCreator( + IKubernetesConfiguration kubernetesConfiguration, IKubernetesPodService podService, IKubernetesPodMonitor podMonitor, IKubernetesSecretService secretService, @@ -31,7 +32,7 @@ public KubernetesRawScriptPodCreator( IHomeConfiguration homeConfiguration, KubernetesPhysicalFileSystem kubernetesPhysicalFileSystem, IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider) - : base(podService, podMonitor, secretService, containerResolver, appInstanceSelector, log, scriptLogProvider, homeConfiguration, kubernetesPhysicalFileSystem, scriptPodLogEncryptionKeyProvider) + : base(kubernetesConfiguration, podService, podMonitor, secretService, containerResolver, appInstanceSelector, log, scriptLogProvider, homeConfiguration, kubernetesPhysicalFileSystem, scriptPodLogEncryptionKeyProvider) { this.containerResolver = containerResolver; } @@ -42,7 +43,7 @@ protected override async Task> CreateInitContainers(StartKube { Name = $"{podName}-init", Image = command.PodImageConfiguration?.Image ?? await containerResolver.GetContainerImageForCluster(), - ImagePullPolicy = KubernetesConfig.ScriptPodPullPolicy, + ImagePullPolicy = KubernetesConfiguration.ScriptPodPullPolicy, Command = new List { "sh", "-c", GetInitExecutionScript("/nfs-mount", homeDir, workspacePath) }, VolumeMounts = new List { new("/nfs-mount", "init-nfs-volume"), new(homeDir, "tentacle-home") }, Resources = new V1ResourceRequirements @@ -80,7 +81,7 @@ protected override IList CreateVolumes(StartKubernetesScriptCommandV1 Name = "init-nfs-volume", PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource { - ClaimName = KubernetesConfig.PodVolumeClaimName + ClaimName = KubernetesConfiguration.PodVolumeClaimName } }, CreateAgentUpgradeSecretVolume(), diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs index 761c713db..f6f896a81 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs @@ -38,8 +38,11 @@ public class KubernetesScriptPodCreator : IKubernetesScriptPodCreator readonly IHomeConfiguration homeConfiguration; readonly KubernetesPhysicalFileSystem kubernetesPhysicalFileSystem; readonly IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider; + + protected IKubernetesConfiguration KubernetesConfiguration { get; } public KubernetesScriptPodCreator( + IKubernetesConfiguration kubernetesConfiguration, IKubernetesPodService podService, IKubernetesPodMonitor podMonitor, IKubernetesSecretService secretService, @@ -51,6 +54,7 @@ public KubernetesScriptPodCreator( KubernetesPhysicalFileSystem kubernetesPhysicalFileSystem, IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider) { + KubernetesConfiguration = kubernetesConfiguration; this.podService = podService; this.podMonitor = podMonitor; this.secretService = secretService; @@ -128,7 +132,7 @@ public async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorks Metadata = new V1ObjectMeta { Name = secretName, - NamespaceProperty = KubernetesConfig.Namespace + NamespaceProperty = KubernetesConfiguration.Namespace }, Data = new Dictionary { @@ -172,19 +176,19 @@ async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorkspace wo LogVerboseToBothLogs($"Creating Kubernetes Pod '{podName}'.", tentacleScriptLog); - workspace.CopyFile(KubernetesConfig.BootstrapRunnerExecutablePath, "bootstrapRunner", true); + workspace.CopyFile(KubernetesConfiguration.BootstrapRunnerExecutablePath, "bootstrapRunner", true); var scriptName = Path.GetFileName(workspace.BootstrapScriptFilePath); var workspacePath = Path.Combine("Work", workspace.ScriptTicket.TaskId); var serviceAccountName = !string.IsNullOrWhiteSpace(command.ScriptPodServiceAccountName) ? command.ScriptPodServiceAccountName - : KubernetesConfig.PodServiceAccountName; + : KubernetesConfiguration.ScriptPodServiceAccountName; // image pull secrets may have been defined in the helm chart (e.g. to avoid docker hub rate limiting) // we put any specified secret name first so it's resolved first var imagePullSecretNames = new[] { imagePullSecretName } - .Concat(KubernetesConfig.PodImagePullSecretNames) + .Concat(KubernetesConfiguration.ScriptPodImagePullSecretNames) .WhereNotNull() .Select(secretName => new V1LocalObjectReference(secretName)) .ToList(); @@ -194,7 +198,7 @@ async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorkspace wo Metadata = new V1ObjectMeta { Name = podName, - NamespaceProperty = KubernetesConfig.Namespace, + NamespaceProperty = KubernetesConfiguration.Namespace, Labels = new Dictionary { ["octopus.com/serverTaskId"] = command.TaskId, @@ -245,7 +249,7 @@ protected virtual IList CreateVolumes(StartKubernetesScriptCommandV1 c Name = "tentacle-home", PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource { - ClaimName = KubernetesConfig.PodVolumeClaimName + ClaimName = KubernetesConfiguration.PodVolumeClaimName } }, CreateAgentUpgradeSecretVolume(), @@ -295,14 +299,14 @@ protected async Task CreateScriptContainer(StartKubernetesScriptCom var envFrom = new List(); //if there is a scrip pod proxies defined - if (!string.IsNullOrWhiteSpace(KubernetesConfig.ScriptPodProxiesSecretName)) + if (!string.IsNullOrWhiteSpace(KubernetesConfiguration.ScriptPodProxiesSecretName)) { //add sourcing environment variables from the secret envFrom.Add(new V1EnvFromSource { SecretRef = new V1SecretEnvSource { - Name = KubernetesConfig.ScriptPodProxiesSecretName + Name = KubernetesConfiguration.ScriptPodProxiesSecretName } }); } @@ -311,7 +315,7 @@ protected async Task CreateScriptContainer(StartKubernetesScriptCom { Name = podName, Image = command.PodImageConfiguration?.Image ?? await containerResolver.GetContainerImageForCluster(), - ImagePullPolicy = KubernetesConfig.ScriptPodPullPolicy, + ImagePullPolicy = KubernetesConfiguration.ScriptPodPullPolicy, Command = new List { "sh" }, Args = new List { @@ -326,12 +330,12 @@ protected async Task CreateScriptContainer(StartKubernetesScriptCom }, Env = new List { - new(KubernetesConfig.NamespaceVariableName, KubernetesConfig.Namespace), - new(KubernetesConfig.HelmReleaseNameVariableName, KubernetesConfig.HelmReleaseName), - new(KubernetesConfig.HelmChartVersionVariableName, KubernetesConfig.HelmChartVersion), - new(KubernetesConfig.ServerCommsAddressesVariableName, string.Join(",", KubernetesConfig.ServerCommsAddresses)), - new(KubernetesConfig.PersistentVolumeFreeBytesVariableName, spaceInformation?.freeSpaceBytes.ToString()), - new(KubernetesConfig.PersistentVolumeSizeBytesVariableName, spaceInformation?.totalSpaceBytes.ToString()), + new(EnvironmentKubernetesConfiguration.VariableNames.NamespaceVariableName, KubernetesConfiguration.Namespace), + new(EnvironmentKubernetesConfiguration.VariableNames.HelmReleaseNameVariableName, KubernetesConfiguration.HelmReleaseName), + new(EnvironmentKubernetesConfiguration.VariableNames.HelmChartVersionVariableName, KubernetesConfiguration.HelmChartVersion), + new(EnvironmentKubernetesConfiguration.VariableNames.ServerCommsAddressesVariableName, string.Join(",", KubernetesConfiguration.ServerCommsAddresses)), + new(EnvironmentKubernetesConfiguration.VariableNames.PersistentVolumeFreeBytesVariableName, spaceInformation?.freeSpaceBytes.ToString()), + new(EnvironmentKubernetesConfiguration.VariableNames.PersistentVolumeSizeBytesVariableName, spaceInformation?.totalSpaceBytes.ToString()), new(EnvironmentVariables.TentacleHome, homeDir), new(EnvironmentVariables.TentacleInstanceName, appInstanceSelector.Current.InstanceName), new(EnvironmentVariables.TentacleVersion, Environment.GetEnvironmentVariable(EnvironmentVariables.TentacleVersion)), @@ -347,7 +351,7 @@ protected async Task CreateScriptContainer(StartKubernetesScriptCom V1ResourceRequirements GetScriptPodResourceRequirements(InMemoryTentacleScriptLog tentacleScriptLog) { - var json = KubernetesConfig.PodResourceJson; + var json = KubernetesConfiguration.PodResourceJson; if (!string.IsNullOrWhiteSpace(json)) { try @@ -356,7 +360,7 @@ V1ResourceRequirements GetScriptPodResourceRequirements(InMemoryTentacleScriptLo } catch (Exception e) { - var message = $"Failed to deserialize env.{KubernetesConfig.PodResourceJsonVariableName} into valid pod resource requirements.{Environment.NewLine}JSON value: {json}{Environment.NewLine}Using default resource requests for script pod."; + var message = $"Failed to deserialize env.{EnvironmentKubernetesConfiguration.VariableNames.PodResourceJsonVariableName} into valid pod resource requirements.{Environment.NewLine}JSON value: {json}{Environment.NewLine}Using default resource requests for script pod."; //if we can't parse the JSON, fall back to the defaults below and warn the user log.WarnFormat(e, message); //write a verbose message to the script log. @@ -378,8 +382,8 @@ V1ResourceRequirements GetScriptPodResourceRequirements(InMemoryTentacleScriptLo V1Affinity ParseScriptPodAffinity(InMemoryTentacleScriptLog tentacleScriptLog) => ParseScriptPodJson( tentacleScriptLog, - KubernetesConfig.PodAffinityJson, - KubernetesConfig.PodAffinityJsonVariableName, + KubernetesConfiguration.PodAffinityJson, + EnvironmentKubernetesConfiguration.VariableNames.PodAffinityJsonVariableName, "pod affinity", //we default to running on linux/arm64 and linux/amd64 nodes new V1Affinity(new V1NodeAffinity(requiredDuringSchedulingIgnoredDuringExecution: new V1NodeSelector(new List @@ -394,15 +398,15 @@ V1Affinity ParseScriptPodAffinity(InMemoryTentacleScriptLog tentacleScriptLog) List? ParseScriptPodTolerations(InMemoryTentacleScriptLog tentacleScriptLog) => ParseScriptPodJson>( tentacleScriptLog, - KubernetesConfig.PodTolerationsJson, - KubernetesConfig.PodTolerationsJsonVariableName, + KubernetesConfiguration.PodTolerationsJson, + EnvironmentKubernetesConfiguration.VariableNames.PodTolerationsJsonVariableName, "pod tolerations"); V1PodSecurityContext? ParseScriptPodSecurityContext(InMemoryTentacleScriptLog tentacleScriptLog) => ParseScriptPodJson( tentacleScriptLog, - KubernetesConfig.PodSecurityContextJson, - KubernetesConfig.PodSecurityContextJsonVariableName, + KubernetesConfiguration.PodSecurityContextJson, + EnvironmentKubernetesConfiguration.VariableNames.PodSecurityContextJsonVariableName, "pod security context"); [return: NotNullIfNotNull("defaultValue")] @@ -430,9 +434,9 @@ V1Affinity ParseScriptPodAffinity(InMemoryTentacleScriptLog tentacleScriptLog) return defaultValue; } - static V1Container? CreateWatchdogContainer(string homeDir) + V1Container? CreateWatchdogContainer(string homeDir) { - if (KubernetesConfig.NfsWatchdogImage is null) + if (KubernetesConfiguration.NfsWatchdogImage is null) { return null; } @@ -440,7 +444,7 @@ V1Affinity ParseScriptPodAffinity(InMemoryTentacleScriptLog tentacleScriptLog) return new V1Container { Name = "nfs-watchdog", - Image = KubernetesConfig.NfsWatchdogImage, + Image = KubernetesConfiguration.NfsWatchdogImage, VolumeMounts = new List { new(homeDir, "tentacle-home"), diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesSecretService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesSecretService.cs index 866c77f40..7574916a8 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesSecretService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesSecretService.cs @@ -19,8 +19,8 @@ public interface IKubernetesSecretService public class KubernetesSecretService : KubernetesService, IKubernetesSecretService { - public KubernetesSecretService(IKubernetesClientConfigProvider configProvider, ISystemLog log) - : base(configProvider, log) + public KubernetesSecretService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) + : base(configProvider,kubernetesConfiguration, log) { } @@ -30,7 +30,7 @@ public KubernetesSecretService(IKubernetesClientConfigProvider configProvider, I { try { - return await Client.ReadNamespacedSecretAsync(name, KubernetesConfig.Namespace, cancellationToken: cancellationToken); + return await Client.ReadNamespacedSecretAsync(name, Namespace, cancellationToken: cancellationToken); } catch (HttpOperationException opException) when (opException.Response.StatusCode == HttpStatusCode.NotFound) @@ -45,7 +45,7 @@ public async Task CreateSecretAsync(V1Secret secret, CancellationToken cancellat AddStandardMetadata(secret); //We only want to retry read/modify operations for now (since they are idempotent) - await Client.CreateNamespacedSecretAsync(secret, KubernetesConfig.Namespace, cancellationToken: cancellationToken); + await Client.CreateNamespacedSecretAsync(secret, Namespace, cancellationToken: cancellationToken); } public async Task UpdateSecretDataAsync(string secretName, IDictionary secretData, CancellationToken cancellationToken) @@ -58,7 +58,7 @@ public async Task UpdateSecretDataAsync(string secretName, IDictionary var patchYaml = KubernetesJson.Serialize(patchSecret); return await RetryPolicy.ExecuteAsync(async () => - await Client.PatchNamespacedSecretAsync(new V1Patch(patchYaml, V1Patch.PatchType.MergePatch), secretName, KubernetesConfig.Namespace, cancellationToken: cancellationToken)); + await Client.PatchNamespacedSecretAsync(new V1Patch(patchYaml, V1Patch.PatchType.MergePatch), secretName, Namespace, cancellationToken: cancellationToken)); } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesService.cs index 27614adee..8a67e0119 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesService.cs @@ -20,9 +20,14 @@ public abstract class KubernetesService protected ISystemLog Log { get; } protected AsyncRetryPolicy RetryPolicy { get; } protected k8sClient Client { get; } + + protected IKubernetesConfiguration KubernetesConfiguration { get; } + + protected string Namespace => KubernetesConfiguration.Namespace; - protected KubernetesService(IKubernetesClientConfigProvider configProvider, ISystemLog log) + protected KubernetesService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) { + KubernetesConfiguration = kubernetesConfiguration; Log = log; Client = new k8sClient(configProvider.Get()); RetryPolicy = Policy.Handle().WaitAndRetryAsync(5, @@ -37,12 +42,12 @@ protected KubernetesService(IKubernetesClientConfigProvider configProvider, ISys protected void AddStandardMetadata(IKubernetesObject k8sObject) { //Everything should be in the main namespace - k8sObject.Metadata.NamespaceProperty = KubernetesConfig.Namespace; + k8sObject.Metadata.NamespaceProperty = Namespace; - //Add helm specific metadata so it's removed if the helm release is uninstalled + //Add helm specific metadata, so it's removed if the helm release is uninstalled k8sObject.Metadata.Annotations ??= new Dictionary(); - k8sObject.Metadata.Annotations["meta.helm.sh/release-name"] = KubernetesConfig.HelmReleaseName; - k8sObject.Metadata.Annotations["meta.helm.sh/release-namespace"] = KubernetesConfig.Namespace; + k8sObject.Metadata.Annotations["meta.helm.sh/release-name"] = KubernetesConfiguration.HelmReleaseName; + k8sObject.Metadata.Annotations["meta.helm.sh/release-namespace"] = Namespace; k8sObject.Metadata.Labels ??= new Dictionary(); k8sObject.Metadata.Labels["app.kubernetes.io/managed-by"] = "Helm"; diff --git a/source/Octopus.Tentacle/Octopus.Tentacle.csproj b/source/Octopus.Tentacle/Octopus.Tentacle.csproj index 6247dc364..4911ca26f 100644 --- a/source/Octopus.Tentacle/Octopus.Tentacle.csproj +++ b/source/Octopus.Tentacle/Octopus.Tentacle.csproj @@ -12,7 +12,7 @@ Octopus.Tentacle net48;net8.0;net8.0-windows true - 9 + default enable diff --git a/source/Tentacle.sln.DotSettings b/source/Tentacle.sln.DotSettings index 0e7802a6d..2c88b318a 100644 --- a/source/Tentacle.sln.DotSettings +++ b/source/Tentacle.sln.DotSettings @@ -183,6 +183,7 @@ True True True + True True True True From 79d735990b63de4620ed228149f2c37d94a9c23f Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Fri, 20 Dec 2024 09:23:58 +1100 Subject: [PATCH 2/9] Fix few more issues and add IKubernetesAgentDetection --- .../EnvironmentKubernetesConfiguration.cs | 37 ++++++++++++------- .../Kubernetes/IKubernetesAgentDetection.cs | 30 +++++++++++++++ .../Kubernetes/IKubernetesConfiguration.cs | 14 +++---- .../Kubernetes/KubernetesScriptPodCreator.cs | 2 +- .../NullableReferenceTypeAttributes.cs | 9 ++++- .../Kubernetes/KubernetesScriptServiceV1.cs | 5 ++- 6 files changed, 73 insertions(+), 24 deletions(-) create mode 100644 source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs diff --git a/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs b/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs index bb1c74c0e..b3c95f910 100644 --- a/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs +++ b/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs @@ -19,10 +19,19 @@ public static class VariableNames public const string HelmChartVersionVariableName = $"{EnvVarPrefix}__HELMCHARTVERSION"; public const string ServerCommsAddressesVariableName = "ServerCommsAddresses"; + public const string ScriptPodResourceJsonVariableName = $"{EnvVarPrefix}__PODRESOURCEJSON"; public const string PodAffinityJsonVariableName = $"{EnvVarPrefix}__PODAFFINITYJSON"; public const string PodTolerationsJsonVariableName = $"{EnvVarPrefix}__PODTOLERATIONSJSON"; public const string PodSecurityContextJsonVariableName = $"{EnvVarPrefix}__PODSECURITYCONTEXTJSON"; public const string PodResourceJsonVariableName = $"{EnvVarPrefix}__PODRESOURCEJSON"; + + + public const string ScriptPodContainerImage = $"{EnvVarPrefix}__SCRIPTPODIMAGE"; + public const string ScriptPodContainerImageTag = $"{EnvVarPrefix}__SCRIPTPODIMAGETAG"; + public const string ScriptPodPullPolicy = $"{EnvVarPrefix}__SCRIPTPODPULLPOLICY"; + + + public const string ScriptPodProxiesSecretNameVariableName = $"{EnvVarPrefix}__PODPROXIESSECRETNAME"; } public string Namespace => GetRequiredEnvVar(VariableNames.NamespaceVariableName, "Unable to determine Kubernetes namespace."); @@ -36,20 +45,19 @@ public static class VariableNames .ToArray() ?? []; public string ScriptPodVolumeClaimName => GetRequiredEnvVar($"{VariableNames.EnvVarPrefix}__PODVOLUMECLAIMNAME", "Unable to determine Kubernetes Pod persistent volume claim name."); - public string? ScriptPodResourceJson => Environment.GetEnvironmentVariable(ScriptPodResourceJsonVariableName); - public string ScriptPodResourceJsonVariableName => $"{VariableNames.EnvVarPrefix}__PODRESOURCEJSON"; + public string? ScriptPodResourceJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodResourceJsonVariableName); + + public string? ScriptPodContainerImage => Environment.GetEnvironmentVariable(VariableNames.ScriptPodContainerImage); + public string ScriptPodContainerImageTag => Environment.GetEnvironmentVariable(VariableNames.ScriptPodContainerImageTag) ?? "latest"; + public string? ScriptPodPullPolicy => Environment.GetEnvironmentVariable(VariableNames.ScriptPodPullPolicy); public static string NfsWatchdogImageVariableName => $"{VariableNames.EnvVarPrefix}__NFSWATCHDOGIMAGE"; public string? NfsWatchdogImage => Environment.GetEnvironmentVariable(NfsWatchdogImageVariableName); + + public string HelmReleaseName => GetRequiredEnvVar(VariableNames.HelmReleaseNameVariableName, "Unable to determine Helm release name."); + public string HelmChartVersion => GetRequiredEnvVar(VariableNames.HelmChartVersionVariableName, "Unable to determine Helm chart version."); - public static string HelmReleaseNameVariableName => $"{VariableNames.EnvVarPrefix}__HELMRELEASENAME"; - public static string HelmChartVersionVariableName => $"{VariableNames.EnvVarPrefix}__HELMCHARTVERSION"; const string ServerCommsAddressVariableName = "ServerCommsAddress"; - public const string ServerCommsAddressesVariableName = "ServerCommsAddresses"; - - public string HelmReleaseName => GetRequiredEnvVar(HelmReleaseNameVariableName, "Unable to determine Helm release name."); - public string HelmChartVersion => GetRequiredEnvVar(HelmChartVersionVariableName, "Unable to determine Helm chart version."); - public string[] ServerCommsAddresses { get @@ -60,7 +68,7 @@ public string[] ServerCommsAddresses addresses.Add(addressString); } - if (Environment.GetEnvironmentVariable(ServerCommsAddressesVariableName) is { } addressesString) + if (Environment.GetEnvironmentVariable(VariableNames.ServerCommsAddressesVariableName) is { } addressesString) { addresses.AddRange(addressesString.Split(',').Where(a => !a.IsNullOrEmpty())); } @@ -73,11 +81,14 @@ public string[] ServerCommsAddresses public int? PodMonitorTimeoutSeconds => int.TryParse(Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__PODMONITORTIMEOUT"), out var podMonitorTimeout) ? podMonitorTimeout : 10 * 60; //10min public TimeSpan PodsConsideredOrphanedAfterTimeSpan => TimeSpan.FromMinutes(int.TryParse(Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__PODSCONSIDEREDORPHANEDAFTERMINUTES"), out var podsConsideredOrphanedAfterTimeSpan) ? podsConsideredOrphanedAfterTimeSpan : 10); public bool DisableAutomaticPodCleanup => bool.TryParse(Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__DISABLEAUTOPODCLEANUP"), out var disableAutoCleanup) && disableAutoCleanup; - - public static string PersistentVolumeSizeVariableName => $"{VariableNames.EnvVarPrefix}__PERSISTENTVOLUMESIZE"; - public string PersistentVolumeSize => GetRequiredEnvVar(PersistentVolumeSizeVariableName, "Unable to determine Persistent Volume Size"); + public bool DisablePodEventsInTaskLog => bool.TryParse(Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__DISABLEPODEVENTSINTASKLOG"), out var disable) && disable; + public string PersistentVolumeSize => GetRequiredEnvVar($"{VariableNames.EnvVarPrefix}__PERSISTENTVOLUMESIZE", "Unable to determine Persistent Volume Size"); public bool IsMetricsEnabled => !bool.TryParse(Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__ENABLEMETRICSCAPTURE"), out var enableMetrics) || enableMetrics; + public string? PodAffinityJson => Environment.GetEnvironmentVariable(VariableNames.PodAffinityJsonVariableName); + public string? PodTolerationsJson => Environment.GetEnvironmentVariable(VariableNames.PodTolerationsJsonVariableName); + public string? PodSecurityContextJson => Environment.GetEnvironmentVariable(VariableNames.PodSecurityContextJsonVariableName); + public string? ScriptPodProxiesSecretName => Environment.GetEnvironmentVariable(VariableNames.ScriptPodProxiesSecretNameVariableName); static string GetRequiredEnvVar(string variable, string errorMessage) => Environment.GetEnvironmentVariable(variable) diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs new file mode 100644 index 000000000..4250dbb0e --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs @@ -0,0 +1,30 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Octopus.Tentacle.Kubernetes; + +public interface IKubernetesAgentDetection +{ + /// + /// Indicates if the Tentacle is running inside a Kubernetes cluster as the Kubernetes Agent + /// + [MemberNotNullWhen(true, nameof(Namespace))] + bool IsRunningAsKubernetesAgent { get; } + + /// + /// The Kubernetes namespace the agent is running under, null if not running as a Kubernetes agent + /// + string? Namespace { get; } +} + +public class KubernetesAgentDetection : IKubernetesAgentDetection +{ + public static bool IsRunningAsKubernetesAgent => !string.IsNullOrWhiteSpace(Namespace); + public static string? Namespace => Environment.GetEnvironmentVariable(EnvironmentKubernetesConfiguration.VariableNames.NamespaceVariableName); + + /// + bool IKubernetesAgentDetection.IsRunningAsKubernetesAgent => IsRunningAsKubernetesAgent; + + /// + string? IKubernetesAgentDetection.Namespace => Namespace; +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs index e63c71e81..dc0f6454a 100644 --- a/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs +++ b/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs @@ -12,10 +12,9 @@ public interface IKubernetesConfiguration IEnumerable ScriptPodImagePullSecretNames { get; } string ScriptPodVolumeClaimName { get; } string? ScriptPodResourceJson { get; } - string ScriptPodResourceJsonVariableName { get; } - string ScriptPodContainerImage { get; } + string? ScriptPodContainerImage { get; } string ScriptPodContainerImageTag { get;} - string ScriptPodPullPolicy { get; set; } + string? ScriptPodPullPolicy { get; } string? NfsWatchdogImage { get; } string HelmReleaseName { get; } string HelmChartVersion { get; } @@ -27,10 +26,9 @@ public interface IKubernetesConfiguration bool DisablePodEventsInTaskLog { get; } string PersistentVolumeSize { get; } bool IsMetricsEnabled { get; } - string? PodResourceJson { get; set; } - string? PodAffinityJson { get; set; } - string? PodTolerationsJson { get; set; } - string? PodSecurityContextJson { get; set; } - string? ScriptPodProxiesSecretName { get; set; } + string? PodAffinityJson { get; } + string? PodTolerationsJson { get; } + string? PodSecurityContextJson { get; } + string? ScriptPodProxiesSecretName { get; } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs index f6f896a81..735be5f5e 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs @@ -351,7 +351,7 @@ protected async Task CreateScriptContainer(StartKubernetesScriptCom V1ResourceRequirements GetScriptPodResourceRequirements(InMemoryTentacleScriptLog tentacleScriptLog) { - var json = KubernetesConfiguration.PodResourceJson; + var json = KubernetesConfiguration.ScriptPodResourceJson; if (!string.IsNullOrWhiteSpace(json)) { try diff --git a/source/Octopus.Tentacle/NullableReferenceTypeAttributes.cs b/source/Octopus.Tentacle/NullableReferenceTypeAttributes.cs index 004ec67b0..73d8279c6 100644 --- a/source/Octopus.Tentacle/NullableReferenceTypeAttributes.cs +++ b/source/Octopus.Tentacle/NullableReferenceTypeAttributes.cs @@ -5,7 +5,7 @@ /// These attributes replicate the ones from System.Diagnostics.CodeAnalysis, and are here so we can still compile against the older frameworks. /// -namespace Octopus.Tentacle +namespace System.Diagnostics.CodeAnalysis { [AttributeUsage(AttributeTargets.Property | AttributeTargets.Parameter | AttributeTargets.ReturnValue, AllowMultiple = true)] public sealed class NotNullIfNotNullAttribute : Attribute @@ -63,5 +63,12 @@ public sealed class MemberNotNullAttribute : Attribute /// Gets field or property member names. public string[] Members { get; } } + + [AttributeUsage(AttributeTargets.Method | AttributeTargets.Property, AllowMultiple = true)] + public sealed class MemberNotNullWhenAttribute : Attribute + { + public MemberNotNullWhenAttribute(bool returnValue, params string[] members) { } + public MemberNotNullWhenAttribute(bool returnValue, string member) { } + } } #endif \ No newline at end of file diff --git a/source/Octopus.Tentacle/Services/Scripts/Kubernetes/KubernetesScriptServiceV1.cs b/source/Octopus.Tentacle/Services/Scripts/Kubernetes/KubernetesScriptServiceV1.cs index e6ed60700..c472427dd 100644 --- a/source/Octopus.Tentacle/Services/Scripts/Kubernetes/KubernetesScriptServiceV1.cs +++ b/source/Octopus.Tentacle/Services/Scripts/Kubernetes/KubernetesScriptServiceV1.cs @@ -16,6 +16,7 @@ namespace Octopus.Tentacle.Services.Scripts.Kubernetes [KubernetesService(typeof(IKubernetesScriptServiceV1))] public class KubernetesScriptServiceV1 : IAsyncKubernetesScriptServiceV1, IRunningScriptReporter { + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesPodService podService; readonly IScriptWorkspaceFactory workspaceFactory; readonly IKubernetesPodStatusProvider podStatusProvider; @@ -28,6 +29,7 @@ public class KubernetesScriptServiceV1 : IAsyncKubernetesScriptServiceV1, IRunni readonly IKeyedSemaphore keyedSemaphore; public KubernetesScriptServiceV1( + IKubernetesConfiguration kubernetesConfiguration, IKubernetesPodService podService, IScriptWorkspaceFactory workspaceFactory, IKubernetesPodStatusProvider podStatusProvider, @@ -39,6 +41,7 @@ public KubernetesScriptServiceV1( IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider, IKeyedSemaphore keyedSemaphore) { + this.kubernetesConfiguration = kubernetesConfiguration; this.podService = podService; this.workspaceFactory = workspaceFactory; this.podStatusProvider = podStatusProvider; @@ -129,7 +132,7 @@ public async Task CompleteScriptAsync(CompleteKubernetesScriptCommandV1 command, scriptPodSinceTimeStore.Delete(command.ScriptTicket); scriptPodLogEncryptionKeyProvider.Delete(command.ScriptTicket); - if (!KubernetesConfig.DisableAutomaticPodCleanup) + if (!kubernetesConfiguration.DisableAutomaticPodCleanup) await podService.DeleteIfExists(command.ScriptTicket, cancellationToken); } From dfcbcef61d7b3a708e5b23b2c750e96fc8a68330 Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Fri, 20 Dec 2024 09:45:09 +1100 Subject: [PATCH 3/9] Complete compilation --- .../KubernetesAgentMetricsIntegrationTest.cs | 29 +++++++------ ...ntacle.Kubernetes.Tests.Integration.csproj | 1 + .../Support/TestSupportConfigMapService.cs | 5 +-- .../CapabilitiesServiceV2Fixture.cs | 13 +++--- .../ApplicationInstanceSelectorFixture.cs | 9 ++-- ...etesDirectoryInformationProviderFixture.cs | 41 +++++++++---------- .../KubernetesOrphanedPodCleanerTests.cs | 13 ++++-- .../KubernetesPodContainerResolverTests.cs | 12 +++--- .../Crypto/LinuxGeneratedMachineKey.cs | 3 +- .../Instances/ApplicationInstanceManager.cs | 9 ++-- .../Instances/ApplicationInstanceSelector.cs | 10 +++-- .../Kubernetes/IKubernetesAgentDetection.cs | 4 ++ source/Octopus.Tentacle/Program.cs | 4 +- .../Capabilities/CapabilitiesServiceV2.cs | 10 ++++- .../Services/ServicesModule.cs | 3 +- .../Startup/OctopusProgram.cs | 4 +- .../Util/OctopusFileSystemModule.cs | 2 +- .../Util/OctopusPhysicalFileSystem.cs | 4 -- .../Util/PlatformDetection.cs | 9 ---- 19 files changed, 105 insertions(+), 80 deletions(-) diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs index 724f5269f..0826c821e 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs @@ -2,6 +2,7 @@ using FluentAssertions; using k8s; using Newtonsoft.Json; +using NSubstitute; using Octopus.Diagnostics; using Octopus.Tentacle.Diagnostics; using Octopus.Tentacle.Kubernetes.Diagnostics; @@ -26,19 +27,20 @@ public KubernetesClientConfiguration Get() return KubernetesClientConfiguration.BuildConfigFromConfigFile(filename); } } - + [Test] public async Task FetchingTimestampFromEmptyConfigMapEntryShouldBeMinValue() { //Arrange + var config = Substitute.For(); var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath); - var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); - var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", configMapService); + var configMapService = new Support.TestSupportConfigMapService(config, kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); + var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", config, configMapService); var metrics = new KubernetesAgentMetrics(persistenceProvider, ConfigMapNames.AgentMetricsConfigMapKey, systemLog); //Act var result = await metrics.GetLatestEventTimestamp(CancellationToken.None); - + //Assert result.Should().Be(DateTimeOffset.MinValue); } @@ -47,14 +49,15 @@ public async Task FetchingTimestampFromEmptyConfigMapEntryShouldBeMinValue() public async Task FetchingLatestEventTimestampFromNonexistentConfigMapThrowsException() { //Arrange + var config = Substitute.For(); var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath); - var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); - var persistenceProvider = new PersistenceProvider("nonexistent-config-map", configMapService); + var configMapService = new Support.TestSupportConfigMapService(config, kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); + var persistenceProvider = new PersistenceProvider("nonexistent-config-map", config, configMapService); var metrics = new KubernetesAgentMetrics(persistenceProvider, "metrics", systemLog); //Act Func func = async () => await metrics.GetLatestEventTimestamp(CancellationToken.None); - + //Assert await func.Should().ThrowAsync(); } @@ -63,9 +66,10 @@ public async Task FetchingLatestEventTimestampFromNonexistentConfigMapThrowsExce public async Task WritingEventToNonExistentConfigMapShouldFailSilently() { //Arrange + var config = Substitute.For(); var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath); - var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); - var persistenceProvider = new PersistenceProvider("nonexistent-config-map", configMapService); + var configMapService = new Support.TestSupportConfigMapService(config, kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); + var persistenceProvider = new PersistenceProvider("nonexistent-config-map", config, configMapService); var metrics = new KubernetesAgentMetrics(persistenceProvider, ConfigMapNames.AgentMetricsConfigMapKey, systemLog); //Act @@ -79,15 +83,16 @@ public async Task WritingEventToNonExistentConfigMapShouldFailSilently() public async Task WritingEventToExistingConfigMapShouldPersistJsonEntry() { //Arrange + var config = Substitute.For(); var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath); - var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); - var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", configMapService); + var configMapService = new Support.TestSupportConfigMapService(config, kubernetesConfigClient, systemLog, KubernetesAgentInstaller.Namespace); + var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", config, configMapService); var metrics = new KubernetesAgentMetrics(persistenceProvider, ConfigMapNames.AgentMetricsConfigMapKey, systemLog); //Act var eventTimestamp = DateTimeOffset.Now; await metrics.TrackEvent("reason", "source", eventTimestamp, CancellationToken.None); - + //Assert var persistedDictionary = await persistenceProvider.ReadValues(CancellationToken.None); var metricsData = persistedDictionary[ConfigMapNames.AgentMetricsConfigMapKey]; diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj index 412854765..f2e545276 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj @@ -18,6 +18,7 @@ + diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Support/TestSupportConfigMapService.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Support/TestSupportConfigMapService.cs index af9b58fda..e6d99b2b4 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Support/TestSupportConfigMapService.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Support/TestSupportConfigMapService.cs @@ -7,15 +7,14 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration.Support { - // This is a copy of the production ConfigMapService, but allows the namespace to be explicitly // defined. public class TestSupportConfigMapService : KubernetesService, IKubernetesConfigMapService { readonly string targetNamespace; - public TestSupportConfigMapService(IKubernetesClientConfigProvider configProvider, ISystemLog log, string targetNamespace) - : base(configProvider, log) + public TestSupportConfigMapService(IKubernetesConfiguration kubernetesConfiguration, IKubernetesClientConfigProvider configProvider, ISystemLog log, string targetNamespace) + : base(configProvider, kubernetesConfiguration, log) { this.targetNamespace = targetNamespace; } diff --git a/source/Octopus.Tentacle.Tests/Capabilities/CapabilitiesServiceV2Fixture.cs b/source/Octopus.Tentacle.Tests/Capabilities/CapabilitiesServiceV2Fixture.cs index 0c6326a05..de4e8ea2a 100644 --- a/source/Octopus.Tentacle.Tests/Capabilities/CapabilitiesServiceV2Fixture.cs +++ b/source/Octopus.Tentacle.Tests/Capabilities/CapabilitiesServiceV2Fixture.cs @@ -2,6 +2,7 @@ using System.Threading; using System.Threading.Tasks; using FluentAssertions; +using NSubstitute; using NUnit.Framework; using Octopus.Tentacle.Contracts; using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1; @@ -16,7 +17,10 @@ public class CapabilitiesServiceV2Fixture [Test] public async Task CapabilitiesAreReturned() { - var capabilities = (await new CapabilitiesServiceV2() + var k8sDetection = Substitute.For(); + k8sDetection.IsRunningAsKubernetesAgent.Returns(false); + + var capabilities = (await new CapabilitiesServiceV2(k8sDetection) .GetCapabilitiesAsync(CancellationToken.None)) .SupportedCapabilities; @@ -29,9 +33,10 @@ public async Task CapabilitiesAreReturned() [Test] public async Task OnlyKubernetesScriptServicesAreReturnedWhenRunningAsKubernetesAgent() { - Environment.SetEnvironmentVariable(KubernetesConfig.NamespaceVariableName, "ABC"); + var k8sDetection = Substitute.For(); + k8sDetection.IsRunningAsKubernetesAgent.Returns(true); - var capabilities = (await new CapabilitiesServiceV2() + var capabilities = (await new CapabilitiesServiceV2(k8sDetection) .GetCapabilitiesAsync(CancellationToken.None)) .SupportedCapabilities; @@ -39,8 +44,6 @@ public async Task OnlyKubernetesScriptServicesAreReturnedWhenRunningAsKubernetes capabilities.Count.Should().Be(2); capabilities.Should().NotContainMatch("IScriptService*"); - - Environment.SetEnvironmentVariable(KubernetesConfig.NamespaceVariableName, null); } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests/Configuration/ApplicationInstanceSelectorFixture.cs b/source/Octopus.Tentacle.Tests/Configuration/ApplicationInstanceSelectorFixture.cs index e34be43b1..cdf1e856b 100644 --- a/source/Octopus.Tentacle.Tests/Configuration/ApplicationInstanceSelectorFixture.cs +++ b/source/Octopus.Tentacle.Tests/Configuration/ApplicationInstanceSelectorFixture.cs @@ -188,13 +188,16 @@ ApplicationInstanceSelector CreateApplicationInstanceSelector(StartUpInstanceReq IApplicationConfigurationContributor[]? additionalConfigurations = null) { var log = Substitute.For(); + var kubernetesConfig = Substitute.For(); + var k8sDetection = Substitute.For(); return new ApplicationInstanceSelector(ApplicationName.Tentacle, applicationInstanceStore, instanceRequest ?? new StartUpDynamicInstanceRequest(), - additionalConfigurations ?? new IApplicationConfigurationContributor[0], - new Lazy(() => new ConfigMapKeyValueStore(Substitute.For(), Substitute.For())), + additionalConfigurations ?? Array.Empty(), + new Lazy(() => new ConfigMapKeyValueStore(kubernetesConfig,Substitute.For(), Substitute.For())), octopusFileSystem, - log); + log, + k8sDetection); } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesDirectoryInformationProviderFixture.cs b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesDirectoryInformationProviderFixture.cs index 5f89b8126..558c4cfbc 100644 --- a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesDirectoryInformationProviderFixture.cs +++ b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesDirectoryInformationProviderFixture.cs @@ -18,7 +18,7 @@ public class KubernetesDirectoryInformationProviderFixture { // Sizes const ulong Megabyte = 1000 * 1000; - + [Test] public void DuOutputParses() { @@ -30,10 +30,10 @@ public void DuOutputParses() x.ArgAt>(3).Invoke($"{usedSize}\t/octopus"); }); var memoryCache = new MemoryCache(new MemoryCacheOptions()); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(usedSize); } - + [Test] public void DuOutputParsesWithMultipleLines() { @@ -44,10 +44,10 @@ public void DuOutputParsesWithMultipleLines() { x.ArgAt>(3).Invoke($"500\t/octopus/extradir"); x.ArgAt>(3).Invoke($"{usedSize}\t/octopus"); - x.ArgAt>(3).Invoke($"{usedSize+1000}\tTotal"); + x.ArgAt>(3).Invoke($"{usedSize + 1000}\tTotal"); }); var memoryCache = new MemoryCache(new MemoryCacheOptions()); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(usedSize); } @@ -64,10 +64,10 @@ public void IfDuFailsWeStillGetData() }); spr.ReturnsForAll(1); var memoryCache = new MemoryCache(new MemoryCacheOptions()); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(usedSize); } - + [Test] public void IfDuFailsWeLogCorrectly() { @@ -81,16 +81,16 @@ public void IfDuFailsWeLogCorrectly() // stdout x.ArgAt>(3).Invoke("500\t/octopus"); x.ArgAt>(3).Invoke($"{usedSize}\t/octopus"); - + // stderr x.ArgAt>(4).Invoke("no permission for foo"); x.ArgAt>(4).Invoke("also no permission for bar"); }); spr.ReturnsForAll(1); var memoryCache = new MemoryCache(new MemoryCacheOptions()); - var sut = new KubernetesDirectoryInformationProvider(systemLog, spr, memoryCache); + var sut = new KubernetesDirectoryInformationProvider(systemLog, Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(usedSize); - + systemLog.GetLogsForCategory(LogCategory.Warning).Should().Contain("Could not reliably get disk space using du. Getting best approximation..."); systemLog.GetLogsForCategory(LogCategory.Info).Should().Contain($"Du stdout returned 500\t/octopus, {usedSize}\t/octopus"); systemLog.GetLogsForCategory(LogCategory.Info).Should().Contain("Du stderr returned no permission for foo, also no permission for bar"); @@ -102,10 +102,10 @@ public void IfDuFailsCompletelyReturnNull() var spr = Substitute.For(); spr.ReturnsForAll(1); var memoryCache = new MemoryCache(new MemoryCacheOptions()); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(null); } - + [Test] public void ReturnedValueShouldBeCached() { @@ -113,10 +113,10 @@ public void ReturnedValueShouldBeCached() spr.ReturnsForAll(1); var baseTime = DateTimeOffset.UtcNow; var clock = new TestClock(baseTime); - var memoryCache = new MemoryCache(new MemoryCacheOptions(){ Clock = clock}); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var memoryCache = new MemoryCache(new MemoryCacheOptions() { Clock = clock }); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(null); - + const ulong usedSize = 500 * Megabyte; spr.When(x => x.ExecuteCommand("du", "-s -B 1 /octopus", "/", Arg.Any>(), Arg.Any>())) .Do(x => @@ -128,7 +128,7 @@ public void ReturnedValueShouldBeCached() sut.GetPathUsedBytes("/octopus").Should().Be(null); } - + [Test] public void DuCacheExpiresAfter30Seconds() { @@ -136,10 +136,10 @@ public void DuCacheExpiresAfter30Seconds() spr.ReturnsForAll(1); var baseTime = DateTimeOffset.UtcNow; var clock = new TestClock(baseTime); - var memoryCache = new MemoryCache(new MemoryCacheOptions(){ Clock = clock}); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var memoryCache = new MemoryCache(new MemoryCacheOptions() { Clock = clock }); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(null); - + const ulong usedSize = 500 * Megabyte; spr.When(x => x.ExecuteCommand("du", "-s -B 1 /octopus", "/", Arg.Any>(), Arg.Any>())) .Do(x => @@ -149,10 +149,9 @@ public void DuCacheExpiresAfter30Seconds() }); clock.UtcNow = baseTime + TimeSpan.FromSeconds(30); - + sut.GetPathUsedBytes("/octopus").Should().Be(usedSize); } - } public class TestClock : ISystemClock diff --git a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs index bc026e045..461f2b9d2 100644 --- a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs +++ b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs @@ -28,6 +28,7 @@ public class KubernetesOrphanedPodCleanerTests ITentacleScriptLogProvider scriptLogProvider; IScriptPodSinceTimeStore scriptPodSinceTimeStore; IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider; + IKubernetesConfiguration config; [SetUp] public void Setup() @@ -42,8 +43,12 @@ public void Setup() scriptPodLogEncryptionKeyProvider = Substitute.For(); monitor = Substitute.For(); scriptTicket = new ScriptTicket(Guid.NewGuid().ToString()); + + config = Substitute.For(); + config.PodsConsideredOrphanedAfterTimeSpan.Returns(TimeSpan.FromMinutes(10)); + config.DisableAutomaticPodCleanup.Returns(false); - cleaner = new KubernetesOrphanedPodCleaner(monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore, scriptPodLogEncryptionKeyProvider); + cleaner = new KubernetesOrphanedPodCleaner(config, monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore, scriptPodLogEncryptionKeyProvider); overCutoff = cleaner.CompletedPodConsideredOrphanedAfterTimeSpan + 1.Minutes(); underCutoff = cleaner.CompletedPodConsideredOrphanedAfterTimeSpan - 1.Minutes(); @@ -149,7 +154,7 @@ public async Task OrphanedPodNotCleanedUpIfOnly9MinutesHavePassed() public async Task OrphanedPodNotCleanedUpIfPodCleanupIsDisabled() { //Arrange - Environment.SetEnvironmentVariable("OCTOPUS__K8STENTACLE__DISABLEAUTOPODCLEANUP", "true"); + config.DisableAutomaticPodCleanup.Returns(true); var pods = new List { CreatePod(TrackedScriptPodState.Succeeded(0, startTime)) @@ -172,10 +177,10 @@ public async Task OrphanedPodNotCleanedUpIfPodCleanupIsDisabled() public async Task EnvironmentVariableDictatesWhenPodsAreConsideredOrphaned(int checkAfterMinutes, bool shouldDelete) { //Arrange - Environment.SetEnvironmentVariable("OCTOPUS__K8STENTACLE__PODSCONSIDEREDORPHANEDAFTERMINUTES", "2"); + config.PodsConsideredOrphanedAfterTimeSpan.Returns(TimeSpan.FromMinutes(2)); // We need to reinitialise the sut after changing the env var value - cleaner = new KubernetesOrphanedPodCleaner(monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore, scriptPodLogEncryptionKeyProvider); + cleaner = new KubernetesOrphanedPodCleaner(config, monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore, scriptPodLogEncryptionKeyProvider); var pods = new List { CreatePod(TrackedScriptPodState.Succeeded(0, startTime)) diff --git a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesPodContainerResolverTests.cs b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesPodContainerResolverTests.cs index 216938fdc..97b3a92bc 100644 --- a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesPodContainerResolverTests.cs +++ b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesPodContainerResolverTests.cs @@ -26,11 +26,13 @@ public class KubernetesPodContainerResolverTests }); readonly IToolsImageVersionMetadataProvider mockToolsImageVersionMetadataProvider = Substitute.For(); + IKubernetesConfiguration kubernetesConfig; [SetUp] public void Init() { mockToolsImageVersionMetadataProvider.TryGetVersionMetadata().Returns(testVersionMetadata); + kubernetesConfig = Substitute.For(); } [TestCase(30)] @@ -42,7 +44,7 @@ public async Task GetContainerImageForCluster_VersionMetadataExists_ClusterVersi var clusterService = Substitute.For(); clusterService.GetClusterVersion().Returns(new ClusterVersion(1, clusterMinorVersion)); - var podContainerResolver = new KubernetesPodContainerResolver(clusterService, mockToolsImageVersionMetadataProvider); + var podContainerResolver = new KubernetesPodContainerResolver(kubernetesConfig, clusterService, mockToolsImageVersionMetadataProvider); // Act var result = await podContainerResolver.GetContainerImageForCluster(); @@ -59,7 +61,7 @@ public async Task GetContainerImageForCluster_VersionMetadataExists_ClusterVersi var clusterService = Substitute.For(); clusterService.GetClusterVersion().Returns(new ClusterVersion(1, clusterMinorVersion)); - var podContainerResolver = new KubernetesPodContainerResolver(clusterService, mockToolsImageVersionMetadataProvider); + var podContainerResolver = new KubernetesPodContainerResolver(kubernetesConfig, clusterService, mockToolsImageVersionMetadataProvider); // Act var result = await podContainerResolver.GetContainerImageForCluster(); @@ -75,7 +77,7 @@ public async Task GetContainerImageForCluster_VersionMetadataExists_ClusterVersi var clusterService = Substitute.For(); clusterService.GetClusterVersion().Returns(new ClusterVersion(1, 31)); - var podContainerResolver = new KubernetesPodContainerResolver(clusterService, mockToolsImageVersionMetadataProvider); + var podContainerResolver = new KubernetesPodContainerResolver(kubernetesConfig, clusterService, mockToolsImageVersionMetadataProvider); // Act var result = await podContainerResolver.GetContainerImageForCluster(); @@ -91,7 +93,7 @@ public async Task GetContainerImageForCluster_VersionMetadataExists_ClusterVersi var clusterService = Substitute.For(); clusterService.GetClusterVersion().Returns(new ClusterVersion(1, 40)); - var podContainerResolver = new KubernetesPodContainerResolver(clusterService, mockToolsImageVersionMetadataProvider); + var podContainerResolver = new KubernetesPodContainerResolver(kubernetesConfig, clusterService, mockToolsImageVersionMetadataProvider); // Act var result = await podContainerResolver.GetContainerImageForCluster(); @@ -114,7 +116,7 @@ public async Task GetContainerImageForCluster_VersionMetadataNotFound_FallBackTo clusterService.GetClusterVersion().Returns(new ClusterVersion(1, clusterMinorVersion)); mockToolsImageVersionMetadataProvider.TryGetVersionMetadata().ReturnsNull(); - var podContainerResolver = new KubernetesPodContainerResolver(clusterService, mockToolsImageVersionMetadataProvider); + var podContainerResolver = new KubernetesPodContainerResolver(kubernetesConfig, clusterService, mockToolsImageVersionMetadataProvider); // Act var result = await podContainerResolver.GetContainerImageForCluster(); diff --git a/source/Octopus.Tentacle/Configuration/Crypto/LinuxGeneratedMachineKey.cs b/source/Octopus.Tentacle/Configuration/Crypto/LinuxGeneratedMachineKey.cs index b5324d983..6088e8ca4 100644 --- a/source/Octopus.Tentacle/Configuration/Crypto/LinuxGeneratedMachineKey.cs +++ b/source/Octopus.Tentacle/Configuration/Crypto/LinuxGeneratedMachineKey.cs @@ -2,6 +2,7 @@ using System.IO; using System.Security.Cryptography; using Octopus.Diagnostics; +using Octopus.Tentacle.Kubernetes; using Octopus.Tentacle.Util; using Octopus.Tentacle.Variables; @@ -13,7 +14,7 @@ public class LinuxGeneratedMachineKey: ICryptoKeyNixSource readonly IOctopusFileSystem fileSystem; static string FileName => - PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent + KubernetesAgentDetection.IsRunningAsKubernetesAgent //if we are running in K8S, we want to save the machine key to the home directory, which is likely on a network drives ? Path.Combine(Environment.GetEnvironmentVariable(EnvironmentVariables.TentacleHome)!, "machinekey") : "/etc/octopus/machinekey"; diff --git a/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceManager.cs b/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceManager.cs index ae5482e0f..a9f1a8aa6 100644 --- a/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceManager.cs +++ b/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceManager.cs @@ -11,6 +11,7 @@ class ApplicationInstanceManager : IApplicationInstanceManager readonly IOctopusFileSystem fileSystem; readonly ISystemLog log; readonly IApplicationInstanceStore instanceStore; + readonly IKubernetesAgentDetection kubernetesAgentDetection; readonly Lazy homeConfiguration; readonly ApplicationName applicationName; readonly StartUpInstanceRequest startUpInstanceRequest; @@ -21,6 +22,7 @@ public ApplicationInstanceManager( IOctopusFileSystem fileSystem, ISystemLog log, IApplicationInstanceStore instanceStore, + IKubernetesAgentDetection kubernetesAgentDetection, Lazy homeConfiguration) { this.applicationName = applicationName; @@ -28,6 +30,7 @@ public ApplicationInstanceManager( this.fileSystem = fileSystem; this.log = log; this.instanceStore = instanceStore; + this.kubernetesAgentDetection = kubernetesAgentDetection; this.homeConfiguration = homeConfiguration; } @@ -67,11 +70,11 @@ void WriteHomeDirectory(string homeDirectory) void EnsureConfigurationFileExists(string configurationFile, string homeDirectory) { - //Skip this step if we're running on Kubernetes - if (PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) return; + //Skip this step if we're running as a Kubernetes Agent + if (kubernetesAgentDetection.IsRunningAsKubernetesAgent) return; // Ensure we can write configuration file - string configurationDirectory = Path.GetDirectoryName(configurationFile) ?? homeDirectory; + var configurationDirectory = Path.GetDirectoryName(configurationFile) ?? homeDirectory; fileSystem.EnsureDirectoryExists(configurationDirectory); if (!fileSystem.FileExists(configurationFile)) { diff --git a/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceSelector.cs b/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceSelector.cs index 5a3765ae7..56f9724fe 100644 --- a/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceSelector.cs +++ b/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceSelector.cs @@ -15,6 +15,7 @@ class ApplicationInstanceSelector : IApplicationInstanceSelector readonly IApplicationConfigurationContributor[] instanceStrategies; readonly IOctopusFileSystem fileSystem; readonly ISystemLog log; + readonly IKubernetesAgentDetection kubernetesAgentDetection; readonly object @lock = new object(); ApplicationInstanceConfiguration? current; Lazy configMapStoreFactory; @@ -26,13 +27,15 @@ public ApplicationInstanceSelector( IApplicationConfigurationContributor[] instanceStrategies, Lazy configMapStoreFactory, IOctopusFileSystem fileSystem, - ISystemLog log) + ISystemLog log, + IKubernetesAgentDetection kubernetesAgentDetection) { this.applicationInstanceStore = applicationInstanceStore; this.startUpInstanceRequest = startUpInstanceRequest; this.instanceStrategies = instanceStrategies; this.fileSystem = fileSystem; this.log = log; + this.kubernetesAgentDetection = kubernetesAgentDetection; ApplicationName = applicationName; this.configMapStoreFactory = configMapStoreFactory; } @@ -78,10 +81,9 @@ ApplicationInstanceConfiguration LoadInstance() (IKeyValueStore, IWritableKeyValueStore) LoadConfigurationStore((string? instanceName, string? configurationpath) appInstance) { - if (appInstance is { instanceName: not null, configurationpath: null } && - PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) + if (appInstance is { instanceName: not null, configurationpath: null } && kubernetesAgentDetection.IsRunningAsKubernetesAgent) { - log.Verbose($"Loading configuration from ConfigMap for namespace {KubernetesConfig.Namespace}"); + log.Verbose($"Loading configuration from ConfigMap for namespace {kubernetesAgentDetection.Namespace}"); var configMapWritableStore = configMapStoreFactory.Value; return (ContributeAdditionalConfiguration(configMapWritableStore), configMapWritableStore); } diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs index 4250dbb0e..5c1ac81bf 100644 --- a/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs +++ b/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs @@ -17,6 +17,10 @@ public interface IKubernetesAgentDetection string? Namespace { get; } } +/// +/// Used for detection if the tentacle is running as part of the Kubernetes agent helm chart +/// Can be used with dependency injection via or statically +/// public class KubernetesAgentDetection : IKubernetesAgentDetection { public static bool IsRunningAsKubernetesAgent => !string.IsNullOrWhiteSpace(Namespace); diff --git a/source/Octopus.Tentacle/Program.cs b/source/Octopus.Tentacle/Program.cs index a9a05a74f..774b1097d 100644 --- a/source/Octopus.Tentacle/Program.cs +++ b/source/Octopus.Tentacle/Program.cs @@ -58,7 +58,9 @@ public override IContainer BuildContainer(StartUpInstanceRequest startUpInstance builder.RegisterModule(new VersioningModule(GetType().Assembly)); builder.RegisterModule(new MaintenanceModule()); - if (PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) + //register kubernetes agent detection + builder.RegisterType().As().SingleInstance(); + if (KubernetesAgentDetection.IsRunningAsKubernetesAgent) { builder.RegisterModule(); } diff --git a/source/Octopus.Tentacle/Services/Capabilities/CapabilitiesServiceV2.cs b/source/Octopus.Tentacle/Services/Capabilities/CapabilitiesServiceV2.cs index 6ab67d9b3..d138ab40d 100644 --- a/source/Octopus.Tentacle/Services/Capabilities/CapabilitiesServiceV2.cs +++ b/source/Octopus.Tentacle/Services/Capabilities/CapabilitiesServiceV2.cs @@ -5,6 +5,7 @@ using Octopus.Tentacle.Contracts.Capabilities; using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1; using Octopus.Tentacle.Contracts.ScriptServiceV2; +using Octopus.Tentacle.Kubernetes; using Octopus.Tentacle.Util; namespace Octopus.Tentacle.Services.Capabilities @@ -12,12 +13,19 @@ namespace Octopus.Tentacle.Services.Capabilities [Service(typeof(ICapabilitiesServiceV2))] public class CapabilitiesServiceV2 : IAsyncCapabilitiesServiceV2 { + readonly IKubernetesAgentDetection kubernetesAgentDetection; + + public CapabilitiesServiceV2(IKubernetesAgentDetection kubernetesAgentDetection) + { + this.kubernetesAgentDetection = kubernetesAgentDetection; + } + public async Task GetCapabilitiesAsync(CancellationToken cancellationToken) { await Task.CompletedTask; //the kubernetes agent only supports the kubernetes script services - if (PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) + if (kubernetesAgentDetection.IsRunningAsKubernetesAgent) { return new CapabilitiesResponseV2(new List { nameof(IFileTransferService), nameof(IKubernetesScriptServiceV1) }); } diff --git a/source/Octopus.Tentacle/Services/ServicesModule.cs b/source/Octopus.Tentacle/Services/ServicesModule.cs index 4e24c14e4..e4157130f 100644 --- a/source/Octopus.Tentacle/Services/ServicesModule.cs +++ b/source/Octopus.Tentacle/Services/ServicesModule.cs @@ -4,6 +4,7 @@ using System.Reflection; using Autofac; using Octopus.Tentacle.Communications; +using Octopus.Tentacle.Kubernetes; using Octopus.Tentacle.Packages; using Octopus.Tentacle.Scripts; using Octopus.Tentacle.Util; @@ -27,7 +28,7 @@ protected override void Load(ContainerBuilder builder) RegisterHalibutServices(builder, allTypes); //only register kubernetes services when - if (PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) + if (KubernetesAgentDetection.IsRunningAsKubernetesAgent) { RegisterHalibutServices(builder, allTypes); } diff --git a/source/Octopus.Tentacle/Startup/OctopusProgram.cs b/source/Octopus.Tentacle/Startup/OctopusProgram.cs index bc7b6687f..5c4f5c5d4 100644 --- a/source/Octopus.Tentacle/Startup/OctopusProgram.cs +++ b/source/Octopus.Tentacle/Startup/OctopusProgram.cs @@ -308,7 +308,7 @@ void InitializeLogging() Target.Register("EventLog"); #endif #if REQUIRES_EXPLICIT_LOG_CONFIG - var nLogFileExtension = !PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent + var nLogFileExtension = !KubernetesAgentDetection.IsRunningAsKubernetesAgent ? "exe.nlog" : "exe.k8s.nlog"; @@ -384,7 +384,7 @@ StartUpInstanceRequest TryLoadInstanceNameFromCommandLineArguments(string[] comm if (!string.IsNullOrWhiteSpace(instanceName)) { - return PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent + return KubernetesAgentDetection.IsRunningAsKubernetesAgent ? new StartUpKubernetesConfigMapInstanceRequest(instanceName) : new StartUpRegistryInstanceRequest(instanceName); } diff --git a/source/Octopus.Tentacle/Util/OctopusFileSystemModule.cs b/source/Octopus.Tentacle/Util/OctopusFileSystemModule.cs index 53ab25598..d98451dfd 100644 --- a/source/Octopus.Tentacle/Util/OctopusFileSystemModule.cs +++ b/source/Octopus.Tentacle/Util/OctopusFileSystemModule.cs @@ -9,7 +9,7 @@ public class OctopusFileSystemModule : Module protected override void Load(ContainerBuilder builder) { base.Load(builder); - if (PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) + if (KubernetesAgentDetection.IsRunningAsKubernetesAgent) { builder.RegisterType().AsSelf().As(); } diff --git a/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs b/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs index fe5f93a6b..5d37b9470 100644 --- a/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs +++ b/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs @@ -265,10 +265,6 @@ public virtual void EnsureDiskHasEnoughFreeSpace(string directoryPath, long requ if (!Path.IsPathRooted(directoryPath)) return; - //We can't perform this check in Kubernetes due to how drives are mounted and reported (always returns 0 byte sized drives) - if(PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) - return; - var driveInfo = SafelyGetDriveInfo(directoryPath); var required = requiredSpaceInBytes < 0 ? 0 : (ulong)requiredSpaceInBytes; diff --git a/source/Octopus.Tentacle/Util/PlatformDetection.cs b/source/Octopus.Tentacle/Util/PlatformDetection.cs index 801e10bfe..3d7b219d7 100644 --- a/source/Octopus.Tentacle/Util/PlatformDetection.cs +++ b/source/Octopus.Tentacle/Util/PlatformDetection.cs @@ -1,6 +1,5 @@ using System; using System.Runtime.InteropServices; -using Octopus.Tentacle.Kubernetes; namespace Octopus.Tentacle.Util { @@ -9,13 +8,5 @@ public static class PlatformDetection public static bool IsRunningOnNix => RuntimeInformation.IsOSPlatform(OSPlatform.Linux); public static bool IsRunningOnWindows => RuntimeInformation.IsOSPlatform(OSPlatform.Windows); public static bool IsRunningOnMac => RuntimeInformation.IsOSPlatform(OSPlatform.OSX); - - public static class Kubernetes - { - /// - /// Indicates if the Tentacle is running inside a Kubernetes cluster as the Kubernetes Agent. This is done by checking if the namespace environment variable is set - /// - public static bool IsRunningAsKubernetesAgent => !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(KubernetesConfig.NamespaceVariableName)); - } } } \ No newline at end of file From b084c60b42d8fdd33e1ec8f9c7e4f5fc9c80595d Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Fri, 20 Dec 2024 09:53:57 +1100 Subject: [PATCH 4/9] Add config for injection --- source/Octopus.Tentacle/Program.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/source/Octopus.Tentacle/Program.cs b/source/Octopus.Tentacle/Program.cs index 774b1097d..ef470cf96 100644 --- a/source/Octopus.Tentacle/Program.cs +++ b/source/Octopus.Tentacle/Program.cs @@ -60,6 +60,7 @@ public override IContainer BuildContainer(StartUpInstanceRequest startUpInstance //register kubernetes agent detection builder.RegisterType().As().SingleInstance(); + builder.RegisterType().As().SingleInstance(); if (KubernetesAgentDetection.IsRunningAsKubernetesAgent) { builder.RegisterModule(); From f36a96cc888259bd5cfc49983cba1911641705c1 Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Fri, 20 Dec 2024 10:42:53 +1100 Subject: [PATCH 5/9] Validate that EnvironmentKubernetesConfiguration matches old KubernetesConfig --- .../EnvironmentKubernetesConfiguration.cs | 94 +++++++++------- .../Kubernetes/IKubernetesAgentDetection.cs | 2 +- .../Kubernetes/IKubernetesConfiguration.cs | 1 - .../Kubernetes/KubernetesConfig.cs | 104 ------------------ .../KubernetesRawScriptPodCreator.cs | 2 +- .../Kubernetes/KubernetesScriptPodCreator.cs | 22 ++-- 6 files changed, 68 insertions(+), 157 deletions(-) delete mode 100644 source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs diff --git a/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs b/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs index b3c95f910..3798c453f 100644 --- a/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs +++ b/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs @@ -9,21 +9,29 @@ public class EnvironmentKubernetesConfiguration : IKubernetesConfiguration { public static class VariableNames { - public const string EnvVarPrefix = "OCTOPUS__K8STENTACLE"; + const string EnvVarPrefix = "OCTOPUS__K8STENTACLE"; + + public const string Namespace = $"{EnvVarPrefix}__NAMESPACE"; - public const string PersistentVolumeSizeBytesVariableName = $"{EnvVarPrefix}__PERSISTENTVOLUMETOTALBYTES"; - public const string PersistentVolumeFreeBytesVariableName = $"{EnvVarPrefix}__PERSISTENTVOLUMEFREEBYTES"; - public const string NamespaceVariableName = $"{EnvVarPrefix}__NAMESPACE"; + public const string BoolstrapRunnerExecutablePath = "BOOTSTRAPRUNNEREXECUTABLEPATH"; - public const string HelmReleaseNameVariableName = $"{EnvVarPrefix}__HELMRELEASENAME"; - public const string HelmChartVersionVariableName = $"{EnvVarPrefix}__HELMCHARTVERSION"; - public const string ServerCommsAddressesVariableName = "ServerCommsAddresses"; + public const string PersistentVolumeSizeBytes = $"{EnvVarPrefix}__PERSISTENTVOLUMETOTALBYTES"; + public const string PersistentVolumeFreeBytes = $"{EnvVarPrefix}__PERSISTENTVOLUMEFREEBYTES"; - public const string ScriptPodResourceJsonVariableName = $"{EnvVarPrefix}__PODRESOURCEJSON"; - public const string PodAffinityJsonVariableName = $"{EnvVarPrefix}__PODAFFINITYJSON"; - public const string PodTolerationsJsonVariableName = $"{EnvVarPrefix}__PODTOLERATIONSJSON"; - public const string PodSecurityContextJsonVariableName = $"{EnvVarPrefix}__PODSECURITYCONTEXTJSON"; - public const string PodResourceJsonVariableName = $"{EnvVarPrefix}__PODRESOURCEJSON"; + public const string HelmReleaseName = $"{EnvVarPrefix}__HELMRELEASENAME"; + public const string HelmChartVersion = $"{EnvVarPrefix}__HELMCHARTVERSION"; + + public const string ServerCommsAddress = "ServerCommsAddress"; + public const string ServerCommsAddresses = "ServerCommsAddresses"; + + public const string ScriptPodServiceAccountName = $"{EnvVarPrefix}__SCRIPTPODSERVICEACCOUNTNAME"; + public const string ScriptPodImagePullSecretNames = $"{EnvVarPrefix}__PODIMAGEPULLSECRETNAMES"; + public const string ScriptPodVolumeClaimName = $"{EnvVarPrefix}__PODVOLUMECLAIMNAME"; + + public const string ScriptPodResourceJson = $"{EnvVarPrefix}__PODRESOURCEJSON"; + public const string ScriptPodAffinityJson = $"{EnvVarPrefix}__PODAFFINITYJSON"; + public const string ScriptPodTolerationsJson = $"{EnvVarPrefix}__PODTOLERATIONSJSON"; + public const string ScriptPodSecurityContextJson = $"{EnvVarPrefix}__PODSECURITYCONTEXTJSON"; public const string ScriptPodContainerImage = $"{EnvVarPrefix}__SCRIPTPODIMAGE"; @@ -31,44 +39,53 @@ public static class VariableNames public const string ScriptPodPullPolicy = $"{EnvVarPrefix}__SCRIPTPODPULLPOLICY"; - public const string ScriptPodProxiesSecretNameVariableName = $"{EnvVarPrefix}__PODPROXIESSECRETNAME"; + public const string ScriptPodProxiesSecretName = $"{EnvVarPrefix}__PODPROXIESSECRETNAME"; + + + public const string NfsWatchdogImage = $"{EnvVarPrefix}__NFSWATCHDOGIMAGE"; + public const string ScriptPodMonitorTimeout = $"{EnvVarPrefix}__PODMONITORTIMEOUT"; + public const string ScriptPodConsideredOrphanedAfterTimeSpan = $"{EnvVarPrefix}__PODSCONSIDEREDORPHANEDAFTERMINUTES"; + + public const string DisableAutomaticPodCleanup = $"{EnvVarPrefix}__DISABLEAUTOPODCLEANUP"; + public const string DisablePodEventsInTaskLog = $"{EnvVarPrefix}__DISABLEPODEVENTSINTASKLOG"; + + public const string PersistentVolumeSize = $"{EnvVarPrefix}__PERSISTENTVOLUMESIZE"; + + public const string IsMetricsEnabled = $"{EnvVarPrefix}__ENABLEMETRICSCAPTURE"; } - public string Namespace => GetRequiredEnvVar(VariableNames.NamespaceVariableName, "Unable to determine Kubernetes namespace."); - public string BootstrapRunnerExecutablePath => GetRequiredEnvVar("BOOTSTRAPRUNNEREXECUTABLEPATH", "Unable to determine Bootstrap Runner Executable Path"); - public string ScriptPodServiceAccountName => GetRequiredEnvVar($"{VariableNames.EnvVarPrefix}__SCRIPTPODSERVICEACCOUNTNAME", "Unable to determine Kubernetes Pod service account name."); + public string Namespace => GetRequiredEnvVar(VariableNames.Namespace, "Unable to determine Kubernetes namespace."); + public string BootstrapRunnerExecutablePath => GetRequiredEnvVar(VariableNames.BoolstrapRunnerExecutablePath, "Unable to determine Bootstrap Runner Executable Path"); + public string ScriptPodServiceAccountName => GetRequiredEnvVar(VariableNames.ScriptPodServiceAccountName, "Unable to determine Kubernetes Pod service account name."); - public IEnumerable ScriptPodImagePullSecretNames => Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__PODIMAGEPULLSECRETNAMES") + public IEnumerable ScriptPodImagePullSecretNames => Environment.GetEnvironmentVariable(VariableNames.ScriptPodImagePullSecretNames) ?.Split(',') .Select(str => str.Trim()) .WhereNotNullOrWhiteSpace() .ToArray() ?? []; - public string ScriptPodVolumeClaimName => GetRequiredEnvVar($"{VariableNames.EnvVarPrefix}__PODVOLUMECLAIMNAME", "Unable to determine Kubernetes Pod persistent volume claim name."); - public string? ScriptPodResourceJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodResourceJsonVariableName); + public string ScriptPodVolumeClaimName => GetRequiredEnvVar(VariableNames.ScriptPodVolumeClaimName, "Unable to determine Kubernetes Pod persistent volume claim name."); + public string? ScriptPodResourceJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodResourceJson); public string? ScriptPodContainerImage => Environment.GetEnvironmentVariable(VariableNames.ScriptPodContainerImage); public string ScriptPodContainerImageTag => Environment.GetEnvironmentVariable(VariableNames.ScriptPodContainerImageTag) ?? "latest"; public string? ScriptPodPullPolicy => Environment.GetEnvironmentVariable(VariableNames.ScriptPodPullPolicy); - - public static string NfsWatchdogImageVariableName => $"{VariableNames.EnvVarPrefix}__NFSWATCHDOGIMAGE"; - public string? NfsWatchdogImage => Environment.GetEnvironmentVariable(NfsWatchdogImageVariableName); + public string? NfsWatchdogImage => Environment.GetEnvironmentVariable(VariableNames.NfsWatchdogImage); - public string HelmReleaseName => GetRequiredEnvVar(VariableNames.HelmReleaseNameVariableName, "Unable to determine Helm release name."); - public string HelmChartVersion => GetRequiredEnvVar(VariableNames.HelmChartVersionVariableName, "Unable to determine Helm chart version."); + public string HelmReleaseName => GetRequiredEnvVar(VariableNames.HelmReleaseName, "Unable to determine Helm release name."); + public string HelmChartVersion => GetRequiredEnvVar(VariableNames.HelmChartVersion, "Unable to determine Helm chart version."); - const string ServerCommsAddressVariableName = "ServerCommsAddress"; public string[] ServerCommsAddresses { get { var addresses = new List(); - if (Environment.GetEnvironmentVariable(ServerCommsAddressVariableName) is { Length: > 0 } addressString) + if (Environment.GetEnvironmentVariable(VariableNames.ServerCommsAddress) is { Length: > 0 } addressString) { addresses.Add(addressString); } - if (Environment.GetEnvironmentVariable(VariableNames.ServerCommsAddressesVariableName) is { } addressesString) + if (Environment.GetEnvironmentVariable(VariableNames.ServerCommsAddresses) is { } addressesString) { addresses.AddRange(addressesString.Split(',').Where(a => !a.IsNullOrEmpty())); } @@ -77,18 +94,17 @@ public string[] ServerCommsAddresses } } - public string PodVolumeClaimName => GetRequiredEnvVar($"{VariableNames.EnvVarPrefix}__PODVOLUMECLAIMNAME", "Unable to determine Kubernetes Pod persistent volume claim name."); - public int? PodMonitorTimeoutSeconds => int.TryParse(Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__PODMONITORTIMEOUT"), out var podMonitorTimeout) ? podMonitorTimeout : 10 * 60; //10min - public TimeSpan PodsConsideredOrphanedAfterTimeSpan => TimeSpan.FromMinutes(int.TryParse(Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__PODSCONSIDEREDORPHANEDAFTERMINUTES"), out var podsConsideredOrphanedAfterTimeSpan) ? podsConsideredOrphanedAfterTimeSpan : 10); - public bool DisableAutomaticPodCleanup => bool.TryParse(Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__DISABLEAUTOPODCLEANUP"), out var disableAutoCleanup) && disableAutoCleanup; - public bool DisablePodEventsInTaskLog => bool.TryParse(Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__DISABLEPODEVENTSINTASKLOG"), out var disable) && disable; - public string PersistentVolumeSize => GetRequiredEnvVar($"{VariableNames.EnvVarPrefix}__PERSISTENTVOLUMESIZE", "Unable to determine Persistent Volume Size"); - - public bool IsMetricsEnabled => !bool.TryParse(Environment.GetEnvironmentVariable($"{VariableNames.EnvVarPrefix}__ENABLEMETRICSCAPTURE"), out var enableMetrics) || enableMetrics; - public string? PodAffinityJson => Environment.GetEnvironmentVariable(VariableNames.PodAffinityJsonVariableName); - public string? PodTolerationsJson => Environment.GetEnvironmentVariable(VariableNames.PodTolerationsJsonVariableName); - public string? PodSecurityContextJson => Environment.GetEnvironmentVariable(VariableNames.PodSecurityContextJsonVariableName); - public string? ScriptPodProxiesSecretName => Environment.GetEnvironmentVariable(VariableNames.ScriptPodProxiesSecretNameVariableName); + public int? PodMonitorTimeoutSeconds => int.TryParse(Environment.GetEnvironmentVariable(VariableNames.ScriptPodMonitorTimeout), out var podMonitorTimeout) ? podMonitorTimeout : 10 * 60; //10min + public TimeSpan PodsConsideredOrphanedAfterTimeSpan => TimeSpan.FromMinutes(int.TryParse(Environment.GetEnvironmentVariable(VariableNames.ScriptPodConsideredOrphanedAfterTimeSpan), out var podsConsideredOrphanedAfterTimeSpan) ? podsConsideredOrphanedAfterTimeSpan : 10); + public bool DisableAutomaticPodCleanup => bool.TryParse(Environment.GetEnvironmentVariable(VariableNames.DisableAutomaticPodCleanup), out var disableAutoCleanup) && disableAutoCleanup; + public bool DisablePodEventsInTaskLog => bool.TryParse(Environment.GetEnvironmentVariable(VariableNames.DisablePodEventsInTaskLog), out var disable) && disable; + public string PersistentVolumeSize => GetRequiredEnvVar(VariableNames.PersistentVolumeSize, "Unable to determine Persistent Volume Size"); + + public bool IsMetricsEnabled => !bool.TryParse(Environment.GetEnvironmentVariable(VariableNames.IsMetricsEnabled), out var enableMetrics) || enableMetrics; + public string? PodAffinityJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodAffinityJson); + public string? PodTolerationsJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodTolerationsJson); + public string? PodSecurityContextJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodSecurityContextJson); + public string? ScriptPodProxiesSecretName => Environment.GetEnvironmentVariable(VariableNames.ScriptPodProxiesSecretName); static string GetRequiredEnvVar(string variable, string errorMessage) => Environment.GetEnvironmentVariable(variable) diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs index 5c1ac81bf..aab2df643 100644 --- a/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs +++ b/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs @@ -24,7 +24,7 @@ public interface IKubernetesAgentDetection public class KubernetesAgentDetection : IKubernetesAgentDetection { public static bool IsRunningAsKubernetesAgent => !string.IsNullOrWhiteSpace(Namespace); - public static string? Namespace => Environment.GetEnvironmentVariable(EnvironmentKubernetesConfiguration.VariableNames.NamespaceVariableName); + public static string? Namespace => Environment.GetEnvironmentVariable(EnvironmentKubernetesConfiguration.VariableNames.Namespace); /// bool IKubernetesAgentDetection.IsRunningAsKubernetesAgent => IsRunningAsKubernetesAgent; diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs index dc0f6454a..cdd41ef8c 100644 --- a/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs +++ b/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs @@ -19,7 +19,6 @@ public interface IKubernetesConfiguration string HelmReleaseName { get; } string HelmChartVersion { get; } string[] ServerCommsAddresses { get; } - string PodVolumeClaimName { get; } int? PodMonitorTimeoutSeconds { get; } TimeSpan PodsConsideredOrphanedAfterTimeSpan { get; } bool DisableAutomaticPodCleanup { get; } diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs deleted file mode 100644 index 03aee6f70..000000000 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs +++ /dev/null @@ -1,104 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Octopus.Tentacle.Util; - -namespace Octopus.Tentacle.Kubernetes -{ - public static class KubernetesConfigz - { - const string ServerCommsAddressVariableName = "ServerCommsAddress"; - const string EnvVarPrefix = "OCTOPUS__K8STENTACLE"; - - public static string NamespaceVariableName => $"{EnvVarPrefix}__NAMESPACE"; - public static string Namespace => GetRequiredEnvVar(NamespaceVariableName, "Unable to determine Kubernetes namespace."); - public static string PodServiceAccountName => GetRequiredEnvVar($"{EnvVarPrefix}__PODSERVICEACCOUNTNAME", "Unable to determine Kubernetes Pod service account name."); - public static string PodVolumeClaimName => GetRequiredEnvVar($"{EnvVarPrefix}__PODVOLUMECLAIMNAME", "Unable to determine Kubernetes Pod persistent volume claim name."); - - public static int PodMonitorTimeoutSeconds => int.TryParse(Environment.GetEnvironmentVariable($"{EnvVarPrefix}__PODMONITORTIMEOUT"), out var podMonitorTimeout) ? podMonitorTimeout : 10 * 60; //10min - public static string NfsWatchdogImageVariableName => $"{EnvVarPrefix}__NFSWATCHDOGIMAGE"; - public static string? NfsWatchdogImage => Environment.GetEnvironmentVariable(NfsWatchdogImageVariableName); - - public static TimeSpan PodsConsideredOrphanedAfterTimeSpan => TimeSpan.FromMinutes(int.TryParse(Environment.GetEnvironmentVariable($"{EnvVarPrefix}__PODSCONSIDEREDORPHANEDAFTERMINUTES"), out var podsConsideredOrphanedAfterTimeSpan) ? podsConsideredOrphanedAfterTimeSpan : 10); - public static bool DisableAutomaticPodCleanup => bool.TryParse(Environment.GetEnvironmentVariable($"{EnvVarPrefix}__DISABLEAUTOPODCLEANUP"), out var disableAutoCleanup) && disableAutoCleanup; - public static bool DisablePodEventsInTaskLog => bool.TryParse(Environment.GetEnvironmentVariable($"{EnvVarPrefix}__DISABLEPODEVENTSINTASKLOG"), out var disable) && disable; - - public static string HelmReleaseNameVariableName => $"{EnvVarPrefix}__HELMRELEASENAME"; - public static string HelmReleaseName => GetRequiredEnvVar(HelmReleaseNameVariableName, "Unable to determine Helm release name."); - - public static string HelmChartVersionVariableName => $"{EnvVarPrefix}__HELMCHARTVERSION"; - public static string HelmChartVersion => GetRequiredEnvVar(HelmChartVersionVariableName, "Unable to determine Helm chart version."); - - public static string BootstrapRunnerExecutablePath => GetRequiredEnvVar("BOOTSTRAPRUNNEREXECUTABLEPATH", "Unable to determine Bootstrap Runner Executable Path"); - - public static string PersistentVolumeSizeVariableName => $"{EnvVarPrefix}__PERSISTENTVOLUMESIZE"; - public static string PersistentVolumeSize => GetRequiredEnvVar(PersistentVolumeSizeVariableName, "Unable to determine Persistent Volume Size"); - - public static string PersistentVolumeSizeBytesVariableName => $"{EnvVarPrefix}__PERSISTENTVOLUMETOTALBYTES"; - public static string PersistentVolumeFreeBytesVariableName => $"{EnvVarPrefix}__PERSISTENTVOLUMEFREEBYTES"; - - public const string ServerCommsAddressesVariableName = "ServerCommsAddresses"; - - public static string? ScriptPodContainerImage => Environment.GetEnvironmentVariable($"{EnvVarPrefix}__SCRIPTPODIMAGE"); - public static string ScriptPodContainerImageTag => Environment.GetEnvironmentVariable($"{EnvVarPrefix}__SCRIPTPODIMAGETAG") ?? "latest"; - public static string? ScriptPodPullPolicy => Environment.GetEnvironmentVariable($"{EnvVarPrefix}__SCRIPTPODPULLPOLICY"); - public static string? ScriptPodProxiesSecretName => Environment.GetEnvironmentVariable($"{EnvVarPrefix}__PODPROXIESSECRETNAME"); - - public static IEnumerable PodImagePullSecretNames => Environment.GetEnvironmentVariable($"{EnvVarPrefix}__PODIMAGEPULLSECRETNAMES") - ?.Split(',') - .Select(str => str.Trim()) - .WhereNotNullOrWhiteSpace() - .ToArray() ?? Array.Empty(); - - public static readonly string PodResourceJsonVariableName = $"{EnvVarPrefix}__PODRESOURCEJSON"; - public static string? PodResourceJson => Environment.GetEnvironmentVariable(PodResourceJsonVariableName); - - public static readonly string PodAffinityJsonVariableName = $"{EnvVarPrefix}__PODAFFINITYJSON"; - public static string? PodAffinityJson => Environment.GetEnvironmentVariable(PodAffinityJsonVariableName); - - public static readonly string PodTolerationsJsonVariableName = $"{EnvVarPrefix}__PODTOLERATIONSJSON"; - public static string? PodTolerationsJson => Environment.GetEnvironmentVariable(PodTolerationsJsonVariableName); - - public static readonly string PodSecurityContextJsonVariableName = $"{EnvVarPrefix}__PODSECURITYCONTEXTJSON"; - public static string? PodSecurityContextJson => Environment.GetEnvironmentVariable(PodSecurityContextJsonVariableName); - - public static string MetricsEnableVariableName => $"{EnvVarPrefix}__ENABLEMETRICSCAPTURE"; - - public static bool MetricsIsEnabled - { - get - { - var envContent = Environment.GetEnvironmentVariable(MetricsEnableVariableName); - if (bool.TryParse(envContent, out var result)) - { - return result; - } - - return true; - } - } - - public static string[] ServerCommsAddresses - { - get - { - var addresses = new List(); - if (Environment.GetEnvironmentVariable(ServerCommsAddressVariableName) is { Length: > 0 } addressString) - { - addresses.Add(addressString); - } - - if (Environment.GetEnvironmentVariable(ServerCommsAddressesVariableName) is { } addressesString) - { - addresses.AddRange(addressesString.Split(',').Where(a => !a.IsNullOrEmpty())); - } - - return addresses.ToArray(); - } - } - - static string GetRequiredEnvVar(string variable, string errorMessage) - => Environment.GetEnvironmentVariable(variable) - ?? throw new InvalidOperationException($"{errorMessage} The environment variable '{variable}' must be defined with a non-null value."); - } -} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs index 34cb80ec2..f41405506 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs @@ -81,7 +81,7 @@ protected override IList CreateVolumes(StartKubernetesScriptCommandV1 Name = "init-nfs-volume", PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource { - ClaimName = KubernetesConfiguration.PodVolumeClaimName + ClaimName = KubernetesConfiguration.ScriptPodVolumeClaimName } }, CreateAgentUpgradeSecretVolume(), diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs index 735be5f5e..2ada348fd 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs @@ -249,7 +249,7 @@ protected virtual IList CreateVolumes(StartKubernetesScriptCommandV1 c Name = "tentacle-home", PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource { - ClaimName = KubernetesConfiguration.PodVolumeClaimName + ClaimName = KubernetesConfiguration.ScriptPodVolumeClaimName } }, CreateAgentUpgradeSecretVolume(), @@ -330,12 +330,12 @@ protected async Task CreateScriptContainer(StartKubernetesScriptCom }, Env = new List { - new(EnvironmentKubernetesConfiguration.VariableNames.NamespaceVariableName, KubernetesConfiguration.Namespace), - new(EnvironmentKubernetesConfiguration.VariableNames.HelmReleaseNameVariableName, KubernetesConfiguration.HelmReleaseName), - new(EnvironmentKubernetesConfiguration.VariableNames.HelmChartVersionVariableName, KubernetesConfiguration.HelmChartVersion), - new(EnvironmentKubernetesConfiguration.VariableNames.ServerCommsAddressesVariableName, string.Join(",", KubernetesConfiguration.ServerCommsAddresses)), - new(EnvironmentKubernetesConfiguration.VariableNames.PersistentVolumeFreeBytesVariableName, spaceInformation?.freeSpaceBytes.ToString()), - new(EnvironmentKubernetesConfiguration.VariableNames.PersistentVolumeSizeBytesVariableName, spaceInformation?.totalSpaceBytes.ToString()), + new(EnvironmentKubernetesConfiguration.VariableNames.Namespace, KubernetesConfiguration.Namespace), + new(EnvironmentKubernetesConfiguration.VariableNames.HelmReleaseName, KubernetesConfiguration.HelmReleaseName), + new(EnvironmentKubernetesConfiguration.VariableNames.HelmChartVersion, KubernetesConfiguration.HelmChartVersion), + new(EnvironmentKubernetesConfiguration.VariableNames.ServerCommsAddresses, string.Join(",", KubernetesConfiguration.ServerCommsAddresses)), + new(EnvironmentKubernetesConfiguration.VariableNames.PersistentVolumeFreeBytes, spaceInformation?.freeSpaceBytes.ToString()), + new(EnvironmentKubernetesConfiguration.VariableNames.PersistentVolumeSizeBytes, spaceInformation?.totalSpaceBytes.ToString()), new(EnvironmentVariables.TentacleHome, homeDir), new(EnvironmentVariables.TentacleInstanceName, appInstanceSelector.Current.InstanceName), new(EnvironmentVariables.TentacleVersion, Environment.GetEnvironmentVariable(EnvironmentVariables.TentacleVersion)), @@ -360,7 +360,7 @@ V1ResourceRequirements GetScriptPodResourceRequirements(InMemoryTentacleScriptLo } catch (Exception e) { - var message = $"Failed to deserialize env.{EnvironmentKubernetesConfiguration.VariableNames.PodResourceJsonVariableName} into valid pod resource requirements.{Environment.NewLine}JSON value: {json}{Environment.NewLine}Using default resource requests for script pod."; + var message = $"Failed to deserialize env.{EnvironmentKubernetesConfiguration.VariableNames.ScriptPodResourceJson} into valid pod resource requirements.{Environment.NewLine}JSON value: {json}{Environment.NewLine}Using default resource requests for script pod."; //if we can't parse the JSON, fall back to the defaults below and warn the user log.WarnFormat(e, message); //write a verbose message to the script log. @@ -383,7 +383,7 @@ V1Affinity ParseScriptPodAffinity(InMemoryTentacleScriptLog tentacleScriptLog) => ParseScriptPodJson( tentacleScriptLog, KubernetesConfiguration.PodAffinityJson, - EnvironmentKubernetesConfiguration.VariableNames.PodAffinityJsonVariableName, + EnvironmentKubernetesConfiguration.VariableNames.ScriptPodAffinityJson, "pod affinity", //we default to running on linux/arm64 and linux/amd64 nodes new V1Affinity(new V1NodeAffinity(requiredDuringSchedulingIgnoredDuringExecution: new V1NodeSelector(new List @@ -399,14 +399,14 @@ V1Affinity ParseScriptPodAffinity(InMemoryTentacleScriptLog tentacleScriptLog) => ParseScriptPodJson>( tentacleScriptLog, KubernetesConfiguration.PodTolerationsJson, - EnvironmentKubernetesConfiguration.VariableNames.PodTolerationsJsonVariableName, + EnvironmentKubernetesConfiguration.VariableNames.ScriptPodTolerationsJson, "pod tolerations"); V1PodSecurityContext? ParseScriptPodSecurityContext(InMemoryTentacleScriptLog tentacleScriptLog) => ParseScriptPodJson( tentacleScriptLog, KubernetesConfiguration.PodSecurityContextJson, - EnvironmentKubernetesConfiguration.VariableNames.PodSecurityContextJsonVariableName, + EnvironmentKubernetesConfiguration.VariableNames.ScriptPodSecurityContextJson, "pod security context"); [return: NotNullIfNotNull("defaultValue")] From eef899221db020087165b9850da2acf7593381ab Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Fri, 20 Dec 2024 11:02:55 +1100 Subject: [PATCH 6/9] Revert language version update --- .../EnvironmentKubernetesConfiguration.cs | 56 +++++++++---------- .../Kubernetes/IKubernetesAgentDetection.cs | 49 ++++++++-------- .../Octopus.Tentacle/Octopus.Tentacle.csproj | 2 +- 3 files changed, 54 insertions(+), 53 deletions(-) diff --git a/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs b/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs index 3798c453f..34ee4f3b2 100644 --- a/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs +++ b/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs @@ -11,58 +11,58 @@ public static class VariableNames { const string EnvVarPrefix = "OCTOPUS__K8STENTACLE"; - public const string Namespace = $"{EnvVarPrefix}__NAMESPACE"; + public static readonly string Namespace = $"{EnvVarPrefix}__NAMESPACE"; - public const string BoolstrapRunnerExecutablePath = "BOOTSTRAPRUNNEREXECUTABLEPATH"; + public static readonly string BootstrapRunnerExecutablePath = "BOOTSTRAPRUNNEREXECUTABLEPATH"; - public const string PersistentVolumeSizeBytes = $"{EnvVarPrefix}__PERSISTENTVOLUMETOTALBYTES"; - public const string PersistentVolumeFreeBytes = $"{EnvVarPrefix}__PERSISTENTVOLUMEFREEBYTES"; + public static readonly string PersistentVolumeSizeBytes = $"{EnvVarPrefix}__PERSISTENTVOLUMETOTALBYTES"; + public static readonly string PersistentVolumeFreeBytes = $"{EnvVarPrefix}__PERSISTENTVOLUMEFREEBYTES"; - public const string HelmReleaseName = $"{EnvVarPrefix}__HELMRELEASENAME"; - public const string HelmChartVersion = $"{EnvVarPrefix}__HELMCHARTVERSION"; + public static readonly string HelmReleaseName = $"{EnvVarPrefix}__HELMRELEASENAME"; + public static readonly string HelmChartVersion = $"{EnvVarPrefix}__HELMCHARTVERSION"; - public const string ServerCommsAddress = "ServerCommsAddress"; - public const string ServerCommsAddresses = "ServerCommsAddresses"; + public static readonly string ServerCommsAddress = "ServerCommsAddress"; + public static readonly string ServerCommsAddresses = "ServerCommsAddresses"; - public const string ScriptPodServiceAccountName = $"{EnvVarPrefix}__SCRIPTPODSERVICEACCOUNTNAME"; - public const string ScriptPodImagePullSecretNames = $"{EnvVarPrefix}__PODIMAGEPULLSECRETNAMES"; - public const string ScriptPodVolumeClaimName = $"{EnvVarPrefix}__PODVOLUMECLAIMNAME"; + public static readonly string ScriptPodServiceAccountName = $"{EnvVarPrefix}__SCRIPTPODSERVICEACCOUNTNAME"; + public static readonly string ScriptPodImagePullSecretNames = $"{EnvVarPrefix}__PODIMAGEPULLSECRETNAMES"; + public static readonly string ScriptPodVolumeClaimName = $"{EnvVarPrefix}__PODVOLUMECLAIMNAME"; - public const string ScriptPodResourceJson = $"{EnvVarPrefix}__PODRESOURCEJSON"; - public const string ScriptPodAffinityJson = $"{EnvVarPrefix}__PODAFFINITYJSON"; - public const string ScriptPodTolerationsJson = $"{EnvVarPrefix}__PODTOLERATIONSJSON"; - public const string ScriptPodSecurityContextJson = $"{EnvVarPrefix}__PODSECURITYCONTEXTJSON"; + public static readonly string ScriptPodResourceJson = $"{EnvVarPrefix}__PODRESOURCEJSON"; + public static readonly string ScriptPodAffinityJson = $"{EnvVarPrefix}__PODAFFINITYJSON"; + public static readonly string ScriptPodTolerationsJson = $"{EnvVarPrefix}__PODTOLERATIONSJSON"; + public static readonly string ScriptPodSecurityContextJson = $"{EnvVarPrefix}__PODSECURITYCONTEXTJSON"; - public const string ScriptPodContainerImage = $"{EnvVarPrefix}__SCRIPTPODIMAGE"; - public const string ScriptPodContainerImageTag = $"{EnvVarPrefix}__SCRIPTPODIMAGETAG"; - public const string ScriptPodPullPolicy = $"{EnvVarPrefix}__SCRIPTPODPULLPOLICY"; + public static readonly string ScriptPodContainerImage = $"{EnvVarPrefix}__SCRIPTPODIMAGE"; + public static readonly string ScriptPodContainerImageTag = $"{EnvVarPrefix}__SCRIPTPODIMAGETAG"; + public static readonly string ScriptPodPullPolicy = $"{EnvVarPrefix}__SCRIPTPODPULLPOLICY"; - public const string ScriptPodProxiesSecretName = $"{EnvVarPrefix}__PODPROXIESSECRETNAME"; + public static readonly string ScriptPodProxiesSecretName = $"{EnvVarPrefix}__PODPROXIESSECRETNAME"; - public const string NfsWatchdogImage = $"{EnvVarPrefix}__NFSWATCHDOGIMAGE"; - public const string ScriptPodMonitorTimeout = $"{EnvVarPrefix}__PODMONITORTIMEOUT"; - public const string ScriptPodConsideredOrphanedAfterTimeSpan = $"{EnvVarPrefix}__PODSCONSIDEREDORPHANEDAFTERMINUTES"; + public static readonly string NfsWatchdogImage = $"{EnvVarPrefix}__NFSWATCHDOGIMAGE"; + public static readonly string ScriptPodMonitorTimeout = $"{EnvVarPrefix}__PODMONITORTIMEOUT"; + public static readonly string ScriptPodConsideredOrphanedAfterTimeSpan = $"{EnvVarPrefix}__PODSCONSIDEREDORPHANEDAFTERMINUTES"; - public const string DisableAutomaticPodCleanup = $"{EnvVarPrefix}__DISABLEAUTOPODCLEANUP"; - public const string DisablePodEventsInTaskLog = $"{EnvVarPrefix}__DISABLEPODEVENTSINTASKLOG"; + public static readonly string DisableAutomaticPodCleanup = $"{EnvVarPrefix}__DISABLEAUTOPODCLEANUP"; + public static readonly string DisablePodEventsInTaskLog = $"{EnvVarPrefix}__DISABLEPODEVENTSINTASKLOG"; - public const string PersistentVolumeSize = $"{EnvVarPrefix}__PERSISTENTVOLUMESIZE"; + public static readonly string PersistentVolumeSize = $"{EnvVarPrefix}__PERSISTENTVOLUMESIZE"; - public const string IsMetricsEnabled = $"{EnvVarPrefix}__ENABLEMETRICSCAPTURE"; + public static readonly string IsMetricsEnabled = $"{EnvVarPrefix}__ENABLEMETRICSCAPTURE"; } public string Namespace => GetRequiredEnvVar(VariableNames.Namespace, "Unable to determine Kubernetes namespace."); - public string BootstrapRunnerExecutablePath => GetRequiredEnvVar(VariableNames.BoolstrapRunnerExecutablePath, "Unable to determine Bootstrap Runner Executable Path"); + public string BootstrapRunnerExecutablePath => GetRequiredEnvVar(VariableNames.BootstrapRunnerExecutablePath, "Unable to determine Bootstrap Runner Executable Path"); public string ScriptPodServiceAccountName => GetRequiredEnvVar(VariableNames.ScriptPodServiceAccountName, "Unable to determine Kubernetes Pod service account name."); public IEnumerable ScriptPodImagePullSecretNames => Environment.GetEnvironmentVariable(VariableNames.ScriptPodImagePullSecretNames) ?.Split(',') .Select(str => str.Trim()) .WhereNotNullOrWhiteSpace() - .ToArray() ?? []; + .ToArray() ?? Array.Empty(); public string ScriptPodVolumeClaimName => GetRequiredEnvVar(VariableNames.ScriptPodVolumeClaimName, "Unable to determine Kubernetes Pod persistent volume claim name."); public string? ScriptPodResourceJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodResourceJson); diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs index aab2df643..509e8a776 100644 --- a/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs +++ b/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs @@ -1,34 +1,35 @@ using System; using System.Diagnostics.CodeAnalysis; -namespace Octopus.Tentacle.Kubernetes; - -public interface IKubernetesAgentDetection +namespace Octopus.Tentacle.Kubernetes { - /// - /// Indicates if the Tentacle is running inside a Kubernetes cluster as the Kubernetes Agent - /// - [MemberNotNullWhen(true, nameof(Namespace))] - bool IsRunningAsKubernetesAgent { get; } + public interface IKubernetesAgentDetection + { + /// + /// Indicates if the Tentacle is running inside a Kubernetes cluster as the Kubernetes Agent + /// + [MemberNotNullWhen(true, nameof(Namespace))] + bool IsRunningAsKubernetesAgent { get; } + /// + /// The Kubernetes namespace the agent is running under, null if not running as a Kubernetes agent + /// + string? Namespace { get; } + } + /// - /// The Kubernetes namespace the agent is running under, null if not running as a Kubernetes agent + /// Used for detection if the tentacle is running as part of the Kubernetes agent helm chart + /// Can be used with dependency injection via or statically /// - string? Namespace { get; } -} - -/// -/// Used for detection if the tentacle is running as part of the Kubernetes agent helm chart -/// Can be used with dependency injection via or statically -/// -public class KubernetesAgentDetection : IKubernetesAgentDetection -{ - public static bool IsRunningAsKubernetesAgent => !string.IsNullOrWhiteSpace(Namespace); - public static string? Namespace => Environment.GetEnvironmentVariable(EnvironmentKubernetesConfiguration.VariableNames.Namespace); + public class KubernetesAgentDetection : IKubernetesAgentDetection + { + public static bool IsRunningAsKubernetesAgent => !string.IsNullOrWhiteSpace(Namespace); + public static string? Namespace => Environment.GetEnvironmentVariable(EnvironmentKubernetesConfiguration.VariableNames.Namespace); - /// - bool IKubernetesAgentDetection.IsRunningAsKubernetesAgent => IsRunningAsKubernetesAgent; + /// + bool IKubernetesAgentDetection.IsRunningAsKubernetesAgent => IsRunningAsKubernetesAgent; - /// - string? IKubernetesAgentDetection.Namespace => Namespace; + /// + string? IKubernetesAgentDetection.Namespace => Namespace; + } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Octopus.Tentacle.csproj b/source/Octopus.Tentacle/Octopus.Tentacle.csproj index 4911ca26f..6247dc364 100644 --- a/source/Octopus.Tentacle/Octopus.Tentacle.csproj +++ b/source/Octopus.Tentacle/Octopus.Tentacle.csproj @@ -12,7 +12,7 @@ Octopus.Tentacle net48;net8.0;net8.0-windows true - default + 9 enable From 76f297604b97a061cc850cf50706b4573b31f629 Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Fri, 20 Dec 2024 11:17:49 +1100 Subject: [PATCH 7/9] Reorder and rename cleanup --- .../KubernetesOrphanedPodCleanerTests.cs | 4 +-- .../EnvironmentKubernetesConfiguration.cs | 10 +++--- .../Kubernetes/IKubernetesAgentDetection.cs | 10 +++++- .../Kubernetes/IKubernetesConfiguration.cs | 35 +++++++++---------- .../KubernetesOrphanedPodCleaner.cs | 2 +- .../Kubernetes/KubernetesPodService.cs | 2 +- .../Kubernetes/KubernetesScriptPodCreator.cs | 6 ++-- 7 files changed, 38 insertions(+), 31 deletions(-) diff --git a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs index 461f2b9d2..ef9410ab8 100644 --- a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs +++ b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs @@ -45,7 +45,7 @@ public void Setup() scriptTicket = new ScriptTicket(Guid.NewGuid().ToString()); config = Substitute.For(); - config.PodsConsideredOrphanedAfterTimeSpan.Returns(TimeSpan.FromMinutes(10)); + config.ScriptPodConsideredOrphanedAfterTimeSpan.Returns(TimeSpan.FromMinutes(10)); config.DisableAutomaticPodCleanup.Returns(false); cleaner = new KubernetesOrphanedPodCleaner(config, monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore, scriptPodLogEncryptionKeyProvider); @@ -177,7 +177,7 @@ public async Task OrphanedPodNotCleanedUpIfPodCleanupIsDisabled() public async Task EnvironmentVariableDictatesWhenPodsAreConsideredOrphaned(int checkAfterMinutes, bool shouldDelete) { //Arrange - config.PodsConsideredOrphanedAfterTimeSpan.Returns(TimeSpan.FromMinutes(2)); + config.ScriptPodConsideredOrphanedAfterTimeSpan.Returns(TimeSpan.FromMinutes(2)); // We need to reinitialise the sut after changing the env var value cleaner = new KubernetesOrphanedPodCleaner(config, monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore, scriptPodLogEncryptionKeyProvider); diff --git a/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs b/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs index 34ee4f3b2..7e2cb26b0 100644 --- a/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs +++ b/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs @@ -94,16 +94,16 @@ public string[] ServerCommsAddresses } } - public int? PodMonitorTimeoutSeconds => int.TryParse(Environment.GetEnvironmentVariable(VariableNames.ScriptPodMonitorTimeout), out var podMonitorTimeout) ? podMonitorTimeout : 10 * 60; //10min - public TimeSpan PodsConsideredOrphanedAfterTimeSpan => TimeSpan.FromMinutes(int.TryParse(Environment.GetEnvironmentVariable(VariableNames.ScriptPodConsideredOrphanedAfterTimeSpan), out var podsConsideredOrphanedAfterTimeSpan) ? podsConsideredOrphanedAfterTimeSpan : 10); + public int? ScriptPodMonitorTimeoutSeconds => int.TryParse(Environment.GetEnvironmentVariable(VariableNames.ScriptPodMonitorTimeout), out var podMonitorTimeout) ? podMonitorTimeout : 10 * 60; //10min + public TimeSpan ScriptPodConsideredOrphanedAfterTimeSpan => TimeSpan.FromMinutes(int.TryParse(Environment.GetEnvironmentVariable(VariableNames.ScriptPodConsideredOrphanedAfterTimeSpan), out var podsConsideredOrphanedAfterTimeSpan) ? podsConsideredOrphanedAfterTimeSpan : 10); public bool DisableAutomaticPodCleanup => bool.TryParse(Environment.GetEnvironmentVariable(VariableNames.DisableAutomaticPodCleanup), out var disableAutoCleanup) && disableAutoCleanup; public bool DisablePodEventsInTaskLog => bool.TryParse(Environment.GetEnvironmentVariable(VariableNames.DisablePodEventsInTaskLog), out var disable) && disable; public string PersistentVolumeSize => GetRequiredEnvVar(VariableNames.PersistentVolumeSize, "Unable to determine Persistent Volume Size"); public bool IsMetricsEnabled => !bool.TryParse(Environment.GetEnvironmentVariable(VariableNames.IsMetricsEnabled), out var enableMetrics) || enableMetrics; - public string? PodAffinityJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodAffinityJson); - public string? PodTolerationsJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodTolerationsJson); - public string? PodSecurityContextJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodSecurityContextJson); + public string? ScriptPodAffinityJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodAffinityJson); + public string? ScriptPodTolerationsJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodTolerationsJson); + public string? ScriptPodSecurityContextJson => Environment.GetEnvironmentVariable(VariableNames.ScriptPodSecurityContextJson); public string? ScriptPodProxiesSecretName => Environment.GetEnvironmentVariable(VariableNames.ScriptPodProxiesSecretName); static string GetRequiredEnvVar(string variable, string errorMessage) diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs index 509e8a776..996c86ea8 100644 --- a/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs +++ b/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs @@ -19,11 +19,19 @@ public interface IKubernetesAgentDetection /// /// Used for detection if the tentacle is running as part of the Kubernetes agent helm chart - /// Can be used with dependency injection via or statically + /// Can be used with dependency injection via or statically (in scenarios where dependency injection isn't available) /// public class KubernetesAgentDetection : IKubernetesAgentDetection { + /// + /// Indicates if the Tentacle is running inside a Kubernetes cluster as the Kubernetes Agent + /// + [MemberNotNullWhen(true, nameof(Namespace))] public static bool IsRunningAsKubernetesAgent => !string.IsNullOrWhiteSpace(Namespace); + + /// + /// The Kubernetes namespace the Kubernetes Agent is running under, null if not running as a Kubernetes agent + /// public static string? Namespace => Environment.GetEnvironmentVariable(EnvironmentKubernetesConfiguration.VariableNames.Namespace); /// diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs index cdd41ef8c..879d52043 100644 --- a/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs +++ b/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs @@ -5,29 +5,28 @@ namespace Octopus.Tentacle.Kubernetes { public interface IKubernetesConfiguration { - string Namespace { get;} - string BootstrapRunnerExecutablePath { get; } - - string ScriptPodServiceAccountName { get; } - IEnumerable ScriptPodImagePullSecretNames { get; } + string Namespace { get; } + string BootstrapRunnerExecutablePath { get; } + string ScriptPodServiceAccountName { get; } + IEnumerable ScriptPodImagePullSecretNames { get; } string ScriptPodVolumeClaimName { get; } - string? ScriptPodResourceJson { get; } + string? ScriptPodResourceJson { get; } + string? ScriptPodAffinityJson { get; } + string? ScriptPodTolerationsJson { get; } + string? ScriptPodSecurityContextJson { get; } + string? ScriptPodProxiesSecretName { get; } string? ScriptPodContainerImage { get; } - string ScriptPodContainerImageTag { get;} + string ScriptPodContainerImageTag { get; } string? ScriptPodPullPolicy { get; } - string? NfsWatchdogImage { get; } + string? NfsWatchdogImage { get; } string HelmReleaseName { get; } - string HelmChartVersion { get; } + string HelmChartVersion { get; } string[] ServerCommsAddresses { get; } - int? PodMonitorTimeoutSeconds { get; } - TimeSpan PodsConsideredOrphanedAfterTimeSpan { get; } - bool DisableAutomaticPodCleanup { get; } + int? ScriptPodMonitorTimeoutSeconds { get; } + TimeSpan ScriptPodConsideredOrphanedAfterTimeSpan { get; } + bool DisableAutomaticPodCleanup { get; } bool DisablePodEventsInTaskLog { get; } - string PersistentVolumeSize { get; } - bool IsMetricsEnabled { get; } - string? PodAffinityJson { get; } - string? PodTolerationsJson { get; } - string? PodSecurityContextJson { get; } - string? ScriptPodProxiesSecretName { get; } + string PersistentVolumeSize { get; } + bool IsMetricsEnabled { get; } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs index 060c929ee..192ba4c13 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs @@ -27,7 +27,7 @@ public class KubernetesOrphanedPodCleaner : IKubernetesOrphanedPodCleaner readonly IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider; readonly TimeSpan initialDelay = TimeSpan.FromMinutes(1); - internal TimeSpan CompletedPodConsideredOrphanedAfterTimeSpan => kubernetesConfiguration.PodsConsideredOrphanedAfterTimeSpan; + internal TimeSpan CompletedPodConsideredOrphanedAfterTimeSpan => kubernetesConfiguration.ScriptPodConsideredOrphanedAfterTimeSpan; public KubernetesOrphanedPodCleaner( IKubernetesConfiguration kubernetesConfiguration, diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs index 81bb8f0be..508552df1 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs @@ -39,7 +39,7 @@ public async Task WatchAllPods(string initialResourceVersion, Func ParseScriptPodJson( tentacleScriptLog, - KubernetesConfiguration.PodAffinityJson, + KubernetesConfiguration.ScriptPodAffinityJson, EnvironmentKubernetesConfiguration.VariableNames.ScriptPodAffinityJson, "pod affinity", //we default to running on linux/arm64 and linux/amd64 nodes @@ -398,14 +398,14 @@ V1Affinity ParseScriptPodAffinity(InMemoryTentacleScriptLog tentacleScriptLog) List? ParseScriptPodTolerations(InMemoryTentacleScriptLog tentacleScriptLog) => ParseScriptPodJson>( tentacleScriptLog, - KubernetesConfiguration.PodTolerationsJson, + KubernetesConfiguration.ScriptPodTolerationsJson, EnvironmentKubernetesConfiguration.VariableNames.ScriptPodTolerationsJson, "pod tolerations"); V1PodSecurityContext? ParseScriptPodSecurityContext(InMemoryTentacleScriptLog tentacleScriptLog) => ParseScriptPodJson( tentacleScriptLog, - KubernetesConfiguration.PodSecurityContextJson, + KubernetesConfiguration.ScriptPodSecurityContextJson, EnvironmentKubernetesConfiguration.VariableNames.ScriptPodSecurityContextJson, "pod security context"); From 3fa3e47f3d9ae0745aa94f5dbeb808289a1183b4 Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Fri, 20 Dec 2024 11:51:19 +1100 Subject: [PATCH 8/9] Fix KubernetesEventMonitor DI issues --- .../KubernetesEventMonitorFixture.cs | 27 ++++++++++--------- .../Kubernetes/KubernetesEventMonitor.cs | 12 ++++----- .../Kubernetes/KubernetesEventMonitorTask.cs | 2 -- .../Kubernetes/KubernetesEventService.cs | 14 +++++----- .../Kubernetes/KubernetesPodLogService.cs | 2 +- 5 files changed, 26 insertions(+), 31 deletions(-) diff --git a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesEventMonitorFixture.cs b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesEventMonitorFixture.cs index 8c980203a..5840cdaa5 100644 --- a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesEventMonitorFixture.cs +++ b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesEventMonitorFixture.cs @@ -51,6 +51,7 @@ public class KubernetesEventMonitorFixture readonly CancellationTokenSource tokenSource = new(); readonly DateTimeOffset testEpoch = DateTimeOffset.Now; readonly ISystemLog log = Substitute.For(); + readonly IKubernetesConfiguration kubernetesConfiguration = Substitute.For(); [Test] public async Task NoEntriesAreSentToMetricsWhenEventListIsEmpty() @@ -58,7 +59,7 @@ public async Task NoEntriesAreSentToMetricsWhenEventListIsEmpty() var agentMetrics = Substitute.For(); agentMetrics.GetLatestEventTimestamp(Arg.Any()).ReturnsForAnyArgs(testEpoch); var eventService = Substitute.For(); - var sut = new KubernetesEventMonitor(agentMetrics, eventService, "arbitraryNamespace", new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); + var sut = new KubernetesEventMonitor(kubernetesConfiguration, agentMetrics, eventService, new IEventMapper[] { new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper() }, log); await sut.CacheNewEvents(tokenSource.Token); @@ -71,7 +72,7 @@ public async Task NfsPodKillingEventsAreTrackedInMetrics() //Arrange var agentMetrics = new StubbedAgentMetrics(testEpoch); var eventService = Substitute.For(); - eventService.FetchAllEventsAsync(Arg.Any(), Arg.Any()).ReturnsForAnyArgs( + eventService.FetchAllEventsAsync(Arg.Any()).ReturnsForAnyArgs( new Corev1EventList(new List { new() @@ -85,7 +86,7 @@ public async Task NfsPodKillingEventsAreTrackedInMetrics() LastTimestamp = testEpoch.DateTime.AddMinutes(1) } })); - var sut = new KubernetesEventMonitor(agentMetrics, eventService, "arbitraryNamespace", new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); + var sut = new KubernetesEventMonitor(kubernetesConfiguration, agentMetrics, eventService, new IEventMapper[] { new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper() }, log); //Act await sut.CacheNewEvents(tokenSource.Token); @@ -104,7 +105,7 @@ public async Task NfsWatchDogEventsAreTrackedInMetrics() var podName = "octopus-script-123412341234.123412341234"; var agentMetrics = new StubbedAgentMetrics(testEpoch); var eventService = Substitute.For(); - eventService.FetchAllEventsAsync(Arg.Any(), Arg.Any()).ReturnsForAnyArgs( + eventService.FetchAllEventsAsync(Arg.Any()).ReturnsForAnyArgs( new Corev1EventList(new List { new() @@ -119,7 +120,7 @@ public async Task NfsWatchDogEventsAreTrackedInMetrics() } })); - var sut = new KubernetesEventMonitor(agentMetrics, eventService, "arbitraryNamespace", new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); + var sut = new KubernetesEventMonitor(kubernetesConfiguration, agentMetrics, eventService, new IEventMapper[] { new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper() }, log); //Act await sut.CacheNewEvents(tokenSource.Token); @@ -138,7 +139,7 @@ public async Task EventsOlderThanOrEqualToMetricsTimestampCursorAreNotAddedToMet var podName = "octopus-script-123412341234.123412341234"; var agentMetrics = new StubbedAgentMetrics(testEpoch); var eventService = Substitute.For(); - eventService.FetchAllEventsAsync(Arg.Any(), Arg.Any()).ReturnsForAnyArgs( + eventService.FetchAllEventsAsync(Arg.Any()).ReturnsForAnyArgs( new Corev1EventList(new List { new() @@ -152,11 +153,11 @@ public async Task EventsOlderThanOrEqualToMetricsTimestampCursorAreNotAddedToMet LastTimestamp = testEpoch.DateTime } })); - - var sut = new KubernetesEventMonitor(agentMetrics, eventService, "arbitraryNamespace", new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); + + var sut = new KubernetesEventMonitor(kubernetesConfiguration, agentMetrics, eventService, new IEventMapper[] { new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper() }, log); //Act await sut.CacheNewEvents(tokenSource.Token); - + //Assert agentMetrics.Events.Should().BeEquivalentTo(new Dictionary>>()); } @@ -168,7 +169,7 @@ public async Task NewestTimeStampInEventIsUsedToDetermineAgeAndAsMetricsValue() var podName = "octopus-script-123412341234.123412341234"; var agentMetrics = new StubbedAgentMetrics(testEpoch); var eventService = Substitute.For(); - eventService.FetchAllEventsAsync(Arg.Any(), Arg.Any()).ReturnsForAnyArgs( + eventService.FetchAllEventsAsync(Arg.Any()).ReturnsForAnyArgs( new Corev1EventList(new List { new() @@ -183,11 +184,11 @@ public async Task NewestTimeStampInEventIsUsedToDetermineAgeAndAsMetricsValue() EventTime = testEpoch.DateTime.AddMinutes(1) } })); - - var sut = new KubernetesEventMonitor(agentMetrics, eventService, "arbitraryNamespace", new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); + + var sut = new KubernetesEventMonitor(kubernetesConfiguration, agentMetrics, eventService, new IEventMapper[] { new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper() }, log); //Act await sut.CacheNewEvents(tokenSource.Token); - + //Assert // The event.EventTime is newest event Time stamp, and is larger than the last metric date (TestEpoch) as such // the event should factored into the metrics, and should report this latest time value. diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitor.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitor.cs index 7342416b8..7a3eff08f 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitor.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitor.cs @@ -18,27 +18,25 @@ public interface IKubernetesEventMonitor public class KubernetesEventMonitor : IKubernetesEventMonitor { - public delegate KubernetesEventMonitor Factory(string kubernetesNamespace); - + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesAgentMetrics agentMetrics; readonly IKubernetesEventService eventService; - readonly string kubernetesNamespace; readonly IEventMapper[] eventMappers; readonly ISystemLog log; - public KubernetesEventMonitor(IKubernetesAgentMetrics agentMetrics, IKubernetesEventService eventService, string kubernetesNamespace, IEventMapper[] eventMappers, ISystemLog log) + public KubernetesEventMonitor(IKubernetesConfiguration kubernetesConfiguration, IKubernetesAgentMetrics agentMetrics, IKubernetesEventService eventService, IEventMapper[] eventMappers, ISystemLog log) { + this.kubernetesConfiguration = kubernetesConfiguration; this.agentMetrics = agentMetrics; this.eventService = eventService; - this.kubernetesNamespace = kubernetesNamespace; this.eventMappers = eventMappers; this.log = log; } public async Task CacheNewEvents(CancellationToken cancellationToken) { - log.Info($"Parsing kubernetes event list for namespace {kubernetesNamespace}."); - var allEvents = await eventService.FetchAllEventsAsync(kubernetesNamespace, cancellationToken) ?? new Corev1EventList(new List()); + log.Info($"Parsing kubernetes event list for namespace {kubernetesConfiguration.Namespace}."); + var allEvents = await eventService.FetchAllEventsAsync(cancellationToken) ?? new Corev1EventList(new List()); var lastCachedEventTimeStamp = await agentMetrics.GetLatestEventTimestamp(cancellationToken); diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs index 289fa04e4..2c7454720 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs @@ -9,8 +9,6 @@ namespace Octopus.Tentacle.Kubernetes { public class KubernetesEventMonitorTask : BackgroundTask { - public delegate KubernetesEventMonitorTask Factory(IKubernetesEventMonitor eventMonitor); - readonly IKubernetesEventMonitor eventMonitor; readonly ISystemLog log; readonly IKubernetesConfiguration kubernetesConfiguration; diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs index 861378294..fc1cdcf5c 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs @@ -1,4 +1,3 @@ -using System.Collections.Generic; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -6,14 +5,13 @@ using k8s.Autorest; using k8s.Models; using Octopus.Diagnostics; -using Octopus.Tentacle.Contracts; namespace Octopus.Tentacle.Kubernetes { public interface IKubernetesEventService { - Task FetchAllEventsAsync(string kubernetesNamespace, CancellationToken cancellationToken); - Task FetchAllEventsAsync(string kubernetesNamespace, string podName, CancellationToken cancellationToken); + Task FetchAllEventsAsync(CancellationToken cancellationToken); + Task FetchAllEventsAsync(string podName, CancellationToken cancellationToken); } public class KubernetesEventService : KubernetesService, IKubernetesEventService @@ -23,13 +21,13 @@ public KubernetesEventService(IKubernetesClientConfigProvider configProvider, IK { } - public async Task FetchAllEventsAsync(string kubernetesNamespace, CancellationToken cancellationToken) + public async Task FetchAllEventsAsync(CancellationToken cancellationToken) { return await RetryPolicy.ExecuteAsync(async () => { try { - return await Client.CoreV1.ListNamespacedEventAsync(kubernetesNamespace, cancellationToken: cancellationToken); + return await Client.CoreV1.ListNamespacedEventAsync(Namespace, cancellationToken: cancellationToken); } catch (HttpOperationException opException) when (opException.Response.StatusCode == HttpStatusCode.NotFound) @@ -39,7 +37,7 @@ public KubernetesEventService(IKubernetesClientConfigProvider configProvider, IK }); } - public async Task FetchAllEventsAsync(string kubernetesNamespace, string podName, CancellationToken cancellationToken) + public async Task FetchAllEventsAsync(string podName, CancellationToken cancellationToken) { return await RetryPolicy.ExecuteAsync(async () => { @@ -47,7 +45,7 @@ public KubernetesEventService(IKubernetesClientConfigProvider configProvider, IK { //get all the events for a specific script pod return await Client.CoreV1.ListNamespacedEventAsync( - kubernetesNamespace, + Namespace, fieldSelector: $"involvedObject.name={podName}", cancellationToken: cancellationToken); } diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesPodLogService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesPodLogService.cs index aa9b49203..7d8ac165b 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesPodLogService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesPodLogService.cs @@ -138,7 +138,7 @@ async Task> GetPodEvents(ScriptTicket scriptTicket, s var sinceTime = scriptPodSinceTimeStore.GetPodEventsSinceTime(scriptTicket); - var allEvents = await eventService.FetchAllEventsAsync(Namespace, podName, cancellationToken); + var allEvents = await eventService.FetchAllEventsAsync(podName, cancellationToken); if (allEvents is null) { return Array.Empty(); From a24ab714caa2040bbcfca7a696f93b8886dff5a1 Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Fri, 20 Dec 2024 12:03:04 +1100 Subject: [PATCH 9/9] Fix env var name --- .../Kubernetes/EnvironmentKubernetesConfiguration.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs b/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs index 7e2cb26b0..4ab85cb96 100644 --- a/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs +++ b/source/Octopus.Tentacle/Kubernetes/EnvironmentKubernetesConfiguration.cs @@ -24,7 +24,7 @@ public static class VariableNames public static readonly string ServerCommsAddress = "ServerCommsAddress"; public static readonly string ServerCommsAddresses = "ServerCommsAddresses"; - public static readonly string ScriptPodServiceAccountName = $"{EnvVarPrefix}__SCRIPTPODSERVICEACCOUNTNAME"; + public static readonly string ScriptPodServiceAccountName = $"{EnvVarPrefix}__PODSERVICEACCOUNTNAME"; public static readonly string ScriptPodImagePullSecretNames = $"{EnvVarPrefix}__PODIMAGEPULLSECRETNAMES"; public static readonly string ScriptPodVolumeClaimName = $"{EnvVarPrefix}__PODVOLUMECLAIMNAME"; @@ -38,10 +38,8 @@ public static class VariableNames public static readonly string ScriptPodContainerImageTag = $"{EnvVarPrefix}__SCRIPTPODIMAGETAG"; public static readonly string ScriptPodPullPolicy = $"{EnvVarPrefix}__SCRIPTPODPULLPOLICY"; - public static readonly string ScriptPodProxiesSecretName = $"{EnvVarPrefix}__PODPROXIESSECRETNAME"; - public static readonly string NfsWatchdogImage = $"{EnvVarPrefix}__NFSWATCHDOGIMAGE"; public static readonly string ScriptPodMonitorTimeout = $"{EnvVarPrefix}__PODMONITORTIMEOUT"; public static readonly string ScriptPodConsideredOrphanedAfterTimeSpan = $"{EnvVarPrefix}__PODSCONSIDEREDORPHANEDAFTERMINUTES";