From b755d1264c04235c3c96c30ffb174f133b9c00a6 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Tue, 24 Sep 2024 16:15:45 +1200 Subject: [PATCH 01/34] Just set the timeout to 10 minutes hardcoded for now. Can set through env variable later --- .../Support/IntegrationTestTimeout.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTestTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTestTimeout.cs index be3b7079a..b88a537bf 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTestTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTestTimeout.cs @@ -21,7 +21,7 @@ public static int TestTimeoutInMilliseconds() return (int)TimeSpan.FromHours(1).TotalMilliseconds; } - return (int)TimeSpan.FromMinutes(2).TotalMilliseconds; + return (int)TimeSpan.FromMinutes(10).TotalMilliseconds; } } } \ No newline at end of file From e616806ed7ec0ad8ac2c8d9a3cc7180ca0b5a944 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Thu, 26 Sep 2024 11:40:33 +1200 Subject: [PATCH 02/34] Add IntegrationTest_Timeout_Minutes to enable configurable timeout --- .../Support/IntegrationTestTimeout.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTestTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTestTimeout.cs index b88a537bf..db030a65c 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTestTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTestTimeout.cs @@ -13,6 +13,8 @@ public IntegrationTestTimeout(int timeoutInSeconds) : base((int)TimeSpan.FromSec public IntegrationTestTimeout() : base(TestTimeoutInMilliseconds()) { } + + public static int TestTimeoutInMilliseconds() { @@ -21,7 +23,17 @@ public static int TestTimeoutInMilliseconds() return (int)TimeSpan.FromHours(1).TotalMilliseconds; } - return (int)TimeSpan.FromMinutes(10).TotalMilliseconds; + return GetTimeoutFromEnvironmentVariable() ?? (int)TimeSpan.FromMinutes(10).TotalMilliseconds; + } + + static int? GetTimeoutFromEnvironmentVariable() + { + if (int.TryParse(Environment.GetEnvironmentVariable("IntegrationTest_Timeout_Minutes"), out var timeoutFromEnv)) + { + return (int)TimeSpan.FromMinutes(timeoutFromEnv).TotalMilliseconds; + } + + return null; } } } \ No newline at end of file From 0afdea17642fdb4c0a19fbf0206ffce7ad38b231 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Thu, 26 Sep 2024 11:52:03 +1200 Subject: [PATCH 03/34] Set default back to 2 mins --- .../Support/IntegrationTestTimeout.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTestTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTestTimeout.cs index db030a65c..c42709be1 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTestTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTestTimeout.cs @@ -13,8 +13,6 @@ public IntegrationTestTimeout(int timeoutInSeconds) : base((int)TimeSpan.FromSec public IntegrationTestTimeout() : base(TestTimeoutInMilliseconds()) { } - - public static int TestTimeoutInMilliseconds() { @@ -23,7 +21,7 @@ public static int TestTimeoutInMilliseconds() return (int)TimeSpan.FromHours(1).TotalMilliseconds; } - return GetTimeoutFromEnvironmentVariable() ?? (int)TimeSpan.FromMinutes(10).TotalMilliseconds; + return GetTimeoutFromEnvironmentVariable() ?? (int)TimeSpan.FromMinutes(2).TotalMilliseconds; } static int? GetTimeoutFromEnvironmentVariable() From cc42fc75c3e983bedfa745eb35d9e1f47b8d4ef7 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Thu, 26 Sep 2024 15:28:33 +1200 Subject: [PATCH 04/34] Disallow building and restoring for disk space issues resolution --- build/Build.Tests.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build/Build.Tests.cs b/build/Build.Tests.cs index 08919d18e..b40b8dc60 100644 --- a/build/Build.Tests.cs +++ b/build/Build.Tests.cs @@ -287,7 +287,8 @@ void RunTests(string testFramework, string testRuntime) DotNetTasks.DotNetTest(settings => settings .SetProjectFile(projectPath) .SetFramework(testFramework) - .SetLoggers("console;verbosity=normal", "teamcity")) + .SetLoggers("console;verbosity=normal", "teamcity") + ) ); } catch (Exception e) @@ -319,7 +320,9 @@ void RunIntegrationTests(string testFramework, string testRuntime, string filter .SetProjectFile(projectPath) .SetFramework(testFramework) .SetFilter(filter) - .SetLoggers("console;verbosity=normal", "teamcity")) + .SetLoggers("console;verbosity=normal", "teamcity") + .SetNoBuild(true) + .SetNoRestore(true)) ); } catch (Exception e) From 5f4745354da55dd6b3df9ad88d84c9899370ffd3 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Fri, 27 Sep 2024 19:33:09 +1200 Subject: [PATCH 05/34] Wait longer for tentacle to start --- .../Support/TentacleBuilder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/TentacleBuilder.cs b/source/Octopus.Tentacle.Tests.Integration/Support/TentacleBuilder.cs index 952ed4bcd..e617914c9 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Support/TentacleBuilder.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Support/TentacleBuilder.cs @@ -293,7 +293,7 @@ await RunCommandOutOfProcess( } }, cancellationToken); - await Task.WhenAny(runningTentacle, WaitHandleAsyncFactory.FromWaitHandle(hasTentacleStarted.WaitHandle, TimeSpan.FromMinutes(1), cancellationToken)); + await Task.WhenAny(runningTentacle, WaitHandleAsyncFactory.FromWaitHandle(hasTentacleStarted.WaitHandle, TimeSpan.FromMinutes(5), cancellationToken)); // Will throw. if (runningTentacle.IsCompleted) From d26055e2007bdbaf04bc7d612503b10d96188c61 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Sat, 28 Sep 2024 19:39:19 +1200 Subject: [PATCH 06/34] Skip some known problem tests on Windows 2012 (an environment with known performance issues) --- .../ClientScriptExecutionRetriesTimeout.cs | 2 ++ .../FileTransferServiceTests.cs | 3 +++ .../ScriptServiceV2IntegrationTest.cs | 2 ++ ...entsWithKnownPerformanceIssuesAttribute.cs | 25 +++++++++++++++++++ .../Util/SilentProcessRunnerFixture.cs | 1 + 5 files changed, 33 insertions(+) create mode 100644 source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs index edfb8893e..6ff609a88 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs @@ -8,6 +8,7 @@ using Octopus.Tentacle.Contracts.ClientServices; using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; using Octopus.Tentacle.Tests.Integration.Util; using Octopus.Tentacle.Tests.Integration.Util.Builders; using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; @@ -217,6 +218,7 @@ public async Task WhenRpcRetriesTimeOut_DuringStartScript_TheRpcCallIsCancelled( [Test] [TentacleConfigurations] + [SkipOnEnvironmentsWithKnownPerformanceIssues] public async Task WhenStartScriptFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) { var retryDuration = TimeSpan.FromSeconds(15); diff --git a/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs b/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs index bce59c8cc..8d3bfb513 100644 --- a/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs @@ -6,6 +6,7 @@ using NUnit.Framework; using Octopus.Tentacle.CommonTestUtils; using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; using Octopus.Tentacle.Tests.Integration.Util; namespace Octopus.Tentacle.Tests.Integration @@ -15,6 +16,7 @@ public class FileTransferServiceTests : IntegrationTest { [Test] [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + [SkipOnEnvironmentsWithKnownPerformanceIssues] public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase) { using var fileToUpload = new RandomTemporaryFileBuilder().Build(); @@ -42,6 +44,7 @@ public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleC [Test] [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + [SkipOnEnvironmentsWithKnownPerformanceIssues] public async Task DownloadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase) { using var fileToDownload = new RandomTemporaryFileBuilder().Build(); diff --git a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs index 4fb62eb72..d24f90e6e 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs @@ -9,6 +9,7 @@ using Octopus.Tentacle.Contracts.ClientServices; using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; using Octopus.Tentacle.Tests.Integration.Util; using Octopus.Tentacle.Tests.Integration.Util.Builders; using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; @@ -132,6 +133,7 @@ await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, Cancellati [Test] [TentacleConfigurations] + [SkipOnEnvironmentsWithKnownPerformanceIssues] public async Task WhenALongRunningScriptIsCancelled_TheScriptShouldStop(TentacleConfigurationTestCase tentacleConfigurationTestCase) { await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs b/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs new file mode 100644 index 000000000..096322039 --- /dev/null +++ b/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs @@ -0,0 +1,25 @@ +using System; +using System.Linq; +using NUnit.Framework; +using NUnit.Framework.Interfaces; +using NUnit.Framework.Internal; +using Octopus.Tentacle.Util; + +namespace Octopus.Tentacle.Tests.Integration.Support.TestAttributes +{ + [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)] + public class SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute : NUnitAttribute, IApplyToTest + { + public void ApplyToTest(Test test) + { + if (test.RunState == RunState.NotRunnable || test.RunState == RunState.Ignored) + return; + + if (bool.TryParse(Environment.GetEnvironmentVariable("Has_Known_Performance_Issues"), out _)) + { + test.RunState = RunState.Skipped; + test.Properties.Add("_SKIPREASON", "This test only runs on environments without performance issues"); + } + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs b/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs index 8169263a4..a7a9983dd 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs @@ -83,6 +83,7 @@ public void DebugLogging_ShouldContainDiagnosticsInfo_ForDefault() [Test] [Retry(3)] + [SkipOnEnvironmentsWithKnownPerformanceIssues] public void CancellationToken_ShouldForceKillTheProcess() { // Terminate the process after a very short time so the test doesn't run forever From f9a3643a61e74b8097da8d82b9e683be248090ab Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Sat, 28 Sep 2024 21:08:51 +1200 Subject: [PATCH 07/34] Add HelpForInstanceSpecificCommandsAlwaysWorks to skipped tests on Windows 2012 --- .../TentacleCommandLineTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs b/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs index fba7ceedb..f0fc20bbf 100644 --- a/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs @@ -272,6 +272,7 @@ public async Task CommandSpecificHelpAsJsonLooksSensibleToHumans(TentacleConfigu [Test] [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + [SkipOnEnvironmentsWithKnownPerformanceIssues] public async Task HelpForInstanceSpecificCommandsAlwaysWorks(TentacleConfigurationTestCase tc) { var (_, stdout, stderr) = await RunCommand(tc, null, "help", "--format=json"); From c9ccd99571873aa8db1de1491506bd8d9a5d5109 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Sun, 29 Sep 2024 22:53:40 +1300 Subject: [PATCH 08/34] Revert "Disallow building and restoring for disk space issues resolution" This reverts commit 12d6da93dd75040fcbede9831c5ed90b0835aff5. --- build/Build.Tests.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/build/Build.Tests.cs b/build/Build.Tests.cs index b40b8dc60..08919d18e 100644 --- a/build/Build.Tests.cs +++ b/build/Build.Tests.cs @@ -287,8 +287,7 @@ void RunTests(string testFramework, string testRuntime) DotNetTasks.DotNetTest(settings => settings .SetProjectFile(projectPath) .SetFramework(testFramework) - .SetLoggers("console;verbosity=normal", "teamcity") - ) + .SetLoggers("console;verbosity=normal", "teamcity")) ); } catch (Exception e) @@ -320,9 +319,7 @@ void RunIntegrationTests(string testFramework, string testRuntime, string filter .SetProjectFile(projectPath) .SetFramework(testFramework) .SetFilter(filter) - .SetLoggers("console;verbosity=normal", "teamcity") - .SetNoBuild(true) - .SetNoRestore(true)) + .SetLoggers("console;verbosity=normal", "teamcity")) ); } catch (Exception e) From 7cc73a8223cf421964da317e5228c677aa2995bf Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Mon, 30 Sep 2024 00:12:27 +1300 Subject: [PATCH 09/34] Revert "Wait longer for tentacle to start" This reverts commit 5f3bbfd12333effca570a0332519676edd170c03. --- .../Support/TentacleBuilder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/TentacleBuilder.cs b/source/Octopus.Tentacle.Tests.Integration/Support/TentacleBuilder.cs index e617914c9..952ed4bcd 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Support/TentacleBuilder.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Support/TentacleBuilder.cs @@ -293,7 +293,7 @@ await RunCommandOutOfProcess( } }, cancellationToken); - await Task.WhenAny(runningTentacle, WaitHandleAsyncFactory.FromWaitHandle(hasTentacleStarted.WaitHandle, TimeSpan.FromMinutes(5), cancellationToken)); + await Task.WhenAny(runningTentacle, WaitHandleAsyncFactory.FromWaitHandle(hasTentacleStarted.WaitHandle, TimeSpan.FromMinutes(1), cancellationToken)); // Will throw. if (runningTentacle.IsCompleted) From 4da45555c17fdc8e22f44e8cfc29474744aae779 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Mon, 30 Sep 2024 10:08:17 +1300 Subject: [PATCH 10/34] Put back lines to disallow building and restoring (to resolve disk space issues on CI) --- build/Build.Tests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/Build.Tests.cs b/build/Build.Tests.cs index 08919d18e..38d55fea3 100644 --- a/build/Build.Tests.cs +++ b/build/Build.Tests.cs @@ -319,7 +319,9 @@ void RunIntegrationTests(string testFramework, string testRuntime, string filter .SetProjectFile(projectPath) .SetFramework(testFramework) .SetFilter(filter) - .SetLoggers("console;verbosity=normal", "teamcity")) + .SetLoggers("console;verbosity=normal", "teamcity") + .SetNoBuild(true) + .SetNoRestore(true)) ); } catch (Exception e) From 3f4a29774639f64b432e4faecbe5c11f979e3b47 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Mon, 30 Sep 2024 14:50:53 +1300 Subject: [PATCH 11/34] Ignore more tests that rely on timing. Enable tests that usually fail due to disk space issues --- .../ClientFileTransferRetriesTimeout.cs | 2 ++ ...ntFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs | 2 ++ ...ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs | 2 ++ .../ClientScriptExecutionRetriesTimeout.cs | 2 +- ...criptExecutorObservesScriptObserverBackoffStrategy.cs | 2 ++ .../FileTransferServiceTests.cs | 2 -- .../ScriptServiceV2IntegrationTest.cs | 2 +- ...pOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs | 9 ++++++++- .../TentacleCommandLineTests.cs | 2 +- .../Util/SilentProcessRunnerFixture.cs | 8 +++++++- 10 files changed, 26 insertions(+), 7 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs index d2e5784d1..f33ba0bf0 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs @@ -11,6 +11,7 @@ using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Support; using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; using Octopus.Tentacle.Tests.Integration.Util; using Octopus.Tentacle.Tests.Integration.Util.Builders; using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; @@ -24,6 +25,7 @@ namespace Octopus.Tentacle.Tests.Integration /// from RPC calls when they are being retried and the rpc timeout period elapses. /// [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public class ClientFileTransferRetriesTimeout : IntegrationTest { readonly TimeSpan retryIfRemainingDurationAtLeastBuffer = TimeSpan.FromSeconds(1); diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs index b8ebd1261..cb5876c7f 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs @@ -7,6 +7,7 @@ using Octopus.Tentacle.Contracts.ClientServices; using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; using Octopus.Tentacle.Tests.Integration.Util.Builders; using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; @@ -14,6 +15,7 @@ namespace Octopus.Tentacle.Tests.Integration { [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public class ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled : IntegrationTest { [Test] diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs index df282f0fd..eef4f972f 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs @@ -10,6 +10,7 @@ using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Support; using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; using Octopus.Tentacle.Tests.Integration.Util.Builders; using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; @@ -20,6 +21,7 @@ public class ClientFileTransfersAreRetriedWhenRetriesAreEnabled : IntegrationTes { [Test] [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public async Task FailedUploadsAreRetriedAndIsEventuallySuccessful(TentacleConfigurationTestCase tentacleConfigurationTestCase) { await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs index 6ff609a88..2ea063101 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs @@ -218,7 +218,7 @@ public async Task WhenRpcRetriesTimeOut_DuringStartScript_TheRpcCallIsCancelled( [Test] [TentacleConfigurations] - [SkipOnEnvironmentsWithKnownPerformanceIssues] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public async Task WhenStartScriptFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) { var retryDuration = TimeSpan.FromSeconds(15); diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs index b61e5aeb0..949d586ae 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs @@ -6,6 +6,7 @@ using Octopus.Tentacle.Contracts.ClientServices; using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; using Octopus.Tentacle.Tests.Integration.Util; using Octopus.Tentacle.Tests.Integration.Util.Builders; using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; @@ -13,6 +14,7 @@ namespace Octopus.Tentacle.Tests.Integration { [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public class ClientScriptExecutorObservesScriptObserverBackoffStrategy : IntegrationTest { [Test] diff --git a/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs b/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs index 8d3bfb513..dd166d8bc 100644 --- a/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs @@ -16,7 +16,6 @@ public class FileTransferServiceTests : IntegrationTest { [Test] [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - [SkipOnEnvironmentsWithKnownPerformanceIssues] public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase) { using var fileToUpload = new RandomTemporaryFileBuilder().Build(); @@ -44,7 +43,6 @@ public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleC [Test] [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - [SkipOnEnvironmentsWithKnownPerformanceIssues] public async Task DownloadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase) { using var fileToDownload = new RandomTemporaryFileBuilder().Build(); diff --git a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs index d24f90e6e..816318950 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs @@ -133,7 +133,7 @@ await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, Cancellati [Test] [TentacleConfigurations] - [SkipOnEnvironmentsWithKnownPerformanceIssues] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public async Task WhenALongRunningScriptIsCancelled_TheScriptShouldStop(TentacleConfigurationTestCase tentacleConfigurationTestCase) { await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs b/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs index 096322039..493e6cbbc 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs @@ -10,6 +10,13 @@ namespace Octopus.Tentacle.Tests.Integration.Support.TestAttributes [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)] public class SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute : NUnitAttribute, IApplyToTest { + string Reason { get; } + + public SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute(string reason) + { + Reason = reason; + } + public void ApplyToTest(Test test) { if (test.RunState == RunState.NotRunnable || test.RunState == RunState.Ignored) @@ -18,7 +25,7 @@ public void ApplyToTest(Test test) if (bool.TryParse(Environment.GetEnvironmentVariable("Has_Known_Performance_Issues"), out _)) { test.RunState = RunState.Skipped; - test.Properties.Add("_SKIPREASON", "This test only runs on environments without performance issues"); + test.Properties.Add("_SKIPREASON", $"This test only runs on environments without performance issues because {Reason}"); } } } diff --git a/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs b/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs index f0fc20bbf..42adb51d8 100644 --- a/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs @@ -28,6 +28,7 @@ namespace Octopus.Tentacle.Tests.Integration /// Please review any changes to the assertions made by these tests carefully. /// [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("we don't expect customers to execute it in this environment in a situation where it needs to behave within a reasonable timeframe.")] public class TentacleCommandLineTests : IntegrationTest { [Test] @@ -272,7 +273,6 @@ public async Task CommandSpecificHelpAsJsonLooksSensibleToHumans(TentacleConfigu [Test] [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - [SkipOnEnvironmentsWithKnownPerformanceIssues] public async Task HelpForInstanceSpecificCommandsAlwaysWorks(TentacleConfigurationTestCase tc) { var (_, stdout, stderr) = await RunCommand(tc, null, "help", "--format=json"); diff --git a/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs b/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs index a7a9983dd..4dd44b0a9 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs @@ -33,6 +33,7 @@ public void SetUpLocal() } [Test] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void ExitCode_ShouldBeReturned() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) @@ -56,6 +57,7 @@ public void ExitCode_ShouldBeReturned() } [Test] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void DebugLogging_ShouldContainDiagnosticsInfo_ForDefault() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) @@ -83,7 +85,7 @@ public void DebugLogging_ShouldContainDiagnosticsInfo_ForDefault() [Test] [Retry(3)] - [SkipOnEnvironmentsWithKnownPerformanceIssues] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void CancellationToken_ShouldForceKillTheProcess() { // Terminate the process after a very short time so the test doesn't run forever @@ -116,6 +118,7 @@ public void CancellationToken_ShouldForceKillTheProcess() } [Test] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void EchoHello_ShouldWriteToStdOut() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) @@ -138,6 +141,7 @@ public void EchoHello_ShouldWriteToStdOut() } [Test] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void EchoError_ShouldWriteToStdErr() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) @@ -160,6 +164,7 @@ public void EchoError_ShouldWriteToStdErr() } [Test] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void RunAsCurrentUser_ShouldWork() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) @@ -186,6 +191,7 @@ public void RunAsCurrentUser_ShouldWork() [Test] [WindowsTest] [TestCase("powershell.exe", "-command \"Write-Host $env:userdomain\\$env:username\"")] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void RunAsCurrentUser_PowerShell_ShouldWork(string command, string arguments) { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) From 1c13439726a538c13002090726edecca8539e075 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Wed, 23 Oct 2024 10:12:44 +1300 Subject: [PATCH 12/34] Comment out all tests other than file transfer tests --- .../CapabilitiesServiceV2Test.cs | 252 +-- .../ClientFileTransferRetriesTimeout.cs | 528 +++--- ...fersAreNotRetriedWhenRetriesAreDisabled.cs | 198 +-- ...ransfersAreRetriedWhenRetriesAreEnabled.cs | 216 +-- .../ClientGathersRpcCallMetrics.cs | 454 ++--- .../ClientScriptExecutionAdditionalScripts.cs | 100 +- ...ionCanBeCancelledWhenRetriesAreDisabled.cs | 868 +++++----- ...tionCanBeCancelledWhenRetriesAreEnabled.cs | 1344 +++++++-------- ...iptExecutionCanRecoverFromNetworkIssues.cs | 616 +++---- .../ClientScriptExecutionIsolationMutex.cs | 364 ++-- .../ClientScriptExecutionRetriesTimeout.cs | 1096 ++++++------ ...lientScriptExecutionScriptArgumentsWork.cs | 82 +- ...ClientScriptExecutionScriptFilesAreSent.cs | 76 +- ...NonV1IsNotRetriedWhenRetriesAreDisabled.cs | 584 +++---- ...iptExecutionScriptServiceV1IsNotRetried.cs | 502 +++--- ...criptExecutionWorksWithMultipleVersions.cs | 78 +- ...orObservesScriptObserverBackoffStrategy.cs | 88 +- .../MachineConfigurationHomeDirectoryTests.cs | 108 +- .../ScriptServiceTests.cs | 272 +-- .../ScriptServiceV2IntegrationTest.cs | 432 ++--- .../TentacleClientObserver.cs | 320 ++-- .../TentacleCommandLineTests.cs | 1504 ++++++++--------- .../TentacleStartupAndShutdownTests.cs | 144 +- .../WorkspaceCleanerTests.cs | 514 +++--- 24 files changed, 5370 insertions(+), 5370 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/CapabilitiesServiceV2Test.cs b/source/Octopus.Tentacle.Tests.Integration/CapabilitiesServiceV2Test.cs index 6f9b7300a..bbe894108 100644 --- a/source/Octopus.Tentacle.Tests.Integration/CapabilitiesServiceV2Test.cs +++ b/source/Octopus.Tentacle.Tests.Integration/CapabilitiesServiceV2Test.cs @@ -1,126 +1,126 @@ -#nullable enable -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using FluentAssertions; -using Halibut; -using NUnit.Framework; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Contracts.Capabilities; -using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1; -using Octopus.Tentacle.Contracts.ScriptServiceV2; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Util.Builders; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class CapabilitiesServiceV2Test : IntegrationTest - { - [Test] - [TentacleConfigurations(testCapabilitiesServiceVersions: true)] - public async Task CapabilitiesFromAnOlderTentacleWhichHasNoCapabilitiesService_WorksWithTheBackwardsCompatabilityDecorator(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - Version? version = tentacleConfigurationTestCase.Version; - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); - - var capabilities = (await clientAndTentacle.TentacleClient.CapabilitiesServiceV2.GetCapabilitiesAsync(new(CancellationToken))).SupportedCapabilities; - - capabilities.Should().Contain(nameof(IScriptService)); - capabilities.Should().Contain(nameof(IFileTransferService)); - - //all versions have ScriptServiceV1 & IFileTransferService - var expectedCapabilitiesCount = 2; - if (version.HasScriptServiceV2()) - { - capabilities.Should().Contain(nameof(IScriptServiceV2)); - expectedCapabilitiesCount++; - } - - capabilities.Count.Should().Be(expectedCapabilitiesCount); - } - - [Test] - [TentacleConfigurations] - public async Task CapabilitiesServiceDoesNotReturnKubernetesScriptServiceForNonKubernetesTentacle(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var version = tentacleConfigurationTestCase.Version; - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); - - var capabilities = (await clientAndTentacle.TentacleClient.CapabilitiesServiceV2.GetCapabilitiesAsync(new(CancellationToken))).SupportedCapabilities; - - capabilities.Should().Contain(nameof(IScriptService)); - capabilities.Should().Contain(nameof(IFileTransferService)); - - //all versions have ScriptServiceV1 & IFileTransferService - var expectedCapabilitiesCount = 2; - if (version.HasScriptServiceV2()) - { - capabilities.Should().Contain(nameof(IScriptServiceV2)); - expectedCapabilitiesCount++; - } - - capabilities.Should().NotContain(nameof(IKubernetesScriptServiceV1)); - - capabilities.Count.Should().Be(expectedCapabilitiesCount); - } - - [Test] - [TentacleConfigurations(testCapabilitiesServiceVersions: true)] - public async Task CapabilitiesResponseShouldBeCached(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var capabilitiesResponses = new List(); - var resumePortForwarder = false; - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPortForwarder(out var portForwarder) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .DecorateCapabilitiesServiceV2With(d => d - .DecorateGetCapabilitiesWith( - async (inner, options) => - { - var response = await inner.GetCapabilitiesAsync(options); - - capabilitiesResponses.Add(response); - - if (resumePortForwarder) - { - // (2) Once a get capabilities call has been made which uses the cached response then resume normal RPC calls - // to allow script execution to continue - portForwarder.Value.ReturnToNormalMode(); - } - - return response; - })) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Print("Running...")) - .Build(); - - await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - - // (1) Kill new and existing connections to ensure no RPC calls can be made - clientAndTentacle.PortForwarder!.EnterKillNewAndExistingConnectionsMode(); - resumePortForwarder = true; - - try - { - await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - } - catch (HalibutClientException) when (tentacleConfigurationTestCase.TentacleType == TentacleType.Polling && tentacleConfigurationTestCase.Version != null) - { - // For script execution on a tentacle without ScriptServiceV2 and retries a polling request can be de-queued into a broken TCP Connection - // By the time this happens we will have already called gt capabilities and got the cached response so we can safely ignore. - } - - capabilitiesResponses.Should().HaveCount(2); - capabilitiesResponses[0].Should().BeEquivalentTo(capabilitiesResponses[1]); - } - } -} +// #nullable enable +// using System; +// using System.Collections.Generic; +// using System.Threading.Tasks; +// using FluentAssertions; +// using Halibut; +// using NUnit.Framework; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Contracts.Capabilities; +// using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1; +// using Octopus.Tentacle.Contracts.ScriptServiceV2; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class CapabilitiesServiceV2Test : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(testCapabilitiesServiceVersions: true)] +// public async Task CapabilitiesFromAnOlderTentacleWhichHasNoCapabilitiesService_WorksWithTheBackwardsCompatabilityDecorator(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// Version? version = tentacleConfigurationTestCase.Version; +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); +// +// var capabilities = (await clientAndTentacle.TentacleClient.CapabilitiesServiceV2.GetCapabilitiesAsync(new(CancellationToken))).SupportedCapabilities; +// +// capabilities.Should().Contain(nameof(IScriptService)); +// capabilities.Should().Contain(nameof(IFileTransferService)); +// +// //all versions have ScriptServiceV1 & IFileTransferService +// var expectedCapabilitiesCount = 2; +// if (version.HasScriptServiceV2()) +// { +// capabilities.Should().Contain(nameof(IScriptServiceV2)); +// expectedCapabilitiesCount++; +// } +// +// capabilities.Count.Should().Be(expectedCapabilitiesCount); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task CapabilitiesServiceDoesNotReturnKubernetesScriptServiceForNonKubernetesTentacle(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var version = tentacleConfigurationTestCase.Version; +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); +// +// var capabilities = (await clientAndTentacle.TentacleClient.CapabilitiesServiceV2.GetCapabilitiesAsync(new(CancellationToken))).SupportedCapabilities; +// +// capabilities.Should().Contain(nameof(IScriptService)); +// capabilities.Should().Contain(nameof(IFileTransferService)); +// +// //all versions have ScriptServiceV1 & IFileTransferService +// var expectedCapabilitiesCount = 2; +// if (version.HasScriptServiceV2()) +// { +// capabilities.Should().Contain(nameof(IScriptServiceV2)); +// expectedCapabilitiesCount++; +// } +// +// capabilities.Should().NotContain(nameof(IKubernetesScriptServiceV1)); +// +// capabilities.Count.Should().Be(expectedCapabilitiesCount); +// } +// +// [Test] +// [TentacleConfigurations(testCapabilitiesServiceVersions: true)] +// public async Task CapabilitiesResponseShouldBeCached(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var capabilitiesResponses = new List(); +// var resumePortForwarder = false; +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPortForwarder(out var portForwarder) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .DecorateCapabilitiesServiceV2With(d => d +// .DecorateGetCapabilitiesWith( +// async (inner, options) => +// { +// var response = await inner.GetCapabilitiesAsync(options); +// +// capabilitiesResponses.Add(response); +// +// if (resumePortForwarder) +// { +// // (2) Once a get capabilities call has been made which uses the cached response then resume normal RPC calls +// // to allow script execution to continue +// portForwarder.Value.ReturnToNormalMode(); +// } +// +// return response; +// })) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Print("Running...")) +// .Build(); +// +// await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// +// // (1) Kill new and existing connections to ensure no RPC calls can be made +// clientAndTentacle.PortForwarder!.EnterKillNewAndExistingConnectionsMode(); +// resumePortForwarder = true; +// +// try +// { +// await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// } +// catch (HalibutClientException) when (tentacleConfigurationTestCase.TentacleType == TentacleType.Polling && tentacleConfigurationTestCase.Version != null) +// { +// // For script execution on a tentacle without ScriptServiceV2 and retries a polling request can be de-queued into a broken TCP Connection +// // By the time this happens we will have already called gt capabilities and got the cached response so we can safely ignore. +// } +// +// capabilitiesResponses.Should().HaveCount(2); +// capabilitiesResponses[0].Should().BeEquivalentTo(capabilitiesResponses[1]); +// } +// } +// } diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs index f33ba0bf0..f8659b33e 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs @@ -1,264 +1,264 @@ -using System; -using System.Collections; -using System.Diagnostics; -using System.IO; -using System.Threading.Tasks; -using FluentAssertions; -using Halibut; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils.Diagnostics; -using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; -using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; -using Octopus.TestPortForwarder; - -namespace Octopus.Tentacle.Tests.Integration -{ - /// - /// These tests make sure that we can cancel or walk away (if code does not cooperate with cancellation tokens) - /// from RPC calls when they are being retried and the rpc timeout period elapses. - /// - [IntegrationTestTimeout] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] - public class ClientFileTransferRetriesTimeout : IntegrationTest - { - readonly TimeSpan retryIfRemainingDurationAtLeastBuffer = TimeSpan.FromSeconds(1); - readonly TimeSpan retryBackoffBuffer = TimeSpan.FromSeconds(2); - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None, additionalParameterTypes: new object[] { typeof(StopPortForwarderAfterFirstCallValues) })] - public async Task WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, bool stopPortForwarderAfterFirstCall) - { - PortForwarder portForwarder = null!; - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - // Set a short retry duration so we cancel fairly quickly - .WithRetryDuration(TimeSpan.FromSeconds(15)) - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var methodUsages) - .DecorateFileTransferServiceWith(d => d - .BeforeUploadFile( - async () => - { - if (methodUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException is null) - { - // Ensure there is an active connection so it can be killed correctly - await tcpConnectionUtilities.RestartTcpConnection(); - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - else - { - if (stopPortForwarderAfterFirstCall) - { - // Kill the port forwarder so the next requests are in the connecting state when retries timeout - Logger.Information("Killing PortForwarder"); - portForwarder!.EnterKillNewAndExistingConnectionsMode(); - } - else - { - // Ensure there is an active connection so it can be killed correctly - await tcpConnectionUtilities.RestartTcpConnection(); - // Pause the port forwarder so the next requests are in-flight when retries timeout - responseMessageTcpKiller.PauseConnectionOnNextResponse(); - } - } - })) - .Build()) - .Build(CancellationToken); - - portForwarder = clientAndTentacle.PortForwarder; - - var inMemoryLog = new InMemoryLog(); - - var remotePath = Path.Combine(clientAndTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); - var dataStream = DataStream.FromString("The Stream"); - - // Start the script which will wait for a file to exist - var duration = Stopwatch.StartNew(); - var executeScriptTask = clientAndTentacle.TentacleClient.UploadFile(remotePath, dataStream, CancellationToken, inMemoryLog); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - duration.Stop(); - - methodUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().BeGreaterOrEqualTo(2); - methodUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(0); - - // Ensure we actually waited and retried until the timeout policy kicked in - duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); - - inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task WhenUploadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var retryDuration = TimeSpan.FromSeconds(15); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - // Set a short retry duration so we cancel fairly quickly - .WithRetryDuration(retryDuration) - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var methodUsages) - .DecorateFileTransferServiceWith(d => d - .BeforeUploadFile( - async () => - { - await tcpConnectionUtilities.RestartTcpConnection(); - - // Sleep to make the initial RPC call take longer than the allowed retry duration - await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); - - // Kill the first UploadFile call to force the rpc call into retries - responseMessageTcpKiller.KillConnectionOnNextResponse(); - })) - .Build()) - .Build(CancellationToken); - - var inMemoryLog = new InMemoryLog(); - var remotePath = Path.Combine(clientAndTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); - var dataStream = DataStream.FromString("The Stream"); - var executeScriptTask = clientAndTentacle.TentacleClient.UploadFile(remotePath, dataStream, CancellationToken, inMemoryLog); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - methodUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(1); - methodUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(0); - - inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None, additionalParameterTypes: new object[] { typeof(StopPortForwarderAfterFirstCallValues) })] - public async Task WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, bool stopPortForwarderAfterFirstCall) - { - PortForwarder portForwarder = null!; - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - // Set a short retry duration so we cancel fairly quickly - .WithRetryDuration(TimeSpan.FromSeconds(15)) - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var recordedUsages) - .DecorateFileTransferServiceWith(d => d - .BeforeDownloadFile( - async () => - { - if (recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException is null) - { - // Ensure there is an active connection so it can be killed correctly - await tcpConnectionUtilities.RestartTcpConnection(); - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - else - { - - if (stopPortForwarderAfterFirstCall) - { - // Kill the port forwarder so the next requests are in the connecting state when retries timeout - Logger.Information("Killing PortForwarder"); - portForwarder!.Dispose(); - } - else - { - // Ensure there is an active connection so it can be killed correctly - await tcpConnectionUtilities.RestartTcpConnection(); - // Pause the port forwarder so the next requests are in-flight when retries timeout - responseMessageTcpKiller.PauseConnectionOnNextResponse(); - } - } - })) - .Build()) - .Build(CancellationToken); - - portForwarder = clientAndTentacle.PortForwarder; - - using var tempFile = new RandomTemporaryFileBuilder().Build(); - - var inMemoryLog = new InMemoryLog(); - - // Start the script which will wait for a file to exist - var duration = Stopwatch.StartNew(); - var executeScriptTask = clientAndTentacle.TentacleClient.DownloadFile(tempFile.File.FullName, CancellationToken, inMemoryLog); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - duration.Stop(); - - recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().BeGreaterOrEqualTo(2); - recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(0); - - // Ensure we actually waited and retried until the timeout policy kicked in - duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); - - inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var retryDuration = TimeSpan.FromSeconds(15); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - // Set a short retry duration so we cancel fairly quickly - .WithRetryDuration(retryDuration) - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var recordedUsages) - .DecorateFileTransferServiceWith(d => d - .BeforeDownloadFile( - async () => - { - await tcpConnectionUtilities.RestartTcpConnection(); - - // Sleep to make the initial RPC call take longer than the allowed retry duration - await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); - - // Kill the first DownloadFile call to force the rpc call into retries - responseMessageTcpKiller.KillConnectionOnNextResponse(); - })) - .Build()) - .Build(CancellationToken); - - using var tempFile = new RandomTemporaryFileBuilder().Build(); - var inMemoryLog = new InMemoryLog(); - var executeScriptTask = clientAndTentacle.TentacleClient.DownloadFile(tempFile.File.FullName, CancellationToken, inMemoryLog); - - Func> action = async () => await executeScriptTask; - await action.Should().ThrowAsync(); - - recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(0); - - inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); - } - - class StopPortForwarderAfterFirstCallValues : IEnumerable - { - public IEnumerator GetEnumerator() - { - yield return true; - yield return false; - } - } - } -} \ No newline at end of file +// using System; +// using System.Collections; +// using System.Diagnostics; +// using System.IO; +// using System.Threading.Tasks; +// using FluentAssertions; +// using Halibut; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils.Diagnostics; +// using Octopus.Tentacle.Contracts.ClientServices; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; +// using Octopus.TestPortForwarder; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// /// +// /// These tests make sure that we can cancel or walk away (if code does not cooperate with cancellation tokens) +// /// from RPC calls when they are being retried and the rpc timeout period elapses. +// /// +// [IntegrationTestTimeout] +// [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] +// public class ClientFileTransferRetriesTimeout : IntegrationTest +// { +// readonly TimeSpan retryIfRemainingDurationAtLeastBuffer = TimeSpan.FromSeconds(1); +// readonly TimeSpan retryBackoffBuffer = TimeSpan.FromSeconds(2); +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None, additionalParameterTypes: new object[] { typeof(StopPortForwarderAfterFirstCallValues) })] +// public async Task WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, bool stopPortForwarderAfterFirstCall) +// { +// PortForwarder portForwarder = null!; +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// // Set a short retry duration so we cancel fairly quickly +// .WithRetryDuration(TimeSpan.FromSeconds(15)) +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var methodUsages) +// .DecorateFileTransferServiceWith(d => d +// .BeforeUploadFile( +// async () => +// { +// if (methodUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException is null) +// { +// // Ensure there is an active connection so it can be killed correctly +// await tcpConnectionUtilities.RestartTcpConnection(); +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// else +// { +// if (stopPortForwarderAfterFirstCall) +// { +// // Kill the port forwarder so the next requests are in the connecting state when retries timeout +// Logger.Information("Killing PortForwarder"); +// portForwarder!.EnterKillNewAndExistingConnectionsMode(); +// } +// else +// { +// // Ensure there is an active connection so it can be killed correctly +// await tcpConnectionUtilities.RestartTcpConnection(); +// // Pause the port forwarder so the next requests are in-flight when retries timeout +// responseMessageTcpKiller.PauseConnectionOnNextResponse(); +// } +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// portForwarder = clientAndTentacle.PortForwarder; +// +// var inMemoryLog = new InMemoryLog(); +// +// var remotePath = Path.Combine(clientAndTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); +// var dataStream = DataStream.FromString("The Stream"); +// +// // Start the script which will wait for a file to exist +// var duration = Stopwatch.StartNew(); +// var executeScriptTask = clientAndTentacle.TentacleClient.UploadFile(remotePath, dataStream, CancellationToken, inMemoryLog); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// duration.Stop(); +// +// methodUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().BeGreaterOrEqualTo(2); +// methodUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(0); +// +// // Ensure we actually waited and retried until the timeout policy kicked in +// duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); +// +// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task WhenUploadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var retryDuration = TimeSpan.FromSeconds(15); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// // Set a short retry duration so we cancel fairly quickly +// .WithRetryDuration(retryDuration) +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var methodUsages) +// .DecorateFileTransferServiceWith(d => d +// .BeforeUploadFile( +// async () => +// { +// await tcpConnectionUtilities.RestartTcpConnection(); +// +// // Sleep to make the initial RPC call take longer than the allowed retry duration +// await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); +// +// // Kill the first UploadFile call to force the rpc call into retries +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// })) +// .Build()) +// .Build(CancellationToken); +// +// var inMemoryLog = new InMemoryLog(); +// var remotePath = Path.Combine(clientAndTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); +// var dataStream = DataStream.FromString("The Stream"); +// var executeScriptTask = clientAndTentacle.TentacleClient.UploadFile(remotePath, dataStream, CancellationToken, inMemoryLog); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// methodUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(1); +// methodUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(0); +// +// inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None, additionalParameterTypes: new object[] { typeof(StopPortForwarderAfterFirstCallValues) })] +// public async Task WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, bool stopPortForwarderAfterFirstCall) +// { +// PortForwarder portForwarder = null!; +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// // Set a short retry duration so we cancel fairly quickly +// .WithRetryDuration(TimeSpan.FromSeconds(15)) +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var recordedUsages) +// .DecorateFileTransferServiceWith(d => d +// .BeforeDownloadFile( +// async () => +// { +// if (recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException is null) +// { +// // Ensure there is an active connection so it can be killed correctly +// await tcpConnectionUtilities.RestartTcpConnection(); +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// else +// { +// +// if (stopPortForwarderAfterFirstCall) +// { +// // Kill the port forwarder so the next requests are in the connecting state when retries timeout +// Logger.Information("Killing PortForwarder"); +// portForwarder!.Dispose(); +// } +// else +// { +// // Ensure there is an active connection so it can be killed correctly +// await tcpConnectionUtilities.RestartTcpConnection(); +// // Pause the port forwarder so the next requests are in-flight when retries timeout +// responseMessageTcpKiller.PauseConnectionOnNextResponse(); +// } +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// portForwarder = clientAndTentacle.PortForwarder; +// +// using var tempFile = new RandomTemporaryFileBuilder().Build(); +// +// var inMemoryLog = new InMemoryLog(); +// +// // Start the script which will wait for a file to exist +// var duration = Stopwatch.StartNew(); +// var executeScriptTask = clientAndTentacle.TentacleClient.DownloadFile(tempFile.File.FullName, CancellationToken, inMemoryLog); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// duration.Stop(); +// +// recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().BeGreaterOrEqualTo(2); +// recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(0); +// +// // Ensure we actually waited and retried until the timeout policy kicked in +// duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); +// +// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var retryDuration = TimeSpan.FromSeconds(15); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// // Set a short retry duration so we cancel fairly quickly +// .WithRetryDuration(retryDuration) +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var recordedUsages) +// .DecorateFileTransferServiceWith(d => d +// .BeforeDownloadFile( +// async () => +// { +// await tcpConnectionUtilities.RestartTcpConnection(); +// +// // Sleep to make the initial RPC call take longer than the allowed retry duration +// await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); +// +// // Kill the first DownloadFile call to force the rpc call into retries +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// })) +// .Build()) +// .Build(CancellationToken); +// +// using var tempFile = new RandomTemporaryFileBuilder().Build(); +// var inMemoryLog = new InMemoryLog(); +// var executeScriptTask = clientAndTentacle.TentacleClient.DownloadFile(tempFile.File.FullName, CancellationToken, inMemoryLog); +// +// Func> action = async () => await executeScriptTask; +// await action.Should().ThrowAsync(); +// +// recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(0); +// +// inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); +// } +// +// class StopPortForwarderAfterFirstCallValues : IEnumerable +// { +// public IEnumerator GetEnumerator() +// { +// yield return true; +// yield return false; +// } +// } +// } +// } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs index cb5876c7f..df7969d42 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs @@ -1,99 +1,99 @@ -using System; -using System.IO; -using System.Threading.Tasks; -using FluentAssertions; -using Halibut; -using NUnit.Framework; -using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] - public class ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled : IntegrationTest - { - [Test] - [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task FailedUploadsAreNotRetriedAndFail(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithRetriesDisabled() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var recordedUsages) - .DecorateFileTransferServiceWith(d => d - .BeforeUploadFile( - async () => - { - await tcpConnectionUtilities.RestartTcpConnection(); - - // Only kill the connection the first time, causing the upload - // to succeed - and therefore failing the test - if retries are attempted - if (recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException is null) - { - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - })) - .Build()) - .Build(CancellationToken); - - var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); - var uploadFileTask = clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); - - await AssertionExtensions.Should(async () => await uploadFileTask).ThrowExceptionContractAsync(expectedException); - - recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException.Should().NotBeNull(); - recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(1); - } - - [Test] - [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task FailedDownloadsAreNotRetriedAndFail(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithRetriesDisabled() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var recordedUsages) - .DecorateFileTransferServiceWith(d => d - .BeforeDownloadFile( - async () => - { - await tcpConnectionUtilities.RestartTcpConnection(); - - // Only kill the connection the first time, causing the upload - // to succeed - and therefore failing the test - if retries are attempted - if (recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException is null) - { - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - })) - .Build()) - .Build(CancellationToken); - - var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); - - await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); - var downloadFileTask = clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); - - await AssertionExtensions.Should(async () => await downloadFileTask).ThrowExceptionContractAsync(expectedException); - - recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException.Should().NotBeNull(); - recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(1); - } - } -} \ No newline at end of file +// using System; +// using System.IO; +// using System.Threading.Tasks; +// using FluentAssertions; +// using Halibut; +// using NUnit.Framework; +// using Octopus.Tentacle.Contracts.ClientServices; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] +// public class ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task FailedUploadsAreNotRetriedAndFail(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithRetriesDisabled() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var recordedUsages) +// .DecorateFileTransferServiceWith(d => d +// .BeforeUploadFile( +// async () => +// { +// await tcpConnectionUtilities.RestartTcpConnection(); +// +// // Only kill the connection the first time, causing the upload +// // to succeed - and therefore failing the test - if retries are attempted +// if (recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException is null) +// { +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); +// var uploadFileTask = clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await uploadFileTask).ThrowExceptionContractAsync(expectedException); +// +// recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException.Should().NotBeNull(); +// recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(1); +// } +// +// [Test] +// [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task FailedDownloadsAreNotRetriedAndFail(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithRetriesDisabled() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var recordedUsages) +// .DecorateFileTransferServiceWith(d => d +// .BeforeDownloadFile( +// async () => +// { +// await tcpConnectionUtilities.RestartTcpConnection(); +// +// // Only kill the connection the first time, causing the upload +// // to succeed - and therefore failing the test - if retries are attempted +// if (recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException is null) +// { +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); +// +// await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); +// var downloadFileTask = clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await downloadFileTask).ThrowExceptionContractAsync(expectedException); +// +// recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException.Should().NotBeNull(); +// recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(1); +// } +// } +// } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs index eef4f972f..c823c40e1 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs @@ -1,108 +1,108 @@ -using System; -using System.IO; -using System.Threading.Tasks; -using FluentAssertions; -using Halibut; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils; -using Octopus.Tentacle.CommonTestUtils.Diagnostics; -using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; -using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; - -namespace Octopus.Tentacle.Tests.Integration -{ - public class ClientFileTransfersAreRetriedWhenRetriesAreEnabled : IntegrationTest - { - [Test] - [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] - public async Task FailedUploadsAreRetriedAndIsEventuallySuccessful(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var recordedUsages) - .DecorateFileTransferServiceWith(d => d - .BeforeUploadFile( - async () => - { - await tcpConnectionUtilities.RestartTcpConnection(); - - // Only kill the connection the first time, causing the upload - // to succeed - and therefore failing the test - if retries are attempted - if (recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException is null) - { - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - })) - .Build()) - .Build(CancellationToken); - - var inMemoryLog = new InMemoryLog(); - - var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); - - var res = await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken, inMemoryLog); - res.Length.Should().Be(5); - - recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException.Should().NotBeNull(); - recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(2); - - var downloadFile = await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); - var actuallySent = await downloadFile.GetUtf8String(CancellationToken); - actuallySent.Should().Be("Hello"); - - inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); - } - - [Test] - [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task FailedDownloadsAreRetriedAndIsEventuallySuccessful(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var recordedUsages) - .DecorateFileTransferServiceWith(d => d - .BeforeDownloadFile( - async () => - { - await tcpConnectionUtilities.RestartTcpConnection(); - - // Only kill the connection the first time, causing the upload - // to succeed - and therefore failing the test - if retries are attempted - if (recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException is null) - { - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - })) - .Build()) - .Build(CancellationToken); - - var inMemoryLog = new InMemoryLog(); - - var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); - - await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); - var downloadFile = await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken, inMemoryLog); - var actuallySent = await downloadFile.GetUtf8String(CancellationToken); - - recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException.Should().NotBeNull(); - recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(2); - - actuallySent.Should().Be("Hello"); - - inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); - } - } -} +// using System; +// using System.IO; +// using System.Threading.Tasks; +// using FluentAssertions; +// using Halibut; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils; +// using Octopus.Tentacle.CommonTestUtils.Diagnostics; +// using Octopus.Tentacle.Contracts.ClientServices; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// public class ClientFileTransfersAreRetriedWhenRetriesAreEnabled : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] +// [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] +// public async Task FailedUploadsAreRetriedAndIsEventuallySuccessful(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var recordedUsages) +// .DecorateFileTransferServiceWith(d => d +// .BeforeUploadFile( +// async () => +// { +// await tcpConnectionUtilities.RestartTcpConnection(); +// +// // Only kill the connection the first time, causing the upload +// // to succeed - and therefore failing the test - if retries are attempted +// if (recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException is null) +// { +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var inMemoryLog = new InMemoryLog(); +// +// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); +// +// var res = await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken, inMemoryLog); +// res.Length.Should().Be(5); +// +// recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException.Should().NotBeNull(); +// recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(2); +// +// var downloadFile = await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); +// var actuallySent = await downloadFile.GetUtf8String(CancellationToken); +// actuallySent.Should().Be("Hello"); +// +// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); +// } +// +// [Test] +// [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task FailedDownloadsAreRetriedAndIsEventuallySuccessful(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var recordedUsages) +// .DecorateFileTransferServiceWith(d => d +// .BeforeDownloadFile( +// async () => +// { +// await tcpConnectionUtilities.RestartTcpConnection(); +// +// // Only kill the connection the first time, causing the upload +// // to succeed - and therefore failing the test - if retries are attempted +// if (recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException is null) +// { +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var inMemoryLog = new InMemoryLog(); +// +// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); +// +// await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); +// var downloadFile = await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken, inMemoryLog); +// var actuallySent = await downloadFile.GetUtf8String(CancellationToken); +// +// recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException.Should().NotBeNull(); +// recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(2); +// +// actuallySent.Should().Be("Hello"); +// +// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); +// } +// } +// } diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientGathersRpcCallMetrics.cs b/source/Octopus.Tentacle.Tests.Integration/ClientGathersRpcCallMetrics.cs index 5975f8ded..4c76806fc 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientGathersRpcCallMetrics.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientGathersRpcCallMetrics.cs @@ -1,227 +1,227 @@ -using System; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using FluentAssertions; -using Halibut; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Contracts.Capabilities; -using Octopus.Tentacle.Contracts.Observability; -using Octopus.Tentacle.Contracts.ScriptServiceV2; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class ClientGathersRpcCallMetrics : IntegrationTest - { - [Test] - [TentacleConfigurations(testCommonVersions: true)] - public async Task ExecuteScriptShouldGatherMetrics_WhenSucceeds(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - // Arrange - var tentacleClientObserver = new TestTentacleClientObserver(); - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleClientObserver(tentacleClientObserver) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b.Print("Hello")) - .Build(); - - // Act - var (finalResponse, _) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - - // Assert - finalResponse.State.Should().Be(ProcessState.Complete); - - var executeScriptMetrics = tentacleClientObserver.ExecuteScriptMetrics.Should().ContainSingle().Subject; - ThenClientOperationMetricsShouldBeSuccessful(executeScriptMetrics); - - var expectedScriptService = tentacleConfigurationTestCase.ScriptServiceToTest == TentacleConfigurationTestCases.ScriptServiceV2Type - ? nameof(IScriptServiceV2) - : nameof(IScriptService); - - tentacleClientObserver.RpcCallMetrics.Should().NotBeEmpty(); - tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == nameof(ICapabilitiesServiceV2.GetCapabilities) && m.RpcCall.Service == nameof(ICapabilitiesServiceV2)); - tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == nameof(IScriptServiceV2.StartScript) && m.RpcCall.Service == expectedScriptService); - tentacleClientObserver.RpcCallMetrics.Should().Contain(m => m.RpcCall.Name == nameof(IScriptServiceV2.GetStatus) && m.RpcCall.Service == expectedScriptService); - tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == nameof(IScriptServiceV2.CompleteScript) && m.RpcCall.Service == expectedScriptService); - tentacleClientObserver.RpcCallMetrics.Should().AllSatisfy(m => m.Succeeded.Should().BeTrue()); - } - - [Test] - [TentacleConfigurations(testCommonVersions: true)] - public async Task ExecuteScriptShouldGatherMetrics_WhenFails(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - // Arrange - var tentacleClientObserver = new TestTentacleClientObserver(); - var exception = new HalibutClientException("Error"); - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleClientObserver(tentacleClientObserver) - .WithRetryDuration(TimeSpan.FromSeconds(1)) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .DecorateAllScriptServicesWith(u => u - .BeforeStartScript( - () => throw exception)) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b.Print("Hello")) - .Build(); - - // Act - await AssertionExtensions.Should(() => clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken)).ThrowAsync(); - - // Assert - var executeScriptMetrics = tentacleClientObserver.ExecuteScriptMetrics.Should().ContainSingle().Subject; - ThenClientOperationMetricsShouldBeFailed(executeScriptMetrics, exception); - - tentacleClientObserver.RpcCallMetrics.Should().NotBeEmpty(); - var startScriptMetric = tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == "StartScript").Subject; - startScriptMetric.Succeeded.Should().BeFalse(); - } - - [Test] - [TentacleConfigurations(testCommonVersions: true)] - public async Task UploadFileShouldGatherMetrics_WhenSucceeds(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - // Arrange - var tentacleClientObserver = new TestTentacleClientObserver(); - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleClientObserver(tentacleClientObserver) - .Build(CancellationToken); - - var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); - - // Act - await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); - - // Assert - var uploadFileMetrics = tentacleClientObserver.UploadFileMetrics.Should().ContainSingle().Subject; - ThenClientOperationMetricsShouldBeSuccessful(uploadFileMetrics); - - tentacleClientObserver.RpcCallMetrics.Should().HaveCountGreaterThan(0); - var metric = tentacleClientObserver.RpcCallMetrics.Last(); - metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.UploadFile)); - metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); - metric.Succeeded.Should().BeTrue(); - } - - [Test] - [TentacleConfigurations(testCommonVersions: true)] - public async Task UploadFileShouldGatherMetrics_WhenFails(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - // Arrange - var tentacleClientObserver = new TestTentacleClientObserver(); - var exception = new HalibutClientException("Error"); - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleClientObserver(tentacleClientObserver) - .WithRetryDuration(TimeSpan.FromSeconds(1)) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .DecorateFileTransferServiceWith(d => d.BeforeUploadFile(() => throw exception)) - .Build()) - .Build(CancellationToken); - - var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); - - // Act - await AssertionExtensions.Should(() => clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken)).ThrowAsync(); - - // Assert - var uploadFileMetrics = tentacleClientObserver.UploadFileMetrics.Should().ContainSingle().Subject; - ThenClientOperationMetricsShouldBeFailed(uploadFileMetrics, exception); - - tentacleClientObserver.RpcCallMetrics.Should().NotBeEmpty(); - var metric = tentacleClientObserver.RpcCallMetrics.Last(); - metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.UploadFile)); - metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); - metric.Succeeded.Should().BeFalse(); - } - - [Test] - [TentacleConfigurations(testCommonVersions: true)] - public async Task DownloadFileShouldGatherMetrics_WhenSucceeds(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - // Arrange - var tentacleClientObserver = new TestTentacleClientObserver(); - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleClientObserver(tentacleClientObserver) - .Build(CancellationToken); - - var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "DownloadFile.txt"); - await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); - - // Act - await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); - - // Assert - var downloadFileMetrics = tentacleClientObserver.DownloadFileMetrics.Should().ContainSingle().Subject; - ThenClientOperationMetricsShouldBeSuccessful(downloadFileMetrics); - - tentacleClientObserver.RpcCallMetrics.Should().HaveCountGreaterThan(1); // the first one will be the upload - var metric = tentacleClientObserver.RpcCallMetrics.Last(); - metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.DownloadFile)); - metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); - metric.Succeeded.Should().BeTrue(); - } - - [Test] - [TentacleConfigurations(testCommonVersions: true)] - public async Task DownloadFileShouldGatherMetrics_WhenFails(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - // Arrange - var tentacleClientObserver = new TestTentacleClientObserver(); - var exception = new HalibutClientException("Error"); - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleClientObserver(tentacleClientObserver) - .WithRetryDuration(TimeSpan.FromSeconds(1)) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .DecorateFileTransferServiceWith(d => d.BeforeDownloadFile(() => throw exception)) - .Build()) - .Build(CancellationToken); - - var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "DownloadFile.txt"); - await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); - - // Act - await AssertionExtensions.Should(() => clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken)).ThrowAsync(); - - // Assert - var downloadFileMetrics = tentacleClientObserver.DownloadFileMetrics.Should().ContainSingle().Subject; - ThenClientOperationMetricsShouldBeFailed(downloadFileMetrics, exception); - - tentacleClientObserver.RpcCallMetrics.Should().HaveCountGreaterThan(1); // the first one will be the upload - var metric = tentacleClientObserver.RpcCallMetrics.Last(); - metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.DownloadFile)); - metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); - metric.Succeeded.Should().BeFalse(); - } - - private static void ThenClientOperationMetricsShouldBeSuccessful(ClientOperationMetrics metric) - { - metric.Succeeded.Should().BeTrue(); - metric.Exception.Should().BeNull(); - metric.WasCancelled.Should().BeFalse(); - - metric.End.Should().BeOnOrAfter(metric.Start); - metric.Duration.Should().Be(metric.End - metric.Start); - } - - private static void ThenClientOperationMetricsShouldBeFailed(ClientOperationMetrics metric, Exception expectedException) - { - metric.Succeeded.Should().BeFalse(); - metric.Exception.Should().BeEquivalentTo(expectedException); - metric.WasCancelled.Should().BeFalse(); - - metric.End.Should().BeOnOrAfter(metric.Start); - metric.Duration.Should().Be(metric.End - metric.Start); - } - } -} \ No newline at end of file +// using System; +// using System.IO; +// using System.Linq; +// using System.Threading.Tasks; +// using FluentAssertions; +// using Halibut; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Contracts.Capabilities; +// using Octopus.Tentacle.Contracts.Observability; +// using Octopus.Tentacle.Contracts.ScriptServiceV2; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class ClientGathersRpcCallMetrics : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(testCommonVersions: true)] +// public async Task ExecuteScriptShouldGatherMetrics_WhenSucceeds(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// // Arrange +// var tentacleClientObserver = new TestTentacleClientObserver(); +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleClientObserver(tentacleClientObserver) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b.Print("Hello")) +// .Build(); +// +// // Act +// var (finalResponse, _) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// +// // Assert +// finalResponse.State.Should().Be(ProcessState.Complete); +// +// var executeScriptMetrics = tentacleClientObserver.ExecuteScriptMetrics.Should().ContainSingle().Subject; +// ThenClientOperationMetricsShouldBeSuccessful(executeScriptMetrics); +// +// var expectedScriptService = tentacleConfigurationTestCase.ScriptServiceToTest == TentacleConfigurationTestCases.ScriptServiceV2Type +// ? nameof(IScriptServiceV2) +// : nameof(IScriptService); +// +// tentacleClientObserver.RpcCallMetrics.Should().NotBeEmpty(); +// tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == nameof(ICapabilitiesServiceV2.GetCapabilities) && m.RpcCall.Service == nameof(ICapabilitiesServiceV2)); +// tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == nameof(IScriptServiceV2.StartScript) && m.RpcCall.Service == expectedScriptService); +// tentacleClientObserver.RpcCallMetrics.Should().Contain(m => m.RpcCall.Name == nameof(IScriptServiceV2.GetStatus) && m.RpcCall.Service == expectedScriptService); +// tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == nameof(IScriptServiceV2.CompleteScript) && m.RpcCall.Service == expectedScriptService); +// tentacleClientObserver.RpcCallMetrics.Should().AllSatisfy(m => m.Succeeded.Should().BeTrue()); +// } +// +// [Test] +// [TentacleConfigurations(testCommonVersions: true)] +// public async Task ExecuteScriptShouldGatherMetrics_WhenFails(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// // Arrange +// var tentacleClientObserver = new TestTentacleClientObserver(); +// var exception = new HalibutClientException("Error"); +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleClientObserver(tentacleClientObserver) +// .WithRetryDuration(TimeSpan.FromSeconds(1)) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .DecorateAllScriptServicesWith(u => u +// .BeforeStartScript( +// () => throw exception)) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b.Print("Hello")) +// .Build(); +// +// // Act +// await AssertionExtensions.Should(() => clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken)).ThrowAsync(); +// +// // Assert +// var executeScriptMetrics = tentacleClientObserver.ExecuteScriptMetrics.Should().ContainSingle().Subject; +// ThenClientOperationMetricsShouldBeFailed(executeScriptMetrics, exception); +// +// tentacleClientObserver.RpcCallMetrics.Should().NotBeEmpty(); +// var startScriptMetric = tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == "StartScript").Subject; +// startScriptMetric.Succeeded.Should().BeFalse(); +// } +// +// [Test] +// [TentacleConfigurations(testCommonVersions: true)] +// public async Task UploadFileShouldGatherMetrics_WhenSucceeds(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// // Arrange +// var tentacleClientObserver = new TestTentacleClientObserver(); +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleClientObserver(tentacleClientObserver) +// .Build(CancellationToken); +// +// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); +// +// // Act +// await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); +// +// // Assert +// var uploadFileMetrics = tentacleClientObserver.UploadFileMetrics.Should().ContainSingle().Subject; +// ThenClientOperationMetricsShouldBeSuccessful(uploadFileMetrics); +// +// tentacleClientObserver.RpcCallMetrics.Should().HaveCountGreaterThan(0); +// var metric = tentacleClientObserver.RpcCallMetrics.Last(); +// metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.UploadFile)); +// metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); +// metric.Succeeded.Should().BeTrue(); +// } +// +// [Test] +// [TentacleConfigurations(testCommonVersions: true)] +// public async Task UploadFileShouldGatherMetrics_WhenFails(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// // Arrange +// var tentacleClientObserver = new TestTentacleClientObserver(); +// var exception = new HalibutClientException("Error"); +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleClientObserver(tentacleClientObserver) +// .WithRetryDuration(TimeSpan.FromSeconds(1)) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .DecorateFileTransferServiceWith(d => d.BeforeUploadFile(() => throw exception)) +// .Build()) +// .Build(CancellationToken); +// +// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); +// +// // Act +// await AssertionExtensions.Should(() => clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken)).ThrowAsync(); +// +// // Assert +// var uploadFileMetrics = tentacleClientObserver.UploadFileMetrics.Should().ContainSingle().Subject; +// ThenClientOperationMetricsShouldBeFailed(uploadFileMetrics, exception); +// +// tentacleClientObserver.RpcCallMetrics.Should().NotBeEmpty(); +// var metric = tentacleClientObserver.RpcCallMetrics.Last(); +// metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.UploadFile)); +// metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); +// metric.Succeeded.Should().BeFalse(); +// } +// +// [Test] +// [TentacleConfigurations(testCommonVersions: true)] +// public async Task DownloadFileShouldGatherMetrics_WhenSucceeds(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// // Arrange +// var tentacleClientObserver = new TestTentacleClientObserver(); +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleClientObserver(tentacleClientObserver) +// .Build(CancellationToken); +// +// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "DownloadFile.txt"); +// await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); +// +// // Act +// await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); +// +// // Assert +// var downloadFileMetrics = tentacleClientObserver.DownloadFileMetrics.Should().ContainSingle().Subject; +// ThenClientOperationMetricsShouldBeSuccessful(downloadFileMetrics); +// +// tentacleClientObserver.RpcCallMetrics.Should().HaveCountGreaterThan(1); // the first one will be the upload +// var metric = tentacleClientObserver.RpcCallMetrics.Last(); +// metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.DownloadFile)); +// metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); +// metric.Succeeded.Should().BeTrue(); +// } +// +// [Test] +// [TentacleConfigurations(testCommonVersions: true)] +// public async Task DownloadFileShouldGatherMetrics_WhenFails(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// // Arrange +// var tentacleClientObserver = new TestTentacleClientObserver(); +// var exception = new HalibutClientException("Error"); +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleClientObserver(tentacleClientObserver) +// .WithRetryDuration(TimeSpan.FromSeconds(1)) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .DecorateFileTransferServiceWith(d => d.BeforeDownloadFile(() => throw exception)) +// .Build()) +// .Build(CancellationToken); +// +// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "DownloadFile.txt"); +// await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); +// +// // Act +// await AssertionExtensions.Should(() => clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken)).ThrowAsync(); +// +// // Assert +// var downloadFileMetrics = tentacleClientObserver.DownloadFileMetrics.Should().ContainSingle().Subject; +// ThenClientOperationMetricsShouldBeFailed(downloadFileMetrics, exception); +// +// tentacleClientObserver.RpcCallMetrics.Should().HaveCountGreaterThan(1); // the first one will be the upload +// var metric = tentacleClientObserver.RpcCallMetrics.Last(); +// metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.DownloadFile)); +// metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); +// metric.Succeeded.Should().BeFalse(); +// } +// +// private static void ThenClientOperationMetricsShouldBeSuccessful(ClientOperationMetrics metric) +// { +// metric.Succeeded.Should().BeTrue(); +// metric.Exception.Should().BeNull(); +// metric.WasCancelled.Should().BeFalse(); +// +// metric.End.Should().BeOnOrAfter(metric.Start); +// metric.Duration.Should().Be(metric.End - metric.Start); +// } +// +// private static void ThenClientOperationMetricsShouldBeFailed(ClientOperationMetrics metric, Exception expectedException) +// { +// metric.Succeeded.Should().BeFalse(); +// metric.Exception.Should().BeEquivalentTo(expectedException); +// metric.WasCancelled.Should().BeFalse(); +// +// metric.End.Should().BeOnOrAfter(metric.Start); +// metric.Duration.Should().Be(metric.End - metric.Start); +// } +// } +// } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionAdditionalScripts.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionAdditionalScripts.cs index 24e6ea8ca..27e1c6d60 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionAdditionalScripts.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionAdditionalScripts.cs @@ -1,50 +1,50 @@ -using System; -using System.IO; -using System.Threading.Tasks; -using FluentAssertions; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class ClientScriptExecutionAdditionalScripts : IntegrationTest - { - [Test] - [TentacleConfigurations(testCommonVersions: true)] - public async Task AdditionalScriptsWork(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - using var tmp = new TemporaryDirectory(); - var path = Path.Combine(tmp.DirectoryPath, "file"); - - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .Build(CancellationToken); - - var scriptBuilder = new ScriptBuilder() - .CreateFile(path) // How files are made are different in bash and powershell, doing this ensures the client and tentacle really are using the correct script. - .Print("Hello"); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .WithAdditionalScriptType(ScriptType.Bash, scriptBuilder.BuildBashScript()) - // Additional Scripts don't actually work on tentacle for anything other than bash. - // Below is what we would have expected to tentacle to work with. - //.WithAdditionalScriptTypes(ScriptType.PowerShell, scriptBuilder.BuildPowershellScript()) - // But instead we need to send the powershell in the scriptbody. - .WithScriptBody(scriptBuilder.BuildPowershellScript()) - .Build(); - - var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(0); - - var allLogs = logs.JoinLogs(); - - allLogs.Should().Contain("Hello"); - } - } -} +// using System; +// using System.IO; +// using System.Threading.Tasks; +// using FluentAssertions; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class ClientScriptExecutionAdditionalScripts : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(testCommonVersions: true)] +// public async Task AdditionalScriptsWork(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// using var tmp = new TemporaryDirectory(); +// var path = Path.Combine(tmp.DirectoryPath, "file"); +// +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .Build(CancellationToken); +// +// var scriptBuilder = new ScriptBuilder() +// .CreateFile(path) // How files are made are different in bash and powershell, doing this ensures the client and tentacle really are using the correct script. +// .Print("Hello"); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .WithAdditionalScriptType(ScriptType.Bash, scriptBuilder.BuildBashScript()) +// // Additional Scripts don't actually work on tentacle for anything other than bash. +// // Below is what we would have expected to tentacle to work with. +// //.WithAdditionalScriptTypes(ScriptType.PowerShell, scriptBuilder.BuildPowershellScript()) +// // But instead we need to send the powershell in the scriptbody. +// .WithScriptBody(scriptBuilder.BuildPowershellScript()) +// .Build(); +// +// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// +// finalResponse.State.Should().Be(ProcessState.Complete); +// finalResponse.ExitCode.Should().Be(0); +// +// var allLogs = logs.JoinLogs(); +// +// allLogs.Should().Contain("Hello"); +// } +// } +// } diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreDisabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreDisabled.cs index 91aedbf7e..472a9893a 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreDisabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreDisabled.cs @@ -1,434 +1,434 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using Halibut; -using NUnit.Framework; -using Octopus.Tentacle.Client.Scripts; -using Octopus.Tentacle.Client.Scripts.Models; -using Octopus.Tentacle.CommonTestUtils.Builders; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Contracts.Capabilities; -using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators.Proxies; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Util.PendingRequestQueueHelpers; -using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; -using Octopus.Tentacle.Util; -using Octopus.TestPortForwarder; - -namespace Octopus.Tentacle.Tests.Integration -{ - /// - /// These tests make sure that we can cancel the ExecuteScript operation when using Tentacle Client with RPC retries disabled. - /// - [IntegrationTestTimeout] - public class ClientScriptExecutionCanBeCancelledWhenRetriesAreDisabled : IntegrationTest - { - [Test] - [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] - public async Task DuringGetCapabilities_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) - { - // ARRANGE - var rpcCallHasStarted = new Reference(false); - var hasPausedOrStoppedPortForwarder = false; - var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue - .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) - .WithRetriesDisabled() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithPortForwarder(out var portForwarder) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var capabilitiesMethodUsages) - .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) - .DecorateCapabilitiesServiceV2With(d => d - .BeforeGetCapabilities( - async () => - { - if (!hasPausedOrStoppedPortForwarder) - { - hasPausedOrStoppedPortForwarder = true; - await tcpConnectionUtilities.RestartTcpConnection(); - - PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); - if (rpcCallStage == RpcCallStage.Connecting) - { - await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); - } - } - - ensureCancellationOccursDuringAnRpcCall.Release(); - }) - .AfterGetCapabilities( - async _ => - { - await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); - })) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Print("Should not run this script") - .Sleep(TimeSpan.FromHours(1))) - .Build(); - - // ACT - var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); - - // ASSERT - // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring - capabilitiesMethodUsages.ForGetCapabilitiesAsync().LastException.Should().BeRequestCancelledException(rpcCallStage); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - actualException.ShouldMatchExceptionContract(expectedException); - - cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(15)); - - capabilitiesMethodUsages.ForGetCapabilitiesAsync().Started.Should().Be(1); - scriptMethodUsages.ForStartScriptAsync().Started.Should().Be(0, "Should not have proceeded past GetCapabilities"); - scriptMethodUsages.ForCancelScriptAsync().Started.Should().Be(0, "Should not have tried to call CancelScript"); - } - - [Test] - [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] - public async Task DuringStartScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) - { - // ARRANGE - var rpcCallHasStarted = new Reference(false); - var restartedPortForwarderForCancel = false; - var hasPausedOrStoppedPortForwarder = false; - var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue - .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) - .WithRetriesDisabled() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithPortForwarder(out var portForwarder) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeStartScript( - async () => - { - if (!hasPausedOrStoppedPortForwarder) - { - hasPausedOrStoppedPortForwarder = true; - await tcpConnectionUtilities.RestartTcpConnection(); - PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); - if (rpcCallStage == RpcCallStage.Connecting) - { - await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); - } - } - - ensureCancellationOccursDuringAnRpcCall.Release(); - }) - .BeforeCancelScript( - async () => - { - await Task.CompletedTask; - - if (!restartedPortForwarderForCancel) - { - restartedPortForwarderForCancel = true; - UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); - } - })) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Print("The script") - .Sleep(TimeSpan.FromHours(1))) - .Build(); - - // ACT - var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); - - // ASSERT - // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring - recordedUsages.ForStartScriptAsync().LastException.Should().BeRequestCancelledException(rpcCallStage); - - if (rpcCallStage == RpcCallStage.Connecting) - { - // Should have cancelled the RPC call and exited immediately - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - - actualException.ShouldMatchExceptionContract(expectedException); - - // We should have cancelled the RPC call quickly and existed - cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(15)); - - recordedUsages.ForStartScriptAsync().Started.Should().Be(1); - recordedUsages.ForGetStatusAsync().Started.Should().Be(0, "Test should not have not proceeded past StartScript before being Cancelled"); - recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); - recordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); - } - else // Transferring - { - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - actualException.ShouldMatchExceptionContract(expectedException); - - // Assert the CancelScript and CompleteScript flow happened fairly quickly - cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); - - recordedUsages.ForStartScriptAsync().Started.Should().Be(1); - recordedUsages.ForGetStatusAsync().Started.Should().Be(0, "Test should not have not proceeded past StartScript before being Cancelled"); - recordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(1); - recordedUsages.ForCompleteScriptAsync().Started.Should().Be(1); - } - } - - [Test] - [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] - public async Task DuringGetStatus_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) - { - // ARRANGE - var rpcCallHasStarted = new Reference(false); - var restartedPortForwarderForCancel = false; - var hasPausedOrStoppedPortForwarder = false; - var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue - .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) - .WithRetriesDisabled() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithPortForwarder(out var portForwarder) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeGetStatus( - async () => - { - if (!hasPausedOrStoppedPortForwarder) - { - hasPausedOrStoppedPortForwarder = true; - await tcpConnectionUtilities.RestartTcpConnection(); - PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); - if (rpcCallStage == RpcCallStage.Connecting) - { - await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); - } - } - - ensureCancellationOccursDuringAnRpcCall.Release(); - }) - .AfterGetStatus( - async () => - { - await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); - }) - .BeforeCancelScript( - async () => - { - await Task.CompletedTask; - - if (!restartedPortForwarderForCancel) - { - restartedPortForwarderForCancel = true; - UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); - } - })) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Print("The script") - .Sleep(TimeSpan.FromHours(1))) - .Build(); - - // ACT - var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); - - // ASSERT - // Assert that script execution was cancelled - actualException.Should().BeScriptExecutionCancelledException(); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - actualException.ShouldMatchExceptionContract(expectedException); - - // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring - recordedUsages.ForGetStatusAsync().LastException.Should().BeRequestCancelledException(rpcCallStage); - - // Assert the CancelScript and CompleteScript flow happened fairly quickly - cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); - - // The expected RPC calls were made - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().BeGreaterOrEqualTo(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); - } - - [Test] - [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] - public async Task DuringCompleteScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) - { - // ARRANGE - var rpcCallHasStarted = new Reference(false); - var hasPausedOrStoppedPortForwarder = false; - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue - .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) - .WithRetriesDisabled() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithPortForwarder(out var portForwarder) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeCompleteScript( - async () => - { - if (!hasPausedOrStoppedPortForwarder) - { - hasPausedOrStoppedPortForwarder = true; - await tcpConnectionUtilities.RestartTcpConnection(); - PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); - if (rpcCallStage == RpcCallStage.Connecting) - { - await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); - } - } - })) - .Build()) - .Build(CancellationToken); - - clientAndTentacle.TentacleClient.OnCancellationAbandonCompleteScriptAfter = TimeSpan.FromSeconds(20); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Print("The script") - .Sleep(TimeSpan.FromSeconds(5))) - .Build(); - - // ACT - var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, new SemaphoreSlim(int.MaxValue, int.MaxValue)); - - // ASSERT - // The actual exception may be null if the script completed before the cancellation was observed - if (actualException != null) - { - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - actualException.ShouldMatchExceptionContract(expectedException); - } - - // Halibut Errors were recorded on CompleteScript - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).LastException?.Should().Match(x => x is HalibutClientException || x is OperationCanceledException || x is TaskCanceledException); // Complete Script was cancelled quickly - cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); - - // The expected RPC calls were made - recordedUsages.ForStartScriptAsync().Started.Should().Be(1); - recordedUsages.ForGetStatusAsync().Started.Should().BeGreaterThanOrEqualTo(1); - recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); - recordedUsages.ForCompleteScriptAsync().Started.Should().BeGreaterOrEqualTo(1); - } - - void PauseOrStopPortForwarder(RpcCallStage rpcCallStage, PortForwarder portForwarder, IResponseMessageTcpKiller responseMessageTcpKiller, Reference rpcCallHasStarted) - { - if (rpcCallStage == RpcCallStage.Connecting) - { - Logger.Information("Killing the port forwarder so the next RPCs are in the connecting state when being cancelled"); - //portForwarder.Stop(); - portForwarder.EnterKillNewAndExistingConnectionsMode(); - rpcCallHasStarted.Value = true; - } - else - { - Logger.Information("Will Pause the port forwarder on next response so the next RPC is in-flight when being cancelled"); - responseMessageTcpKiller.PauseConnectionOnNextResponse(() => rpcCallHasStarted.Value = true); - } - } - - void UnPauseOrRestartPortForwarder(TentacleType tentacleType, RpcCallStage rpcCallStage, Reference portForwarder) - { - if (rpcCallStage == RpcCallStage.Connecting) - { - Logger.Information("Starting the PortForwarder as we stopped it to get the StartScript RPC call in the Connecting state"); - //portForwarder.Value.Start(); - portForwarder.Value.ReturnToNormalMode(); - } - else if (tentacleType == TentacleType.Polling) - { - Logger.Information("UnPausing the PortForwarder as we paused the connections which means Polling will be stalled"); - portForwarder.Value.UnPauseExistingConnections(); - portForwarder.Value.CloseExistingConnections(); - } - } - - async Task<(ScriptExecutionResult response, Exception? actualException, TimeSpan cancellationDuration)> ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted( - ClientAndTentacle clientAndTentacle, - ExecuteScriptCommand executeScriptCommand, - Reference rpcCallHasStarted, - SemaphoreSlim whenTheRequestCanBeCancelled) - { - Logger.Information("Start of ExecuteScriptThenCancel"); - var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); - - var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( - executeScriptCommand, - cancelExecutionCancellationTokenSource.Token); - - Logger.Information("Create action"); - Func)>> action = async () => await executeScriptTask; - - Logger.Information("Waiting for the RPC Call to start"); - await Wait.For( - () => rpcCallHasStarted.Value, - TimeSpan.FromSeconds(30), - () => throw new Exception("RPC call did not start") - ,CancellationToken); - Logger.Information("RPC Call has start"); - - await Task.Delay(TimeSpan.FromSeconds(6), CancellationToken); - - var cancellationDuration = new Stopwatch(); - await whenTheRequestCanBeCancelled.WithLockAsync(() => - { - Logger.Information("Cancelling ExecuteScript"); - cancelExecutionCancellationTokenSource.Cancel(); - cancellationDuration.Start(); - }, CancellationToken); - - Exception? actualException = null; - (ScriptExecutionResult Response, List Logs)? responseAndLogs = null; - try - { - responseAndLogs = await action(); - } - catch (Exception ex) - { - actualException = ex; - } - - cancellationDuration.Stop(); - return (responseAndLogs?.Response, actualException, cancellationDuration.Elapsed); - } - } -} \ No newline at end of file +// using System; +// using System.Collections.Generic; +// using System.Diagnostics; +// using System.Threading; +// using System.Threading.Tasks; +// using FluentAssertions; +// using Halibut; +// using NUnit.Framework; +// using Octopus.Tentacle.Client.Scripts; +// using Octopus.Tentacle.Client.Scripts.Models; +// using Octopus.Tentacle.CommonTestUtils.Builders; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Contracts.Capabilities; +// using Octopus.Tentacle.Contracts.ClientServices; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators.Proxies; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Util.PendingRequestQueueHelpers; +// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; +// using Octopus.Tentacle.Util; +// using Octopus.TestPortForwarder; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// /// +// /// These tests make sure that we can cancel the ExecuteScript operation when using Tentacle Client with RPC retries disabled. +// /// +// [IntegrationTestTimeout] +// public class ClientScriptExecutionCanBeCancelledWhenRetriesAreDisabled : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] +// public async Task DuringGetCapabilities_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) +// { +// // ARRANGE +// var rpcCallHasStarted = new Reference(false); +// var hasPausedOrStoppedPortForwarder = false; +// var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue +// .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) +// .WithRetriesDisabled() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithPortForwarder(out var portForwarder) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var capabilitiesMethodUsages) +// .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) +// .DecorateCapabilitiesServiceV2With(d => d +// .BeforeGetCapabilities( +// async () => +// { +// if (!hasPausedOrStoppedPortForwarder) +// { +// hasPausedOrStoppedPortForwarder = true; +// await tcpConnectionUtilities.RestartTcpConnection(); +// +// PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); +// } +// } +// +// ensureCancellationOccursDuringAnRpcCall.Release(); +// }) +// .AfterGetCapabilities( +// async _ => +// { +// await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); +// })) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Print("Should not run this script") +// .Sleep(TimeSpan.FromHours(1))) +// .Build(); +// +// // ACT +// var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); +// +// // ASSERT +// // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring +// capabilitiesMethodUsages.ForGetCapabilitiesAsync().LastException.Should().BeRequestCancelledException(rpcCallStage); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// actualException.ShouldMatchExceptionContract(expectedException); +// +// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(15)); +// +// capabilitiesMethodUsages.ForGetCapabilitiesAsync().Started.Should().Be(1); +// scriptMethodUsages.ForStartScriptAsync().Started.Should().Be(0, "Should not have proceeded past GetCapabilities"); +// scriptMethodUsages.ForCancelScriptAsync().Started.Should().Be(0, "Should not have tried to call CancelScript"); +// } +// +// [Test] +// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] +// public async Task DuringStartScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) +// { +// // ARRANGE +// var rpcCallHasStarted = new Reference(false); +// var restartedPortForwarderForCancel = false; +// var hasPausedOrStoppedPortForwarder = false; +// var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue +// .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) +// .WithRetriesDisabled() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithPortForwarder(out var portForwarder) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeStartScript( +// async () => +// { +// if (!hasPausedOrStoppedPortForwarder) +// { +// hasPausedOrStoppedPortForwarder = true; +// await tcpConnectionUtilities.RestartTcpConnection(); +// PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); +// } +// } +// +// ensureCancellationOccursDuringAnRpcCall.Release(); +// }) +// .BeforeCancelScript( +// async () => +// { +// await Task.CompletedTask; +// +// if (!restartedPortForwarderForCancel) +// { +// restartedPortForwarderForCancel = true; +// UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Print("The script") +// .Sleep(TimeSpan.FromHours(1))) +// .Build(); +// +// // ACT +// var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); +// +// // ASSERT +// // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring +// recordedUsages.ForStartScriptAsync().LastException.Should().BeRequestCancelledException(rpcCallStage); +// +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// // Should have cancelled the RPC call and exited immediately +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// +// actualException.ShouldMatchExceptionContract(expectedException); +// +// // We should have cancelled the RPC call quickly and existed +// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(15)); +// +// recordedUsages.ForStartScriptAsync().Started.Should().Be(1); +// recordedUsages.ForGetStatusAsync().Started.Should().Be(0, "Test should not have not proceeded past StartScript before being Cancelled"); +// recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); +// recordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); +// } +// else // Transferring +// { +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// actualException.ShouldMatchExceptionContract(expectedException); +// +// // Assert the CancelScript and CompleteScript flow happened fairly quickly +// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); +// +// recordedUsages.ForStartScriptAsync().Started.Should().Be(1); +// recordedUsages.ForGetStatusAsync().Started.Should().Be(0, "Test should not have not proceeded past StartScript before being Cancelled"); +// recordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(1); +// recordedUsages.ForCompleteScriptAsync().Started.Should().Be(1); +// } +// } +// +// [Test] +// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] +// public async Task DuringGetStatus_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) +// { +// // ARRANGE +// var rpcCallHasStarted = new Reference(false); +// var restartedPortForwarderForCancel = false; +// var hasPausedOrStoppedPortForwarder = false; +// var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue +// .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) +// .WithRetriesDisabled() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithPortForwarder(out var portForwarder) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeGetStatus( +// async () => +// { +// if (!hasPausedOrStoppedPortForwarder) +// { +// hasPausedOrStoppedPortForwarder = true; +// await tcpConnectionUtilities.RestartTcpConnection(); +// PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); +// } +// } +// +// ensureCancellationOccursDuringAnRpcCall.Release(); +// }) +// .AfterGetStatus( +// async () => +// { +// await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); +// }) +// .BeforeCancelScript( +// async () => +// { +// await Task.CompletedTask; +// +// if (!restartedPortForwarderForCancel) +// { +// restartedPortForwarderForCancel = true; +// UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Print("The script") +// .Sleep(TimeSpan.FromHours(1))) +// .Build(); +// +// // ACT +// var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); +// +// // ASSERT +// // Assert that script execution was cancelled +// actualException.Should().BeScriptExecutionCancelledException(); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// actualException.ShouldMatchExceptionContract(expectedException); +// +// // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring +// recordedUsages.ForGetStatusAsync().LastException.Should().BeRequestCancelledException(rpcCallStage); +// +// // Assert the CancelScript and CompleteScript flow happened fairly quickly +// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); +// +// // The expected RPC calls were made +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().BeGreaterOrEqualTo(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); +// } +// +// [Test] +// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] +// public async Task DuringCompleteScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) +// { +// // ARRANGE +// var rpcCallHasStarted = new Reference(false); +// var hasPausedOrStoppedPortForwarder = false; +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue +// .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) +// .WithRetriesDisabled() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithPortForwarder(out var portForwarder) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeCompleteScript( +// async () => +// { +// if (!hasPausedOrStoppedPortForwarder) +// { +// hasPausedOrStoppedPortForwarder = true; +// await tcpConnectionUtilities.RestartTcpConnection(); +// PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); +// } +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// clientAndTentacle.TentacleClient.OnCancellationAbandonCompleteScriptAfter = TimeSpan.FromSeconds(20); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Print("The script") +// .Sleep(TimeSpan.FromSeconds(5))) +// .Build(); +// +// // ACT +// var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, new SemaphoreSlim(int.MaxValue, int.MaxValue)); +// +// // ASSERT +// // The actual exception may be null if the script completed before the cancellation was observed +// if (actualException != null) +// { +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// actualException.ShouldMatchExceptionContract(expectedException); +// } +// +// // Halibut Errors were recorded on CompleteScript +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).LastException?.Should().Match(x => x is HalibutClientException || x is OperationCanceledException || x is TaskCanceledException); // Complete Script was cancelled quickly +// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); +// +// // The expected RPC calls were made +// recordedUsages.ForStartScriptAsync().Started.Should().Be(1); +// recordedUsages.ForGetStatusAsync().Started.Should().BeGreaterThanOrEqualTo(1); +// recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); +// recordedUsages.ForCompleteScriptAsync().Started.Should().BeGreaterOrEqualTo(1); +// } +// +// void PauseOrStopPortForwarder(RpcCallStage rpcCallStage, PortForwarder portForwarder, IResponseMessageTcpKiller responseMessageTcpKiller, Reference rpcCallHasStarted) +// { +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// Logger.Information("Killing the port forwarder so the next RPCs are in the connecting state when being cancelled"); +// //portForwarder.Stop(); +// portForwarder.EnterKillNewAndExistingConnectionsMode(); +// rpcCallHasStarted.Value = true; +// } +// else +// { +// Logger.Information("Will Pause the port forwarder on next response so the next RPC is in-flight when being cancelled"); +// responseMessageTcpKiller.PauseConnectionOnNextResponse(() => rpcCallHasStarted.Value = true); +// } +// } +// +// void UnPauseOrRestartPortForwarder(TentacleType tentacleType, RpcCallStage rpcCallStage, Reference portForwarder) +// { +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// Logger.Information("Starting the PortForwarder as we stopped it to get the StartScript RPC call in the Connecting state"); +// //portForwarder.Value.Start(); +// portForwarder.Value.ReturnToNormalMode(); +// } +// else if (tentacleType == TentacleType.Polling) +// { +// Logger.Information("UnPausing the PortForwarder as we paused the connections which means Polling will be stalled"); +// portForwarder.Value.UnPauseExistingConnections(); +// portForwarder.Value.CloseExistingConnections(); +// } +// } +// +// async Task<(ScriptExecutionResult response, Exception? actualException, TimeSpan cancellationDuration)> ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted( +// ClientAndTentacle clientAndTentacle, +// ExecuteScriptCommand executeScriptCommand, +// Reference rpcCallHasStarted, +// SemaphoreSlim whenTheRequestCanBeCancelled) +// { +// Logger.Information("Start of ExecuteScriptThenCancel"); +// var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); +// +// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( +// executeScriptCommand, +// cancelExecutionCancellationTokenSource.Token); +// +// Logger.Information("Create action"); +// Func)>> action = async () => await executeScriptTask; +// +// Logger.Information("Waiting for the RPC Call to start"); +// await Wait.For( +// () => rpcCallHasStarted.Value, +// TimeSpan.FromSeconds(30), +// () => throw new Exception("RPC call did not start") +// ,CancellationToken); +// Logger.Information("RPC Call has start"); +// +// await Task.Delay(TimeSpan.FromSeconds(6), CancellationToken); +// +// var cancellationDuration = new Stopwatch(); +// await whenTheRequestCanBeCancelled.WithLockAsync(() => +// { +// Logger.Information("Cancelling ExecuteScript"); +// cancelExecutionCancellationTokenSource.Cancel(); +// cancellationDuration.Start(); +// }, CancellationToken); +// +// Exception? actualException = null; +// (ScriptExecutionResult Response, List Logs)? responseAndLogs = null; +// try +// { +// responseAndLogs = await action(); +// } +// catch (Exception ex) +// { +// actualException = ex; +// } +// +// cancellationDuration.Stop(); +// return (responseAndLogs?.Response, actualException, cancellationDuration.Elapsed); +// } +// } +// } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreEnabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreEnabled.cs index 693adbd07..4d855bda2 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreEnabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreEnabled.cs @@ -1,672 +1,672 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using Halibut; -using Halibut.Diagnostics; -using NUnit.Framework; -using Octopus.Tentacle.Client.Scripts; -using Octopus.Tentacle.Client.Scripts.Models; -using Octopus.Tentacle.CommonTestUtils.Builders; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Contracts.Capabilities; -using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators.Proxies; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; -using Octopus.Tentacle.Tests.Integration.Support.PendingRequestQueueFactories; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Util.PendingRequestQueueHelpers; -using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; -using Octopus.Tentacle.Util; -using Octopus.TestPortForwarder; - -namespace Octopus.Tentacle.Tests.Integration -{ - /// - /// These tests make sure that we can cancel the ExecuteScript operation when using Tentacle Client with RPC retries enabled. - /// - [IntegrationTestTimeout] - public class ClientScriptExecutionCanBeCancelledWhenRetriesAreEnabled : IntegrationTest - { - [Test] - [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage), typeof(RpcCall) })] - public async Task DuringGetCapabilities_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage, RpcCall rpcCall) - { - // ARRANGE - var rpcCallHasStarted = new Reference(false); - var hasPausedOrStoppedPortForwarder = false; - var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue - .WithServiceEndpointModifier(point => - { - if (rpcCall == RpcCall.FirstCall) point.TryAndConnectForALongTime(); - }) - .WithRetryDuration(TimeSpan.FromHours(1)) - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithPortForwarder(out var portForwarder) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var recordedUsages) - .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) - .DecorateCapabilitiesServiceV2With(d => d - .BeforeGetCapabilities( - async () => - { - if (rpcCall == RpcCall.RetryingCall && - recordedUsages.ForGetCapabilitiesAsync().LastException == null) - { - await tcpConnectionUtilities.RestartTcpConnection(); - - // Kill the first GetCapabilities call to force the rpc call into retries - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - else if (!hasPausedOrStoppedPortForwarder) - { - hasPausedOrStoppedPortForwarder = true; - await tcpConnectionUtilities.RestartTcpConnection(); - - await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); - if (rpcCallStage == RpcCallStage.Connecting && tentacleConfigurationTestCase.TentacleType == TentacleType.Polling) - { - await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); - } - } - - ensureCancellationOccursDuringAnRpcCall.Release(); - }) - .AfterGetCapabilities( - async _ => - { - await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); - })) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Print("Should not run this script") - .Sleep(TimeSpan.FromHours(1))) - .Build(); - - // ACT - var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); - - // ASSERT - // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring - var latestException = recordedUsages.ForGetCapabilitiesAsync().LastException; - if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) - { - Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + - "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); - } - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - actualException.ShouldMatchExceptionContract(expectedException); - - latestException.Should().BeRequestCancelledException(rpcCallStage); - cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(10), "The RPC call should have been cancelled quickly"); - - if (rpcCall == RpcCall.FirstCall) - { - recordedUsages.ForGetCapabilitiesAsync().Started.Should().Be(1); - } - else - { - recordedUsages.ForGetCapabilitiesAsync().Started.Should().BeGreaterOrEqualTo(2); - } - scriptMethodUsages.ForStartScriptAsync().Started.Should().Be(0, "Should not have proceeded past GetCapabilities"); - scriptMethodUsages.ForCancelScriptAsync().Started.Should().Be(0, "Should not have tried to call CancelScript"); - scriptMethodUsages.ForCompleteScriptAsync().Started.Should().Be(0, "Should not have tried to call CompleteScript"); - } - - /// - /// This test, and probably others in this test class do not correctly test the Connecting Scenario. The port forwarder is used to kill new and existing connections but this can - /// result in Halibut thinking a Request is transferring, where we want it to be connecting. These tests need to be rewritten to ensure they test the correct scenario. - /// - [Test] - [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCall), typeof(RpcCallStage) })] - public async Task DuringStartScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCall rpcCall, RpcCallStage rpcCallStage) - { - var rpcCallHasStarted = new Reference(false); - var restartedPortForwarderForCancel = false; - var hasPausedOrStoppedPortForwarder = false; - var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue - .WithServiceEndpointModifier(point => - { - if (rpcCall == RpcCall.FirstCall) point.TryAndConnectForALongTime(); - }) - .WithRetryDuration(TimeSpan.FromHours(1)) - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithPortForwarder(out var portForwarder) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeStartScript( - async () => - { - if (rpcCall == RpcCall.RetryingCall && recordedUsages.ForStartScriptAsync().LastException is null) - { - await tcpConnectionUtilities.RestartTcpConnection(); - // Kill the first StartScript call to force the rpc call into retries - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - else - { - if (!hasPausedOrStoppedPortForwarder) - { - hasPausedOrStoppedPortForwarder = true; - await tcpConnectionUtilities.RestartTcpConnection(); - await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); - if (rpcCallStage == RpcCallStage.Connecting) - { - await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); - } - } - } - - ensureCancellationOccursDuringAnRpcCall.Release(); - }) - .AfterStartScript( - async () => - { - await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); - }) - .BeforeCancelScript( - async () => - { - await Task.CompletedTask; - - if (!restartedPortForwarderForCancel) - { - restartedPortForwarderForCancel = true; - UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); - } - })) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Print("The script") - .Sleep(TimeSpan.FromHours(1))) - .Build(); - - // ACT - var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); - - // ASSERT - // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring - var latestException = recordedUsages.ForStartScriptAsync().LastException; - if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) - { - Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + - "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); - } - latestException.Should().BeRequestCancelledException(rpcCallStage); - - if (rpcCall == RpcCall.FirstCall && rpcCallStage == RpcCallStage.Connecting) - { - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - actualException.ShouldMatchExceptionContract(expectedException); - - // We should have cancelled the RPC call quickly - cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(15)); - - recordedUsages.ForStartScriptAsync().Started.Should().Be(1); - recordedUsages.ForGetStatusAsync().Started.Should().Be(0); - recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); - recordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); - } - else if((rpcCall == RpcCall.RetryingCall && rpcCallStage == RpcCallStage.Connecting) || rpcCallStage == RpcCallStage.InFlight) - { - // Assert that script execution was cancelled - actualException.Should().BeScriptExecutionCancelledException(); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - actualException.ShouldMatchExceptionContract(expectedException); - - - // Assert the CancelScript and CompleteScript flow happened fairly quickly - cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); - - if (rpcCall == RpcCall.FirstCall) - { - recordedUsages.ForStartScriptAsync().Started.Should().Be(1); - } - else - { - recordedUsages.ForStartScriptAsync().Started.Should().BeGreaterOrEqualTo(2); - } - recordedUsages.ForGetStatusAsync().Started.Should().Be(0); - recordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(1); - recordedUsages.ForCompleteScriptAsync().Started.Should().Be(1); - } - else - { - throw new ArgumentOutOfRangeException(); - } - } - - [Test] - [TentacleConfigurations(testListening: false)] - public async Task DuringStartScript_ForPollingTentacle_ThatIsRetryingTheRpc_AndConnecting_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var halibutTimeoutsAndLimits = HalibutTimeoutsAndLimits.RecommendedValues(); - halibutTimeoutsAndLimits.PollingQueueWaitTimeout = TimeSpan.FromSeconds(4); - halibutTimeoutsAndLimits.PollingRequestQueueTimeout = TimeSpan.FromSeconds(3); - - var started = false; - var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithRetryDuration(TimeSpan.FromHours(1)) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder().RecordMethodUsages(tentacleConfigurationTestCase, out var scriptServiceRecordedUsages).Build()) - .WithPendingRequestQueueFactory(new CancelWhenRequestQueuedPendingRequestQueueFactory( - cancelExecutionCancellationTokenSource, - halibutTimeoutsAndLimits, - // Cancel the execution when the StartScript RPC call is being retries - shouldCancel: async () => - { - await Task.CompletedTask; - - if (started && scriptServiceRecordedUsages.ForStartScriptAsync().Started >= 2) - { - Logger.Information("Cancelling Execute Script"); - return true; - } - - return false; - })) - .WithHalibutTimeoutsAndLimits(halibutTimeoutsAndLimits) - .Build(CancellationToken); - - // Arrange - Logger.Information("Execute a script so that GetCapabilities will be cached"); - await clientAndTentacle.TentacleClient.ExecuteScript( - new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("The script")).Build(), - cancelExecutionCancellationTokenSource.Token); - - Logger.Information("Stop Tentacle so no more requests are picked up"); - await clientAndTentacle.RunningTentacle.Stop(CancellationToken); - - var delay = clientAndTentacle.Server.ServerHalibutRuntime.TimeoutsAndLimits.PollingQueueWaitTimeout + TimeSpan.FromSeconds(2); - Logger.Information($"Waiting for {delay} for any active PendingRequestQueues for the Tentacle to drop the latest poll connection"); - await Task.Delay(delay, CancellationToken); - - scriptServiceRecordedUsages.Reset(); - started = true; - - // ACT - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b.Print("The script").Sleep(TimeSpan.FromHours(1))) - .Build(); - - Logger.Information("Start Executing the Script"); - var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, cancelExecutionCancellationTokenSource.Token); - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - // Assert - scriptServiceRecordedUsages.ForStartScriptAsync().Completed.Should().BeGreaterOrEqualTo(2); - scriptServiceRecordedUsages.ForGetStatusAsync().Started.Should().Be(0); - scriptServiceRecordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(0, "Script Execution does not need to be cancelled on Tentacle as it has not started"); - scriptServiceRecordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); - } - - [Test] - [TentacleConfigurations(testPolling: false)] - public async Task DuringStartScript_ForListeningTentacle_ThatIsRetryingTheRpc_AndConnecting_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var halibutTimeoutsAndLimits = HalibutTimeoutsAndLimits.RecommendedValues(); - halibutTimeoutsAndLimits.RetryCountLimit = 1; - halibutTimeoutsAndLimits.ConnectionErrorRetryTimeout = TimeSpan.FromSeconds(4); - halibutTimeoutsAndLimits.RetryListeningSleepInterval = TimeSpan.Zero; - halibutTimeoutsAndLimits.TcpClientConnectTimeout = TimeSpan.FromSeconds(4); - halibutTimeoutsAndLimits.TcpClientPooledConnectionTimeout = TimeSpan.FromSeconds(5); - - var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithRetryDuration(TimeSpan.FromHours(1)) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder().RecordMethodUsages(tentacleConfigurationTestCase, out var scriptServiceRecordedUsages).Build()) - .WithHalibutTimeoutsAndLimits(halibutTimeoutsAndLimits) - .WithPortForwarder(out var portForwarder) - .Build(CancellationToken); - - // Arrange - Logger.Information("Execute a script so that GetCapabilities will be cached"); - await clientAndTentacle.TentacleClient.ExecuteScript( - new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("The script")).Build(), - cancelExecutionCancellationTokenSource.Token); - - Logger.Information("Stop Tentacle so no more requests are picked up"); - await clientAndTentacle.RunningTentacle.Stop(CancellationToken); - portForwarder.Value.KillNewConnectionsImmediatlyMode = true; - portForwarder.Value.CloseExistingConnections(); - - var delay = clientAndTentacle.Server.ServerHalibutRuntime.TimeoutsAndLimits.SafeTcpClientPooledConnectionTimeout + TimeSpan.FromSeconds(2); - Logger.Information($"Waiting for {delay} for any active Pooled Connections for the Tentacle to expire"); - await Task.Delay(delay, CancellationToken); - - scriptServiceRecordedUsages.Reset(); - - // ACT - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b.Print("The script").Sleep(TimeSpan.FromHours(1))) - .Build(); - - Logger.Information("Start Executing the Script"); - var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, cancelExecutionCancellationTokenSource.Token); - var cancellationTask = Task.Run(async () => - { - while (true) - { - if (scriptServiceRecordedUsages.ForStartScriptAsync().Started >= 2) - { - cancelExecutionCancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(2)); - Logger.Information("Cancelling cancellation token source after 2 seconds."); - return; - } - - await Task.Delay(TimeSpan.FromSeconds(0.5), CancellationToken); - } - }, CancellationToken); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - await cancellationTask; - - // Assert - scriptServiceRecordedUsages.ForStartScriptAsync().Completed.Should().BeGreaterOrEqualTo(2); - scriptServiceRecordedUsages.ForGetStatusAsync().Started.Should().Be(0); - scriptServiceRecordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(0, "Script Execution does not need to be cancelled on Tentacle as it has not started"); - scriptServiceRecordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); - } - - [Test] - [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCall), typeof(RpcCallStage) })] - public async Task DuringGetStatus_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCall rpcCall, RpcCallStage rpcCallStage) - { - // ARRANGE - var rpcCallHasStarted = new Reference(false); - var restartedPortForwarderForCancel = false; - var hasPausedOrStoppedPortForwarder = false; - var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue - .WithServiceEndpointModifier(point => - { - if (rpcCall == RpcCall.FirstCall) point.TryAndConnectForALongTime(); - }) - .WithRetryDuration(TimeSpan.FromHours(1)) - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithPortForwarder(out var portForwarder) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeGetStatus( - async () => - { - if (rpcCall == RpcCall.RetryingCall && - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException is null) - { - await tcpConnectionUtilities.RestartTcpConnection(); - // Kill the first StartScript call to force the rpc call into retries - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - else - { - if (!hasPausedOrStoppedPortForwarder) - { - hasPausedOrStoppedPortForwarder = true; - await tcpConnectionUtilities.RestartTcpConnection(); - await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); - if (rpcCallStage == RpcCallStage.Connecting) - { - await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); - } - } - } - - ensureCancellationOccursDuringAnRpcCall.Release(); - }) - .AfterGetStatus( - async () => - { - await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); - }) - .BeforeCancelScript( - async () => - { - await Task.CompletedTask; - - if (!restartedPortForwarderForCancel) - { - restartedPortForwarderForCancel = true; - UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); - } - })) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Print("The script") - .Sleep(TimeSpan.FromHours(1))) - .Build(); - - // ACT - var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); - - // ASSERT - // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring - var latestException = recordedUsages.ForGetStatusAsync().LastException; - if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) - { - Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + - "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); - } - - latestException.Should().BeRequestCancelledException(rpcCallStage); - - // Assert that script execution was cancelled - actualException.Should().BeScriptExecutionCancelledException(); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - actualException.ShouldMatchExceptionContract(expectedException); - - - // Script Execution should cancel quickly - cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); - - recordedUsages.ForStartScriptAsync().Started.Should().Be(1); - if (rpcCall == RpcCall.FirstCall) - { - recordedUsages.ForGetStatusAsync().Started.Should().Be(1); - } - else - { - recordedUsages.ForGetStatusAsync().Started.Should().BeGreaterOrEqualTo(2); - } - recordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(1); - recordedUsages.ForCompleteScriptAsync().Started.Should().Be(1); - } - - [Test] - [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] - public async Task DuringCompleteScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) - { - // ARRANGE - var rpcCallHasStarted = new Reference(false); - var hasPausedOrStoppedPortForwarder = false; - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue - .WithRetryDuration(TimeSpan.FromHours(1)) - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithPortForwarder(out var portForwarder) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeCompleteScript( - async () => - { - if (!hasPausedOrStoppedPortForwarder) - { - hasPausedOrStoppedPortForwarder = true; - await tcpConnectionUtilities.RestartTcpConnection(); - await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); - if (rpcCallStage == RpcCallStage.Connecting) - { - await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); - } - } - })) - .Build()) - .Build(CancellationToken); - - clientAndTentacle.TentacleClient.OnCancellationAbandonCompleteScriptAfter = TimeSpan.FromSeconds(20); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Print("The script") - .Sleep(TimeSpan.FromSeconds(5))) - .Build(); - - // ACT - var (responseAndLogs, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, new SemaphoreSlim(int.MaxValue, int.MaxValue)); - - // ASSERT - - // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring - var latestException = recordedUsages.ForCompleteScriptAsync().LastException; - if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) - { - Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + - "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); - } - - // The actual exception may be null if the script completed before the cancellation was observed - if (actualException != null) - { - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - actualException.ShouldMatchExceptionContract(expectedException); - } - - latestException.Should().BeRequestCancelledException(rpcCallStage); - - // Complete Script was cancelled quickly - cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); - - recordedUsages.ForStartScriptAsync().Started.Should().Be(1); - recordedUsages.ForGetStatusAsync().Started.Should().BeGreaterThanOrEqualTo(1); - recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); - recordedUsages.ForCompleteScriptAsync().Started.Should().BeGreaterOrEqualTo(1); - } - - async Task PauseOrStopPortForwarder(RpcCallStage rpcCallStage, PortForwarder portForwarder, IResponseMessageTcpKiller responseMessageTcpKiller, Reference rpcCallHasStarted) - { - if (rpcCallStage == RpcCallStage.Connecting) - { - Logger.Information("Killing the port forwarder so the next RPCs are in the connecting state when being cancelled"); - portForwarder.EnterKillNewAndExistingConnectionsMode(); - - await SetRpcCallWeAreInterestedInAsStarted(rpcCallHasStarted); - } - else - { - Logger.Information("Will Pause the port forwarder on next response so the next RPC is in-flight when being cancelled"); -#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed - responseMessageTcpKiller.PauseConnectionOnNextResponse(() => SetRpcCallWeAreInterestedInAsStarted(rpcCallHasStarted)); -#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed - } - - static async Task SetRpcCallWeAreInterestedInAsStarted(Reference rpcCallHasStarted) - { - // Allow the port forwarder some time to stop - await Task.Delay(TimeSpan.FromSeconds(5)); - rpcCallHasStarted.Value = true; - } - } - - void UnPauseOrRestartPortForwarder(TentacleType tentacleType, RpcCallStage rpcCallStage, Reference portForwarder) - { - if (rpcCallStage == RpcCallStage.Connecting) - { - Logger.Information("Starting the PortForwarder as we stopped it to get the StartScript RPC call in the Connecting state"); - portForwarder.Value.ReturnToNormalMode(); - } - else if (tentacleType == TentacleType.Polling) - { - Logger.Information("UnPausing the PortForwarder as we paused the connections which means Polling will be stalled"); - portForwarder.Value.UnPauseExistingConnections(); - portForwarder.Value.CloseExistingConnections(); - } - } - - async Task<(ScriptExecutionResult response, Exception? actualException, TimeSpan cancellationDuration)> ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted( - ClientAndTentacle clientAndTentacle, - ExecuteScriptCommand executeScriptCommand, - Reference rpcCallHasStarted, - SemaphoreSlim whenTheRequestCanBeCancelled) - { - var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); - - var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( - executeScriptCommand, - cancelExecutionCancellationTokenSource.Token); - - Logger.Information("Waiting for the RPC Call to start"); - await Wait.For(() => rpcCallHasStarted.Value, - TimeSpan.FromSeconds(30), - () => throw new Exception("RPC call did not start"), - CancellationToken); - Logger.Information("RPC Call has start"); - - var cancellationDuration = new Stopwatch(); - await whenTheRequestCanBeCancelled.WithLockAsync(async () => - { - await Task.Delay(TimeSpan.FromSeconds(6), CancellationToken); - Logger.Information("Cancelling ExecuteScript"); - cancelExecutionCancellationTokenSource.Cancel(); - cancellationDuration.Start(); - }, CancellationToken); - - Exception? actualException = null; - (ScriptExecutionResult Response, List Logs)? responseAndLogs = null; - try - { - responseAndLogs = await executeScriptTask; - } - catch (Exception ex) - { - actualException = ex; - } - - cancellationDuration.Stop(); - return (responseAndLogs?.Response, actualException, cancellationDuration.Elapsed); - } - } -} \ No newline at end of file +// using System; +// using System.Collections.Generic; +// using System.Diagnostics; +// using System.Threading; +// using System.Threading.Tasks; +// using FluentAssertions; +// using Halibut; +// using Halibut.Diagnostics; +// using NUnit.Framework; +// using Octopus.Tentacle.Client.Scripts; +// using Octopus.Tentacle.Client.Scripts.Models; +// using Octopus.Tentacle.CommonTestUtils.Builders; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Contracts.Capabilities; +// using Octopus.Tentacle.Contracts.ClientServices; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators.Proxies; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +// using Octopus.Tentacle.Tests.Integration.Support.PendingRequestQueueFactories; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Util.PendingRequestQueueHelpers; +// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; +// using Octopus.Tentacle.Util; +// using Octopus.TestPortForwarder; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// /// +// /// These tests make sure that we can cancel the ExecuteScript operation when using Tentacle Client with RPC retries enabled. +// /// +// [IntegrationTestTimeout] +// public class ClientScriptExecutionCanBeCancelledWhenRetriesAreEnabled : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage), typeof(RpcCall) })] +// public async Task DuringGetCapabilities_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage, RpcCall rpcCall) +// { +// // ARRANGE +// var rpcCallHasStarted = new Reference(false); +// var hasPausedOrStoppedPortForwarder = false; +// var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue +// .WithServiceEndpointModifier(point => +// { +// if (rpcCall == RpcCall.FirstCall) point.TryAndConnectForALongTime(); +// }) +// .WithRetryDuration(TimeSpan.FromHours(1)) +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithPortForwarder(out var portForwarder) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var recordedUsages) +// .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) +// .DecorateCapabilitiesServiceV2With(d => d +// .BeforeGetCapabilities( +// async () => +// { +// if (rpcCall == RpcCall.RetryingCall && +// recordedUsages.ForGetCapabilitiesAsync().LastException == null) +// { +// await tcpConnectionUtilities.RestartTcpConnection(); +// +// // Kill the first GetCapabilities call to force the rpc call into retries +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// else if (!hasPausedOrStoppedPortForwarder) +// { +// hasPausedOrStoppedPortForwarder = true; +// await tcpConnectionUtilities.RestartTcpConnection(); +// +// await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); +// if (rpcCallStage == RpcCallStage.Connecting && tentacleConfigurationTestCase.TentacleType == TentacleType.Polling) +// { +// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); +// } +// } +// +// ensureCancellationOccursDuringAnRpcCall.Release(); +// }) +// .AfterGetCapabilities( +// async _ => +// { +// await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); +// })) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Print("Should not run this script") +// .Sleep(TimeSpan.FromHours(1))) +// .Build(); +// +// // ACT +// var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); +// +// // ASSERT +// // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring +// var latestException = recordedUsages.ForGetCapabilitiesAsync().LastException; +// if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) +// { +// Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + +// "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); +// } +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// actualException.ShouldMatchExceptionContract(expectedException); +// +// latestException.Should().BeRequestCancelledException(rpcCallStage); +// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(10), "The RPC call should have been cancelled quickly"); +// +// if (rpcCall == RpcCall.FirstCall) +// { +// recordedUsages.ForGetCapabilitiesAsync().Started.Should().Be(1); +// } +// else +// { +// recordedUsages.ForGetCapabilitiesAsync().Started.Should().BeGreaterOrEqualTo(2); +// } +// scriptMethodUsages.ForStartScriptAsync().Started.Should().Be(0, "Should not have proceeded past GetCapabilities"); +// scriptMethodUsages.ForCancelScriptAsync().Started.Should().Be(0, "Should not have tried to call CancelScript"); +// scriptMethodUsages.ForCompleteScriptAsync().Started.Should().Be(0, "Should not have tried to call CompleteScript"); +// } +// +// /// +// /// This test, and probably others in this test class do not correctly test the Connecting Scenario. The port forwarder is used to kill new and existing connections but this can +// /// result in Halibut thinking a Request is transferring, where we want it to be connecting. These tests need to be rewritten to ensure they test the correct scenario. +// /// +// [Test] +// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCall), typeof(RpcCallStage) })] +// public async Task DuringStartScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCall rpcCall, RpcCallStage rpcCallStage) +// { +// var rpcCallHasStarted = new Reference(false); +// var restartedPortForwarderForCancel = false; +// var hasPausedOrStoppedPortForwarder = false; +// var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue +// .WithServiceEndpointModifier(point => +// { +// if (rpcCall == RpcCall.FirstCall) point.TryAndConnectForALongTime(); +// }) +// .WithRetryDuration(TimeSpan.FromHours(1)) +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithPortForwarder(out var portForwarder) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeStartScript( +// async () => +// { +// if (rpcCall == RpcCall.RetryingCall && recordedUsages.ForStartScriptAsync().LastException is null) +// { +// await tcpConnectionUtilities.RestartTcpConnection(); +// // Kill the first StartScript call to force the rpc call into retries +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// else +// { +// if (!hasPausedOrStoppedPortForwarder) +// { +// hasPausedOrStoppedPortForwarder = true; +// await tcpConnectionUtilities.RestartTcpConnection(); +// await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); +// } +// } +// } +// +// ensureCancellationOccursDuringAnRpcCall.Release(); +// }) +// .AfterStartScript( +// async () => +// { +// await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); +// }) +// .BeforeCancelScript( +// async () => +// { +// await Task.CompletedTask; +// +// if (!restartedPortForwarderForCancel) +// { +// restartedPortForwarderForCancel = true; +// UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Print("The script") +// .Sleep(TimeSpan.FromHours(1))) +// .Build(); +// +// // ACT +// var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); +// +// // ASSERT +// // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring +// var latestException = recordedUsages.ForStartScriptAsync().LastException; +// if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) +// { +// Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + +// "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); +// } +// latestException.Should().BeRequestCancelledException(rpcCallStage); +// +// if (rpcCall == RpcCall.FirstCall && rpcCallStage == RpcCallStage.Connecting) +// { +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// actualException.ShouldMatchExceptionContract(expectedException); +// +// // We should have cancelled the RPC call quickly +// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(15)); +// +// recordedUsages.ForStartScriptAsync().Started.Should().Be(1); +// recordedUsages.ForGetStatusAsync().Started.Should().Be(0); +// recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); +// recordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); +// } +// else if((rpcCall == RpcCall.RetryingCall && rpcCallStage == RpcCallStage.Connecting) || rpcCallStage == RpcCallStage.InFlight) +// { +// // Assert that script execution was cancelled +// actualException.Should().BeScriptExecutionCancelledException(); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// actualException.ShouldMatchExceptionContract(expectedException); +// +// +// // Assert the CancelScript and CompleteScript flow happened fairly quickly +// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); +// +// if (rpcCall == RpcCall.FirstCall) +// { +// recordedUsages.ForStartScriptAsync().Started.Should().Be(1); +// } +// else +// { +// recordedUsages.ForStartScriptAsync().Started.Should().BeGreaterOrEqualTo(2); +// } +// recordedUsages.ForGetStatusAsync().Started.Should().Be(0); +// recordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(1); +// recordedUsages.ForCompleteScriptAsync().Started.Should().Be(1); +// } +// else +// { +// throw new ArgumentOutOfRangeException(); +// } +// } +// +// [Test] +// [TentacleConfigurations(testListening: false)] +// public async Task DuringStartScript_ForPollingTentacle_ThatIsRetryingTheRpc_AndConnecting_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var halibutTimeoutsAndLimits = HalibutTimeoutsAndLimits.RecommendedValues(); +// halibutTimeoutsAndLimits.PollingQueueWaitTimeout = TimeSpan.FromSeconds(4); +// halibutTimeoutsAndLimits.PollingRequestQueueTimeout = TimeSpan.FromSeconds(3); +// +// var started = false; +// var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithRetryDuration(TimeSpan.FromHours(1)) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder().RecordMethodUsages(tentacleConfigurationTestCase, out var scriptServiceRecordedUsages).Build()) +// .WithPendingRequestQueueFactory(new CancelWhenRequestQueuedPendingRequestQueueFactory( +// cancelExecutionCancellationTokenSource, +// halibutTimeoutsAndLimits, +// // Cancel the execution when the StartScript RPC call is being retries +// shouldCancel: async () => +// { +// await Task.CompletedTask; +// +// if (started && scriptServiceRecordedUsages.ForStartScriptAsync().Started >= 2) +// { +// Logger.Information("Cancelling Execute Script"); +// return true; +// } +// +// return false; +// })) +// .WithHalibutTimeoutsAndLimits(halibutTimeoutsAndLimits) +// .Build(CancellationToken); +// +// // Arrange +// Logger.Information("Execute a script so that GetCapabilities will be cached"); +// await clientAndTentacle.TentacleClient.ExecuteScript( +// new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("The script")).Build(), +// cancelExecutionCancellationTokenSource.Token); +// +// Logger.Information("Stop Tentacle so no more requests are picked up"); +// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); +// +// var delay = clientAndTentacle.Server.ServerHalibutRuntime.TimeoutsAndLimits.PollingQueueWaitTimeout + TimeSpan.FromSeconds(2); +// Logger.Information($"Waiting for {delay} for any active PendingRequestQueues for the Tentacle to drop the latest poll connection"); +// await Task.Delay(delay, CancellationToken); +// +// scriptServiceRecordedUsages.Reset(); +// started = true; +// +// // ACT +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b.Print("The script").Sleep(TimeSpan.FromHours(1))) +// .Build(); +// +// Logger.Information("Start Executing the Script"); +// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, cancelExecutionCancellationTokenSource.Token); +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// // Assert +// scriptServiceRecordedUsages.ForStartScriptAsync().Completed.Should().BeGreaterOrEqualTo(2); +// scriptServiceRecordedUsages.ForGetStatusAsync().Started.Should().Be(0); +// scriptServiceRecordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(0, "Script Execution does not need to be cancelled on Tentacle as it has not started"); +// scriptServiceRecordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); +// } +// +// [Test] +// [TentacleConfigurations(testPolling: false)] +// public async Task DuringStartScript_ForListeningTentacle_ThatIsRetryingTheRpc_AndConnecting_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var halibutTimeoutsAndLimits = HalibutTimeoutsAndLimits.RecommendedValues(); +// halibutTimeoutsAndLimits.RetryCountLimit = 1; +// halibutTimeoutsAndLimits.ConnectionErrorRetryTimeout = TimeSpan.FromSeconds(4); +// halibutTimeoutsAndLimits.RetryListeningSleepInterval = TimeSpan.Zero; +// halibutTimeoutsAndLimits.TcpClientConnectTimeout = TimeSpan.FromSeconds(4); +// halibutTimeoutsAndLimits.TcpClientPooledConnectionTimeout = TimeSpan.FromSeconds(5); +// +// var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithRetryDuration(TimeSpan.FromHours(1)) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder().RecordMethodUsages(tentacleConfigurationTestCase, out var scriptServiceRecordedUsages).Build()) +// .WithHalibutTimeoutsAndLimits(halibutTimeoutsAndLimits) +// .WithPortForwarder(out var portForwarder) +// .Build(CancellationToken); +// +// // Arrange +// Logger.Information("Execute a script so that GetCapabilities will be cached"); +// await clientAndTentacle.TentacleClient.ExecuteScript( +// new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("The script")).Build(), +// cancelExecutionCancellationTokenSource.Token); +// +// Logger.Information("Stop Tentacle so no more requests are picked up"); +// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); +// portForwarder.Value.KillNewConnectionsImmediatlyMode = true; +// portForwarder.Value.CloseExistingConnections(); +// +// var delay = clientAndTentacle.Server.ServerHalibutRuntime.TimeoutsAndLimits.SafeTcpClientPooledConnectionTimeout + TimeSpan.FromSeconds(2); +// Logger.Information($"Waiting for {delay} for any active Pooled Connections for the Tentacle to expire"); +// await Task.Delay(delay, CancellationToken); +// +// scriptServiceRecordedUsages.Reset(); +// +// // ACT +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b.Print("The script").Sleep(TimeSpan.FromHours(1))) +// .Build(); +// +// Logger.Information("Start Executing the Script"); +// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, cancelExecutionCancellationTokenSource.Token); +// var cancellationTask = Task.Run(async () => +// { +// while (true) +// { +// if (scriptServiceRecordedUsages.ForStartScriptAsync().Started >= 2) +// { +// cancelExecutionCancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(2)); +// Logger.Information("Cancelling cancellation token source after 2 seconds."); +// return; +// } +// +// await Task.Delay(TimeSpan.FromSeconds(0.5), CancellationToken); +// } +// }, CancellationToken); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// await cancellationTask; +// +// // Assert +// scriptServiceRecordedUsages.ForStartScriptAsync().Completed.Should().BeGreaterOrEqualTo(2); +// scriptServiceRecordedUsages.ForGetStatusAsync().Started.Should().Be(0); +// scriptServiceRecordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(0, "Script Execution does not need to be cancelled on Tentacle as it has not started"); +// scriptServiceRecordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); +// } +// +// [Test] +// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCall), typeof(RpcCallStage) })] +// public async Task DuringGetStatus_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCall rpcCall, RpcCallStage rpcCallStage) +// { +// // ARRANGE +// var rpcCallHasStarted = new Reference(false); +// var restartedPortForwarderForCancel = false; +// var hasPausedOrStoppedPortForwarder = false; +// var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue +// .WithServiceEndpointModifier(point => +// { +// if (rpcCall == RpcCall.FirstCall) point.TryAndConnectForALongTime(); +// }) +// .WithRetryDuration(TimeSpan.FromHours(1)) +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithPortForwarder(out var portForwarder) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeGetStatus( +// async () => +// { +// if (rpcCall == RpcCall.RetryingCall && +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException is null) +// { +// await tcpConnectionUtilities.RestartTcpConnection(); +// // Kill the first StartScript call to force the rpc call into retries +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// else +// { +// if (!hasPausedOrStoppedPortForwarder) +// { +// hasPausedOrStoppedPortForwarder = true; +// await tcpConnectionUtilities.RestartTcpConnection(); +// await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); +// } +// } +// } +// +// ensureCancellationOccursDuringAnRpcCall.Release(); +// }) +// .AfterGetStatus( +// async () => +// { +// await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); +// }) +// .BeforeCancelScript( +// async () => +// { +// await Task.CompletedTask; +// +// if (!restartedPortForwarderForCancel) +// { +// restartedPortForwarderForCancel = true; +// UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Print("The script") +// .Sleep(TimeSpan.FromHours(1))) +// .Build(); +// +// // ACT +// var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); +// +// // ASSERT +// // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring +// var latestException = recordedUsages.ForGetStatusAsync().LastException; +// if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) +// { +// Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + +// "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); +// } +// +// latestException.Should().BeRequestCancelledException(rpcCallStage); +// +// // Assert that script execution was cancelled +// actualException.Should().BeScriptExecutionCancelledException(); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// actualException.ShouldMatchExceptionContract(expectedException); +// +// +// // Script Execution should cancel quickly +// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); +// +// recordedUsages.ForStartScriptAsync().Started.Should().Be(1); +// if (rpcCall == RpcCall.FirstCall) +// { +// recordedUsages.ForGetStatusAsync().Started.Should().Be(1); +// } +// else +// { +// recordedUsages.ForGetStatusAsync().Started.Should().BeGreaterOrEqualTo(2); +// } +// recordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(1); +// recordedUsages.ForCompleteScriptAsync().Started.Should().Be(1); +// } +// +// [Test] +// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] +// public async Task DuringCompleteScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) +// { +// // ARRANGE +// var rpcCallHasStarted = new Reference(false); +// var hasPausedOrStoppedPortForwarder = false; +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue +// .WithRetryDuration(TimeSpan.FromHours(1)) +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithPortForwarder(out var portForwarder) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeCompleteScript( +// async () => +// { +// if (!hasPausedOrStoppedPortForwarder) +// { +// hasPausedOrStoppedPortForwarder = true; +// await tcpConnectionUtilities.RestartTcpConnection(); +// await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); +// } +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// clientAndTentacle.TentacleClient.OnCancellationAbandonCompleteScriptAfter = TimeSpan.FromSeconds(20); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Print("The script") +// .Sleep(TimeSpan.FromSeconds(5))) +// .Build(); +// +// // ACT +// var (responseAndLogs, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, new SemaphoreSlim(int.MaxValue, int.MaxValue)); +// +// // ASSERT +// +// // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring +// var latestException = recordedUsages.ForCompleteScriptAsync().LastException; +// if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) +// { +// Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + +// "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); +// } +// +// // The actual exception may be null if the script completed before the cancellation was observed +// if (actualException != null) +// { +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// actualException.ShouldMatchExceptionContract(expectedException); +// } +// +// latestException.Should().BeRequestCancelledException(rpcCallStage); +// +// // Complete Script was cancelled quickly +// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); +// +// recordedUsages.ForStartScriptAsync().Started.Should().Be(1); +// recordedUsages.ForGetStatusAsync().Started.Should().BeGreaterThanOrEqualTo(1); +// recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); +// recordedUsages.ForCompleteScriptAsync().Started.Should().BeGreaterOrEqualTo(1); +// } +// +// async Task PauseOrStopPortForwarder(RpcCallStage rpcCallStage, PortForwarder portForwarder, IResponseMessageTcpKiller responseMessageTcpKiller, Reference rpcCallHasStarted) +// { +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// Logger.Information("Killing the port forwarder so the next RPCs are in the connecting state when being cancelled"); +// portForwarder.EnterKillNewAndExistingConnectionsMode(); +// +// await SetRpcCallWeAreInterestedInAsStarted(rpcCallHasStarted); +// } +// else +// { +// Logger.Information("Will Pause the port forwarder on next response so the next RPC is in-flight when being cancelled"); +// #pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed +// responseMessageTcpKiller.PauseConnectionOnNextResponse(() => SetRpcCallWeAreInterestedInAsStarted(rpcCallHasStarted)); +// #pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed +// } +// +// static async Task SetRpcCallWeAreInterestedInAsStarted(Reference rpcCallHasStarted) +// { +// // Allow the port forwarder some time to stop +// await Task.Delay(TimeSpan.FromSeconds(5)); +// rpcCallHasStarted.Value = true; +// } +// } +// +// void UnPauseOrRestartPortForwarder(TentacleType tentacleType, RpcCallStage rpcCallStage, Reference portForwarder) +// { +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// Logger.Information("Starting the PortForwarder as we stopped it to get the StartScript RPC call in the Connecting state"); +// portForwarder.Value.ReturnToNormalMode(); +// } +// else if (tentacleType == TentacleType.Polling) +// { +// Logger.Information("UnPausing the PortForwarder as we paused the connections which means Polling will be stalled"); +// portForwarder.Value.UnPauseExistingConnections(); +// portForwarder.Value.CloseExistingConnections(); +// } +// } +// +// async Task<(ScriptExecutionResult response, Exception? actualException, TimeSpan cancellationDuration)> ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted( +// ClientAndTentacle clientAndTentacle, +// ExecuteScriptCommand executeScriptCommand, +// Reference rpcCallHasStarted, +// SemaphoreSlim whenTheRequestCanBeCancelled) +// { +// var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); +// +// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( +// executeScriptCommand, +// cancelExecutionCancellationTokenSource.Token); +// +// Logger.Information("Waiting for the RPC Call to start"); +// await Wait.For(() => rpcCallHasStarted.Value, +// TimeSpan.FromSeconds(30), +// () => throw new Exception("RPC call did not start"), +// CancellationToken); +// Logger.Information("RPC Call has start"); +// +// var cancellationDuration = new Stopwatch(); +// await whenTheRequestCanBeCancelled.WithLockAsync(async () => +// { +// await Task.Delay(TimeSpan.FromSeconds(6), CancellationToken); +// Logger.Information("Cancelling ExecuteScript"); +// cancelExecutionCancellationTokenSource.Cancel(); +// cancellationDuration.Start(); +// }, CancellationToken); +// +// Exception? actualException = null; +// (ScriptExecutionResult Response, List Logs)? responseAndLogs = null; +// try +// { +// responseAndLogs = await executeScriptTask; +// } +// catch (Exception ex) +// { +// actualException = ex; +// } +// +// cancellationDuration.Stop(); +// return (responseAndLogs?.Response, actualException, cancellationDuration.Elapsed); +// } +// } +// } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanRecoverFromNetworkIssues.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanRecoverFromNetworkIssues.cs index 4f0afb1b6..7511c8a45 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanRecoverFromNetworkIssues.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanRecoverFromNetworkIssues.cs @@ -1,308 +1,308 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using NUnit.Framework; -using Octopus.Tentacle.Client; -using Octopus.Tentacle.CommonTestUtils; -using Octopus.Tentacle.CommonTestUtils.Diagnostics; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; -using Octopus.TestPortForwarder; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class ClientScriptExecutionCanRecoverFromNetworkIssues : IntegrationTest - { - [Test] - [TentacleConfigurations] - public async Task WhenANetworkFailureOccurs_DuringStartScript_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPortForwarder() - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .Build()) - .Build(CancellationToken); - - var scriptHasStartFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "scripthasstarted"); - var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .CreateFile(scriptHasStartFile) - .WaitForFileToExist(waitForFile) - .Print("hello")) - // Configure the start script command to wait a long time, so we have plenty of time to kill the connection. - .WithDurationStartScriptCanWaitForScriptToFinish(TimeSpan.FromHours(1)) - .Build(); - - var inMemoryLog = new InMemoryLog(); - - var execScriptTask = Task.Run( - async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog), - CancellationToken); - - // Wait for the script to start. - await Wait.For(() => File.Exists(scriptHasStartFile), - TimeSpan.FromSeconds(30), - () => throw new Exception("Script did not start"), - CancellationToken); - - // Now it has started, kill active connections killing the start script request. - clientTentacle.PortForwarder.CloseExistingConnections(); - - // Let the script finish. - File.WriteAllText(waitForFile, ""); - - var (finalResponse, logs) = await execScriptTask; - - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(0); - - var allLogs = logs.JoinLogs(); - allLogs.Should().Contain("hello"); - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().BeGreaterThan(1); - - inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); - } - - [Test] - [TentacleConfigurations] - public async Task WhenANetworkFailureOccurs_DuringGetStatus_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeGetStatus( - async () => - { - await Task.CompletedTask; - - if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException is null) - { - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - })) - .Build()) - .Build(CancellationToken); - - var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .Print("hello") - .WaitForFileToExist(waitForFile) - .Print("AllDone")) - .Build(); - - var inMemoryLog = new InMemoryLog(); - - var execScriptTask = Task.Run( - async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog), - CancellationToken); - - await Wait.For(() => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException != null, - TimeSpan.FromSeconds(60), - () => throw new Exception("GetStatus did not error"), - CancellationToken); - - // Let the script finish. - File.WriteAllText(waitForFile, ""); - - var (finalResponse, logs) = await execScriptTask; - - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(0); - - var allLogs = logs.JoinLogs(); - allLogs.Should().Contain("hello"); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException.Should().NotBeNull(); - - inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); - } - - [Test] - [TentacleConfigurations] - public async Task WhenANetworkFailureOccurs_DuringCompleteScript_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - bool completeScriptWasCalled = false; - PortForwarder? portForwarder = null; - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPortForwarderDataLogging() - .WithServiceEndpointModifier(serviceEndpoint => - { - serviceEndpoint.PollingRequestQueueTimeout = TimeSpan.FromSeconds(10); - serviceEndpoint.RetryCountLimit = 1; - }) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .DecorateAllScriptServicesWith(u => u - .BeforeCompleteScript( - async () => - { - - await Task.CompletedTask; - - completeScriptWasCalled = true; - // A successfully CompleteScript call is not required for the script to be completed. - // So it should be the case that the tentacle can be no longer contactable at this point, - // yet the script execution is marked as successful. - portForwarder.Dispose(); - })) - .Build()) - .Build(CancellationToken); - portForwarder = clientTentacle.PortForwarder; - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder().Print("hello").Sleep(TimeSpan.FromSeconds(1))) - .Build(); - - var inMemoryLog = new InMemoryLog(); - var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); - - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(0); - - var allLogs = logs.JoinLogs(); - allLogs.Should().Contain("hello"); - completeScriptWasCalled.Should().BeTrue("The tests expects that the client actually called this"); - - inMemoryLog.ShouldNotHaveLoggedRetryAttemptsOrRetryFailures(); - } - - [Test] - [TentacleConfigurations] - public async Task WhenANetworkFailureOccurs_DuringCancelScript_TheClientIsAbleToSuccessfullyCancelTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var cts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); - using var tmp = new TemporaryDirectory(); - var scriptIsRunningFlag = Path.Combine(tmp.DirectoryPath, "scriptisrunning"); - - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeGetStatus( - async () => - { - await Wait.For(() => File.Exists(scriptIsRunningFlag), - TimeSpan.FromSeconds(30), - () => throw new Exception("Script did not start"), - CancellationToken); - cts.Cancel(); - }) - .BeforeCancelScript( - async () => - { - await Task.CompletedTask; - - if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException == null) - { - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - })) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .Print("hello") - .CreateFile(scriptIsRunningFlag) - .Sleep(TimeSpan.FromMinutes(2)) - .Print("AllDone")) - .Build(); - - Exception? actualException = null; - var logs = new List(); - var inMemoryLog = new InMemoryLog(); - - try - { - await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, - onScriptStatusResponseReceived => - { - logs.AddRange(onScriptStatusResponseReceived.Logs); - }, - _ => Task.CompletedTask, - new SerilogLoggerBuilder().Build().ForContext().ToITentacleTaskLog().Chain(inMemoryLog), - cts.Token); - } - catch (Exception ex) - { - actualException = ex; - } - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); - - actualException!.ShouldMatchExceptionContract(expectedException); - - var allLogs = logs.JoinLogs(); - allLogs.Should().Contain("hello"); - allLogs.Should().NotContain("AllDone"); - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException.Should().NotBeNull(); - - inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); - } - - [Test] - [TentacleConfigurations] - public async Task WhenANetworkFailureOccurs_DuringGetCapabilities_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var recordedUsages) - .DecorateCapabilitiesServiceV2With(d => d - .BeforeGetCapabilities( - async () => - { - if (recordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException == null) - { - await tcpConnectionUtilities.RestartTcpConnection(); - - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - })) - .Build()) - .Build(CancellationToken); - - var inMemoryLog = new InMemoryLog(); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder().Print("hello")) - .Build(); - - var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); - - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(0); - - var allLogs = logs.JoinLogs(); - allLogs.Should().Contain("hello"); - - recordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException.Should().NotBeNull(); - recordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).Started.Should().Be(2); - - inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); - } - } -} +// using System; +// using System.Collections.Generic; +// using System.IO; +// using System.Threading; +// using System.Threading.Tasks; +// using FluentAssertions; +// using NUnit.Framework; +// using Octopus.Tentacle.Client; +// using Octopus.Tentacle.CommonTestUtils; +// using Octopus.Tentacle.CommonTestUtils.Diagnostics; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Contracts.ClientServices; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; +// using Octopus.TestPortForwarder; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class ClientScriptExecutionCanRecoverFromNetworkIssues : IntegrationTest +// { +// [Test] +// [TentacleConfigurations] +// public async Task WhenANetworkFailureOccurs_DuringStartScript_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPortForwarder() +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .Build()) +// .Build(CancellationToken); +// +// var scriptHasStartFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "scripthasstarted"); +// var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .CreateFile(scriptHasStartFile) +// .WaitForFileToExist(waitForFile) +// .Print("hello")) +// // Configure the start script command to wait a long time, so we have plenty of time to kill the connection. +// .WithDurationStartScriptCanWaitForScriptToFinish(TimeSpan.FromHours(1)) +// .Build(); +// +// var inMemoryLog = new InMemoryLog(); +// +// var execScriptTask = Task.Run( +// async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog), +// CancellationToken); +// +// // Wait for the script to start. +// await Wait.For(() => File.Exists(scriptHasStartFile), +// TimeSpan.FromSeconds(30), +// () => throw new Exception("Script did not start"), +// CancellationToken); +// +// // Now it has started, kill active connections killing the start script request. +// clientTentacle.PortForwarder.CloseExistingConnections(); +// +// // Let the script finish. +// File.WriteAllText(waitForFile, ""); +// +// var (finalResponse, logs) = await execScriptTask; +// +// finalResponse.State.Should().Be(ProcessState.Complete); +// finalResponse.ExitCode.Should().Be(0); +// +// var allLogs = logs.JoinLogs(); +// allLogs.Should().Contain("hello"); +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().BeGreaterThan(1); +// +// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task WhenANetworkFailureOccurs_DuringGetStatus_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeGetStatus( +// async () => +// { +// await Task.CompletedTask; +// +// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException is null) +// { +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .Print("hello") +// .WaitForFileToExist(waitForFile) +// .Print("AllDone")) +// .Build(); +// +// var inMemoryLog = new InMemoryLog(); +// +// var execScriptTask = Task.Run( +// async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog), +// CancellationToken); +// +// await Wait.For(() => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException != null, +// TimeSpan.FromSeconds(60), +// () => throw new Exception("GetStatus did not error"), +// CancellationToken); +// +// // Let the script finish. +// File.WriteAllText(waitForFile, ""); +// +// var (finalResponse, logs) = await execScriptTask; +// +// finalResponse.State.Should().Be(ProcessState.Complete); +// finalResponse.ExitCode.Should().Be(0); +// +// var allLogs = logs.JoinLogs(); +// allLogs.Should().Contain("hello"); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException.Should().NotBeNull(); +// +// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task WhenANetworkFailureOccurs_DuringCompleteScript_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// bool completeScriptWasCalled = false; +// PortForwarder? portForwarder = null; +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPortForwarderDataLogging() +// .WithServiceEndpointModifier(serviceEndpoint => +// { +// serviceEndpoint.PollingRequestQueueTimeout = TimeSpan.FromSeconds(10); +// serviceEndpoint.RetryCountLimit = 1; +// }) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .DecorateAllScriptServicesWith(u => u +// .BeforeCompleteScript( +// async () => +// { +// +// await Task.CompletedTask; +// +// completeScriptWasCalled = true; +// // A successfully CompleteScript call is not required for the script to be completed. +// // So it should be the case that the tentacle can be no longer contactable at this point, +// // yet the script execution is marked as successful. +// portForwarder.Dispose(); +// })) +// .Build()) +// .Build(CancellationToken); +// portForwarder = clientTentacle.PortForwarder; +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder().Print("hello").Sleep(TimeSpan.FromSeconds(1))) +// .Build(); +// +// var inMemoryLog = new InMemoryLog(); +// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); +// +// finalResponse.State.Should().Be(ProcessState.Complete); +// finalResponse.ExitCode.Should().Be(0); +// +// var allLogs = logs.JoinLogs(); +// allLogs.Should().Contain("hello"); +// completeScriptWasCalled.Should().BeTrue("The tests expects that the client actually called this"); +// +// inMemoryLog.ShouldNotHaveLoggedRetryAttemptsOrRetryFailures(); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task WhenANetworkFailureOccurs_DuringCancelScript_TheClientIsAbleToSuccessfullyCancelTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var cts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); +// using var tmp = new TemporaryDirectory(); +// var scriptIsRunningFlag = Path.Combine(tmp.DirectoryPath, "scriptisrunning"); +// +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeGetStatus( +// async () => +// { +// await Wait.For(() => File.Exists(scriptIsRunningFlag), +// TimeSpan.FromSeconds(30), +// () => throw new Exception("Script did not start"), +// CancellationToken); +// cts.Cancel(); +// }) +// .BeforeCancelScript( +// async () => +// { +// await Task.CompletedTask; +// +// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException == null) +// { +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .Print("hello") +// .CreateFile(scriptIsRunningFlag) +// .Sleep(TimeSpan.FromMinutes(2)) +// .Print("AllDone")) +// .Build(); +// +// Exception? actualException = null; +// var logs = new List(); +// var inMemoryLog = new InMemoryLog(); +// +// try +// { +// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, +// onScriptStatusResponseReceived => +// { +// logs.AddRange(onScriptStatusResponseReceived.Logs); +// }, +// _ => Task.CompletedTask, +// new SerilogLoggerBuilder().Build().ForContext().ToITentacleTaskLog().Chain(inMemoryLog), +// cts.Token); +// } +// catch (Exception ex) +// { +// actualException = ex; +// } +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); +// +// actualException!.ShouldMatchExceptionContract(expectedException); +// +// var allLogs = logs.JoinLogs(); +// allLogs.Should().Contain("hello"); +// allLogs.Should().NotContain("AllDone"); +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException.Should().NotBeNull(); +// +// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task WhenANetworkFailureOccurs_DuringGetCapabilities_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var recordedUsages) +// .DecorateCapabilitiesServiceV2With(d => d +// .BeforeGetCapabilities( +// async () => +// { +// if (recordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException == null) +// { +// await tcpConnectionUtilities.RestartTcpConnection(); +// +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var inMemoryLog = new InMemoryLog(); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder().Print("hello")) +// .Build(); +// +// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); +// +// finalResponse.State.Should().Be(ProcessState.Complete); +// finalResponse.ExitCode.Should().Be(0); +// +// var allLogs = logs.JoinLogs(); +// allLogs.Should().Contain("hello"); +// +// recordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException.Should().NotBeNull(); +// recordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).Started.Should().Be(2); +// +// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); +// } +// } +// } diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionIsolationMutex.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionIsolationMutex.cs index baf50164c..cdcac998b 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionIsolationMutex.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionIsolationMutex.cs @@ -1,182 +1,182 @@ -using System; -using System.Collections; -using System.IO; -using System.Threading.Tasks; -using FluentAssertions; -using NUnit.Framework; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class ClientScriptExecutionIsolationMutex : IntegrationTest - { - [Test] - [TentacleConfigurations(testScriptIsolationLevelVersions: true, additionalParameterTypes: new object[] { typeof(ScriptIsolationLevel)})] - public async Task ScriptIsolationMutexFull_EnsuresTwoDifferentScriptsDontRunAtTheSameTime(TentacleConfigurationTestCase tentacleConfigurationTestCase, ScriptIsolationLevel levelOfSecondScript) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .Build()) - .Build(CancellationToken); - - var firstScriptStartFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptStartFile"); - var firstScriptWaitFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptWaitFile"); - - var secondScriptStart = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "secondScriptStartFile"); - - var firstStartScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .CreateFile(firstScriptStartFile) - .WaitForFileToExist(firstScriptWaitFile)) - .WithIsolationLevel(ScriptIsolationLevel.FullIsolation) - .WithIsolationMutexName("mymutex") - .Build(); - - var secondStartScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder().CreateFile(secondScriptStart)) - .WithIsolationLevel(levelOfSecondScript) - .WithIsolationMutexName("mymutex") - .Build(); - - var tentacleClient = clientTentacle.TentacleClient; - var firstScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(firstStartScriptCommand, CancellationToken)); - - // Wait for the first script to start running - await Wait.For(() => File.Exists(firstScriptStartFile), - TimeSpan.FromSeconds(30), - () => throw new Exception("Script did not start"), - CancellationToken); - Logger.Information("First script is now running"); - - var secondScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(secondStartScriptCommand, CancellationToken)); - - // Wait for the second script start script RPC call to return. - await Wait.For(() => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Completed == 2, - TimeSpan.FromSeconds(60), - () => throw new Exception("Second execute script call did not complete"), - CancellationToken); - - // Give Tentacle some more time to run the script (although it should not). - await Task.Delay(TimeSpan.FromSeconds(2)); - - File.Exists(secondScriptStart).Should().BeFalse("The second script must not be started while the first is running with a FullIsolationMutex"); - - // Let the first script finish. - File.WriteAllText(firstScriptWaitFile, ""); - - var (finalResponseFirstScript, _) = await firstScriptExecution; - var (finalResponseSecondScript, _) = await secondScriptExecution; - - File.Exists(secondScriptStart).Should().BeTrue("The second should now have run."); - - finalResponseFirstScript.ExitCode.Should().Be(0); - finalResponseSecondScript.ExitCode.Should().Be(0); - } - - [Test] - [TentacleConfigurations(testScriptIsolationLevelVersions: true, additionalParameterTypes: new object[] {typeof(ScriptsInParallelTestCases)})] - public async Task ScriptIsolationMutexFull_IsOnlyExclusiveWhenFullAndWhenTheMutexNameIsTheSame(TentacleConfigurationTestCase tentacleConfigurationTestCase, ScriptsInParallelTestCase scriptsInParallelTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); - - var firstScriptStartFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptStartFile"); - var firstScriptWaitFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptWaitFile"); - - var secondScriptStart = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "secondScriptStartFile"); - - var firstStartScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .CreateFile(firstScriptStartFile) - .WaitForFileToExist(firstScriptWaitFile)) - .WithIsolationLevel(scriptsInParallelTestCase.LevelOfFirstScript) - .WithIsolationMutexName(scriptsInParallelTestCase.MutexForFirstScript) - .Build(); - - var secondStartScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder().CreateFile(secondScriptStart)) - .WithIsolationLevel(scriptsInParallelTestCase.LevelOfSecondScript) - .WithIsolationMutexName(scriptsInParallelTestCase.MutexForSecondScript) - .Build(); - - var tentacleClient = clientTentacle.TentacleClient; - var firstScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(firstStartScriptCommand, CancellationToken)); - - // Wait for the first script to start running - await Wait.For(() => File.Exists(firstScriptStartFile), - TimeSpan.FromSeconds(30), - () => throw new Exception("Script did not start running"), - CancellationToken); - - var secondScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(secondStartScriptCommand, CancellationToken)); - - // Wait for the second script to start - await Wait.For(() => File.Exists(secondScriptStart), - TimeSpan.FromSeconds(30), - () => throw new Exception("Second script did not start"), - CancellationToken); - // Both scripts are now running in parallel - - // Let the first script finish. - File.WriteAllText(firstScriptWaitFile, ""); - - var (finalResponseFirstScript, _) = await firstScriptExecution; - var (finalResponseSecondScript, _) = await secondScriptExecution; - - File.Exists(secondScriptStart).Should().BeTrue("The second script must not be started while the first is running with a FullIsolationMutex"); - - finalResponseFirstScript.ExitCode.Should().Be(0); - finalResponseSecondScript.ExitCode.Should().Be(0); - } - - public class ScriptsInParallelTestCases : IEnumerable - { - public IEnumerator GetEnumerator() - { - // Scripts with the same mutex name can run at the same time if they both has no isolation. - yield return ScriptsInParallelTestCase.NoIsolationSameMutex; - // Scripts with different mutex names can run at the same time. - yield return ScriptsInParallelTestCase.FullIsolationDifferentMutex; - } - } - - public class ScriptsInParallelTestCase - { - public static ScriptsInParallelTestCase NoIsolationSameMutex => new(ScriptIsolationLevel.NoIsolation, "sameMutex", ScriptIsolationLevel.NoIsolation, "sameMutex", nameof(NoIsolationSameMutex)); - public static ScriptsInParallelTestCase FullIsolationDifferentMutex => new(ScriptIsolationLevel.FullIsolation, "mutex", ScriptIsolationLevel.FullIsolation, "differentMutex", nameof(FullIsolationDifferentMutex)); - - public readonly ScriptIsolationLevel LevelOfFirstScript; - public readonly string MutexForFirstScript; - public readonly ScriptIsolationLevel LevelOfSecondScript; - public readonly string MutexForSecondScript; - - private readonly string stringValue; - - private ScriptsInParallelTestCase( - ScriptIsolationLevel levelOfFirstScript, - string mutexForFirstScript, - ScriptIsolationLevel levelOfSecondScript, - string mutexForSecondScript, - string stringValue) - { - LevelOfFirstScript = levelOfFirstScript; - MutexForFirstScript = mutexForFirstScript; - LevelOfSecondScript = levelOfSecondScript; - MutexForSecondScript = mutexForSecondScript; - this.stringValue = stringValue; - } - - public override string ToString() - { - return stringValue; - } - } - } -} +// using System; +// using System.Collections; +// using System.IO; +// using System.Threading.Tasks; +// using FluentAssertions; +// using NUnit.Framework; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Contracts.ClientServices; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class ClientScriptExecutionIsolationMutex : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(testScriptIsolationLevelVersions: true, additionalParameterTypes: new object[] { typeof(ScriptIsolationLevel)})] +// public async Task ScriptIsolationMutexFull_EnsuresTwoDifferentScriptsDontRunAtTheSameTime(TentacleConfigurationTestCase tentacleConfigurationTestCase, ScriptIsolationLevel levelOfSecondScript) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .Build()) +// .Build(CancellationToken); +// +// var firstScriptStartFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptStartFile"); +// var firstScriptWaitFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptWaitFile"); +// +// var secondScriptStart = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "secondScriptStartFile"); +// +// var firstStartScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .CreateFile(firstScriptStartFile) +// .WaitForFileToExist(firstScriptWaitFile)) +// .WithIsolationLevel(ScriptIsolationLevel.FullIsolation) +// .WithIsolationMutexName("mymutex") +// .Build(); +// +// var secondStartScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder().CreateFile(secondScriptStart)) +// .WithIsolationLevel(levelOfSecondScript) +// .WithIsolationMutexName("mymutex") +// .Build(); +// +// var tentacleClient = clientTentacle.TentacleClient; +// var firstScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(firstStartScriptCommand, CancellationToken)); +// +// // Wait for the first script to start running +// await Wait.For(() => File.Exists(firstScriptStartFile), +// TimeSpan.FromSeconds(30), +// () => throw new Exception("Script did not start"), +// CancellationToken); +// Logger.Information("First script is now running"); +// +// var secondScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(secondStartScriptCommand, CancellationToken)); +// +// // Wait for the second script start script RPC call to return. +// await Wait.For(() => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Completed == 2, +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Second execute script call did not complete"), +// CancellationToken); +// +// // Give Tentacle some more time to run the script (although it should not). +// await Task.Delay(TimeSpan.FromSeconds(2)); +// +// File.Exists(secondScriptStart).Should().BeFalse("The second script must not be started while the first is running with a FullIsolationMutex"); +// +// // Let the first script finish. +// File.WriteAllText(firstScriptWaitFile, ""); +// +// var (finalResponseFirstScript, _) = await firstScriptExecution; +// var (finalResponseSecondScript, _) = await secondScriptExecution; +// +// File.Exists(secondScriptStart).Should().BeTrue("The second should now have run."); +// +// finalResponseFirstScript.ExitCode.Should().Be(0); +// finalResponseSecondScript.ExitCode.Should().Be(0); +// } +// +// [Test] +// [TentacleConfigurations(testScriptIsolationLevelVersions: true, additionalParameterTypes: new object[] {typeof(ScriptsInParallelTestCases)})] +// public async Task ScriptIsolationMutexFull_IsOnlyExclusiveWhenFullAndWhenTheMutexNameIsTheSame(TentacleConfigurationTestCase tentacleConfigurationTestCase, ScriptsInParallelTestCase scriptsInParallelTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); +// +// var firstScriptStartFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptStartFile"); +// var firstScriptWaitFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptWaitFile"); +// +// var secondScriptStart = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "secondScriptStartFile"); +// +// var firstStartScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .CreateFile(firstScriptStartFile) +// .WaitForFileToExist(firstScriptWaitFile)) +// .WithIsolationLevel(scriptsInParallelTestCase.LevelOfFirstScript) +// .WithIsolationMutexName(scriptsInParallelTestCase.MutexForFirstScript) +// .Build(); +// +// var secondStartScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder().CreateFile(secondScriptStart)) +// .WithIsolationLevel(scriptsInParallelTestCase.LevelOfSecondScript) +// .WithIsolationMutexName(scriptsInParallelTestCase.MutexForSecondScript) +// .Build(); +// +// var tentacleClient = clientTentacle.TentacleClient; +// var firstScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(firstStartScriptCommand, CancellationToken)); +// +// // Wait for the first script to start running +// await Wait.For(() => File.Exists(firstScriptStartFile), +// TimeSpan.FromSeconds(30), +// () => throw new Exception("Script did not start running"), +// CancellationToken); +// +// var secondScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(secondStartScriptCommand, CancellationToken)); +// +// // Wait for the second script to start +// await Wait.For(() => File.Exists(secondScriptStart), +// TimeSpan.FromSeconds(30), +// () => throw new Exception("Second script did not start"), +// CancellationToken); +// // Both scripts are now running in parallel +// +// // Let the first script finish. +// File.WriteAllText(firstScriptWaitFile, ""); +// +// var (finalResponseFirstScript, _) = await firstScriptExecution; +// var (finalResponseSecondScript, _) = await secondScriptExecution; +// +// File.Exists(secondScriptStart).Should().BeTrue("The second script must not be started while the first is running with a FullIsolationMutex"); +// +// finalResponseFirstScript.ExitCode.Should().Be(0); +// finalResponseSecondScript.ExitCode.Should().Be(0); +// } +// +// public class ScriptsInParallelTestCases : IEnumerable +// { +// public IEnumerator GetEnumerator() +// { +// // Scripts with the same mutex name can run at the same time if they both has no isolation. +// yield return ScriptsInParallelTestCase.NoIsolationSameMutex; +// // Scripts with different mutex names can run at the same time. +// yield return ScriptsInParallelTestCase.FullIsolationDifferentMutex; +// } +// } +// +// public class ScriptsInParallelTestCase +// { +// public static ScriptsInParallelTestCase NoIsolationSameMutex => new(ScriptIsolationLevel.NoIsolation, "sameMutex", ScriptIsolationLevel.NoIsolation, "sameMutex", nameof(NoIsolationSameMutex)); +// public static ScriptsInParallelTestCase FullIsolationDifferentMutex => new(ScriptIsolationLevel.FullIsolation, "mutex", ScriptIsolationLevel.FullIsolation, "differentMutex", nameof(FullIsolationDifferentMutex)); +// +// public readonly ScriptIsolationLevel LevelOfFirstScript; +// public readonly string MutexForFirstScript; +// public readonly ScriptIsolationLevel LevelOfSecondScript; +// public readonly string MutexForSecondScript; +// +// private readonly string stringValue; +// +// private ScriptsInParallelTestCase( +// ScriptIsolationLevel levelOfFirstScript, +// string mutexForFirstScript, +// ScriptIsolationLevel levelOfSecondScript, +// string mutexForSecondScript, +// string stringValue) +// { +// LevelOfFirstScript = levelOfFirstScript; +// MutexForFirstScript = mutexForFirstScript; +// LevelOfSecondScript = levelOfSecondScript; +// MutexForSecondScript = mutexForSecondScript; +// this.stringValue = stringValue; +// } +// +// public override string ToString() +// { +// return stringValue; +// } +// } +// } +// } diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs index 2ea063101..0a5e2c447 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs @@ -1,548 +1,548 @@ -using System; -using System.Diagnostics; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils.Diagnostics; -using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; - -namespace Octopus.Tentacle.Tests.Integration -{ - /// - /// These tests make sure that we can cancel or walk away (if code does not cooperate with cancellation tokens) - /// from RPC calls when they are being retried and the rpc timeout period elapses. - /// - [IntegrationTestTimeout] - public class ClientScriptExecutionRetriesTimeout : IntegrationTest - { - readonly TimeSpan retryIfRemainingDurationAtLeastBuffer = TimeSpan.FromSeconds(1); - readonly TimeSpan retryBackoffBuffer = TimeSpan.FromSeconds(2); - - [Test] - [TentacleConfigurations(additionalParameterTypes: new object[] {typeof(RpcCallStage)})] - public async Task WhenRpcRetriesTimeOut_DuringGetCapabilities_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) - { - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - // Set a short retry duration so we cancel fairly quickly - .WithRetryDuration(TimeSpan.FromSeconds(15)) - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithPortForwarder(out var portForwarder) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var capabilitiesMethodUsages) - .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) - .DecorateCapabilitiesServiceV2With(d => d - .BeforeGetCapabilities( - async () => - { - // Kill the first GetCapabilities call to force the rpc call into retries - if (capabilitiesMethodUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException is null) - { - // Ensure there is an active connection so it can be killed correctly - await tcpConnectionUtilities.RestartTcpConnection(); - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - else - { - if (rpcCallStage == RpcCallStage.Connecting) - { - // Kill the port forwarder so the next requests are in the connecting state when retries timeout - Logger.Information("Killing PortForwarder"); - portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); - } - else - { - // Ensure there is an active connection so it can be killed correctly - await tcpConnectionUtilities.RestartTcpConnection(); - // Pause the port forwarder so the next requests are in-flight when retries timeout - responseMessageTcpKiller.PauseConnectionOnNextResponse(); - } - } - })) - .Build()) - .Build(CancellationToken); - - var inMemoryLog = new InMemoryLog(); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Print("Should not run this script")) - .Build(); - - var duration = Stopwatch.StartNew(); - - var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - duration.Stop(); - - capabilitiesMethodUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).Started.Should().BeGreaterOrEqualTo(2); - scriptMethodUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(0, "Test should not have not proceeded past GetCapabilities"); - - // Ensure we actually waited and retried until the timeout policy kicked in - duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); - - inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); - } - - [Test] - [TentacleConfigurations] - public async Task WhenGetCapabilitiesFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var retryDuration = TimeSpan.FromSeconds(15); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - // Set a short retry duration so we cancel fairly quickly - .WithRetryDuration(retryDuration) - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var capabilitiesMethodUsages) - .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) - .DecorateCapabilitiesServiceV2With(d => d - .BeforeGetCapabilities( - async () => - { - await tcpConnectionUtilities.RestartTcpConnection(); - - // Sleep to make the initial RPC call take longer than the allowed retry duration - await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); - - // Kill the first GetCapabilities call to force the rpc call into retries - responseMessageTcpKiller.KillConnectionOnNextResponse(); - })) - .Build()) - .Build(CancellationToken); - - var inMemoryLog = new InMemoryLog(); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .Build(); - - var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - capabilitiesMethodUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).Started.Should().Be(1); - scriptMethodUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(0, "Test should not have not proceeded past GetCapabilities"); - - inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); - } - - [Test] - [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage)})] - public async Task WhenRpcRetriesTimeOut_DuringStartScript_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) - { - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - // Set a short retry duration so we cancel fairly quickly - .WithRetryDuration(TimeSpan.FromSeconds(15)) - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithPortForwarder(out var portForwarder) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeStartScript( - async () => - { - // Kill the first StartScript call to force the rpc call into retries - if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).LastException == null) - { - // Ensure there is an active connection so it can be killed correctly - await tcpConnectionUtilities.RestartTcpConnection(); - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - else - { - if (rpcCallStage == RpcCallStage.Connecting) - { - // Kill the port forwarder so the next requests are in the connecting state when retries timeout - Logger.Information("Killing PortForwarder"); - portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); - } - else - { - // Ensure there is an active connection so it can be killed correctly - await tcpConnectionUtilities.RestartTcpConnection(); - // Pause the port forwarder so the next requests are in-flight when retries timeout - responseMessageTcpKiller.PauseConnectionOnNextResponse(); - } - } - })) - .Build()) - .Build(CancellationToken); - - var inMemoryLog = new InMemoryLog(); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Print("Start Script") - .Sleep(TimeSpan.FromHours(1)) - .Print("End Script")) - .Build(); - - var duration = Stopwatch.StartNew(); - var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - duration.Stop(); - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().BeGreaterOrEqualTo(2); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); - - // Ensure we actually waited and retried until the timeout policy kicked in - duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); - - inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); - } - - [Test] - [TentacleConfigurations] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] - public async Task WhenStartScriptFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var retryDuration = TimeSpan.FromSeconds(15); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - // Set a short retry duration so we cancel fairly quickly - .WithRetryDuration(retryDuration) - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeStartScript( - async () => - { - // Sleep to make the initial RPC call take longer than the allowed retry duration - await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); - - // Kill the first StartScript call to force the rpc call into retries - responseMessageTcpKiller.KillConnectionOnNextResponse(); - })) - .Build()) - .Build(CancellationToken); - - var inMemoryLog = new InMemoryLog(); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Sleep(TimeSpan.FromHours(1))) - .Build(); - - var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); - - inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); - } - - [Test] - [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage)})] - public async Task WhenRpcRetriesTimeOut_DuringGetStatus_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) - { - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - // Set a short retry duration so we cancel fairly quickly - .WithRetryDuration(TimeSpan.FromSeconds(15)) - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithPortForwarder(out var portForwarder) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeGetStatus( - async () => - { - // Kill the first GetStatus call to force the rpc call into retries - if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException == null) - { - // Ensure there is an active connection so it can be killed correctly - await tcpConnectionUtilities.RestartTcpConnection(); - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - else - { - if (rpcCallStage == RpcCallStage.Connecting) - { - // Kill the port forwarder so the next requests are in the connecting state when retries timeout - Logger.Information("Killing PortForwarder"); - portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); - } - else - { - // Ensure there is an active connection so it can be killed correctly - await tcpConnectionUtilities.RestartTcpConnection(); - // Pause the port forwarder so the next requests are in-flight when retries timeout - responseMessageTcpKiller.PauseConnectionOnNextResponse(); - } - } - })) - .Build()) - .Build(CancellationToken); - - var inMemoryLog = new InMemoryLog(); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Print("Start Script") - .Sleep(TimeSpan.FromHours(1)) - .Print("End Script")) - // Don't wait in start script as we want to tst get status - .WithDurationStartScriptCanWaitForScriptToFinish(null) - .Build(); - - var duration = Stopwatch.StartNew(); - var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - duration.Stop(); - - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterOrEqualTo(2); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); - - // Ensure we actually waited and retried until the timeout policy kicked in - duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); - - inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); - } - - [Test] - [TentacleConfigurations] - public async Task WhenGetStatusFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var retryDuration = TimeSpan.FromSeconds(15); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - // Set a short retry duration so we cancel fairly quickly - .WithRetryDuration(retryDuration) - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeGetStatus( - async () => - { - // Sleep to make the initial RPC call take longer than the allowed retry duration - await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); - - // Kill the first GetStatus call to force the rpc call into retries - responseMessageTcpKiller.KillConnectionOnNextResponse(); - })) - .Build()) - .Build(CancellationToken); - - var inMemoryLog = new InMemoryLog(); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Sleep(TimeSpan.FromHours(1))) - // Don't wait in start script as we want to test get status - .WithDurationStartScriptCanWaitForScriptToFinish(null) - .Build(); - - var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); - - inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); - } - - [Test] - [TentacleConfigurations(additionalParameterTypes: new object[] {typeof(RpcCallStage)})] - public async Task WhenRpcRetriesTimeOut_DuringCancelScript_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) - { - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - // Set a short retry duration so we cancel fairly quickly - .WithRetryDuration(TimeSpan.FromSeconds(15)) - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithPortForwarder(out var portForwarder) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeCancelScript( - async () => - { - // Kill the first CancelScript call to force the rpc call into retries - if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException == null) - { - // Ensure there is an active connection so it can be killed correctly - await tcpConnectionUtilities.RestartTcpConnection(); - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - else - { - if (rpcCallStage == RpcCallStage.Connecting) - { - // Kill the port forwarder so the next requests are in the connecting state when retries timeout - Logger.Information("Killing PortForwarder"); - portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); - } - else - { - // Ensure there is an active connection so it can be killed correctly - await tcpConnectionUtilities.RestartTcpConnection(); - // Pause the port forwarder so the next requests are in-flight when retries timeout - responseMessageTcpKiller.PauseConnectionOnNextResponse(); - } - } - })) - .Build()) - .Build(CancellationToken); - - var inMemoryLog = new InMemoryLog(); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Print("Start Script") - .Sleep(TimeSpan.FromHours(1)) - .Print("End Script")) - // Don't wait in start script as we want to tst get status - .WithDurationStartScriptCanWaitForScriptToFinish(null) - .Build(); - - // Start the script which will wait for a file to exist - var testCancellationTokenSource = new CancellationTokenSource(); - - var duration = Stopwatch.StartNew(); - var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( - startScriptCommand, - CancellationTokenSource.CreateLinkedTokenSource(CancellationToken, testCancellationTokenSource.Token).Token, - null, - inMemoryLog); - - await Wait.For(() => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Completed > 0, - TimeSpan.FromSeconds(60), - () => throw new Exception("Script execution did not complete"), - CancellationToken); - - // We cancel script execution via the cancellation token. This should trigger the CancelScript RPC call to be made - testCancellationTokenSource.Cancel(); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - duration.Stop(); - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterOrEqualTo(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().BeGreaterOrEqualTo(2); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); - - // Ensure we actually waited and retried until the timeout policy kicked in - duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); - - inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); - } - - [Test] - [TentacleConfigurations] - public async Task WhenCancelScriptFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var retryDuration = TimeSpan.FromSeconds(15); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - // Set a short retry duration so we cancel fairly quickly - .WithRetryDuration(retryDuration) - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeCancelScript( - async () => - { - // Sleep to make the initial RPC call take longer than the allowed retry duration - await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); - - // Kill the first CancelScript call to force the rpc call into retries - responseMessageTcpKiller.KillConnectionOnNextResponse(); - })) - .Build()) - .Build(CancellationToken); - - var inMemoryLog = new InMemoryLog(); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b - .Sleep(TimeSpan.FromHours(1))) - // Don't wait in start script as we want to test get status - .WithDurationStartScriptCanWaitForScriptToFinish(null) - .Build(); - - // Start the script which will wait for a file to exist - var testCancellationTokenSource = new CancellationTokenSource(); - - var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( - startScriptCommand, - CancellationTokenSource.CreateLinkedTokenSource(CancellationToken, testCancellationTokenSource.Token).Token, - null, - inMemoryLog); - - Func action = async () => await executeScriptTask; - await Wait.For( - () => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Completed > 0, - TimeSpan.FromSeconds(60), - () => throw new Exception("Script Execution did not complete"), - CancellationToken); - - // We cancel script execution via the cancellation token. This should trigger the CancelScript RPC call to be made - testCancellationTokenSource.Cancel(); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterOrEqualTo(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); - - inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); - } - } -} +// using System; +// using System.Diagnostics; +// using System.Threading; +// using System.Threading.Tasks; +// using FluentAssertions; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils.Diagnostics; +// using Octopus.Tentacle.Contracts.ClientServices; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// /// +// /// These tests make sure that we can cancel or walk away (if code does not cooperate with cancellation tokens) +// /// from RPC calls when they are being retried and the rpc timeout period elapses. +// /// +// [IntegrationTestTimeout] +// public class ClientScriptExecutionRetriesTimeout : IntegrationTest +// { +// readonly TimeSpan retryIfRemainingDurationAtLeastBuffer = TimeSpan.FromSeconds(1); +// readonly TimeSpan retryBackoffBuffer = TimeSpan.FromSeconds(2); +// +// [Test] +// [TentacleConfigurations(additionalParameterTypes: new object[] {typeof(RpcCallStage)})] +// public async Task WhenRpcRetriesTimeOut_DuringGetCapabilities_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) +// { +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// // Set a short retry duration so we cancel fairly quickly +// .WithRetryDuration(TimeSpan.FromSeconds(15)) +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithPortForwarder(out var portForwarder) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var capabilitiesMethodUsages) +// .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) +// .DecorateCapabilitiesServiceV2With(d => d +// .BeforeGetCapabilities( +// async () => +// { +// // Kill the first GetCapabilities call to force the rpc call into retries +// if (capabilitiesMethodUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException is null) +// { +// // Ensure there is an active connection so it can be killed correctly +// await tcpConnectionUtilities.RestartTcpConnection(); +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// else +// { +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// // Kill the port forwarder so the next requests are in the connecting state when retries timeout +// Logger.Information("Killing PortForwarder"); +// portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); +// } +// else +// { +// // Ensure there is an active connection so it can be killed correctly +// await tcpConnectionUtilities.RestartTcpConnection(); +// // Pause the port forwarder so the next requests are in-flight when retries timeout +// responseMessageTcpKiller.PauseConnectionOnNextResponse(); +// } +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var inMemoryLog = new InMemoryLog(); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Print("Should not run this script")) +// .Build(); +// +// var duration = Stopwatch.StartNew(); +// +// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// duration.Stop(); +// +// capabilitiesMethodUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).Started.Should().BeGreaterOrEqualTo(2); +// scriptMethodUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(0, "Test should not have not proceeded past GetCapabilities"); +// +// // Ensure we actually waited and retried until the timeout policy kicked in +// duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); +// +// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task WhenGetCapabilitiesFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var retryDuration = TimeSpan.FromSeconds(15); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// // Set a short retry duration so we cancel fairly quickly +// .WithRetryDuration(retryDuration) +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var capabilitiesMethodUsages) +// .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) +// .DecorateCapabilitiesServiceV2With(d => d +// .BeforeGetCapabilities( +// async () => +// { +// await tcpConnectionUtilities.RestartTcpConnection(); +// +// // Sleep to make the initial RPC call take longer than the allowed retry duration +// await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); +// +// // Kill the first GetCapabilities call to force the rpc call into retries +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// })) +// .Build()) +// .Build(CancellationToken); +// +// var inMemoryLog = new InMemoryLog(); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .Build(); +// +// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// capabilitiesMethodUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).Started.Should().Be(1); +// scriptMethodUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(0, "Test should not have not proceeded past GetCapabilities"); +// +// inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); +// } +// +// [Test] +// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage)})] +// public async Task WhenRpcRetriesTimeOut_DuringStartScript_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) +// { +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// // Set a short retry duration so we cancel fairly quickly +// .WithRetryDuration(TimeSpan.FromSeconds(15)) +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithPortForwarder(out var portForwarder) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeStartScript( +// async () => +// { +// // Kill the first StartScript call to force the rpc call into retries +// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).LastException == null) +// { +// // Ensure there is an active connection so it can be killed correctly +// await tcpConnectionUtilities.RestartTcpConnection(); +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// else +// { +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// // Kill the port forwarder so the next requests are in the connecting state when retries timeout +// Logger.Information("Killing PortForwarder"); +// portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); +// } +// else +// { +// // Ensure there is an active connection so it can be killed correctly +// await tcpConnectionUtilities.RestartTcpConnection(); +// // Pause the port forwarder so the next requests are in-flight when retries timeout +// responseMessageTcpKiller.PauseConnectionOnNextResponse(); +// } +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var inMemoryLog = new InMemoryLog(); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Print("Start Script") +// .Sleep(TimeSpan.FromHours(1)) +// .Print("End Script")) +// .Build(); +// +// var duration = Stopwatch.StartNew(); +// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// duration.Stop(); +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().BeGreaterOrEqualTo(2); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); +// +// // Ensure we actually waited and retried until the timeout policy kicked in +// duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); +// +// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); +// } +// +// [Test] +// [TentacleConfigurations] +// [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] +// public async Task WhenStartScriptFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var retryDuration = TimeSpan.FromSeconds(15); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// // Set a short retry duration so we cancel fairly quickly +// .WithRetryDuration(retryDuration) +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeStartScript( +// async () => +// { +// // Sleep to make the initial RPC call take longer than the allowed retry duration +// await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); +// +// // Kill the first StartScript call to force the rpc call into retries +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// })) +// .Build()) +// .Build(CancellationToken); +// +// var inMemoryLog = new InMemoryLog(); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Sleep(TimeSpan.FromHours(1))) +// .Build(); +// +// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); +// +// inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); +// } +// +// [Test] +// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage)})] +// public async Task WhenRpcRetriesTimeOut_DuringGetStatus_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) +// { +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// // Set a short retry duration so we cancel fairly quickly +// .WithRetryDuration(TimeSpan.FromSeconds(15)) +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithPortForwarder(out var portForwarder) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeGetStatus( +// async () => +// { +// // Kill the first GetStatus call to force the rpc call into retries +// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException == null) +// { +// // Ensure there is an active connection so it can be killed correctly +// await tcpConnectionUtilities.RestartTcpConnection(); +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// else +// { +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// // Kill the port forwarder so the next requests are in the connecting state when retries timeout +// Logger.Information("Killing PortForwarder"); +// portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); +// } +// else +// { +// // Ensure there is an active connection so it can be killed correctly +// await tcpConnectionUtilities.RestartTcpConnection(); +// // Pause the port forwarder so the next requests are in-flight when retries timeout +// responseMessageTcpKiller.PauseConnectionOnNextResponse(); +// } +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var inMemoryLog = new InMemoryLog(); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Print("Start Script") +// .Sleep(TimeSpan.FromHours(1)) +// .Print("End Script")) +// // Don't wait in start script as we want to tst get status +// .WithDurationStartScriptCanWaitForScriptToFinish(null) +// .Build(); +// +// var duration = Stopwatch.StartNew(); +// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// duration.Stop(); +// +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterOrEqualTo(2); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); +// +// // Ensure we actually waited and retried until the timeout policy kicked in +// duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); +// +// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task WhenGetStatusFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var retryDuration = TimeSpan.FromSeconds(15); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// // Set a short retry duration so we cancel fairly quickly +// .WithRetryDuration(retryDuration) +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeGetStatus( +// async () => +// { +// // Sleep to make the initial RPC call take longer than the allowed retry duration +// await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); +// +// // Kill the first GetStatus call to force the rpc call into retries +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// })) +// .Build()) +// .Build(CancellationToken); +// +// var inMemoryLog = new InMemoryLog(); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Sleep(TimeSpan.FromHours(1))) +// // Don't wait in start script as we want to test get status +// .WithDurationStartScriptCanWaitForScriptToFinish(null) +// .Build(); +// +// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); +// +// inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); +// } +// +// [Test] +// [TentacleConfigurations(additionalParameterTypes: new object[] {typeof(RpcCallStage)})] +// public async Task WhenRpcRetriesTimeOut_DuringCancelScript_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) +// { +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// // Set a short retry duration so we cancel fairly quickly +// .WithRetryDuration(TimeSpan.FromSeconds(15)) +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithPortForwarder(out var portForwarder) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeCancelScript( +// async () => +// { +// // Kill the first CancelScript call to force the rpc call into retries +// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException == null) +// { +// // Ensure there is an active connection so it can be killed correctly +// await tcpConnectionUtilities.RestartTcpConnection(); +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// else +// { +// if (rpcCallStage == RpcCallStage.Connecting) +// { +// // Kill the port forwarder so the next requests are in the connecting state when retries timeout +// Logger.Information("Killing PortForwarder"); +// portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); +// } +// else +// { +// // Ensure there is an active connection so it can be killed correctly +// await tcpConnectionUtilities.RestartTcpConnection(); +// // Pause the port forwarder so the next requests are in-flight when retries timeout +// responseMessageTcpKiller.PauseConnectionOnNextResponse(); +// } +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var inMemoryLog = new InMemoryLog(); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Print("Start Script") +// .Sleep(TimeSpan.FromHours(1)) +// .Print("End Script")) +// // Don't wait in start script as we want to tst get status +// .WithDurationStartScriptCanWaitForScriptToFinish(null) +// .Build(); +// +// // Start the script which will wait for a file to exist +// var testCancellationTokenSource = new CancellationTokenSource(); +// +// var duration = Stopwatch.StartNew(); +// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( +// startScriptCommand, +// CancellationTokenSource.CreateLinkedTokenSource(CancellationToken, testCancellationTokenSource.Token).Token, +// null, +// inMemoryLog); +// +// await Wait.For(() => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Completed > 0, +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Script execution did not complete"), +// CancellationToken); +// +// // We cancel script execution via the cancellation token. This should trigger the CancelScript RPC call to be made +// testCancellationTokenSource.Cancel(); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// duration.Stop(); +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterOrEqualTo(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().BeGreaterOrEqualTo(2); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); +// +// // Ensure we actually waited and retried until the timeout policy kicked in +// duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); +// +// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task WhenCancelScriptFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var retryDuration = TimeSpan.FromSeconds(15); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// // Set a short retry duration so we cancel fairly quickly +// .WithRetryDuration(retryDuration) +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeCancelScript( +// async () => +// { +// // Sleep to make the initial RPC call take longer than the allowed retry duration +// await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); +// +// // Kill the first CancelScript call to force the rpc call into retries +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// })) +// .Build()) +// .Build(CancellationToken); +// +// var inMemoryLog = new InMemoryLog(); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b +// .Sleep(TimeSpan.FromHours(1))) +// // Don't wait in start script as we want to test get status +// .WithDurationStartScriptCanWaitForScriptToFinish(null) +// .Build(); +// +// // Start the script which will wait for a file to exist +// var testCancellationTokenSource = new CancellationTokenSource(); +// +// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( +// startScriptCommand, +// CancellationTokenSource.CreateLinkedTokenSource(CancellationToken, testCancellationTokenSource.Token).Token, +// null, +// inMemoryLog); +// +// Func action = async () => await executeScriptTask; +// await Wait.For( +// () => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Completed > 0, +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Script Execution did not complete"), +// CancellationToken); +// +// // We cancel script execution via the cancellation token. This should trigger the CancelScript RPC call to be made +// testCancellationTokenSource.Cancel(); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterOrEqualTo(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); +// +// inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); +// } +// } +// } diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptArgumentsWork.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptArgumentsWork.cs index cdce531e7..1127d47f0 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptArgumentsWork.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptArgumentsWork.cs @@ -1,41 +1,41 @@ -using System; -using System.Threading.Tasks; -using FluentAssertions; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils.Builders; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class ClientScriptExecutionScriptArgumentsWork : IntegrationTest - { - [Test] - [TentacleConfigurations(testCommonVersions: true)] - public async Task ArgumentsArePassedToTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder().PrintArguments()) - .WithArguments(new[] { "First", "Second", "AndSpacesAreNotHandledWellInTentacle" }) - .Build(); - - var tentacleServicesDecorator = new TentacleServiceDecoratorBuilder().Build(); - - var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(0); - - var allLogs = logs.JoinLogs(); - - allLogs.Should().MatchRegex(".*Argument: First\n.*Argument: Second\n.*Argument: AndSpacesAreNotHandledWellInTentacle\n"); - } - } -} \ No newline at end of file +// using System; +// using System.Threading.Tasks; +// using FluentAssertions; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils.Builders; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class ClientScriptExecutionScriptArgumentsWork : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(testCommonVersions: true)] +// public async Task ArgumentsArePassedToTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder().PrintArguments()) +// .WithArguments(new[] { "First", "Second", "AndSpacesAreNotHandledWellInTentacle" }) +// .Build(); +// +// var tentacleServicesDecorator = new TentacleServiceDecoratorBuilder().Build(); +// +// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// +// finalResponse.State.Should().Be(ProcessState.Complete); +// finalResponse.ExitCode.Should().Be(0); +// +// var allLogs = logs.JoinLogs(); +// +// allLogs.Should().MatchRegex(".*Argument: First\n.*Argument: Second\n.*Argument: AndSpacesAreNotHandledWellInTentacle\n"); +// } +// } +// } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs index 181ceeef0..f5423050c 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs @@ -1,38 +1,38 @@ -using System; -using System.Threading.Tasks; -using FluentAssertions; -using Halibut; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils.Builders; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class ClientScriptExecutionScriptFilesAreSent : IntegrationTest - { - [Test] - [TentacleConfigurations(testCommonVersions: true)] - public async Task ScriptFilesAreSent(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder().PrintFileContents("foo.txt")) - .WithScriptFile(new ScriptFile("foo.txt", DataStream.FromString("The File Contents"))) - .Build(); - - var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(0); - - var allLogs = logs.JoinLogs(); - - allLogs.Should().Contain("The File Contents"); - } - } -} +// using System; +// using System.Threading.Tasks; +// using FluentAssertions; +// using Halibut; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils.Builders; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class ClientScriptExecutionScriptFilesAreSent : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(testCommonVersions: true)] +// public async Task ScriptFilesAreSent(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder().PrintFileContents("foo.txt")) +// .WithScriptFile(new ScriptFile("foo.txt", DataStream.FromString("The File Contents"))) +// .Build(); +// +// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// +// finalResponse.State.Should().Be(ProcessState.Complete); +// finalResponse.ExitCode.Should().Be(0); +// +// var allLogs = logs.JoinLogs(); +// +// allLogs.Should().Contain("The File Contents"); +// } +// } +// } diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled.cs index 2986460e6..3cc14d6ca 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled.cs @@ -1,292 +1,292 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils.Builders; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled : IntegrationTest - { - [Test] - [TentacleConfigurations] - public async Task WhenNetworkFailureOccurs_DuringGetCapabilities_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithRetriesDisabled() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithRetryDuration(TimeSpan.FromMinutes(4)) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var capabilitiesRecordedUsages) - .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptRecordedUsages) - .DecorateCapabilitiesServiceV2With(d => d - .BeforeGetCapabilities( - async () => - { - // Due to the GetCapabilities response getting cached, we must - // use a different service to ensure Tentacle is connected to Server. - // Otherwise, the response to the 'ensure connection' will get cached - // and any subsequent calls will succeed w/o using the network. - await tcpConnectionUtilities.RestartTcpConnection(); - - if (capabilitiesRecordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException is null) - { - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - })) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder().Print("hello")).Build(); - - var logs = new List(); - var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - var allLogs = logs.JoinLogs(); - - allLogs.Should().NotContain("hello"); - - var getCapabilitiesUsages = capabilitiesRecordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)); - getCapabilitiesUsages.LastException.Should().NotBeNull(); - getCapabilitiesUsages.Started.Should().Be(1); - getCapabilitiesUsages.Completed.Should().Be(1); - - scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); - scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Completed.Should().Be(0); - scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(0); - scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Completed.Should().Be(0); - } - - [Test] - [TentacleConfigurations] - public async Task WhenANetworkFailureOccurs_DuringStartScript_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithRetriesDisabled() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithRetryDuration(TimeSpan.FromMinutes(4)) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeStartScript( - async () => - { - await Task.CompletedTask; - - if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).LastException is null) - { - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - })) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .Print("hello") - .Print("AllDone")) - .Build(); - - var logs = new List(); - var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - var allLogs = logs.JoinLogs(); - - allLogs.Should().NotContain("AllDone"); - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).LastException.Should().NotBeNull(); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); - - // We must ensure all script are complete, otherwise if we shutdown tentacle while running a script the build can hang. - // Ensure the script is finished by running the script again - await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); - } - - [Test] - [TentacleConfigurations] - public async Task WhenANetworkFailureOccurs_DuringGetStatus_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithRetriesDisabled() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithRetryDuration(TimeSpan.FromMinutes(4)) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeGetStatus( - async () => - { - await Task.CompletedTask; - - if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException is null) - { - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - })) - .Build()) - .Build(CancellationToken); - - var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .Print("hello") - .WaitForFileToExist(waitForFile) - .Print("AllDone")) - .Build(); - - var logs = new List(); - - var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - var allLogs = logs.JoinLogs(); - - allLogs.Should().NotContain("AllDone"); - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException.Should().NotBeNull(); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); - - // Let the script finish. - File.WriteAllText(waitForFile, ""); - await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); - } - - [Test] - [TentacleConfigurations] - public async Task WhenANetworkFailureOccurs_DuringCancelScript_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithRetriesDisabled() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithRetryDuration(TimeSpan.FromMinutes(4)) - .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeGetStatus( - async () => - { - await Task.CompletedTask; - - cts.Cancel(); - }) - .BeforeCancelScript( - async () => - { - if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException is null) - { - await tcpConnectionUtilities.RestartTcpConnection(); - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - })) - .Build()) - .Build(CancellationToken); - - var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .Print("hello") - .WaitForFileToExist(waitForFile) - .Print("AllDone")) - .Build(); - - var logs = new List(); - var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, cts.Token); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - var allLogs = logs.JoinLogs(); - - allLogs.Should().NotContain("AllDone"); - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException.Should().NotBeNull(); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); - - // Let the script finish. - File.WriteAllText(waitForFile, ""); - await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); - } - - [Test] - [TentacleConfigurations] - public async Task WhenANetworkFailureOccurs_DuringCompleteScript_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithRetriesDisabled() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithRetryDuration(TimeSpan.FromMinutes(4)) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateAllScriptServicesWith(u => u - .BeforeCompleteScript( - async () => - { - await Task.CompletedTask; - - if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).LastException is null) - { - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - })) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .Print("hello") - .Print("AllDone")) - .Build(); - - var logs = new List(); - await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); - - var allLogs = logs.JoinLogs(); - allLogs.Should().Contain("AllDone"); - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).LastException.Should().NotBeNull(); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); - } - } -} \ No newline at end of file +// using System; +// using System.Collections.Generic; +// using System.IO; +// using System.Threading; +// using System.Threading.Tasks; +// using FluentAssertions; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils.Builders; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Contracts.ClientServices; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled : IntegrationTest +// { +// [Test] +// [TentacleConfigurations] +// public async Task WhenNetworkFailureOccurs_DuringGetCapabilities_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithRetriesDisabled() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithRetryDuration(TimeSpan.FromMinutes(4)) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var capabilitiesRecordedUsages) +// .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptRecordedUsages) +// .DecorateCapabilitiesServiceV2With(d => d +// .BeforeGetCapabilities( +// async () => +// { +// // Due to the GetCapabilities response getting cached, we must +// // use a different service to ensure Tentacle is connected to Server. +// // Otherwise, the response to the 'ensure connection' will get cached +// // and any subsequent calls will succeed w/o using the network. +// await tcpConnectionUtilities.RestartTcpConnection(); +// +// if (capabilitiesRecordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException is null) +// { +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder().Print("hello")).Build(); +// +// var logs = new List(); +// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// var allLogs = logs.JoinLogs(); +// +// allLogs.Should().NotContain("hello"); +// +// var getCapabilitiesUsages = capabilitiesRecordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)); +// getCapabilitiesUsages.LastException.Should().NotBeNull(); +// getCapabilitiesUsages.Started.Should().Be(1); +// getCapabilitiesUsages.Completed.Should().Be(1); +// +// scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); +// scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Completed.Should().Be(0); +// scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(0); +// scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Completed.Should().Be(0); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task WhenANetworkFailureOccurs_DuringStartScript_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithRetriesDisabled() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithRetryDuration(TimeSpan.FromMinutes(4)) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeStartScript( +// async () => +// { +// await Task.CompletedTask; +// +// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).LastException is null) +// { +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .Print("hello") +// .Print("AllDone")) +// .Build(); +// +// var logs = new List(); +// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// var allLogs = logs.JoinLogs(); +// +// allLogs.Should().NotContain("AllDone"); +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).LastException.Should().NotBeNull(); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); +// +// // We must ensure all script are complete, otherwise if we shutdown tentacle while running a script the build can hang. +// // Ensure the script is finished by running the script again +// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task WhenANetworkFailureOccurs_DuringGetStatus_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithRetriesDisabled() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithRetryDuration(TimeSpan.FromMinutes(4)) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeGetStatus( +// async () => +// { +// await Task.CompletedTask; +// +// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException is null) +// { +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .Print("hello") +// .WaitForFileToExist(waitForFile) +// .Print("AllDone")) +// .Build(); +// +// var logs = new List(); +// +// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// var allLogs = logs.JoinLogs(); +// +// allLogs.Should().NotContain("AllDone"); +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException.Should().NotBeNull(); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); +// +// // Let the script finish. +// File.WriteAllText(waitForFile, ""); +// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task WhenANetworkFailureOccurs_DuringCancelScript_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithRetriesDisabled() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithRetryDuration(TimeSpan.FromMinutes(4)) +// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeGetStatus( +// async () => +// { +// await Task.CompletedTask; +// +// cts.Cancel(); +// }) +// .BeforeCancelScript( +// async () => +// { +// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException is null) +// { +// await tcpConnectionUtilities.RestartTcpConnection(); +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .Print("hello") +// .WaitForFileToExist(waitForFile) +// .Print("AllDone")) +// .Build(); +// +// var logs = new List(); +// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, cts.Token); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// var allLogs = logs.JoinLogs(); +// +// allLogs.Should().NotContain("AllDone"); +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException.Should().NotBeNull(); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); +// +// // Let the script finish. +// File.WriteAllText(waitForFile, ""); +// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task WhenANetworkFailureOccurs_DuringCompleteScript_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithRetriesDisabled() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithRetryDuration(TimeSpan.FromMinutes(4)) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateAllScriptServicesWith(u => u +// .BeforeCompleteScript( +// async () => +// { +// await Task.CompletedTask; +// +// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).LastException is null) +// { +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// })) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .Print("hello") +// .Print("AllDone")) +// .Build(); +// +// var logs = new List(); +// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); +// +// var allLogs = logs.JoinLogs(); +// allLogs.Should().Contain("AllDone"); +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).LastException.Should().NotBeNull(); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); +// } +// } +// } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceV1IsNotRetried.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceV1IsNotRetried.cs index 4d117c972..8be5b7864 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceV1IsNotRetried.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceV1IsNotRetried.cs @@ -1,251 +1,251 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using NUnit.Framework; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class ClientScriptExecutionScriptServiceV1IsNotRetried : IntegrationTest - { - [Test] - [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] - public async Task WhenANetworkFailureOccurs_DuringStartScript_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithRetryDuration(TimeSpan.FromMinutes(4)) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var recordedUsages) - .HookServiceMethod( - nameof(IAsyncClientScriptService.StartScriptAsync), - async (_, _) => - { - await Task.CompletedTask; - - if (recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).LastException == null) - { - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - }) - .Build()) - .Build(CancellationToken); - - var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .Print("hello") - .WaitForFileToExist(waitForFile) - .Print("AllDone")) - .WithIsolationLevel(ScriptIsolationLevel.FullIsolation) - .WithIsolationMutexName("bob") - .Build(); - - var logs = new List(); - var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - // Let the script finish. - File.WriteAllText(waitForFile, ""); - - var allLogs = logs.JoinLogs(); - - allLogs.Should().NotContain("AllDone"); - recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).LastException.Should().NotBeNull(); - recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).Started.Should().Be(0); - recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(0); - - // We must ensure all script are complete, otherwise if we shutdown tentacle while running a script the build can hang. - // Ensure the script is finished by running the script again, the isolation mutex will ensure this second script runs after the first is complete. - await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); - } - - [Test] - [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] - public async Task WhenANetworkFailureOccurs_DuringGetStatus_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - ScriptStatusRequest? scriptStatusRequest = null; - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithRetryDuration(TimeSpan.FromMinutes(4)) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var recordedUsages) - .HookServiceMethod( - nameof(IAsyncClientScriptService.GetStatusAsync), - async (_, request) => - { - await Task.CompletedTask; - - scriptStatusRequest = request; - if (recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).LastException == null) - { - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - }) - .Build()) - .Build(CancellationToken); - - var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .Print("hello") - .WaitForFileToExist(waitForFile) - .Print("AllDone")) - .Build(); - - var logs = new List(); - var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - // Let the script finish. - File.WriteAllText(waitForFile, ""); - var legacyTentacleClient = clientTentacle.LegacyTentacleClientBuilder().Build(); - - await Wait.For( - async () => (await legacyTentacleClient.ScriptService.GetStatusAsync(scriptStatusRequest, new(CancellationToken))).State == ProcessState.Complete, - TimeSpan.FromSeconds(60), - () => throw new Exception("Script Execution did not complete"), - CancellationToken); - - var allLogs = logs.JoinLogs(); - - allLogs.Should().NotContain("AllDone"); - recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).LastException.Should().NotBeNull(); - recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(0); - } - - [Test] - [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] - public async Task WhenANetworkFailureOccurs_DuringCancelScript_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - ScriptStatusRequest? scriptStatusRequest = null; - CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithRetryDuration(TimeSpan.FromMinutes(4)) - .WithPortForwarderDataLogging() - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var recordedUsages) - .HookServiceMethod( - nameof(IAsyncClientScriptService.GetStatusAsync), - async (_, request) => - { - await Task.CompletedTask; - - cts.Cancel(); - scriptStatusRequest = request; - }) - .HookServiceMethod( - nameof(IAsyncClientScriptService.CancelScriptAsync), - async (_, _) => - { - await Task.CompletedTask; - - if (recordedUsages.For(nameof(IAsyncClientScriptService.CancelScriptAsync)).LastException == null) - { - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - }) - .Build()) - .Build(CancellationToken); - - var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .Print("hello") - .WaitForFileToExist(waitForFile) - .Print("AllDone")) - .Build(); - - var logs = new List(); - - var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, cts.Token); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - var allLogs = logs.JoinLogs(); - allLogs.Should().NotContain("AllDone"); - recordedUsages.For(nameof(IAsyncClientScriptService.CancelScriptAsync)).LastException.Should().NotBeNull(); - recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptService.CancelScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(0); - - // Let the script finish. - File.WriteAllText(waitForFile, ""); - var legacyTentacleClient = clientTentacle.LegacyTentacleClientBuilder().Build(); - - await Wait.For( - async () => (await legacyTentacleClient.ScriptService.GetStatusAsync(scriptStatusRequest, new(CancellationToken))).State == ProcessState.Complete, - TimeSpan.FromSeconds(60), - () => throw new Exception("Script Execution did not complete"), - CancellationToken); - } - - [Test] - [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] - public async Task WhenANetworkFailureOccurs_DuringCompleteScript_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithPortForwarderDataLogging() - .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) - .WithRetryDuration(TimeSpan.FromMinutes(4)) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var recordedUsages) - .HookServiceMethod( - nameof(IAsyncClientScriptService.CompleteScriptAsync), - async (_, _) => - { - await Task.CompletedTask; - - if (recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).LastException == null) - { - responseMessageTcpKiller.KillConnectionOnNextResponse(); - } - }) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(new ScriptBuilder().Print("hello")).Build(); - - var logs = new List(); - var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); - - var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); - - await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); - - // We Can not verify what will be in the logs because of race conditions in tentacle. - // The last complete script which we fail might come back with the logs. - recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).LastException.Should().NotBeNull(); - recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(1); - } - } -} \ No newline at end of file +// using System; +// using System.Collections.Generic; +// using System.IO; +// using System.Threading; +// using System.Threading.Tasks; +// using FluentAssertions; +// using NUnit.Framework; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Contracts.ClientServices; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class ClientScriptExecutionScriptServiceV1IsNotRetried : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] +// public async Task WhenANetworkFailureOccurs_DuringStartScript_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithRetryDuration(TimeSpan.FromMinutes(4)) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var recordedUsages) +// .HookServiceMethod( +// nameof(IAsyncClientScriptService.StartScriptAsync), +// async (_, _) => +// { +// await Task.CompletedTask; +// +// if (recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).LastException == null) +// { +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// }) +// .Build()) +// .Build(CancellationToken); +// +// var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .Print("hello") +// .WaitForFileToExist(waitForFile) +// .Print("AllDone")) +// .WithIsolationLevel(ScriptIsolationLevel.FullIsolation) +// .WithIsolationMutexName("bob") +// .Build(); +// +// var logs = new List(); +// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// // Let the script finish. +// File.WriteAllText(waitForFile, ""); +// +// var allLogs = logs.JoinLogs(); +// +// allLogs.Should().NotContain("AllDone"); +// recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).LastException.Should().NotBeNull(); +// recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).Started.Should().Be(0); +// recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(0); +// +// // We must ensure all script are complete, otherwise if we shutdown tentacle while running a script the build can hang. +// // Ensure the script is finished by running the script again, the isolation mutex will ensure this second script runs after the first is complete. +// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); +// } +// +// [Test] +// [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] +// public async Task WhenANetworkFailureOccurs_DuringGetStatus_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// ScriptStatusRequest? scriptStatusRequest = null; +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithRetryDuration(TimeSpan.FromMinutes(4)) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var recordedUsages) +// .HookServiceMethod( +// nameof(IAsyncClientScriptService.GetStatusAsync), +// async (_, request) => +// { +// await Task.CompletedTask; +// +// scriptStatusRequest = request; +// if (recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).LastException == null) +// { +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// }) +// .Build()) +// .Build(CancellationToken); +// +// var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .Print("hello") +// .WaitForFileToExist(waitForFile) +// .Print("AllDone")) +// .Build(); +// +// var logs = new List(); +// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// // Let the script finish. +// File.WriteAllText(waitForFile, ""); +// var legacyTentacleClient = clientTentacle.LegacyTentacleClientBuilder().Build(); +// +// await Wait.For( +// async () => (await legacyTentacleClient.ScriptService.GetStatusAsync(scriptStatusRequest, new(CancellationToken))).State == ProcessState.Complete, +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Script Execution did not complete"), +// CancellationToken); +// +// var allLogs = logs.JoinLogs(); +// +// allLogs.Should().NotContain("AllDone"); +// recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).LastException.Should().NotBeNull(); +// recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(0); +// } +// +// [Test] +// [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] +// public async Task WhenANetworkFailureOccurs_DuringCancelScript_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// ScriptStatusRequest? scriptStatusRequest = null; +// CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithRetryDuration(TimeSpan.FromMinutes(4)) +// .WithPortForwarderDataLogging() +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var recordedUsages) +// .HookServiceMethod( +// nameof(IAsyncClientScriptService.GetStatusAsync), +// async (_, request) => +// { +// await Task.CompletedTask; +// +// cts.Cancel(); +// scriptStatusRequest = request; +// }) +// .HookServiceMethod( +// nameof(IAsyncClientScriptService.CancelScriptAsync), +// async (_, _) => +// { +// await Task.CompletedTask; +// +// if (recordedUsages.For(nameof(IAsyncClientScriptService.CancelScriptAsync)).LastException == null) +// { +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// }) +// .Build()) +// .Build(CancellationToken); +// +// var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .Print("hello") +// .WaitForFileToExist(waitForFile) +// .Print("AllDone")) +// .Build(); +// +// var logs = new List(); +// +// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, cts.Token); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// var allLogs = logs.JoinLogs(); +// allLogs.Should().NotContain("AllDone"); +// recordedUsages.For(nameof(IAsyncClientScriptService.CancelScriptAsync)).LastException.Should().NotBeNull(); +// recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptService.CancelScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(0); +// +// // Let the script finish. +// File.WriteAllText(waitForFile, ""); +// var legacyTentacleClient = clientTentacle.LegacyTentacleClientBuilder().Build(); +// +// await Wait.For( +// async () => (await legacyTentacleClient.ScriptService.GetStatusAsync(scriptStatusRequest, new(CancellationToken))).State == ProcessState.Complete, +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Script Execution did not complete"), +// CancellationToken); +// } +// +// [Test] +// [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] +// public async Task WhenANetworkFailureOccurs_DuringCompleteScript_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithPortForwarderDataLogging() +// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) +// .WithRetryDuration(TimeSpan.FromMinutes(4)) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var recordedUsages) +// .HookServiceMethod( +// nameof(IAsyncClientScriptService.CompleteScriptAsync), +// async (_, _) => +// { +// await Task.CompletedTask; +// +// if (recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).LastException == null) +// { +// responseMessageTcpKiller.KillConnectionOnNextResponse(); +// } +// }) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(new ScriptBuilder().Print("hello")).Build(); +// +// var logs = new List(); +// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); +// +// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); +// +// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); +// +// // We Can not verify what will be in the logs because of race conditions in tentacle. +// // The last complete script which we fail might come back with the logs. +// recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).LastException.Should().NotBeNull(); +// recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(1); +// } +// } +// } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionWorksWithMultipleVersions.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionWorksWithMultipleVersions.cs index a28ed3bde..e10dcbfa4 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionWorksWithMultipleVersions.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionWorksWithMultipleVersions.cs @@ -1,39 +1,39 @@ -using System; -using System.Threading.Tasks; -using FluentAssertions; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils.Builders; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class ClientScriptExecutionWorksWithMultipleVersions : IntegrationTest - { - [Test] - [TentacleConfigurations(testCommonVersions: true)] - public async Task CanRunScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .Print("hello") - .PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1)) - .Print("AllDone")) - .Build(); - - var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(0); - - var allLogs = logs.JoinLogs(); - - allLogs.Should().MatchRegex(".*hello\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nAllDone"); - } - } -} +// using System; +// using System.Threading.Tasks; +// using FluentAssertions; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils.Builders; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class ClientScriptExecutionWorksWithMultipleVersions : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(testCommonVersions: true)] +// public async Task CanRunScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .Print("hello") +// .PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1)) +// .Print("AllDone")) +// .Build(); +// +// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// +// finalResponse.State.Should().Be(ProcessState.Complete); +// finalResponse.ExitCode.Should().Be(0); +// +// var allLogs = logs.JoinLogs(); +// +// allLogs.Should().MatchRegex(".*hello\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nAllDone"); +// } +// } +// } diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs index 949d586ae..839741063 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs @@ -1,44 +1,44 @@ -using System; -using System.Threading.Tasks; -using FluentAssertions; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils.Builders; -using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] - public class ClientScriptExecutorObservesScriptObserverBackoffStrategy : IntegrationTest - { - [Test] - [TentacleConfigurations(testCommonVersions: true)] - public async Task TheScriptObserverBackoffShouldBeRespected(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithScriptObserverBackoffStrategy(new FuncScriptObserverBackoffStrategy(iters => TimeSpan.FromSeconds(20))) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder().PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1))) - .Build(); - - var (_, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started - .Should() - .BeGreaterThan(0) - .And - .BeLessThan(3); - } - } -} +// using System; +// using System.Threading.Tasks; +// using FluentAssertions; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils.Builders; +// using Octopus.Tentacle.Contracts.ClientServices; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] +// public class ClientScriptExecutorObservesScriptObserverBackoffStrategy : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(testCommonVersions: true)] +// public async Task TheScriptObserverBackoffShouldBeRespected(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithScriptObserverBackoffStrategy(new FuncScriptObserverBackoffStrategy(iters => TimeSpan.FromSeconds(20))) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder().PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1))) +// .Build(); +// +// var (_, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started +// .Should() +// .BeGreaterThan(0) +// .And +// .BeLessThan(3); +// } +// } +// } diff --git a/source/Octopus.Tentacle.Tests.Integration/MachineConfigurationHomeDirectoryTests.cs b/source/Octopus.Tentacle.Tests.Integration/MachineConfigurationHomeDirectoryTests.cs index aabcde625..b0d1558be 100644 --- a/source/Octopus.Tentacle.Tests.Integration/MachineConfigurationHomeDirectoryTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/MachineConfigurationHomeDirectoryTests.cs @@ -1,54 +1,54 @@ -using System; -using System.IO; -using System.Threading.Tasks; -using FluentAssertions; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Variables; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class MachineConfigurationHomeDirectoryTests : IntegrationTest - { - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task ShouldUseTheDefaultMachineConfigurationHomeDirectoryWhenACustomLocationIsNotProvided(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using (var clientAndTentacle = await tentacleConfigurationTestCase - .CreateBuilder() - .UseDefaultMachineConfigurationHomeDirectory() - .Build(CancellationToken)) - { - - var defaultMachineConfigurationHomeDirectory = PlatformDetection.IsRunningOnWindows ? - Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), "Octopus", "Tentacle", "Instances") : - "/etc/octopus/Tentacle/Instances"; - - var expectedInstanceConfigurationFilePath = new FileInfo(Path.Combine(defaultMachineConfigurationHomeDirectory, $"{clientAndTentacle.RunningTentacle.InstanceName}.config")); - - expectedInstanceConfigurationFilePath.Exists.Should().BeTrue($"Instance configuration file should exist {expectedInstanceConfigurationFilePath.FullName}"); - } - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task ShouldUseTheCustomMachineConfigurationHomeDirectoryWhenACustomLocationIsProvided(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - using var tempDirectory = new TemporaryDirectory(); - await using var clientAndTentacle = await tentacleConfigurationTestCase - .CreateBuilder() - .UseDefaultMachineConfigurationHomeDirectory() - .WithTentacle(x => - { - x.WithRunTentacleEnvironmentVariable(EnvironmentVariables.TentacleMachineConfigurationHomeDirectory, tempDirectory.DirectoryPath); - }) - .Build(CancellationToken); - - var expectedInstanceConfigurationFilePath = new FileInfo(Path.Combine(tempDirectory.DirectoryPath, "Tentacle", "Instances", $"{clientAndTentacle.RunningTentacle.InstanceName}.config")); - - expectedInstanceConfigurationFilePath.Exists.Should().BeTrue($"Instance configuration file should exist {expectedInstanceConfigurationFilePath.FullName}"); - } - } -} +// using System; +// using System.IO; +// using System.Threading.Tasks; +// using FluentAssertions; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Variables; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class MachineConfigurationHomeDirectoryTests : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task ShouldUseTheDefaultMachineConfigurationHomeDirectoryWhenACustomLocationIsNotProvided(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using (var clientAndTentacle = await tentacleConfigurationTestCase +// .CreateBuilder() +// .UseDefaultMachineConfigurationHomeDirectory() +// .Build(CancellationToken)) +// { +// +// var defaultMachineConfigurationHomeDirectory = PlatformDetection.IsRunningOnWindows ? +// Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), "Octopus", "Tentacle", "Instances") : +// "/etc/octopus/Tentacle/Instances"; +// +// var expectedInstanceConfigurationFilePath = new FileInfo(Path.Combine(defaultMachineConfigurationHomeDirectory, $"{clientAndTentacle.RunningTentacle.InstanceName}.config")); +// +// expectedInstanceConfigurationFilePath.Exists.Should().BeTrue($"Instance configuration file should exist {expectedInstanceConfigurationFilePath.FullName}"); +// } +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task ShouldUseTheCustomMachineConfigurationHomeDirectoryWhenACustomLocationIsProvided(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// using var tempDirectory = new TemporaryDirectory(); +// await using var clientAndTentacle = await tentacleConfigurationTestCase +// .CreateBuilder() +// .UseDefaultMachineConfigurationHomeDirectory() +// .WithTentacle(x => +// { +// x.WithRunTentacleEnvironmentVariable(EnvironmentVariables.TentacleMachineConfigurationHomeDirectory, tempDirectory.DirectoryPath); +// }) +// .Build(CancellationToken); +// +// var expectedInstanceConfigurationFilePath = new FileInfo(Path.Combine(tempDirectory.DirectoryPath, "Tentacle", "Instances", $"{clientAndTentacle.RunningTentacle.InstanceName}.config")); +// +// expectedInstanceConfigurationFilePath.Exists.Should().BeTrue($"Instance configuration file should exist {expectedInstanceConfigurationFilePath.FullName}"); +// } +// } +// } diff --git a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceTests.cs b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceTests.cs index b855caeb9..7d1847411 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceTests.cs @@ -1,136 +1,136 @@ -using System; -using System.Diagnostics; -using System.Linq; -using System.Threading.Tasks; -using FluentAssertions; -using Halibut.Logging; -using NUnit.Framework; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Support.Legacy; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class ScriptServiceTests : IntegrationTest - { - [Test] - [TentacleConfigurations] - public async Task RunScriptWithSuccess(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var windowsScript = @" - Write-Host ""This is the start of the script"" - Write-Host ""The answer is"" (6 * 7) - Start-Sleep -Seconds 3 - Write-Host ""This is the end of the script"""; - - var nixScript = @" - echo This is the start of the script - val=6 - ((theAnswer=$val*7)) - echo The answer is $theAnswer - sleep 3 - echo This is the end of the script"; - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); - - var scriptStatusResponse = await new ScriptExecutionOrchestrator(clientAndTentacle.TentacleClient, Logger) - .ExecuteScript(windowsScript, nixScript, CancellationToken); - - DumpLog(scriptStatusResponse); - - scriptStatusResponse.State.Should().Be(ProcessState.Complete); - scriptStatusResponse.ExitCode.Should().Be(0); - scriptStatusResponse.Logs.Select(x => x.Text).Should().Contain("The answer is 42"); - } - - [Test] - [TentacleConfigurations] - public async Task RunScriptWithErrors(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var windowsScript = @" - Write-Host ""This is the start of the script"" - Start-Sleep -Seconds 3 - throw ""Whoopsy Daisy!"" - Write-Host ""This is the end of the script"""; - - var nixScript = @" - echo This is the start of the script - sleep 3 - echo ""Whoopsy Daisy!"" - exit 1 - echo This is the end of the script"; - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); - - var scriptStatusResponse = await new ScriptExecutionOrchestrator(clientAndTentacle.TentacleClient, Logger) - .ExecuteScript(windowsScript, nixScript, CancellationToken); - - DumpLog(scriptStatusResponse); - - scriptStatusResponse.State.Should().Be(ProcessState.Complete); - scriptStatusResponse.ExitCode.Should().NotBe(0); - scriptStatusResponse.Logs.Select(x => x.Text).Should().Contain("Whoopsy Daisy!"); - scriptStatusResponse.Logs.Select(x => x.Text).Should().NotContain("This is the end of the script"); - } - - [Test] - [TentacleConfigurations] - public async Task CancelScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var windowsScript = @"Write-Host ""This is the start of the script"" - & ping.exe 127.0.0.1 -n 100 - Write-Host ""This is the end of the script"""; - - var nixScript = @"echo This is the start of the script - ping 127.0.0.1 -c 100 - echo This is the end of the script"; - - await using var clientAndTentacle = await tentacleConfigurationTestCase - .CreateLegacyBuilder() - .WithHalibutLoggingLevel(LogLevel.Trace) - .Build(CancellationToken); - - var scriptExecutor = new ScriptExecutionOrchestrator(clientAndTentacle.TentacleClient, Logger); - - Logger.Information("Starting script execution"); - var ticket = await scriptExecutor.StartScript(windowsScript, nixScript, CancellationToken); - - // Possible Tentacle BUG: If we just observe until the first output is received then sometimes the script will fail to Cancel - await scriptExecutor.ObserverUntilScriptOutputReceived(ticket, "This is the start of the script", CancellationToken); - - Logger.Information("Cancelling script execution"); - await clientAndTentacle.TentacleClient.ScriptService.CancelScriptAsync(new CancelScriptCommand(ticket, 0), new(CancellationToken)); - - var cancellationDuration = Stopwatch.StartNew(); - - Logger.Information("Waiting for Script Execution to complete"); - var finalScriptStatusResponse = await scriptExecutor.ObserverUntilComplete(ticket, CancellationToken); - cancellationDuration.Stop(); - - Logger.Information("Completing script execution"); - var finalStatus = await scriptExecutor.CompleteScript(finalScriptStatusResponse, CancellationToken); - - DumpLog(finalStatus); - - finalStatus.State.Should().Be(ProcessState.Complete); - finalStatus.ExitCode.Should().NotBe(0, "Expected ExitCode to be non-zero"); - finalStatus.Logs.Count.Should().BeGreaterThan(0, "Expected something in the logs"); - - finalStatus.Logs.Select(x => x.Text).Should().Contain("This is the start of the script"); - finalStatus.Logs.Select(x => x.Text).Should().NotContain("This is the end of the script"); - cancellationDuration.Elapsed.TotalSeconds.Should().BeLessThanOrEqualTo(20); - } - - private static void DumpLog(ScriptStatusResponse finalStatus) - { - Console.WriteLine("### Start of script result logs ###"); - foreach (var log in finalStatus.Logs) - { - Console.WriteLine(log.Text); - } - - Console.WriteLine("### End of script result logs ###"); - } - } -} +// using System; +// using System.Diagnostics; +// using System.Linq; +// using System.Threading.Tasks; +// using FluentAssertions; +// using Halibut.Logging; +// using NUnit.Framework; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Support.Legacy; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class ScriptServiceTests : IntegrationTest +// { +// [Test] +// [TentacleConfigurations] +// public async Task RunScriptWithSuccess(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var windowsScript = @" +// Write-Host ""This is the start of the script"" +// Write-Host ""The answer is"" (6 * 7) +// Start-Sleep -Seconds 3 +// Write-Host ""This is the end of the script"""; +// +// var nixScript = @" +// echo This is the start of the script +// val=6 +// ((theAnswer=$val*7)) +// echo The answer is $theAnswer +// sleep 3 +// echo This is the end of the script"; +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); +// +// var scriptStatusResponse = await new ScriptExecutionOrchestrator(clientAndTentacle.TentacleClient, Logger) +// .ExecuteScript(windowsScript, nixScript, CancellationToken); +// +// DumpLog(scriptStatusResponse); +// +// scriptStatusResponse.State.Should().Be(ProcessState.Complete); +// scriptStatusResponse.ExitCode.Should().Be(0); +// scriptStatusResponse.Logs.Select(x => x.Text).Should().Contain("The answer is 42"); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task RunScriptWithErrors(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var windowsScript = @" +// Write-Host ""This is the start of the script"" +// Start-Sleep -Seconds 3 +// throw ""Whoopsy Daisy!"" +// Write-Host ""This is the end of the script"""; +// +// var nixScript = @" +// echo This is the start of the script +// sleep 3 +// echo ""Whoopsy Daisy!"" +// exit 1 +// echo This is the end of the script"; +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); +// +// var scriptStatusResponse = await new ScriptExecutionOrchestrator(clientAndTentacle.TentacleClient, Logger) +// .ExecuteScript(windowsScript, nixScript, CancellationToken); +// +// DumpLog(scriptStatusResponse); +// +// scriptStatusResponse.State.Should().Be(ProcessState.Complete); +// scriptStatusResponse.ExitCode.Should().NotBe(0); +// scriptStatusResponse.Logs.Select(x => x.Text).Should().Contain("Whoopsy Daisy!"); +// scriptStatusResponse.Logs.Select(x => x.Text).Should().NotContain("This is the end of the script"); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task CancelScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var windowsScript = @"Write-Host ""This is the start of the script"" +// & ping.exe 127.0.0.1 -n 100 +// Write-Host ""This is the end of the script"""; +// +// var nixScript = @"echo This is the start of the script +// ping 127.0.0.1 -c 100 +// echo This is the end of the script"; +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase +// .CreateLegacyBuilder() +// .WithHalibutLoggingLevel(LogLevel.Trace) +// .Build(CancellationToken); +// +// var scriptExecutor = new ScriptExecutionOrchestrator(clientAndTentacle.TentacleClient, Logger); +// +// Logger.Information("Starting script execution"); +// var ticket = await scriptExecutor.StartScript(windowsScript, nixScript, CancellationToken); +// +// // Possible Tentacle BUG: If we just observe until the first output is received then sometimes the script will fail to Cancel +// await scriptExecutor.ObserverUntilScriptOutputReceived(ticket, "This is the start of the script", CancellationToken); +// +// Logger.Information("Cancelling script execution"); +// await clientAndTentacle.TentacleClient.ScriptService.CancelScriptAsync(new CancelScriptCommand(ticket, 0), new(CancellationToken)); +// +// var cancellationDuration = Stopwatch.StartNew(); +// +// Logger.Information("Waiting for Script Execution to complete"); +// var finalScriptStatusResponse = await scriptExecutor.ObserverUntilComplete(ticket, CancellationToken); +// cancellationDuration.Stop(); +// +// Logger.Information("Completing script execution"); +// var finalStatus = await scriptExecutor.CompleteScript(finalScriptStatusResponse, CancellationToken); +// +// DumpLog(finalStatus); +// +// finalStatus.State.Should().Be(ProcessState.Complete); +// finalStatus.ExitCode.Should().NotBe(0, "Expected ExitCode to be non-zero"); +// finalStatus.Logs.Count.Should().BeGreaterThan(0, "Expected something in the logs"); +// +// finalStatus.Logs.Select(x => x.Text).Should().Contain("This is the start of the script"); +// finalStatus.Logs.Select(x => x.Text).Should().NotContain("This is the end of the script"); +// cancellationDuration.Elapsed.TotalSeconds.Should().BeLessThanOrEqualTo(20); +// } +// +// private static void DumpLog(ScriptStatusResponse finalStatus) +// { +// Console.WriteLine("### Start of script result logs ###"); +// foreach (var log in finalStatus.Logs) +// { +// Console.WriteLine(log.Text); +// } +// +// Console.WriteLine("### End of script result logs ###"); +// } +// } +// } diff --git a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs index 816318950..ef9bdeaba 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs @@ -1,216 +1,216 @@ -using System; -using System.Diagnostics; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils.Builders; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class ScriptServiceV2IntegrationTest : IntegrationTest - { - [Test] - [TentacleConfigurations] - public async Task CanRunScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var methodUsages) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .Print("Lets do it") - .PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1)) - .Print("All done")) - .Build(); - - var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(0); - - var allLogs = logs.JoinLogs(); - - allLogs.Should().MatchRegex(".*Lets do it\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nAll done.*"); - - methodUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); - methodUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterThan(2).And.BeLessThan(30); - methodUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); - methodUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); - } - - [Test] - [TentacleConfigurations] - public async Task DelayInStartScriptSavesNetworkCalls(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var recordedUsages) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .Print("Lets do it") - .PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1)) - .Print("All done")) - .WithDurationStartScriptCanWaitForScriptToFinish(TimeSpan.FromMinutes(1)) - .Build(); - - var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(0); - - var allLogs = logs.JoinLogs(); - - allLogs.Should().MatchRegex(".*Lets do it\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nAll done.*"); - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0, "Since start script should wait for the script to finish so we don't need to call get status"); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); - } - - [Test] - [TentacleConfigurations] - public async Task WhenTentacleRestartsWhileRunningAScript_TheExitCodeShouldBe_UnknownResultExitCode(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var recordedUsages) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .Print("hello") - .Sleep(TimeSpan.FromSeconds(1)) - .Print("waitingtobestopped") - .Sleep(TimeSpan.FromSeconds(100))) - .Build(); - - var semaphoreSlim = new SemaphoreSlim(0, 1); - - var executingScript = Task.Run(async () => - await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, onScriptStatusResponseReceived => - { - if (onScriptStatusResponseReceived.Logs.JoinLogs().Contains("waitingtobestopped")) - { - semaphoreSlim.Release(); - } - })); - - await semaphoreSlim.WaitAsync(CancellationToken); - - Logger.Information("Stopping and starting tentacle now."); - await clientTentacle.RunningTentacle.Restart(CancellationToken); - - var (finalResponse, logs) = await executingScript; - - finalResponse.Should().NotBeNull(); - logs.JoinLogs().Should().Contain("waitingtobestopped"); - finalResponse.State.Should().Be(ProcessState.Complete); // This is technically a lie, the process is still running on linux - finalResponse.ExitCode.Should().Be(ScriptExitCodes.UnknownResultExitCode); - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterThan(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); - } - - [Test] - [TentacleConfigurations] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] - public async Task WhenALongRunningScriptIsCancelled_TheScriptShouldStop(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(out var recordedUsages) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder() - .Print("hello") - .Sleep(TimeSpan.FromSeconds(1)) - .Print("waitingtobestopped") - .Sleep(TimeSpan.FromSeconds(100))) - .Build(); - - var scriptCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); - var stopWatch = Stopwatch.StartNew(); - Exception? actualException = null; - - try - { - await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, scriptCancellationTokenSource.Token, onScriptStatusResponseReceived => - { - if (onScriptStatusResponseReceived.Logs.JoinLogs().Contains("waitingtobestopped")) - { - scriptCancellationTokenSource.Cancel(); - } - }); - } - catch (Exception ex) - { - actualException = ex; - } - - stopWatch.Stop(); - - actualException.Should().NotBeNull().And.BeOfType().And.Match(x => x.Message == "Script execution was cancelled"); - stopWatch.Elapsed.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(10)); - - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterThan(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().BeGreaterThanOrEqualTo(1); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.Version2)] - public async Task WhenOnCompleteTakesLongerThan_OnCancellationAbandonCompleteScriptAfter_AndTheExecutionIsNotCancelled_TheOrchestratorWaitsForCleanUpToComplete(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - bool calledWithNonCancelledCT = false; - - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .DecorateScriptServiceV2With(b => b - .BeforeCompleteScript(async (_, _, halibutProxyRequestOptions) => - { - await Task.Delay(TimeSpan.FromSeconds(2), CancellationToken); - calledWithNonCancelledCT = !halibutProxyRequestOptions.RequestCancellationToken.IsCancellationRequested; - }) - .Build()) - .Build()) - .Build(CancellationToken); - - var scriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b.Print("Hello")) - .Build(); - - var tentacleClient = clientTentacle.TentacleClient; - - tentacleClient.OnCancellationAbandonCompleteScriptAfter = TimeSpan.FromMilliseconds(1); - - await tentacleClient.ExecuteScript(scriptCommand, CancellationToken); - - - calledWithNonCancelledCT.Should().Be(true); - } - } -} +// using System; +// using System.Diagnostics; +// using System.Threading; +// using System.Threading.Tasks; +// using FluentAssertions; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils.Builders; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Contracts.ClientServices; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class ScriptServiceV2IntegrationTest : IntegrationTest +// { +// [Test] +// [TentacleConfigurations] +// public async Task CanRunScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var methodUsages) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .Print("Lets do it") +// .PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1)) +// .Print("All done")) +// .Build(); +// +// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// +// finalResponse.State.Should().Be(ProcessState.Complete); +// finalResponse.ExitCode.Should().Be(0); +// +// var allLogs = logs.JoinLogs(); +// +// allLogs.Should().MatchRegex(".*Lets do it\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nAll done.*"); +// +// methodUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); +// methodUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterThan(2).And.BeLessThan(30); +// methodUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); +// methodUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task DelayInStartScriptSavesNetworkCalls(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var recordedUsages) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .Print("Lets do it") +// .PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1)) +// .Print("All done")) +// .WithDurationStartScriptCanWaitForScriptToFinish(TimeSpan.FromMinutes(1)) +// .Build(); +// +// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// +// finalResponse.State.Should().Be(ProcessState.Complete); +// finalResponse.ExitCode.Should().Be(0); +// +// var allLogs = logs.JoinLogs(); +// +// allLogs.Should().MatchRegex(".*Lets do it\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nAll done.*"); +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0, "Since start script should wait for the script to finish so we don't need to call get status"); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task WhenTentacleRestartsWhileRunningAScript_TheExitCodeShouldBe_UnknownResultExitCode(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var recordedUsages) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .Print("hello") +// .Sleep(TimeSpan.FromSeconds(1)) +// .Print("waitingtobestopped") +// .Sleep(TimeSpan.FromSeconds(100))) +// .Build(); +// +// var semaphoreSlim = new SemaphoreSlim(0, 1); +// +// var executingScript = Task.Run(async () => +// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, onScriptStatusResponseReceived => +// { +// if (onScriptStatusResponseReceived.Logs.JoinLogs().Contains("waitingtobestopped")) +// { +// semaphoreSlim.Release(); +// } +// })); +// +// await semaphoreSlim.WaitAsync(CancellationToken); +// +// Logger.Information("Stopping and starting tentacle now."); +// await clientTentacle.RunningTentacle.Restart(CancellationToken); +// +// var (finalResponse, logs) = await executingScript; +// +// finalResponse.Should().NotBeNull(); +// logs.JoinLogs().Should().Contain("waitingtobestopped"); +// finalResponse.State.Should().Be(ProcessState.Complete); // This is technically a lie, the process is still running on linux +// finalResponse.ExitCode.Should().Be(ScriptExitCodes.UnknownResultExitCode); +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterThan(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); +// } +// +// [Test] +// [TentacleConfigurations] +// [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] +// public async Task WhenALongRunningScriptIsCancelled_TheScriptShouldStop(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(out var recordedUsages) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder() +// .Print("hello") +// .Sleep(TimeSpan.FromSeconds(1)) +// .Print("waitingtobestopped") +// .Sleep(TimeSpan.FromSeconds(100))) +// .Build(); +// +// var scriptCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); +// var stopWatch = Stopwatch.StartNew(); +// Exception? actualException = null; +// +// try +// { +// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, scriptCancellationTokenSource.Token, onScriptStatusResponseReceived => +// { +// if (onScriptStatusResponseReceived.Logs.JoinLogs().Contains("waitingtobestopped")) +// { +// scriptCancellationTokenSource.Cancel(); +// } +// }); +// } +// catch (Exception ex) +// { +// actualException = ex; +// } +// +// stopWatch.Stop(); +// +// actualException.Should().NotBeNull().And.BeOfType().And.Match(x => x.Message == "Script execution was cancelled"); +// stopWatch.Elapsed.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(10)); +// +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterThan(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().BeGreaterThanOrEqualTo(1); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.Version2)] +// public async Task WhenOnCompleteTakesLongerThan_OnCancellationAbandonCompleteScriptAfter_AndTheExecutionIsNotCancelled_TheOrchestratorWaitsForCleanUpToComplete(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// bool calledWithNonCancelledCT = false; +// +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .DecorateScriptServiceV2With(b => b +// .BeforeCompleteScript(async (_, _, halibutProxyRequestOptions) => +// { +// await Task.Delay(TimeSpan.FromSeconds(2), CancellationToken); +// calledWithNonCancelledCT = !halibutProxyRequestOptions.RequestCancellationToken.IsCancellationRequested; +// }) +// .Build()) +// .Build()) +// .Build(CancellationToken); +// +// var scriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b.Print("Hello")) +// .Build(); +// +// var tentacleClient = clientTentacle.TentacleClient; +// +// tentacleClient.OnCancellationAbandonCompleteScriptAfter = TimeSpan.FromMilliseconds(1); +// +// await tentacleClient.ExecuteScript(scriptCommand, CancellationToken); +// +// +// calledWithNonCancelledCT.Should().Be(true); +// } +// } +// } diff --git a/source/Octopus.Tentacle.Tests.Integration/TentacleClientObserver.cs b/source/Octopus.Tentacle.Tests.Integration/TentacleClientObserver.cs index bbc2955a9..1bc2b8a81 100644 --- a/source/Octopus.Tentacle.Tests.Integration/TentacleClientObserver.cs +++ b/source/Octopus.Tentacle.Tests.Integration/TentacleClientObserver.cs @@ -1,160 +1,160 @@ -using System; -using System.IO; -using System.Threading.Tasks; -using FluentAssertions; -using Halibut; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils.Builders; -using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Contracts.Logging; -using Octopus.Tentacle.Contracts.Observability; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class TentacleClientObserver : IntegrationTest - { - [Test] - [TentacleConfigurations] - public async Task AnErrorDuringTheCallbackTo_ExecuteScriptCompleted_ShouldNotThrowOrStopScriptExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - // Arrange - var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnExecuteScriptCompleted: true); - - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleClientObserver(tentacleClientObserver) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b.Print("Hello")) - .Build(); - - // Act - await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - - // Assert - // We should have completed the script and not failed due to the error thrown by the TentacleClientObserver - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); - } - - [Test] - [TentacleConfigurations] - public async Task AnErrorDuringTheCallbackTo_RpcCallComplete_ShouldNotThrowOrStopScriptExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - // Arrange - var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnRpcCallCompleted: true); - - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleClientObserver(tentacleClientObserver) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) - .Build()) - .Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(b => b.Print("Hello")) - .Build(); - - // Act - await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - - // Assert - // We should have completed the script and not failed due to the error thrown by the TentacleClientObserver - recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); - } - - [Test] - [TentacleConfigurations] - public async Task AnErrorDuringTheCallbackTo_UploadFileCompleted_ShouldNotThrowAnExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - // Arrange - var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnUploadFileCompleted: true); - - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleClientObserver(tentacleClientObserver) - .Build(CancellationToken); - - var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); - - // Act + Assert - await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); - } - - [Test] - [TentacleConfigurations] - public async Task AnErrorDuringTheCallbackTo_DownloadFileCompleted_ShouldNotThrowAnExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - // Arrange - var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnDownloadFileCompleted: true); - - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacleClientObserver(tentacleClientObserver) - .Build(CancellationToken); - - var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "DownloadFile.txt"); - await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); - - // Act + Assert - await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); - } - - class BrokenTentacleClientObserver : ITentacleClientObserver - { - private readonly bool errorOnRpcCallCompleted; - private readonly bool errorOnUploadFileCompleted; - private readonly bool errorOnDownloadFileCompleted; - private readonly bool errorOnExecuteScriptCompleted; - - public BrokenTentacleClientObserver( - bool errorOnRpcCallCompleted = false, - bool errorOnUploadFileCompleted = false, - bool errorOnDownloadFileCompleted = false, - bool errorOnExecuteScriptCompleted = false) - { - this.errorOnRpcCallCompleted = errorOnRpcCallCompleted; - this.errorOnUploadFileCompleted = errorOnUploadFileCompleted; - this.errorOnDownloadFileCompleted = errorOnDownloadFileCompleted; - this.errorOnExecuteScriptCompleted = errorOnExecuteScriptCompleted; - } - - public void RpcCallCompleted(RpcCallMetrics metrics, ITentacleClientTaskLog logger) - { - if (errorOnRpcCallCompleted) - { - throw new Exception($"RpcCallCompleted {Guid.NewGuid()}"); - } - } - - public void UploadFileCompleted(ClientOperationMetrics clientOperationMetrics, ITentacleClientTaskLog logger) - { - if (errorOnUploadFileCompleted) - { - throw new Exception($"UploadFileCompleted {Guid.NewGuid()}"); - } - } - - public void DownloadFileCompleted(ClientOperationMetrics clientOperationMetrics, ITentacleClientTaskLog logger) - { - if (errorOnDownloadFileCompleted) - { - throw new Exception($"DownloadFileCompleted {Guid.NewGuid()}"); - } - } - - public void ExecuteScriptCompleted(ClientOperationMetrics clientOperationMetrics, ITentacleClientTaskLog logger) - { - if (errorOnExecuteScriptCompleted) - { - throw new Exception($"ExecuteScriptCompleted {Guid.NewGuid()}"); - } - } - } - } -} +// using System; +// using System.IO; +// using System.Threading.Tasks; +// using FluentAssertions; +// using Halibut; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils.Builders; +// using Octopus.Tentacle.Contracts.ClientServices; +// using Octopus.Tentacle.Contracts.Logging; +// using Octopus.Tentacle.Contracts.Observability; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class TentacleClientObserver : IntegrationTest +// { +// [Test] +// [TentacleConfigurations] +// public async Task AnErrorDuringTheCallbackTo_ExecuteScriptCompleted_ShouldNotThrowOrStopScriptExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// // Arrange +// var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnExecuteScriptCompleted: true); +// +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleClientObserver(tentacleClientObserver) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b.Print("Hello")) +// .Build(); +// +// // Act +// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// +// // Assert +// // We should have completed the script and not failed due to the error thrown by the TentacleClientObserver +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task AnErrorDuringTheCallbackTo_RpcCallComplete_ShouldNotThrowOrStopScriptExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// // Arrange +// var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnRpcCallCompleted: true); +// +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleClientObserver(tentacleClientObserver) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) +// .Build()) +// .Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(b => b.Print("Hello")) +// .Build(); +// +// // Act +// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// +// // Assert +// // We should have completed the script and not failed due to the error thrown by the TentacleClientObserver +// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task AnErrorDuringTheCallbackTo_UploadFileCompleted_ShouldNotThrowAnExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// // Arrange +// var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnUploadFileCompleted: true); +// +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleClientObserver(tentacleClientObserver) +// .Build(CancellationToken); +// +// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); +// +// // Act + Assert +// await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); +// } +// +// [Test] +// [TentacleConfigurations] +// public async Task AnErrorDuringTheCallbackTo_DownloadFileCompleted_ShouldNotThrowAnExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// // Arrange +// var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnDownloadFileCompleted: true); +// +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacleClientObserver(tentacleClientObserver) +// .Build(CancellationToken); +// +// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "DownloadFile.txt"); +// await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); +// +// // Act + Assert +// await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); +// } +// +// class BrokenTentacleClientObserver : ITentacleClientObserver +// { +// private readonly bool errorOnRpcCallCompleted; +// private readonly bool errorOnUploadFileCompleted; +// private readonly bool errorOnDownloadFileCompleted; +// private readonly bool errorOnExecuteScriptCompleted; +// +// public BrokenTentacleClientObserver( +// bool errorOnRpcCallCompleted = false, +// bool errorOnUploadFileCompleted = false, +// bool errorOnDownloadFileCompleted = false, +// bool errorOnExecuteScriptCompleted = false) +// { +// this.errorOnRpcCallCompleted = errorOnRpcCallCompleted; +// this.errorOnUploadFileCompleted = errorOnUploadFileCompleted; +// this.errorOnDownloadFileCompleted = errorOnDownloadFileCompleted; +// this.errorOnExecuteScriptCompleted = errorOnExecuteScriptCompleted; +// } +// +// public void RpcCallCompleted(RpcCallMetrics metrics, ITentacleClientTaskLog logger) +// { +// if (errorOnRpcCallCompleted) +// { +// throw new Exception($"RpcCallCompleted {Guid.NewGuid()}"); +// } +// } +// +// public void UploadFileCompleted(ClientOperationMetrics clientOperationMetrics, ITentacleClientTaskLog logger) +// { +// if (errorOnUploadFileCompleted) +// { +// throw new Exception($"UploadFileCompleted {Guid.NewGuid()}"); +// } +// } +// +// public void DownloadFileCompleted(ClientOperationMetrics clientOperationMetrics, ITentacleClientTaskLog logger) +// { +// if (errorOnDownloadFileCompleted) +// { +// throw new Exception($"DownloadFileCompleted {Guid.NewGuid()}"); +// } +// } +// +// public void ExecuteScriptCompleted(ClientOperationMetrics clientOperationMetrics, ITentacleClientTaskLog logger) +// { +// if (errorOnExecuteScriptCompleted) +// { +// throw new Exception($"ExecuteScriptCompleted {Guid.NewGuid()}"); +// } +// } +// } +// } +// } diff --git a/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs b/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs index 42adb51d8..2817386b5 100644 --- a/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs @@ -1,752 +1,752 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Runtime.InteropServices; -using System.Security.Principal; -using System.Text; -using System.Threading.Tasks; -using CliWrap; -using CliWrap.Exceptions; -using FluentAssertions; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -using Octopus.Tentacle.Util; -using Octopus.Tentacle.Variables; -using Polly; -using PlatformDetection = Octopus.Tentacle.Util.PlatformDetection; - -namespace Octopus.Tentacle.Tests.Integration -{ - /// - /// These tests provide guarantees around how our command-line interface works, especially for scenarios where people automate setup. - /// Please review any changes to the assertions made by these tests carefully. - /// - [IntegrationTestTimeout] - [SkipOnEnvironmentsWithKnownPerformanceIssues("we don't expect customers to execute it in this environment in a situation where it needs to behave within a reasonable timeframe.")] - public class TentacleCommandLineTests : IntegrationTest - { - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task TentacleExeNoArguments(TentacleConfigurationTestCase tc) - { - var (exitCode, stdout, stderr) = await RunCommand(tc, null); - - exitCode.Should().Be(2, "the exit code should be 2 if the command wasn't understood"); - stdout.Should().StartWithEquivalentOf("Usage: Tentacle []", "should show help by default if no other commands are specified"); - stdout.Should().ContainEquivalentOf("Or use --help for more details.", "should provide the hint for command-specific help"); - stderr.Should().BeNullOrEmpty(); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task UnknownCommand(TentacleConfigurationTestCase tc) - { - var (exitCode, stdout, stderr) = await RunCommand(tc, null, "unknown-command"); - - exitCode.Should().Be(2, "the exit code should be 2 if the command wasn't understood"); - stderr.Should().StartWithEquivalentOf("Command 'unknown-command' is not supported", "the error should clearly indicate the command which is not understood"); - stdout.Should().StartWithEquivalentOf("See 'Tentacle help'", "should provide the hint to use help"); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task UnknownArgument(TentacleConfigurationTestCase tc) - { - var (exitCode, stdout, stderr) = await RunCommand(tc, null, "version", "--unknown=argument"); - - exitCode.Should().Be(1, "the exit code should be 1 if the command has unknown arguments"); - stdout.Should().BeNullOrEmpty("the error message should be written to stderr, not stdout"); - stderr.Should().ContainEquivalentOf("Unrecognized command line arguments: --unknown=argument", "the error message (written to stderr) should clearly indicate which arguments couldn't be parsed."); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task InvalidArgument(TentacleConfigurationTestCase tc) - { - var (exitCode, stdout, stderr) = await RunCommand(tc, null, "version", "--format=unsupported"); - - exitCode.Should().Be(1, "the exit code should be 1 if the command has unknown arguments"); - stdout.Should().BeNullOrEmpty("the error message should be written to stderr, not stdout"); - stderr.Should().ContainEquivalentOf("The format 'unsupported' is not supported. Try text or json.", "the error message (written to stderr) should clearly indicate which argument was invalid."); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task NoConsoleLoggingSwitchStillSilentlySupportedForBackwardsCompat(TentacleConfigurationTestCase tc) - { - var (_, _, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--noconsolelogging"); - - stderr.Should().BeNullOrEmpty(); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task NoLogoSwitchStillSilentlySupportedForBackwardsCompat(TentacleConfigurationTestCase tc) - { - var (_, _, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--nologo"); - - stderr.Should().BeNullOrEmpty(); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task ConsoleSwitchStillSilentlySupportedForBackwardsCompat(TentacleConfigurationTestCase tc) - { - var (_, _, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--console"); - - stderr.Should().BeNullOrEmpty(); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task ShouldSupportFuzzyCommandParsing(TentacleConfigurationTestCase tc) - { - await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version"); - await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "--version"); - await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "/version"); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task VersionCommandTextFormat(TentacleConfigurationTestCase tc) - { - var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version"); - - var expectedVersion = GetVersionInfo(tc); - - stdout.Should().Be(expectedVersion.ProductVersion, "The version command should print the informational version as text"); - stderr.Should().BeNullOrEmpty(); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task VersionCommandJsonFormat(TentacleConfigurationTestCase tc) - { - var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--format=json"); - - var expectedVersion = GetVersionInfo(tc); - var output = JObject.Parse(stdout); - - output["InformationalVersion"].Value().Should().Be(expectedVersion.ProductVersion, "The version command should print the informational version in the JSON output"); - output["MajorMinorPatch"].Value().Should().Be($"{expectedVersion.FileMajorPart}.{expectedVersion.FileMinorPart}.{expectedVersion.FileBuildPart}", "The version command should print the version in the json output"); - output["NuGetVersion"].Value().Should().NotBeNull("The version command should print the NuGet version in the JSON output"); - output["SourceBranchName"].Value().Should().NotBeNull("The version command should print the source branch in the JSON output"); - - stderr.Should().BeNullOrEmpty(); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task CanGetHelpForHelp(TentacleConfigurationTestCase tc) - { - var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "help", "--help"); - stderr.Should().BeNullOrEmpty(); - - stdout.Should().Be( -@"Usage: Tentacle help [] - -Where [] is any of: - - --format=VALUE The format of the output (text,json). Defaults - to text. - -Or one of the common options: - - --help Show detailed help for this command -"); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task HelpAsSwitchShouldShowCommandSpecificHelp(TentacleConfigurationTestCase tc) - { - var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--help"); - stderr.Should().BeNullOrEmpty(); - - stdout.Should().Be( - @"Usage: Tentacle version [] - -Where [] is any of: - - --format=VALUE The format of the output (text,json). Defaults - to text. - -Or one of the common options: - - --help Show detailed help for this command -"); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task GeneralHelpAsJsonCanBeParsedByAutomationScripts(TentacleConfigurationTestCase tc) - { - var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "help", "--format=json"); - - stderr.Should().BeNullOrEmpty(); - var help = JsonConvert.DeserializeAnonymousType( - stdout, - new - { - Commands = new[] - { - new - { - Name = "", - Description = "", - Aliases = Array.Empty() - } - } - }); - - help.Commands.Select(c => c.Name) - .Should() - .Contain( - "configure", - "help", - "run", - "version", - "show-master-key", - "show-thumbprint"); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task CommandSpecificHelpAsJsonCanBeParsedByAutomationScripts(TentacleConfigurationTestCase tc) - { - var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--help", "--format=json"); - - stderr.Should().BeNullOrEmpty(); - var help = JsonConvert.DeserializeAnonymousType( - stdout, - new - { - Name = "", - Description = "", - Aliases = Array.Empty(), - Options = new[] - { - new - { - Name = "", - Description = "" - } - } - }); - - help.Name.Should().Be("version"); - help.Options.Select(o => o.Name).Should().Contain("format"); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task CommandSpecificHelpAsJsonLooksSensibleToHumans(TentacleConfigurationTestCase tc) - { - var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--help", "--format=json"); - stderr.Should().BeNullOrEmpty(); - - stdout.Should().Be( -@"{ - ""Name"": ""version"", - ""Description"": ""Show the Tentacle version information"", - ""Aliases"": [], - ""Options"": [ - { - ""Name"": ""format"", - ""Description"": ""The format of the output (text,json). Defaults to text."" - } - ], - ""CommonOptions"": [ - { - ""Name"": ""help"", - ""Description"": ""Show detailed help for this command"" - } - ] -}"); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task HelpForInstanceSpecificCommandsAlwaysWorks(TentacleConfigurationTestCase tc) - { - var (_, stdout, stderr) = await RunCommand(tc, null, "help", "--format=json"); - - stderr.Should().BeNullOrEmpty(); - var help = JsonConvert.DeserializeAnonymousType( - stdout, - new - { - Commands = new[] - { - new - { - Name = "", - Description = "", - Aliases = new string[0] - } - } - }); - - help.Commands.Should().HaveCountGreaterThan(0); - - var failed = help.Commands.Select(async c => - { - var (exitCode2, stdout2, stderr2) = await RunCommand(tc, null,$"{c.Name}", "--help"); - return new - { - Command = c, - ExitCode = exitCode2, - StdOut = stdout2, - StdErr = stderr2, - HasExpectedExitCode = exitCode2 == 0, - HasExpectedHelpMessage = stdout2.StartsWith($"Usage: Tentacle {c.Name} []") - }; - }) - .Where(r => !(r.Result.HasExpectedExitCode && r.Result.HasExpectedHelpMessage)) - .ToArray(); - - if (failed.Any()) - { - var failureDetails = string.Empty; - - foreach (var failure in failed) - { - failureDetails += $@"{failure.Result.Command.Name} -StdErr:{failure.Result.StdErr} -StdOut:{failure.Result.StdOut}"; - } - - Assert.Fail( -$@"The following commands cannot show help without specifying the --instance argument: -{failureDetails} -The details are logged above. These commands probably need to take Lazy dependencies so they can be instantiated for showing help without requiring every dependency to be resolvable."); - } - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - // Run these tests in serial to avoid conflicts - [NonParallelizable] - public async Task InvalidInstance(TentacleConfigurationTestCase tc) - { - var (exitCode, stdout, stderr) = await RunCommand( - tc, - null, - "show-thumbprint", "--instance=invalidinstance"); - - exitCode.Should().Be(1, $"the exit code should be 1 if the instance is not able to be resolved"); - stderr.Should().ContainEquivalentOf("Instance invalidinstance of tentacle has not been configured", "the error message should make it clear the instance has not been configured"); - stderr.Should().ContainEquivalentOf("Available instances:", "should provide a hint as to which instances are available on the machine"); - stdout.Should().BeNullOrEmpty(); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - // Run these tests in serial to avoid conflicts - [NonParallelizable] - public async Task ShowThumbprintCommandText(TentacleConfigurationTestCase tc) - { - await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); - await clientAndTentacle.RunningTentacle.Stop(CancellationToken); - var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( - tc, - clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, - "show-thumbprint", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); - - exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); - stdout.Should().Be(TestCertificates.TentaclePublicThumbprint, "the thumbprint should be written directly to stdout"); - stderr.Should().BeNullOrEmpty(); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - // Run these tests in serial to avoid conflicts - [NonParallelizable] - public async Task ShowThumbprintCommandJson(TentacleConfigurationTestCase tc) - { - await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); - await clientAndTentacle.RunningTentacle.Stop(CancellationToken); - var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( - tc, - clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, - "show-thumbprint", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}", "--format=json"); - - exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); - stdout.Should().Be(JsonConvert.SerializeObject(new { Thumbprint = TestCertificates.TentaclePublicThumbprint }), "the thumbprint should be written directly to stdout as JSON"); - stderr.Should().BeNullOrEmpty(); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - // Run these tests in serial to avoid conflicts - [NonParallelizable] - public async Task ListInstancesCommandText(TentacleConfigurationTestCase tc) - { - await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); - await clientAndTentacle.RunningTentacle.Stop(CancellationToken); - var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( - tc, - clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, - "list-instances", "--format=text"); - - exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); - var configPath = Path.Combine(clientAndTentacle.RunningTentacle.HomeDirectory, clientAndTentacle.RunningTentacle.InstanceName + ".cfg"); - stdout.Should().Contain($"Instance '{clientAndTentacle.RunningTentacle.InstanceName}' uses configuration '{configPath}'.", "the current instance should be listed"); - stderr.Should().BeNullOrEmpty(); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - // Run these tests in serial to avoid conflicts - [NonParallelizable] - public async Task ListInstancesCommandJson(TentacleConfigurationTestCase tc) - { - await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); - await clientAndTentacle.RunningTentacle.Stop(CancellationToken); - var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( - tc, - clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, - "list-instances", "--format=json"); - - stdout.Should().Contain($"\"InstanceName\": \"{clientAndTentacle.RunningTentacle.InstanceName}\"", "the current instance should be listed"); - var configPath = Path.Combine(clientAndTentacle.RunningTentacle.HomeDirectory, clientAndTentacle.RunningTentacle.InstanceName + ".cfg"); - var jsonFormattedPath = JsonFormattedPath(configPath); - stdout.Should().Contain($"\"ConfigurationFilePath\": \"{jsonFormattedPath}\"", "the path to the config file for the current instance should be listed"); - stderr.Should().BeNullOrEmpty(); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - // Run these tests in serial to avoid conflicts - [NonParallelizable] - public async Task ShouldLogStartupDiagnosticsToInstanceLogFileOnly(TentacleConfigurationTestCase tc) - { - await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); - await clientAndTentacle.RunningTentacle.Stop(CancellationToken); - - var startingLogText = clientAndTentacle.RunningTentacle.ReadAllLogFileText(); - - var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( - tc, - clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, - "show-thumbprint", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); - - try - { - var logFileText = Policy - .Handle() - .WaitAndRetry( - 20, - i => TimeSpan.FromMilliseconds(100 * i), - (exception, _) => { Logger.Information($"Failed to get new log content: {exception.Message}. Retrying!"); }) - .Execute( - () => - { - var wholeLog = clientAndTentacle.RunningTentacle.ReadAllLogFileText(); - var newLog = wholeLog.Replace(startingLogText, string.Empty); - if (string.IsNullOrWhiteSpace(newLog) || !newLog.Contains("CommandLine:")) - { - throw new NotLoggedYetException(); - } - return newLog; - }); - - logFileText.Should().ContainEquivalentOf($"OperatingSystem: {RuntimeInformation.OSDescription}", "the OSVersion should be in our diagnostics"); - logFileText.Should().ContainEquivalentOf("OperatingSystem:", "the OSVersion should be in our diagnostics"); - logFileText.Should().ContainEquivalentOf($"OsBitVersion: {(Environment.Is64BitOperatingSystem ? "x64" : "x86")}", "the OsBitVersion should be in our diagnostics"); - logFileText.Should().ContainEquivalentOf($"Is64BitProcess: {Environment.Is64BitProcess}", "the Is64BitProcess should be in our diagnostics"); - - if (PlatformDetection.IsRunningOnWindows) - { -#pragma warning disable CA1416 - logFileText.Should().ContainEquivalentOf($"CurrentUser: {WindowsIdentity.GetCurrent().Name}", "the CurrentUser should be in our diagnostics"); -#pragma warning disable CA1416 - } - else - { - logFileText.Should().ContainEquivalentOf($"CurrentUser: {Environment.UserName}", "the CurrentUser should be in our diagnostics"); - } - - logFileText.Should().ContainEquivalentOf($"MachineName: {Environment.MachineName}", "the MachineName should be in our diagnostics"); - logFileText.Should().ContainEquivalentOf($"ProcessorCount: {Environment.ProcessorCount}", "the ProcessorCount should be in our diagnostics"); - logFileText.Should().ContainEquivalentOf($"CurrentDirectory: {Directory.GetCurrentDirectory()}", "the CurrentDirectory should be in our diagnostics"); - logFileText.Should().ContainEquivalentOf($"TempDirectory: {Path.GetTempPath()}", "the TempDirectory should be in our diagnostics"); - logFileText.Should().ContainEquivalentOf("HostProcessName: ", "the HostProcessName should be in our diagnostics"); - stdout.Should().NotContainEquivalentOf($"{RuntimeInformation.OSDescription}", "the OSVersion should not be written to stdout"); - await Task.CompletedTask; - } - catch (NotLoggedYetException) - { - Logger.Error("Failed to get new log content"); - Logger.Error($"Process exit code {exitCode}"); - Logger.Error($"Starting log text: {Environment.NewLine}{startingLogText}"); - Logger.Error($"Current log text: {Environment.NewLine}{clientAndTentacle.RunningTentacle.ReadAllLogFileText()}"); - Logger.Error($"Command StdOut: {Environment.NewLine}{stdout}"); - Logger.Error($"Command StdErr: {Environment.NewLine}{stderr}"); - - throw; - } - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - public async Task HelpAsFirstArgumentShouldShowCommandSpecificHelp(TentacleConfigurationTestCase tc) - { - var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "help", "version"); - stderr.Should().BeNullOrEmpty(); - - stdout.Should().Be( -@"Usage: Tentacle version [] - -Where [] is any of: - - --format=VALUE The format of the output (text,json). Defaults - to text. - -Or one of the common options: - - --help Show detailed help for this command -"); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - // Run these tests in serial to avoid conflicts - [NonParallelizable] - public async Task ShowConfigurationCommand(TentacleConfigurationTestCase tc) - { - await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); - await clientAndTentacle.RunningTentacle.Stop(CancellationToken); - var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( - tc, - clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, - "show-configuration", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); - - stderr.Should().BeNullOrEmpty(); - - // Actually parse and query the document just like our consumer will - dynamic? settings = JsonConvert.DeserializeObject(stdout); - - ((string)settings.Octopus.Home).Should().Be(clientAndTentacle.RunningTentacle.HomeDirectory, "the home directory should match"); - ((string)settings.Tentacle.Deployment.ApplicationDirectory).Should().Be(clientAndTentacle.RunningTentacle.ApplicationDirectory, "the application directory should match"); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - // Run these tests in serial to avoid conflicts - [NonParallelizable] - public async Task ShowConfigurationCommandOnPartiallyConfiguredTentacle(TentacleConfigurationTestCase tc) - { - using var homeDirectory = new TemporaryDirectory(); - var environmentVariables = new Dictionary { { EnvironmentVariables.TentacleMachineConfigurationHomeDirectory, homeDirectory.DirectoryPath } }; - - var instanceId = Guid.NewGuid().ToString(); - using var temporaryDirectory = new TemporaryDirectory(); - await RunCommandAndAssertExitsWithSuccessExitCode( - tc, - environmentVariables, - "create-instance", $"--instance={instanceId}", "--config", Path.Combine(temporaryDirectory.DirectoryPath, instanceId + ".cfg")); - - var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( - tc, - environmentVariables, - "show-configuration", - $"--instance={instanceId}"); - - stderr.Should().BeNullOrEmpty(); - - // Actually parse and query the document just like our consumer will - dynamic? settings = JsonConvert.DeserializeObject(stdout); - ((string)settings.Octopus.Home).Should().Be(temporaryDirectory.DirectoryPath, "the home directory should match"); - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - // Run these tests in serial to avoid conflicts - [NonParallelizable] - public async Task ShowConfigurationCommandLooksSensibleToHumans(TentacleConfigurationTestCase tc) - { - await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); - await clientAndTentacle.RunningTentacle.Stop(CancellationToken); - var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( - tc, - clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, - "show-configuration", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); - - stderr.Should().BeNullOrEmpty(); - - if (tc.TentacleType == TentacleType.Polling) - { - stdout.Should().Be($@"{{ - ""Octopus"": {{ - ""Home"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.HomeDirectory)}"", - ""Watchdog"": {{ - ""Enabled"": false, - ""Instances"": ""*"", - ""Interval"": 0 - }} - }}, - ""Tentacle"": {{ - ""CertificateThumbprint"": ""{clientAndTentacle.RunningTentacle.Thumbprint}"", - ""Communication"": {{ - ""TrustedOctopusServers"": [ - {{ - ""Thumbprint"": ""{clientAndTentacle.Server.Thumbprint}"", - ""CommunicationStyle"": 2, - ""Address"": ""https://localhost:{clientAndTentacle.Server.ServerListeningPort}"", - ""Squid"": null, - ""SubscriptionId"": ""{clientAndTentacle.RunningTentacle.ServiceUri}"" - }} - ] - }}, - ""Deployment"": {{ - ""ApplicationDirectory"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.ApplicationDirectory)}"" - }}, - ""Services"": {{ - ""ListenIP"": null, - ""NoListen"": true, - ""PortNumber"": 10933 - }} - }} -}} -"); - } - else - { - stdout.Should().Be($@"{{ - ""Octopus"": {{ - ""Home"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.HomeDirectory)}"", - ""Watchdog"": {{ - ""Enabled"": false, - ""Instances"": ""*"", - ""Interval"": 0 - }} - }}, - ""Tentacle"": {{ - ""CertificateThumbprint"": ""{clientAndTentacle.RunningTentacle.Thumbprint}"", - ""Communication"": {{ - ""TrustedOctopusServers"": [ - {{ - ""Thumbprint"": ""{clientAndTentacle.Server.Thumbprint}"", - ""CommunicationStyle"": 1, - ""Address"": null, - ""Squid"": null, - ""SubscriptionId"": null - }} - ] - }}, - ""Deployment"": {{ - ""ApplicationDirectory"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.ApplicationDirectory)}"" - }}, - ""Services"": {{ - ""ListenIP"": null, - ""NoListen"": false, - ""PortNumber"": {clientAndTentacle.RunningTentacle.ServiceUri.Port} - }} - }} -}} -"); - } - - await Task.CompletedTask; - } - - [Test] - [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] - [WindowsTest] - // Run these tests in serial to avoid conflicts - [NonParallelizable] - public async Task WatchdogCreateAndDeleteCommand(TentacleConfigurationTestCase tc) - { - await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); - await clientAndTentacle.RunningTentacle.Stop(CancellationToken); - var create = await RunCommandAndAssertExitsWithSuccessExitCode( - tc, - clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, - "watchdog", "--create", $"--instances={clientAndTentacle.RunningTentacle.InstanceName}"); - - create.StdError.Should().BeNullOrEmpty(); - create.StdOut.Should().ContainEquivalentOf("Creating watchdog task"); - var delete = await RunCommandAndAssertExitsWithSuccessExitCode( - tc, - clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, - "watchdog", "--delete", $"--instances={clientAndTentacle.RunningTentacle.InstanceName}"); - - delete.StdError.Should().BeNullOrEmpty(); - delete.StdOut.Should().ContainEquivalentOf("Removing watchdog task"); - } - - FileVersionInfo GetVersionInfo(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var tentacleExe = TentacleExeFinder.FindTentacleExe(tentacleConfigurationTestCase.TentacleRuntime); - - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - return FileVersionInfo.GetVersionInfo(tentacleExe); - } - - //todo: change this to trust the value set in the context.TentacleExePath (will need a renovation of ExePathResolver to be non windows specific) - return FileVersionInfo.GetVersionInfo($"{tentacleExe}.dll"); - } - - async Task<(int ExitCode, string StdOut, string StdError)> RunCommandAndAssertExitsWithSuccessExitCode( - TentacleConfigurationTestCase tentacleConfigurationTestCase, - IReadOnlyDictionary? environmentVariables, - params string[] arguments) - { - var (exitCode, stdout, stderr) = await RunCommand(tentacleConfigurationTestCase, environmentVariables, arguments); - exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); - return (exitCode, stdout, stderr); - } - - async Task<(int ExitCode, string StdOut, string StdError)> RunCommand( - TentacleConfigurationTestCase tentacleConfigurationTestCase, - IReadOnlyDictionary? environmentVariables, - params string[] arguments) - { - using var tempDirectory = new TemporaryDirectory(); - - var environmentVariablesToRunTentacleWith = new Dictionary(); - - if (environmentVariables?.Any() == true) - { - environmentVariablesToRunTentacleWith.AddRange(environmentVariables); - } - - if (!environmentVariablesToRunTentacleWith.ContainsKey(EnvironmentVariables.TentacleMachineConfigurationHomeDirectory)) - { - environmentVariablesToRunTentacleWith.Add(EnvironmentVariables.TentacleMachineConfigurationHomeDirectory, tempDirectory.DirectoryPath); - } - - var tentacleExe = TentacleExeFinder.FindTentacleExe(tentacleConfigurationTestCase.TentacleRuntime); - var output = new StringBuilder(); - var errorOut = new StringBuilder(); - - var result = await RetryHelper.RetryAsync( - () => Cli.Wrap(tentacleExe) - .WithArguments(arguments) - .WithValidation(CommandResultValidation.None) - .WithStandardOutputPipe(PipeTarget.ToStringBuilder(output)) - .WithStandardErrorPipe(PipeTarget.ToStringBuilder(errorOut)) - .WithEnvironmentVariables(environmentVariablesToRunTentacleWith) - .ExecuteAsync(CancellationToken)); - - return (result.ExitCode, output.ToString(), errorOut.ToString()); - } - - static string JsonFormattedPath(string path) - { - return path.Replace(@"\", @"\\"); - } - - public class NotLoggedYetException : Exception - { - } - } -} +// using System; +// using System.Collections.Generic; +// using System.Diagnostics; +// using System.IO; +// using System.Linq; +// using System.Runtime.InteropServices; +// using System.Security.Principal; +// using System.Text; +// using System.Threading.Tasks; +// using CliWrap; +// using CliWrap.Exceptions; +// using FluentAssertions; +// using Newtonsoft.Json; +// using Newtonsoft.Json.Linq; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +// using Octopus.Tentacle.Util; +// using Octopus.Tentacle.Variables; +// using Polly; +// using PlatformDetection = Octopus.Tentacle.Util.PlatformDetection; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// /// +// /// These tests provide guarantees around how our command-line interface works, especially for scenarios where people automate setup. +// /// Please review any changes to the assertions made by these tests carefully. +// /// +// [IntegrationTestTimeout] +// [SkipOnEnvironmentsWithKnownPerformanceIssues("we don't expect customers to execute it in this environment in a situation where it needs to behave within a reasonable timeframe.")] +// public class TentacleCommandLineTests : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task TentacleExeNoArguments(TentacleConfigurationTestCase tc) +// { +// var (exitCode, stdout, stderr) = await RunCommand(tc, null); +// +// exitCode.Should().Be(2, "the exit code should be 2 if the command wasn't understood"); +// stdout.Should().StartWithEquivalentOf("Usage: Tentacle []", "should show help by default if no other commands are specified"); +// stdout.Should().ContainEquivalentOf("Or use --help for more details.", "should provide the hint for command-specific help"); +// stderr.Should().BeNullOrEmpty(); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task UnknownCommand(TentacleConfigurationTestCase tc) +// { +// var (exitCode, stdout, stderr) = await RunCommand(tc, null, "unknown-command"); +// +// exitCode.Should().Be(2, "the exit code should be 2 if the command wasn't understood"); +// stderr.Should().StartWithEquivalentOf("Command 'unknown-command' is not supported", "the error should clearly indicate the command which is not understood"); +// stdout.Should().StartWithEquivalentOf("See 'Tentacle help'", "should provide the hint to use help"); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task UnknownArgument(TentacleConfigurationTestCase tc) +// { +// var (exitCode, stdout, stderr) = await RunCommand(tc, null, "version", "--unknown=argument"); +// +// exitCode.Should().Be(1, "the exit code should be 1 if the command has unknown arguments"); +// stdout.Should().BeNullOrEmpty("the error message should be written to stderr, not stdout"); +// stderr.Should().ContainEquivalentOf("Unrecognized command line arguments: --unknown=argument", "the error message (written to stderr) should clearly indicate which arguments couldn't be parsed."); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task InvalidArgument(TentacleConfigurationTestCase tc) +// { +// var (exitCode, stdout, stderr) = await RunCommand(tc, null, "version", "--format=unsupported"); +// +// exitCode.Should().Be(1, "the exit code should be 1 if the command has unknown arguments"); +// stdout.Should().BeNullOrEmpty("the error message should be written to stderr, not stdout"); +// stderr.Should().ContainEquivalentOf("The format 'unsupported' is not supported. Try text or json.", "the error message (written to stderr) should clearly indicate which argument was invalid."); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task NoConsoleLoggingSwitchStillSilentlySupportedForBackwardsCompat(TentacleConfigurationTestCase tc) +// { +// var (_, _, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--noconsolelogging"); +// +// stderr.Should().BeNullOrEmpty(); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task NoLogoSwitchStillSilentlySupportedForBackwardsCompat(TentacleConfigurationTestCase tc) +// { +// var (_, _, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--nologo"); +// +// stderr.Should().BeNullOrEmpty(); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task ConsoleSwitchStillSilentlySupportedForBackwardsCompat(TentacleConfigurationTestCase tc) +// { +// var (_, _, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--console"); +// +// stderr.Should().BeNullOrEmpty(); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task ShouldSupportFuzzyCommandParsing(TentacleConfigurationTestCase tc) +// { +// await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version"); +// await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "--version"); +// await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "/version"); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task VersionCommandTextFormat(TentacleConfigurationTestCase tc) +// { +// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version"); +// +// var expectedVersion = GetVersionInfo(tc); +// +// stdout.Should().Be(expectedVersion.ProductVersion, "The version command should print the informational version as text"); +// stderr.Should().BeNullOrEmpty(); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task VersionCommandJsonFormat(TentacleConfigurationTestCase tc) +// { +// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--format=json"); +// +// var expectedVersion = GetVersionInfo(tc); +// var output = JObject.Parse(stdout); +// +// output["InformationalVersion"].Value().Should().Be(expectedVersion.ProductVersion, "The version command should print the informational version in the JSON output"); +// output["MajorMinorPatch"].Value().Should().Be($"{expectedVersion.FileMajorPart}.{expectedVersion.FileMinorPart}.{expectedVersion.FileBuildPart}", "The version command should print the version in the json output"); +// output["NuGetVersion"].Value().Should().NotBeNull("The version command should print the NuGet version in the JSON output"); +// output["SourceBranchName"].Value().Should().NotBeNull("The version command should print the source branch in the JSON output"); +// +// stderr.Should().BeNullOrEmpty(); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task CanGetHelpForHelp(TentacleConfigurationTestCase tc) +// { +// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "help", "--help"); +// stderr.Should().BeNullOrEmpty(); +// +// stdout.Should().Be( +// @"Usage: Tentacle help [] +// +// Where [] is any of: +// +// --format=VALUE The format of the output (text,json). Defaults +// to text. +// +// Or one of the common options: +// +// --help Show detailed help for this command +// "); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task HelpAsSwitchShouldShowCommandSpecificHelp(TentacleConfigurationTestCase tc) +// { +// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--help"); +// stderr.Should().BeNullOrEmpty(); +// +// stdout.Should().Be( +// @"Usage: Tentacle version [] +// +// Where [] is any of: +// +// --format=VALUE The format of the output (text,json). Defaults +// to text. +// +// Or one of the common options: +// +// --help Show detailed help for this command +// "); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task GeneralHelpAsJsonCanBeParsedByAutomationScripts(TentacleConfigurationTestCase tc) +// { +// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "help", "--format=json"); +// +// stderr.Should().BeNullOrEmpty(); +// var help = JsonConvert.DeserializeAnonymousType( +// stdout, +// new +// { +// Commands = new[] +// { +// new +// { +// Name = "", +// Description = "", +// Aliases = Array.Empty() +// } +// } +// }); +// +// help.Commands.Select(c => c.Name) +// .Should() +// .Contain( +// "configure", +// "help", +// "run", +// "version", +// "show-master-key", +// "show-thumbprint"); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task CommandSpecificHelpAsJsonCanBeParsedByAutomationScripts(TentacleConfigurationTestCase tc) +// { +// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--help", "--format=json"); +// +// stderr.Should().BeNullOrEmpty(); +// var help = JsonConvert.DeserializeAnonymousType( +// stdout, +// new +// { +// Name = "", +// Description = "", +// Aliases = Array.Empty(), +// Options = new[] +// { +// new +// { +// Name = "", +// Description = "" +// } +// } +// }); +// +// help.Name.Should().Be("version"); +// help.Options.Select(o => o.Name).Should().Contain("format"); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task CommandSpecificHelpAsJsonLooksSensibleToHumans(TentacleConfigurationTestCase tc) +// { +// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--help", "--format=json"); +// stderr.Should().BeNullOrEmpty(); +// +// stdout.Should().Be( +// @"{ +// ""Name"": ""version"", +// ""Description"": ""Show the Tentacle version information"", +// ""Aliases"": [], +// ""Options"": [ +// { +// ""Name"": ""format"", +// ""Description"": ""The format of the output (text,json). Defaults to text."" +// } +// ], +// ""CommonOptions"": [ +// { +// ""Name"": ""help"", +// ""Description"": ""Show detailed help for this command"" +// } +// ] +// }"); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task HelpForInstanceSpecificCommandsAlwaysWorks(TentacleConfigurationTestCase tc) +// { +// var (_, stdout, stderr) = await RunCommand(tc, null, "help", "--format=json"); +// +// stderr.Should().BeNullOrEmpty(); +// var help = JsonConvert.DeserializeAnonymousType( +// stdout, +// new +// { +// Commands = new[] +// { +// new +// { +// Name = "", +// Description = "", +// Aliases = new string[0] +// } +// } +// }); +// +// help.Commands.Should().HaveCountGreaterThan(0); +// +// var failed = help.Commands.Select(async c => +// { +// var (exitCode2, stdout2, stderr2) = await RunCommand(tc, null,$"{c.Name}", "--help"); +// return new +// { +// Command = c, +// ExitCode = exitCode2, +// StdOut = stdout2, +// StdErr = stderr2, +// HasExpectedExitCode = exitCode2 == 0, +// HasExpectedHelpMessage = stdout2.StartsWith($"Usage: Tentacle {c.Name} []") +// }; +// }) +// .Where(r => !(r.Result.HasExpectedExitCode && r.Result.HasExpectedHelpMessage)) +// .ToArray(); +// +// if (failed.Any()) +// { +// var failureDetails = string.Empty; +// +// foreach (var failure in failed) +// { +// failureDetails += $@"{failure.Result.Command.Name} +// StdErr:{failure.Result.StdErr} +// StdOut:{failure.Result.StdOut}"; +// } +// +// Assert.Fail( +// $@"The following commands cannot show help without specifying the --instance argument: +// {failureDetails} +// The details are logged above. These commands probably need to take Lazy dependencies so they can be instantiated for showing help without requiring every dependency to be resolvable."); +// } +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// // Run these tests in serial to avoid conflicts +// [NonParallelizable] +// public async Task InvalidInstance(TentacleConfigurationTestCase tc) +// { +// var (exitCode, stdout, stderr) = await RunCommand( +// tc, +// null, +// "show-thumbprint", "--instance=invalidinstance"); +// +// exitCode.Should().Be(1, $"the exit code should be 1 if the instance is not able to be resolved"); +// stderr.Should().ContainEquivalentOf("Instance invalidinstance of tentacle has not been configured", "the error message should make it clear the instance has not been configured"); +// stderr.Should().ContainEquivalentOf("Available instances:", "should provide a hint as to which instances are available on the machine"); +// stdout.Should().BeNullOrEmpty(); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// // Run these tests in serial to avoid conflicts +// [NonParallelizable] +// public async Task ShowThumbprintCommandText(TentacleConfigurationTestCase tc) +// { +// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); +// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); +// var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( +// tc, +// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, +// "show-thumbprint", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); +// +// exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); +// stdout.Should().Be(TestCertificates.TentaclePublicThumbprint, "the thumbprint should be written directly to stdout"); +// stderr.Should().BeNullOrEmpty(); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// // Run these tests in serial to avoid conflicts +// [NonParallelizable] +// public async Task ShowThumbprintCommandJson(TentacleConfigurationTestCase tc) +// { +// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); +// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); +// var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( +// tc, +// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, +// "show-thumbprint", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}", "--format=json"); +// +// exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); +// stdout.Should().Be(JsonConvert.SerializeObject(new { Thumbprint = TestCertificates.TentaclePublicThumbprint }), "the thumbprint should be written directly to stdout as JSON"); +// stderr.Should().BeNullOrEmpty(); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// // Run these tests in serial to avoid conflicts +// [NonParallelizable] +// public async Task ListInstancesCommandText(TentacleConfigurationTestCase tc) +// { +// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); +// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); +// var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( +// tc, +// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, +// "list-instances", "--format=text"); +// +// exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); +// var configPath = Path.Combine(clientAndTentacle.RunningTentacle.HomeDirectory, clientAndTentacle.RunningTentacle.InstanceName + ".cfg"); +// stdout.Should().Contain($"Instance '{clientAndTentacle.RunningTentacle.InstanceName}' uses configuration '{configPath}'.", "the current instance should be listed"); +// stderr.Should().BeNullOrEmpty(); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// // Run these tests in serial to avoid conflicts +// [NonParallelizable] +// public async Task ListInstancesCommandJson(TentacleConfigurationTestCase tc) +// { +// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); +// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); +// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( +// tc, +// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, +// "list-instances", "--format=json"); +// +// stdout.Should().Contain($"\"InstanceName\": \"{clientAndTentacle.RunningTentacle.InstanceName}\"", "the current instance should be listed"); +// var configPath = Path.Combine(clientAndTentacle.RunningTentacle.HomeDirectory, clientAndTentacle.RunningTentacle.InstanceName + ".cfg"); +// var jsonFormattedPath = JsonFormattedPath(configPath); +// stdout.Should().Contain($"\"ConfigurationFilePath\": \"{jsonFormattedPath}\"", "the path to the config file for the current instance should be listed"); +// stderr.Should().BeNullOrEmpty(); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// // Run these tests in serial to avoid conflicts +// [NonParallelizable] +// public async Task ShouldLogStartupDiagnosticsToInstanceLogFileOnly(TentacleConfigurationTestCase tc) +// { +// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); +// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); +// +// var startingLogText = clientAndTentacle.RunningTentacle.ReadAllLogFileText(); +// +// var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( +// tc, +// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, +// "show-thumbprint", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); +// +// try +// { +// var logFileText = Policy +// .Handle() +// .WaitAndRetry( +// 20, +// i => TimeSpan.FromMilliseconds(100 * i), +// (exception, _) => { Logger.Information($"Failed to get new log content: {exception.Message}. Retrying!"); }) +// .Execute( +// () => +// { +// var wholeLog = clientAndTentacle.RunningTentacle.ReadAllLogFileText(); +// var newLog = wholeLog.Replace(startingLogText, string.Empty); +// if (string.IsNullOrWhiteSpace(newLog) || !newLog.Contains("CommandLine:")) +// { +// throw new NotLoggedYetException(); +// } +// return newLog; +// }); +// +// logFileText.Should().ContainEquivalentOf($"OperatingSystem: {RuntimeInformation.OSDescription}", "the OSVersion should be in our diagnostics"); +// logFileText.Should().ContainEquivalentOf("OperatingSystem:", "the OSVersion should be in our diagnostics"); +// logFileText.Should().ContainEquivalentOf($"OsBitVersion: {(Environment.Is64BitOperatingSystem ? "x64" : "x86")}", "the OsBitVersion should be in our diagnostics"); +// logFileText.Should().ContainEquivalentOf($"Is64BitProcess: {Environment.Is64BitProcess}", "the Is64BitProcess should be in our diagnostics"); +// +// if (PlatformDetection.IsRunningOnWindows) +// { +// #pragma warning disable CA1416 +// logFileText.Should().ContainEquivalentOf($"CurrentUser: {WindowsIdentity.GetCurrent().Name}", "the CurrentUser should be in our diagnostics"); +// #pragma warning disable CA1416 +// } +// else +// { +// logFileText.Should().ContainEquivalentOf($"CurrentUser: {Environment.UserName}", "the CurrentUser should be in our diagnostics"); +// } +// +// logFileText.Should().ContainEquivalentOf($"MachineName: {Environment.MachineName}", "the MachineName should be in our diagnostics"); +// logFileText.Should().ContainEquivalentOf($"ProcessorCount: {Environment.ProcessorCount}", "the ProcessorCount should be in our diagnostics"); +// logFileText.Should().ContainEquivalentOf($"CurrentDirectory: {Directory.GetCurrentDirectory()}", "the CurrentDirectory should be in our diagnostics"); +// logFileText.Should().ContainEquivalentOf($"TempDirectory: {Path.GetTempPath()}", "the TempDirectory should be in our diagnostics"); +// logFileText.Should().ContainEquivalentOf("HostProcessName: ", "the HostProcessName should be in our diagnostics"); +// stdout.Should().NotContainEquivalentOf($"{RuntimeInformation.OSDescription}", "the OSVersion should not be written to stdout"); +// await Task.CompletedTask; +// } +// catch (NotLoggedYetException) +// { +// Logger.Error("Failed to get new log content"); +// Logger.Error($"Process exit code {exitCode}"); +// Logger.Error($"Starting log text: {Environment.NewLine}{startingLogText}"); +// Logger.Error($"Current log text: {Environment.NewLine}{clientAndTentacle.RunningTentacle.ReadAllLogFileText()}"); +// Logger.Error($"Command StdOut: {Environment.NewLine}{stdout}"); +// Logger.Error($"Command StdErr: {Environment.NewLine}{stderr}"); +// +// throw; +// } +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// public async Task HelpAsFirstArgumentShouldShowCommandSpecificHelp(TentacleConfigurationTestCase tc) +// { +// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "help", "version"); +// stderr.Should().BeNullOrEmpty(); +// +// stdout.Should().Be( +// @"Usage: Tentacle version [] +// +// Where [] is any of: +// +// --format=VALUE The format of the output (text,json). Defaults +// to text. +// +// Or one of the common options: +// +// --help Show detailed help for this command +// "); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// // Run these tests in serial to avoid conflicts +// [NonParallelizable] +// public async Task ShowConfigurationCommand(TentacleConfigurationTestCase tc) +// { +// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); +// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); +// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( +// tc, +// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, +// "show-configuration", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); +// +// stderr.Should().BeNullOrEmpty(); +// +// // Actually parse and query the document just like our consumer will +// dynamic? settings = JsonConvert.DeserializeObject(stdout); +// +// ((string)settings.Octopus.Home).Should().Be(clientAndTentacle.RunningTentacle.HomeDirectory, "the home directory should match"); +// ((string)settings.Tentacle.Deployment.ApplicationDirectory).Should().Be(clientAndTentacle.RunningTentacle.ApplicationDirectory, "the application directory should match"); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// // Run these tests in serial to avoid conflicts +// [NonParallelizable] +// public async Task ShowConfigurationCommandOnPartiallyConfiguredTentacle(TentacleConfigurationTestCase tc) +// { +// using var homeDirectory = new TemporaryDirectory(); +// var environmentVariables = new Dictionary { { EnvironmentVariables.TentacleMachineConfigurationHomeDirectory, homeDirectory.DirectoryPath } }; +// +// var instanceId = Guid.NewGuid().ToString(); +// using var temporaryDirectory = new TemporaryDirectory(); +// await RunCommandAndAssertExitsWithSuccessExitCode( +// tc, +// environmentVariables, +// "create-instance", $"--instance={instanceId}", "--config", Path.Combine(temporaryDirectory.DirectoryPath, instanceId + ".cfg")); +// +// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( +// tc, +// environmentVariables, +// "show-configuration", +// $"--instance={instanceId}"); +// +// stderr.Should().BeNullOrEmpty(); +// +// // Actually parse and query the document just like our consumer will +// dynamic? settings = JsonConvert.DeserializeObject(stdout); +// ((string)settings.Octopus.Home).Should().Be(temporaryDirectory.DirectoryPath, "the home directory should match"); +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// // Run these tests in serial to avoid conflicts +// [NonParallelizable] +// public async Task ShowConfigurationCommandLooksSensibleToHumans(TentacleConfigurationTestCase tc) +// { +// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); +// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); +// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( +// tc, +// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, +// "show-configuration", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); +// +// stderr.Should().BeNullOrEmpty(); +// +// if (tc.TentacleType == TentacleType.Polling) +// { +// stdout.Should().Be($@"{{ +// ""Octopus"": {{ +// ""Home"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.HomeDirectory)}"", +// ""Watchdog"": {{ +// ""Enabled"": false, +// ""Instances"": ""*"", +// ""Interval"": 0 +// }} +// }}, +// ""Tentacle"": {{ +// ""CertificateThumbprint"": ""{clientAndTentacle.RunningTentacle.Thumbprint}"", +// ""Communication"": {{ +// ""TrustedOctopusServers"": [ +// {{ +// ""Thumbprint"": ""{clientAndTentacle.Server.Thumbprint}"", +// ""CommunicationStyle"": 2, +// ""Address"": ""https://localhost:{clientAndTentacle.Server.ServerListeningPort}"", +// ""Squid"": null, +// ""SubscriptionId"": ""{clientAndTentacle.RunningTentacle.ServiceUri}"" +// }} +// ] +// }}, +// ""Deployment"": {{ +// ""ApplicationDirectory"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.ApplicationDirectory)}"" +// }}, +// ""Services"": {{ +// ""ListenIP"": null, +// ""NoListen"": true, +// ""PortNumber"": 10933 +// }} +// }} +// }} +// "); +// } +// else +// { +// stdout.Should().Be($@"{{ +// ""Octopus"": {{ +// ""Home"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.HomeDirectory)}"", +// ""Watchdog"": {{ +// ""Enabled"": false, +// ""Instances"": ""*"", +// ""Interval"": 0 +// }} +// }}, +// ""Tentacle"": {{ +// ""CertificateThumbprint"": ""{clientAndTentacle.RunningTentacle.Thumbprint}"", +// ""Communication"": {{ +// ""TrustedOctopusServers"": [ +// {{ +// ""Thumbprint"": ""{clientAndTentacle.Server.Thumbprint}"", +// ""CommunicationStyle"": 1, +// ""Address"": null, +// ""Squid"": null, +// ""SubscriptionId"": null +// }} +// ] +// }}, +// ""Deployment"": {{ +// ""ApplicationDirectory"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.ApplicationDirectory)}"" +// }}, +// ""Services"": {{ +// ""ListenIP"": null, +// ""NoListen"": false, +// ""PortNumber"": {clientAndTentacle.RunningTentacle.ServiceUri.Port} +// }} +// }} +// }} +// "); +// } +// +// await Task.CompletedTask; +// } +// +// [Test] +// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] +// [WindowsTest] +// // Run these tests in serial to avoid conflicts +// [NonParallelizable] +// public async Task WatchdogCreateAndDeleteCommand(TentacleConfigurationTestCase tc) +// { +// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); +// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); +// var create = await RunCommandAndAssertExitsWithSuccessExitCode( +// tc, +// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, +// "watchdog", "--create", $"--instances={clientAndTentacle.RunningTentacle.InstanceName}"); +// +// create.StdError.Should().BeNullOrEmpty(); +// create.StdOut.Should().ContainEquivalentOf("Creating watchdog task"); +// var delete = await RunCommandAndAssertExitsWithSuccessExitCode( +// tc, +// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, +// "watchdog", "--delete", $"--instances={clientAndTentacle.RunningTentacle.InstanceName}"); +// +// delete.StdError.Should().BeNullOrEmpty(); +// delete.StdOut.Should().ContainEquivalentOf("Removing watchdog task"); +// } +// +// FileVersionInfo GetVersionInfo(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var tentacleExe = TentacleExeFinder.FindTentacleExe(tentacleConfigurationTestCase.TentacleRuntime); +// +// if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) +// { +// return FileVersionInfo.GetVersionInfo(tentacleExe); +// } +// +// //todo: change this to trust the value set in the context.TentacleExePath (will need a renovation of ExePathResolver to be non windows specific) +// return FileVersionInfo.GetVersionInfo($"{tentacleExe}.dll"); +// } +// +// async Task<(int ExitCode, string StdOut, string StdError)> RunCommandAndAssertExitsWithSuccessExitCode( +// TentacleConfigurationTestCase tentacleConfigurationTestCase, +// IReadOnlyDictionary? environmentVariables, +// params string[] arguments) +// { +// var (exitCode, stdout, stderr) = await RunCommand(tentacleConfigurationTestCase, environmentVariables, arguments); +// exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); +// return (exitCode, stdout, stderr); +// } +// +// async Task<(int ExitCode, string StdOut, string StdError)> RunCommand( +// TentacleConfigurationTestCase tentacleConfigurationTestCase, +// IReadOnlyDictionary? environmentVariables, +// params string[] arguments) +// { +// using var tempDirectory = new TemporaryDirectory(); +// +// var environmentVariablesToRunTentacleWith = new Dictionary(); +// +// if (environmentVariables?.Any() == true) +// { +// environmentVariablesToRunTentacleWith.AddRange(environmentVariables); +// } +// +// if (!environmentVariablesToRunTentacleWith.ContainsKey(EnvironmentVariables.TentacleMachineConfigurationHomeDirectory)) +// { +// environmentVariablesToRunTentacleWith.Add(EnvironmentVariables.TentacleMachineConfigurationHomeDirectory, tempDirectory.DirectoryPath); +// } +// +// var tentacleExe = TentacleExeFinder.FindTentacleExe(tentacleConfigurationTestCase.TentacleRuntime); +// var output = new StringBuilder(); +// var errorOut = new StringBuilder(); +// +// var result = await RetryHelper.RetryAsync( +// () => Cli.Wrap(tentacleExe) +// .WithArguments(arguments) +// .WithValidation(CommandResultValidation.None) +// .WithStandardOutputPipe(PipeTarget.ToStringBuilder(output)) +// .WithStandardErrorPipe(PipeTarget.ToStringBuilder(errorOut)) +// .WithEnvironmentVariables(environmentVariablesToRunTentacleWith) +// .ExecuteAsync(CancellationToken)); +// +// return (result.ExitCode, output.ToString(), errorOut.ToString()); +// } +// +// static string JsonFormattedPath(string path) +// { +// return path.Replace(@"\", @"\\"); +// } +// +// public class NotLoggedYetException : Exception +// { +// } +// } +// } diff --git a/source/Octopus.Tentacle.Tests.Integration/TentacleStartupAndShutdownTests.cs b/source/Octopus.Tentacle.Tests.Integration/TentacleStartupAndShutdownTests.cs index 7cd833c4f..9eaa56113 100644 --- a/source/Octopus.Tentacle.Tests.Integration/TentacleStartupAndShutdownTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/TentacleStartupAndShutdownTests.cs @@ -1,72 +1,72 @@ -#nullable enable -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using FluentAssertions; -using Halibut.Exceptions; -using NUnit.Framework; -using Octopus.Tentacle.Client.Scripts; -using Octopus.Tentacle.CommonTestUtils.Builders; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; -using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -using Octopus.Tentacle.Tests.Integration.Util.Builders; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class TentacleStartupAndShutdownTests : IntegrationTest - { - [Test] - [TentacleConfigurations] - [RequiresSudoOnLinux] - public async Task WhenRunningTentacleAsAServiceItShouldBeAbleToRestartItself(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using (var clientAndTentacle = await tentacleConfigurationTestCase - .CreateBuilder() - .InstallAsAService() - .Build(CancellationToken)) - { - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBodyForCurrentOs( -$@"cd ""{clientAndTentacle.RunningTentacle.TentacleExe.DirectoryName}"" -.\Tentacle.exe service --instance {clientAndTentacle.RunningTentacle.InstanceName} --stop --start", -$@"#!/bin/sh -cd ""{clientAndTentacle.RunningTentacle.TentacleExe.DirectoryName}"" -./Tentacle service --instance {clientAndTentacle.RunningTentacle.InstanceName} --stop --start") - .Build(); - - (Client.Scripts.Models.ScriptExecutionResult ScriptExecutionResult, List ProcessOutput) result; - - try - { - result = await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - } - catch (ServiceInvocationHalibutClientException ex) - { - Logger.Information(ex, "ServiceInvocationHalibutClientException thrown while Tentacle was restarting itself. This can be ignored for the purpose of this test."); - - // Making Tentacle restart itself can cause internal errors with Script Service - // Execute the script again to get the final result and logs. This will not rerun the script. - result = await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - } - - result.LogExecuteScriptOutput(Logger); - - result.ProcessOutput.Any(x => x.Text.Contains("Stopping service")).Should().BeTrue("Stopping service should be logged"); - result.ScriptExecutionResult.State.Should().Be(ProcessState.Complete); - - startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder().Print("Running...")) - .Build(); - - result = await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - result.ProcessOutput.Any(x => x.Text.Contains("Running...")).Should().BeTrue("Running... should be logged"); - result.ScriptExecutionResult.ExitCode.Should().Be(0); - result.ScriptExecutionResult.State.Should().Be(ProcessState.Complete); - } - } - } -} +// #nullable enable +// using System.Collections.Generic; +// using System.Linq; +// using System.Threading.Tasks; +// using FluentAssertions; +// using Halibut.Exceptions; +// using NUnit.Framework; +// using Octopus.Tentacle.Client.Scripts; +// using Octopus.Tentacle.CommonTestUtils.Builders; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class TentacleStartupAndShutdownTests : IntegrationTest +// { +// [Test] +// [TentacleConfigurations] +// [RequiresSudoOnLinux] +// public async Task WhenRunningTentacleAsAServiceItShouldBeAbleToRestartItself(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using (var clientAndTentacle = await tentacleConfigurationTestCase +// .CreateBuilder() +// .InstallAsAService() +// .Build(CancellationToken)) +// { +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBodyForCurrentOs( +// $@"cd ""{clientAndTentacle.RunningTentacle.TentacleExe.DirectoryName}"" +// .\Tentacle.exe service --instance {clientAndTentacle.RunningTentacle.InstanceName} --stop --start", +// $@"#!/bin/sh +// cd ""{clientAndTentacle.RunningTentacle.TentacleExe.DirectoryName}"" +// ./Tentacle service --instance {clientAndTentacle.RunningTentacle.InstanceName} --stop --start") +// .Build(); +// +// (Client.Scripts.Models.ScriptExecutionResult ScriptExecutionResult, List ProcessOutput) result; +// +// try +// { +// result = await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// } +// catch (ServiceInvocationHalibutClientException ex) +// { +// Logger.Information(ex, "ServiceInvocationHalibutClientException thrown while Tentacle was restarting itself. This can be ignored for the purpose of this test."); +// +// // Making Tentacle restart itself can cause internal errors with Script Service +// // Execute the script again to get the final result and logs. This will not rerun the script. +// result = await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// } +// +// result.LogExecuteScriptOutput(Logger); +// +// result.ProcessOutput.Any(x => x.Text.Contains("Stopping service")).Should().BeTrue("Stopping service should be logged"); +// result.ScriptExecutionResult.State.Should().Be(ProcessState.Complete); +// +// startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder().Print("Running...")) +// .Build(); +// +// result = await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// result.ProcessOutput.Any(x => x.Text.Contains("Running...")).Should().BeTrue("Running... should be logged"); +// result.ScriptExecutionResult.ExitCode.Should().Be(0); +// result.ScriptExecutionResult.State.Should().Be(ProcessState.Complete); +// } +// } +// } +// } diff --git a/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs b/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs index 8eb83f876..378f1e2e9 100644 --- a/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs @@ -1,257 +1,257 @@ -using System; -using System.IO; -using System.Threading.Tasks; -using FluentAssertions; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils; -using Octopus.Tentacle.CommonTestUtils.Diagnostics; -using Octopus.Tentacle.Scripts; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class WorkspaceCleanerTests : IntegrationTest - { - [Test] - [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] - public async Task WhenScriptServiceIsRunningAndWritesLogFile_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var cleanerDelay = TimeSpan.FromMilliseconds(500); - var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); - - var existingHomeDirectory = new TemporaryDirectory(); - - var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); - var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); - var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacle(b => - { - b.WithHomeDirectory(existingHomeDirectory) - .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); - }) - .Build(CancellationToken); - - // Start task - var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); - await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), - TimeSpan.FromSeconds(60), - () => throw new Exception("Workspace directory did not get created"), - CancellationToken); - - // Ensure Workspace Cleaning Has Run - var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); - await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); - await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), - TimeSpan.FromSeconds(60), - () => throw new Exception("Workspace directory did not get deleted"), - CancellationToken); - - Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); - - await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); - await runningScriptTask; - } - - [Test] - [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] - public async Task WhenScriptServiceIsRunningAndWritesBootstrapScript_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var cleanerDelay = TimeSpan.FromMilliseconds(500); - var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); - - var existingHomeDirectory = new TemporaryDirectory(); - - var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); - var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); - var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacle(b => - { - b.WithHomeDirectory(existingHomeDirectory) - .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); - }) - .Build(CancellationToken); - - // Start task - var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); - await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), - TimeSpan.FromSeconds(60), - () => throw new Exception("Workspace directory did not get created"), - CancellationToken); - - // Ensure Workspace Cleaning Has Run - var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); - await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectory), "Existing bootstrap file"); - await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), - TimeSpan.FromSeconds(60), - () => throw new Exception("Workspace directory did not get deleted"), - CancellationToken); - - Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); - - await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); - await runningScriptTask; - } - - [Test] - [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] - public async Task WhenCompleteScriptIsNotCalled_ThenWorkspaceShouldGetDeletedWhenScriptFinishesRunning(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var cleanerDelay = TimeSpan.FromMilliseconds(500); - var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("Hello")).Build(); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacle(b => - { - b.WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); - }) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .DecorateAllScriptServicesWith(u => u - .BeforeCompleteScript( - () => throw new NotImplementedException("Force failure to simulate tentacle client crashing, and ensure we do not complete the script"))) - .Build()) - .Build(CancellationToken); - - await AssertionExtensions - .Should(() => clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog())) - .ThrowAsync(); - - var workspaceDirectory = GetWorkspaceDirectoryPath(clientAndTentacle.RunningTentacle.HomeDirectory, startScriptCommand.ScriptTicket.TaskId); - - await Wait.For(() => !Directory.Exists(workspaceDirectory), - TimeSpan.FromSeconds(20), - () => - { - try - { - Directory.Delete(workspaceDirectory, true); - } - catch (Exception) - { - // Deleting a worksapce is best effort and can silently fail if it is in use / locked by something. - // If the cleaner failed to delete the directory and we cannot delete it in the test we can assume that it - // is a valid failure and the test was successful. - - return; - } - - throw new Exception("Workspace directory did not get deleted by the workspace cleaner but the test was able to delete it. This indicates there is an issue in the Tentacle code."); - }, - CancellationToken); - } - - [Test] - [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] - public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithLogFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var cleanerDelay = TimeSpan.FromMilliseconds(500); - var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); - - var existingHomeDirectory = new TemporaryDirectory(); - - var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); - var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); - await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectoryWithLogFile), "Existing log file"); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacle(b => - { - b.WithHomeDirectory(existingHomeDirectory) - .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); - }) - .Build(CancellationToken); - - await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), - TimeSpan.FromSeconds(60), - () => throw new Exception("Workspace directory did not get deleted"), - CancellationToken); - Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); - } - - [Test] - [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] - public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithBootstrapFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var cleanerDelay = TimeSpan.FromMilliseconds(500); - var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); - - var existingHomeDirectory = new TemporaryDirectory(); - - var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); - var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); - await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectoryWithLogFile), "Existing bootstrap file"); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacle(b => - { - b.WithHomeDirectory(existingHomeDirectory) - .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); - }) - .Build(CancellationToken); - - await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), - TimeSpan.FromSeconds(60), - () => throw new Exception("Workspace directory did not get deleted"), - CancellationToken); - Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); - } - - [Test] - [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] - public async Task WhenTentacleStarts_WithWorkspaceYoungerThanThreshold_ThenWorkspaceIsLeftAlone(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var cleanerDelay = TimeSpan.FromMilliseconds(500); - var deleteWorkspacesOlderThan = TimeSpan.FromMinutes(30); - - var existingHomeDirectory = new TemporaryDirectory(); - - var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); - await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacle(b => - { - b.WithHomeDirectory(existingHomeDirectory) - .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); - }) - .Build(CancellationToken); - - await Task.Delay(1000, CancellationToken); - - Directory.Exists(existingWorkspaceDirectory).Should().BeTrue(); - } - - static string GetWorkspaceDirectoryPath(string homeDirectory, string scriptTicket) - { - var workspaceDirectory = Path.Combine( - homeDirectory, - ScriptWorkspaceFactory.WorkDirectory, - scriptTicket); - return workspaceDirectory; - } - - static string GivenExistingWorkspaceExists(TemporaryDirectory existingHomeDirectory) - { - var existingWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, Guid.NewGuid().ToString()); - Directory.CreateDirectory(existingWorkspaceDirectory); - return existingWorkspaceDirectory; - } - - static string GetBootstrapScriptFilePath(string workingDirectory) - { - return !PlatformDetection.IsRunningOnWindows - ? BashScriptWorkspace.GetBashBootstrapScriptFilePath(workingDirectory) - : ScriptWorkspace.GetBootstrapScriptFilePath(workingDirectory); - } - } -} \ No newline at end of file +// using System; +// using System.IO; +// using System.Threading.Tasks; +// using FluentAssertions; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils; +// using Octopus.Tentacle.CommonTestUtils.Diagnostics; +// using Octopus.Tentacle.Scripts; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class WorkspaceCleanerTests : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] +// public async Task WhenScriptServiceIsRunningAndWritesLogFile_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var cleanerDelay = TimeSpan.FromMilliseconds(500); +// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); +// +// var existingHomeDirectory = new TemporaryDirectory(); +// +// var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); +// var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacle(b => +// { +// b.WithHomeDirectory(existingHomeDirectory) +// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); +// }) +// .Build(CancellationToken); +// +// // Start task +// var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); +// await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Workspace directory did not get created"), +// CancellationToken); +// +// // Ensure Workspace Cleaning Has Run +// var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); +// await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); +// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Workspace directory did not get deleted"), +// CancellationToken); +// +// Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); +// +// await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); +// await runningScriptTask; +// } +// +// [Test] +// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] +// public async Task WhenScriptServiceIsRunningAndWritesBootstrapScript_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var cleanerDelay = TimeSpan.FromMilliseconds(500); +// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); +// +// var existingHomeDirectory = new TemporaryDirectory(); +// +// var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); +// var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacle(b => +// { +// b.WithHomeDirectory(existingHomeDirectory) +// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); +// }) +// .Build(CancellationToken); +// +// // Start task +// var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); +// await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Workspace directory did not get created"), +// CancellationToken); +// +// // Ensure Workspace Cleaning Has Run +// var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); +// await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectory), "Existing bootstrap file"); +// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Workspace directory did not get deleted"), +// CancellationToken); +// +// Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); +// +// await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); +// await runningScriptTask; +// } +// +// [Test] +// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] +// public async Task WhenCompleteScriptIsNotCalled_ThenWorkspaceShouldGetDeletedWhenScriptFinishesRunning(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var cleanerDelay = TimeSpan.FromMilliseconds(500); +// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("Hello")).Build(); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacle(b => +// { +// b.WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); +// }) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .DecorateAllScriptServicesWith(u => u +// .BeforeCompleteScript( +// () => throw new NotImplementedException("Force failure to simulate tentacle client crashing, and ensure we do not complete the script"))) +// .Build()) +// .Build(CancellationToken); +// +// await AssertionExtensions +// .Should(() => clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog())) +// .ThrowAsync(); +// +// var workspaceDirectory = GetWorkspaceDirectoryPath(clientAndTentacle.RunningTentacle.HomeDirectory, startScriptCommand.ScriptTicket.TaskId); +// +// await Wait.For(() => !Directory.Exists(workspaceDirectory), +// TimeSpan.FromSeconds(20), +// () => +// { +// try +// { +// Directory.Delete(workspaceDirectory, true); +// } +// catch (Exception) +// { +// // Deleting a worksapce is best effort and can silently fail if it is in use / locked by something. +// // If the cleaner failed to delete the directory and we cannot delete it in the test we can assume that it +// // is a valid failure and the test was successful. +// +// return; +// } +// +// throw new Exception("Workspace directory did not get deleted by the workspace cleaner but the test was able to delete it. This indicates there is an issue in the Tentacle code."); +// }, +// CancellationToken); +// } +// +// [Test] +// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] +// public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithLogFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var cleanerDelay = TimeSpan.FromMilliseconds(500); +// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); +// +// var existingHomeDirectory = new TemporaryDirectory(); +// +// var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); +// var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); +// await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectoryWithLogFile), "Existing log file"); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacle(b => +// { +// b.WithHomeDirectory(existingHomeDirectory) +// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); +// }) +// .Build(CancellationToken); +// +// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Workspace directory did not get deleted"), +// CancellationToken); +// Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); +// } +// +// [Test] +// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] +// public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithBootstrapFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var cleanerDelay = TimeSpan.FromMilliseconds(500); +// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); +// +// var existingHomeDirectory = new TemporaryDirectory(); +// +// var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); +// var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); +// await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectoryWithLogFile), "Existing bootstrap file"); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacle(b => +// { +// b.WithHomeDirectory(existingHomeDirectory) +// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); +// }) +// .Build(CancellationToken); +// +// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Workspace directory did not get deleted"), +// CancellationToken); +// Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); +// } +// +// [Test] +// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] +// public async Task WhenTentacleStarts_WithWorkspaceYoungerThanThreshold_ThenWorkspaceIsLeftAlone(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var cleanerDelay = TimeSpan.FromMilliseconds(500); +// var deleteWorkspacesOlderThan = TimeSpan.FromMinutes(30); +// +// var existingHomeDirectory = new TemporaryDirectory(); +// +// var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); +// await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacle(b => +// { +// b.WithHomeDirectory(existingHomeDirectory) +// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); +// }) +// .Build(CancellationToken); +// +// await Task.Delay(1000, CancellationToken); +// +// Directory.Exists(existingWorkspaceDirectory).Should().BeTrue(); +// } +// +// static string GetWorkspaceDirectoryPath(string homeDirectory, string scriptTicket) +// { +// var workspaceDirectory = Path.Combine( +// homeDirectory, +// ScriptWorkspaceFactory.WorkDirectory, +// scriptTicket); +// return workspaceDirectory; +// } +// +// static string GivenExistingWorkspaceExists(TemporaryDirectory existingHomeDirectory) +// { +// var existingWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, Guid.NewGuid().ToString()); +// Directory.CreateDirectory(existingWorkspaceDirectory); +// return existingWorkspaceDirectory; +// } +// +// static string GetBootstrapScriptFilePath(string workingDirectory) +// { +// return !PlatformDetection.IsRunningOnWindows +// ? BashScriptWorkspace.GetBashBootstrapScriptFilePath(workingDirectory) +// : ScriptWorkspace.GetBootstrapScriptFilePath(workingDirectory); +// } +// } +// } \ No newline at end of file From aa3202cb1442d1c3eacf6314cfb785bc9259aa72 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Wed, 23 Oct 2024 12:22:25 +1300 Subject: [PATCH 13/34] Put back CapabilitiesServiceV2Test.cs --- .../CapabilitiesServiceV2Test.cs | 252 +++++++++--------- 1 file changed, 126 insertions(+), 126 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/CapabilitiesServiceV2Test.cs b/source/Octopus.Tentacle.Tests.Integration/CapabilitiesServiceV2Test.cs index bbe894108..6f9b7300a 100644 --- a/source/Octopus.Tentacle.Tests.Integration/CapabilitiesServiceV2Test.cs +++ b/source/Octopus.Tentacle.Tests.Integration/CapabilitiesServiceV2Test.cs @@ -1,126 +1,126 @@ -// #nullable enable -// using System; -// using System.Collections.Generic; -// using System.Threading.Tasks; -// using FluentAssertions; -// using Halibut; -// using NUnit.Framework; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Contracts.Capabilities; -// using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1; -// using Octopus.Tentacle.Contracts.ScriptServiceV2; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class CapabilitiesServiceV2Test : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(testCapabilitiesServiceVersions: true)] -// public async Task CapabilitiesFromAnOlderTentacleWhichHasNoCapabilitiesService_WorksWithTheBackwardsCompatabilityDecorator(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// Version? version = tentacleConfigurationTestCase.Version; -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); -// -// var capabilities = (await clientAndTentacle.TentacleClient.CapabilitiesServiceV2.GetCapabilitiesAsync(new(CancellationToken))).SupportedCapabilities; -// -// capabilities.Should().Contain(nameof(IScriptService)); -// capabilities.Should().Contain(nameof(IFileTransferService)); -// -// //all versions have ScriptServiceV1 & IFileTransferService -// var expectedCapabilitiesCount = 2; -// if (version.HasScriptServiceV2()) -// { -// capabilities.Should().Contain(nameof(IScriptServiceV2)); -// expectedCapabilitiesCount++; -// } -// -// capabilities.Count.Should().Be(expectedCapabilitiesCount); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task CapabilitiesServiceDoesNotReturnKubernetesScriptServiceForNonKubernetesTentacle(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var version = tentacleConfigurationTestCase.Version; -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); -// -// var capabilities = (await clientAndTentacle.TentacleClient.CapabilitiesServiceV2.GetCapabilitiesAsync(new(CancellationToken))).SupportedCapabilities; -// -// capabilities.Should().Contain(nameof(IScriptService)); -// capabilities.Should().Contain(nameof(IFileTransferService)); -// -// //all versions have ScriptServiceV1 & IFileTransferService -// var expectedCapabilitiesCount = 2; -// if (version.HasScriptServiceV2()) -// { -// capabilities.Should().Contain(nameof(IScriptServiceV2)); -// expectedCapabilitiesCount++; -// } -// -// capabilities.Should().NotContain(nameof(IKubernetesScriptServiceV1)); -// -// capabilities.Count.Should().Be(expectedCapabilitiesCount); -// } -// -// [Test] -// [TentacleConfigurations(testCapabilitiesServiceVersions: true)] -// public async Task CapabilitiesResponseShouldBeCached(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var capabilitiesResponses = new List(); -// var resumePortForwarder = false; -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPortForwarder(out var portForwarder) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .DecorateCapabilitiesServiceV2With(d => d -// .DecorateGetCapabilitiesWith( -// async (inner, options) => -// { -// var response = await inner.GetCapabilitiesAsync(options); -// -// capabilitiesResponses.Add(response); -// -// if (resumePortForwarder) -// { -// // (2) Once a get capabilities call has been made which uses the cached response then resume normal RPC calls -// // to allow script execution to continue -// portForwarder.Value.ReturnToNormalMode(); -// } -// -// return response; -// })) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Print("Running...")) -// .Build(); -// -// await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// -// // (1) Kill new and existing connections to ensure no RPC calls can be made -// clientAndTentacle.PortForwarder!.EnterKillNewAndExistingConnectionsMode(); -// resumePortForwarder = true; -// -// try -// { -// await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// } -// catch (HalibutClientException) when (tentacleConfigurationTestCase.TentacleType == TentacleType.Polling && tentacleConfigurationTestCase.Version != null) -// { -// // For script execution on a tentacle without ScriptServiceV2 and retries a polling request can be de-queued into a broken TCP Connection -// // By the time this happens we will have already called gt capabilities and got the cached response so we can safely ignore. -// } -// -// capabilitiesResponses.Should().HaveCount(2); -// capabilitiesResponses[0].Should().BeEquivalentTo(capabilitiesResponses[1]); -// } -// } -// } +#nullable enable +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using FluentAssertions; +using Halibut; +using NUnit.Framework; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.Capabilities; +using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1; +using Octopus.Tentacle.Contracts.ScriptServiceV2; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Util.Builders; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + public class CapabilitiesServiceV2Test : IntegrationTest + { + [Test] + [TentacleConfigurations(testCapabilitiesServiceVersions: true)] + public async Task CapabilitiesFromAnOlderTentacleWhichHasNoCapabilitiesService_WorksWithTheBackwardsCompatabilityDecorator(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + Version? version = tentacleConfigurationTestCase.Version; + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); + + var capabilities = (await clientAndTentacle.TentacleClient.CapabilitiesServiceV2.GetCapabilitiesAsync(new(CancellationToken))).SupportedCapabilities; + + capabilities.Should().Contain(nameof(IScriptService)); + capabilities.Should().Contain(nameof(IFileTransferService)); + + //all versions have ScriptServiceV1 & IFileTransferService + var expectedCapabilitiesCount = 2; + if (version.HasScriptServiceV2()) + { + capabilities.Should().Contain(nameof(IScriptServiceV2)); + expectedCapabilitiesCount++; + } + + capabilities.Count.Should().Be(expectedCapabilitiesCount); + } + + [Test] + [TentacleConfigurations] + public async Task CapabilitiesServiceDoesNotReturnKubernetesScriptServiceForNonKubernetesTentacle(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var version = tentacleConfigurationTestCase.Version; + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); + + var capabilities = (await clientAndTentacle.TentacleClient.CapabilitiesServiceV2.GetCapabilitiesAsync(new(CancellationToken))).SupportedCapabilities; + + capabilities.Should().Contain(nameof(IScriptService)); + capabilities.Should().Contain(nameof(IFileTransferService)); + + //all versions have ScriptServiceV1 & IFileTransferService + var expectedCapabilitiesCount = 2; + if (version.HasScriptServiceV2()) + { + capabilities.Should().Contain(nameof(IScriptServiceV2)); + expectedCapabilitiesCount++; + } + + capabilities.Should().NotContain(nameof(IKubernetesScriptServiceV1)); + + capabilities.Count.Should().Be(expectedCapabilitiesCount); + } + + [Test] + [TentacleConfigurations(testCapabilitiesServiceVersions: true)] + public async Task CapabilitiesResponseShouldBeCached(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var capabilitiesResponses = new List(); + var resumePortForwarder = false; + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPortForwarder(out var portForwarder) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .DecorateCapabilitiesServiceV2With(d => d + .DecorateGetCapabilitiesWith( + async (inner, options) => + { + var response = await inner.GetCapabilitiesAsync(options); + + capabilitiesResponses.Add(response); + + if (resumePortForwarder) + { + // (2) Once a get capabilities call has been made which uses the cached response then resume normal RPC calls + // to allow script execution to continue + portForwarder.Value.ReturnToNormalMode(); + } + + return response; + })) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Print("Running...")) + .Build(); + + await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + + // (1) Kill new and existing connections to ensure no RPC calls can be made + clientAndTentacle.PortForwarder!.EnterKillNewAndExistingConnectionsMode(); + resumePortForwarder = true; + + try + { + await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + } + catch (HalibutClientException) when (tentacleConfigurationTestCase.TentacleType == TentacleType.Polling && tentacleConfigurationTestCase.Version != null) + { + // For script execution on a tentacle without ScriptServiceV2 and retries a polling request can be de-queued into a broken TCP Connection + // By the time this happens we will have already called gt capabilities and got the cached response so we can safely ignore. + } + + capabilitiesResponses.Should().HaveCount(2); + capabilitiesResponses[0].Should().BeEquivalentTo(capabilitiesResponses[1]); + } + } +} From fc2c6e53573a4b1d101705b2b3f5444b4ea6ff58 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Wed, 23 Oct 2024 13:59:03 +1300 Subject: [PATCH 14/34] Put back ClientFileTransferRetriesTimeout.cs --- .../ClientFileTransferRetriesTimeout.cs | 528 +++++++++--------- 1 file changed, 264 insertions(+), 264 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs index f8659b33e..f33ba0bf0 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs @@ -1,264 +1,264 @@ -// using System; -// using System.Collections; -// using System.Diagnostics; -// using System.IO; -// using System.Threading.Tasks; -// using FluentAssertions; -// using Halibut; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils.Diagnostics; -// using Octopus.Tentacle.Contracts.ClientServices; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; -// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; -// using Octopus.TestPortForwarder; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// /// -// /// These tests make sure that we can cancel or walk away (if code does not cooperate with cancellation tokens) -// /// from RPC calls when they are being retried and the rpc timeout period elapses. -// /// -// [IntegrationTestTimeout] -// [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] -// public class ClientFileTransferRetriesTimeout : IntegrationTest -// { -// readonly TimeSpan retryIfRemainingDurationAtLeastBuffer = TimeSpan.FromSeconds(1); -// readonly TimeSpan retryBackoffBuffer = TimeSpan.FromSeconds(2); -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None, additionalParameterTypes: new object[] { typeof(StopPortForwarderAfterFirstCallValues) })] -// public async Task WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, bool stopPortForwarderAfterFirstCall) -// { -// PortForwarder portForwarder = null!; -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// // Set a short retry duration so we cancel fairly quickly -// .WithRetryDuration(TimeSpan.FromSeconds(15)) -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var methodUsages) -// .DecorateFileTransferServiceWith(d => d -// .BeforeUploadFile( -// async () => -// { -// if (methodUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException is null) -// { -// // Ensure there is an active connection so it can be killed correctly -// await tcpConnectionUtilities.RestartTcpConnection(); -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// else -// { -// if (stopPortForwarderAfterFirstCall) -// { -// // Kill the port forwarder so the next requests are in the connecting state when retries timeout -// Logger.Information("Killing PortForwarder"); -// portForwarder!.EnterKillNewAndExistingConnectionsMode(); -// } -// else -// { -// // Ensure there is an active connection so it can be killed correctly -// await tcpConnectionUtilities.RestartTcpConnection(); -// // Pause the port forwarder so the next requests are in-flight when retries timeout -// responseMessageTcpKiller.PauseConnectionOnNextResponse(); -// } -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// portForwarder = clientAndTentacle.PortForwarder; -// -// var inMemoryLog = new InMemoryLog(); -// -// var remotePath = Path.Combine(clientAndTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); -// var dataStream = DataStream.FromString("The Stream"); -// -// // Start the script which will wait for a file to exist -// var duration = Stopwatch.StartNew(); -// var executeScriptTask = clientAndTentacle.TentacleClient.UploadFile(remotePath, dataStream, CancellationToken, inMemoryLog); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// duration.Stop(); -// -// methodUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().BeGreaterOrEqualTo(2); -// methodUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(0); -// -// // Ensure we actually waited and retried until the timeout policy kicked in -// duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); -// -// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task WhenUploadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var retryDuration = TimeSpan.FromSeconds(15); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// // Set a short retry duration so we cancel fairly quickly -// .WithRetryDuration(retryDuration) -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var methodUsages) -// .DecorateFileTransferServiceWith(d => d -// .BeforeUploadFile( -// async () => -// { -// await tcpConnectionUtilities.RestartTcpConnection(); -// -// // Sleep to make the initial RPC call take longer than the allowed retry duration -// await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); -// -// // Kill the first UploadFile call to force the rpc call into retries -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// })) -// .Build()) -// .Build(CancellationToken); -// -// var inMemoryLog = new InMemoryLog(); -// var remotePath = Path.Combine(clientAndTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); -// var dataStream = DataStream.FromString("The Stream"); -// var executeScriptTask = clientAndTentacle.TentacleClient.UploadFile(remotePath, dataStream, CancellationToken, inMemoryLog); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// methodUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(1); -// methodUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(0); -// -// inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None, additionalParameterTypes: new object[] { typeof(StopPortForwarderAfterFirstCallValues) })] -// public async Task WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, bool stopPortForwarderAfterFirstCall) -// { -// PortForwarder portForwarder = null!; -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// // Set a short retry duration so we cancel fairly quickly -// .WithRetryDuration(TimeSpan.FromSeconds(15)) -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var recordedUsages) -// .DecorateFileTransferServiceWith(d => d -// .BeforeDownloadFile( -// async () => -// { -// if (recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException is null) -// { -// // Ensure there is an active connection so it can be killed correctly -// await tcpConnectionUtilities.RestartTcpConnection(); -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// else -// { -// -// if (stopPortForwarderAfterFirstCall) -// { -// // Kill the port forwarder so the next requests are in the connecting state when retries timeout -// Logger.Information("Killing PortForwarder"); -// portForwarder!.Dispose(); -// } -// else -// { -// // Ensure there is an active connection so it can be killed correctly -// await tcpConnectionUtilities.RestartTcpConnection(); -// // Pause the port forwarder so the next requests are in-flight when retries timeout -// responseMessageTcpKiller.PauseConnectionOnNextResponse(); -// } -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// portForwarder = clientAndTentacle.PortForwarder; -// -// using var tempFile = new RandomTemporaryFileBuilder().Build(); -// -// var inMemoryLog = new InMemoryLog(); -// -// // Start the script which will wait for a file to exist -// var duration = Stopwatch.StartNew(); -// var executeScriptTask = clientAndTentacle.TentacleClient.DownloadFile(tempFile.File.FullName, CancellationToken, inMemoryLog); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// duration.Stop(); -// -// recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().BeGreaterOrEqualTo(2); -// recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(0); -// -// // Ensure we actually waited and retried until the timeout policy kicked in -// duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); -// -// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var retryDuration = TimeSpan.FromSeconds(15); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// // Set a short retry duration so we cancel fairly quickly -// .WithRetryDuration(retryDuration) -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var recordedUsages) -// .DecorateFileTransferServiceWith(d => d -// .BeforeDownloadFile( -// async () => -// { -// await tcpConnectionUtilities.RestartTcpConnection(); -// -// // Sleep to make the initial RPC call take longer than the allowed retry duration -// await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); -// -// // Kill the first DownloadFile call to force the rpc call into retries -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// })) -// .Build()) -// .Build(CancellationToken); -// -// using var tempFile = new RandomTemporaryFileBuilder().Build(); -// var inMemoryLog = new InMemoryLog(); -// var executeScriptTask = clientAndTentacle.TentacleClient.DownloadFile(tempFile.File.FullName, CancellationToken, inMemoryLog); -// -// Func> action = async () => await executeScriptTask; -// await action.Should().ThrowAsync(); -// -// recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(0); -// -// inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); -// } -// -// class StopPortForwarderAfterFirstCallValues : IEnumerable -// { -// public IEnumerator GetEnumerator() -// { -// yield return true; -// yield return false; -// } -// } -// } -// } \ No newline at end of file +using System; +using System.Collections; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using Halibut; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils.Diagnostics; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; +using Octopus.TestPortForwarder; + +namespace Octopus.Tentacle.Tests.Integration +{ + /// + /// These tests make sure that we can cancel or walk away (if code does not cooperate with cancellation tokens) + /// from RPC calls when they are being retried and the rpc timeout period elapses. + /// + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] + public class ClientFileTransferRetriesTimeout : IntegrationTest + { + readonly TimeSpan retryIfRemainingDurationAtLeastBuffer = TimeSpan.FromSeconds(1); + readonly TimeSpan retryBackoffBuffer = TimeSpan.FromSeconds(2); + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None, additionalParameterTypes: new object[] { typeof(StopPortForwarderAfterFirstCallValues) })] + public async Task WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, bool stopPortForwarderAfterFirstCall) + { + PortForwarder portForwarder = null!; + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + // Set a short retry duration so we cancel fairly quickly + .WithRetryDuration(TimeSpan.FromSeconds(15)) + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var methodUsages) + .DecorateFileTransferServiceWith(d => d + .BeforeUploadFile( + async () => + { + if (methodUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException is null) + { + // Ensure there is an active connection so it can be killed correctly + await tcpConnectionUtilities.RestartTcpConnection(); + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + else + { + if (stopPortForwarderAfterFirstCall) + { + // Kill the port forwarder so the next requests are in the connecting state when retries timeout + Logger.Information("Killing PortForwarder"); + portForwarder!.EnterKillNewAndExistingConnectionsMode(); + } + else + { + // Ensure there is an active connection so it can be killed correctly + await tcpConnectionUtilities.RestartTcpConnection(); + // Pause the port forwarder so the next requests are in-flight when retries timeout + responseMessageTcpKiller.PauseConnectionOnNextResponse(); + } + } + })) + .Build()) + .Build(CancellationToken); + + portForwarder = clientAndTentacle.PortForwarder; + + var inMemoryLog = new InMemoryLog(); + + var remotePath = Path.Combine(clientAndTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); + var dataStream = DataStream.FromString("The Stream"); + + // Start the script which will wait for a file to exist + var duration = Stopwatch.StartNew(); + var executeScriptTask = clientAndTentacle.TentacleClient.UploadFile(remotePath, dataStream, CancellationToken, inMemoryLog); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + duration.Stop(); + + methodUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().BeGreaterOrEqualTo(2); + methodUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(0); + + // Ensure we actually waited and retried until the timeout policy kicked in + duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); + + inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task WhenUploadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var retryDuration = TimeSpan.FromSeconds(15); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + // Set a short retry duration so we cancel fairly quickly + .WithRetryDuration(retryDuration) + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var methodUsages) + .DecorateFileTransferServiceWith(d => d + .BeforeUploadFile( + async () => + { + await tcpConnectionUtilities.RestartTcpConnection(); + + // Sleep to make the initial RPC call take longer than the allowed retry duration + await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); + + // Kill the first UploadFile call to force the rpc call into retries + responseMessageTcpKiller.KillConnectionOnNextResponse(); + })) + .Build()) + .Build(CancellationToken); + + var inMemoryLog = new InMemoryLog(); + var remotePath = Path.Combine(clientAndTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); + var dataStream = DataStream.FromString("The Stream"); + var executeScriptTask = clientAndTentacle.TentacleClient.UploadFile(remotePath, dataStream, CancellationToken, inMemoryLog); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + methodUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(1); + methodUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(0); + + inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None, additionalParameterTypes: new object[] { typeof(StopPortForwarderAfterFirstCallValues) })] + public async Task WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, bool stopPortForwarderAfterFirstCall) + { + PortForwarder portForwarder = null!; + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + // Set a short retry duration so we cancel fairly quickly + .WithRetryDuration(TimeSpan.FromSeconds(15)) + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var recordedUsages) + .DecorateFileTransferServiceWith(d => d + .BeforeDownloadFile( + async () => + { + if (recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException is null) + { + // Ensure there is an active connection so it can be killed correctly + await tcpConnectionUtilities.RestartTcpConnection(); + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + else + { + + if (stopPortForwarderAfterFirstCall) + { + // Kill the port forwarder so the next requests are in the connecting state when retries timeout + Logger.Information("Killing PortForwarder"); + portForwarder!.Dispose(); + } + else + { + // Ensure there is an active connection so it can be killed correctly + await tcpConnectionUtilities.RestartTcpConnection(); + // Pause the port forwarder so the next requests are in-flight when retries timeout + responseMessageTcpKiller.PauseConnectionOnNextResponse(); + } + } + })) + .Build()) + .Build(CancellationToken); + + portForwarder = clientAndTentacle.PortForwarder; + + using var tempFile = new RandomTemporaryFileBuilder().Build(); + + var inMemoryLog = new InMemoryLog(); + + // Start the script which will wait for a file to exist + var duration = Stopwatch.StartNew(); + var executeScriptTask = clientAndTentacle.TentacleClient.DownloadFile(tempFile.File.FullName, CancellationToken, inMemoryLog); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + duration.Stop(); + + recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().BeGreaterOrEqualTo(2); + recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(0); + + // Ensure we actually waited and retried until the timeout policy kicked in + duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); + + inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var retryDuration = TimeSpan.FromSeconds(15); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + // Set a short retry duration so we cancel fairly quickly + .WithRetryDuration(retryDuration) + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var recordedUsages) + .DecorateFileTransferServiceWith(d => d + .BeforeDownloadFile( + async () => + { + await tcpConnectionUtilities.RestartTcpConnection(); + + // Sleep to make the initial RPC call take longer than the allowed retry duration + await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); + + // Kill the first DownloadFile call to force the rpc call into retries + responseMessageTcpKiller.KillConnectionOnNextResponse(); + })) + .Build()) + .Build(CancellationToken); + + using var tempFile = new RandomTemporaryFileBuilder().Build(); + var inMemoryLog = new InMemoryLog(); + var executeScriptTask = clientAndTentacle.TentacleClient.DownloadFile(tempFile.File.FullName, CancellationToken, inMemoryLog); + + Func> action = async () => await executeScriptTask; + await action.Should().ThrowAsync(); + + recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(0); + + inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); + } + + class StopPortForwarderAfterFirstCallValues : IEnumerable + { + public IEnumerator GetEnumerator() + { + yield return true; + yield return false; + } + } + } +} \ No newline at end of file From 250f894c010f076cab55b6fc8e4f7a6186078585 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Wed, 23 Oct 2024 14:40:14 +1300 Subject: [PATCH 15/34] Put back ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs and ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs --- ...fersAreNotRetriedWhenRetriesAreDisabled.cs | 198 ++++++++-------- ...ransfersAreRetriedWhenRetriesAreEnabled.cs | 216 +++++++++--------- 2 files changed, 207 insertions(+), 207 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs index df7969d42..cb5876c7f 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs @@ -1,99 +1,99 @@ -// using System; -// using System.IO; -// using System.Threading.Tasks; -// using FluentAssertions; -// using Halibut; -// using NUnit.Framework; -// using Octopus.Tentacle.Contracts.ClientServices; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] -// public class ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task FailedUploadsAreNotRetriedAndFail(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithRetriesDisabled() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var recordedUsages) -// .DecorateFileTransferServiceWith(d => d -// .BeforeUploadFile( -// async () => -// { -// await tcpConnectionUtilities.RestartTcpConnection(); -// -// // Only kill the connection the first time, causing the upload -// // to succeed - and therefore failing the test - if retries are attempted -// if (recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException is null) -// { -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); -// var uploadFileTask = clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await uploadFileTask).ThrowExceptionContractAsync(expectedException); -// -// recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException.Should().NotBeNull(); -// recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(1); -// } -// -// [Test] -// [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task FailedDownloadsAreNotRetriedAndFail(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithRetriesDisabled() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var recordedUsages) -// .DecorateFileTransferServiceWith(d => d -// .BeforeDownloadFile( -// async () => -// { -// await tcpConnectionUtilities.RestartTcpConnection(); -// -// // Only kill the connection the first time, causing the upload -// // to succeed - and therefore failing the test - if retries are attempted -// if (recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException is null) -// { -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); -// -// await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); -// var downloadFileTask = clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await downloadFileTask).ThrowExceptionContractAsync(expectedException); -// -// recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException.Should().NotBeNull(); -// recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(1); -// } -// } -// } \ No newline at end of file +using System; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using Halibut; +using NUnit.Framework; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] + public class ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled : IntegrationTest + { + [Test] + [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task FailedUploadsAreNotRetriedAndFail(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithRetriesDisabled() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var recordedUsages) + .DecorateFileTransferServiceWith(d => d + .BeforeUploadFile( + async () => + { + await tcpConnectionUtilities.RestartTcpConnection(); + + // Only kill the connection the first time, causing the upload + // to succeed - and therefore failing the test - if retries are attempted + if (recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException is null) + { + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + })) + .Build()) + .Build(CancellationToken); + + var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); + var uploadFileTask = clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); + + await AssertionExtensions.Should(async () => await uploadFileTask).ThrowExceptionContractAsync(expectedException); + + recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException.Should().NotBeNull(); + recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(1); + } + + [Test] + [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task FailedDownloadsAreNotRetriedAndFail(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithRetriesDisabled() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var recordedUsages) + .DecorateFileTransferServiceWith(d => d + .BeforeDownloadFile( + async () => + { + await tcpConnectionUtilities.RestartTcpConnection(); + + // Only kill the connection the first time, causing the upload + // to succeed - and therefore failing the test - if retries are attempted + if (recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException is null) + { + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + })) + .Build()) + .Build(CancellationToken); + + var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); + + await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); + var downloadFileTask = clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); + + await AssertionExtensions.Should(async () => await downloadFileTask).ThrowExceptionContractAsync(expectedException); + + recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException.Should().NotBeNull(); + recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(1); + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs index c823c40e1..eef4f972f 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs @@ -1,108 +1,108 @@ -// using System; -// using System.IO; -// using System.Threading.Tasks; -// using FluentAssertions; -// using Halibut; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils; -// using Octopus.Tentacle.CommonTestUtils.Diagnostics; -// using Octopus.Tentacle.Contracts.ClientServices; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; -// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// public class ClientFileTransfersAreRetriedWhenRetriesAreEnabled : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] -// [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] -// public async Task FailedUploadsAreRetriedAndIsEventuallySuccessful(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var recordedUsages) -// .DecorateFileTransferServiceWith(d => d -// .BeforeUploadFile( -// async () => -// { -// await tcpConnectionUtilities.RestartTcpConnection(); -// -// // Only kill the connection the first time, causing the upload -// // to succeed - and therefore failing the test - if retries are attempted -// if (recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException is null) -// { -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var inMemoryLog = new InMemoryLog(); -// -// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); -// -// var res = await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken, inMemoryLog); -// res.Length.Should().Be(5); -// -// recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException.Should().NotBeNull(); -// recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(2); -// -// var downloadFile = await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); -// var actuallySent = await downloadFile.GetUtf8String(CancellationToken); -// actuallySent.Should().Be("Hello"); -// -// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); -// } -// -// [Test] -// [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task FailedDownloadsAreRetriedAndIsEventuallySuccessful(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var recordedUsages) -// .DecorateFileTransferServiceWith(d => d -// .BeforeDownloadFile( -// async () => -// { -// await tcpConnectionUtilities.RestartTcpConnection(); -// -// // Only kill the connection the first time, causing the upload -// // to succeed - and therefore failing the test - if retries are attempted -// if (recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException is null) -// { -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var inMemoryLog = new InMemoryLog(); -// -// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); -// -// await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); -// var downloadFile = await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken, inMemoryLog); -// var actuallySent = await downloadFile.GetUtf8String(CancellationToken); -// -// recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException.Should().NotBeNull(); -// recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(2); -// -// actuallySent.Should().Be("Hello"); -// -// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); -// } -// } -// } +using System; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using Halibut; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils; +using Octopus.Tentacle.CommonTestUtils.Diagnostics; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; + +namespace Octopus.Tentacle.Tests.Integration +{ + public class ClientFileTransfersAreRetriedWhenRetriesAreEnabled : IntegrationTest + { + [Test] + [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] + public async Task FailedUploadsAreRetriedAndIsEventuallySuccessful(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var recordedUsages) + .DecorateFileTransferServiceWith(d => d + .BeforeUploadFile( + async () => + { + await tcpConnectionUtilities.RestartTcpConnection(); + + // Only kill the connection the first time, causing the upload + // to succeed - and therefore failing the test - if retries are attempted + if (recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException is null) + { + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + })) + .Build()) + .Build(CancellationToken); + + var inMemoryLog = new InMemoryLog(); + + var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); + + var res = await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken, inMemoryLog); + res.Length.Should().Be(5); + + recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).LastException.Should().NotBeNull(); + recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(2); + + var downloadFile = await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); + var actuallySent = await downloadFile.GetUtf8String(CancellationToken); + actuallySent.Should().Be("Hello"); + + inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); + } + + [Test] + [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task FailedDownloadsAreRetriedAndIsEventuallySuccessful(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var recordedUsages) + .DecorateFileTransferServiceWith(d => d + .BeforeDownloadFile( + async () => + { + await tcpConnectionUtilities.RestartTcpConnection(); + + // Only kill the connection the first time, causing the upload + // to succeed - and therefore failing the test - if retries are attempted + if (recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException is null) + { + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + })) + .Build()) + .Build(CancellationToken); + + var inMemoryLog = new InMemoryLog(); + + var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); + + await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); + var downloadFile = await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken, inMemoryLog); + var actuallySent = await downloadFile.GetUtf8String(CancellationToken); + + recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).LastException.Should().NotBeNull(); + recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(2); + + actuallySent.Should().Be("Hello"); + + inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); + } + } +} From 14ac9943eb7ccdaa63d75c55efc305f255e56f3f Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Wed, 23 Oct 2024 17:10:16 +1300 Subject: [PATCH 16/34] Put back WorkspaceCleanerTests.cs --- .../WorkspaceCleanerTests.cs | 514 +++++++++--------- 1 file changed, 257 insertions(+), 257 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs b/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs index 378f1e2e9..8eb83f876 100644 --- a/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs @@ -1,257 +1,257 @@ -// using System; -// using System.IO; -// using System.Threading.Tasks; -// using FluentAssertions; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils; -// using Octopus.Tentacle.CommonTestUtils.Diagnostics; -// using Octopus.Tentacle.Scripts; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class WorkspaceCleanerTests : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] -// public async Task WhenScriptServiceIsRunningAndWritesLogFile_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var cleanerDelay = TimeSpan.FromMilliseconds(500); -// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); -// -// var existingHomeDirectory = new TemporaryDirectory(); -// -// var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); -// var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacle(b => -// { -// b.WithHomeDirectory(existingHomeDirectory) -// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); -// }) -// .Build(CancellationToken); -// -// // Start task -// var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); -// await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Workspace directory did not get created"), -// CancellationToken); -// -// // Ensure Workspace Cleaning Has Run -// var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); -// await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); -// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Workspace directory did not get deleted"), -// CancellationToken); -// -// Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); -// -// await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); -// await runningScriptTask; -// } -// -// [Test] -// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] -// public async Task WhenScriptServiceIsRunningAndWritesBootstrapScript_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var cleanerDelay = TimeSpan.FromMilliseconds(500); -// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); -// -// var existingHomeDirectory = new TemporaryDirectory(); -// -// var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); -// var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacle(b => -// { -// b.WithHomeDirectory(existingHomeDirectory) -// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); -// }) -// .Build(CancellationToken); -// -// // Start task -// var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); -// await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Workspace directory did not get created"), -// CancellationToken); -// -// // Ensure Workspace Cleaning Has Run -// var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); -// await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectory), "Existing bootstrap file"); -// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Workspace directory did not get deleted"), -// CancellationToken); -// -// Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); -// -// await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); -// await runningScriptTask; -// } -// -// [Test] -// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] -// public async Task WhenCompleteScriptIsNotCalled_ThenWorkspaceShouldGetDeletedWhenScriptFinishesRunning(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var cleanerDelay = TimeSpan.FromMilliseconds(500); -// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("Hello")).Build(); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacle(b => -// { -// b.WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); -// }) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .DecorateAllScriptServicesWith(u => u -// .BeforeCompleteScript( -// () => throw new NotImplementedException("Force failure to simulate tentacle client crashing, and ensure we do not complete the script"))) -// .Build()) -// .Build(CancellationToken); -// -// await AssertionExtensions -// .Should(() => clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog())) -// .ThrowAsync(); -// -// var workspaceDirectory = GetWorkspaceDirectoryPath(clientAndTentacle.RunningTentacle.HomeDirectory, startScriptCommand.ScriptTicket.TaskId); -// -// await Wait.For(() => !Directory.Exists(workspaceDirectory), -// TimeSpan.FromSeconds(20), -// () => -// { -// try -// { -// Directory.Delete(workspaceDirectory, true); -// } -// catch (Exception) -// { -// // Deleting a worksapce is best effort and can silently fail if it is in use / locked by something. -// // If the cleaner failed to delete the directory and we cannot delete it in the test we can assume that it -// // is a valid failure and the test was successful. -// -// return; -// } -// -// throw new Exception("Workspace directory did not get deleted by the workspace cleaner but the test was able to delete it. This indicates there is an issue in the Tentacle code."); -// }, -// CancellationToken); -// } -// -// [Test] -// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] -// public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithLogFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var cleanerDelay = TimeSpan.FromMilliseconds(500); -// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); -// -// var existingHomeDirectory = new TemporaryDirectory(); -// -// var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); -// var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); -// await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectoryWithLogFile), "Existing log file"); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacle(b => -// { -// b.WithHomeDirectory(existingHomeDirectory) -// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); -// }) -// .Build(CancellationToken); -// -// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Workspace directory did not get deleted"), -// CancellationToken); -// Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); -// } -// -// [Test] -// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] -// public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithBootstrapFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var cleanerDelay = TimeSpan.FromMilliseconds(500); -// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); -// -// var existingHomeDirectory = new TemporaryDirectory(); -// -// var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); -// var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); -// await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectoryWithLogFile), "Existing bootstrap file"); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacle(b => -// { -// b.WithHomeDirectory(existingHomeDirectory) -// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); -// }) -// .Build(CancellationToken); -// -// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Workspace directory did not get deleted"), -// CancellationToken); -// Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); -// } -// -// [Test] -// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] -// public async Task WhenTentacleStarts_WithWorkspaceYoungerThanThreshold_ThenWorkspaceIsLeftAlone(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var cleanerDelay = TimeSpan.FromMilliseconds(500); -// var deleteWorkspacesOlderThan = TimeSpan.FromMinutes(30); -// -// var existingHomeDirectory = new TemporaryDirectory(); -// -// var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); -// await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacle(b => -// { -// b.WithHomeDirectory(existingHomeDirectory) -// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); -// }) -// .Build(CancellationToken); -// -// await Task.Delay(1000, CancellationToken); -// -// Directory.Exists(existingWorkspaceDirectory).Should().BeTrue(); -// } -// -// static string GetWorkspaceDirectoryPath(string homeDirectory, string scriptTicket) -// { -// var workspaceDirectory = Path.Combine( -// homeDirectory, -// ScriptWorkspaceFactory.WorkDirectory, -// scriptTicket); -// return workspaceDirectory; -// } -// -// static string GivenExistingWorkspaceExists(TemporaryDirectory existingHomeDirectory) -// { -// var existingWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, Guid.NewGuid().ToString()); -// Directory.CreateDirectory(existingWorkspaceDirectory); -// return existingWorkspaceDirectory; -// } -// -// static string GetBootstrapScriptFilePath(string workingDirectory) -// { -// return !PlatformDetection.IsRunningOnWindows -// ? BashScriptWorkspace.GetBashBootstrapScriptFilePath(workingDirectory) -// : ScriptWorkspace.GetBootstrapScriptFilePath(workingDirectory); -// } -// } -// } \ No newline at end of file +using System; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils; +using Octopus.Tentacle.CommonTestUtils.Diagnostics; +using Octopus.Tentacle.Scripts; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + public class WorkspaceCleanerTests : IntegrationTest + { + [Test] + [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] + public async Task WhenScriptServiceIsRunningAndWritesLogFile_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var cleanerDelay = TimeSpan.FromMilliseconds(500); + var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); + + var existingHomeDirectory = new TemporaryDirectory(); + + var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); + var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); + var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacle(b => + { + b.WithHomeDirectory(existingHomeDirectory) + .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); + }) + .Build(CancellationToken); + + // Start task + var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); + await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), + TimeSpan.FromSeconds(60), + () => throw new Exception("Workspace directory did not get created"), + CancellationToken); + + // Ensure Workspace Cleaning Has Run + var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); + await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); + await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), + TimeSpan.FromSeconds(60), + () => throw new Exception("Workspace directory did not get deleted"), + CancellationToken); + + Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); + + await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); + await runningScriptTask; + } + + [Test] + [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] + public async Task WhenScriptServiceIsRunningAndWritesBootstrapScript_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var cleanerDelay = TimeSpan.FromMilliseconds(500); + var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); + + var existingHomeDirectory = new TemporaryDirectory(); + + var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); + var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); + var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacle(b => + { + b.WithHomeDirectory(existingHomeDirectory) + .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); + }) + .Build(CancellationToken); + + // Start task + var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); + await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), + TimeSpan.FromSeconds(60), + () => throw new Exception("Workspace directory did not get created"), + CancellationToken); + + // Ensure Workspace Cleaning Has Run + var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); + await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectory), "Existing bootstrap file"); + await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), + TimeSpan.FromSeconds(60), + () => throw new Exception("Workspace directory did not get deleted"), + CancellationToken); + + Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); + + await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); + await runningScriptTask; + } + + [Test] + [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] + public async Task WhenCompleteScriptIsNotCalled_ThenWorkspaceShouldGetDeletedWhenScriptFinishesRunning(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var cleanerDelay = TimeSpan.FromMilliseconds(500); + var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("Hello")).Build(); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacle(b => + { + b.WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); + }) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .DecorateAllScriptServicesWith(u => u + .BeforeCompleteScript( + () => throw new NotImplementedException("Force failure to simulate tentacle client crashing, and ensure we do not complete the script"))) + .Build()) + .Build(CancellationToken); + + await AssertionExtensions + .Should(() => clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog())) + .ThrowAsync(); + + var workspaceDirectory = GetWorkspaceDirectoryPath(clientAndTentacle.RunningTentacle.HomeDirectory, startScriptCommand.ScriptTicket.TaskId); + + await Wait.For(() => !Directory.Exists(workspaceDirectory), + TimeSpan.FromSeconds(20), + () => + { + try + { + Directory.Delete(workspaceDirectory, true); + } + catch (Exception) + { + // Deleting a worksapce is best effort and can silently fail if it is in use / locked by something. + // If the cleaner failed to delete the directory and we cannot delete it in the test we can assume that it + // is a valid failure and the test was successful. + + return; + } + + throw new Exception("Workspace directory did not get deleted by the workspace cleaner but the test was able to delete it. This indicates there is an issue in the Tentacle code."); + }, + CancellationToken); + } + + [Test] + [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] + public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithLogFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var cleanerDelay = TimeSpan.FromMilliseconds(500); + var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); + + var existingHomeDirectory = new TemporaryDirectory(); + + var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); + var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); + await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectoryWithLogFile), "Existing log file"); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacle(b => + { + b.WithHomeDirectory(existingHomeDirectory) + .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); + }) + .Build(CancellationToken); + + await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), + TimeSpan.FromSeconds(60), + () => throw new Exception("Workspace directory did not get deleted"), + CancellationToken); + Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); + } + + [Test] + [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] + public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithBootstrapFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var cleanerDelay = TimeSpan.FromMilliseconds(500); + var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); + + var existingHomeDirectory = new TemporaryDirectory(); + + var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); + var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); + await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectoryWithLogFile), "Existing bootstrap file"); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacle(b => + { + b.WithHomeDirectory(existingHomeDirectory) + .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); + }) + .Build(CancellationToken); + + await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), + TimeSpan.FromSeconds(60), + () => throw new Exception("Workspace directory did not get deleted"), + CancellationToken); + Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); + } + + [Test] + [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] + public async Task WhenTentacleStarts_WithWorkspaceYoungerThanThreshold_ThenWorkspaceIsLeftAlone(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var cleanerDelay = TimeSpan.FromMilliseconds(500); + var deleteWorkspacesOlderThan = TimeSpan.FromMinutes(30); + + var existingHomeDirectory = new TemporaryDirectory(); + + var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); + await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacle(b => + { + b.WithHomeDirectory(existingHomeDirectory) + .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); + }) + .Build(CancellationToken); + + await Task.Delay(1000, CancellationToken); + + Directory.Exists(existingWorkspaceDirectory).Should().BeTrue(); + } + + static string GetWorkspaceDirectoryPath(string homeDirectory, string scriptTicket) + { + var workspaceDirectory = Path.Combine( + homeDirectory, + ScriptWorkspaceFactory.WorkDirectory, + scriptTicket); + return workspaceDirectory; + } + + static string GivenExistingWorkspaceExists(TemporaryDirectory existingHomeDirectory) + { + var existingWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, Guid.NewGuid().ToString()); + Directory.CreateDirectory(existingWorkspaceDirectory); + return existingWorkspaceDirectory; + } + + static string GetBootstrapScriptFilePath(string workingDirectory) + { + return !PlatformDetection.IsRunningOnWindows + ? BashScriptWorkspace.GetBashBootstrapScriptFilePath(workingDirectory) + : ScriptWorkspace.GetBootstrapScriptFilePath(workingDirectory); + } + } +} \ No newline at end of file From a63ab990d387caae15a21586deeffbccb16f8a82 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Thu, 24 Oct 2024 03:39:50 +1300 Subject: [PATCH 17/34] Revert "Put back WorkspaceCleanerTests.cs" This reverts commit 14ac9943eb7ccdaa63d75c55efc305f255e56f3f. --- .../WorkspaceCleanerTests.cs | 514 +++++++++--------- 1 file changed, 257 insertions(+), 257 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs b/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs index 8eb83f876..378f1e2e9 100644 --- a/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs @@ -1,257 +1,257 @@ -using System; -using System.IO; -using System.Threading.Tasks; -using FluentAssertions; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils; -using Octopus.Tentacle.CommonTestUtils.Diagnostics; -using Octopus.Tentacle.Scripts; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class WorkspaceCleanerTests : IntegrationTest - { - [Test] - [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] - public async Task WhenScriptServiceIsRunningAndWritesLogFile_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var cleanerDelay = TimeSpan.FromMilliseconds(500); - var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); - - var existingHomeDirectory = new TemporaryDirectory(); - - var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); - var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); - var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacle(b => - { - b.WithHomeDirectory(existingHomeDirectory) - .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); - }) - .Build(CancellationToken); - - // Start task - var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); - await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), - TimeSpan.FromSeconds(60), - () => throw new Exception("Workspace directory did not get created"), - CancellationToken); - - // Ensure Workspace Cleaning Has Run - var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); - await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); - await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), - TimeSpan.FromSeconds(60), - () => throw new Exception("Workspace directory did not get deleted"), - CancellationToken); - - Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); - - await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); - await runningScriptTask; - } - - [Test] - [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] - public async Task WhenScriptServiceIsRunningAndWritesBootstrapScript_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var cleanerDelay = TimeSpan.FromMilliseconds(500); - var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); - - var existingHomeDirectory = new TemporaryDirectory(); - - var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); - var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); - var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacle(b => - { - b.WithHomeDirectory(existingHomeDirectory) - .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); - }) - .Build(CancellationToken); - - // Start task - var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); - await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), - TimeSpan.FromSeconds(60), - () => throw new Exception("Workspace directory did not get created"), - CancellationToken); - - // Ensure Workspace Cleaning Has Run - var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); - await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectory), "Existing bootstrap file"); - await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), - TimeSpan.FromSeconds(60), - () => throw new Exception("Workspace directory did not get deleted"), - CancellationToken); - - Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); - - await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); - await runningScriptTask; - } - - [Test] - [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] - public async Task WhenCompleteScriptIsNotCalled_ThenWorkspaceShouldGetDeletedWhenScriptFinishesRunning(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var cleanerDelay = TimeSpan.FromMilliseconds(500); - var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("Hello")).Build(); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacle(b => - { - b.WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); - }) - .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() - .DecorateAllScriptServicesWith(u => u - .BeforeCompleteScript( - () => throw new NotImplementedException("Force failure to simulate tentacle client crashing, and ensure we do not complete the script"))) - .Build()) - .Build(CancellationToken); - - await AssertionExtensions - .Should(() => clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog())) - .ThrowAsync(); - - var workspaceDirectory = GetWorkspaceDirectoryPath(clientAndTentacle.RunningTentacle.HomeDirectory, startScriptCommand.ScriptTicket.TaskId); - - await Wait.For(() => !Directory.Exists(workspaceDirectory), - TimeSpan.FromSeconds(20), - () => - { - try - { - Directory.Delete(workspaceDirectory, true); - } - catch (Exception) - { - // Deleting a worksapce is best effort and can silently fail if it is in use / locked by something. - // If the cleaner failed to delete the directory and we cannot delete it in the test we can assume that it - // is a valid failure and the test was successful. - - return; - } - - throw new Exception("Workspace directory did not get deleted by the workspace cleaner but the test was able to delete it. This indicates there is an issue in the Tentacle code."); - }, - CancellationToken); - } - - [Test] - [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] - public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithLogFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var cleanerDelay = TimeSpan.FromMilliseconds(500); - var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); - - var existingHomeDirectory = new TemporaryDirectory(); - - var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); - var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); - await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectoryWithLogFile), "Existing log file"); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacle(b => - { - b.WithHomeDirectory(existingHomeDirectory) - .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); - }) - .Build(CancellationToken); - - await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), - TimeSpan.FromSeconds(60), - () => throw new Exception("Workspace directory did not get deleted"), - CancellationToken); - Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); - } - - [Test] - [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] - public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithBootstrapFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var cleanerDelay = TimeSpan.FromMilliseconds(500); - var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); - - var existingHomeDirectory = new TemporaryDirectory(); - - var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); - var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); - await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectoryWithLogFile), "Existing bootstrap file"); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacle(b => - { - b.WithHomeDirectory(existingHomeDirectory) - .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); - }) - .Build(CancellationToken); - - await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), - TimeSpan.FromSeconds(60), - () => throw new Exception("Workspace directory did not get deleted"), - CancellationToken); - Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); - } - - [Test] - [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] - public async Task WhenTentacleStarts_WithWorkspaceYoungerThanThreshold_ThenWorkspaceIsLeftAlone(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - var cleanerDelay = TimeSpan.FromMilliseconds(500); - var deleteWorkspacesOlderThan = TimeSpan.FromMinutes(30); - - var existingHomeDirectory = new TemporaryDirectory(); - - var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); - await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); - - await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() - .WithTentacle(b => - { - b.WithHomeDirectory(existingHomeDirectory) - .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); - }) - .Build(CancellationToken); - - await Task.Delay(1000, CancellationToken); - - Directory.Exists(existingWorkspaceDirectory).Should().BeTrue(); - } - - static string GetWorkspaceDirectoryPath(string homeDirectory, string scriptTicket) - { - var workspaceDirectory = Path.Combine( - homeDirectory, - ScriptWorkspaceFactory.WorkDirectory, - scriptTicket); - return workspaceDirectory; - } - - static string GivenExistingWorkspaceExists(TemporaryDirectory existingHomeDirectory) - { - var existingWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, Guid.NewGuid().ToString()); - Directory.CreateDirectory(existingWorkspaceDirectory); - return existingWorkspaceDirectory; - } - - static string GetBootstrapScriptFilePath(string workingDirectory) - { - return !PlatformDetection.IsRunningOnWindows - ? BashScriptWorkspace.GetBashBootstrapScriptFilePath(workingDirectory) - : ScriptWorkspace.GetBootstrapScriptFilePath(workingDirectory); - } - } -} \ No newline at end of file +// using System; +// using System.IO; +// using System.Threading.Tasks; +// using FluentAssertions; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils; +// using Octopus.Tentacle.CommonTestUtils.Diagnostics; +// using Octopus.Tentacle.Scripts; +// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class WorkspaceCleanerTests : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] +// public async Task WhenScriptServiceIsRunningAndWritesLogFile_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var cleanerDelay = TimeSpan.FromMilliseconds(500); +// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); +// +// var existingHomeDirectory = new TemporaryDirectory(); +// +// var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); +// var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacle(b => +// { +// b.WithHomeDirectory(existingHomeDirectory) +// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); +// }) +// .Build(CancellationToken); +// +// // Start task +// var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); +// await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Workspace directory did not get created"), +// CancellationToken); +// +// // Ensure Workspace Cleaning Has Run +// var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); +// await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); +// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Workspace directory did not get deleted"), +// CancellationToken); +// +// Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); +// +// await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); +// await runningScriptTask; +// } +// +// [Test] +// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] +// public async Task WhenScriptServiceIsRunningAndWritesBootstrapScript_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var cleanerDelay = TimeSpan.FromMilliseconds(500); +// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); +// +// var existingHomeDirectory = new TemporaryDirectory(); +// +// var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); +// var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacle(b => +// { +// b.WithHomeDirectory(existingHomeDirectory) +// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); +// }) +// .Build(CancellationToken); +// +// // Start task +// var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); +// await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Workspace directory did not get created"), +// CancellationToken); +// +// // Ensure Workspace Cleaning Has Run +// var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); +// await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectory), "Existing bootstrap file"); +// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Workspace directory did not get deleted"), +// CancellationToken); +// +// Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); +// +// await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); +// await runningScriptTask; +// } +// +// [Test] +// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] +// public async Task WhenCompleteScriptIsNotCalled_ThenWorkspaceShouldGetDeletedWhenScriptFinishesRunning(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var cleanerDelay = TimeSpan.FromMilliseconds(500); +// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("Hello")).Build(); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacle(b => +// { +// b.WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); +// }) +// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() +// .DecorateAllScriptServicesWith(u => u +// .BeforeCompleteScript( +// () => throw new NotImplementedException("Force failure to simulate tentacle client crashing, and ensure we do not complete the script"))) +// .Build()) +// .Build(CancellationToken); +// +// await AssertionExtensions +// .Should(() => clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog())) +// .ThrowAsync(); +// +// var workspaceDirectory = GetWorkspaceDirectoryPath(clientAndTentacle.RunningTentacle.HomeDirectory, startScriptCommand.ScriptTicket.TaskId); +// +// await Wait.For(() => !Directory.Exists(workspaceDirectory), +// TimeSpan.FromSeconds(20), +// () => +// { +// try +// { +// Directory.Delete(workspaceDirectory, true); +// } +// catch (Exception) +// { +// // Deleting a worksapce is best effort and can silently fail if it is in use / locked by something. +// // If the cleaner failed to delete the directory and we cannot delete it in the test we can assume that it +// // is a valid failure and the test was successful. +// +// return; +// } +// +// throw new Exception("Workspace directory did not get deleted by the workspace cleaner but the test was able to delete it. This indicates there is an issue in the Tentacle code."); +// }, +// CancellationToken); +// } +// +// [Test] +// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] +// public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithLogFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var cleanerDelay = TimeSpan.FromMilliseconds(500); +// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); +// +// var existingHomeDirectory = new TemporaryDirectory(); +// +// var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); +// var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); +// await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectoryWithLogFile), "Existing log file"); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacle(b => +// { +// b.WithHomeDirectory(existingHomeDirectory) +// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); +// }) +// .Build(CancellationToken); +// +// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Workspace directory did not get deleted"), +// CancellationToken); +// Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); +// } +// +// [Test] +// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] +// public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithBootstrapFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var cleanerDelay = TimeSpan.FromMilliseconds(500); +// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); +// +// var existingHomeDirectory = new TemporaryDirectory(); +// +// var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); +// var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); +// await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectoryWithLogFile), "Existing bootstrap file"); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacle(b => +// { +// b.WithHomeDirectory(existingHomeDirectory) +// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); +// }) +// .Build(CancellationToken); +// +// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), +// TimeSpan.FromSeconds(60), +// () => throw new Exception("Workspace directory did not get deleted"), +// CancellationToken); +// Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); +// } +// +// [Test] +// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] +// public async Task WhenTentacleStarts_WithWorkspaceYoungerThanThreshold_ThenWorkspaceIsLeftAlone(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// var cleanerDelay = TimeSpan.FromMilliseconds(500); +// var deleteWorkspacesOlderThan = TimeSpan.FromMinutes(30); +// +// var existingHomeDirectory = new TemporaryDirectory(); +// +// var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); +// await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); +// +// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() +// .WithTentacle(b => +// { +// b.WithHomeDirectory(existingHomeDirectory) +// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); +// }) +// .Build(CancellationToken); +// +// await Task.Delay(1000, CancellationToken); +// +// Directory.Exists(existingWorkspaceDirectory).Should().BeTrue(); +// } +// +// static string GetWorkspaceDirectoryPath(string homeDirectory, string scriptTicket) +// { +// var workspaceDirectory = Path.Combine( +// homeDirectory, +// ScriptWorkspaceFactory.WorkDirectory, +// scriptTicket); +// return workspaceDirectory; +// } +// +// static string GivenExistingWorkspaceExists(TemporaryDirectory existingHomeDirectory) +// { +// var existingWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, Guid.NewGuid().ToString()); +// Directory.CreateDirectory(existingWorkspaceDirectory); +// return existingWorkspaceDirectory; +// } +// +// static string GetBootstrapScriptFilePath(string workingDirectory) +// { +// return !PlatformDetection.IsRunningOnWindows +// ? BashScriptWorkspace.GetBashBootstrapScriptFilePath(workingDirectory) +// : ScriptWorkspace.GetBootstrapScriptFilePath(workingDirectory); +// } +// } +// } \ No newline at end of file From be784e1d03a6b4d1a5b33a2e62c1d73e911b1b83 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Thu, 24 Oct 2024 04:32:21 +1300 Subject: [PATCH 18/34] Put back ClientScriptExecutionAdditionalScripts.cs --- .../ClientScriptExecutionAdditionalScripts.cs | 100 +++++++++--------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionAdditionalScripts.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionAdditionalScripts.cs index 27e1c6d60..24e6ea8ca 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionAdditionalScripts.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionAdditionalScripts.cs @@ -1,50 +1,50 @@ -// using System; -// using System.IO; -// using System.Threading.Tasks; -// using FluentAssertions; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class ClientScriptExecutionAdditionalScripts : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(testCommonVersions: true)] -// public async Task AdditionalScriptsWork(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// using var tmp = new TemporaryDirectory(); -// var path = Path.Combine(tmp.DirectoryPath, "file"); -// -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .Build(CancellationToken); -// -// var scriptBuilder = new ScriptBuilder() -// .CreateFile(path) // How files are made are different in bash and powershell, doing this ensures the client and tentacle really are using the correct script. -// .Print("Hello"); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .WithAdditionalScriptType(ScriptType.Bash, scriptBuilder.BuildBashScript()) -// // Additional Scripts don't actually work on tentacle for anything other than bash. -// // Below is what we would have expected to tentacle to work with. -// //.WithAdditionalScriptTypes(ScriptType.PowerShell, scriptBuilder.BuildPowershellScript()) -// // But instead we need to send the powershell in the scriptbody. -// .WithScriptBody(scriptBuilder.BuildPowershellScript()) -// .Build(); -// -// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// -// finalResponse.State.Should().Be(ProcessState.Complete); -// finalResponse.ExitCode.Should().Be(0); -// -// var allLogs = logs.JoinLogs(); -// -// allLogs.Should().Contain("Hello"); -// } -// } -// } +using System; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + public class ClientScriptExecutionAdditionalScripts : IntegrationTest + { + [Test] + [TentacleConfigurations(testCommonVersions: true)] + public async Task AdditionalScriptsWork(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + using var tmp = new TemporaryDirectory(); + var path = Path.Combine(tmp.DirectoryPath, "file"); + + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .Build(CancellationToken); + + var scriptBuilder = new ScriptBuilder() + .CreateFile(path) // How files are made are different in bash and powershell, doing this ensures the client and tentacle really are using the correct script. + .Print("Hello"); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .WithAdditionalScriptType(ScriptType.Bash, scriptBuilder.BuildBashScript()) + // Additional Scripts don't actually work on tentacle for anything other than bash. + // Below is what we would have expected to tentacle to work with. + //.WithAdditionalScriptTypes(ScriptType.PowerShell, scriptBuilder.BuildPowershellScript()) + // But instead we need to send the powershell in the scriptbody. + .WithScriptBody(scriptBuilder.BuildPowershellScript()) + .Build(); + + var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = logs.JoinLogs(); + + allLogs.Should().Contain("Hello"); + } + } +} From 26ef76a752361e8187b3b1b35c7adca036789e2c Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Thu, 24 Oct 2024 06:32:12 +1300 Subject: [PATCH 19/34] Put back rest of ClientScriptExecution tests --- .../ClientScriptExecutionIsolationMutex.cs | 364 +++++++++--------- ...lientScriptExecutionScriptArgumentsWork.cs | 82 ++-- ...ClientScriptExecutionScriptFilesAreSent.cs | 76 ++-- 3 files changed, 261 insertions(+), 261 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionIsolationMutex.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionIsolationMutex.cs index cdcac998b..baf50164c 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionIsolationMutex.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionIsolationMutex.cs @@ -1,182 +1,182 @@ -// using System; -// using System.Collections; -// using System.IO; -// using System.Threading.Tasks; -// using FluentAssertions; -// using NUnit.Framework; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Contracts.ClientServices; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class ClientScriptExecutionIsolationMutex : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(testScriptIsolationLevelVersions: true, additionalParameterTypes: new object[] { typeof(ScriptIsolationLevel)})] -// public async Task ScriptIsolationMutexFull_EnsuresTwoDifferentScriptsDontRunAtTheSameTime(TentacleConfigurationTestCase tentacleConfigurationTestCase, ScriptIsolationLevel levelOfSecondScript) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .Build()) -// .Build(CancellationToken); -// -// var firstScriptStartFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptStartFile"); -// var firstScriptWaitFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptWaitFile"); -// -// var secondScriptStart = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "secondScriptStartFile"); -// -// var firstStartScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .CreateFile(firstScriptStartFile) -// .WaitForFileToExist(firstScriptWaitFile)) -// .WithIsolationLevel(ScriptIsolationLevel.FullIsolation) -// .WithIsolationMutexName("mymutex") -// .Build(); -// -// var secondStartScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder().CreateFile(secondScriptStart)) -// .WithIsolationLevel(levelOfSecondScript) -// .WithIsolationMutexName("mymutex") -// .Build(); -// -// var tentacleClient = clientTentacle.TentacleClient; -// var firstScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(firstStartScriptCommand, CancellationToken)); -// -// // Wait for the first script to start running -// await Wait.For(() => File.Exists(firstScriptStartFile), -// TimeSpan.FromSeconds(30), -// () => throw new Exception("Script did not start"), -// CancellationToken); -// Logger.Information("First script is now running"); -// -// var secondScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(secondStartScriptCommand, CancellationToken)); -// -// // Wait for the second script start script RPC call to return. -// await Wait.For(() => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Completed == 2, -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Second execute script call did not complete"), -// CancellationToken); -// -// // Give Tentacle some more time to run the script (although it should not). -// await Task.Delay(TimeSpan.FromSeconds(2)); -// -// File.Exists(secondScriptStart).Should().BeFalse("The second script must not be started while the first is running with a FullIsolationMutex"); -// -// // Let the first script finish. -// File.WriteAllText(firstScriptWaitFile, ""); -// -// var (finalResponseFirstScript, _) = await firstScriptExecution; -// var (finalResponseSecondScript, _) = await secondScriptExecution; -// -// File.Exists(secondScriptStart).Should().BeTrue("The second should now have run."); -// -// finalResponseFirstScript.ExitCode.Should().Be(0); -// finalResponseSecondScript.ExitCode.Should().Be(0); -// } -// -// [Test] -// [TentacleConfigurations(testScriptIsolationLevelVersions: true, additionalParameterTypes: new object[] {typeof(ScriptsInParallelTestCases)})] -// public async Task ScriptIsolationMutexFull_IsOnlyExclusiveWhenFullAndWhenTheMutexNameIsTheSame(TentacleConfigurationTestCase tentacleConfigurationTestCase, ScriptsInParallelTestCase scriptsInParallelTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); -// -// var firstScriptStartFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptStartFile"); -// var firstScriptWaitFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptWaitFile"); -// -// var secondScriptStart = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "secondScriptStartFile"); -// -// var firstStartScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .CreateFile(firstScriptStartFile) -// .WaitForFileToExist(firstScriptWaitFile)) -// .WithIsolationLevel(scriptsInParallelTestCase.LevelOfFirstScript) -// .WithIsolationMutexName(scriptsInParallelTestCase.MutexForFirstScript) -// .Build(); -// -// var secondStartScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder().CreateFile(secondScriptStart)) -// .WithIsolationLevel(scriptsInParallelTestCase.LevelOfSecondScript) -// .WithIsolationMutexName(scriptsInParallelTestCase.MutexForSecondScript) -// .Build(); -// -// var tentacleClient = clientTentacle.TentacleClient; -// var firstScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(firstStartScriptCommand, CancellationToken)); -// -// // Wait for the first script to start running -// await Wait.For(() => File.Exists(firstScriptStartFile), -// TimeSpan.FromSeconds(30), -// () => throw new Exception("Script did not start running"), -// CancellationToken); -// -// var secondScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(secondStartScriptCommand, CancellationToken)); -// -// // Wait for the second script to start -// await Wait.For(() => File.Exists(secondScriptStart), -// TimeSpan.FromSeconds(30), -// () => throw new Exception("Second script did not start"), -// CancellationToken); -// // Both scripts are now running in parallel -// -// // Let the first script finish. -// File.WriteAllText(firstScriptWaitFile, ""); -// -// var (finalResponseFirstScript, _) = await firstScriptExecution; -// var (finalResponseSecondScript, _) = await secondScriptExecution; -// -// File.Exists(secondScriptStart).Should().BeTrue("The second script must not be started while the first is running with a FullIsolationMutex"); -// -// finalResponseFirstScript.ExitCode.Should().Be(0); -// finalResponseSecondScript.ExitCode.Should().Be(0); -// } -// -// public class ScriptsInParallelTestCases : IEnumerable -// { -// public IEnumerator GetEnumerator() -// { -// // Scripts with the same mutex name can run at the same time if they both has no isolation. -// yield return ScriptsInParallelTestCase.NoIsolationSameMutex; -// // Scripts with different mutex names can run at the same time. -// yield return ScriptsInParallelTestCase.FullIsolationDifferentMutex; -// } -// } -// -// public class ScriptsInParallelTestCase -// { -// public static ScriptsInParallelTestCase NoIsolationSameMutex => new(ScriptIsolationLevel.NoIsolation, "sameMutex", ScriptIsolationLevel.NoIsolation, "sameMutex", nameof(NoIsolationSameMutex)); -// public static ScriptsInParallelTestCase FullIsolationDifferentMutex => new(ScriptIsolationLevel.FullIsolation, "mutex", ScriptIsolationLevel.FullIsolation, "differentMutex", nameof(FullIsolationDifferentMutex)); -// -// public readonly ScriptIsolationLevel LevelOfFirstScript; -// public readonly string MutexForFirstScript; -// public readonly ScriptIsolationLevel LevelOfSecondScript; -// public readonly string MutexForSecondScript; -// -// private readonly string stringValue; -// -// private ScriptsInParallelTestCase( -// ScriptIsolationLevel levelOfFirstScript, -// string mutexForFirstScript, -// ScriptIsolationLevel levelOfSecondScript, -// string mutexForSecondScript, -// string stringValue) -// { -// LevelOfFirstScript = levelOfFirstScript; -// MutexForFirstScript = mutexForFirstScript; -// LevelOfSecondScript = levelOfSecondScript; -// MutexForSecondScript = mutexForSecondScript; -// this.stringValue = stringValue; -// } -// -// public override string ToString() -// { -// return stringValue; -// } -// } -// } -// } +using System; +using System.Collections; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + public class ClientScriptExecutionIsolationMutex : IntegrationTest + { + [Test] + [TentacleConfigurations(testScriptIsolationLevelVersions: true, additionalParameterTypes: new object[] { typeof(ScriptIsolationLevel)})] + public async Task ScriptIsolationMutexFull_EnsuresTwoDifferentScriptsDontRunAtTheSameTime(TentacleConfigurationTestCase tentacleConfigurationTestCase, ScriptIsolationLevel levelOfSecondScript) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .Build()) + .Build(CancellationToken); + + var firstScriptStartFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptStartFile"); + var firstScriptWaitFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptWaitFile"); + + var secondScriptStart = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "secondScriptStartFile"); + + var firstStartScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .CreateFile(firstScriptStartFile) + .WaitForFileToExist(firstScriptWaitFile)) + .WithIsolationLevel(ScriptIsolationLevel.FullIsolation) + .WithIsolationMutexName("mymutex") + .Build(); + + var secondStartScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder().CreateFile(secondScriptStart)) + .WithIsolationLevel(levelOfSecondScript) + .WithIsolationMutexName("mymutex") + .Build(); + + var tentacleClient = clientTentacle.TentacleClient; + var firstScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(firstStartScriptCommand, CancellationToken)); + + // Wait for the first script to start running + await Wait.For(() => File.Exists(firstScriptStartFile), + TimeSpan.FromSeconds(30), + () => throw new Exception("Script did not start"), + CancellationToken); + Logger.Information("First script is now running"); + + var secondScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(secondStartScriptCommand, CancellationToken)); + + // Wait for the second script start script RPC call to return. + await Wait.For(() => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Completed == 2, + TimeSpan.FromSeconds(60), + () => throw new Exception("Second execute script call did not complete"), + CancellationToken); + + // Give Tentacle some more time to run the script (although it should not). + await Task.Delay(TimeSpan.FromSeconds(2)); + + File.Exists(secondScriptStart).Should().BeFalse("The second script must not be started while the first is running with a FullIsolationMutex"); + + // Let the first script finish. + File.WriteAllText(firstScriptWaitFile, ""); + + var (finalResponseFirstScript, _) = await firstScriptExecution; + var (finalResponseSecondScript, _) = await secondScriptExecution; + + File.Exists(secondScriptStart).Should().BeTrue("The second should now have run."); + + finalResponseFirstScript.ExitCode.Should().Be(0); + finalResponseSecondScript.ExitCode.Should().Be(0); + } + + [Test] + [TentacleConfigurations(testScriptIsolationLevelVersions: true, additionalParameterTypes: new object[] {typeof(ScriptsInParallelTestCases)})] + public async Task ScriptIsolationMutexFull_IsOnlyExclusiveWhenFullAndWhenTheMutexNameIsTheSame(TentacleConfigurationTestCase tentacleConfigurationTestCase, ScriptsInParallelTestCase scriptsInParallelTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); + + var firstScriptStartFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptStartFile"); + var firstScriptWaitFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "firstScriptWaitFile"); + + var secondScriptStart = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "secondScriptStartFile"); + + var firstStartScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .CreateFile(firstScriptStartFile) + .WaitForFileToExist(firstScriptWaitFile)) + .WithIsolationLevel(scriptsInParallelTestCase.LevelOfFirstScript) + .WithIsolationMutexName(scriptsInParallelTestCase.MutexForFirstScript) + .Build(); + + var secondStartScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder().CreateFile(secondScriptStart)) + .WithIsolationLevel(scriptsInParallelTestCase.LevelOfSecondScript) + .WithIsolationMutexName(scriptsInParallelTestCase.MutexForSecondScript) + .Build(); + + var tentacleClient = clientTentacle.TentacleClient; + var firstScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(firstStartScriptCommand, CancellationToken)); + + // Wait for the first script to start running + await Wait.For(() => File.Exists(firstScriptStartFile), + TimeSpan.FromSeconds(30), + () => throw new Exception("Script did not start running"), + CancellationToken); + + var secondScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(secondStartScriptCommand, CancellationToken)); + + // Wait for the second script to start + await Wait.For(() => File.Exists(secondScriptStart), + TimeSpan.FromSeconds(30), + () => throw new Exception("Second script did not start"), + CancellationToken); + // Both scripts are now running in parallel + + // Let the first script finish. + File.WriteAllText(firstScriptWaitFile, ""); + + var (finalResponseFirstScript, _) = await firstScriptExecution; + var (finalResponseSecondScript, _) = await secondScriptExecution; + + File.Exists(secondScriptStart).Should().BeTrue("The second script must not be started while the first is running with a FullIsolationMutex"); + + finalResponseFirstScript.ExitCode.Should().Be(0); + finalResponseSecondScript.ExitCode.Should().Be(0); + } + + public class ScriptsInParallelTestCases : IEnumerable + { + public IEnumerator GetEnumerator() + { + // Scripts with the same mutex name can run at the same time if they both has no isolation. + yield return ScriptsInParallelTestCase.NoIsolationSameMutex; + // Scripts with different mutex names can run at the same time. + yield return ScriptsInParallelTestCase.FullIsolationDifferentMutex; + } + } + + public class ScriptsInParallelTestCase + { + public static ScriptsInParallelTestCase NoIsolationSameMutex => new(ScriptIsolationLevel.NoIsolation, "sameMutex", ScriptIsolationLevel.NoIsolation, "sameMutex", nameof(NoIsolationSameMutex)); + public static ScriptsInParallelTestCase FullIsolationDifferentMutex => new(ScriptIsolationLevel.FullIsolation, "mutex", ScriptIsolationLevel.FullIsolation, "differentMutex", nameof(FullIsolationDifferentMutex)); + + public readonly ScriptIsolationLevel LevelOfFirstScript; + public readonly string MutexForFirstScript; + public readonly ScriptIsolationLevel LevelOfSecondScript; + public readonly string MutexForSecondScript; + + private readonly string stringValue; + + private ScriptsInParallelTestCase( + ScriptIsolationLevel levelOfFirstScript, + string mutexForFirstScript, + ScriptIsolationLevel levelOfSecondScript, + string mutexForSecondScript, + string stringValue) + { + LevelOfFirstScript = levelOfFirstScript; + MutexForFirstScript = mutexForFirstScript; + LevelOfSecondScript = levelOfSecondScript; + MutexForSecondScript = mutexForSecondScript; + this.stringValue = stringValue; + } + + public override string ToString() + { + return stringValue; + } + } + } +} diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptArgumentsWork.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptArgumentsWork.cs index 1127d47f0..cdce531e7 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptArgumentsWork.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptArgumentsWork.cs @@ -1,41 +1,41 @@ -// using System; -// using System.Threading.Tasks; -// using FluentAssertions; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils.Builders; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class ClientScriptExecutionScriptArgumentsWork : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(testCommonVersions: true)] -// public async Task ArgumentsArePassedToTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder().PrintArguments()) -// .WithArguments(new[] { "First", "Second", "AndSpacesAreNotHandledWellInTentacle" }) -// .Build(); -// -// var tentacleServicesDecorator = new TentacleServiceDecoratorBuilder().Build(); -// -// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// -// finalResponse.State.Should().Be(ProcessState.Complete); -// finalResponse.ExitCode.Should().Be(0); -// -// var allLogs = logs.JoinLogs(); -// -// allLogs.Should().MatchRegex(".*Argument: First\n.*Argument: Second\n.*Argument: AndSpacesAreNotHandledWellInTentacle\n"); -// } -// } -// } \ No newline at end of file +using System; +using System.Threading.Tasks; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils.Builders; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + public class ClientScriptExecutionScriptArgumentsWork : IntegrationTest + { + [Test] + [TentacleConfigurations(testCommonVersions: true)] + public async Task ArgumentsArePassedToTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder().PrintArguments()) + .WithArguments(new[] { "First", "Second", "AndSpacesAreNotHandledWellInTentacle" }) + .Build(); + + var tentacleServicesDecorator = new TentacleServiceDecoratorBuilder().Build(); + + var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = logs.JoinLogs(); + + allLogs.Should().MatchRegex(".*Argument: First\n.*Argument: Second\n.*Argument: AndSpacesAreNotHandledWellInTentacle\n"); + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs index f5423050c..181ceeef0 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs @@ -1,38 +1,38 @@ -// using System; -// using System.Threading.Tasks; -// using FluentAssertions; -// using Halibut; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils.Builders; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class ClientScriptExecutionScriptFilesAreSent : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(testCommonVersions: true)] -// public async Task ScriptFilesAreSent(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder().PrintFileContents("foo.txt")) -// .WithScriptFile(new ScriptFile("foo.txt", DataStream.FromString("The File Contents"))) -// .Build(); -// -// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// -// finalResponse.State.Should().Be(ProcessState.Complete); -// finalResponse.ExitCode.Should().Be(0); -// -// var allLogs = logs.JoinLogs(); -// -// allLogs.Should().Contain("The File Contents"); -// } -// } -// } +using System; +using System.Threading.Tasks; +using FluentAssertions; +using Halibut; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils.Builders; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + public class ClientScriptExecutionScriptFilesAreSent : IntegrationTest + { + [Test] + [TentacleConfigurations(testCommonVersions: true)] + public async Task ScriptFilesAreSent(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder().PrintFileContents("foo.txt")) + .WithScriptFile(new ScriptFile("foo.txt", DataStream.FromString("The File Contents"))) + .Build(); + + var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = logs.JoinLogs(); + + allLogs.Should().Contain("The File Contents"); + } + } +} From b3c364de851c410618153e446eedf80eaaa05af4 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Thu, 24 Oct 2024 08:10:23 +1300 Subject: [PATCH 20/34] Roll back ClientScriptExecutionScriptFilesAreSent.cs --- ...ClientScriptExecutionScriptFilesAreSent.cs | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs index 181ceeef0..f5423050c 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs @@ -1,38 +1,38 @@ -using System; -using System.Threading.Tasks; -using FluentAssertions; -using Halibut; -using NUnit.Framework; -using Octopus.Tentacle.CommonTestUtils.Builders; -using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Util; -using Octopus.Tentacle.Tests.Integration.Util.Builders; - -namespace Octopus.Tentacle.Tests.Integration -{ - [IntegrationTestTimeout] - public class ClientScriptExecutionScriptFilesAreSent : IntegrationTest - { - [Test] - [TentacleConfigurations(testCommonVersions: true)] - public async Task ScriptFilesAreSent(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { - await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); - - var startScriptCommand = new TestExecuteShellScriptCommandBuilder() - .SetScriptBody(new ScriptBuilder().PrintFileContents("foo.txt")) - .WithScriptFile(new ScriptFile("foo.txt", DataStream.FromString("The File Contents"))) - .Build(); - - var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); - - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(0); - - var allLogs = logs.JoinLogs(); - - allLogs.Should().Contain("The File Contents"); - } - } -} +// using System; +// using System.Threading.Tasks; +// using FluentAssertions; +// using Halibut; +// using NUnit.Framework; +// using Octopus.Tentacle.CommonTestUtils.Builders; +// using Octopus.Tentacle.Contracts; +// using Octopus.Tentacle.Tests.Integration.Support; +// using Octopus.Tentacle.Tests.Integration.Util; +// using Octopus.Tentacle.Tests.Integration.Util.Builders; +// +// namespace Octopus.Tentacle.Tests.Integration +// { +// [IntegrationTestTimeout] +// public class ClientScriptExecutionScriptFilesAreSent : IntegrationTest +// { +// [Test] +// [TentacleConfigurations(testCommonVersions: true)] +// public async Task ScriptFilesAreSent(TentacleConfigurationTestCase tentacleConfigurationTestCase) +// { +// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); +// +// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() +// .SetScriptBody(new ScriptBuilder().PrintFileContents("foo.txt")) +// .WithScriptFile(new ScriptFile("foo.txt", DataStream.FromString("The File Contents"))) +// .Build(); +// +// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); +// +// finalResponse.State.Should().Be(ProcessState.Complete); +// finalResponse.ExitCode.Should().Be(0); +// +// var allLogs = logs.JoinLogs(); +// +// allLogs.Should().Contain("The File Contents"); +// } +// } +// } From 85c388ef91b27e4805f61364817d98064e875024 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Fri, 8 Nov 2024 10:18:09 +1300 Subject: [PATCH 21/34] Trigger Build From 6f8e8c9c542f3446ec7888f6a3d261c7574ca381 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Fri, 8 Nov 2024 12:24:57 +1300 Subject: [PATCH 22/34] Add logging to file transfer service tests --- .../FileTransferServiceTests.cs | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs b/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs index dd166d8bc..dad377c10 100644 --- a/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs @@ -1,9 +1,11 @@ using System; using System.IO; +using System.Linq; using System.Threading.Tasks; using FluentAssertions; using Halibut; using NUnit.Framework; +using Octopus.Client.Extensions; using Octopus.Tentacle.CommonTestUtils; using Octopus.Tentacle.Tests.Integration.Support; using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; @@ -18,10 +20,17 @@ public class FileTransferServiceTests : IntegrationTest [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase) { + var driveInfos = DriveInfo.GetDrives(); + + Logger.Information($"UploadFileSuccessfully Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); + using var fileToUpload = new RandomTemporaryFileBuilder().Build(); + Logger.Information($"UploadFileSuccessfully Available Disk space before CreateLegacyBuilder: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); + Logger.Information($"UploadFileSuccessfully Available Disk space after CreateLegacyBuilder: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); + var dataStream = new DataStream( fileToUpload.File.Length, async (stream, ct) => @@ -29,14 +38,20 @@ public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleC using var fileStream = File.OpenRead(fileToUpload.File.FullName); await fileStream.CopyToAsync(stream); }); + + Logger.Information($"UploadFileSuccessfully Available Disk after creating a DataStream: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); var uploadResult = await clientAndTentacle.TentacleClient.FileTransferService.UploadFileAsync("the_remote_uploaded_file", dataStream, new(CancellationToken)); + + Logger.Information($"UploadFileSuccessfully Available Disk space after UploadFileAsync: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); Console.WriteLine($"Source: {fileToUpload.File.FullName}"); Console.WriteLine($"Destination: {uploadResult.FullPath}"); var sourceBytes = File.ReadAllBytes(fileToUpload.File.FullName); var destinationBytes = File.ReadAllBytes(uploadResult.FullPath); + + Logger.Information($"UploadFileSuccessfully Available Disk space after assertion: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); sourceBytes.Should().BeEquivalentTo(destinationBytes); } @@ -44,19 +59,27 @@ public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleC [Test] [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] public async Task DownloadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { + { + var driveInfos = DriveInfo.GetDrives(); + using var fileToDownload = new RandomTemporaryFileBuilder().Build(); + Logger.Information($"DownloadFileSuccessfully Available Disk space before RandomTemporaryFileBuilder: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); + Logger.Information($"DownloadFileSuccessfully Available Disk space after CreateLegacyBuilder: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); var downloadedData = await clientAndTentacle.TentacleClient.FileTransferService.DownloadFileAsync( fileToDownload.File.FullName, new(CancellationToken)); + Logger.Information($"DownloadFileSuccessfully Available Disk space after DownloadFileAsync: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); var sourceBytes = File.ReadAllBytes(fileToDownload.File.FullName); var destinationBytes = await downloadedData.ToBytes(CancellationToken); + Logger.Information($"DownloadFileSuccessfully Available Disk space after downloadedData: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); destinationBytes.Should().BeEquivalentTo(sourceBytes); + Logger.Information($"DownloadFileSuccessfully Available Disk space after assertion: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); + } } } From 777d6f2bd69f9a66a41a752166d6d73e4dc6a071 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Fri, 8 Nov 2024 12:49:08 +1300 Subject: [PATCH 23/34] More logging to determine disk space issue --- .../ClientFileTransferRetriesTimeout.cs | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs index f33ba0bf0..f7e20c692 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs @@ -2,10 +2,12 @@ using System.Collections; using System.Diagnostics; using System.IO; +using System.Linq; using System.Threading.Tasks; using FluentAssertions; using Halibut; using NUnit.Framework; +using Octopus.Client.Extensions; using Octopus.Tentacle.CommonTestUtils.Diagnostics; using Octopus.Tentacle.Contracts.ClientServices; using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; @@ -146,6 +148,10 @@ public async Task WhenUploadFileFails_AndTakesLongerThanTheRetryDuration_TheCall [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None, additionalParameterTypes: new object[] { typeof(StopPortForwarderAfterFirstCallValues) })] public async Task WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, bool stopPortForwarderAfterFirstCall) { + var driveInfos = DriveInfo.GetDrives(); + + Logger.Information($"WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); + PortForwarder portForwarder = null!; await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() // Set a short retry duration so we cancel fairly quickly @@ -187,14 +193,18 @@ public async Task WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled .Build(CancellationToken); portForwarder = clientAndTentacle.PortForwarder; + Logger.Information($"WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled Available Disk space after CreateLegacyBuilder: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); using var tempFile = new RandomTemporaryFileBuilder().Build(); + Logger.Information($"WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled Available Disk space after RandomTemporaryFileBuilder: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); var inMemoryLog = new InMemoryLog(); - + // Start the script which will wait for a file to exist var duration = Stopwatch.StartNew(); var executeScriptTask = clientAndTentacle.TentacleClient.DownloadFile(tempFile.File.FullName, CancellationToken, inMemoryLog); + + Logger.Information($"WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled Available Disk space after TentacleClient.DownloadFile: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); @@ -204,6 +214,8 @@ public async Task WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().BeGreaterOrEqualTo(2); recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(0); + + Logger.Information($"WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled Available Disk space after recordedUsages: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); // Ensure we actually waited and retried until the timeout policy kicked in duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); @@ -215,6 +227,10 @@ public async Task WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] public async Task WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) { + var driveInfos = DriveInfo.GetDrives(); + + Logger.Information($"WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); + var retryDuration = TimeSpan.FromSeconds(15); await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() @@ -240,14 +256,20 @@ public async Task WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCa .Build(CancellationToken); using var tempFile = new RandomTemporaryFileBuilder().Build(); + Logger.Information($"WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut Available Disk space after RandomTemporaryFileBuilder: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); + var inMemoryLog = new InMemoryLog(); var executeScriptTask = clientAndTentacle.TentacleClient.DownloadFile(tempFile.File.FullName, CancellationToken, inMemoryLog); + + Logger.Information($"WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled Available Disk space after TentacleClient.DownloadFile: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); Func> action = async () => await executeScriptTask; await action.Should().ThrowAsync(); recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(1); recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(0); + + Logger.Information($"WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut Available Disk space after recordedUsages: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); } From fdb6f036e2ea3e97fa41b96cd608b38741d862c9 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Fri, 8 Nov 2024 12:52:44 +1300 Subject: [PATCH 24/34] Further logging --- .../ClientFileTransferRetriesTimeout.cs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs index f7e20c692..01ce7f04d 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs @@ -37,6 +37,10 @@ public class ClientFileTransferRetriesTimeout : IntegrationTest [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None, additionalParameterTypes: new object[] { typeof(StopPortForwarderAfterFirstCallValues) })] public async Task WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, bool stopPortForwarderAfterFirstCall) { + var driveInfos = DriveInfo.GetDrives(); + + Logger.Information($"WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); + PortForwarder portForwarder = null!; await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() // Set a short retry duration so we cancel fairly quickly @@ -77,6 +81,7 @@ public async Task WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled(T .Build(CancellationToken); portForwarder = clientAndTentacle.PortForwarder; + Logger.Information($"WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled Available Disk space after clientAndTentacle.PortForwarder: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); var inMemoryLog = new InMemoryLog(); @@ -87,6 +92,8 @@ public async Task WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled(T var duration = Stopwatch.StartNew(); var executeScriptTask = clientAndTentacle.TentacleClient.UploadFile(remotePath, dataStream, CancellationToken, inMemoryLog); + Logger.Information($"WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled Available Disk space after TentacleClient.UploadFile: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); @@ -94,6 +101,8 @@ public async Task WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled(T methodUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().BeGreaterOrEqualTo(2); methodUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(0); + + Logger.Information($"WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled Available Disk space after assertion: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); // Ensure we actually waited and retried until the timeout policy kicked in duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); @@ -104,7 +113,12 @@ public async Task WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled(T [Test] [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] public async Task WhenUploadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) - { + { + + var driveInfos = DriveInfo.GetDrives(); + + Logger.Information($"WhenUploadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); + var retryDuration = TimeSpan.FromSeconds(15); await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() @@ -135,11 +149,14 @@ public async Task WhenUploadFileFails_AndTakesLongerThanTheRetryDuration_TheCall var executeScriptTask = clientAndTentacle.TentacleClient.UploadFile(remotePath, dataStream, CancellationToken, inMemoryLog); var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + Logger.Information($"WhenUploadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut Available Disk space after expectedException: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); methodUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(1); methodUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(0); + + Logger.Information($"WhenUploadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut Available Disk space after assertions: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); } From c4e8bb786656b6bbb44e81362df6c2d74215122e Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Fri, 8 Nov 2024 14:13:19 +1300 Subject: [PATCH 25/34] Add free disk space logs --- .../Support/IntegrationTest.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTest.cs b/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTest.cs index 79c83541d..5b1e1177b 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTest.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTest.cs @@ -1,9 +1,11 @@ using System; using System.IO; +using System.Linq; using System.Threading; using Halibut.Util; using NUnit.Framework; using NUnit.Framework.Interfaces; +using Octopus.Client.Extensions; using Octopus.Tentacle.Tests.Integration.Util; using Serilog; @@ -20,7 +22,8 @@ public abstract class IntegrationTest : IDisposable public void SetUp() { Logger = new SerilogLoggerBuilder().Build().ForContext(GetType()); - Logger.Information("Test started"); + var driveInfos = DriveInfo.GetDrives(); + Logger.Information($"Test started. Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); // Time out the cancellation token so we cancel the test if it takes too long // The IntegrationTestTimeout attribute will also cancel the test if it takes too long, but nunit will not call TearDown on the test @@ -40,7 +43,9 @@ public void TearDown() Logger.Information("Disposing CancellationTokenSource"); cancellationTokenSource?.Dispose(); cancellationTokenSource = null; - Logger.Information("Finished Test Tearing Down"); + var driveInfos = DriveInfo.GetDrives(); + Logger.Information($"Finished Test Tearing Down. Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); + } void WriteTentacleLogToOutputIfTestHasFailed() From 4f36272df2d4fd17a1d70338e6e9ea97093e1686 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Fri, 8 Nov 2024 14:13:58 +1300 Subject: [PATCH 26/34] Unskip test that fails on disk space issue --- .../ClientFileTransferRetriesTimeout.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs index 01ce7f04d..a31e9df60 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs @@ -27,7 +27,7 @@ namespace Octopus.Tentacle.Tests.Integration /// from RPC calls when they are being retried and the rpc timeout period elapses. /// [IntegrationTestTimeout] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] + // [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public class ClientFileTransferRetriesTimeout : IntegrationTest { readonly TimeSpan retryIfRemainingDurationAtLeastBuffer = TimeSpan.FromSeconds(1); From cb38ac56c1594ff2d244d55a99cdb49feb4638fb Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Fri, 8 Nov 2024 15:54:02 +1300 Subject: [PATCH 27/34] Correct drive not being ready --- .../ClientFileTransferRetriesTimeout.cs | 8 ++++---- .../FileTransferServiceTests.cs | 4 ++-- .../Support/ClientAndTentacle.cs | 7 +++++++ .../Support/IntegrationTest.cs | 4 ++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs index a31e9df60..6021d2cd9 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs @@ -37,7 +37,7 @@ public class ClientFileTransferRetriesTimeout : IntegrationTest [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None, additionalParameterTypes: new object[] { typeof(StopPortForwarderAfterFirstCallValues) })] public async Task WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, bool stopPortForwarderAfterFirstCall) { - var driveInfos = DriveInfo.GetDrives(); + var driveInfos = DriveInfo.GetDrives().Where(d => d.IsReady); Logger.Information($"WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); @@ -115,7 +115,7 @@ public async Task WhenRpcRetriesTimeOut_DuringUploadFile_TheRpcCallIsCancelled(T public async Task WhenUploadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) { - var driveInfos = DriveInfo.GetDrives(); + var driveInfos = DriveInfo.GetDrives().Where(d => d.IsReady); Logger.Information($"WhenUploadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); @@ -165,7 +165,7 @@ public async Task WhenUploadFileFails_AndTakesLongerThanTheRetryDuration_TheCall [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None, additionalParameterTypes: new object[] { typeof(StopPortForwarderAfterFirstCallValues) })] public async Task WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, bool stopPortForwarderAfterFirstCall) { - var driveInfos = DriveInfo.GetDrives(); + var driveInfos = DriveInfo.GetDrives().Where(d => d.IsReady); Logger.Information($"WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); @@ -244,7 +244,7 @@ public async Task WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] public async Task WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) { - var driveInfos = DriveInfo.GetDrives(); + var driveInfos = DriveInfo.GetDrives().Where(d => d.IsReady); Logger.Information($"WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); diff --git a/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs b/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs index dad377c10..de5f41b6d 100644 --- a/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs @@ -20,7 +20,7 @@ public class FileTransferServiceTests : IntegrationTest [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase) { - var driveInfos = DriveInfo.GetDrives(); + var driveInfos = DriveInfo.GetDrives().Where(d => d.IsReady); Logger.Information($"UploadFileSuccessfully Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); @@ -60,7 +60,7 @@ public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleC [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] public async Task DownloadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase) { - var driveInfos = DriveInfo.GetDrives(); + var driveInfos = DriveInfo.GetDrives().Where(d => d.IsReady); using var fileToDownload = new RandomTemporaryFileBuilder().Build(); Logger.Information($"DownloadFileSuccessfully Available Disk space before RandomTemporaryFileBuilder: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/ClientAndTentacle.cs b/source/Octopus.Tentacle.Tests.Integration/Support/ClientAndTentacle.cs index 9d4e9a523..9ed091e42 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Support/ClientAndTentacle.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Support/ClientAndTentacle.cs @@ -1,8 +1,10 @@ using System; using System.IO; +using System.Linq; using System.Text; using System.Threading.Tasks; using Halibut; +using Octopus.Client.Extensions; using Octopus.Tentacle.Client; using Octopus.Tentacle.Client.Retries; using Octopus.Tentacle.CommonTestUtils; @@ -53,6 +55,11 @@ public ClientAndTentacle(IHalibutRuntime halibutRuntime, public async ValueTask DisposeAsync() { + + var driveInfos = DriveInfo.GetDrives().Where(d => d.IsReady); + + logger.Information($"DisposeAsync() Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); + SafelyMoveTentacleLogFileToSharedLocation(); var banner = new StringBuilder(); diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTest.cs b/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTest.cs index 5b1e1177b..1e6cd61ce 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTest.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Support/IntegrationTest.cs @@ -22,7 +22,7 @@ public abstract class IntegrationTest : IDisposable public void SetUp() { Logger = new SerilogLoggerBuilder().Build().ForContext(GetType()); - var driveInfos = DriveInfo.GetDrives(); + var driveInfos = DriveInfo.GetDrives().Where(d => d.IsReady); Logger.Information($"Test started. Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); // Time out the cancellation token so we cancel the test if it takes too long @@ -43,7 +43,7 @@ public void TearDown() Logger.Information("Disposing CancellationTokenSource"); cancellationTokenSource?.Dispose(); cancellationTokenSource = null; - var driveInfos = DriveInfo.GetDrives(); + var driveInfos = DriveInfo.GetDrives().Where(d => d.IsReady); Logger.Information($"Finished Test Tearing Down. Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}"); } From 22fa5e74506a5c2a2c4d7a4ad42f84435933073e Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Mon, 11 Nov 2024 15:31:54 +1300 Subject: [PATCH 28/34] Unskip some tests --- ...fersAreNotRetriedWhenRetriesAreDisabled.cs | 2 +- ...ransfersAreRetriedWhenRetriesAreEnabled.cs | 2 +- .../TentacleCommandLineTests.cs | 1504 ++++++++--------- 3 files changed, 754 insertions(+), 754 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs index cb5876c7f..b6f9fc713 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs @@ -15,7 +15,7 @@ namespace Octopus.Tentacle.Tests.Integration { [IntegrationTestTimeout] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] + // [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public class ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled : IntegrationTest { [Test] diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs index eef4f972f..12cb89c2e 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs @@ -21,7 +21,7 @@ public class ClientFileTransfersAreRetriedWhenRetriesAreEnabled : IntegrationTes { [Test] [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] + // [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public async Task FailedUploadsAreRetriedAndIsEventuallySuccessful(TentacleConfigurationTestCase tentacleConfigurationTestCase) { await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() diff --git a/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs b/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs index 2817386b5..42adb51d8 100644 --- a/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs @@ -1,752 +1,752 @@ -// using System; -// using System.Collections.Generic; -// using System.Diagnostics; -// using System.IO; -// using System.Linq; -// using System.Runtime.InteropServices; -// using System.Security.Principal; -// using System.Text; -// using System.Threading.Tasks; -// using CliWrap; -// using CliWrap.Exceptions; -// using FluentAssertions; -// using Newtonsoft.Json; -// using Newtonsoft.Json.Linq; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -// using Octopus.Tentacle.Util; -// using Octopus.Tentacle.Variables; -// using Polly; -// using PlatformDetection = Octopus.Tentacle.Util.PlatformDetection; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// /// -// /// These tests provide guarantees around how our command-line interface works, especially for scenarios where people automate setup. -// /// Please review any changes to the assertions made by these tests carefully. -// /// -// [IntegrationTestTimeout] -// [SkipOnEnvironmentsWithKnownPerformanceIssues("we don't expect customers to execute it in this environment in a situation where it needs to behave within a reasonable timeframe.")] -// public class TentacleCommandLineTests : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task TentacleExeNoArguments(TentacleConfigurationTestCase tc) -// { -// var (exitCode, stdout, stderr) = await RunCommand(tc, null); -// -// exitCode.Should().Be(2, "the exit code should be 2 if the command wasn't understood"); -// stdout.Should().StartWithEquivalentOf("Usage: Tentacle []", "should show help by default if no other commands are specified"); -// stdout.Should().ContainEquivalentOf("Or use --help for more details.", "should provide the hint for command-specific help"); -// stderr.Should().BeNullOrEmpty(); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task UnknownCommand(TentacleConfigurationTestCase tc) -// { -// var (exitCode, stdout, stderr) = await RunCommand(tc, null, "unknown-command"); -// -// exitCode.Should().Be(2, "the exit code should be 2 if the command wasn't understood"); -// stderr.Should().StartWithEquivalentOf("Command 'unknown-command' is not supported", "the error should clearly indicate the command which is not understood"); -// stdout.Should().StartWithEquivalentOf("See 'Tentacle help'", "should provide the hint to use help"); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task UnknownArgument(TentacleConfigurationTestCase tc) -// { -// var (exitCode, stdout, stderr) = await RunCommand(tc, null, "version", "--unknown=argument"); -// -// exitCode.Should().Be(1, "the exit code should be 1 if the command has unknown arguments"); -// stdout.Should().BeNullOrEmpty("the error message should be written to stderr, not stdout"); -// stderr.Should().ContainEquivalentOf("Unrecognized command line arguments: --unknown=argument", "the error message (written to stderr) should clearly indicate which arguments couldn't be parsed."); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task InvalidArgument(TentacleConfigurationTestCase tc) -// { -// var (exitCode, stdout, stderr) = await RunCommand(tc, null, "version", "--format=unsupported"); -// -// exitCode.Should().Be(1, "the exit code should be 1 if the command has unknown arguments"); -// stdout.Should().BeNullOrEmpty("the error message should be written to stderr, not stdout"); -// stderr.Should().ContainEquivalentOf("The format 'unsupported' is not supported. Try text or json.", "the error message (written to stderr) should clearly indicate which argument was invalid."); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task NoConsoleLoggingSwitchStillSilentlySupportedForBackwardsCompat(TentacleConfigurationTestCase tc) -// { -// var (_, _, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--noconsolelogging"); -// -// stderr.Should().BeNullOrEmpty(); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task NoLogoSwitchStillSilentlySupportedForBackwardsCompat(TentacleConfigurationTestCase tc) -// { -// var (_, _, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--nologo"); -// -// stderr.Should().BeNullOrEmpty(); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task ConsoleSwitchStillSilentlySupportedForBackwardsCompat(TentacleConfigurationTestCase tc) -// { -// var (_, _, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--console"); -// -// stderr.Should().BeNullOrEmpty(); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task ShouldSupportFuzzyCommandParsing(TentacleConfigurationTestCase tc) -// { -// await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version"); -// await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "--version"); -// await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "/version"); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task VersionCommandTextFormat(TentacleConfigurationTestCase tc) -// { -// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version"); -// -// var expectedVersion = GetVersionInfo(tc); -// -// stdout.Should().Be(expectedVersion.ProductVersion, "The version command should print the informational version as text"); -// stderr.Should().BeNullOrEmpty(); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task VersionCommandJsonFormat(TentacleConfigurationTestCase tc) -// { -// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--format=json"); -// -// var expectedVersion = GetVersionInfo(tc); -// var output = JObject.Parse(stdout); -// -// output["InformationalVersion"].Value().Should().Be(expectedVersion.ProductVersion, "The version command should print the informational version in the JSON output"); -// output["MajorMinorPatch"].Value().Should().Be($"{expectedVersion.FileMajorPart}.{expectedVersion.FileMinorPart}.{expectedVersion.FileBuildPart}", "The version command should print the version in the json output"); -// output["NuGetVersion"].Value().Should().NotBeNull("The version command should print the NuGet version in the JSON output"); -// output["SourceBranchName"].Value().Should().NotBeNull("The version command should print the source branch in the JSON output"); -// -// stderr.Should().BeNullOrEmpty(); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task CanGetHelpForHelp(TentacleConfigurationTestCase tc) -// { -// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "help", "--help"); -// stderr.Should().BeNullOrEmpty(); -// -// stdout.Should().Be( -// @"Usage: Tentacle help [] -// -// Where [] is any of: -// -// --format=VALUE The format of the output (text,json). Defaults -// to text. -// -// Or one of the common options: -// -// --help Show detailed help for this command -// "); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task HelpAsSwitchShouldShowCommandSpecificHelp(TentacleConfigurationTestCase tc) -// { -// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--help"); -// stderr.Should().BeNullOrEmpty(); -// -// stdout.Should().Be( -// @"Usage: Tentacle version [] -// -// Where [] is any of: -// -// --format=VALUE The format of the output (text,json). Defaults -// to text. -// -// Or one of the common options: -// -// --help Show detailed help for this command -// "); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task GeneralHelpAsJsonCanBeParsedByAutomationScripts(TentacleConfigurationTestCase tc) -// { -// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "help", "--format=json"); -// -// stderr.Should().BeNullOrEmpty(); -// var help = JsonConvert.DeserializeAnonymousType( -// stdout, -// new -// { -// Commands = new[] -// { -// new -// { -// Name = "", -// Description = "", -// Aliases = Array.Empty() -// } -// } -// }); -// -// help.Commands.Select(c => c.Name) -// .Should() -// .Contain( -// "configure", -// "help", -// "run", -// "version", -// "show-master-key", -// "show-thumbprint"); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task CommandSpecificHelpAsJsonCanBeParsedByAutomationScripts(TentacleConfigurationTestCase tc) -// { -// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--help", "--format=json"); -// -// stderr.Should().BeNullOrEmpty(); -// var help = JsonConvert.DeserializeAnonymousType( -// stdout, -// new -// { -// Name = "", -// Description = "", -// Aliases = Array.Empty(), -// Options = new[] -// { -// new -// { -// Name = "", -// Description = "" -// } -// } -// }); -// -// help.Name.Should().Be("version"); -// help.Options.Select(o => o.Name).Should().Contain("format"); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task CommandSpecificHelpAsJsonLooksSensibleToHumans(TentacleConfigurationTestCase tc) -// { -// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--help", "--format=json"); -// stderr.Should().BeNullOrEmpty(); -// -// stdout.Should().Be( -// @"{ -// ""Name"": ""version"", -// ""Description"": ""Show the Tentacle version information"", -// ""Aliases"": [], -// ""Options"": [ -// { -// ""Name"": ""format"", -// ""Description"": ""The format of the output (text,json). Defaults to text."" -// } -// ], -// ""CommonOptions"": [ -// { -// ""Name"": ""help"", -// ""Description"": ""Show detailed help for this command"" -// } -// ] -// }"); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task HelpForInstanceSpecificCommandsAlwaysWorks(TentacleConfigurationTestCase tc) -// { -// var (_, stdout, stderr) = await RunCommand(tc, null, "help", "--format=json"); -// -// stderr.Should().BeNullOrEmpty(); -// var help = JsonConvert.DeserializeAnonymousType( -// stdout, -// new -// { -// Commands = new[] -// { -// new -// { -// Name = "", -// Description = "", -// Aliases = new string[0] -// } -// } -// }); -// -// help.Commands.Should().HaveCountGreaterThan(0); -// -// var failed = help.Commands.Select(async c => -// { -// var (exitCode2, stdout2, stderr2) = await RunCommand(tc, null,$"{c.Name}", "--help"); -// return new -// { -// Command = c, -// ExitCode = exitCode2, -// StdOut = stdout2, -// StdErr = stderr2, -// HasExpectedExitCode = exitCode2 == 0, -// HasExpectedHelpMessage = stdout2.StartsWith($"Usage: Tentacle {c.Name} []") -// }; -// }) -// .Where(r => !(r.Result.HasExpectedExitCode && r.Result.HasExpectedHelpMessage)) -// .ToArray(); -// -// if (failed.Any()) -// { -// var failureDetails = string.Empty; -// -// foreach (var failure in failed) -// { -// failureDetails += $@"{failure.Result.Command.Name} -// StdErr:{failure.Result.StdErr} -// StdOut:{failure.Result.StdOut}"; -// } -// -// Assert.Fail( -// $@"The following commands cannot show help without specifying the --instance argument: -// {failureDetails} -// The details are logged above. These commands probably need to take Lazy dependencies so they can be instantiated for showing help without requiring every dependency to be resolvable."); -// } -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// // Run these tests in serial to avoid conflicts -// [NonParallelizable] -// public async Task InvalidInstance(TentacleConfigurationTestCase tc) -// { -// var (exitCode, stdout, stderr) = await RunCommand( -// tc, -// null, -// "show-thumbprint", "--instance=invalidinstance"); -// -// exitCode.Should().Be(1, $"the exit code should be 1 if the instance is not able to be resolved"); -// stderr.Should().ContainEquivalentOf("Instance invalidinstance of tentacle has not been configured", "the error message should make it clear the instance has not been configured"); -// stderr.Should().ContainEquivalentOf("Available instances:", "should provide a hint as to which instances are available on the machine"); -// stdout.Should().BeNullOrEmpty(); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// // Run these tests in serial to avoid conflicts -// [NonParallelizable] -// public async Task ShowThumbprintCommandText(TentacleConfigurationTestCase tc) -// { -// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); -// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); -// var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( -// tc, -// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, -// "show-thumbprint", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); -// -// exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); -// stdout.Should().Be(TestCertificates.TentaclePublicThumbprint, "the thumbprint should be written directly to stdout"); -// stderr.Should().BeNullOrEmpty(); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// // Run these tests in serial to avoid conflicts -// [NonParallelizable] -// public async Task ShowThumbprintCommandJson(TentacleConfigurationTestCase tc) -// { -// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); -// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); -// var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( -// tc, -// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, -// "show-thumbprint", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}", "--format=json"); -// -// exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); -// stdout.Should().Be(JsonConvert.SerializeObject(new { Thumbprint = TestCertificates.TentaclePublicThumbprint }), "the thumbprint should be written directly to stdout as JSON"); -// stderr.Should().BeNullOrEmpty(); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// // Run these tests in serial to avoid conflicts -// [NonParallelizable] -// public async Task ListInstancesCommandText(TentacleConfigurationTestCase tc) -// { -// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); -// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); -// var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( -// tc, -// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, -// "list-instances", "--format=text"); -// -// exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); -// var configPath = Path.Combine(clientAndTentacle.RunningTentacle.HomeDirectory, clientAndTentacle.RunningTentacle.InstanceName + ".cfg"); -// stdout.Should().Contain($"Instance '{clientAndTentacle.RunningTentacle.InstanceName}' uses configuration '{configPath}'.", "the current instance should be listed"); -// stderr.Should().BeNullOrEmpty(); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// // Run these tests in serial to avoid conflicts -// [NonParallelizable] -// public async Task ListInstancesCommandJson(TentacleConfigurationTestCase tc) -// { -// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); -// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); -// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( -// tc, -// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, -// "list-instances", "--format=json"); -// -// stdout.Should().Contain($"\"InstanceName\": \"{clientAndTentacle.RunningTentacle.InstanceName}\"", "the current instance should be listed"); -// var configPath = Path.Combine(clientAndTentacle.RunningTentacle.HomeDirectory, clientAndTentacle.RunningTentacle.InstanceName + ".cfg"); -// var jsonFormattedPath = JsonFormattedPath(configPath); -// stdout.Should().Contain($"\"ConfigurationFilePath\": \"{jsonFormattedPath}\"", "the path to the config file for the current instance should be listed"); -// stderr.Should().BeNullOrEmpty(); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// // Run these tests in serial to avoid conflicts -// [NonParallelizable] -// public async Task ShouldLogStartupDiagnosticsToInstanceLogFileOnly(TentacleConfigurationTestCase tc) -// { -// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); -// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); -// -// var startingLogText = clientAndTentacle.RunningTentacle.ReadAllLogFileText(); -// -// var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( -// tc, -// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, -// "show-thumbprint", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); -// -// try -// { -// var logFileText = Policy -// .Handle() -// .WaitAndRetry( -// 20, -// i => TimeSpan.FromMilliseconds(100 * i), -// (exception, _) => { Logger.Information($"Failed to get new log content: {exception.Message}. Retrying!"); }) -// .Execute( -// () => -// { -// var wholeLog = clientAndTentacle.RunningTentacle.ReadAllLogFileText(); -// var newLog = wholeLog.Replace(startingLogText, string.Empty); -// if (string.IsNullOrWhiteSpace(newLog) || !newLog.Contains("CommandLine:")) -// { -// throw new NotLoggedYetException(); -// } -// return newLog; -// }); -// -// logFileText.Should().ContainEquivalentOf($"OperatingSystem: {RuntimeInformation.OSDescription}", "the OSVersion should be in our diagnostics"); -// logFileText.Should().ContainEquivalentOf("OperatingSystem:", "the OSVersion should be in our diagnostics"); -// logFileText.Should().ContainEquivalentOf($"OsBitVersion: {(Environment.Is64BitOperatingSystem ? "x64" : "x86")}", "the OsBitVersion should be in our diagnostics"); -// logFileText.Should().ContainEquivalentOf($"Is64BitProcess: {Environment.Is64BitProcess}", "the Is64BitProcess should be in our diagnostics"); -// -// if (PlatformDetection.IsRunningOnWindows) -// { -// #pragma warning disable CA1416 -// logFileText.Should().ContainEquivalentOf($"CurrentUser: {WindowsIdentity.GetCurrent().Name}", "the CurrentUser should be in our diagnostics"); -// #pragma warning disable CA1416 -// } -// else -// { -// logFileText.Should().ContainEquivalentOf($"CurrentUser: {Environment.UserName}", "the CurrentUser should be in our diagnostics"); -// } -// -// logFileText.Should().ContainEquivalentOf($"MachineName: {Environment.MachineName}", "the MachineName should be in our diagnostics"); -// logFileText.Should().ContainEquivalentOf($"ProcessorCount: {Environment.ProcessorCount}", "the ProcessorCount should be in our diagnostics"); -// logFileText.Should().ContainEquivalentOf($"CurrentDirectory: {Directory.GetCurrentDirectory()}", "the CurrentDirectory should be in our diagnostics"); -// logFileText.Should().ContainEquivalentOf($"TempDirectory: {Path.GetTempPath()}", "the TempDirectory should be in our diagnostics"); -// logFileText.Should().ContainEquivalentOf("HostProcessName: ", "the HostProcessName should be in our diagnostics"); -// stdout.Should().NotContainEquivalentOf($"{RuntimeInformation.OSDescription}", "the OSVersion should not be written to stdout"); -// await Task.CompletedTask; -// } -// catch (NotLoggedYetException) -// { -// Logger.Error("Failed to get new log content"); -// Logger.Error($"Process exit code {exitCode}"); -// Logger.Error($"Starting log text: {Environment.NewLine}{startingLogText}"); -// Logger.Error($"Current log text: {Environment.NewLine}{clientAndTentacle.RunningTentacle.ReadAllLogFileText()}"); -// Logger.Error($"Command StdOut: {Environment.NewLine}{stdout}"); -// Logger.Error($"Command StdErr: {Environment.NewLine}{stderr}"); -// -// throw; -// } -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task HelpAsFirstArgumentShouldShowCommandSpecificHelp(TentacleConfigurationTestCase tc) -// { -// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "help", "version"); -// stderr.Should().BeNullOrEmpty(); -// -// stdout.Should().Be( -// @"Usage: Tentacle version [] -// -// Where [] is any of: -// -// --format=VALUE The format of the output (text,json). Defaults -// to text. -// -// Or one of the common options: -// -// --help Show detailed help for this command -// "); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// // Run these tests in serial to avoid conflicts -// [NonParallelizable] -// public async Task ShowConfigurationCommand(TentacleConfigurationTestCase tc) -// { -// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); -// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); -// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( -// tc, -// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, -// "show-configuration", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); -// -// stderr.Should().BeNullOrEmpty(); -// -// // Actually parse and query the document just like our consumer will -// dynamic? settings = JsonConvert.DeserializeObject(stdout); -// -// ((string)settings.Octopus.Home).Should().Be(clientAndTentacle.RunningTentacle.HomeDirectory, "the home directory should match"); -// ((string)settings.Tentacle.Deployment.ApplicationDirectory).Should().Be(clientAndTentacle.RunningTentacle.ApplicationDirectory, "the application directory should match"); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// // Run these tests in serial to avoid conflicts -// [NonParallelizable] -// public async Task ShowConfigurationCommandOnPartiallyConfiguredTentacle(TentacleConfigurationTestCase tc) -// { -// using var homeDirectory = new TemporaryDirectory(); -// var environmentVariables = new Dictionary { { EnvironmentVariables.TentacleMachineConfigurationHomeDirectory, homeDirectory.DirectoryPath } }; -// -// var instanceId = Guid.NewGuid().ToString(); -// using var temporaryDirectory = new TemporaryDirectory(); -// await RunCommandAndAssertExitsWithSuccessExitCode( -// tc, -// environmentVariables, -// "create-instance", $"--instance={instanceId}", "--config", Path.Combine(temporaryDirectory.DirectoryPath, instanceId + ".cfg")); -// -// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( -// tc, -// environmentVariables, -// "show-configuration", -// $"--instance={instanceId}"); -// -// stderr.Should().BeNullOrEmpty(); -// -// // Actually parse and query the document just like our consumer will -// dynamic? settings = JsonConvert.DeserializeObject(stdout); -// ((string)settings.Octopus.Home).Should().Be(temporaryDirectory.DirectoryPath, "the home directory should match"); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// // Run these tests in serial to avoid conflicts -// [NonParallelizable] -// public async Task ShowConfigurationCommandLooksSensibleToHumans(TentacleConfigurationTestCase tc) -// { -// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); -// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); -// var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( -// tc, -// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, -// "show-configuration", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); -// -// stderr.Should().BeNullOrEmpty(); -// -// if (tc.TentacleType == TentacleType.Polling) -// { -// stdout.Should().Be($@"{{ -// ""Octopus"": {{ -// ""Home"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.HomeDirectory)}"", -// ""Watchdog"": {{ -// ""Enabled"": false, -// ""Instances"": ""*"", -// ""Interval"": 0 -// }} -// }}, -// ""Tentacle"": {{ -// ""CertificateThumbprint"": ""{clientAndTentacle.RunningTentacle.Thumbprint}"", -// ""Communication"": {{ -// ""TrustedOctopusServers"": [ -// {{ -// ""Thumbprint"": ""{clientAndTentacle.Server.Thumbprint}"", -// ""CommunicationStyle"": 2, -// ""Address"": ""https://localhost:{clientAndTentacle.Server.ServerListeningPort}"", -// ""Squid"": null, -// ""SubscriptionId"": ""{clientAndTentacle.RunningTentacle.ServiceUri}"" -// }} -// ] -// }}, -// ""Deployment"": {{ -// ""ApplicationDirectory"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.ApplicationDirectory)}"" -// }}, -// ""Services"": {{ -// ""ListenIP"": null, -// ""NoListen"": true, -// ""PortNumber"": 10933 -// }} -// }} -// }} -// "); -// } -// else -// { -// stdout.Should().Be($@"{{ -// ""Octopus"": {{ -// ""Home"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.HomeDirectory)}"", -// ""Watchdog"": {{ -// ""Enabled"": false, -// ""Instances"": ""*"", -// ""Interval"": 0 -// }} -// }}, -// ""Tentacle"": {{ -// ""CertificateThumbprint"": ""{clientAndTentacle.RunningTentacle.Thumbprint}"", -// ""Communication"": {{ -// ""TrustedOctopusServers"": [ -// {{ -// ""Thumbprint"": ""{clientAndTentacle.Server.Thumbprint}"", -// ""CommunicationStyle"": 1, -// ""Address"": null, -// ""Squid"": null, -// ""SubscriptionId"": null -// }} -// ] -// }}, -// ""Deployment"": {{ -// ""ApplicationDirectory"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.ApplicationDirectory)}"" -// }}, -// ""Services"": {{ -// ""ListenIP"": null, -// ""NoListen"": false, -// ""PortNumber"": {clientAndTentacle.RunningTentacle.ServiceUri.Port} -// }} -// }} -// }} -// "); -// } -// -// await Task.CompletedTask; -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// [WindowsTest] -// // Run these tests in serial to avoid conflicts -// [NonParallelizable] -// public async Task WatchdogCreateAndDeleteCommand(TentacleConfigurationTestCase tc) -// { -// await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); -// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); -// var create = await RunCommandAndAssertExitsWithSuccessExitCode( -// tc, -// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, -// "watchdog", "--create", $"--instances={clientAndTentacle.RunningTentacle.InstanceName}"); -// -// create.StdError.Should().BeNullOrEmpty(); -// create.StdOut.Should().ContainEquivalentOf("Creating watchdog task"); -// var delete = await RunCommandAndAssertExitsWithSuccessExitCode( -// tc, -// clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, -// "watchdog", "--delete", $"--instances={clientAndTentacle.RunningTentacle.InstanceName}"); -// -// delete.StdError.Should().BeNullOrEmpty(); -// delete.StdOut.Should().ContainEquivalentOf("Removing watchdog task"); -// } -// -// FileVersionInfo GetVersionInfo(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var tentacleExe = TentacleExeFinder.FindTentacleExe(tentacleConfigurationTestCase.TentacleRuntime); -// -// if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) -// { -// return FileVersionInfo.GetVersionInfo(tentacleExe); -// } -// -// //todo: change this to trust the value set in the context.TentacleExePath (will need a renovation of ExePathResolver to be non windows specific) -// return FileVersionInfo.GetVersionInfo($"{tentacleExe}.dll"); -// } -// -// async Task<(int ExitCode, string StdOut, string StdError)> RunCommandAndAssertExitsWithSuccessExitCode( -// TentacleConfigurationTestCase tentacleConfigurationTestCase, -// IReadOnlyDictionary? environmentVariables, -// params string[] arguments) -// { -// var (exitCode, stdout, stderr) = await RunCommand(tentacleConfigurationTestCase, environmentVariables, arguments); -// exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); -// return (exitCode, stdout, stderr); -// } -// -// async Task<(int ExitCode, string StdOut, string StdError)> RunCommand( -// TentacleConfigurationTestCase tentacleConfigurationTestCase, -// IReadOnlyDictionary? environmentVariables, -// params string[] arguments) -// { -// using var tempDirectory = new TemporaryDirectory(); -// -// var environmentVariablesToRunTentacleWith = new Dictionary(); -// -// if (environmentVariables?.Any() == true) -// { -// environmentVariablesToRunTentacleWith.AddRange(environmentVariables); -// } -// -// if (!environmentVariablesToRunTentacleWith.ContainsKey(EnvironmentVariables.TentacleMachineConfigurationHomeDirectory)) -// { -// environmentVariablesToRunTentacleWith.Add(EnvironmentVariables.TentacleMachineConfigurationHomeDirectory, tempDirectory.DirectoryPath); -// } -// -// var tentacleExe = TentacleExeFinder.FindTentacleExe(tentacleConfigurationTestCase.TentacleRuntime); -// var output = new StringBuilder(); -// var errorOut = new StringBuilder(); -// -// var result = await RetryHelper.RetryAsync( -// () => Cli.Wrap(tentacleExe) -// .WithArguments(arguments) -// .WithValidation(CommandResultValidation.None) -// .WithStandardOutputPipe(PipeTarget.ToStringBuilder(output)) -// .WithStandardErrorPipe(PipeTarget.ToStringBuilder(errorOut)) -// .WithEnvironmentVariables(environmentVariablesToRunTentacleWith) -// .ExecuteAsync(CancellationToken)); -// -// return (result.ExitCode, output.ToString(), errorOut.ToString()); -// } -// -// static string JsonFormattedPath(string path) -// { -// return path.Replace(@"\", @"\\"); -// } -// -// public class NotLoggedYetException : Exception -// { -// } -// } -// } +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Security.Principal; +using System.Text; +using System.Threading.Tasks; +using CliWrap; +using CliWrap.Exceptions; +using FluentAssertions; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Util; +using Octopus.Tentacle.Variables; +using Polly; +using PlatformDetection = Octopus.Tentacle.Util.PlatformDetection; + +namespace Octopus.Tentacle.Tests.Integration +{ + /// + /// These tests provide guarantees around how our command-line interface works, especially for scenarios where people automate setup. + /// Please review any changes to the assertions made by these tests carefully. + /// + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("we don't expect customers to execute it in this environment in a situation where it needs to behave within a reasonable timeframe.")] + public class TentacleCommandLineTests : IntegrationTest + { + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task TentacleExeNoArguments(TentacleConfigurationTestCase tc) + { + var (exitCode, stdout, stderr) = await RunCommand(tc, null); + + exitCode.Should().Be(2, "the exit code should be 2 if the command wasn't understood"); + stdout.Should().StartWithEquivalentOf("Usage: Tentacle []", "should show help by default if no other commands are specified"); + stdout.Should().ContainEquivalentOf("Or use --help for more details.", "should provide the hint for command-specific help"); + stderr.Should().BeNullOrEmpty(); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task UnknownCommand(TentacleConfigurationTestCase tc) + { + var (exitCode, stdout, stderr) = await RunCommand(tc, null, "unknown-command"); + + exitCode.Should().Be(2, "the exit code should be 2 if the command wasn't understood"); + stderr.Should().StartWithEquivalentOf("Command 'unknown-command' is not supported", "the error should clearly indicate the command which is not understood"); + stdout.Should().StartWithEquivalentOf("See 'Tentacle help'", "should provide the hint to use help"); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task UnknownArgument(TentacleConfigurationTestCase tc) + { + var (exitCode, stdout, stderr) = await RunCommand(tc, null, "version", "--unknown=argument"); + + exitCode.Should().Be(1, "the exit code should be 1 if the command has unknown arguments"); + stdout.Should().BeNullOrEmpty("the error message should be written to stderr, not stdout"); + stderr.Should().ContainEquivalentOf("Unrecognized command line arguments: --unknown=argument", "the error message (written to stderr) should clearly indicate which arguments couldn't be parsed."); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task InvalidArgument(TentacleConfigurationTestCase tc) + { + var (exitCode, stdout, stderr) = await RunCommand(tc, null, "version", "--format=unsupported"); + + exitCode.Should().Be(1, "the exit code should be 1 if the command has unknown arguments"); + stdout.Should().BeNullOrEmpty("the error message should be written to stderr, not stdout"); + stderr.Should().ContainEquivalentOf("The format 'unsupported' is not supported. Try text or json.", "the error message (written to stderr) should clearly indicate which argument was invalid."); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task NoConsoleLoggingSwitchStillSilentlySupportedForBackwardsCompat(TentacleConfigurationTestCase tc) + { + var (_, _, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--noconsolelogging"); + + stderr.Should().BeNullOrEmpty(); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task NoLogoSwitchStillSilentlySupportedForBackwardsCompat(TentacleConfigurationTestCase tc) + { + var (_, _, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--nologo"); + + stderr.Should().BeNullOrEmpty(); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task ConsoleSwitchStillSilentlySupportedForBackwardsCompat(TentacleConfigurationTestCase tc) + { + var (_, _, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--console"); + + stderr.Should().BeNullOrEmpty(); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task ShouldSupportFuzzyCommandParsing(TentacleConfigurationTestCase tc) + { + await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version"); + await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "--version"); + await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "/version"); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task VersionCommandTextFormat(TentacleConfigurationTestCase tc) + { + var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version"); + + var expectedVersion = GetVersionInfo(tc); + + stdout.Should().Be(expectedVersion.ProductVersion, "The version command should print the informational version as text"); + stderr.Should().BeNullOrEmpty(); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task VersionCommandJsonFormat(TentacleConfigurationTestCase tc) + { + var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--format=json"); + + var expectedVersion = GetVersionInfo(tc); + var output = JObject.Parse(stdout); + + output["InformationalVersion"].Value().Should().Be(expectedVersion.ProductVersion, "The version command should print the informational version in the JSON output"); + output["MajorMinorPatch"].Value().Should().Be($"{expectedVersion.FileMajorPart}.{expectedVersion.FileMinorPart}.{expectedVersion.FileBuildPart}", "The version command should print the version in the json output"); + output["NuGetVersion"].Value().Should().NotBeNull("The version command should print the NuGet version in the JSON output"); + output["SourceBranchName"].Value().Should().NotBeNull("The version command should print the source branch in the JSON output"); + + stderr.Should().BeNullOrEmpty(); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task CanGetHelpForHelp(TentacleConfigurationTestCase tc) + { + var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "help", "--help"); + stderr.Should().BeNullOrEmpty(); + + stdout.Should().Be( +@"Usage: Tentacle help [] + +Where [] is any of: + + --format=VALUE The format of the output (text,json). Defaults + to text. + +Or one of the common options: + + --help Show detailed help for this command +"); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task HelpAsSwitchShouldShowCommandSpecificHelp(TentacleConfigurationTestCase tc) + { + var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--help"); + stderr.Should().BeNullOrEmpty(); + + stdout.Should().Be( + @"Usage: Tentacle version [] + +Where [] is any of: + + --format=VALUE The format of the output (text,json). Defaults + to text. + +Or one of the common options: + + --help Show detailed help for this command +"); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task GeneralHelpAsJsonCanBeParsedByAutomationScripts(TentacleConfigurationTestCase tc) + { + var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "help", "--format=json"); + + stderr.Should().BeNullOrEmpty(); + var help = JsonConvert.DeserializeAnonymousType( + stdout, + new + { + Commands = new[] + { + new + { + Name = "", + Description = "", + Aliases = Array.Empty() + } + } + }); + + help.Commands.Select(c => c.Name) + .Should() + .Contain( + "configure", + "help", + "run", + "version", + "show-master-key", + "show-thumbprint"); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task CommandSpecificHelpAsJsonCanBeParsedByAutomationScripts(TentacleConfigurationTestCase tc) + { + var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--help", "--format=json"); + + stderr.Should().BeNullOrEmpty(); + var help = JsonConvert.DeserializeAnonymousType( + stdout, + new + { + Name = "", + Description = "", + Aliases = Array.Empty(), + Options = new[] + { + new + { + Name = "", + Description = "" + } + } + }); + + help.Name.Should().Be("version"); + help.Options.Select(o => o.Name).Should().Contain("format"); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task CommandSpecificHelpAsJsonLooksSensibleToHumans(TentacleConfigurationTestCase tc) + { + var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "version", "--help", "--format=json"); + stderr.Should().BeNullOrEmpty(); + + stdout.Should().Be( +@"{ + ""Name"": ""version"", + ""Description"": ""Show the Tentacle version information"", + ""Aliases"": [], + ""Options"": [ + { + ""Name"": ""format"", + ""Description"": ""The format of the output (text,json). Defaults to text."" + } + ], + ""CommonOptions"": [ + { + ""Name"": ""help"", + ""Description"": ""Show detailed help for this command"" + } + ] +}"); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task HelpForInstanceSpecificCommandsAlwaysWorks(TentacleConfigurationTestCase tc) + { + var (_, stdout, stderr) = await RunCommand(tc, null, "help", "--format=json"); + + stderr.Should().BeNullOrEmpty(); + var help = JsonConvert.DeserializeAnonymousType( + stdout, + new + { + Commands = new[] + { + new + { + Name = "", + Description = "", + Aliases = new string[0] + } + } + }); + + help.Commands.Should().HaveCountGreaterThan(0); + + var failed = help.Commands.Select(async c => + { + var (exitCode2, stdout2, stderr2) = await RunCommand(tc, null,$"{c.Name}", "--help"); + return new + { + Command = c, + ExitCode = exitCode2, + StdOut = stdout2, + StdErr = stderr2, + HasExpectedExitCode = exitCode2 == 0, + HasExpectedHelpMessage = stdout2.StartsWith($"Usage: Tentacle {c.Name} []") + }; + }) + .Where(r => !(r.Result.HasExpectedExitCode && r.Result.HasExpectedHelpMessage)) + .ToArray(); + + if (failed.Any()) + { + var failureDetails = string.Empty; + + foreach (var failure in failed) + { + failureDetails += $@"{failure.Result.Command.Name} +StdErr:{failure.Result.StdErr} +StdOut:{failure.Result.StdOut}"; + } + + Assert.Fail( +$@"The following commands cannot show help without specifying the --instance argument: +{failureDetails} +The details are logged above. These commands probably need to take Lazy dependencies so they can be instantiated for showing help without requiring every dependency to be resolvable."); + } + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + // Run these tests in serial to avoid conflicts + [NonParallelizable] + public async Task InvalidInstance(TentacleConfigurationTestCase tc) + { + var (exitCode, stdout, stderr) = await RunCommand( + tc, + null, + "show-thumbprint", "--instance=invalidinstance"); + + exitCode.Should().Be(1, $"the exit code should be 1 if the instance is not able to be resolved"); + stderr.Should().ContainEquivalentOf("Instance invalidinstance of tentacle has not been configured", "the error message should make it clear the instance has not been configured"); + stderr.Should().ContainEquivalentOf("Available instances:", "should provide a hint as to which instances are available on the machine"); + stdout.Should().BeNullOrEmpty(); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + // Run these tests in serial to avoid conflicts + [NonParallelizable] + public async Task ShowThumbprintCommandText(TentacleConfigurationTestCase tc) + { + await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); + await clientAndTentacle.RunningTentacle.Stop(CancellationToken); + var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( + tc, + clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, + "show-thumbprint", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); + + exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); + stdout.Should().Be(TestCertificates.TentaclePublicThumbprint, "the thumbprint should be written directly to stdout"); + stderr.Should().BeNullOrEmpty(); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + // Run these tests in serial to avoid conflicts + [NonParallelizable] + public async Task ShowThumbprintCommandJson(TentacleConfigurationTestCase tc) + { + await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); + await clientAndTentacle.RunningTentacle.Stop(CancellationToken); + var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( + tc, + clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, + "show-thumbprint", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}", "--format=json"); + + exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); + stdout.Should().Be(JsonConvert.SerializeObject(new { Thumbprint = TestCertificates.TentaclePublicThumbprint }), "the thumbprint should be written directly to stdout as JSON"); + stderr.Should().BeNullOrEmpty(); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + // Run these tests in serial to avoid conflicts + [NonParallelizable] + public async Task ListInstancesCommandText(TentacleConfigurationTestCase tc) + { + await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); + await clientAndTentacle.RunningTentacle.Stop(CancellationToken); + var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( + tc, + clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, + "list-instances", "--format=text"); + + exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); + var configPath = Path.Combine(clientAndTentacle.RunningTentacle.HomeDirectory, clientAndTentacle.RunningTentacle.InstanceName + ".cfg"); + stdout.Should().Contain($"Instance '{clientAndTentacle.RunningTentacle.InstanceName}' uses configuration '{configPath}'.", "the current instance should be listed"); + stderr.Should().BeNullOrEmpty(); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + // Run these tests in serial to avoid conflicts + [NonParallelizable] + public async Task ListInstancesCommandJson(TentacleConfigurationTestCase tc) + { + await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); + await clientAndTentacle.RunningTentacle.Stop(CancellationToken); + var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( + tc, + clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, + "list-instances", "--format=json"); + + stdout.Should().Contain($"\"InstanceName\": \"{clientAndTentacle.RunningTentacle.InstanceName}\"", "the current instance should be listed"); + var configPath = Path.Combine(clientAndTentacle.RunningTentacle.HomeDirectory, clientAndTentacle.RunningTentacle.InstanceName + ".cfg"); + var jsonFormattedPath = JsonFormattedPath(configPath); + stdout.Should().Contain($"\"ConfigurationFilePath\": \"{jsonFormattedPath}\"", "the path to the config file for the current instance should be listed"); + stderr.Should().BeNullOrEmpty(); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + // Run these tests in serial to avoid conflicts + [NonParallelizable] + public async Task ShouldLogStartupDiagnosticsToInstanceLogFileOnly(TentacleConfigurationTestCase tc) + { + await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); + await clientAndTentacle.RunningTentacle.Stop(CancellationToken); + + var startingLogText = clientAndTentacle.RunningTentacle.ReadAllLogFileText(); + + var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( + tc, + clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, + "show-thumbprint", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); + + try + { + var logFileText = Policy + .Handle() + .WaitAndRetry( + 20, + i => TimeSpan.FromMilliseconds(100 * i), + (exception, _) => { Logger.Information($"Failed to get new log content: {exception.Message}. Retrying!"); }) + .Execute( + () => + { + var wholeLog = clientAndTentacle.RunningTentacle.ReadAllLogFileText(); + var newLog = wholeLog.Replace(startingLogText, string.Empty); + if (string.IsNullOrWhiteSpace(newLog) || !newLog.Contains("CommandLine:")) + { + throw new NotLoggedYetException(); + } + return newLog; + }); + + logFileText.Should().ContainEquivalentOf($"OperatingSystem: {RuntimeInformation.OSDescription}", "the OSVersion should be in our diagnostics"); + logFileText.Should().ContainEquivalentOf("OperatingSystem:", "the OSVersion should be in our diagnostics"); + logFileText.Should().ContainEquivalentOf($"OsBitVersion: {(Environment.Is64BitOperatingSystem ? "x64" : "x86")}", "the OsBitVersion should be in our diagnostics"); + logFileText.Should().ContainEquivalentOf($"Is64BitProcess: {Environment.Is64BitProcess}", "the Is64BitProcess should be in our diagnostics"); + + if (PlatformDetection.IsRunningOnWindows) + { +#pragma warning disable CA1416 + logFileText.Should().ContainEquivalentOf($"CurrentUser: {WindowsIdentity.GetCurrent().Name}", "the CurrentUser should be in our diagnostics"); +#pragma warning disable CA1416 + } + else + { + logFileText.Should().ContainEquivalentOf($"CurrentUser: {Environment.UserName}", "the CurrentUser should be in our diagnostics"); + } + + logFileText.Should().ContainEquivalentOf($"MachineName: {Environment.MachineName}", "the MachineName should be in our diagnostics"); + logFileText.Should().ContainEquivalentOf($"ProcessorCount: {Environment.ProcessorCount}", "the ProcessorCount should be in our diagnostics"); + logFileText.Should().ContainEquivalentOf($"CurrentDirectory: {Directory.GetCurrentDirectory()}", "the CurrentDirectory should be in our diagnostics"); + logFileText.Should().ContainEquivalentOf($"TempDirectory: {Path.GetTempPath()}", "the TempDirectory should be in our diagnostics"); + logFileText.Should().ContainEquivalentOf("HostProcessName: ", "the HostProcessName should be in our diagnostics"); + stdout.Should().NotContainEquivalentOf($"{RuntimeInformation.OSDescription}", "the OSVersion should not be written to stdout"); + await Task.CompletedTask; + } + catch (NotLoggedYetException) + { + Logger.Error("Failed to get new log content"); + Logger.Error($"Process exit code {exitCode}"); + Logger.Error($"Starting log text: {Environment.NewLine}{startingLogText}"); + Logger.Error($"Current log text: {Environment.NewLine}{clientAndTentacle.RunningTentacle.ReadAllLogFileText()}"); + Logger.Error($"Command StdOut: {Environment.NewLine}{stdout}"); + Logger.Error($"Command StdErr: {Environment.NewLine}{stderr}"); + + throw; + } + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task HelpAsFirstArgumentShouldShowCommandSpecificHelp(TentacleConfigurationTestCase tc) + { + var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, null, "help", "version"); + stderr.Should().BeNullOrEmpty(); + + stdout.Should().Be( +@"Usage: Tentacle version [] + +Where [] is any of: + + --format=VALUE The format of the output (text,json). Defaults + to text. + +Or one of the common options: + + --help Show detailed help for this command +"); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + // Run these tests in serial to avoid conflicts + [NonParallelizable] + public async Task ShowConfigurationCommand(TentacleConfigurationTestCase tc) + { + await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); + await clientAndTentacle.RunningTentacle.Stop(CancellationToken); + var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( + tc, + clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, + "show-configuration", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); + + stderr.Should().BeNullOrEmpty(); + + // Actually parse and query the document just like our consumer will + dynamic? settings = JsonConvert.DeserializeObject(stdout); + + ((string)settings.Octopus.Home).Should().Be(clientAndTentacle.RunningTentacle.HomeDirectory, "the home directory should match"); + ((string)settings.Tentacle.Deployment.ApplicationDirectory).Should().Be(clientAndTentacle.RunningTentacle.ApplicationDirectory, "the application directory should match"); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + // Run these tests in serial to avoid conflicts + [NonParallelizable] + public async Task ShowConfigurationCommandOnPartiallyConfiguredTentacle(TentacleConfigurationTestCase tc) + { + using var homeDirectory = new TemporaryDirectory(); + var environmentVariables = new Dictionary { { EnvironmentVariables.TentacleMachineConfigurationHomeDirectory, homeDirectory.DirectoryPath } }; + + var instanceId = Guid.NewGuid().ToString(); + using var temporaryDirectory = new TemporaryDirectory(); + await RunCommandAndAssertExitsWithSuccessExitCode( + tc, + environmentVariables, + "create-instance", $"--instance={instanceId}", "--config", Path.Combine(temporaryDirectory.DirectoryPath, instanceId + ".cfg")); + + var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( + tc, + environmentVariables, + "show-configuration", + $"--instance={instanceId}"); + + stderr.Should().BeNullOrEmpty(); + + // Actually parse and query the document just like our consumer will + dynamic? settings = JsonConvert.DeserializeObject(stdout); + ((string)settings.Octopus.Home).Should().Be(temporaryDirectory.DirectoryPath, "the home directory should match"); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + // Run these tests in serial to avoid conflicts + [NonParallelizable] + public async Task ShowConfigurationCommandLooksSensibleToHumans(TentacleConfigurationTestCase tc) + { + await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); + await clientAndTentacle.RunningTentacle.Stop(CancellationToken); + var (_, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode( + tc, + clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, + "show-configuration", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}"); + + stderr.Should().BeNullOrEmpty(); + + if (tc.TentacleType == TentacleType.Polling) + { + stdout.Should().Be($@"{{ + ""Octopus"": {{ + ""Home"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.HomeDirectory)}"", + ""Watchdog"": {{ + ""Enabled"": false, + ""Instances"": ""*"", + ""Interval"": 0 + }} + }}, + ""Tentacle"": {{ + ""CertificateThumbprint"": ""{clientAndTentacle.RunningTentacle.Thumbprint}"", + ""Communication"": {{ + ""TrustedOctopusServers"": [ + {{ + ""Thumbprint"": ""{clientAndTentacle.Server.Thumbprint}"", + ""CommunicationStyle"": 2, + ""Address"": ""https://localhost:{clientAndTentacle.Server.ServerListeningPort}"", + ""Squid"": null, + ""SubscriptionId"": ""{clientAndTentacle.RunningTentacle.ServiceUri}"" + }} + ] + }}, + ""Deployment"": {{ + ""ApplicationDirectory"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.ApplicationDirectory)}"" + }}, + ""Services"": {{ + ""ListenIP"": null, + ""NoListen"": true, + ""PortNumber"": 10933 + }} + }} +}} +"); + } + else + { + stdout.Should().Be($@"{{ + ""Octopus"": {{ + ""Home"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.HomeDirectory)}"", + ""Watchdog"": {{ + ""Enabled"": false, + ""Instances"": ""*"", + ""Interval"": 0 + }} + }}, + ""Tentacle"": {{ + ""CertificateThumbprint"": ""{clientAndTentacle.RunningTentacle.Thumbprint}"", + ""Communication"": {{ + ""TrustedOctopusServers"": [ + {{ + ""Thumbprint"": ""{clientAndTentacle.Server.Thumbprint}"", + ""CommunicationStyle"": 1, + ""Address"": null, + ""Squid"": null, + ""SubscriptionId"": null + }} + ] + }}, + ""Deployment"": {{ + ""ApplicationDirectory"": ""{JsonFormattedPath(clientAndTentacle.RunningTentacle.ApplicationDirectory)}"" + }}, + ""Services"": {{ + ""ListenIP"": null, + ""NoListen"": false, + ""PortNumber"": {clientAndTentacle.RunningTentacle.ServiceUri.Port} + }} + }} +}} +"); + } + + await Task.CompletedTask; + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + [WindowsTest] + // Run these tests in serial to avoid conflicts + [NonParallelizable] + public async Task WatchdogCreateAndDeleteCommand(TentacleConfigurationTestCase tc) + { + await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken); + await clientAndTentacle.RunningTentacle.Stop(CancellationToken); + var create = await RunCommandAndAssertExitsWithSuccessExitCode( + tc, + clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, + "watchdog", "--create", $"--instances={clientAndTentacle.RunningTentacle.InstanceName}"); + + create.StdError.Should().BeNullOrEmpty(); + create.StdOut.Should().ContainEquivalentOf("Creating watchdog task"); + var delete = await RunCommandAndAssertExitsWithSuccessExitCode( + tc, + clientAndTentacle.RunningTentacle.RunTentacleEnvironmentVariables, + "watchdog", "--delete", $"--instances={clientAndTentacle.RunningTentacle.InstanceName}"); + + delete.StdError.Should().BeNullOrEmpty(); + delete.StdOut.Should().ContainEquivalentOf("Removing watchdog task"); + } + + FileVersionInfo GetVersionInfo(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var tentacleExe = TentacleExeFinder.FindTentacleExe(tentacleConfigurationTestCase.TentacleRuntime); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return FileVersionInfo.GetVersionInfo(tentacleExe); + } + + //todo: change this to trust the value set in the context.TentacleExePath (will need a renovation of ExePathResolver to be non windows specific) + return FileVersionInfo.GetVersionInfo($"{tentacleExe}.dll"); + } + + async Task<(int ExitCode, string StdOut, string StdError)> RunCommandAndAssertExitsWithSuccessExitCode( + TentacleConfigurationTestCase tentacleConfigurationTestCase, + IReadOnlyDictionary? environmentVariables, + params string[] arguments) + { + var (exitCode, stdout, stderr) = await RunCommand(tentacleConfigurationTestCase, environmentVariables, arguments); + exitCode.Should().Be(0, $"we expected the command to succeed.\r\nStdErr: '{stderr}'\r\nStdOut: '{stdout}'"); + return (exitCode, stdout, stderr); + } + + async Task<(int ExitCode, string StdOut, string StdError)> RunCommand( + TentacleConfigurationTestCase tentacleConfigurationTestCase, + IReadOnlyDictionary? environmentVariables, + params string[] arguments) + { + using var tempDirectory = new TemporaryDirectory(); + + var environmentVariablesToRunTentacleWith = new Dictionary(); + + if (environmentVariables?.Any() == true) + { + environmentVariablesToRunTentacleWith.AddRange(environmentVariables); + } + + if (!environmentVariablesToRunTentacleWith.ContainsKey(EnvironmentVariables.TentacleMachineConfigurationHomeDirectory)) + { + environmentVariablesToRunTentacleWith.Add(EnvironmentVariables.TentacleMachineConfigurationHomeDirectory, tempDirectory.DirectoryPath); + } + + var tentacleExe = TentacleExeFinder.FindTentacleExe(tentacleConfigurationTestCase.TentacleRuntime); + var output = new StringBuilder(); + var errorOut = new StringBuilder(); + + var result = await RetryHelper.RetryAsync( + () => Cli.Wrap(tentacleExe) + .WithArguments(arguments) + .WithValidation(CommandResultValidation.None) + .WithStandardOutputPipe(PipeTarget.ToStringBuilder(output)) + .WithStandardErrorPipe(PipeTarget.ToStringBuilder(errorOut)) + .WithEnvironmentVariables(environmentVariablesToRunTentacleWith) + .ExecuteAsync(CancellationToken)); + + return (result.ExitCode, output.ToString(), errorOut.ToString()); + } + + static string JsonFormattedPath(string path) + { + return path.Replace(@"\", @"\\"); + } + + public class NotLoggedYetException : Exception + { + } + } +} From 2b63f18b5e4de0846ad899c2933c63bc7137a769 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Mon, 11 Nov 2024 17:51:52 +1300 Subject: [PATCH 29/34] Don't comment out tests - skip them using the attribute --- .../ClientGathersRpcCallMetrics.cs | 456 +++--- ...ionCanBeCancelledWhenRetriesAreDisabled.cs | 870 +++++------ ...tionCanBeCancelledWhenRetriesAreEnabled.cs | 1346 +++++++++-------- ...iptExecutionCanRecoverFromNetworkIssues.cs | 618 ++++---- .../ClientScriptExecutionRetriesTimeout.cs | 1097 +++++++------- ...ClientScriptExecutionScriptFilesAreSent.cs | 78 +- ...NonV1IsNotRetriedWhenRetriesAreDisabled.cs | 586 +++---- ...iptExecutionScriptServiceV1IsNotRetried.cs | 504 +++--- ...criptExecutionWorksWithMultipleVersions.cs | 80 +- ...orObservesScriptObserverBackoffStrategy.cs | 88 +- .../MachineConfigurationHomeDirectoryTests.cs | 110 +- .../ScriptServiceTests.cs | 274 ++-- .../ScriptServiceV2IntegrationTest.cs | 432 +++--- .../TentacleClientObserver.cs | 322 ++-- .../TentacleStartupAndShutdownTests.cs | 145 +- .../WorkspaceCleanerTests.cs | 516 +++---- 16 files changed, 3774 insertions(+), 3748 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientGathersRpcCallMetrics.cs b/source/Octopus.Tentacle.Tests.Integration/ClientGathersRpcCallMetrics.cs index 4c76806fc..c06b15995 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientGathersRpcCallMetrics.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientGathersRpcCallMetrics.cs @@ -1,227 +1,229 @@ -// using System; -// using System.IO; -// using System.Linq; -// using System.Threading.Tasks; -// using FluentAssertions; -// using Halibut; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Contracts.Capabilities; -// using Octopus.Tentacle.Contracts.Observability; -// using Octopus.Tentacle.Contracts.ScriptServiceV2; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class ClientGathersRpcCallMetrics : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(testCommonVersions: true)] -// public async Task ExecuteScriptShouldGatherMetrics_WhenSucceeds(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// // Arrange -// var tentacleClientObserver = new TestTentacleClientObserver(); -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleClientObserver(tentacleClientObserver) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b.Print("Hello")) -// .Build(); -// -// // Act -// var (finalResponse, _) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// -// // Assert -// finalResponse.State.Should().Be(ProcessState.Complete); -// -// var executeScriptMetrics = tentacleClientObserver.ExecuteScriptMetrics.Should().ContainSingle().Subject; -// ThenClientOperationMetricsShouldBeSuccessful(executeScriptMetrics); -// -// var expectedScriptService = tentacleConfigurationTestCase.ScriptServiceToTest == TentacleConfigurationTestCases.ScriptServiceV2Type -// ? nameof(IScriptServiceV2) -// : nameof(IScriptService); -// -// tentacleClientObserver.RpcCallMetrics.Should().NotBeEmpty(); -// tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == nameof(ICapabilitiesServiceV2.GetCapabilities) && m.RpcCall.Service == nameof(ICapabilitiesServiceV2)); -// tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == nameof(IScriptServiceV2.StartScript) && m.RpcCall.Service == expectedScriptService); -// tentacleClientObserver.RpcCallMetrics.Should().Contain(m => m.RpcCall.Name == nameof(IScriptServiceV2.GetStatus) && m.RpcCall.Service == expectedScriptService); -// tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == nameof(IScriptServiceV2.CompleteScript) && m.RpcCall.Service == expectedScriptService); -// tentacleClientObserver.RpcCallMetrics.Should().AllSatisfy(m => m.Succeeded.Should().BeTrue()); -// } -// -// [Test] -// [TentacleConfigurations(testCommonVersions: true)] -// public async Task ExecuteScriptShouldGatherMetrics_WhenFails(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// // Arrange -// var tentacleClientObserver = new TestTentacleClientObserver(); -// var exception = new HalibutClientException("Error"); -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleClientObserver(tentacleClientObserver) -// .WithRetryDuration(TimeSpan.FromSeconds(1)) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .DecorateAllScriptServicesWith(u => u -// .BeforeStartScript( -// () => throw exception)) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b.Print("Hello")) -// .Build(); -// -// // Act -// await AssertionExtensions.Should(() => clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken)).ThrowAsync(); -// -// // Assert -// var executeScriptMetrics = tentacleClientObserver.ExecuteScriptMetrics.Should().ContainSingle().Subject; -// ThenClientOperationMetricsShouldBeFailed(executeScriptMetrics, exception); -// -// tentacleClientObserver.RpcCallMetrics.Should().NotBeEmpty(); -// var startScriptMetric = tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == "StartScript").Subject; -// startScriptMetric.Succeeded.Should().BeFalse(); -// } -// -// [Test] -// [TentacleConfigurations(testCommonVersions: true)] -// public async Task UploadFileShouldGatherMetrics_WhenSucceeds(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// // Arrange -// var tentacleClientObserver = new TestTentacleClientObserver(); -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleClientObserver(tentacleClientObserver) -// .Build(CancellationToken); -// -// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); -// -// // Act -// await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); -// -// // Assert -// var uploadFileMetrics = tentacleClientObserver.UploadFileMetrics.Should().ContainSingle().Subject; -// ThenClientOperationMetricsShouldBeSuccessful(uploadFileMetrics); -// -// tentacleClientObserver.RpcCallMetrics.Should().HaveCountGreaterThan(0); -// var metric = tentacleClientObserver.RpcCallMetrics.Last(); -// metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.UploadFile)); -// metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); -// metric.Succeeded.Should().BeTrue(); -// } -// -// [Test] -// [TentacleConfigurations(testCommonVersions: true)] -// public async Task UploadFileShouldGatherMetrics_WhenFails(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// // Arrange -// var tentacleClientObserver = new TestTentacleClientObserver(); -// var exception = new HalibutClientException("Error"); -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleClientObserver(tentacleClientObserver) -// .WithRetryDuration(TimeSpan.FromSeconds(1)) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .DecorateFileTransferServiceWith(d => d.BeforeUploadFile(() => throw exception)) -// .Build()) -// .Build(CancellationToken); -// -// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); -// -// // Act -// await AssertionExtensions.Should(() => clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken)).ThrowAsync(); -// -// // Assert -// var uploadFileMetrics = tentacleClientObserver.UploadFileMetrics.Should().ContainSingle().Subject; -// ThenClientOperationMetricsShouldBeFailed(uploadFileMetrics, exception); -// -// tentacleClientObserver.RpcCallMetrics.Should().NotBeEmpty(); -// var metric = tentacleClientObserver.RpcCallMetrics.Last(); -// metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.UploadFile)); -// metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); -// metric.Succeeded.Should().BeFalse(); -// } -// -// [Test] -// [TentacleConfigurations(testCommonVersions: true)] -// public async Task DownloadFileShouldGatherMetrics_WhenSucceeds(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// // Arrange -// var tentacleClientObserver = new TestTentacleClientObserver(); -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleClientObserver(tentacleClientObserver) -// .Build(CancellationToken); -// -// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "DownloadFile.txt"); -// await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); -// -// // Act -// await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); -// -// // Assert -// var downloadFileMetrics = tentacleClientObserver.DownloadFileMetrics.Should().ContainSingle().Subject; -// ThenClientOperationMetricsShouldBeSuccessful(downloadFileMetrics); -// -// tentacleClientObserver.RpcCallMetrics.Should().HaveCountGreaterThan(1); // the first one will be the upload -// var metric = tentacleClientObserver.RpcCallMetrics.Last(); -// metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.DownloadFile)); -// metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); -// metric.Succeeded.Should().BeTrue(); -// } -// -// [Test] -// [TentacleConfigurations(testCommonVersions: true)] -// public async Task DownloadFileShouldGatherMetrics_WhenFails(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// // Arrange -// var tentacleClientObserver = new TestTentacleClientObserver(); -// var exception = new HalibutClientException("Error"); -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleClientObserver(tentacleClientObserver) -// .WithRetryDuration(TimeSpan.FromSeconds(1)) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .DecorateFileTransferServiceWith(d => d.BeforeDownloadFile(() => throw exception)) -// .Build()) -// .Build(CancellationToken); -// -// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "DownloadFile.txt"); -// await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); -// -// // Act -// await AssertionExtensions.Should(() => clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken)).ThrowAsync(); -// -// // Assert -// var downloadFileMetrics = tentacleClientObserver.DownloadFileMetrics.Should().ContainSingle().Subject; -// ThenClientOperationMetricsShouldBeFailed(downloadFileMetrics, exception); -// -// tentacleClientObserver.RpcCallMetrics.Should().HaveCountGreaterThan(1); // the first one will be the upload -// var metric = tentacleClientObserver.RpcCallMetrics.Last(); -// metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.DownloadFile)); -// metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); -// metric.Succeeded.Should().BeFalse(); -// } -// -// private static void ThenClientOperationMetricsShouldBeSuccessful(ClientOperationMetrics metric) -// { -// metric.Succeeded.Should().BeTrue(); -// metric.Exception.Should().BeNull(); -// metric.WasCancelled.Should().BeFalse(); -// -// metric.End.Should().BeOnOrAfter(metric.Start); -// metric.Duration.Should().Be(metric.End - metric.Start); -// } -// -// private static void ThenClientOperationMetricsShouldBeFailed(ClientOperationMetrics metric, Exception expectedException) -// { -// metric.Succeeded.Should().BeFalse(); -// metric.Exception.Should().BeEquivalentTo(expectedException); -// metric.WasCancelled.Should().BeFalse(); -// -// metric.End.Should().BeOnOrAfter(metric.Start); -// metric.Duration.Should().Be(metric.End - metric.Start); -// } -// } -// } \ No newline at end of file +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Halibut; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.Capabilities; +using Octopus.Tentacle.Contracts.Observability; +using Octopus.Tentacle.Contracts.ScriptServiceV2; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + public class ClientGathersRpcCallMetrics : IntegrationTest + { + [Test] + [TentacleConfigurations(testCommonVersions: true)] + [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] + public async Task ExecuteScriptShouldGatherMetrics_WhenSucceeds(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + // Arrange + var tentacleClientObserver = new TestTentacleClientObserver(); + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleClientObserver(tentacleClientObserver) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b.Print("Hello")) + .Build(); + + // Act + var (finalResponse, _) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + + // Assert + finalResponse.State.Should().Be(ProcessState.Complete); + + var executeScriptMetrics = tentacleClientObserver.ExecuteScriptMetrics.Should().ContainSingle().Subject; + ThenClientOperationMetricsShouldBeSuccessful(executeScriptMetrics); + + var expectedScriptService = tentacleConfigurationTestCase.ScriptServiceToTest == TentacleConfigurationTestCases.ScriptServiceV2Type + ? nameof(IScriptServiceV2) + : nameof(IScriptService); + + tentacleClientObserver.RpcCallMetrics.Should().NotBeEmpty(); + tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == nameof(ICapabilitiesServiceV2.GetCapabilities) && m.RpcCall.Service == nameof(ICapabilitiesServiceV2)); + tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == nameof(IScriptServiceV2.StartScript) && m.RpcCall.Service == expectedScriptService); + tentacleClientObserver.RpcCallMetrics.Should().Contain(m => m.RpcCall.Name == nameof(IScriptServiceV2.GetStatus) && m.RpcCall.Service == expectedScriptService); + tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == nameof(IScriptServiceV2.CompleteScript) && m.RpcCall.Service == expectedScriptService); + tentacleClientObserver.RpcCallMetrics.Should().AllSatisfy(m => m.Succeeded.Should().BeTrue()); + } + + [Test] + [TentacleConfigurations(testCommonVersions: true)] + public async Task ExecuteScriptShouldGatherMetrics_WhenFails(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + // Arrange + var tentacleClientObserver = new TestTentacleClientObserver(); + var exception = new HalibutClientException("Error"); + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleClientObserver(tentacleClientObserver) + .WithRetryDuration(TimeSpan.FromSeconds(1)) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .DecorateAllScriptServicesWith(u => u + .BeforeStartScript( + () => throw exception)) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b.Print("Hello")) + .Build(); + + // Act + await AssertionExtensions.Should(() => clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken)).ThrowAsync(); + + // Assert + var executeScriptMetrics = tentacleClientObserver.ExecuteScriptMetrics.Should().ContainSingle().Subject; + ThenClientOperationMetricsShouldBeFailed(executeScriptMetrics, exception); + + tentacleClientObserver.RpcCallMetrics.Should().NotBeEmpty(); + var startScriptMetric = tentacleClientObserver.RpcCallMetrics.Should().ContainSingle(m => m.RpcCall.Name == "StartScript").Subject; + startScriptMetric.Succeeded.Should().BeFalse(); + } + + [Test] + [TentacleConfigurations(testCommonVersions: true)] + public async Task UploadFileShouldGatherMetrics_WhenSucceeds(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + // Arrange + var tentacleClientObserver = new TestTentacleClientObserver(); + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleClientObserver(tentacleClientObserver) + .Build(CancellationToken); + + var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); + + // Act + await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); + + // Assert + var uploadFileMetrics = tentacleClientObserver.UploadFileMetrics.Should().ContainSingle().Subject; + ThenClientOperationMetricsShouldBeSuccessful(uploadFileMetrics); + + tentacleClientObserver.RpcCallMetrics.Should().HaveCountGreaterThan(0); + var metric = tentacleClientObserver.RpcCallMetrics.Last(); + metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.UploadFile)); + metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); + metric.Succeeded.Should().BeTrue(); + } + + [Test] + [TentacleConfigurations(testCommonVersions: true)] + public async Task UploadFileShouldGatherMetrics_WhenFails(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + // Arrange + var tentacleClientObserver = new TestTentacleClientObserver(); + var exception = new HalibutClientException("Error"); + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleClientObserver(tentacleClientObserver) + .WithRetryDuration(TimeSpan.FromSeconds(1)) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .DecorateFileTransferServiceWith(d => d.BeforeUploadFile(() => throw exception)) + .Build()) + .Build(CancellationToken); + + var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); + + // Act + await AssertionExtensions.Should(() => clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken)).ThrowAsync(); + + // Assert + var uploadFileMetrics = tentacleClientObserver.UploadFileMetrics.Should().ContainSingle().Subject; + ThenClientOperationMetricsShouldBeFailed(uploadFileMetrics, exception); + + tentacleClientObserver.RpcCallMetrics.Should().NotBeEmpty(); + var metric = tentacleClientObserver.RpcCallMetrics.Last(); + metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.UploadFile)); + metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); + metric.Succeeded.Should().BeFalse(); + } + + [Test] + [TentacleConfigurations(testCommonVersions: true)] + public async Task DownloadFileShouldGatherMetrics_WhenSucceeds(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + // Arrange + var tentacleClientObserver = new TestTentacleClientObserver(); + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleClientObserver(tentacleClientObserver) + .Build(CancellationToken); + + var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "DownloadFile.txt"); + await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); + + // Act + await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); + + // Assert + var downloadFileMetrics = tentacleClientObserver.DownloadFileMetrics.Should().ContainSingle().Subject; + ThenClientOperationMetricsShouldBeSuccessful(downloadFileMetrics); + + tentacleClientObserver.RpcCallMetrics.Should().HaveCountGreaterThan(1); // the first one will be the upload + var metric = tentacleClientObserver.RpcCallMetrics.Last(); + metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.DownloadFile)); + metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); + metric.Succeeded.Should().BeTrue(); + } + + [Test] + [TentacleConfigurations(testCommonVersions: true)] + public async Task DownloadFileShouldGatherMetrics_WhenFails(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + // Arrange + var tentacleClientObserver = new TestTentacleClientObserver(); + var exception = new HalibutClientException("Error"); + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleClientObserver(tentacleClientObserver) + .WithRetryDuration(TimeSpan.FromSeconds(1)) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .DecorateFileTransferServiceWith(d => d.BeforeDownloadFile(() => throw exception)) + .Build()) + .Build(CancellationToken); + + var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "DownloadFile.txt"); + await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); + + // Act + await AssertionExtensions.Should(() => clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken)).ThrowAsync(); + + // Assert + var downloadFileMetrics = tentacleClientObserver.DownloadFileMetrics.Should().ContainSingle().Subject; + ThenClientOperationMetricsShouldBeFailed(downloadFileMetrics, exception); + + tentacleClientObserver.RpcCallMetrics.Should().HaveCountGreaterThan(1); // the first one will be the upload + var metric = tentacleClientObserver.RpcCallMetrics.Last(); + metric.RpcCall.Name.Should().Be(nameof(IFileTransferService.DownloadFile)); + metric.RpcCall.Service.Should().Be(nameof(IFileTransferService)); + metric.Succeeded.Should().BeFalse(); + } + + private static void ThenClientOperationMetricsShouldBeSuccessful(ClientOperationMetrics metric) + { + metric.Succeeded.Should().BeTrue(); + metric.Exception.Should().BeNull(); + metric.WasCancelled.Should().BeFalse(); + + metric.End.Should().BeOnOrAfter(metric.Start); + metric.Duration.Should().Be(metric.End - metric.Start); + } + + private static void ThenClientOperationMetricsShouldBeFailed(ClientOperationMetrics metric, Exception expectedException) + { + metric.Succeeded.Should().BeFalse(); + metric.Exception.Should().BeEquivalentTo(expectedException); + metric.WasCancelled.Should().BeFalse(); + + metric.End.Should().BeOnOrAfter(metric.Start); + metric.Duration.Should().Be(metric.End - metric.Start); + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreDisabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreDisabled.cs index 472a9893a..b72258302 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreDisabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreDisabled.cs @@ -1,434 +1,436 @@ -// using System; -// using System.Collections.Generic; -// using System.Diagnostics; -// using System.Threading; -// using System.Threading.Tasks; -// using FluentAssertions; -// using Halibut; -// using NUnit.Framework; -// using Octopus.Tentacle.Client.Scripts; -// using Octopus.Tentacle.Client.Scripts.Models; -// using Octopus.Tentacle.CommonTestUtils.Builders; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Contracts.Capabilities; -// using Octopus.Tentacle.Contracts.ClientServices; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators.Proxies; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Util.PendingRequestQueueHelpers; -// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; -// using Octopus.Tentacle.Util; -// using Octopus.TestPortForwarder; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// /// -// /// These tests make sure that we can cancel the ExecuteScript operation when using Tentacle Client with RPC retries disabled. -// /// -// [IntegrationTestTimeout] -// public class ClientScriptExecutionCanBeCancelledWhenRetriesAreDisabled : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] -// public async Task DuringGetCapabilities_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) -// { -// // ARRANGE -// var rpcCallHasStarted = new Reference(false); -// var hasPausedOrStoppedPortForwarder = false; -// var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue -// .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) -// .WithRetriesDisabled() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithPortForwarder(out var portForwarder) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var capabilitiesMethodUsages) -// .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) -// .DecorateCapabilitiesServiceV2With(d => d -// .BeforeGetCapabilities( -// async () => -// { -// if (!hasPausedOrStoppedPortForwarder) -// { -// hasPausedOrStoppedPortForwarder = true; -// await tcpConnectionUtilities.RestartTcpConnection(); -// -// PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); -// } -// } -// -// ensureCancellationOccursDuringAnRpcCall.Release(); -// }) -// .AfterGetCapabilities( -// async _ => -// { -// await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); -// })) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Print("Should not run this script") -// .Sleep(TimeSpan.FromHours(1))) -// .Build(); -// -// // ACT -// var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); -// -// // ASSERT -// // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring -// capabilitiesMethodUsages.ForGetCapabilitiesAsync().LastException.Should().BeRequestCancelledException(rpcCallStage); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// actualException.ShouldMatchExceptionContract(expectedException); -// -// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(15)); -// -// capabilitiesMethodUsages.ForGetCapabilitiesAsync().Started.Should().Be(1); -// scriptMethodUsages.ForStartScriptAsync().Started.Should().Be(0, "Should not have proceeded past GetCapabilities"); -// scriptMethodUsages.ForCancelScriptAsync().Started.Should().Be(0, "Should not have tried to call CancelScript"); -// } -// -// [Test] -// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] -// public async Task DuringStartScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) -// { -// // ARRANGE -// var rpcCallHasStarted = new Reference(false); -// var restartedPortForwarderForCancel = false; -// var hasPausedOrStoppedPortForwarder = false; -// var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue -// .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) -// .WithRetriesDisabled() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithPortForwarder(out var portForwarder) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeStartScript( -// async () => -// { -// if (!hasPausedOrStoppedPortForwarder) -// { -// hasPausedOrStoppedPortForwarder = true; -// await tcpConnectionUtilities.RestartTcpConnection(); -// PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); -// } -// } -// -// ensureCancellationOccursDuringAnRpcCall.Release(); -// }) -// .BeforeCancelScript( -// async () => -// { -// await Task.CompletedTask; -// -// if (!restartedPortForwarderForCancel) -// { -// restartedPortForwarderForCancel = true; -// UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Print("The script") -// .Sleep(TimeSpan.FromHours(1))) -// .Build(); -// -// // ACT -// var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); -// -// // ASSERT -// // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring -// recordedUsages.ForStartScriptAsync().LastException.Should().BeRequestCancelledException(rpcCallStage); -// -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// // Should have cancelled the RPC call and exited immediately -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// -// actualException.ShouldMatchExceptionContract(expectedException); -// -// // We should have cancelled the RPC call quickly and existed -// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(15)); -// -// recordedUsages.ForStartScriptAsync().Started.Should().Be(1); -// recordedUsages.ForGetStatusAsync().Started.Should().Be(0, "Test should not have not proceeded past StartScript before being Cancelled"); -// recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); -// recordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); -// } -// else // Transferring -// { -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// actualException.ShouldMatchExceptionContract(expectedException); -// -// // Assert the CancelScript and CompleteScript flow happened fairly quickly -// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); -// -// recordedUsages.ForStartScriptAsync().Started.Should().Be(1); -// recordedUsages.ForGetStatusAsync().Started.Should().Be(0, "Test should not have not proceeded past StartScript before being Cancelled"); -// recordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(1); -// recordedUsages.ForCompleteScriptAsync().Started.Should().Be(1); -// } -// } -// -// [Test] -// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] -// public async Task DuringGetStatus_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) -// { -// // ARRANGE -// var rpcCallHasStarted = new Reference(false); -// var restartedPortForwarderForCancel = false; -// var hasPausedOrStoppedPortForwarder = false; -// var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue -// .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) -// .WithRetriesDisabled() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithPortForwarder(out var portForwarder) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeGetStatus( -// async () => -// { -// if (!hasPausedOrStoppedPortForwarder) -// { -// hasPausedOrStoppedPortForwarder = true; -// await tcpConnectionUtilities.RestartTcpConnection(); -// PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); -// } -// } -// -// ensureCancellationOccursDuringAnRpcCall.Release(); -// }) -// .AfterGetStatus( -// async () => -// { -// await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); -// }) -// .BeforeCancelScript( -// async () => -// { -// await Task.CompletedTask; -// -// if (!restartedPortForwarderForCancel) -// { -// restartedPortForwarderForCancel = true; -// UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Print("The script") -// .Sleep(TimeSpan.FromHours(1))) -// .Build(); -// -// // ACT -// var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); -// -// // ASSERT -// // Assert that script execution was cancelled -// actualException.Should().BeScriptExecutionCancelledException(); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// actualException.ShouldMatchExceptionContract(expectedException); -// -// // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring -// recordedUsages.ForGetStatusAsync().LastException.Should().BeRequestCancelledException(rpcCallStage); -// -// // Assert the CancelScript and CompleteScript flow happened fairly quickly -// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); -// -// // The expected RPC calls were made -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().BeGreaterOrEqualTo(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); -// } -// -// [Test] -// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] -// public async Task DuringCompleteScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) -// { -// // ARRANGE -// var rpcCallHasStarted = new Reference(false); -// var hasPausedOrStoppedPortForwarder = false; -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue -// .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) -// .WithRetriesDisabled() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithPortForwarder(out var portForwarder) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeCompleteScript( -// async () => -// { -// if (!hasPausedOrStoppedPortForwarder) -// { -// hasPausedOrStoppedPortForwarder = true; -// await tcpConnectionUtilities.RestartTcpConnection(); -// PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); -// } -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// clientAndTentacle.TentacleClient.OnCancellationAbandonCompleteScriptAfter = TimeSpan.FromSeconds(20); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Print("The script") -// .Sleep(TimeSpan.FromSeconds(5))) -// .Build(); -// -// // ACT -// var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, new SemaphoreSlim(int.MaxValue, int.MaxValue)); -// -// // ASSERT -// // The actual exception may be null if the script completed before the cancellation was observed -// if (actualException != null) -// { -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// actualException.ShouldMatchExceptionContract(expectedException); -// } -// -// // Halibut Errors were recorded on CompleteScript -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).LastException?.Should().Match(x => x is HalibutClientException || x is OperationCanceledException || x is TaskCanceledException); // Complete Script was cancelled quickly -// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); -// -// // The expected RPC calls were made -// recordedUsages.ForStartScriptAsync().Started.Should().Be(1); -// recordedUsages.ForGetStatusAsync().Started.Should().BeGreaterThanOrEqualTo(1); -// recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); -// recordedUsages.ForCompleteScriptAsync().Started.Should().BeGreaterOrEqualTo(1); -// } -// -// void PauseOrStopPortForwarder(RpcCallStage rpcCallStage, PortForwarder portForwarder, IResponseMessageTcpKiller responseMessageTcpKiller, Reference rpcCallHasStarted) -// { -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// Logger.Information("Killing the port forwarder so the next RPCs are in the connecting state when being cancelled"); -// //portForwarder.Stop(); -// portForwarder.EnterKillNewAndExistingConnectionsMode(); -// rpcCallHasStarted.Value = true; -// } -// else -// { -// Logger.Information("Will Pause the port forwarder on next response so the next RPC is in-flight when being cancelled"); -// responseMessageTcpKiller.PauseConnectionOnNextResponse(() => rpcCallHasStarted.Value = true); -// } -// } -// -// void UnPauseOrRestartPortForwarder(TentacleType tentacleType, RpcCallStage rpcCallStage, Reference portForwarder) -// { -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// Logger.Information("Starting the PortForwarder as we stopped it to get the StartScript RPC call in the Connecting state"); -// //portForwarder.Value.Start(); -// portForwarder.Value.ReturnToNormalMode(); -// } -// else if (tentacleType == TentacleType.Polling) -// { -// Logger.Information("UnPausing the PortForwarder as we paused the connections which means Polling will be stalled"); -// portForwarder.Value.UnPauseExistingConnections(); -// portForwarder.Value.CloseExistingConnections(); -// } -// } -// -// async Task<(ScriptExecutionResult response, Exception? actualException, TimeSpan cancellationDuration)> ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted( -// ClientAndTentacle clientAndTentacle, -// ExecuteScriptCommand executeScriptCommand, -// Reference rpcCallHasStarted, -// SemaphoreSlim whenTheRequestCanBeCancelled) -// { -// Logger.Information("Start of ExecuteScriptThenCancel"); -// var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); -// -// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( -// executeScriptCommand, -// cancelExecutionCancellationTokenSource.Token); -// -// Logger.Information("Create action"); -// Func)>> action = async () => await executeScriptTask; -// -// Logger.Information("Waiting for the RPC Call to start"); -// await Wait.For( -// () => rpcCallHasStarted.Value, -// TimeSpan.FromSeconds(30), -// () => throw new Exception("RPC call did not start") -// ,CancellationToken); -// Logger.Information("RPC Call has start"); -// -// await Task.Delay(TimeSpan.FromSeconds(6), CancellationToken); -// -// var cancellationDuration = new Stopwatch(); -// await whenTheRequestCanBeCancelled.WithLockAsync(() => -// { -// Logger.Information("Cancelling ExecuteScript"); -// cancelExecutionCancellationTokenSource.Cancel(); -// cancellationDuration.Start(); -// }, CancellationToken); -// -// Exception? actualException = null; -// (ScriptExecutionResult Response, List Logs)? responseAndLogs = null; -// try -// { -// responseAndLogs = await action(); -// } -// catch (Exception ex) -// { -// actualException = ex; -// } -// -// cancellationDuration.Stop(); -// return (responseAndLogs?.Response, actualException, cancellationDuration.Elapsed); -// } -// } -// } \ No newline at end of file +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Halibut; +using NUnit.Framework; +using Octopus.Tentacle.Client.Scripts; +using Octopus.Tentacle.Client.Scripts.Models; +using Octopus.Tentacle.CommonTestUtils.Builders; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.Capabilities; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators.Proxies; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Util.PendingRequestQueueHelpers; +using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; +using Octopus.Tentacle.Util; +using Octopus.TestPortForwarder; + +namespace Octopus.Tentacle.Tests.Integration +{ + /// + /// These tests make sure that we can cancel the ExecuteScript operation when using Tentacle Client with RPC retries disabled. + /// + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] + public class ClientScriptExecutionCanBeCancelledWhenRetriesAreDisabled : IntegrationTest + { + [Test] + [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] + public async Task DuringGetCapabilities_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) + { + // ARRANGE + var rpcCallHasStarted = new Reference(false); + var hasPausedOrStoppedPortForwarder = false; + var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue + .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) + .WithRetriesDisabled() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithPortForwarder(out var portForwarder) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var capabilitiesMethodUsages) + .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) + .DecorateCapabilitiesServiceV2With(d => d + .BeforeGetCapabilities( + async () => + { + if (!hasPausedOrStoppedPortForwarder) + { + hasPausedOrStoppedPortForwarder = true; + await tcpConnectionUtilities.RestartTcpConnection(); + + PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); + if (rpcCallStage == RpcCallStage.Connecting) + { + await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); + } + } + + ensureCancellationOccursDuringAnRpcCall.Release(); + }) + .AfterGetCapabilities( + async _ => + { + await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); + })) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Print("Should not run this script") + .Sleep(TimeSpan.FromHours(1))) + .Build(); + + // ACT + var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); + + // ASSERT + // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring + capabilitiesMethodUsages.ForGetCapabilitiesAsync().LastException.Should().BeRequestCancelledException(rpcCallStage); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + actualException.ShouldMatchExceptionContract(expectedException); + + cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(15)); + + capabilitiesMethodUsages.ForGetCapabilitiesAsync().Started.Should().Be(1); + scriptMethodUsages.ForStartScriptAsync().Started.Should().Be(0, "Should not have proceeded past GetCapabilities"); + scriptMethodUsages.ForCancelScriptAsync().Started.Should().Be(0, "Should not have tried to call CancelScript"); + } + + [Test] + [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] + public async Task DuringStartScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) + { + // ARRANGE + var rpcCallHasStarted = new Reference(false); + var restartedPortForwarderForCancel = false; + var hasPausedOrStoppedPortForwarder = false; + var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue + .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) + .WithRetriesDisabled() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithPortForwarder(out var portForwarder) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeStartScript( + async () => + { + if (!hasPausedOrStoppedPortForwarder) + { + hasPausedOrStoppedPortForwarder = true; + await tcpConnectionUtilities.RestartTcpConnection(); + PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); + if (rpcCallStage == RpcCallStage.Connecting) + { + await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); + } + } + + ensureCancellationOccursDuringAnRpcCall.Release(); + }) + .BeforeCancelScript( + async () => + { + await Task.CompletedTask; + + if (!restartedPortForwarderForCancel) + { + restartedPortForwarderForCancel = true; + UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); + } + })) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Print("The script") + .Sleep(TimeSpan.FromHours(1))) + .Build(); + + // ACT + var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); + + // ASSERT + // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring + recordedUsages.ForStartScriptAsync().LastException.Should().BeRequestCancelledException(rpcCallStage); + + if (rpcCallStage == RpcCallStage.Connecting) + { + // Should have cancelled the RPC call and exited immediately + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + + actualException.ShouldMatchExceptionContract(expectedException); + + // We should have cancelled the RPC call quickly and existed + cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(15)); + + recordedUsages.ForStartScriptAsync().Started.Should().Be(1); + recordedUsages.ForGetStatusAsync().Started.Should().Be(0, "Test should not have not proceeded past StartScript before being Cancelled"); + recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); + recordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); + } + else // Transferring + { + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + actualException.ShouldMatchExceptionContract(expectedException); + + // Assert the CancelScript and CompleteScript flow happened fairly quickly + cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); + + recordedUsages.ForStartScriptAsync().Started.Should().Be(1); + recordedUsages.ForGetStatusAsync().Started.Should().Be(0, "Test should not have not proceeded past StartScript before being Cancelled"); + recordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(1); + recordedUsages.ForCompleteScriptAsync().Started.Should().Be(1); + } + } + + [Test] + [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] + public async Task DuringGetStatus_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) + { + // ARRANGE + var rpcCallHasStarted = new Reference(false); + var restartedPortForwarderForCancel = false; + var hasPausedOrStoppedPortForwarder = false; + var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue + .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) + .WithRetriesDisabled() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithPortForwarder(out var portForwarder) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeGetStatus( + async () => + { + if (!hasPausedOrStoppedPortForwarder) + { + hasPausedOrStoppedPortForwarder = true; + await tcpConnectionUtilities.RestartTcpConnection(); + PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); + if (rpcCallStage == RpcCallStage.Connecting) + { + await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); + } + } + + ensureCancellationOccursDuringAnRpcCall.Release(); + }) + .AfterGetStatus( + async () => + { + await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); + }) + .BeforeCancelScript( + async () => + { + await Task.CompletedTask; + + if (!restartedPortForwarderForCancel) + { + restartedPortForwarderForCancel = true; + UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); + } + })) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Print("The script") + .Sleep(TimeSpan.FromHours(1))) + .Build(); + + // ACT + var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); + + // ASSERT + // Assert that script execution was cancelled + actualException.Should().BeScriptExecutionCancelledException(); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + actualException.ShouldMatchExceptionContract(expectedException); + + // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring + recordedUsages.ForGetStatusAsync().LastException.Should().BeRequestCancelledException(rpcCallStage); + + // Assert the CancelScript and CompleteScript flow happened fairly quickly + cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); + + // The expected RPC calls were made + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().BeGreaterOrEqualTo(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); + } + + [Test] + [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] + public async Task DuringCompleteScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) + { + // ARRANGE + var rpcCallHasStarted = new Reference(false); + var hasPausedOrStoppedPortForwarder = false; + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue + .WithServiceEndpointModifier(point => point.TryAndConnectForALongTime()) + .WithRetriesDisabled() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithPortForwarder(out var portForwarder) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeCompleteScript( + async () => + { + if (!hasPausedOrStoppedPortForwarder) + { + hasPausedOrStoppedPortForwarder = true; + await tcpConnectionUtilities.RestartTcpConnection(); + PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); + if (rpcCallStage == RpcCallStage.Connecting) + { + await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); + } + } + })) + .Build()) + .Build(CancellationToken); + + clientAndTentacle.TentacleClient.OnCancellationAbandonCompleteScriptAfter = TimeSpan.FromSeconds(20); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Print("The script") + .Sleep(TimeSpan.FromSeconds(5))) + .Build(); + + // ACT + var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, new SemaphoreSlim(int.MaxValue, int.MaxValue)); + + // ASSERT + // The actual exception may be null if the script completed before the cancellation was observed + if (actualException != null) + { + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + actualException.ShouldMatchExceptionContract(expectedException); + } + + // Halibut Errors were recorded on CompleteScript + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).LastException?.Should().Match(x => x is HalibutClientException || x is OperationCanceledException || x is TaskCanceledException); // Complete Script was cancelled quickly + cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); + + // The expected RPC calls were made + recordedUsages.ForStartScriptAsync().Started.Should().Be(1); + recordedUsages.ForGetStatusAsync().Started.Should().BeGreaterThanOrEqualTo(1); + recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); + recordedUsages.ForCompleteScriptAsync().Started.Should().BeGreaterOrEqualTo(1); + } + + void PauseOrStopPortForwarder(RpcCallStage rpcCallStage, PortForwarder portForwarder, IResponseMessageTcpKiller responseMessageTcpKiller, Reference rpcCallHasStarted) + { + if (rpcCallStage == RpcCallStage.Connecting) + { + Logger.Information("Killing the port forwarder so the next RPCs are in the connecting state when being cancelled"); + //portForwarder.Stop(); + portForwarder.EnterKillNewAndExistingConnectionsMode(); + rpcCallHasStarted.Value = true; + } + else + { + Logger.Information("Will Pause the port forwarder on next response so the next RPC is in-flight when being cancelled"); + responseMessageTcpKiller.PauseConnectionOnNextResponse(() => rpcCallHasStarted.Value = true); + } + } + + void UnPauseOrRestartPortForwarder(TentacleType tentacleType, RpcCallStage rpcCallStage, Reference portForwarder) + { + if (rpcCallStage == RpcCallStage.Connecting) + { + Logger.Information("Starting the PortForwarder as we stopped it to get the StartScript RPC call in the Connecting state"); + //portForwarder.Value.Start(); + portForwarder.Value.ReturnToNormalMode(); + } + else if (tentacleType == TentacleType.Polling) + { + Logger.Information("UnPausing the PortForwarder as we paused the connections which means Polling will be stalled"); + portForwarder.Value.UnPauseExistingConnections(); + portForwarder.Value.CloseExistingConnections(); + } + } + + async Task<(ScriptExecutionResult response, Exception? actualException, TimeSpan cancellationDuration)> ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted( + ClientAndTentacle clientAndTentacle, + ExecuteScriptCommand executeScriptCommand, + Reference rpcCallHasStarted, + SemaphoreSlim whenTheRequestCanBeCancelled) + { + Logger.Information("Start of ExecuteScriptThenCancel"); + var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); + + var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( + executeScriptCommand, + cancelExecutionCancellationTokenSource.Token); + + Logger.Information("Create action"); + Func)>> action = async () => await executeScriptTask; + + Logger.Information("Waiting for the RPC Call to start"); + await Wait.For( + () => rpcCallHasStarted.Value, + TimeSpan.FromSeconds(30), + () => throw new Exception("RPC call did not start") + ,CancellationToken); + Logger.Information("RPC Call has start"); + + await Task.Delay(TimeSpan.FromSeconds(6), CancellationToken); + + var cancellationDuration = new Stopwatch(); + await whenTheRequestCanBeCancelled.WithLockAsync(() => + { + Logger.Information("Cancelling ExecuteScript"); + cancelExecutionCancellationTokenSource.Cancel(); + cancellationDuration.Start(); + }, CancellationToken); + + Exception? actualException = null; + (ScriptExecutionResult Response, List Logs)? responseAndLogs = null; + try + { + responseAndLogs = await action(); + } + catch (Exception ex) + { + actualException = ex; + } + + cancellationDuration.Stop(); + return (responseAndLogs?.Response, actualException, cancellationDuration.Elapsed); + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreEnabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreEnabled.cs index 4d855bda2..2483b92eb 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreEnabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanBeCancelledWhenRetriesAreEnabled.cs @@ -1,672 +1,674 @@ -// using System; -// using System.Collections.Generic; -// using System.Diagnostics; -// using System.Threading; -// using System.Threading.Tasks; -// using FluentAssertions; -// using Halibut; -// using Halibut.Diagnostics; -// using NUnit.Framework; -// using Octopus.Tentacle.Client.Scripts; -// using Octopus.Tentacle.Client.Scripts.Models; -// using Octopus.Tentacle.CommonTestUtils.Builders; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Contracts.Capabilities; -// using Octopus.Tentacle.Contracts.ClientServices; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators.Proxies; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; -// using Octopus.Tentacle.Tests.Integration.Support.PendingRequestQueueFactories; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Util.PendingRequestQueueHelpers; -// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; -// using Octopus.Tentacle.Util; -// using Octopus.TestPortForwarder; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// /// -// /// These tests make sure that we can cancel the ExecuteScript operation when using Tentacle Client with RPC retries enabled. -// /// -// [IntegrationTestTimeout] -// public class ClientScriptExecutionCanBeCancelledWhenRetriesAreEnabled : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage), typeof(RpcCall) })] -// public async Task DuringGetCapabilities_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage, RpcCall rpcCall) -// { -// // ARRANGE -// var rpcCallHasStarted = new Reference(false); -// var hasPausedOrStoppedPortForwarder = false; -// var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue -// .WithServiceEndpointModifier(point => -// { -// if (rpcCall == RpcCall.FirstCall) point.TryAndConnectForALongTime(); -// }) -// .WithRetryDuration(TimeSpan.FromHours(1)) -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithPortForwarder(out var portForwarder) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var recordedUsages) -// .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) -// .DecorateCapabilitiesServiceV2With(d => d -// .BeforeGetCapabilities( -// async () => -// { -// if (rpcCall == RpcCall.RetryingCall && -// recordedUsages.ForGetCapabilitiesAsync().LastException == null) -// { -// await tcpConnectionUtilities.RestartTcpConnection(); -// -// // Kill the first GetCapabilities call to force the rpc call into retries -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// else if (!hasPausedOrStoppedPortForwarder) -// { -// hasPausedOrStoppedPortForwarder = true; -// await tcpConnectionUtilities.RestartTcpConnection(); -// -// await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); -// if (rpcCallStage == RpcCallStage.Connecting && tentacleConfigurationTestCase.TentacleType == TentacleType.Polling) -// { -// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); -// } -// } -// -// ensureCancellationOccursDuringAnRpcCall.Release(); -// }) -// .AfterGetCapabilities( -// async _ => -// { -// await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); -// })) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Print("Should not run this script") -// .Sleep(TimeSpan.FromHours(1))) -// .Build(); -// -// // ACT -// var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); -// -// // ASSERT -// // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring -// var latestException = recordedUsages.ForGetCapabilitiesAsync().LastException; -// if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) -// { -// Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + -// "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); -// } -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// actualException.ShouldMatchExceptionContract(expectedException); -// -// latestException.Should().BeRequestCancelledException(rpcCallStage); -// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(10), "The RPC call should have been cancelled quickly"); -// -// if (rpcCall == RpcCall.FirstCall) -// { -// recordedUsages.ForGetCapabilitiesAsync().Started.Should().Be(1); -// } -// else -// { -// recordedUsages.ForGetCapabilitiesAsync().Started.Should().BeGreaterOrEqualTo(2); -// } -// scriptMethodUsages.ForStartScriptAsync().Started.Should().Be(0, "Should not have proceeded past GetCapabilities"); -// scriptMethodUsages.ForCancelScriptAsync().Started.Should().Be(0, "Should not have tried to call CancelScript"); -// scriptMethodUsages.ForCompleteScriptAsync().Started.Should().Be(0, "Should not have tried to call CompleteScript"); -// } -// -// /// -// /// This test, and probably others in this test class do not correctly test the Connecting Scenario. The port forwarder is used to kill new and existing connections but this can -// /// result in Halibut thinking a Request is transferring, where we want it to be connecting. These tests need to be rewritten to ensure they test the correct scenario. -// /// -// [Test] -// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCall), typeof(RpcCallStage) })] -// public async Task DuringStartScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCall rpcCall, RpcCallStage rpcCallStage) -// { -// var rpcCallHasStarted = new Reference(false); -// var restartedPortForwarderForCancel = false; -// var hasPausedOrStoppedPortForwarder = false; -// var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue -// .WithServiceEndpointModifier(point => -// { -// if (rpcCall == RpcCall.FirstCall) point.TryAndConnectForALongTime(); -// }) -// .WithRetryDuration(TimeSpan.FromHours(1)) -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithPortForwarder(out var portForwarder) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeStartScript( -// async () => -// { -// if (rpcCall == RpcCall.RetryingCall && recordedUsages.ForStartScriptAsync().LastException is null) -// { -// await tcpConnectionUtilities.RestartTcpConnection(); -// // Kill the first StartScript call to force the rpc call into retries -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// else -// { -// if (!hasPausedOrStoppedPortForwarder) -// { -// hasPausedOrStoppedPortForwarder = true; -// await tcpConnectionUtilities.RestartTcpConnection(); -// await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); -// } -// } -// } -// -// ensureCancellationOccursDuringAnRpcCall.Release(); -// }) -// .AfterStartScript( -// async () => -// { -// await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); -// }) -// .BeforeCancelScript( -// async () => -// { -// await Task.CompletedTask; -// -// if (!restartedPortForwarderForCancel) -// { -// restartedPortForwarderForCancel = true; -// UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Print("The script") -// .Sleep(TimeSpan.FromHours(1))) -// .Build(); -// -// // ACT -// var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); -// -// // ASSERT -// // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring -// var latestException = recordedUsages.ForStartScriptAsync().LastException; -// if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) -// { -// Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + -// "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); -// } -// latestException.Should().BeRequestCancelledException(rpcCallStage); -// -// if (rpcCall == RpcCall.FirstCall && rpcCallStage == RpcCallStage.Connecting) -// { -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// actualException.ShouldMatchExceptionContract(expectedException); -// -// // We should have cancelled the RPC call quickly -// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(15)); -// -// recordedUsages.ForStartScriptAsync().Started.Should().Be(1); -// recordedUsages.ForGetStatusAsync().Started.Should().Be(0); -// recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); -// recordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); -// } -// else if((rpcCall == RpcCall.RetryingCall && rpcCallStage == RpcCallStage.Connecting) || rpcCallStage == RpcCallStage.InFlight) -// { -// // Assert that script execution was cancelled -// actualException.Should().BeScriptExecutionCancelledException(); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// actualException.ShouldMatchExceptionContract(expectedException); -// -// -// // Assert the CancelScript and CompleteScript flow happened fairly quickly -// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); -// -// if (rpcCall == RpcCall.FirstCall) -// { -// recordedUsages.ForStartScriptAsync().Started.Should().Be(1); -// } -// else -// { -// recordedUsages.ForStartScriptAsync().Started.Should().BeGreaterOrEqualTo(2); -// } -// recordedUsages.ForGetStatusAsync().Started.Should().Be(0); -// recordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(1); -// recordedUsages.ForCompleteScriptAsync().Started.Should().Be(1); -// } -// else -// { -// throw new ArgumentOutOfRangeException(); -// } -// } -// -// [Test] -// [TentacleConfigurations(testListening: false)] -// public async Task DuringStartScript_ForPollingTentacle_ThatIsRetryingTheRpc_AndConnecting_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var halibutTimeoutsAndLimits = HalibutTimeoutsAndLimits.RecommendedValues(); -// halibutTimeoutsAndLimits.PollingQueueWaitTimeout = TimeSpan.FromSeconds(4); -// halibutTimeoutsAndLimits.PollingRequestQueueTimeout = TimeSpan.FromSeconds(3); -// -// var started = false; -// var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithRetryDuration(TimeSpan.FromHours(1)) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder().RecordMethodUsages(tentacleConfigurationTestCase, out var scriptServiceRecordedUsages).Build()) -// .WithPendingRequestQueueFactory(new CancelWhenRequestQueuedPendingRequestQueueFactory( -// cancelExecutionCancellationTokenSource, -// halibutTimeoutsAndLimits, -// // Cancel the execution when the StartScript RPC call is being retries -// shouldCancel: async () => -// { -// await Task.CompletedTask; -// -// if (started && scriptServiceRecordedUsages.ForStartScriptAsync().Started >= 2) -// { -// Logger.Information("Cancelling Execute Script"); -// return true; -// } -// -// return false; -// })) -// .WithHalibutTimeoutsAndLimits(halibutTimeoutsAndLimits) -// .Build(CancellationToken); -// -// // Arrange -// Logger.Information("Execute a script so that GetCapabilities will be cached"); -// await clientAndTentacle.TentacleClient.ExecuteScript( -// new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("The script")).Build(), -// cancelExecutionCancellationTokenSource.Token); -// -// Logger.Information("Stop Tentacle so no more requests are picked up"); -// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); -// -// var delay = clientAndTentacle.Server.ServerHalibutRuntime.TimeoutsAndLimits.PollingQueueWaitTimeout + TimeSpan.FromSeconds(2); -// Logger.Information($"Waiting for {delay} for any active PendingRequestQueues for the Tentacle to drop the latest poll connection"); -// await Task.Delay(delay, CancellationToken); -// -// scriptServiceRecordedUsages.Reset(); -// started = true; -// -// // ACT -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b.Print("The script").Sleep(TimeSpan.FromHours(1))) -// .Build(); -// -// Logger.Information("Start Executing the Script"); -// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, cancelExecutionCancellationTokenSource.Token); -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// // Assert -// scriptServiceRecordedUsages.ForStartScriptAsync().Completed.Should().BeGreaterOrEqualTo(2); -// scriptServiceRecordedUsages.ForGetStatusAsync().Started.Should().Be(0); -// scriptServiceRecordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(0, "Script Execution does not need to be cancelled on Tentacle as it has not started"); -// scriptServiceRecordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); -// } -// -// [Test] -// [TentacleConfigurations(testPolling: false)] -// public async Task DuringStartScript_ForListeningTentacle_ThatIsRetryingTheRpc_AndConnecting_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var halibutTimeoutsAndLimits = HalibutTimeoutsAndLimits.RecommendedValues(); -// halibutTimeoutsAndLimits.RetryCountLimit = 1; -// halibutTimeoutsAndLimits.ConnectionErrorRetryTimeout = TimeSpan.FromSeconds(4); -// halibutTimeoutsAndLimits.RetryListeningSleepInterval = TimeSpan.Zero; -// halibutTimeoutsAndLimits.TcpClientConnectTimeout = TimeSpan.FromSeconds(4); -// halibutTimeoutsAndLimits.TcpClientPooledConnectionTimeout = TimeSpan.FromSeconds(5); -// -// var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithRetryDuration(TimeSpan.FromHours(1)) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder().RecordMethodUsages(tentacleConfigurationTestCase, out var scriptServiceRecordedUsages).Build()) -// .WithHalibutTimeoutsAndLimits(halibutTimeoutsAndLimits) -// .WithPortForwarder(out var portForwarder) -// .Build(CancellationToken); -// -// // Arrange -// Logger.Information("Execute a script so that GetCapabilities will be cached"); -// await clientAndTentacle.TentacleClient.ExecuteScript( -// new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("The script")).Build(), -// cancelExecutionCancellationTokenSource.Token); -// -// Logger.Information("Stop Tentacle so no more requests are picked up"); -// await clientAndTentacle.RunningTentacle.Stop(CancellationToken); -// portForwarder.Value.KillNewConnectionsImmediatlyMode = true; -// portForwarder.Value.CloseExistingConnections(); -// -// var delay = clientAndTentacle.Server.ServerHalibutRuntime.TimeoutsAndLimits.SafeTcpClientPooledConnectionTimeout + TimeSpan.FromSeconds(2); -// Logger.Information($"Waiting for {delay} for any active Pooled Connections for the Tentacle to expire"); -// await Task.Delay(delay, CancellationToken); -// -// scriptServiceRecordedUsages.Reset(); -// -// // ACT -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b.Print("The script").Sleep(TimeSpan.FromHours(1))) -// .Build(); -// -// Logger.Information("Start Executing the Script"); -// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, cancelExecutionCancellationTokenSource.Token); -// var cancellationTask = Task.Run(async () => -// { -// while (true) -// { -// if (scriptServiceRecordedUsages.ForStartScriptAsync().Started >= 2) -// { -// cancelExecutionCancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(2)); -// Logger.Information("Cancelling cancellation token source after 2 seconds."); -// return; -// } -// -// await Task.Delay(TimeSpan.FromSeconds(0.5), CancellationToken); -// } -// }, CancellationToken); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// await cancellationTask; -// -// // Assert -// scriptServiceRecordedUsages.ForStartScriptAsync().Completed.Should().BeGreaterOrEqualTo(2); -// scriptServiceRecordedUsages.ForGetStatusAsync().Started.Should().Be(0); -// scriptServiceRecordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(0, "Script Execution does not need to be cancelled on Tentacle as it has not started"); -// scriptServiceRecordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); -// } -// -// [Test] -// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCall), typeof(RpcCallStage) })] -// public async Task DuringGetStatus_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCall rpcCall, RpcCallStage rpcCallStage) -// { -// // ARRANGE -// var rpcCallHasStarted = new Reference(false); -// var restartedPortForwarderForCancel = false; -// var hasPausedOrStoppedPortForwarder = false; -// var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue -// .WithServiceEndpointModifier(point => -// { -// if (rpcCall == RpcCall.FirstCall) point.TryAndConnectForALongTime(); -// }) -// .WithRetryDuration(TimeSpan.FromHours(1)) -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithPortForwarder(out var portForwarder) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeGetStatus( -// async () => -// { -// if (rpcCall == RpcCall.RetryingCall && -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException is null) -// { -// await tcpConnectionUtilities.RestartTcpConnection(); -// // Kill the first StartScript call to force the rpc call into retries -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// else -// { -// if (!hasPausedOrStoppedPortForwarder) -// { -// hasPausedOrStoppedPortForwarder = true; -// await tcpConnectionUtilities.RestartTcpConnection(); -// await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); -// } -// } -// } -// -// ensureCancellationOccursDuringAnRpcCall.Release(); -// }) -// .AfterGetStatus( -// async () => -// { -// await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); -// }) -// .BeforeCancelScript( -// async () => -// { -// await Task.CompletedTask; -// -// if (!restartedPortForwarderForCancel) -// { -// restartedPortForwarderForCancel = true; -// UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Print("The script") -// .Sleep(TimeSpan.FromHours(1))) -// .Build(); -// -// // ACT -// var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); -// -// // ASSERT -// // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring -// var latestException = recordedUsages.ForGetStatusAsync().LastException; -// if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) -// { -// Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + -// "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); -// } -// -// latestException.Should().BeRequestCancelledException(rpcCallStage); -// -// // Assert that script execution was cancelled -// actualException.Should().BeScriptExecutionCancelledException(); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// actualException.ShouldMatchExceptionContract(expectedException); -// -// -// // Script Execution should cancel quickly -// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); -// -// recordedUsages.ForStartScriptAsync().Started.Should().Be(1); -// if (rpcCall == RpcCall.FirstCall) -// { -// recordedUsages.ForGetStatusAsync().Started.Should().Be(1); -// } -// else -// { -// recordedUsages.ForGetStatusAsync().Started.Should().BeGreaterOrEqualTo(2); -// } -// recordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(1); -// recordedUsages.ForCompleteScriptAsync().Started.Should().Be(1); -// } -// -// [Test] -// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] -// public async Task DuringCompleteScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) -// { -// // ARRANGE -// var rpcCallHasStarted = new Reference(false); -// var hasPausedOrStoppedPortForwarder = false; -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue -// .WithRetryDuration(TimeSpan.FromHours(1)) -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithPortForwarder(out var portForwarder) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeCompleteScript( -// async () => -// { -// if (!hasPausedOrStoppedPortForwarder) -// { -// hasPausedOrStoppedPortForwarder = true; -// await tcpConnectionUtilities.RestartTcpConnection(); -// await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); -// } -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// clientAndTentacle.TentacleClient.OnCancellationAbandonCompleteScriptAfter = TimeSpan.FromSeconds(20); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Print("The script") -// .Sleep(TimeSpan.FromSeconds(5))) -// .Build(); -// -// // ACT -// var (responseAndLogs, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, new SemaphoreSlim(int.MaxValue, int.MaxValue)); -// -// // ASSERT -// -// // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring -// var latestException = recordedUsages.ForCompleteScriptAsync().LastException; -// if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) -// { -// Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + -// "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); -// } -// -// // The actual exception may be null if the script completed before the cancellation was observed -// if (actualException != null) -// { -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// actualException.ShouldMatchExceptionContract(expectedException); -// } -// -// latestException.Should().BeRequestCancelledException(rpcCallStage); -// -// // Complete Script was cancelled quickly -// cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); -// -// recordedUsages.ForStartScriptAsync().Started.Should().Be(1); -// recordedUsages.ForGetStatusAsync().Started.Should().BeGreaterThanOrEqualTo(1); -// recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); -// recordedUsages.ForCompleteScriptAsync().Started.Should().BeGreaterOrEqualTo(1); -// } -// -// async Task PauseOrStopPortForwarder(RpcCallStage rpcCallStage, PortForwarder portForwarder, IResponseMessageTcpKiller responseMessageTcpKiller, Reference rpcCallHasStarted) -// { -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// Logger.Information("Killing the port forwarder so the next RPCs are in the connecting state when being cancelled"); -// portForwarder.EnterKillNewAndExistingConnectionsMode(); -// -// await SetRpcCallWeAreInterestedInAsStarted(rpcCallHasStarted); -// } -// else -// { -// Logger.Information("Will Pause the port forwarder on next response so the next RPC is in-flight when being cancelled"); -// #pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed -// responseMessageTcpKiller.PauseConnectionOnNextResponse(() => SetRpcCallWeAreInterestedInAsStarted(rpcCallHasStarted)); -// #pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed -// } -// -// static async Task SetRpcCallWeAreInterestedInAsStarted(Reference rpcCallHasStarted) -// { -// // Allow the port forwarder some time to stop -// await Task.Delay(TimeSpan.FromSeconds(5)); -// rpcCallHasStarted.Value = true; -// } -// } -// -// void UnPauseOrRestartPortForwarder(TentacleType tentacleType, RpcCallStage rpcCallStage, Reference portForwarder) -// { -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// Logger.Information("Starting the PortForwarder as we stopped it to get the StartScript RPC call in the Connecting state"); -// portForwarder.Value.ReturnToNormalMode(); -// } -// else if (tentacleType == TentacleType.Polling) -// { -// Logger.Information("UnPausing the PortForwarder as we paused the connections which means Polling will be stalled"); -// portForwarder.Value.UnPauseExistingConnections(); -// portForwarder.Value.CloseExistingConnections(); -// } -// } -// -// async Task<(ScriptExecutionResult response, Exception? actualException, TimeSpan cancellationDuration)> ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted( -// ClientAndTentacle clientAndTentacle, -// ExecuteScriptCommand executeScriptCommand, -// Reference rpcCallHasStarted, -// SemaphoreSlim whenTheRequestCanBeCancelled) -// { -// var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); -// -// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( -// executeScriptCommand, -// cancelExecutionCancellationTokenSource.Token); -// -// Logger.Information("Waiting for the RPC Call to start"); -// await Wait.For(() => rpcCallHasStarted.Value, -// TimeSpan.FromSeconds(30), -// () => throw new Exception("RPC call did not start"), -// CancellationToken); -// Logger.Information("RPC Call has start"); -// -// var cancellationDuration = new Stopwatch(); -// await whenTheRequestCanBeCancelled.WithLockAsync(async () => -// { -// await Task.Delay(TimeSpan.FromSeconds(6), CancellationToken); -// Logger.Information("Cancelling ExecuteScript"); -// cancelExecutionCancellationTokenSource.Cancel(); -// cancellationDuration.Start(); -// }, CancellationToken); -// -// Exception? actualException = null; -// (ScriptExecutionResult Response, List Logs)? responseAndLogs = null; -// try -// { -// responseAndLogs = await executeScriptTask; -// } -// catch (Exception ex) -// { -// actualException = ex; -// } -// -// cancellationDuration.Stop(); -// return (responseAndLogs?.Response, actualException, cancellationDuration.Elapsed); -// } -// } -// } \ No newline at end of file +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Halibut; +using Halibut.Diagnostics; +using NUnit.Framework; +using Octopus.Tentacle.Client.Scripts; +using Octopus.Tentacle.Client.Scripts.Models; +using Octopus.Tentacle.CommonTestUtils.Builders; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.Capabilities; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators.Proxies; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +using Octopus.Tentacle.Tests.Integration.Support.PendingRequestQueueFactories; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Util.PendingRequestQueueHelpers; +using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; +using Octopus.Tentacle.Util; +using Octopus.TestPortForwarder; + +namespace Octopus.Tentacle.Tests.Integration +{ + /// + /// These tests make sure that we can cancel the ExecuteScript operation when using Tentacle Client with RPC retries enabled. + /// + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] + public class ClientScriptExecutionCanBeCancelledWhenRetriesAreEnabled : IntegrationTest + { + [Test] + [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage), typeof(RpcCall) })] + public async Task DuringGetCapabilities_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage, RpcCall rpcCall) + { + // ARRANGE + var rpcCallHasStarted = new Reference(false); + var hasPausedOrStoppedPortForwarder = false; + var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue + .WithServiceEndpointModifier(point => + { + if (rpcCall == RpcCall.FirstCall) point.TryAndConnectForALongTime(); + }) + .WithRetryDuration(TimeSpan.FromHours(1)) + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithPortForwarder(out var portForwarder) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var recordedUsages) + .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) + .DecorateCapabilitiesServiceV2With(d => d + .BeforeGetCapabilities( + async () => + { + if (rpcCall == RpcCall.RetryingCall && + recordedUsages.ForGetCapabilitiesAsync().LastException == null) + { + await tcpConnectionUtilities.RestartTcpConnection(); + + // Kill the first GetCapabilities call to force the rpc call into retries + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + else if (!hasPausedOrStoppedPortForwarder) + { + hasPausedOrStoppedPortForwarder = true; + await tcpConnectionUtilities.RestartTcpConnection(); + + await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); + if (rpcCallStage == RpcCallStage.Connecting && tentacleConfigurationTestCase.TentacleType == TentacleType.Polling) + { + await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); + } + } + + ensureCancellationOccursDuringAnRpcCall.Release(); + }) + .AfterGetCapabilities( + async _ => + { + await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); + })) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Print("Should not run this script") + .Sleep(TimeSpan.FromHours(1))) + .Build(); + + // ACT + var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); + + // ASSERT + // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring + var latestException = recordedUsages.ForGetCapabilitiesAsync().LastException; + if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) + { + Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + + "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); + } + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + actualException.ShouldMatchExceptionContract(expectedException); + + latestException.Should().BeRequestCancelledException(rpcCallStage); + cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(10), "The RPC call should have been cancelled quickly"); + + if (rpcCall == RpcCall.FirstCall) + { + recordedUsages.ForGetCapabilitiesAsync().Started.Should().Be(1); + } + else + { + recordedUsages.ForGetCapabilitiesAsync().Started.Should().BeGreaterOrEqualTo(2); + } + scriptMethodUsages.ForStartScriptAsync().Started.Should().Be(0, "Should not have proceeded past GetCapabilities"); + scriptMethodUsages.ForCancelScriptAsync().Started.Should().Be(0, "Should not have tried to call CancelScript"); + scriptMethodUsages.ForCompleteScriptAsync().Started.Should().Be(0, "Should not have tried to call CompleteScript"); + } + + /// + /// This test, and probably others in this test class do not correctly test the Connecting Scenario. The port forwarder is used to kill new and existing connections but this can + /// result in Halibut thinking a Request is transferring, where we want it to be connecting. These tests need to be rewritten to ensure they test the correct scenario. + /// + [Test] + [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCall), typeof(RpcCallStage) })] + public async Task DuringStartScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCall rpcCall, RpcCallStage rpcCallStage) + { + var rpcCallHasStarted = new Reference(false); + var restartedPortForwarderForCancel = false; + var hasPausedOrStoppedPortForwarder = false; + var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue + .WithServiceEndpointModifier(point => + { + if (rpcCall == RpcCall.FirstCall) point.TryAndConnectForALongTime(); + }) + .WithRetryDuration(TimeSpan.FromHours(1)) + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithPortForwarder(out var portForwarder) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeStartScript( + async () => + { + if (rpcCall == RpcCall.RetryingCall && recordedUsages.ForStartScriptAsync().LastException is null) + { + await tcpConnectionUtilities.RestartTcpConnection(); + // Kill the first StartScript call to force the rpc call into retries + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + else + { + if (!hasPausedOrStoppedPortForwarder) + { + hasPausedOrStoppedPortForwarder = true; + await tcpConnectionUtilities.RestartTcpConnection(); + await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); + if (rpcCallStage == RpcCallStage.Connecting) + { + await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); + } + } + } + + ensureCancellationOccursDuringAnRpcCall.Release(); + }) + .AfterStartScript( + async () => + { + await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); + }) + .BeforeCancelScript( + async () => + { + await Task.CompletedTask; + + if (!restartedPortForwarderForCancel) + { + restartedPortForwarderForCancel = true; + UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); + } + })) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Print("The script") + .Sleep(TimeSpan.FromHours(1))) + .Build(); + + // ACT + var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); + + // ASSERT + // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring + var latestException = recordedUsages.ForStartScriptAsync().LastException; + if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) + { + Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + + "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); + } + latestException.Should().BeRequestCancelledException(rpcCallStage); + + if (rpcCall == RpcCall.FirstCall && rpcCallStage == RpcCallStage.Connecting) + { + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + actualException.ShouldMatchExceptionContract(expectedException); + + // We should have cancelled the RPC call quickly + cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(15)); + + recordedUsages.ForStartScriptAsync().Started.Should().Be(1); + recordedUsages.ForGetStatusAsync().Started.Should().Be(0); + recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); + recordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); + } + else if((rpcCall == RpcCall.RetryingCall && rpcCallStage == RpcCallStage.Connecting) || rpcCallStage == RpcCallStage.InFlight) + { + // Assert that script execution was cancelled + actualException.Should().BeScriptExecutionCancelledException(); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + actualException.ShouldMatchExceptionContract(expectedException); + + + // Assert the CancelScript and CompleteScript flow happened fairly quickly + cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); + + if (rpcCall == RpcCall.FirstCall) + { + recordedUsages.ForStartScriptAsync().Started.Should().Be(1); + } + else + { + recordedUsages.ForStartScriptAsync().Started.Should().BeGreaterOrEqualTo(2); + } + recordedUsages.ForGetStatusAsync().Started.Should().Be(0); + recordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(1); + recordedUsages.ForCompleteScriptAsync().Started.Should().Be(1); + } + else + { + throw new ArgumentOutOfRangeException(); + } + } + + [Test] + [TentacleConfigurations(testListening: false)] + public async Task DuringStartScript_ForPollingTentacle_ThatIsRetryingTheRpc_AndConnecting_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var halibutTimeoutsAndLimits = HalibutTimeoutsAndLimits.RecommendedValues(); + halibutTimeoutsAndLimits.PollingQueueWaitTimeout = TimeSpan.FromSeconds(4); + halibutTimeoutsAndLimits.PollingRequestQueueTimeout = TimeSpan.FromSeconds(3); + + var started = false; + var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithRetryDuration(TimeSpan.FromHours(1)) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder().RecordMethodUsages(tentacleConfigurationTestCase, out var scriptServiceRecordedUsages).Build()) + .WithPendingRequestQueueFactory(new CancelWhenRequestQueuedPendingRequestQueueFactory( + cancelExecutionCancellationTokenSource, + halibutTimeoutsAndLimits, + // Cancel the execution when the StartScript RPC call is being retries + shouldCancel: async () => + { + await Task.CompletedTask; + + if (started && scriptServiceRecordedUsages.ForStartScriptAsync().Started >= 2) + { + Logger.Information("Cancelling Execute Script"); + return true; + } + + return false; + })) + .WithHalibutTimeoutsAndLimits(halibutTimeoutsAndLimits) + .Build(CancellationToken); + + // Arrange + Logger.Information("Execute a script so that GetCapabilities will be cached"); + await clientAndTentacle.TentacleClient.ExecuteScript( + new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("The script")).Build(), + cancelExecutionCancellationTokenSource.Token); + + Logger.Information("Stop Tentacle so no more requests are picked up"); + await clientAndTentacle.RunningTentacle.Stop(CancellationToken); + + var delay = clientAndTentacle.Server.ServerHalibutRuntime.TimeoutsAndLimits.PollingQueueWaitTimeout + TimeSpan.FromSeconds(2); + Logger.Information($"Waiting for {delay} for any active PendingRequestQueues for the Tentacle to drop the latest poll connection"); + await Task.Delay(delay, CancellationToken); + + scriptServiceRecordedUsages.Reset(); + started = true; + + // ACT + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b.Print("The script").Sleep(TimeSpan.FromHours(1))) + .Build(); + + Logger.Information("Start Executing the Script"); + var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, cancelExecutionCancellationTokenSource.Token); + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + // Assert + scriptServiceRecordedUsages.ForStartScriptAsync().Completed.Should().BeGreaterOrEqualTo(2); + scriptServiceRecordedUsages.ForGetStatusAsync().Started.Should().Be(0); + scriptServiceRecordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(0, "Script Execution does not need to be cancelled on Tentacle as it has not started"); + scriptServiceRecordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); + } + + [Test] + [TentacleConfigurations(testPolling: false)] + public async Task DuringStartScript_ForListeningTentacle_ThatIsRetryingTheRpc_AndConnecting_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var halibutTimeoutsAndLimits = HalibutTimeoutsAndLimits.RecommendedValues(); + halibutTimeoutsAndLimits.RetryCountLimit = 1; + halibutTimeoutsAndLimits.ConnectionErrorRetryTimeout = TimeSpan.FromSeconds(4); + halibutTimeoutsAndLimits.RetryListeningSleepInterval = TimeSpan.Zero; + halibutTimeoutsAndLimits.TcpClientConnectTimeout = TimeSpan.FromSeconds(4); + halibutTimeoutsAndLimits.TcpClientPooledConnectionTimeout = TimeSpan.FromSeconds(5); + + var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithRetryDuration(TimeSpan.FromHours(1)) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder().RecordMethodUsages(tentacleConfigurationTestCase, out var scriptServiceRecordedUsages).Build()) + .WithHalibutTimeoutsAndLimits(halibutTimeoutsAndLimits) + .WithPortForwarder(out var portForwarder) + .Build(CancellationToken); + + // Arrange + Logger.Information("Execute a script so that GetCapabilities will be cached"); + await clientAndTentacle.TentacleClient.ExecuteScript( + new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("The script")).Build(), + cancelExecutionCancellationTokenSource.Token); + + Logger.Information("Stop Tentacle so no more requests are picked up"); + await clientAndTentacle.RunningTentacle.Stop(CancellationToken); + portForwarder.Value.KillNewConnectionsImmediatlyMode = true; + portForwarder.Value.CloseExistingConnections(); + + var delay = clientAndTentacle.Server.ServerHalibutRuntime.TimeoutsAndLimits.SafeTcpClientPooledConnectionTimeout + TimeSpan.FromSeconds(2); + Logger.Information($"Waiting for {delay} for any active Pooled Connections for the Tentacle to expire"); + await Task.Delay(delay, CancellationToken); + + scriptServiceRecordedUsages.Reset(); + + // ACT + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b.Print("The script").Sleep(TimeSpan.FromHours(1))) + .Build(); + + Logger.Information("Start Executing the Script"); + var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, cancelExecutionCancellationTokenSource.Token); + var cancellationTask = Task.Run(async () => + { + while (true) + { + if (scriptServiceRecordedUsages.ForStartScriptAsync().Started >= 2) + { + cancelExecutionCancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(2)); + Logger.Information("Cancelling cancellation token source after 2 seconds."); + return; + } + + await Task.Delay(TimeSpan.FromSeconds(0.5), CancellationToken); + } + }, CancellationToken); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + await cancellationTask; + + // Assert + scriptServiceRecordedUsages.ForStartScriptAsync().Completed.Should().BeGreaterOrEqualTo(2); + scriptServiceRecordedUsages.ForGetStatusAsync().Started.Should().Be(0); + scriptServiceRecordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(0, "Script Execution does not need to be cancelled on Tentacle as it has not started"); + scriptServiceRecordedUsages.ForCompleteScriptAsync().Started.Should().Be(0); + } + + [Test] + [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCall), typeof(RpcCallStage) })] + public async Task DuringGetStatus_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCall rpcCall, RpcCallStage rpcCallStage) + { + // ARRANGE + var rpcCallHasStarted = new Reference(false); + var restartedPortForwarderForCancel = false; + var hasPausedOrStoppedPortForwarder = false; + var ensureCancellationOccursDuringAnRpcCall = new SemaphoreSlim(0, 1); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue + .WithServiceEndpointModifier(point => + { + if (rpcCall == RpcCall.FirstCall) point.TryAndConnectForALongTime(); + }) + .WithRetryDuration(TimeSpan.FromHours(1)) + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithPortForwarder(out var portForwarder) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeGetStatus( + async () => + { + if (rpcCall == RpcCall.RetryingCall && + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException is null) + { + await tcpConnectionUtilities.RestartTcpConnection(); + // Kill the first StartScript call to force the rpc call into retries + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + else + { + if (!hasPausedOrStoppedPortForwarder) + { + hasPausedOrStoppedPortForwarder = true; + await tcpConnectionUtilities.RestartTcpConnection(); + await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); + if (rpcCallStage == RpcCallStage.Connecting) + { + await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); + } + } + } + + ensureCancellationOccursDuringAnRpcCall.Release(); + }) + .AfterGetStatus( + async () => + { + await ensureCancellationOccursDuringAnRpcCall.WaitAsync(CancellationToken); + }) + .BeforeCancelScript( + async () => + { + await Task.CompletedTask; + + if (!restartedPortForwarderForCancel) + { + restartedPortForwarderForCancel = true; + UnPauseOrRestartPortForwarder(tentacleConfigurationTestCase.TentacleType, rpcCallStage, portForwarder); + } + })) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Print("The script") + .Sleep(TimeSpan.FromHours(1))) + .Build(); + + // ACT + var (_, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, ensureCancellationOccursDuringAnRpcCall); + + // ASSERT + // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring + var latestException = recordedUsages.ForGetStatusAsync().LastException; + if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) + { + Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + + "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); + } + + latestException.Should().BeRequestCancelledException(rpcCallStage); + + // Assert that script execution was cancelled + actualException.Should().BeScriptExecutionCancelledException(); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + actualException.ShouldMatchExceptionContract(expectedException); + + + // Script Execution should cancel quickly + cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); + + recordedUsages.ForStartScriptAsync().Started.Should().Be(1); + if (rpcCall == RpcCall.FirstCall) + { + recordedUsages.ForGetStatusAsync().Started.Should().Be(1); + } + else + { + recordedUsages.ForGetStatusAsync().Started.Should().BeGreaterOrEqualTo(2); + } + recordedUsages.ForCancelScriptAsync().Started.Should().BeGreaterOrEqualTo(1); + recordedUsages.ForCompleteScriptAsync().Started.Should().Be(1); + } + + [Test] + [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage) })] + public async Task DuringCompleteScript_ScriptExecutionCanBeCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) + { + // ARRANGE + var rpcCallHasStarted = new Reference(false); + var hasPausedOrStoppedPortForwarder = false; + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPendingRequestQueueFactory(new CancellationObservingPendingRequestQueueFactory()) // Partially works around disconnected polling tentacles take work from the queue + .WithRetryDuration(TimeSpan.FromHours(1)) + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithPortForwarder(out var portForwarder) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeCompleteScript( + async () => + { + if (!hasPausedOrStoppedPortForwarder) + { + hasPausedOrStoppedPortForwarder = true; + await tcpConnectionUtilities.RestartTcpConnection(); + await PauseOrStopPortForwarder(rpcCallStage, portForwarder.Value, responseMessageTcpKiller, rpcCallHasStarted); + if (rpcCallStage == RpcCallStage.Connecting) + { + await tcpConnectionUtilities.EnsurePollingQueueWontSendMessageToDisconnectedTentacles(); + } + } + })) + .Build()) + .Build(CancellationToken); + + clientAndTentacle.TentacleClient.OnCancellationAbandonCompleteScriptAfter = TimeSpan.FromSeconds(20); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Print("The script") + .Sleep(TimeSpan.FromSeconds(5))) + .Build(); + + // ACT + var (responseAndLogs, actualException, cancellationDuration) = await ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted(clientAndTentacle, startScriptCommand, rpcCallHasStarted, new SemaphoreSlim(int.MaxValue, int.MaxValue)); + + // ASSERT + + // Assert that the cancellation was performed at the correct state e.g. Connecting or Transferring + var latestException = recordedUsages.ForCompleteScriptAsync().LastException; + if (tentacleConfigurationTestCase.TentacleType == TentacleType.Listening && rpcCallStage == RpcCallStage.Connecting && latestException is HalibutClientException) + { + Assert.Inconclusive("This test is very fragile and often it will often cancel when the client is not in a wait trying to connect but instead gets error responses from the proxy. " + + "This results in a halibut client exception being returned rather than a request cancelled error being returned and is not testing the intended scenario"); + } + + // The actual exception may be null if the script completed before the cancellation was observed + if (actualException != null) + { + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + actualException.ShouldMatchExceptionContract(expectedException); + } + + latestException.Should().BeRequestCancelledException(rpcCallStage); + + // Complete Script was cancelled quickly + cancellationDuration.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(30)); + + recordedUsages.ForStartScriptAsync().Started.Should().Be(1); + recordedUsages.ForGetStatusAsync().Started.Should().BeGreaterThanOrEqualTo(1); + recordedUsages.ForCancelScriptAsync().Started.Should().Be(0); + recordedUsages.ForCompleteScriptAsync().Started.Should().BeGreaterOrEqualTo(1); + } + + async Task PauseOrStopPortForwarder(RpcCallStage rpcCallStage, PortForwarder portForwarder, IResponseMessageTcpKiller responseMessageTcpKiller, Reference rpcCallHasStarted) + { + if (rpcCallStage == RpcCallStage.Connecting) + { + Logger.Information("Killing the port forwarder so the next RPCs are in the connecting state when being cancelled"); + portForwarder.EnterKillNewAndExistingConnectionsMode(); + + await SetRpcCallWeAreInterestedInAsStarted(rpcCallHasStarted); + } + else + { + Logger.Information("Will Pause the port forwarder on next response so the next RPC is in-flight when being cancelled"); +#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed + responseMessageTcpKiller.PauseConnectionOnNextResponse(() => SetRpcCallWeAreInterestedInAsStarted(rpcCallHasStarted)); +#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed + } + + static async Task SetRpcCallWeAreInterestedInAsStarted(Reference rpcCallHasStarted) + { + // Allow the port forwarder some time to stop + await Task.Delay(TimeSpan.FromSeconds(5)); + rpcCallHasStarted.Value = true; + } + } + + void UnPauseOrRestartPortForwarder(TentacleType tentacleType, RpcCallStage rpcCallStage, Reference portForwarder) + { + if (rpcCallStage == RpcCallStage.Connecting) + { + Logger.Information("Starting the PortForwarder as we stopped it to get the StartScript RPC call in the Connecting state"); + portForwarder.Value.ReturnToNormalMode(); + } + else if (tentacleType == TentacleType.Polling) + { + Logger.Information("UnPausing the PortForwarder as we paused the connections which means Polling will be stalled"); + portForwarder.Value.UnPauseExistingConnections(); + portForwarder.Value.CloseExistingConnections(); + } + } + + async Task<(ScriptExecutionResult response, Exception? actualException, TimeSpan cancellationDuration)> ExecuteScriptThenCancelExecutionWhenRpcCallHasStarted( + ClientAndTentacle clientAndTentacle, + ExecuteScriptCommand executeScriptCommand, + Reference rpcCallHasStarted, + SemaphoreSlim whenTheRequestCanBeCancelled) + { + var cancelExecutionCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); + + var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( + executeScriptCommand, + cancelExecutionCancellationTokenSource.Token); + + Logger.Information("Waiting for the RPC Call to start"); + await Wait.For(() => rpcCallHasStarted.Value, + TimeSpan.FromSeconds(30), + () => throw new Exception("RPC call did not start"), + CancellationToken); + Logger.Information("RPC Call has start"); + + var cancellationDuration = new Stopwatch(); + await whenTheRequestCanBeCancelled.WithLockAsync(async () => + { + await Task.Delay(TimeSpan.FromSeconds(6), CancellationToken); + Logger.Information("Cancelling ExecuteScript"); + cancelExecutionCancellationTokenSource.Cancel(); + cancellationDuration.Start(); + }, CancellationToken); + + Exception? actualException = null; + (ScriptExecutionResult Response, List Logs)? responseAndLogs = null; + try + { + responseAndLogs = await executeScriptTask; + } + catch (Exception ex) + { + actualException = ex; + } + + cancellationDuration.Stop(); + return (responseAndLogs?.Response, actualException, cancellationDuration.Elapsed); + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanRecoverFromNetworkIssues.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanRecoverFromNetworkIssues.cs index 7511c8a45..2a488356d 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanRecoverFromNetworkIssues.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionCanRecoverFromNetworkIssues.cs @@ -1,308 +1,310 @@ -// using System; -// using System.Collections.Generic; -// using System.IO; -// using System.Threading; -// using System.Threading.Tasks; -// using FluentAssertions; -// using NUnit.Framework; -// using Octopus.Tentacle.Client; -// using Octopus.Tentacle.CommonTestUtils; -// using Octopus.Tentacle.CommonTestUtils.Diagnostics; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Contracts.ClientServices; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; -// using Octopus.TestPortForwarder; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class ClientScriptExecutionCanRecoverFromNetworkIssues : IntegrationTest -// { -// [Test] -// [TentacleConfigurations] -// public async Task WhenANetworkFailureOccurs_DuringStartScript_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPortForwarder() -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .Build()) -// .Build(CancellationToken); -// -// var scriptHasStartFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "scripthasstarted"); -// var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .CreateFile(scriptHasStartFile) -// .WaitForFileToExist(waitForFile) -// .Print("hello")) -// // Configure the start script command to wait a long time, so we have plenty of time to kill the connection. -// .WithDurationStartScriptCanWaitForScriptToFinish(TimeSpan.FromHours(1)) -// .Build(); -// -// var inMemoryLog = new InMemoryLog(); -// -// var execScriptTask = Task.Run( -// async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog), -// CancellationToken); -// -// // Wait for the script to start. -// await Wait.For(() => File.Exists(scriptHasStartFile), -// TimeSpan.FromSeconds(30), -// () => throw new Exception("Script did not start"), -// CancellationToken); -// -// // Now it has started, kill active connections killing the start script request. -// clientTentacle.PortForwarder.CloseExistingConnections(); -// -// // Let the script finish. -// File.WriteAllText(waitForFile, ""); -// -// var (finalResponse, logs) = await execScriptTask; -// -// finalResponse.State.Should().Be(ProcessState.Complete); -// finalResponse.ExitCode.Should().Be(0); -// -// var allLogs = logs.JoinLogs(); -// allLogs.Should().Contain("hello"); -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().BeGreaterThan(1); -// -// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task WhenANetworkFailureOccurs_DuringGetStatus_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeGetStatus( -// async () => -// { -// await Task.CompletedTask; -// -// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException is null) -// { -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .Print("hello") -// .WaitForFileToExist(waitForFile) -// .Print("AllDone")) -// .Build(); -// -// var inMemoryLog = new InMemoryLog(); -// -// var execScriptTask = Task.Run( -// async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog), -// CancellationToken); -// -// await Wait.For(() => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException != null, -// TimeSpan.FromSeconds(60), -// () => throw new Exception("GetStatus did not error"), -// CancellationToken); -// -// // Let the script finish. -// File.WriteAllText(waitForFile, ""); -// -// var (finalResponse, logs) = await execScriptTask; -// -// finalResponse.State.Should().Be(ProcessState.Complete); -// finalResponse.ExitCode.Should().Be(0); -// -// var allLogs = logs.JoinLogs(); -// allLogs.Should().Contain("hello"); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException.Should().NotBeNull(); -// -// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task WhenANetworkFailureOccurs_DuringCompleteScript_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// bool completeScriptWasCalled = false; -// PortForwarder? portForwarder = null; -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPortForwarderDataLogging() -// .WithServiceEndpointModifier(serviceEndpoint => -// { -// serviceEndpoint.PollingRequestQueueTimeout = TimeSpan.FromSeconds(10); -// serviceEndpoint.RetryCountLimit = 1; -// }) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .DecorateAllScriptServicesWith(u => u -// .BeforeCompleteScript( -// async () => -// { -// -// await Task.CompletedTask; -// -// completeScriptWasCalled = true; -// // A successfully CompleteScript call is not required for the script to be completed. -// // So it should be the case that the tentacle can be no longer contactable at this point, -// // yet the script execution is marked as successful. -// portForwarder.Dispose(); -// })) -// .Build()) -// .Build(CancellationToken); -// portForwarder = clientTentacle.PortForwarder; -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder().Print("hello").Sleep(TimeSpan.FromSeconds(1))) -// .Build(); -// -// var inMemoryLog = new InMemoryLog(); -// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); -// -// finalResponse.State.Should().Be(ProcessState.Complete); -// finalResponse.ExitCode.Should().Be(0); -// -// var allLogs = logs.JoinLogs(); -// allLogs.Should().Contain("hello"); -// completeScriptWasCalled.Should().BeTrue("The tests expects that the client actually called this"); -// -// inMemoryLog.ShouldNotHaveLoggedRetryAttemptsOrRetryFailures(); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task WhenANetworkFailureOccurs_DuringCancelScript_TheClientIsAbleToSuccessfullyCancelTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var cts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); -// using var tmp = new TemporaryDirectory(); -// var scriptIsRunningFlag = Path.Combine(tmp.DirectoryPath, "scriptisrunning"); -// -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeGetStatus( -// async () => -// { -// await Wait.For(() => File.Exists(scriptIsRunningFlag), -// TimeSpan.FromSeconds(30), -// () => throw new Exception("Script did not start"), -// CancellationToken); -// cts.Cancel(); -// }) -// .BeforeCancelScript( -// async () => -// { -// await Task.CompletedTask; -// -// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException == null) -// { -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .Print("hello") -// .CreateFile(scriptIsRunningFlag) -// .Sleep(TimeSpan.FromMinutes(2)) -// .Print("AllDone")) -// .Build(); -// -// Exception? actualException = null; -// var logs = new List(); -// var inMemoryLog = new InMemoryLog(); -// -// try -// { -// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, -// onScriptStatusResponseReceived => -// { -// logs.AddRange(onScriptStatusResponseReceived.Logs); -// }, -// _ => Task.CompletedTask, -// new SerilogLoggerBuilder().Build().ForContext().ToITentacleTaskLog().Chain(inMemoryLog), -// cts.Token); -// } -// catch (Exception ex) -// { -// actualException = ex; -// } -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); -// -// actualException!.ShouldMatchExceptionContract(expectedException); -// -// var allLogs = logs.JoinLogs(); -// allLogs.Should().Contain("hello"); -// allLogs.Should().NotContain("AllDone"); -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException.Should().NotBeNull(); -// -// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task WhenANetworkFailureOccurs_DuringGetCapabilities_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var recordedUsages) -// .DecorateCapabilitiesServiceV2With(d => d -// .BeforeGetCapabilities( -// async () => -// { -// if (recordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException == null) -// { -// await tcpConnectionUtilities.RestartTcpConnection(); -// -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var inMemoryLog = new InMemoryLog(); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder().Print("hello")) -// .Build(); -// -// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); -// -// finalResponse.State.Should().Be(ProcessState.Complete); -// finalResponse.ExitCode.Should().Be(0); -// -// var allLogs = logs.JoinLogs(); -// allLogs.Should().Contain("hello"); -// -// recordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException.Should().NotBeNull(); -// recordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).Started.Should().Be(2); -// -// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); -// } -// } -// } +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Tentacle.Client; +using Octopus.Tentacle.CommonTestUtils; +using Octopus.Tentacle.CommonTestUtils.Diagnostics; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; +using Octopus.TestPortForwarder; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] + public class ClientScriptExecutionCanRecoverFromNetworkIssues : IntegrationTest + { + [Test] + [TentacleConfigurations] + public async Task WhenANetworkFailureOccurs_DuringStartScript_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPortForwarder() + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .Build()) + .Build(CancellationToken); + + var scriptHasStartFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "scripthasstarted"); + var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .CreateFile(scriptHasStartFile) + .WaitForFileToExist(waitForFile) + .Print("hello")) + // Configure the start script command to wait a long time, so we have plenty of time to kill the connection. + .WithDurationStartScriptCanWaitForScriptToFinish(TimeSpan.FromHours(1)) + .Build(); + + var inMemoryLog = new InMemoryLog(); + + var execScriptTask = Task.Run( + async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog), + CancellationToken); + + // Wait for the script to start. + await Wait.For(() => File.Exists(scriptHasStartFile), + TimeSpan.FromSeconds(30), + () => throw new Exception("Script did not start"), + CancellationToken); + + // Now it has started, kill active connections killing the start script request. + clientTentacle.PortForwarder.CloseExistingConnections(); + + // Let the script finish. + File.WriteAllText(waitForFile, ""); + + var (finalResponse, logs) = await execScriptTask; + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = logs.JoinLogs(); + allLogs.Should().Contain("hello"); + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().BeGreaterThan(1); + + inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); + } + + [Test] + [TentacleConfigurations] + public async Task WhenANetworkFailureOccurs_DuringGetStatus_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeGetStatus( + async () => + { + await Task.CompletedTask; + + if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException is null) + { + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + })) + .Build()) + .Build(CancellationToken); + + var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .Print("hello") + .WaitForFileToExist(waitForFile) + .Print("AllDone")) + .Build(); + + var inMemoryLog = new InMemoryLog(); + + var execScriptTask = Task.Run( + async () => await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog), + CancellationToken); + + await Wait.For(() => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException != null, + TimeSpan.FromSeconds(60), + () => throw new Exception("GetStatus did not error"), + CancellationToken); + + // Let the script finish. + File.WriteAllText(waitForFile, ""); + + var (finalResponse, logs) = await execScriptTask; + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = logs.JoinLogs(); + allLogs.Should().Contain("hello"); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException.Should().NotBeNull(); + + inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); + } + + [Test] + [TentacleConfigurations] + public async Task WhenANetworkFailureOccurs_DuringCompleteScript_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + bool completeScriptWasCalled = false; + PortForwarder? portForwarder = null; + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPortForwarderDataLogging() + .WithServiceEndpointModifier(serviceEndpoint => + { + serviceEndpoint.PollingRequestQueueTimeout = TimeSpan.FromSeconds(10); + serviceEndpoint.RetryCountLimit = 1; + }) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .DecorateAllScriptServicesWith(u => u + .BeforeCompleteScript( + async () => + { + + await Task.CompletedTask; + + completeScriptWasCalled = true; + // A successfully CompleteScript call is not required for the script to be completed. + // So it should be the case that the tentacle can be no longer contactable at this point, + // yet the script execution is marked as successful. + portForwarder.Dispose(); + })) + .Build()) + .Build(CancellationToken); + portForwarder = clientTentacle.PortForwarder; + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder().Print("hello").Sleep(TimeSpan.FromSeconds(1))) + .Build(); + + var inMemoryLog = new InMemoryLog(); + var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = logs.JoinLogs(); + allLogs.Should().Contain("hello"); + completeScriptWasCalled.Should().BeTrue("The tests expects that the client actually called this"); + + inMemoryLog.ShouldNotHaveLoggedRetryAttemptsOrRetryFailures(); + } + + [Test] + [TentacleConfigurations] + public async Task WhenANetworkFailureOccurs_DuringCancelScript_TheClientIsAbleToSuccessfullyCancelTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); + using var tmp = new TemporaryDirectory(); + var scriptIsRunningFlag = Path.Combine(tmp.DirectoryPath, "scriptisrunning"); + + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeGetStatus( + async () => + { + await Wait.For(() => File.Exists(scriptIsRunningFlag), + TimeSpan.FromSeconds(30), + () => throw new Exception("Script did not start"), + CancellationToken); + cts.Cancel(); + }) + .BeforeCancelScript( + async () => + { + await Task.CompletedTask; + + if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException == null) + { + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + })) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .Print("hello") + .CreateFile(scriptIsRunningFlag) + .Sleep(TimeSpan.FromMinutes(2)) + .Print("AllDone")) + .Build(); + + Exception? actualException = null; + var logs = new List(); + var inMemoryLog = new InMemoryLog(); + + try + { + await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, + onScriptStatusResponseReceived => + { + logs.AddRange(onScriptStatusResponseReceived.Logs); + }, + _ => Task.CompletedTask, + new SerilogLoggerBuilder().Build().ForContext().ToITentacleTaskLog().Chain(inMemoryLog), + cts.Token); + } + catch (Exception ex) + { + actualException = ex; + } + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ScriptExecutionCancelled, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); + + actualException!.ShouldMatchExceptionContract(expectedException); + + var allLogs = logs.JoinLogs(); + allLogs.Should().Contain("hello"); + allLogs.Should().NotContain("AllDone"); + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException.Should().NotBeNull(); + + inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); + } + + [Test] + [TentacleConfigurations] + public async Task WhenANetworkFailureOccurs_DuringGetCapabilities_TheClientIsAbleToSuccessfullyCompleteTheScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var recordedUsages) + .DecorateCapabilitiesServiceV2With(d => d + .BeforeGetCapabilities( + async () => + { + if (recordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException == null) + { + await tcpConnectionUtilities.RestartTcpConnection(); + + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + })) + .Build()) + .Build(CancellationToken); + + var inMemoryLog = new InMemoryLog(); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder().Print("hello")) + .Build(); + + var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = logs.JoinLogs(); + allLogs.Should().Contain("hello"); + + recordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException.Should().NotBeNull(); + recordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).Started.Should().Be(2); + + inMemoryLog.ShouldHaveLoggedRetryAttemptsAndNoRetryFailures(); + } + } +} diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs index 0a5e2c447..96e82e3ad 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs @@ -1,548 +1,549 @@ -// using System; -// using System.Diagnostics; -// using System.Threading; -// using System.Threading.Tasks; -// using FluentAssertions; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils.Diagnostics; -// using Octopus.Tentacle.Contracts.ClientServices; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// /// -// /// These tests make sure that we can cancel or walk away (if code does not cooperate with cancellation tokens) -// /// from RPC calls when they are being retried and the rpc timeout period elapses. -// /// -// [IntegrationTestTimeout] -// public class ClientScriptExecutionRetriesTimeout : IntegrationTest -// { -// readonly TimeSpan retryIfRemainingDurationAtLeastBuffer = TimeSpan.FromSeconds(1); -// readonly TimeSpan retryBackoffBuffer = TimeSpan.FromSeconds(2); -// -// [Test] -// [TentacleConfigurations(additionalParameterTypes: new object[] {typeof(RpcCallStage)})] -// public async Task WhenRpcRetriesTimeOut_DuringGetCapabilities_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) -// { -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// // Set a short retry duration so we cancel fairly quickly -// .WithRetryDuration(TimeSpan.FromSeconds(15)) -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithPortForwarder(out var portForwarder) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var capabilitiesMethodUsages) -// .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) -// .DecorateCapabilitiesServiceV2With(d => d -// .BeforeGetCapabilities( -// async () => -// { -// // Kill the first GetCapabilities call to force the rpc call into retries -// if (capabilitiesMethodUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException is null) -// { -// // Ensure there is an active connection so it can be killed correctly -// await tcpConnectionUtilities.RestartTcpConnection(); -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// else -// { -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// // Kill the port forwarder so the next requests are in the connecting state when retries timeout -// Logger.Information("Killing PortForwarder"); -// portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); -// } -// else -// { -// // Ensure there is an active connection so it can be killed correctly -// await tcpConnectionUtilities.RestartTcpConnection(); -// // Pause the port forwarder so the next requests are in-flight when retries timeout -// responseMessageTcpKiller.PauseConnectionOnNextResponse(); -// } -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var inMemoryLog = new InMemoryLog(); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Print("Should not run this script")) -// .Build(); -// -// var duration = Stopwatch.StartNew(); -// -// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// duration.Stop(); -// -// capabilitiesMethodUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).Started.Should().BeGreaterOrEqualTo(2); -// scriptMethodUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(0, "Test should not have not proceeded past GetCapabilities"); -// -// // Ensure we actually waited and retried until the timeout policy kicked in -// duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); -// -// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task WhenGetCapabilitiesFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var retryDuration = TimeSpan.FromSeconds(15); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// // Set a short retry duration so we cancel fairly quickly -// .WithRetryDuration(retryDuration) -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var capabilitiesMethodUsages) -// .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) -// .DecorateCapabilitiesServiceV2With(d => d -// .BeforeGetCapabilities( -// async () => -// { -// await tcpConnectionUtilities.RestartTcpConnection(); -// -// // Sleep to make the initial RPC call take longer than the allowed retry duration -// await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); -// -// // Kill the first GetCapabilities call to force the rpc call into retries -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// })) -// .Build()) -// .Build(CancellationToken); -// -// var inMemoryLog = new InMemoryLog(); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .Build(); -// -// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// capabilitiesMethodUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).Started.Should().Be(1); -// scriptMethodUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(0, "Test should not have not proceeded past GetCapabilities"); -// -// inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); -// } -// -// [Test] -// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage)})] -// public async Task WhenRpcRetriesTimeOut_DuringStartScript_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) -// { -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// // Set a short retry duration so we cancel fairly quickly -// .WithRetryDuration(TimeSpan.FromSeconds(15)) -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithPortForwarder(out var portForwarder) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeStartScript( -// async () => -// { -// // Kill the first StartScript call to force the rpc call into retries -// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).LastException == null) -// { -// // Ensure there is an active connection so it can be killed correctly -// await tcpConnectionUtilities.RestartTcpConnection(); -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// else -// { -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// // Kill the port forwarder so the next requests are in the connecting state when retries timeout -// Logger.Information("Killing PortForwarder"); -// portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); -// } -// else -// { -// // Ensure there is an active connection so it can be killed correctly -// await tcpConnectionUtilities.RestartTcpConnection(); -// // Pause the port forwarder so the next requests are in-flight when retries timeout -// responseMessageTcpKiller.PauseConnectionOnNextResponse(); -// } -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var inMemoryLog = new InMemoryLog(); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Print("Start Script") -// .Sleep(TimeSpan.FromHours(1)) -// .Print("End Script")) -// .Build(); -// -// var duration = Stopwatch.StartNew(); -// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// duration.Stop(); -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().BeGreaterOrEqualTo(2); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); -// -// // Ensure we actually waited and retried until the timeout policy kicked in -// duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); -// -// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); -// } -// -// [Test] -// [TentacleConfigurations] -// [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] -// public async Task WhenStartScriptFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var retryDuration = TimeSpan.FromSeconds(15); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// // Set a short retry duration so we cancel fairly quickly -// .WithRetryDuration(retryDuration) -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeStartScript( -// async () => -// { -// // Sleep to make the initial RPC call take longer than the allowed retry duration -// await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); -// -// // Kill the first StartScript call to force the rpc call into retries -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// })) -// .Build()) -// .Build(CancellationToken); -// -// var inMemoryLog = new InMemoryLog(); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Sleep(TimeSpan.FromHours(1))) -// .Build(); -// -// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); -// -// inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); -// } -// -// [Test] -// [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage)})] -// public async Task WhenRpcRetriesTimeOut_DuringGetStatus_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) -// { -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// // Set a short retry duration so we cancel fairly quickly -// .WithRetryDuration(TimeSpan.FromSeconds(15)) -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithPortForwarder(out var portForwarder) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeGetStatus( -// async () => -// { -// // Kill the first GetStatus call to force the rpc call into retries -// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException == null) -// { -// // Ensure there is an active connection so it can be killed correctly -// await tcpConnectionUtilities.RestartTcpConnection(); -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// else -// { -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// // Kill the port forwarder so the next requests are in the connecting state when retries timeout -// Logger.Information("Killing PortForwarder"); -// portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); -// } -// else -// { -// // Ensure there is an active connection so it can be killed correctly -// await tcpConnectionUtilities.RestartTcpConnection(); -// // Pause the port forwarder so the next requests are in-flight when retries timeout -// responseMessageTcpKiller.PauseConnectionOnNextResponse(); -// } -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var inMemoryLog = new InMemoryLog(); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Print("Start Script") -// .Sleep(TimeSpan.FromHours(1)) -// .Print("End Script")) -// // Don't wait in start script as we want to tst get status -// .WithDurationStartScriptCanWaitForScriptToFinish(null) -// .Build(); -// -// var duration = Stopwatch.StartNew(); -// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// duration.Stop(); -// -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterOrEqualTo(2); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); -// -// // Ensure we actually waited and retried until the timeout policy kicked in -// duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); -// -// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task WhenGetStatusFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var retryDuration = TimeSpan.FromSeconds(15); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// // Set a short retry duration so we cancel fairly quickly -// .WithRetryDuration(retryDuration) -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeGetStatus( -// async () => -// { -// // Sleep to make the initial RPC call take longer than the allowed retry duration -// await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); -// -// // Kill the first GetStatus call to force the rpc call into retries -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// })) -// .Build()) -// .Build(CancellationToken); -// -// var inMemoryLog = new InMemoryLog(); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Sleep(TimeSpan.FromHours(1))) -// // Don't wait in start script as we want to test get status -// .WithDurationStartScriptCanWaitForScriptToFinish(null) -// .Build(); -// -// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); -// -// inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); -// } -// -// [Test] -// [TentacleConfigurations(additionalParameterTypes: new object[] {typeof(RpcCallStage)})] -// public async Task WhenRpcRetriesTimeOut_DuringCancelScript_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) -// { -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// // Set a short retry duration so we cancel fairly quickly -// .WithRetryDuration(TimeSpan.FromSeconds(15)) -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithPortForwarder(out var portForwarder) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeCancelScript( -// async () => -// { -// // Kill the first CancelScript call to force the rpc call into retries -// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException == null) -// { -// // Ensure there is an active connection so it can be killed correctly -// await tcpConnectionUtilities.RestartTcpConnection(); -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// else -// { -// if (rpcCallStage == RpcCallStage.Connecting) -// { -// // Kill the port forwarder so the next requests are in the connecting state when retries timeout -// Logger.Information("Killing PortForwarder"); -// portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); -// } -// else -// { -// // Ensure there is an active connection so it can be killed correctly -// await tcpConnectionUtilities.RestartTcpConnection(); -// // Pause the port forwarder so the next requests are in-flight when retries timeout -// responseMessageTcpKiller.PauseConnectionOnNextResponse(); -// } -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var inMemoryLog = new InMemoryLog(); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Print("Start Script") -// .Sleep(TimeSpan.FromHours(1)) -// .Print("End Script")) -// // Don't wait in start script as we want to tst get status -// .WithDurationStartScriptCanWaitForScriptToFinish(null) -// .Build(); -// -// // Start the script which will wait for a file to exist -// var testCancellationTokenSource = new CancellationTokenSource(); -// -// var duration = Stopwatch.StartNew(); -// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( -// startScriptCommand, -// CancellationTokenSource.CreateLinkedTokenSource(CancellationToken, testCancellationTokenSource.Token).Token, -// null, -// inMemoryLog); -// -// await Wait.For(() => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Completed > 0, -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Script execution did not complete"), -// CancellationToken); -// -// // We cancel script execution via the cancellation token. This should trigger the CancelScript RPC call to be made -// testCancellationTokenSource.Cancel(); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// duration.Stop(); -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterOrEqualTo(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().BeGreaterOrEqualTo(2); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); -// -// // Ensure we actually waited and retried until the timeout policy kicked in -// duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); -// -// inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task WhenCancelScriptFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var retryDuration = TimeSpan.FromSeconds(15); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// // Set a short retry duration so we cancel fairly quickly -// .WithRetryDuration(retryDuration) -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeCancelScript( -// async () => -// { -// // Sleep to make the initial RPC call take longer than the allowed retry duration -// await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); -// -// // Kill the first CancelScript call to force the rpc call into retries -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// })) -// .Build()) -// .Build(CancellationToken); -// -// var inMemoryLog = new InMemoryLog(); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b -// .Sleep(TimeSpan.FromHours(1))) -// // Don't wait in start script as we want to test get status -// .WithDurationStartScriptCanWaitForScriptToFinish(null) -// .Build(); -// -// // Start the script which will wait for a file to exist -// var testCancellationTokenSource = new CancellationTokenSource(); -// -// var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( -// startScriptCommand, -// CancellationTokenSource.CreateLinkedTokenSource(CancellationToken, testCancellationTokenSource.Token).Token, -// null, -// inMemoryLog); -// -// Func action = async () => await executeScriptTask; -// await Wait.For( -// () => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Completed > 0, -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Script Execution did not complete"), -// CancellationToken); -// -// // We cancel script execution via the cancellation token. This should trigger the CancelScript RPC call to be made -// testCancellationTokenSource.Cancel(); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterOrEqualTo(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); -// -// inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); -// } -// } -// } +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils.Diagnostics; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; + +namespace Octopus.Tentacle.Tests.Integration +{ + /// + /// These tests make sure that we can cancel or walk away (if code does not cooperate with cancellation tokens) + /// from RPC calls when they are being retried and the rpc timeout period elapses. + /// + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] + public class ClientScriptExecutionRetriesTimeout : IntegrationTest + { + readonly TimeSpan retryIfRemainingDurationAtLeastBuffer = TimeSpan.FromSeconds(1); + readonly TimeSpan retryBackoffBuffer = TimeSpan.FromSeconds(2); + + [Test] + [TentacleConfigurations(additionalParameterTypes: new object[] {typeof(RpcCallStage)})] + public async Task WhenRpcRetriesTimeOut_DuringGetCapabilities_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) + { + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + // Set a short retry duration so we cancel fairly quickly + .WithRetryDuration(TimeSpan.FromSeconds(15)) + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithPortForwarder(out var portForwarder) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var capabilitiesMethodUsages) + .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) + .DecorateCapabilitiesServiceV2With(d => d + .BeforeGetCapabilities( + async () => + { + // Kill the first GetCapabilities call to force the rpc call into retries + if (capabilitiesMethodUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException is null) + { + // Ensure there is an active connection so it can be killed correctly + await tcpConnectionUtilities.RestartTcpConnection(); + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + else + { + if (rpcCallStage == RpcCallStage.Connecting) + { + // Kill the port forwarder so the next requests are in the connecting state when retries timeout + Logger.Information("Killing PortForwarder"); + portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); + } + else + { + // Ensure there is an active connection so it can be killed correctly + await tcpConnectionUtilities.RestartTcpConnection(); + // Pause the port forwarder so the next requests are in-flight when retries timeout + responseMessageTcpKiller.PauseConnectionOnNextResponse(); + } + } + })) + .Build()) + .Build(CancellationToken); + + var inMemoryLog = new InMemoryLog(); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Print("Should not run this script")) + .Build(); + + var duration = Stopwatch.StartNew(); + + var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + duration.Stop(); + + capabilitiesMethodUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).Started.Should().BeGreaterOrEqualTo(2); + scriptMethodUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(0, "Test should not have not proceeded past GetCapabilities"); + + // Ensure we actually waited and retried until the timeout policy kicked in + duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); + + inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); + } + + [Test] + [TentacleConfigurations] + public async Task WhenGetCapabilitiesFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var retryDuration = TimeSpan.FromSeconds(15); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + // Set a short retry duration so we cancel fairly quickly + .WithRetryDuration(retryDuration) + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var capabilitiesMethodUsages) + .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptMethodUsages) + .DecorateCapabilitiesServiceV2With(d => d + .BeforeGetCapabilities( + async () => + { + await tcpConnectionUtilities.RestartTcpConnection(); + + // Sleep to make the initial RPC call take longer than the allowed retry duration + await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); + + // Kill the first GetCapabilities call to force the rpc call into retries + responseMessageTcpKiller.KillConnectionOnNextResponse(); + })) + .Build()) + .Build(CancellationToken); + + var inMemoryLog = new InMemoryLog(); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .Build(); + + var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + capabilitiesMethodUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).Started.Should().Be(1); + scriptMethodUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(0, "Test should not have not proceeded past GetCapabilities"); + + inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); + } + + [Test] + [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage)})] + public async Task WhenRpcRetriesTimeOut_DuringStartScript_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) + { + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + // Set a short retry duration so we cancel fairly quickly + .WithRetryDuration(TimeSpan.FromSeconds(15)) + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithPortForwarder(out var portForwarder) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeStartScript( + async () => + { + // Kill the first StartScript call to force the rpc call into retries + if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).LastException == null) + { + // Ensure there is an active connection so it can be killed correctly + await tcpConnectionUtilities.RestartTcpConnection(); + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + else + { + if (rpcCallStage == RpcCallStage.Connecting) + { + // Kill the port forwarder so the next requests are in the connecting state when retries timeout + Logger.Information("Killing PortForwarder"); + portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); + } + else + { + // Ensure there is an active connection so it can be killed correctly + await tcpConnectionUtilities.RestartTcpConnection(); + // Pause the port forwarder so the next requests are in-flight when retries timeout + responseMessageTcpKiller.PauseConnectionOnNextResponse(); + } + } + })) + .Build()) + .Build(CancellationToken); + + var inMemoryLog = new InMemoryLog(); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Print("Start Script") + .Sleep(TimeSpan.FromHours(1)) + .Print("End Script")) + .Build(); + + var duration = Stopwatch.StartNew(); + var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + duration.Stop(); + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().BeGreaterOrEqualTo(2); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); + + // Ensure we actually waited and retried until the timeout policy kicked in + duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); + + inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); + } + + [Test] + [TentacleConfigurations] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] + public async Task WhenStartScriptFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var retryDuration = TimeSpan.FromSeconds(15); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + // Set a short retry duration so we cancel fairly quickly + .WithRetryDuration(retryDuration) + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeStartScript( + async () => + { + // Sleep to make the initial RPC call take longer than the allowed retry duration + await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); + + // Kill the first StartScript call to force the rpc call into retries + responseMessageTcpKiller.KillConnectionOnNextResponse(); + })) + .Build()) + .Build(CancellationToken); + + var inMemoryLog = new InMemoryLog(); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Sleep(TimeSpan.FromHours(1))) + .Build(); + + var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); + + inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); + } + + [Test] + [TentacleConfigurations(additionalParameterTypes: new object[] { typeof(RpcCallStage)})] + public async Task WhenRpcRetriesTimeOut_DuringGetStatus_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) + { + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + // Set a short retry duration so we cancel fairly quickly + .WithRetryDuration(TimeSpan.FromSeconds(15)) + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithPortForwarder(out var portForwarder) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeGetStatus( + async () => + { + // Kill the first GetStatus call to force the rpc call into retries + if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException == null) + { + // Ensure there is an active connection so it can be killed correctly + await tcpConnectionUtilities.RestartTcpConnection(); + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + else + { + if (rpcCallStage == RpcCallStage.Connecting) + { + // Kill the port forwarder so the next requests are in the connecting state when retries timeout + Logger.Information("Killing PortForwarder"); + portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); + } + else + { + // Ensure there is an active connection so it can be killed correctly + await tcpConnectionUtilities.RestartTcpConnection(); + // Pause the port forwarder so the next requests are in-flight when retries timeout + responseMessageTcpKiller.PauseConnectionOnNextResponse(); + } + } + })) + .Build()) + .Build(CancellationToken); + + var inMemoryLog = new InMemoryLog(); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Print("Start Script") + .Sleep(TimeSpan.FromHours(1)) + .Print("End Script")) + // Don't wait in start script as we want to tst get status + .WithDurationStartScriptCanWaitForScriptToFinish(null) + .Build(); + + var duration = Stopwatch.StartNew(); + var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + duration.Stop(); + + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterOrEqualTo(2); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); + + // Ensure we actually waited and retried until the timeout policy kicked in + duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); + + inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); + } + + [Test] + [TentacleConfigurations] + public async Task WhenGetStatusFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var retryDuration = TimeSpan.FromSeconds(15); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + // Set a short retry duration so we cancel fairly quickly + .WithRetryDuration(retryDuration) + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeGetStatus( + async () => + { + // Sleep to make the initial RPC call take longer than the allowed retry duration + await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); + + // Kill the first GetStatus call to force the rpc call into retries + responseMessageTcpKiller.KillConnectionOnNextResponse(); + })) + .Build()) + .Build(CancellationToken); + + var inMemoryLog = new InMemoryLog(); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Sleep(TimeSpan.FromHours(1))) + // Don't wait in start script as we want to test get status + .WithDurationStartScriptCanWaitForScriptToFinish(null) + .Build(); + + var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, inMemoryLog); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); + + inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); + } + + [Test] + [TentacleConfigurations(additionalParameterTypes: new object[] {typeof(RpcCallStage)})] + public async Task WhenRpcRetriesTimeOut_DuringCancelScript_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, RpcCallStage rpcCallStage) + { + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + // Set a short retry duration so we cancel fairly quickly + .WithRetryDuration(TimeSpan.FromSeconds(15)) + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithPortForwarder(out var portForwarder) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeCancelScript( + async () => + { + // Kill the first CancelScript call to force the rpc call into retries + if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException == null) + { + // Ensure there is an active connection so it can be killed correctly + await tcpConnectionUtilities.RestartTcpConnection(); + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + else + { + if (rpcCallStage == RpcCallStage.Connecting) + { + // Kill the port forwarder so the next requests are in the connecting state when retries timeout + Logger.Information("Killing PortForwarder"); + portForwarder.Value.EnterKillNewAndExistingConnectionsMode(); + } + else + { + // Ensure there is an active connection so it can be killed correctly + await tcpConnectionUtilities.RestartTcpConnection(); + // Pause the port forwarder so the next requests are in-flight when retries timeout + responseMessageTcpKiller.PauseConnectionOnNextResponse(); + } + } + })) + .Build()) + .Build(CancellationToken); + + var inMemoryLog = new InMemoryLog(); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Print("Start Script") + .Sleep(TimeSpan.FromHours(1)) + .Print("End Script")) + // Don't wait in start script as we want to tst get status + .WithDurationStartScriptCanWaitForScriptToFinish(null) + .Build(); + + // Start the script which will wait for a file to exist + var testCancellationTokenSource = new CancellationTokenSource(); + + var duration = Stopwatch.StartNew(); + var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( + startScriptCommand, + CancellationTokenSource.CreateLinkedTokenSource(CancellationToken, testCancellationTokenSource.Token).Token, + null, + inMemoryLog); + + await Wait.For(() => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Completed > 0, + TimeSpan.FromSeconds(60), + () => throw new Exception("Script execution did not complete"), + CancellationToken); + + // We cancel script execution via the cancellation token. This should trigger the CancelScript RPC call to be made + testCancellationTokenSource.Cancel(); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + duration.Stop(); + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterOrEqualTo(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().BeGreaterOrEqualTo(2); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); + + // Ensure we actually waited and retried until the timeout policy kicked in + duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer); + + inMemoryLog.ShouldHaveLoggedRetryAttemptsAndRetryFailure(); + } + + [Test] + [TentacleConfigurations] + public async Task WhenCancelScriptFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var retryDuration = TimeSpan.FromSeconds(15); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + // Set a short retry duration so we cancel fairly quickly + .WithRetryDuration(retryDuration) + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeCancelScript( + async () => + { + // Sleep to make the initial RPC call take longer than the allowed retry duration + await Task.Delay(retryDuration + TimeSpan.FromSeconds(1)); + + // Kill the first CancelScript call to force the rpc call into retries + responseMessageTcpKiller.KillConnectionOnNextResponse(); + })) + .Build()) + .Build(CancellationToken); + + var inMemoryLog = new InMemoryLog(); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b + .Sleep(TimeSpan.FromHours(1))) + // Don't wait in start script as we want to test get status + .WithDurationStartScriptCanWaitForScriptToFinish(null) + .Build(); + + // Start the script which will wait for a file to exist + var testCancellationTokenSource = new CancellationTokenSource(); + + var executeScriptTask = clientAndTentacle.TentacleClient.ExecuteScript( + startScriptCommand, + CancellationTokenSource.CreateLinkedTokenSource(CancellationToken, testCancellationTokenSource.Token).Token, + null, + inMemoryLog); + + Func action = async () => await executeScriptTask; + await Wait.For( + () => recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Completed > 0, + TimeSpan.FromSeconds(60), + () => throw new Exception("Script Execution did not complete"), + CancellationToken); + + // We cancel script execution via the cancellation token. This should trigger the CancelScript RPC call to be made + testCancellationTokenSource.Cancel(); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterOrEqualTo(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); + + inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts(); + } + } +} diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs index f5423050c..9f54aac6d 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs @@ -1,38 +1,40 @@ -// using System; -// using System.Threading.Tasks; -// using FluentAssertions; -// using Halibut; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils.Builders; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class ClientScriptExecutionScriptFilesAreSent : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(testCommonVersions: true)] -// public async Task ScriptFilesAreSent(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder().PrintFileContents("foo.txt")) -// .WithScriptFile(new ScriptFile("foo.txt", DataStream.FromString("The File Contents"))) -// .Build(); -// -// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// -// finalResponse.State.Should().Be(ProcessState.Complete); -// finalResponse.ExitCode.Should().Be(0); -// -// var allLogs = logs.JoinLogs(); -// -// allLogs.Should().Contain("The File Contents"); -// } -// } -// } +using System; +using System.Threading.Tasks; +using FluentAssertions; +using Halibut; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils.Builders; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] + public class ClientScriptExecutionScriptFilesAreSent : IntegrationTest + { + [Test] + [TentacleConfigurations(testCommonVersions: true)] + public async Task ScriptFilesAreSent(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder().PrintFileContents("foo.txt")) + .WithScriptFile(new ScriptFile("foo.txt", DataStream.FromString("The File Contents"))) + .Build(); + + var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = logs.JoinLogs(); + + allLogs.Should().Contain("The File Contents"); + } + } +} diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled.cs index 3cc14d6ca..a75a8dbf3 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled.cs @@ -1,292 +1,294 @@ -// using System; -// using System.Collections.Generic; -// using System.IO; -// using System.Threading; -// using System.Threading.Tasks; -// using FluentAssertions; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils.Builders; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Contracts.ClientServices; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled : IntegrationTest -// { -// [Test] -// [TentacleConfigurations] -// public async Task WhenNetworkFailureOccurs_DuringGetCapabilities_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithRetriesDisabled() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithRetryDuration(TimeSpan.FromMinutes(4)) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var capabilitiesRecordedUsages) -// .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptRecordedUsages) -// .DecorateCapabilitiesServiceV2With(d => d -// .BeforeGetCapabilities( -// async () => -// { -// // Due to the GetCapabilities response getting cached, we must -// // use a different service to ensure Tentacle is connected to Server. -// // Otherwise, the response to the 'ensure connection' will get cached -// // and any subsequent calls will succeed w/o using the network. -// await tcpConnectionUtilities.RestartTcpConnection(); -// -// if (capabilitiesRecordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException is null) -// { -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder().Print("hello")).Build(); -// -// var logs = new List(); -// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// var allLogs = logs.JoinLogs(); -// -// allLogs.Should().NotContain("hello"); -// -// var getCapabilitiesUsages = capabilitiesRecordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)); -// getCapabilitiesUsages.LastException.Should().NotBeNull(); -// getCapabilitiesUsages.Started.Should().Be(1); -// getCapabilitiesUsages.Completed.Should().Be(1); -// -// scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); -// scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Completed.Should().Be(0); -// scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(0); -// scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Completed.Should().Be(0); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task WhenANetworkFailureOccurs_DuringStartScript_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithRetriesDisabled() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithRetryDuration(TimeSpan.FromMinutes(4)) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeStartScript( -// async () => -// { -// await Task.CompletedTask; -// -// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).LastException is null) -// { -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .Print("hello") -// .Print("AllDone")) -// .Build(); -// -// var logs = new List(); -// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// var allLogs = logs.JoinLogs(); -// -// allLogs.Should().NotContain("AllDone"); -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).LastException.Should().NotBeNull(); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); -// -// // We must ensure all script are complete, otherwise if we shutdown tentacle while running a script the build can hang. -// // Ensure the script is finished by running the script again -// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task WhenANetworkFailureOccurs_DuringGetStatus_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithRetriesDisabled() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithRetryDuration(TimeSpan.FromMinutes(4)) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeGetStatus( -// async () => -// { -// await Task.CompletedTask; -// -// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException is null) -// { -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .Print("hello") -// .WaitForFileToExist(waitForFile) -// .Print("AllDone")) -// .Build(); -// -// var logs = new List(); -// -// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// var allLogs = logs.JoinLogs(); -// -// allLogs.Should().NotContain("AllDone"); -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException.Should().NotBeNull(); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); -// -// // Let the script finish. -// File.WriteAllText(waitForFile, ""); -// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task WhenANetworkFailureOccurs_DuringCancelScript_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithRetriesDisabled() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithRetryDuration(TimeSpan.FromMinutes(4)) -// .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeGetStatus( -// async () => -// { -// await Task.CompletedTask; -// -// cts.Cancel(); -// }) -// .BeforeCancelScript( -// async () => -// { -// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException is null) -// { -// await tcpConnectionUtilities.RestartTcpConnection(); -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .Print("hello") -// .WaitForFileToExist(waitForFile) -// .Print("AllDone")) -// .Build(); -// -// var logs = new List(); -// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, cts.Token); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// var allLogs = logs.JoinLogs(); -// -// allLogs.Should().NotContain("AllDone"); -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException.Should().NotBeNull(); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); -// -// // Let the script finish. -// File.WriteAllText(waitForFile, ""); -// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task WhenANetworkFailureOccurs_DuringCompleteScript_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithRetriesDisabled() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithRetryDuration(TimeSpan.FromMinutes(4)) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateAllScriptServicesWith(u => u -// .BeforeCompleteScript( -// async () => -// { -// await Task.CompletedTask; -// -// if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).LastException is null) -// { -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// })) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .Print("hello") -// .Print("AllDone")) -// .Build(); -// -// var logs = new List(); -// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); -// -// var allLogs = logs.JoinLogs(); -// allLogs.Should().Contain("AllDone"); -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).LastException.Should().NotBeNull(); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); -// } -// } -// } \ No newline at end of file +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils.Builders; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] + public class ClientScriptExecutionScriptServiceNonV1IsNotRetriedWhenRetriesAreDisabled : IntegrationTest + { + [Test] + [TentacleConfigurations] + public async Task WhenNetworkFailureOccurs_DuringGetCapabilities_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithRetriesDisabled() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithRetryDuration(TimeSpan.FromMinutes(4)) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var capabilitiesRecordedUsages) + .RecordMethodUsages(tentacleConfigurationTestCase, out var scriptRecordedUsages) + .DecorateCapabilitiesServiceV2With(d => d + .BeforeGetCapabilities( + async () => + { + // Due to the GetCapabilities response getting cached, we must + // use a different service to ensure Tentacle is connected to Server. + // Otherwise, the response to the 'ensure connection' will get cached + // and any subsequent calls will succeed w/o using the network. + await tcpConnectionUtilities.RestartTcpConnection(); + + if (capabilitiesRecordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)).LastException is null) + { + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + })) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder().Print("hello")).Build(); + + var logs = new List(); + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + var allLogs = logs.JoinLogs(); + + allLogs.Should().NotContain("hello"); + + var getCapabilitiesUsages = capabilitiesRecordedUsages.For(nameof(IAsyncClientCapabilitiesServiceV2.GetCapabilitiesAsync)); + getCapabilitiesUsages.LastException.Should().NotBeNull(); + getCapabilitiesUsages.Started.Should().Be(1); + getCapabilitiesUsages.Completed.Should().Be(1); + + scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); + scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Completed.Should().Be(0); + scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(0); + scriptRecordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Completed.Should().Be(0); + } + + [Test] + [TentacleConfigurations] + public async Task WhenANetworkFailureOccurs_DuringStartScript_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithRetriesDisabled() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithRetryDuration(TimeSpan.FromMinutes(4)) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeStartScript( + async () => + { + await Task.CompletedTask; + + if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).LastException is null) + { + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + })) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .Print("hello") + .Print("AllDone")) + .Build(); + + var logs = new List(); + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + var allLogs = logs.JoinLogs(); + + allLogs.Should().NotContain("AllDone"); + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).LastException.Should().NotBeNull(); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); + + // We must ensure all script are complete, otherwise if we shutdown tentacle while running a script the build can hang. + // Ensure the script is finished by running the script again + await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); + } + + [Test] + [TentacleConfigurations] + public async Task WhenANetworkFailureOccurs_DuringGetStatus_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithRetriesDisabled() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithRetryDuration(TimeSpan.FromMinutes(4)) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeGetStatus( + async () => + { + await Task.CompletedTask; + + if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException is null) + { + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + })) + .Build()) + .Build(CancellationToken); + + var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .Print("hello") + .WaitForFileToExist(waitForFile) + .Print("AllDone")) + .Build(); + + var logs = new List(); + + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + var allLogs = logs.JoinLogs(); + + allLogs.Should().NotContain("AllDone"); + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).LastException.Should().NotBeNull(); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); + + // Let the script finish. + File.WriteAllText(waitForFile, ""); + await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); + } + + [Test] + [TentacleConfigurations] + public async Task WhenANetworkFailureOccurs_DuringCancelScript_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithRetriesDisabled() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithRetryDuration(TimeSpan.FromMinutes(4)) + .WithTcpConnectionUtilities(Logger, out var tcpConnectionUtilities) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeGetStatus( + async () => + { + await Task.CompletedTask; + + cts.Cancel(); + }) + .BeforeCancelScript( + async () => + { + if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException is null) + { + await tcpConnectionUtilities.RestartTcpConnection(); + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + })) + .Build()) + .Build(CancellationToken); + + var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .Print("hello") + .WaitForFileToExist(waitForFile) + .Print("AllDone")) + .Build(); + + var logs = new List(); + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, cts.Token); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + var allLogs = logs.JoinLogs(); + + allLogs.Should().NotContain("AllDone"); + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).LastException.Should().NotBeNull(); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(0); + + // Let the script finish. + File.WriteAllText(waitForFile, ""); + await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); + } + + [Test] + [TentacleConfigurations] + public async Task WhenANetworkFailureOccurs_DuringCompleteScript_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithRetriesDisabled() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithRetryDuration(TimeSpan.FromMinutes(4)) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateAllScriptServicesWith(u => u + .BeforeCompleteScript( + async () => + { + await Task.CompletedTask; + + if (recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).LastException is null) + { + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + })) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .Print("hello") + .Print("AllDone")) + .Build(); + + var logs = new List(); + await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); + + var allLogs = logs.JoinLogs(); + allLogs.Should().Contain("AllDone"); + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).LastException.Should().NotBeNull(); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceV1IsNotRetried.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceV1IsNotRetried.cs index 8be5b7864..adf290337 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceV1IsNotRetried.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptServiceV1IsNotRetried.cs @@ -1,251 +1,253 @@ -// using System; -// using System.Collections.Generic; -// using System.IO; -// using System.Threading; -// using System.Threading.Tasks; -// using FluentAssertions; -// using NUnit.Framework; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Contracts.ClientServices; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class ClientScriptExecutionScriptServiceV1IsNotRetried : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] -// public async Task WhenANetworkFailureOccurs_DuringStartScript_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithRetryDuration(TimeSpan.FromMinutes(4)) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var recordedUsages) -// .HookServiceMethod( -// nameof(IAsyncClientScriptService.StartScriptAsync), -// async (_, _) => -// { -// await Task.CompletedTask; -// -// if (recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).LastException == null) -// { -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// }) -// .Build()) -// .Build(CancellationToken); -// -// var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .Print("hello") -// .WaitForFileToExist(waitForFile) -// .Print("AllDone")) -// .WithIsolationLevel(ScriptIsolationLevel.FullIsolation) -// .WithIsolationMutexName("bob") -// .Build(); -// -// var logs = new List(); -// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// // Let the script finish. -// File.WriteAllText(waitForFile, ""); -// -// var allLogs = logs.JoinLogs(); -// -// allLogs.Should().NotContain("AllDone"); -// recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).LastException.Should().NotBeNull(); -// recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).Started.Should().Be(0); -// recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(0); -// -// // We must ensure all script are complete, otherwise if we shutdown tentacle while running a script the build can hang. -// // Ensure the script is finished by running the script again, the isolation mutex will ensure this second script runs after the first is complete. -// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); -// } -// -// [Test] -// [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] -// public async Task WhenANetworkFailureOccurs_DuringGetStatus_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// ScriptStatusRequest? scriptStatusRequest = null; -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithRetryDuration(TimeSpan.FromMinutes(4)) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var recordedUsages) -// .HookServiceMethod( -// nameof(IAsyncClientScriptService.GetStatusAsync), -// async (_, request) => -// { -// await Task.CompletedTask; -// -// scriptStatusRequest = request; -// if (recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).LastException == null) -// { -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// }) -// .Build()) -// .Build(CancellationToken); -// -// var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .Print("hello") -// .WaitForFileToExist(waitForFile) -// .Print("AllDone")) -// .Build(); -// -// var logs = new List(); -// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// // Let the script finish. -// File.WriteAllText(waitForFile, ""); -// var legacyTentacleClient = clientTentacle.LegacyTentacleClientBuilder().Build(); -// -// await Wait.For( -// async () => (await legacyTentacleClient.ScriptService.GetStatusAsync(scriptStatusRequest, new(CancellationToken))).State == ProcessState.Complete, -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Script Execution did not complete"), -// CancellationToken); -// -// var allLogs = logs.JoinLogs(); -// -// allLogs.Should().NotContain("AllDone"); -// recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).LastException.Should().NotBeNull(); -// recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(0); -// } -// -// [Test] -// [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] -// public async Task WhenANetworkFailureOccurs_DuringCancelScript_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// ScriptStatusRequest? scriptStatusRequest = null; -// CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithRetryDuration(TimeSpan.FromMinutes(4)) -// .WithPortForwarderDataLogging() -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var recordedUsages) -// .HookServiceMethod( -// nameof(IAsyncClientScriptService.GetStatusAsync), -// async (_, request) => -// { -// await Task.CompletedTask; -// -// cts.Cancel(); -// scriptStatusRequest = request; -// }) -// .HookServiceMethod( -// nameof(IAsyncClientScriptService.CancelScriptAsync), -// async (_, _) => -// { -// await Task.CompletedTask; -// -// if (recordedUsages.For(nameof(IAsyncClientScriptService.CancelScriptAsync)).LastException == null) -// { -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// }) -// .Build()) -// .Build(CancellationToken); -// -// var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .Print("hello") -// .WaitForFileToExist(waitForFile) -// .Print("AllDone")) -// .Build(); -// -// var logs = new List(); -// -// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, cts.Token); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// var allLogs = logs.JoinLogs(); -// allLogs.Should().NotContain("AllDone"); -// recordedUsages.For(nameof(IAsyncClientScriptService.CancelScriptAsync)).LastException.Should().NotBeNull(); -// recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptService.CancelScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(0); -// -// // Let the script finish. -// File.WriteAllText(waitForFile, ""); -// var legacyTentacleClient = clientTentacle.LegacyTentacleClientBuilder().Build(); -// -// await Wait.For( -// async () => (await legacyTentacleClient.ScriptService.GetStatusAsync(scriptStatusRequest, new(CancellationToken))).State == ProcessState.Complete, -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Script Execution did not complete"), -// CancellationToken); -// } -// -// [Test] -// [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] -// public async Task WhenANetworkFailureOccurs_DuringCompleteScript_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithPortForwarderDataLogging() -// .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) -// .WithRetryDuration(TimeSpan.FromMinutes(4)) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var recordedUsages) -// .HookServiceMethod( -// nameof(IAsyncClientScriptService.CompleteScriptAsync), -// async (_, _) => -// { -// await Task.CompletedTask; -// -// if (recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).LastException == null) -// { -// responseMessageTcpKiller.KillConnectionOnNextResponse(); -// } -// }) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(new ScriptBuilder().Print("hello")).Build(); -// -// var logs = new List(); -// var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); -// -// var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); -// -// await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); -// -// // We Can not verify what will be in the logs because of race conditions in tentacle. -// // The last complete script which we fail might come back with the logs. -// recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).LastException.Should().NotBeNull(); -// recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(1); -// } -// } -// } \ No newline at end of file +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] + public class ClientScriptExecutionScriptServiceV1IsNotRetried : IntegrationTest + { + [Test] + [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] + public async Task WhenANetworkFailureOccurs_DuringStartScript_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithRetryDuration(TimeSpan.FromMinutes(4)) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var recordedUsages) + .HookServiceMethod( + nameof(IAsyncClientScriptService.StartScriptAsync), + async (_, _) => + { + await Task.CompletedTask; + + if (recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).LastException == null) + { + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + }) + .Build()) + .Build(CancellationToken); + + var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .Print("hello") + .WaitForFileToExist(waitForFile) + .Print("AllDone")) + .WithIsolationLevel(ScriptIsolationLevel.FullIsolation) + .WithIsolationMutexName("bob") + .Build(); + + var logs = new List(); + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + // Let the script finish. + File.WriteAllText(waitForFile, ""); + + var allLogs = logs.JoinLogs(); + + allLogs.Should().NotContain("AllDone"); + recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).LastException.Should().NotBeNull(); + recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).Started.Should().Be(0); + recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(0); + + // We must ensure all script are complete, otherwise if we shutdown tentacle while running a script the build can hang. + // Ensure the script is finished by running the script again, the isolation mutex will ensure this second script runs after the first is complete. + await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, new List(), CancellationToken); + } + + [Test] + [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] + public async Task WhenANetworkFailureOccurs_DuringGetStatus_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + ScriptStatusRequest? scriptStatusRequest = null; + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithRetryDuration(TimeSpan.FromMinutes(4)) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var recordedUsages) + .HookServiceMethod( + nameof(IAsyncClientScriptService.GetStatusAsync), + async (_, request) => + { + await Task.CompletedTask; + + scriptStatusRequest = request; + if (recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).LastException == null) + { + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + }) + .Build()) + .Build(CancellationToken); + + var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .Print("hello") + .WaitForFileToExist(waitForFile) + .Print("AllDone")) + .Build(); + + var logs = new List(); + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + // Let the script finish. + File.WriteAllText(waitForFile, ""); + var legacyTentacleClient = clientTentacle.LegacyTentacleClientBuilder().Build(); + + await Wait.For( + async () => (await legacyTentacleClient.ScriptService.GetStatusAsync(scriptStatusRequest, new(CancellationToken))).State == ProcessState.Complete, + TimeSpan.FromSeconds(60), + () => throw new Exception("Script Execution did not complete"), + CancellationToken); + + var allLogs = logs.JoinLogs(); + + allLogs.Should().NotContain("AllDone"); + recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).LastException.Should().NotBeNull(); + recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptService.GetStatusAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(0); + } + + [Test] + [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] + public async Task WhenANetworkFailureOccurs_DuringCancelScript_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + ScriptStatusRequest? scriptStatusRequest = null; + CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithRetryDuration(TimeSpan.FromMinutes(4)) + .WithPortForwarderDataLogging() + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var recordedUsages) + .HookServiceMethod( + nameof(IAsyncClientScriptService.GetStatusAsync), + async (_, request) => + { + await Task.CompletedTask; + + cts.Cancel(); + scriptStatusRequest = request; + }) + .HookServiceMethod( + nameof(IAsyncClientScriptService.CancelScriptAsync), + async (_, _) => + { + await Task.CompletedTask; + + if (recordedUsages.For(nameof(IAsyncClientScriptService.CancelScriptAsync)).LastException == null) + { + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + }) + .Build()) + .Build(CancellationToken); + + var waitForFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "waitforme"); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .Print("hello") + .WaitForFileToExist(waitForFile) + .Print("AllDone")) + .Build(); + + var logs = new List(); + + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, cts.Token); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + var allLogs = logs.JoinLogs(); + allLogs.Should().NotContain("AllDone"); + recordedUsages.For(nameof(IAsyncClientScriptService.CancelScriptAsync)).LastException.Should().NotBeNull(); + recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptService.CancelScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(0); + + // Let the script finish. + File.WriteAllText(waitForFile, ""); + var legacyTentacleClient = clientTentacle.LegacyTentacleClientBuilder().Build(); + + await Wait.For( + async () => (await legacyTentacleClient.ScriptService.GetStatusAsync(scriptStatusRequest, new(CancellationToken))).State == ProcessState.Complete, + TimeSpan.FromSeconds(60), + () => throw new Exception("Script Execution did not complete"), + CancellationToken); + } + + [Test] + [TentacleConfigurations(testNoCapabilitiesServiceVersions: true)] + public async Task WhenANetworkFailureOccurs_DuringCompleteScript_WithATentacleThatOnlySupportsV1ScriptService_TheCallIsNotRetried(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithPortForwarderDataLogging() + .WithResponseMessageTcpKiller(out var responseMessageTcpKiller) + .WithRetryDuration(TimeSpan.FromMinutes(4)) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var recordedUsages) + .HookServiceMethod( + nameof(IAsyncClientScriptService.CompleteScriptAsync), + async (_, _) => + { + await Task.CompletedTask; + + if (recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).LastException == null) + { + responseMessageTcpKiller.KillConnectionOnNextResponse(); + } + }) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(new ScriptBuilder().Print("hello")).Build(); + + var logs = new List(); + var executeScriptTask = clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, logs, CancellationToken); + + var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientTentacle).Build(); + + await AssertionExtensions.Should(async () => await executeScriptTask).ThrowExceptionContractAsync(expectedException); + + // We Can not verify what will be in the logs because of race conditions in tentacle. + // The last complete script which we fail might come back with the logs. + recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).LastException.Should().NotBeNull(); + recordedUsages.For(nameof(IAsyncClientScriptService.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptService.CompleteScriptAsync)).Started.Should().Be(1); + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionWorksWithMultipleVersions.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionWorksWithMultipleVersions.cs index e10dcbfa4..c21d6bf59 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionWorksWithMultipleVersions.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionWorksWithMultipleVersions.cs @@ -1,39 +1,41 @@ -// using System; -// using System.Threading.Tasks; -// using FluentAssertions; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils.Builders; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class ClientScriptExecutionWorksWithMultipleVersions : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(testCommonVersions: true)] -// public async Task CanRunScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .Print("hello") -// .PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1)) -// .Print("AllDone")) -// .Build(); -// -// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// -// finalResponse.State.Should().Be(ProcessState.Complete); -// finalResponse.ExitCode.Should().Be(0); -// -// var allLogs = logs.JoinLogs(); -// -// allLogs.Should().MatchRegex(".*hello\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nAllDone"); -// } -// } -// } +using System; +using System.Threading.Tasks; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils.Builders; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] + public class ClientScriptExecutionWorksWithMultipleVersions : IntegrationTest + { + [Test] + [TentacleConfigurations(testCommonVersions: true)] + public async Task CanRunScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder().Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .Print("hello") + .PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1)) + .Print("AllDone")) + .Build(); + + var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = logs.JoinLogs(); + + allLogs.Should().MatchRegex(".*hello\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nAllDone"); + } + } +} diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs index 839741063..949d586ae 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs @@ -1,44 +1,44 @@ -// using System; -// using System.Threading.Tasks; -// using FluentAssertions; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils.Builders; -// using Octopus.Tentacle.Contracts.ClientServices; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] -// public class ClientScriptExecutorObservesScriptObserverBackoffStrategy : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(testCommonVersions: true)] -// public async Task TheScriptObserverBackoffShouldBeRespected(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithScriptObserverBackoffStrategy(new FuncScriptObserverBackoffStrategy(iters => TimeSpan.FromSeconds(20))) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder().PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1))) -// .Build(); -// -// var (_, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started -// .Should() -// .BeGreaterThan(0) -// .And -// .BeLessThan(3); -// } -// } -// } +using System; +using System.Threading.Tasks; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils.Builders; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] + public class ClientScriptExecutorObservesScriptObserverBackoffStrategy : IntegrationTest + { + [Test] + [TentacleConfigurations(testCommonVersions: true)] + public async Task TheScriptObserverBackoffShouldBeRespected(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithScriptObserverBackoffStrategy(new FuncScriptObserverBackoffStrategy(iters => TimeSpan.FromSeconds(20))) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder().PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1))) + .Build(); + + var (_, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started + .Should() + .BeGreaterThan(0) + .And + .BeLessThan(3); + } + } +} diff --git a/source/Octopus.Tentacle.Tests.Integration/MachineConfigurationHomeDirectoryTests.cs b/source/Octopus.Tentacle.Tests.Integration/MachineConfigurationHomeDirectoryTests.cs index b0d1558be..d6e0efe8a 100644 --- a/source/Octopus.Tentacle.Tests.Integration/MachineConfigurationHomeDirectoryTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/MachineConfigurationHomeDirectoryTests.cs @@ -1,54 +1,56 @@ -// using System; -// using System.IO; -// using System.Threading.Tasks; -// using FluentAssertions; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Variables; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class MachineConfigurationHomeDirectoryTests : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task ShouldUseTheDefaultMachineConfigurationHomeDirectoryWhenACustomLocationIsNotProvided(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using (var clientAndTentacle = await tentacleConfigurationTestCase -// .CreateBuilder() -// .UseDefaultMachineConfigurationHomeDirectory() -// .Build(CancellationToken)) -// { -// -// var defaultMachineConfigurationHomeDirectory = PlatformDetection.IsRunningOnWindows ? -// Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), "Octopus", "Tentacle", "Instances") : -// "/etc/octopus/Tentacle/Instances"; -// -// var expectedInstanceConfigurationFilePath = new FileInfo(Path.Combine(defaultMachineConfigurationHomeDirectory, $"{clientAndTentacle.RunningTentacle.InstanceName}.config")); -// -// expectedInstanceConfigurationFilePath.Exists.Should().BeTrue($"Instance configuration file should exist {expectedInstanceConfigurationFilePath.FullName}"); -// } -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] -// public async Task ShouldUseTheCustomMachineConfigurationHomeDirectoryWhenACustomLocationIsProvided(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// using var tempDirectory = new TemporaryDirectory(); -// await using var clientAndTentacle = await tentacleConfigurationTestCase -// .CreateBuilder() -// .UseDefaultMachineConfigurationHomeDirectory() -// .WithTentacle(x => -// { -// x.WithRunTentacleEnvironmentVariable(EnvironmentVariables.TentacleMachineConfigurationHomeDirectory, tempDirectory.DirectoryPath); -// }) -// .Build(CancellationToken); -// -// var expectedInstanceConfigurationFilePath = new FileInfo(Path.Combine(tempDirectory.DirectoryPath, "Tentacle", "Instances", $"{clientAndTentacle.RunningTentacle.InstanceName}.config")); -// -// expectedInstanceConfigurationFilePath.Exists.Should().BeTrue($"Instance configuration file should exist {expectedInstanceConfigurationFilePath.FullName}"); -// } -// } -// } +using System; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Variables; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] + public class MachineConfigurationHomeDirectoryTests : IntegrationTest + { + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task ShouldUseTheDefaultMachineConfigurationHomeDirectoryWhenACustomLocationIsNotProvided(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using (var clientAndTentacle = await tentacleConfigurationTestCase + .CreateBuilder() + .UseDefaultMachineConfigurationHomeDirectory() + .Build(CancellationToken)) + { + + var defaultMachineConfigurationHomeDirectory = PlatformDetection.IsRunningOnWindows ? + Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), "Octopus", "Tentacle", "Instances") : + "/etc/octopus/Tentacle/Instances"; + + var expectedInstanceConfigurationFilePath = new FileInfo(Path.Combine(defaultMachineConfigurationHomeDirectory, $"{clientAndTentacle.RunningTentacle.InstanceName}.config")); + + expectedInstanceConfigurationFilePath.Exists.Should().BeTrue($"Instance configuration file should exist {expectedInstanceConfigurationFilePath.FullName}"); + } + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)] + public async Task ShouldUseTheCustomMachineConfigurationHomeDirectoryWhenACustomLocationIsProvided(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + using var tempDirectory = new TemporaryDirectory(); + await using var clientAndTentacle = await tentacleConfigurationTestCase + .CreateBuilder() + .UseDefaultMachineConfigurationHomeDirectory() + .WithTentacle(x => + { + x.WithRunTentacleEnvironmentVariable(EnvironmentVariables.TentacleMachineConfigurationHomeDirectory, tempDirectory.DirectoryPath); + }) + .Build(CancellationToken); + + var expectedInstanceConfigurationFilePath = new FileInfo(Path.Combine(tempDirectory.DirectoryPath, "Tentacle", "Instances", $"{clientAndTentacle.RunningTentacle.InstanceName}.config")); + + expectedInstanceConfigurationFilePath.Exists.Should().BeTrue($"Instance configuration file should exist {expectedInstanceConfigurationFilePath.FullName}"); + } + } +} diff --git a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceTests.cs b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceTests.cs index 7d1847411..8822766da 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceTests.cs @@ -1,136 +1,138 @@ -// using System; -// using System.Diagnostics; -// using System.Linq; -// using System.Threading.Tasks; -// using FluentAssertions; -// using Halibut.Logging; -// using NUnit.Framework; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Support.Legacy; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class ScriptServiceTests : IntegrationTest -// { -// [Test] -// [TentacleConfigurations] -// public async Task RunScriptWithSuccess(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var windowsScript = @" -// Write-Host ""This is the start of the script"" -// Write-Host ""The answer is"" (6 * 7) -// Start-Sleep -Seconds 3 -// Write-Host ""This is the end of the script"""; -// -// var nixScript = @" -// echo This is the start of the script -// val=6 -// ((theAnswer=$val*7)) -// echo The answer is $theAnswer -// sleep 3 -// echo This is the end of the script"; -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); -// -// var scriptStatusResponse = await new ScriptExecutionOrchestrator(clientAndTentacle.TentacleClient, Logger) -// .ExecuteScript(windowsScript, nixScript, CancellationToken); -// -// DumpLog(scriptStatusResponse); -// -// scriptStatusResponse.State.Should().Be(ProcessState.Complete); -// scriptStatusResponse.ExitCode.Should().Be(0); -// scriptStatusResponse.Logs.Select(x => x.Text).Should().Contain("The answer is 42"); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task RunScriptWithErrors(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var windowsScript = @" -// Write-Host ""This is the start of the script"" -// Start-Sleep -Seconds 3 -// throw ""Whoopsy Daisy!"" -// Write-Host ""This is the end of the script"""; -// -// var nixScript = @" -// echo This is the start of the script -// sleep 3 -// echo ""Whoopsy Daisy!"" -// exit 1 -// echo This is the end of the script"; -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); -// -// var scriptStatusResponse = await new ScriptExecutionOrchestrator(clientAndTentacle.TentacleClient, Logger) -// .ExecuteScript(windowsScript, nixScript, CancellationToken); -// -// DumpLog(scriptStatusResponse); -// -// scriptStatusResponse.State.Should().Be(ProcessState.Complete); -// scriptStatusResponse.ExitCode.Should().NotBe(0); -// scriptStatusResponse.Logs.Select(x => x.Text).Should().Contain("Whoopsy Daisy!"); -// scriptStatusResponse.Logs.Select(x => x.Text).Should().NotContain("This is the end of the script"); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task CancelScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var windowsScript = @"Write-Host ""This is the start of the script"" -// & ping.exe 127.0.0.1 -n 100 -// Write-Host ""This is the end of the script"""; -// -// var nixScript = @"echo This is the start of the script -// ping 127.0.0.1 -c 100 -// echo This is the end of the script"; -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase -// .CreateLegacyBuilder() -// .WithHalibutLoggingLevel(LogLevel.Trace) -// .Build(CancellationToken); -// -// var scriptExecutor = new ScriptExecutionOrchestrator(clientAndTentacle.TentacleClient, Logger); -// -// Logger.Information("Starting script execution"); -// var ticket = await scriptExecutor.StartScript(windowsScript, nixScript, CancellationToken); -// -// // Possible Tentacle BUG: If we just observe until the first output is received then sometimes the script will fail to Cancel -// await scriptExecutor.ObserverUntilScriptOutputReceived(ticket, "This is the start of the script", CancellationToken); -// -// Logger.Information("Cancelling script execution"); -// await clientAndTentacle.TentacleClient.ScriptService.CancelScriptAsync(new CancelScriptCommand(ticket, 0), new(CancellationToken)); -// -// var cancellationDuration = Stopwatch.StartNew(); -// -// Logger.Information("Waiting for Script Execution to complete"); -// var finalScriptStatusResponse = await scriptExecutor.ObserverUntilComplete(ticket, CancellationToken); -// cancellationDuration.Stop(); -// -// Logger.Information("Completing script execution"); -// var finalStatus = await scriptExecutor.CompleteScript(finalScriptStatusResponse, CancellationToken); -// -// DumpLog(finalStatus); -// -// finalStatus.State.Should().Be(ProcessState.Complete); -// finalStatus.ExitCode.Should().NotBe(0, "Expected ExitCode to be non-zero"); -// finalStatus.Logs.Count.Should().BeGreaterThan(0, "Expected something in the logs"); -// -// finalStatus.Logs.Select(x => x.Text).Should().Contain("This is the start of the script"); -// finalStatus.Logs.Select(x => x.Text).Should().NotContain("This is the end of the script"); -// cancellationDuration.Elapsed.TotalSeconds.Should().BeLessThanOrEqualTo(20); -// } -// -// private static void DumpLog(ScriptStatusResponse finalStatus) -// { -// Console.WriteLine("### Start of script result logs ###"); -// foreach (var log in finalStatus.Logs) -// { -// Console.WriteLine(log.Text); -// } -// -// Console.WriteLine("### End of script result logs ###"); -// } -// } -// } +using System; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Halibut.Logging; +using NUnit.Framework; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.Legacy; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] + public class ScriptServiceTests : IntegrationTest + { + [Test] + [TentacleConfigurations] + public async Task RunScriptWithSuccess(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var windowsScript = @" + Write-Host ""This is the start of the script"" + Write-Host ""The answer is"" (6 * 7) + Start-Sleep -Seconds 3 + Write-Host ""This is the end of the script"""; + + var nixScript = @" + echo This is the start of the script + val=6 + ((theAnswer=$val*7)) + echo The answer is $theAnswer + sleep 3 + echo This is the end of the script"; + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); + + var scriptStatusResponse = await new ScriptExecutionOrchestrator(clientAndTentacle.TentacleClient, Logger) + .ExecuteScript(windowsScript, nixScript, CancellationToken); + + DumpLog(scriptStatusResponse); + + scriptStatusResponse.State.Should().Be(ProcessState.Complete); + scriptStatusResponse.ExitCode.Should().Be(0); + scriptStatusResponse.Logs.Select(x => x.Text).Should().Contain("The answer is 42"); + } + + [Test] + [TentacleConfigurations] + public async Task RunScriptWithErrors(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var windowsScript = @" + Write-Host ""This is the start of the script"" + Start-Sleep -Seconds 3 + throw ""Whoopsy Daisy!"" + Write-Host ""This is the end of the script"""; + + var nixScript = @" + echo This is the start of the script + sleep 3 + echo ""Whoopsy Daisy!"" + exit 1 + echo This is the end of the script"; + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateLegacyBuilder().Build(CancellationToken); + + var scriptStatusResponse = await new ScriptExecutionOrchestrator(clientAndTentacle.TentacleClient, Logger) + .ExecuteScript(windowsScript, nixScript, CancellationToken); + + DumpLog(scriptStatusResponse); + + scriptStatusResponse.State.Should().Be(ProcessState.Complete); + scriptStatusResponse.ExitCode.Should().NotBe(0); + scriptStatusResponse.Logs.Select(x => x.Text).Should().Contain("Whoopsy Daisy!"); + scriptStatusResponse.Logs.Select(x => x.Text).Should().NotContain("This is the end of the script"); + } + + [Test] + [TentacleConfigurations] + public async Task CancelScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var windowsScript = @"Write-Host ""This is the start of the script"" + & ping.exe 127.0.0.1 -n 100 + Write-Host ""This is the end of the script"""; + + var nixScript = @"echo This is the start of the script + ping 127.0.0.1 -c 100 + echo This is the end of the script"; + + await using var clientAndTentacle = await tentacleConfigurationTestCase + .CreateLegacyBuilder() + .WithHalibutLoggingLevel(LogLevel.Trace) + .Build(CancellationToken); + + var scriptExecutor = new ScriptExecutionOrchestrator(clientAndTentacle.TentacleClient, Logger); + + Logger.Information("Starting script execution"); + var ticket = await scriptExecutor.StartScript(windowsScript, nixScript, CancellationToken); + + // Possible Tentacle BUG: If we just observe until the first output is received then sometimes the script will fail to Cancel + await scriptExecutor.ObserverUntilScriptOutputReceived(ticket, "This is the start of the script", CancellationToken); + + Logger.Information("Cancelling script execution"); + await clientAndTentacle.TentacleClient.ScriptService.CancelScriptAsync(new CancelScriptCommand(ticket, 0), new(CancellationToken)); + + var cancellationDuration = Stopwatch.StartNew(); + + Logger.Information("Waiting for Script Execution to complete"); + var finalScriptStatusResponse = await scriptExecutor.ObserverUntilComplete(ticket, CancellationToken); + cancellationDuration.Stop(); + + Logger.Information("Completing script execution"); + var finalStatus = await scriptExecutor.CompleteScript(finalScriptStatusResponse, CancellationToken); + + DumpLog(finalStatus); + + finalStatus.State.Should().Be(ProcessState.Complete); + finalStatus.ExitCode.Should().NotBe(0, "Expected ExitCode to be non-zero"); + finalStatus.Logs.Count.Should().BeGreaterThan(0, "Expected something in the logs"); + + finalStatus.Logs.Select(x => x.Text).Should().Contain("This is the start of the script"); + finalStatus.Logs.Select(x => x.Text).Should().NotContain("This is the end of the script"); + cancellationDuration.Elapsed.TotalSeconds.Should().BeLessThanOrEqualTo(20); + } + + private static void DumpLog(ScriptStatusResponse finalStatus) + { + Console.WriteLine("### Start of script result logs ###"); + foreach (var log in finalStatus.Logs) + { + Console.WriteLine(log.Text); + } + + Console.WriteLine("### End of script result logs ###"); + } + } +} diff --git a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs index ef9bdeaba..760b9209c 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs @@ -1,216 +1,216 @@ -// using System; -// using System.Diagnostics; -// using System.Threading; -// using System.Threading.Tasks; -// using FluentAssertions; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils.Builders; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Contracts.ClientServices; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class ScriptServiceV2IntegrationTest : IntegrationTest -// { -// [Test] -// [TentacleConfigurations] -// public async Task CanRunScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var methodUsages) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .Print("Lets do it") -// .PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1)) -// .Print("All done")) -// .Build(); -// -// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// -// finalResponse.State.Should().Be(ProcessState.Complete); -// finalResponse.ExitCode.Should().Be(0); -// -// var allLogs = logs.JoinLogs(); -// -// allLogs.Should().MatchRegex(".*Lets do it\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nAll done.*"); -// -// methodUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); -// methodUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterThan(2).And.BeLessThan(30); -// methodUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); -// methodUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task DelayInStartScriptSavesNetworkCalls(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var recordedUsages) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .Print("Lets do it") -// .PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1)) -// .Print("All done")) -// .WithDurationStartScriptCanWaitForScriptToFinish(TimeSpan.FromMinutes(1)) -// .Build(); -// -// var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// -// finalResponse.State.Should().Be(ProcessState.Complete); -// finalResponse.ExitCode.Should().Be(0); -// -// var allLogs = logs.JoinLogs(); -// -// allLogs.Should().MatchRegex(".*Lets do it\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nAll done.*"); -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0, "Since start script should wait for the script to finish so we don't need to call get status"); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task WhenTentacleRestartsWhileRunningAScript_TheExitCodeShouldBe_UnknownResultExitCode(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var recordedUsages) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .Print("hello") -// .Sleep(TimeSpan.FromSeconds(1)) -// .Print("waitingtobestopped") -// .Sleep(TimeSpan.FromSeconds(100))) -// .Build(); -// -// var semaphoreSlim = new SemaphoreSlim(0, 1); -// -// var executingScript = Task.Run(async () => -// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, onScriptStatusResponseReceived => -// { -// if (onScriptStatusResponseReceived.Logs.JoinLogs().Contains("waitingtobestopped")) -// { -// semaphoreSlim.Release(); -// } -// })); -// -// await semaphoreSlim.WaitAsync(CancellationToken); -// -// Logger.Information("Stopping and starting tentacle now."); -// await clientTentacle.RunningTentacle.Restart(CancellationToken); -// -// var (finalResponse, logs) = await executingScript; -// -// finalResponse.Should().NotBeNull(); -// logs.JoinLogs().Should().Contain("waitingtobestopped"); -// finalResponse.State.Should().Be(ProcessState.Complete); // This is technically a lie, the process is still running on linux -// finalResponse.ExitCode.Should().Be(ScriptExitCodes.UnknownResultExitCode); -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterThan(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); -// } -// -// [Test] -// [TentacleConfigurations] -// [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] -// public async Task WhenALongRunningScriptIsCancelled_TheScriptShouldStop(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(out var recordedUsages) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder() -// .Print("hello") -// .Sleep(TimeSpan.FromSeconds(1)) -// .Print("waitingtobestopped") -// .Sleep(TimeSpan.FromSeconds(100))) -// .Build(); -// -// var scriptCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); -// var stopWatch = Stopwatch.StartNew(); -// Exception? actualException = null; -// -// try -// { -// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, scriptCancellationTokenSource.Token, onScriptStatusResponseReceived => -// { -// if (onScriptStatusResponseReceived.Logs.JoinLogs().Contains("waitingtobestopped")) -// { -// scriptCancellationTokenSource.Cancel(); -// } -// }); -// } -// catch (Exception ex) -// { -// actualException = ex; -// } -// -// stopWatch.Stop(); -// -// actualException.Should().NotBeNull().And.BeOfType().And.Match(x => x.Message == "Script execution was cancelled"); -// stopWatch.Elapsed.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(10)); -// -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterThan(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().BeGreaterThanOrEqualTo(1); -// } -// -// [Test] -// [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.Version2)] -// public async Task WhenOnCompleteTakesLongerThan_OnCancellationAbandonCompleteScriptAfter_AndTheExecutionIsNotCancelled_TheOrchestratorWaitsForCleanUpToComplete(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// bool calledWithNonCancelledCT = false; -// -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .DecorateScriptServiceV2With(b => b -// .BeforeCompleteScript(async (_, _, halibutProxyRequestOptions) => -// { -// await Task.Delay(TimeSpan.FromSeconds(2), CancellationToken); -// calledWithNonCancelledCT = !halibutProxyRequestOptions.RequestCancellationToken.IsCancellationRequested; -// }) -// .Build()) -// .Build()) -// .Build(CancellationToken); -// -// var scriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b.Print("Hello")) -// .Build(); -// -// var tentacleClient = clientTentacle.TentacleClient; -// -// tentacleClient.OnCancellationAbandonCompleteScriptAfter = TimeSpan.FromMilliseconds(1); -// -// await tentacleClient.ExecuteScript(scriptCommand, CancellationToken); -// -// -// calledWithNonCancelledCT.Should().Be(true); -// } -// } -// } +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils.Builders; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] + public class ScriptServiceV2IntegrationTest : IntegrationTest + { + [Test] + [TentacleConfigurations] + public async Task CanRunScript(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var methodUsages) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .Print("Lets do it") + .PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1)) + .Print("All done")) + .Build(); + + var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = logs.JoinLogs(); + + allLogs.Should().MatchRegex(".*Lets do it\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nAll done.*"); + + methodUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); + methodUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterThan(2).And.BeLessThan(30); + methodUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); + methodUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); + } + + [Test] + [TentacleConfigurations] + public async Task DelayInStartScriptSavesNetworkCalls(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var recordedUsages) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .Print("Lets do it") + .PrintNTimesWithDelay("another one", 10, TimeSpan.FromSeconds(1)) + .Print("All done")) + .WithDurationStartScriptCanWaitForScriptToFinish(TimeSpan.FromMinutes(1)) + .Build(); + + var (finalResponse, logs) = await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = logs.JoinLogs(); + + allLogs.Should().MatchRegex(".*Lets do it\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nanother one\nAll done.*"); + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().Be(0, "Since start script should wait for the script to finish so we don't need to call get status"); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); + } + + [Test] + [TentacleConfigurations] + public async Task WhenTentacleRestartsWhileRunningAScript_TheExitCodeShouldBe_UnknownResultExitCode(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var recordedUsages) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .Print("hello") + .Sleep(TimeSpan.FromSeconds(1)) + .Print("waitingtobestopped") + .Sleep(TimeSpan.FromSeconds(100))) + .Build(); + + var semaphoreSlim = new SemaphoreSlim(0, 1); + + var executingScript = Task.Run(async () => + await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, onScriptStatusResponseReceived => + { + if (onScriptStatusResponseReceived.Logs.JoinLogs().Contains("waitingtobestopped")) + { + semaphoreSlim.Release(); + } + })); + + await semaphoreSlim.WaitAsync(CancellationToken); + + Logger.Information("Stopping and starting tentacle now."); + await clientTentacle.RunningTentacle.Restart(CancellationToken); + + var (finalResponse, logs) = await executingScript; + + finalResponse.Should().NotBeNull(); + logs.JoinLogs().Should().Contain("waitingtobestopped"); + finalResponse.State.Should().Be(ProcessState.Complete); // This is technically a lie, the process is still running on linux + finalResponse.ExitCode.Should().Be(ScriptExitCodes.UnknownResultExitCode); + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterThan(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().Be(0); + } + + [Test] + [TentacleConfigurations] + public async Task WhenALongRunningScriptIsCancelled_TheScriptShouldStop(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(out var recordedUsages) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder() + .Print("hello") + .Sleep(TimeSpan.FromSeconds(1)) + .Print("waitingtobestopped") + .Sleep(TimeSpan.FromSeconds(100))) + .Build(); + + var scriptCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(CancellationToken); + var stopWatch = Stopwatch.StartNew(); + Exception? actualException = null; + + try + { + await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, scriptCancellationTokenSource.Token, onScriptStatusResponseReceived => + { + if (onScriptStatusResponseReceived.Logs.JoinLogs().Contains("waitingtobestopped")) + { + scriptCancellationTokenSource.Cancel(); + } + }); + } + catch (Exception ex) + { + actualException = ex; + } + + stopWatch.Stop(); + + actualException.Should().NotBeNull().And.BeOfType().And.Match(x => x.Message == "Script execution was cancelled"); + stopWatch.Elapsed.Should().BeLessOrEqualTo(TimeSpan.FromSeconds(10)); + + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.StartScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.GetStatusAsync)).Started.Should().BeGreaterThan(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CancelScriptAsync)).Started.Should().BeGreaterThanOrEqualTo(1); + } + + [Test] + [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.Version2)] + public async Task WhenOnCompleteTakesLongerThan_OnCancellationAbandonCompleteScriptAfter_AndTheExecutionIsNotCancelled_TheOrchestratorWaitsForCleanUpToComplete(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + bool calledWithNonCancelledCT = false; + + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .DecorateScriptServiceV2With(b => b + .BeforeCompleteScript(async (_, _, halibutProxyRequestOptions) => + { + await Task.Delay(TimeSpan.FromSeconds(2), CancellationToken); + calledWithNonCancelledCT = !halibutProxyRequestOptions.RequestCancellationToken.IsCancellationRequested; + }) + .Build()) + .Build()) + .Build(CancellationToken); + + var scriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b.Print("Hello")) + .Build(); + + var tentacleClient = clientTentacle.TentacleClient; + + tentacleClient.OnCancellationAbandonCompleteScriptAfter = TimeSpan.FromMilliseconds(1); + + await tentacleClient.ExecuteScript(scriptCommand, CancellationToken); + + + calledWithNonCancelledCT.Should().Be(true); + } + } +} diff --git a/source/Octopus.Tentacle.Tests.Integration/TentacleClientObserver.cs b/source/Octopus.Tentacle.Tests.Integration/TentacleClientObserver.cs index 1bc2b8a81..e1746c7a5 100644 --- a/source/Octopus.Tentacle.Tests.Integration/TentacleClientObserver.cs +++ b/source/Octopus.Tentacle.Tests.Integration/TentacleClientObserver.cs @@ -1,160 +1,162 @@ -// using System; -// using System.IO; -// using System.Threading.Tasks; -// using FluentAssertions; -// using Halibut; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils.Builders; -// using Octopus.Tentacle.Contracts.ClientServices; -// using Octopus.Tentacle.Contracts.Logging; -// using Octopus.Tentacle.Contracts.Observability; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class TentacleClientObserver : IntegrationTest -// { -// [Test] -// [TentacleConfigurations] -// public async Task AnErrorDuringTheCallbackTo_ExecuteScriptCompleted_ShouldNotThrowOrStopScriptExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// // Arrange -// var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnExecuteScriptCompleted: true); -// -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleClientObserver(tentacleClientObserver) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b.Print("Hello")) -// .Build(); -// -// // Act -// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// -// // Assert -// // We should have completed the script and not failed due to the error thrown by the TentacleClientObserver -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task AnErrorDuringTheCallbackTo_RpcCallComplete_ShouldNotThrowOrStopScriptExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// // Arrange -// var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnRpcCallCompleted: true); -// -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleClientObserver(tentacleClientObserver) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) -// .Build()) -// .Build(CancellationToken); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(b => b.Print("Hello")) -// .Build(); -// -// // Act -// await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// -// // Assert -// // We should have completed the script and not failed due to the error thrown by the TentacleClientObserver -// recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task AnErrorDuringTheCallbackTo_UploadFileCompleted_ShouldNotThrowAnExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// // Arrange -// var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnUploadFileCompleted: true); -// -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleClientObserver(tentacleClientObserver) -// .Build(CancellationToken); -// -// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); -// -// // Act + Assert -// await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); -// } -// -// [Test] -// [TentacleConfigurations] -// public async Task AnErrorDuringTheCallbackTo_DownloadFileCompleted_ShouldNotThrowAnExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// // Arrange -// var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnDownloadFileCompleted: true); -// -// await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacleClientObserver(tentacleClientObserver) -// .Build(CancellationToken); -// -// var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "DownloadFile.txt"); -// await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); -// -// // Act + Assert -// await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); -// } -// -// class BrokenTentacleClientObserver : ITentacleClientObserver -// { -// private readonly bool errorOnRpcCallCompleted; -// private readonly bool errorOnUploadFileCompleted; -// private readonly bool errorOnDownloadFileCompleted; -// private readonly bool errorOnExecuteScriptCompleted; -// -// public BrokenTentacleClientObserver( -// bool errorOnRpcCallCompleted = false, -// bool errorOnUploadFileCompleted = false, -// bool errorOnDownloadFileCompleted = false, -// bool errorOnExecuteScriptCompleted = false) -// { -// this.errorOnRpcCallCompleted = errorOnRpcCallCompleted; -// this.errorOnUploadFileCompleted = errorOnUploadFileCompleted; -// this.errorOnDownloadFileCompleted = errorOnDownloadFileCompleted; -// this.errorOnExecuteScriptCompleted = errorOnExecuteScriptCompleted; -// } -// -// public void RpcCallCompleted(RpcCallMetrics metrics, ITentacleClientTaskLog logger) -// { -// if (errorOnRpcCallCompleted) -// { -// throw new Exception($"RpcCallCompleted {Guid.NewGuid()}"); -// } -// } -// -// public void UploadFileCompleted(ClientOperationMetrics clientOperationMetrics, ITentacleClientTaskLog logger) -// { -// if (errorOnUploadFileCompleted) -// { -// throw new Exception($"UploadFileCompleted {Guid.NewGuid()}"); -// } -// } -// -// public void DownloadFileCompleted(ClientOperationMetrics clientOperationMetrics, ITentacleClientTaskLog logger) -// { -// if (errorOnDownloadFileCompleted) -// { -// throw new Exception($"DownloadFileCompleted {Guid.NewGuid()}"); -// } -// } -// -// public void ExecuteScriptCompleted(ClientOperationMetrics clientOperationMetrics, ITentacleClientTaskLog logger) -// { -// if (errorOnExecuteScriptCompleted) -// { -// throw new Exception($"ExecuteScriptCompleted {Guid.NewGuid()}"); -// } -// } -// } -// } -// } +using System; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using Halibut; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils.Builders; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Contracts.Logging; +using Octopus.Tentacle.Contracts.Observability; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] + public class TentacleClientObserver : IntegrationTest + { + [Test] + [TentacleConfigurations] + public async Task AnErrorDuringTheCallbackTo_ExecuteScriptCompleted_ShouldNotThrowOrStopScriptExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + // Arrange + var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnExecuteScriptCompleted: true); + + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleClientObserver(tentacleClientObserver) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b.Print("Hello")) + .Build(); + + // Act + await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + + // Assert + // We should have completed the script and not failed due to the error thrown by the TentacleClientObserver + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); + } + + [Test] + [TentacleConfigurations] + public async Task AnErrorDuringTheCallbackTo_RpcCallComplete_ShouldNotThrowOrStopScriptExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + // Arrange + var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnRpcCallCompleted: true); + + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleClientObserver(tentacleClientObserver) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .RecordMethodUsages(tentacleConfigurationTestCase, out var recordedUsages) + .Build()) + .Build(CancellationToken); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(b => b.Print("Hello")) + .Build(); + + // Act + await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + + // Assert + // We should have completed the script and not failed due to the error thrown by the TentacleClientObserver + recordedUsages.For(nameof(IAsyncClientScriptServiceV2.CompleteScriptAsync)).Started.Should().Be(1); + } + + [Test] + [TentacleConfigurations] + public async Task AnErrorDuringTheCallbackTo_UploadFileCompleted_ShouldNotThrowAnExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + // Arrange + var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnUploadFileCompleted: true); + + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleClientObserver(tentacleClientObserver) + .Build(CancellationToken); + + var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "UploadFile.txt"); + + // Act + Assert + await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); + } + + [Test] + [TentacleConfigurations] + public async Task AnErrorDuringTheCallbackTo_DownloadFileCompleted_ShouldNotThrowAnExecution(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + // Arrange + var tentacleClientObserver = new BrokenTentacleClientObserver(errorOnDownloadFileCompleted: true); + + await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacleClientObserver(tentacleClientObserver) + .Build(CancellationToken); + + var remotePath = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "DownloadFile.txt"); + await clientTentacle.TentacleClient.UploadFile(remotePath, DataStream.FromString("Hello"), CancellationToken); + + // Act + Assert + await clientTentacle.TentacleClient.DownloadFile(remotePath, CancellationToken); + } + + class BrokenTentacleClientObserver : ITentacleClientObserver + { + private readonly bool errorOnRpcCallCompleted; + private readonly bool errorOnUploadFileCompleted; + private readonly bool errorOnDownloadFileCompleted; + private readonly bool errorOnExecuteScriptCompleted; + + public BrokenTentacleClientObserver( + bool errorOnRpcCallCompleted = false, + bool errorOnUploadFileCompleted = false, + bool errorOnDownloadFileCompleted = false, + bool errorOnExecuteScriptCompleted = false) + { + this.errorOnRpcCallCompleted = errorOnRpcCallCompleted; + this.errorOnUploadFileCompleted = errorOnUploadFileCompleted; + this.errorOnDownloadFileCompleted = errorOnDownloadFileCompleted; + this.errorOnExecuteScriptCompleted = errorOnExecuteScriptCompleted; + } + + public void RpcCallCompleted(RpcCallMetrics metrics, ITentacleClientTaskLog logger) + { + if (errorOnRpcCallCompleted) + { + throw new Exception($"RpcCallCompleted {Guid.NewGuid()}"); + } + } + + public void UploadFileCompleted(ClientOperationMetrics clientOperationMetrics, ITentacleClientTaskLog logger) + { + if (errorOnUploadFileCompleted) + { + throw new Exception($"UploadFileCompleted {Guid.NewGuid()}"); + } + } + + public void DownloadFileCompleted(ClientOperationMetrics clientOperationMetrics, ITentacleClientTaskLog logger) + { + if (errorOnDownloadFileCompleted) + { + throw new Exception($"DownloadFileCompleted {Guid.NewGuid()}"); + } + } + + public void ExecuteScriptCompleted(ClientOperationMetrics clientOperationMetrics, ITentacleClientTaskLog logger) + { + if (errorOnExecuteScriptCompleted) + { + throw new Exception($"ExecuteScriptCompleted {Guid.NewGuid()}"); + } + } + } + } +} diff --git a/source/Octopus.Tentacle.Tests.Integration/TentacleStartupAndShutdownTests.cs b/source/Octopus.Tentacle.Tests.Integration/TentacleStartupAndShutdownTests.cs index 9eaa56113..7edfd79a0 100644 --- a/source/Octopus.Tentacle.Tests.Integration/TentacleStartupAndShutdownTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/TentacleStartupAndShutdownTests.cs @@ -1,72 +1,73 @@ -// #nullable enable -// using System.Collections.Generic; -// using System.Linq; -// using System.Threading.Tasks; -// using FluentAssertions; -// using Halibut.Exceptions; -// using NUnit.Framework; -// using Octopus.Tentacle.Client.Scripts; -// using Octopus.Tentacle.CommonTestUtils.Builders; -// using Octopus.Tentacle.Contracts; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; -// using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class TentacleStartupAndShutdownTests : IntegrationTest -// { -// [Test] -// [TentacleConfigurations] -// [RequiresSudoOnLinux] -// public async Task WhenRunningTentacleAsAServiceItShouldBeAbleToRestartItself(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// await using (var clientAndTentacle = await tentacleConfigurationTestCase -// .CreateBuilder() -// .InstallAsAService() -// .Build(CancellationToken)) -// { -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBodyForCurrentOs( -// $@"cd ""{clientAndTentacle.RunningTentacle.TentacleExe.DirectoryName}"" -// .\Tentacle.exe service --instance {clientAndTentacle.RunningTentacle.InstanceName} --stop --start", -// $@"#!/bin/sh -// cd ""{clientAndTentacle.RunningTentacle.TentacleExe.DirectoryName}"" -// ./Tentacle service --instance {clientAndTentacle.RunningTentacle.InstanceName} --stop --start") -// .Build(); -// -// (Client.Scripts.Models.ScriptExecutionResult ScriptExecutionResult, List ProcessOutput) result; -// -// try -// { -// result = await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// } -// catch (ServiceInvocationHalibutClientException ex) -// { -// Logger.Information(ex, "ServiceInvocationHalibutClientException thrown while Tentacle was restarting itself. This can be ignored for the purpose of this test."); -// -// // Making Tentacle restart itself can cause internal errors with Script Service -// // Execute the script again to get the final result and logs. This will not rerun the script. -// result = await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// } -// -// result.LogExecuteScriptOutput(Logger); -// -// result.ProcessOutput.Any(x => x.Text.Contains("Stopping service")).Should().BeTrue("Stopping service should be logged"); -// result.ScriptExecutionResult.State.Should().Be(ProcessState.Complete); -// -// startScriptCommand = new TestExecuteShellScriptCommandBuilder() -// .SetScriptBody(new ScriptBuilder().Print("Running...")) -// .Build(); -// -// result = await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); -// result.ProcessOutput.Any(x => x.Text.Contains("Running...")).Should().BeTrue("Running... should be logged"); -// result.ScriptExecutionResult.ExitCode.Should().Be(0); -// result.ScriptExecutionResult.State.Should().Be(ProcessState.Complete); -// } -// } -// } -// } +#nullable enable +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Halibut.Exceptions; +using NUnit.Framework; +using Octopus.Tentacle.Client.Scripts; +using Octopus.Tentacle.CommonTestUtils.Builders; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util.Builders; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] + public class TentacleStartupAndShutdownTests : IntegrationTest + { + [Test] + [TentacleConfigurations] + [RequiresSudoOnLinux] + public async Task WhenRunningTentacleAsAServiceItShouldBeAbleToRestartItself(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + await using (var clientAndTentacle = await tentacleConfigurationTestCase + .CreateBuilder() + .InstallAsAService() + .Build(CancellationToken)) + { + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBodyForCurrentOs( +$@"cd ""{clientAndTentacle.RunningTentacle.TentacleExe.DirectoryName}"" +.\Tentacle.exe service --instance {clientAndTentacle.RunningTentacle.InstanceName} --stop --start", +$@"#!/bin/sh +cd ""{clientAndTentacle.RunningTentacle.TentacleExe.DirectoryName}"" +./Tentacle service --instance {clientAndTentacle.RunningTentacle.InstanceName} --stop --start") + .Build(); + + (Client.Scripts.Models.ScriptExecutionResult ScriptExecutionResult, List ProcessOutput) result; + + try + { + result = await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + } + catch (ServiceInvocationHalibutClientException ex) + { + Logger.Information(ex, "ServiceInvocationHalibutClientException thrown while Tentacle was restarting itself. This can be ignored for the purpose of this test."); + + // Making Tentacle restart itself can cause internal errors with Script Service + // Execute the script again to get the final result and logs. This will not rerun the script. + result = await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + } + + result.LogExecuteScriptOutput(Logger); + + result.ProcessOutput.Any(x => x.Text.Contains("Stopping service")).Should().BeTrue("Stopping service should be logged"); + result.ScriptExecutionResult.State.Should().Be(ProcessState.Complete); + + startScriptCommand = new TestExecuteShellScriptCommandBuilder() + .SetScriptBody(new ScriptBuilder().Print("Running...")) + .Build(); + + result = await clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken); + result.ProcessOutput.Any(x => x.Text.Contains("Running...")).Should().BeTrue("Running... should be logged"); + result.ScriptExecutionResult.ExitCode.Should().Be(0); + result.ScriptExecutionResult.State.Should().Be(ProcessState.Complete); + } + } + } +} diff --git a/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs b/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs index 378f1e2e9..71ca628e8 100644 --- a/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/WorkspaceCleanerTests.cs @@ -1,257 +1,259 @@ -// using System; -// using System.IO; -// using System.Threading.Tasks; -// using FluentAssertions; -// using NUnit.Framework; -// using Octopus.Tentacle.CommonTestUtils; -// using Octopus.Tentacle.CommonTestUtils.Diagnostics; -// using Octopus.Tentacle.Scripts; -// using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; -// using Octopus.Tentacle.Tests.Integration.Support; -// using Octopus.Tentacle.Tests.Integration.Util; -// using Octopus.Tentacle.Tests.Integration.Util.Builders; -// using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; -// -// namespace Octopus.Tentacle.Tests.Integration -// { -// [IntegrationTestTimeout] -// public class WorkspaceCleanerTests : IntegrationTest -// { -// [Test] -// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] -// public async Task WhenScriptServiceIsRunningAndWritesLogFile_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var cleanerDelay = TimeSpan.FromMilliseconds(500); -// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); -// -// var existingHomeDirectory = new TemporaryDirectory(); -// -// var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); -// var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacle(b => -// { -// b.WithHomeDirectory(existingHomeDirectory) -// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); -// }) -// .Build(CancellationToken); -// -// // Start task -// var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); -// await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Workspace directory did not get created"), -// CancellationToken); -// -// // Ensure Workspace Cleaning Has Run -// var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); -// await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); -// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Workspace directory did not get deleted"), -// CancellationToken); -// -// Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); -// -// await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); -// await runningScriptTask; -// } -// -// [Test] -// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] -// public async Task WhenScriptServiceIsRunningAndWritesBootstrapScript_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var cleanerDelay = TimeSpan.FromMilliseconds(500); -// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); -// -// var existingHomeDirectory = new TemporaryDirectory(); -// -// var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); -// var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacle(b => -// { -// b.WithHomeDirectory(existingHomeDirectory) -// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); -// }) -// .Build(CancellationToken); -// -// // Start task -// var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); -// await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Workspace directory did not get created"), -// CancellationToken); -// -// // Ensure Workspace Cleaning Has Run -// var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); -// await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectory), "Existing bootstrap file"); -// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Workspace directory did not get deleted"), -// CancellationToken); -// -// Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); -// -// await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); -// await runningScriptTask; -// } -// -// [Test] -// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] -// public async Task WhenCompleteScriptIsNotCalled_ThenWorkspaceShouldGetDeletedWhenScriptFinishesRunning(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var cleanerDelay = TimeSpan.FromMilliseconds(500); -// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); -// -// var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("Hello")).Build(); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacle(b => -// { -// b.WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); -// }) -// .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() -// .DecorateAllScriptServicesWith(u => u -// .BeforeCompleteScript( -// () => throw new NotImplementedException("Force failure to simulate tentacle client crashing, and ensure we do not complete the script"))) -// .Build()) -// .Build(CancellationToken); -// -// await AssertionExtensions -// .Should(() => clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog())) -// .ThrowAsync(); -// -// var workspaceDirectory = GetWorkspaceDirectoryPath(clientAndTentacle.RunningTentacle.HomeDirectory, startScriptCommand.ScriptTicket.TaskId); -// -// await Wait.For(() => !Directory.Exists(workspaceDirectory), -// TimeSpan.FromSeconds(20), -// () => -// { -// try -// { -// Directory.Delete(workspaceDirectory, true); -// } -// catch (Exception) -// { -// // Deleting a worksapce is best effort and can silently fail if it is in use / locked by something. -// // If the cleaner failed to delete the directory and we cannot delete it in the test we can assume that it -// // is a valid failure and the test was successful. -// -// return; -// } -// -// throw new Exception("Workspace directory did not get deleted by the workspace cleaner but the test was able to delete it. This indicates there is an issue in the Tentacle code."); -// }, -// CancellationToken); -// } -// -// [Test] -// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] -// public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithLogFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var cleanerDelay = TimeSpan.FromMilliseconds(500); -// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); -// -// var existingHomeDirectory = new TemporaryDirectory(); -// -// var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); -// var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); -// await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectoryWithLogFile), "Existing log file"); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacle(b => -// { -// b.WithHomeDirectory(existingHomeDirectory) -// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); -// }) -// .Build(CancellationToken); -// -// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Workspace directory did not get deleted"), -// CancellationToken); -// Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); -// } -// -// [Test] -// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] -// public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithBootstrapFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var cleanerDelay = TimeSpan.FromMilliseconds(500); -// var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); -// -// var existingHomeDirectory = new TemporaryDirectory(); -// -// var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); -// var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); -// await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectoryWithLogFile), "Existing bootstrap file"); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacle(b => -// { -// b.WithHomeDirectory(existingHomeDirectory) -// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); -// }) -// .Build(CancellationToken); -// -// await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), -// TimeSpan.FromSeconds(60), -// () => throw new Exception("Workspace directory did not get deleted"), -// CancellationToken); -// Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); -// } -// -// [Test] -// [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] -// public async Task WhenTentacleStarts_WithWorkspaceYoungerThanThreshold_ThenWorkspaceIsLeftAlone(TentacleConfigurationTestCase tentacleConfigurationTestCase) -// { -// var cleanerDelay = TimeSpan.FromMilliseconds(500); -// var deleteWorkspacesOlderThan = TimeSpan.FromMinutes(30); -// -// var existingHomeDirectory = new TemporaryDirectory(); -// -// var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); -// await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); -// -// await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() -// .WithTentacle(b => -// { -// b.WithHomeDirectory(existingHomeDirectory) -// .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); -// }) -// .Build(CancellationToken); -// -// await Task.Delay(1000, CancellationToken); -// -// Directory.Exists(existingWorkspaceDirectory).Should().BeTrue(); -// } -// -// static string GetWorkspaceDirectoryPath(string homeDirectory, string scriptTicket) -// { -// var workspaceDirectory = Path.Combine( -// homeDirectory, -// ScriptWorkspaceFactory.WorkDirectory, -// scriptTicket); -// return workspaceDirectory; -// } -// -// static string GivenExistingWorkspaceExists(TemporaryDirectory existingHomeDirectory) -// { -// var existingWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, Guid.NewGuid().ToString()); -// Directory.CreateDirectory(existingWorkspaceDirectory); -// return existingWorkspaceDirectory; -// } -// -// static string GetBootstrapScriptFilePath(string workingDirectory) -// { -// return !PlatformDetection.IsRunningOnWindows -// ? BashScriptWorkspace.GetBashBootstrapScriptFilePath(workingDirectory) -// : ScriptWorkspace.GetBootstrapScriptFilePath(workingDirectory); -// } -// } -// } \ No newline at end of file +using System; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils; +using Octopus.Tentacle.CommonTestUtils.Diagnostics; +using Octopus.Tentacle.Scripts; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Util.Builders; +using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; + +namespace Octopus.Tentacle.Tests.Integration +{ + [IntegrationTestTimeout] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] + public class WorkspaceCleanerTests : IntegrationTest + { + [Test] + [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] + public async Task WhenScriptServiceIsRunningAndWritesLogFile_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var cleanerDelay = TimeSpan.FromMilliseconds(500); + var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); + + var existingHomeDirectory = new TemporaryDirectory(); + + var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); + var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); + var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacle(b => + { + b.WithHomeDirectory(existingHomeDirectory) + .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); + }) + .Build(CancellationToken); + + // Start task + var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); + await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), + TimeSpan.FromSeconds(60), + () => throw new Exception("Workspace directory did not get created"), + CancellationToken); + + // Ensure Workspace Cleaning Has Run + var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); + await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); + await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), + TimeSpan.FromSeconds(60), + () => throw new Exception("Workspace directory did not get deleted"), + CancellationToken); + + Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); + + await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); + await runningScriptTask; + } + + [Test] + [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] + public async Task WhenScriptServiceIsRunningAndWritesBootstrapScript_ThenWorkspaceIsNotDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var cleanerDelay = TimeSpan.FromMilliseconds(500); + var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); + + var existingHomeDirectory = new TemporaryDirectory(); + + var waitBeforeCompletingScriptFile = Path.Combine(existingHomeDirectory.DirectoryPath, "WaitForMeToExist.txt"); + var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.WaitForFileToExist(waitBeforeCompletingScriptFile)).Build(); + var startScriptWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, startScriptCommand.ScriptTicket.TaskId); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacle(b => + { + b.WithHomeDirectory(existingHomeDirectory) + .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); + }) + .Build(CancellationToken); + + // Start task + var runningScriptTask = clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog()); + await Wait.For(() => Directory.Exists(startScriptWorkspaceDirectory), + TimeSpan.FromSeconds(60), + () => throw new Exception("Workspace directory did not get created"), + CancellationToken); + + // Ensure Workspace Cleaning Has Run + var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); + await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectory), "Existing bootstrap file"); + await Wait.For(() => !Directory.Exists(existingWorkspaceDirectory), + TimeSpan.FromSeconds(60), + () => throw new Exception("Workspace directory did not get deleted"), + CancellationToken); + + Directory.Exists(startScriptWorkspaceDirectory).Should().BeTrue("Workspace should not have been cleaned up"); + + await File.WriteAllTextAsync(waitBeforeCompletingScriptFile, "Write file that makes script continue executing"); + await runningScriptTask; + } + + [Test] + [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] + public async Task WhenCompleteScriptIsNotCalled_ThenWorkspaceShouldGetDeletedWhenScriptFinishesRunning(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var cleanerDelay = TimeSpan.FromMilliseconds(500); + var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); + + var startScriptCommand = new TestExecuteShellScriptCommandBuilder().SetScriptBody(b => b.Print("Hello")).Build(); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacle(b => + { + b.WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); + }) + .WithTentacleServiceDecorator(new TentacleServiceDecoratorBuilder() + .DecorateAllScriptServicesWith(u => u + .BeforeCompleteScript( + () => throw new NotImplementedException("Force failure to simulate tentacle client crashing, and ensure we do not complete the script"))) + .Build()) + .Build(CancellationToken); + + await AssertionExtensions + .Should(() => clientAndTentacle.TentacleClient.ExecuteScript(startScriptCommand, CancellationToken, null, new InMemoryLog())) + .ThrowAsync(); + + var workspaceDirectory = GetWorkspaceDirectoryPath(clientAndTentacle.RunningTentacle.HomeDirectory, startScriptCommand.ScriptTicket.TaskId); + + await Wait.For(() => !Directory.Exists(workspaceDirectory), + TimeSpan.FromSeconds(20), + () => + { + try + { + Directory.Delete(workspaceDirectory, true); + } + catch (Exception) + { + // Deleting a worksapce is best effort and can silently fail if it is in use / locked by something. + // If the cleaner failed to delete the directory and we cannot delete it in the test we can assume that it + // is a valid failure and the test was successful. + + return; + } + + throw new Exception("Workspace directory did not get deleted by the workspace cleaner but the test was able to delete it. This indicates there is an issue in the Tentacle code."); + }, + CancellationToken); + } + + [Test] + [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] + public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithLogFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var cleanerDelay = TimeSpan.FromMilliseconds(500); + var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); + + var existingHomeDirectory = new TemporaryDirectory(); + + var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); + var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); + await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectoryWithLogFile), "Existing log file"); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacle(b => + { + b.WithHomeDirectory(existingHomeDirectory) + .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); + }) + .Build(CancellationToken); + + await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), + TimeSpan.FromSeconds(60), + () => throw new Exception("Workspace directory did not get deleted"), + CancellationToken); + Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); + } + + [Test] + [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] + public async Task WhenTentacleStarts_WithWorkspacesOlderThanThreshold_ThenWorkspaceWithBootstrapFileIsDeleted(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var cleanerDelay = TimeSpan.FromMilliseconds(500); + var deleteWorkspacesOlderThan = TimeSpan.FromMilliseconds(500); + + var existingHomeDirectory = new TemporaryDirectory(); + + var existingWorkspaceDirectoryWithoutLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); + var existingWorkspaceDirectoryWithLogFile = GivenExistingWorkspaceExists(existingHomeDirectory); + await File.WriteAllTextAsync(GetBootstrapScriptFilePath(existingWorkspaceDirectoryWithLogFile), "Existing bootstrap file"); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacle(b => + { + b.WithHomeDirectory(existingHomeDirectory) + .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); + }) + .Build(CancellationToken); + + await Wait.For(() => !Directory.Exists(existingWorkspaceDirectoryWithLogFile), + TimeSpan.FromSeconds(60), + () => throw new Exception("Workspace directory did not get deleted"), + CancellationToken); + Directory.Exists(existingWorkspaceDirectoryWithoutLogFile).Should().BeTrue(); + } + + [Test] + [TentacleConfigurations(testDefaultTentacleRuntimeOnly: true)] + public async Task WhenTentacleStarts_WithWorkspaceYoungerThanThreshold_ThenWorkspaceIsLeftAlone(TentacleConfigurationTestCase tentacleConfigurationTestCase) + { + var cleanerDelay = TimeSpan.FromMilliseconds(500); + var deleteWorkspacesOlderThan = TimeSpan.FromMinutes(30); + + var existingHomeDirectory = new TemporaryDirectory(); + + var existingWorkspaceDirectory = GivenExistingWorkspaceExists(existingHomeDirectory); + await File.WriteAllTextAsync(ScriptWorkspace.GetLogFilePath(existingWorkspaceDirectory), "Existing log file"); + + await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder() + .WithTentacle(b => + { + b.WithHomeDirectory(existingHomeDirectory) + .WithWorkspaceCleaningSettings(cleanerDelay, deleteWorkspacesOlderThan); + }) + .Build(CancellationToken); + + await Task.Delay(1000, CancellationToken); + + Directory.Exists(existingWorkspaceDirectory).Should().BeTrue(); + } + + static string GetWorkspaceDirectoryPath(string homeDirectory, string scriptTicket) + { + var workspaceDirectory = Path.Combine( + homeDirectory, + ScriptWorkspaceFactory.WorkDirectory, + scriptTicket); + return workspaceDirectory; + } + + static string GivenExistingWorkspaceExists(TemporaryDirectory existingHomeDirectory) + { + var existingWorkspaceDirectory = GetWorkspaceDirectoryPath(existingHomeDirectory.DirectoryPath, Guid.NewGuid().ToString()); + Directory.CreateDirectory(existingWorkspaceDirectory); + return existingWorkspaceDirectory; + } + + static string GetBootstrapScriptFilePath(string workingDirectory) + { + return !PlatformDetection.IsRunningOnWindows + ? BashScriptWorkspace.GetBashBootstrapScriptFilePath(workingDirectory) + : ScriptWorkspace.GetBootstrapScriptFilePath(workingDirectory); + } + } +} \ No newline at end of file From 26c47bfc9631e08e47bcd7d764ac16088621fffc Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Mon, 11 Nov 2024 22:58:50 +1300 Subject: [PATCH 30/34] Skip tests using class-level attribute --- .../ClientFileTransferRetriesTimeout.cs | 3 --- .../Util/SilentProcessRunnerFixture.cs | 8 +------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs index 6021d2cd9..eb18f0f51 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs @@ -12,11 +12,8 @@ using Octopus.Tentacle.Contracts.ClientServices; using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Support; -using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods; -using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; using Octopus.Tentacle.Tests.Integration.Util; using Octopus.Tentacle.Tests.Integration.Util.Builders; -using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers; using Octopus.TestPortForwarder; diff --git a/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs b/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs index 4dd44b0a9..2e481578d 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs @@ -10,6 +10,7 @@ namespace Octopus.Tentacle.Tests.Integration.Util { [TestFixture] + [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public class SilentProcessRunnerFixture : IntegrationTest { const int SIG_TERM = 143; @@ -33,7 +34,6 @@ public void SetUpLocal() } [Test] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void ExitCode_ShouldBeReturned() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) @@ -57,7 +57,6 @@ public void ExitCode_ShouldBeReturned() } [Test] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void DebugLogging_ShouldContainDiagnosticsInfo_ForDefault() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) @@ -85,7 +84,6 @@ public void DebugLogging_ShouldContainDiagnosticsInfo_ForDefault() [Test] [Retry(3)] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void CancellationToken_ShouldForceKillTheProcess() { // Terminate the process after a very short time so the test doesn't run forever @@ -118,7 +116,6 @@ public void CancellationToken_ShouldForceKillTheProcess() } [Test] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void EchoHello_ShouldWriteToStdOut() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) @@ -141,7 +138,6 @@ public void EchoHello_ShouldWriteToStdOut() } [Test] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void EchoError_ShouldWriteToStdErr() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) @@ -164,7 +160,6 @@ public void EchoError_ShouldWriteToStdErr() } [Test] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void RunAsCurrentUser_ShouldWork() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) @@ -191,7 +186,6 @@ public void RunAsCurrentUser_ShouldWork() [Test] [WindowsTest] [TestCase("powershell.exe", "-command \"Write-Host $env:userdomain\\$env:username\"")] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public void RunAsCurrentUser_PowerShell_ShouldWork(string command, string arguments) { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) From 8a70a05240292c1b631d4ccc1fe263e97761983b Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Mon, 11 Nov 2024 23:04:45 +1300 Subject: [PATCH 31/34] Try to see if the disk space issue comes back --- build/Build.Tests.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build/Build.Tests.cs b/build/Build.Tests.cs index 38d55fea3..08919d18e 100644 --- a/build/Build.Tests.cs +++ b/build/Build.Tests.cs @@ -319,9 +319,7 @@ void RunIntegrationTests(string testFramework, string testRuntime, string filter .SetProjectFile(projectPath) .SetFramework(testFramework) .SetFilter(filter) - .SetLoggers("console;verbosity=normal", "teamcity") - .SetNoBuild(true) - .SetNoRestore(true)) + .SetLoggers("console;verbosity=normal", "teamcity")) ); } catch (Exception e) From 848a8c00eb54b4a9fea48b6b6822d54bf4ed6c04 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Wed, 13 Nov 2024 08:19:03 +1100 Subject: [PATCH 32/34] Remove commented code --- .../ClientFileTransferRetriesTimeout.cs | 1 - .../ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs | 1 - .../ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs | 1 - 3 files changed, 3 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs index eb18f0f51..10e1ca10f 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs @@ -24,7 +24,6 @@ namespace Octopus.Tentacle.Tests.Integration /// from RPC calls when they are being retried and the rpc timeout period elapses. /// [IntegrationTestTimeout] - // [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public class ClientFileTransferRetriesTimeout : IntegrationTest { readonly TimeSpan retryIfRemainingDurationAtLeastBuffer = TimeSpan.FromSeconds(1); diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs index b6f9fc713..921ade73f 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs @@ -15,7 +15,6 @@ namespace Octopus.Tentacle.Tests.Integration { [IntegrationTestTimeout] - // [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public class ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled : IntegrationTest { [Test] diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs index 12cb89c2e..e1db7b2be 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs @@ -21,7 +21,6 @@ public class ClientFileTransfersAreRetriedWhenRetriesAreEnabled : IntegrationTes { [Test] [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)] - // [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public async Task FailedUploadsAreRetriedAndIsEventuallySuccessful(TentacleConfigurationTestCase tentacleConfigurationTestCase) { await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder() From c0e5ee2f4eddc7c6248e48e2158deac1be76eb07 Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Wed, 13 Nov 2024 08:45:29 +1100 Subject: [PATCH 33/34] Put back some tests and see if disk space issues return --- .../ClientScriptExecutionScriptFilesAreSent.cs | 2 +- .../ClientScriptExecutionWorksWithMultipleVersions.cs | 2 +- .../MachineConfigurationHomeDirectoryTests.cs | 2 +- source/Octopus.Tentacle.Tests.Integration/ScriptServiceTests.cs | 2 +- .../ScriptServiceV2IntegrationTest.cs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs index 9f54aac6d..22a3daf7b 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionScriptFilesAreSent.cs @@ -13,7 +13,7 @@ namespace Octopus.Tentacle.Tests.Integration { [IntegrationTestTimeout] - [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] + // [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] public class ClientScriptExecutionScriptFilesAreSent : IntegrationTest { [Test] diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionWorksWithMultipleVersions.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionWorksWithMultipleVersions.cs index c21d6bf59..eb56e9088 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionWorksWithMultipleVersions.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionWorksWithMultipleVersions.cs @@ -12,7 +12,7 @@ namespace Octopus.Tentacle.Tests.Integration { [IntegrationTestTimeout] - [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] + // [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] public class ClientScriptExecutionWorksWithMultipleVersions : IntegrationTest { [Test] diff --git a/source/Octopus.Tentacle.Tests.Integration/MachineConfigurationHomeDirectoryTests.cs b/source/Octopus.Tentacle.Tests.Integration/MachineConfigurationHomeDirectoryTests.cs index d6e0efe8a..f6c672102 100644 --- a/source/Octopus.Tentacle.Tests.Integration/MachineConfigurationHomeDirectoryTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/MachineConfigurationHomeDirectoryTests.cs @@ -11,7 +11,7 @@ namespace Octopus.Tentacle.Tests.Integration { [IntegrationTestTimeout] - [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] + // [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] public class MachineConfigurationHomeDirectoryTests : IntegrationTest { [Test] diff --git a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceTests.cs b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceTests.cs index 8822766da..1ecbe7a55 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceTests.cs @@ -13,7 +13,7 @@ namespace Octopus.Tentacle.Tests.Integration { [IntegrationTestTimeout] - [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] + // [SkipOnEnvironmentsWithKnownPerformanceIssues("we keep facing issues with disk space running out")] public class ScriptServiceTests : IntegrationTest { [Test] diff --git a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs index 760b9209c..8b7723f69 100644 --- a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs +++ b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs @@ -17,7 +17,7 @@ namespace Octopus.Tentacle.Tests.Integration { [IntegrationTestTimeout] - [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] + // [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")] public class ScriptServiceV2IntegrationTest : IntegrationTest { [Test] From 3f3e3b5e8e22b7d920c8254f825c4b83c15b212b Mon Sep 17 00:00:00 2001 From: Samdanae Imran Date: Wed, 13 Nov 2024 09:01:58 +1100 Subject: [PATCH 34/34] Change env variable name --- .../SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs b/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs index 493e6cbbc..9a36ab6c5 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs @@ -22,10 +22,10 @@ public void ApplyToTest(Test test) if (test.RunState == RunState.NotRunnable || test.RunState == RunState.Ignored) return; - if (bool.TryParse(Environment.GetEnvironmentVariable("Has_Known_Performance_Issues"), out _)) + if (bool.TryParse(Environment.GetEnvironmentVariable("Has_NET8_Compatibility_Issues"), out _)) { test.RunState = RunState.Skipped; - test.Properties.Add("_SKIPREASON", $"This test only runs on environments without performance issues because {Reason}"); + test.Properties.Add("_SKIPREASON", $"This test does not run on environments with .NET 8 performance issues because {Reason}"); } } }