Feature: experiment.EnsureControlRunsFirst()

README.md

@@ -29,7 +29,7 @@ public bool CanAccess(IUser user)
 Wrap a `Use` block around the code's original behavior, and wrap `Try` around the new behavior. Invoking `Scientist.Science<T>` will always return whatever the `Use` block returns, but it does a bunch of stuff behind the scenes:
 
 * It decides whether or not to run the `Try` block,
-* Randomizes the order in which `Use` and `Try` blocks are run,
+* Randomizes the order in which `Use` and `Try` blocks are run (Also see [Ensure control runs first](#control-first) option), 
 * Measures the durations of all behaviors,
 * Compares the result of `Try` to the result of `Use`,
 * Swallows (but records) any exceptions raised in the `Try` block, and
@@ -341,6 +341,18 @@ public bool CanAccess(IUser user)
 }
 ```
 
+### Ensuring Control is ran first {#control-first}
+
+Sometimes you've got to run the control first and then the candidate(s) and we get it! 
+```csharp
+Scientist.Science<int>("ExperimentN", experiment => 
+{
+    experiment.EnsureControlRunsFirst();
+    // ...
+});
+```
+> The candidates are still ran in a randomised order, its just the control get ran first.
+
 ## Alternatives
 
 Here are other implementations of Scientist available in different languages.

src/Scientist/IExperiment.cs

@@ -32,6 +32,11 @@ public interface IExperiment
         /// </summary>
         /// <param name="block">The delegate to handle exceptions thrown from an experiment.</param>
         void Thrown(Action<Operation, Exception> block);
+
+        /// <summary>
+        /// Sets the control to run first and candidates after. Does not affect candidate randomisation.
+        /// </summary>
+        void EnsureControlRunsFirst();
     }
 
     /// <summary>

src/Scientist/Internals/Experiment.cs

@@ -28,6 +28,8 @@ private static readonly Action<Operation, Exception> _alwaysThrow
         private readonly Dictionary<string, dynamic> _contexts = new Dictionary<string, dynamic>();
         private readonly IResultPublisher _resultPublisher;
 
+        private bool _ensureControlRunsFirst = false;
+
         public Experiment(string name, Func<Task<bool>> enabled, int concurrentTasks, IResultPublisher resultPublisher)
         {
             if (concurrentTasks <= 0)
@@ -130,7 +132,8 @@ internal ExperimentInstance<T, TClean> Build() =>
                 RunIf = _runIf,
                 Thrown = _thrown,
                 ThrowOnMismatches = ThrowOnMismatches,
-                ResultPublisher = _resultPublisher
+                ResultPublisher = _resultPublisher,
+                EnsureControlRunsFirst = _ensureControlRunsFirst,
             });
 
         public void Compare(Func<T, T, bool> comparison)
@@ -156,5 +159,9 @@ public void BeforeRun(Func<Task> action)
         {
             _beforeRun = action;
         }
+
+        public void EnsureControlRunsFirst() {
+            _ensureControlRunsFirst = true;
+        }
     }
 }

src/Scientist/Internals/ExperimentInstance.cs

@@ -16,7 +16,8 @@ internal class ExperimentInstance<T, TClean>
 
         internal readonly string Name;
         internal readonly int ConcurrentTasks;
-        internal readonly List<NamedBehavior> Behaviors;
+        internal readonly NamedBehavior Control;
+        internal readonly List<NamedBehavior> Candidates = new List<NamedBehavior>();
         internal readonly Func<T, TClean> Cleaner;
         internal readonly Func<T, T, bool> Comparator;
         internal readonly Func<Task> BeforeRun;
@@ -27,18 +28,15 @@ internal class ExperimentInstance<T, TClean>
         internal readonly Action<Operation, Exception> Thrown;
         internal readonly bool ThrowOnMismatches;
         internal readonly IResultPublisher ResultPublisher;
-
+        internal readonly bool EnsureControlRunsFirst;
+
         static Random _random = new Random(DateTimeOffset.UtcNow.Millisecond);
-        
+
         public ExperimentInstance(ExperimentSettings<T, TClean> settings)
         {
             Name = settings.Name;
-
-            Behaviors = new List<NamedBehavior>
-            {
-                new NamedBehavior(ControlExperimentName, settings.Control),
-            };
-            Behaviors.AddRange(
+            Control = new NamedBehavior(ControlExperimentName, settings.Control);
+            Candidates.AddRange(
                 settings.Candidates.Select(c => new NamedBehavior(c.Key, c.Value)));
 
             BeforeRun = settings.BeforeRun;
@@ -52,6 +50,7 @@ public ExperimentInstance(ExperimentSettings<T, TClean> settings)
             Thrown = settings.Thrown;
             ThrowOnMismatches = settings.ThrowOnMismatches;
             ResultPublisher = settings.ResultPublisher;
+            EnsureControlRunsFirst = settings.EnsureControlRunsFirst;
         }
 
         public async Task<T> Run()
@@ -60,27 +59,35 @@ public async Task<T> Run()
             if (!await ShouldExperimentRun().ConfigureAwait(false))
             {
                 // Run the control behavior.
-                return await Behaviors[0].Behavior().ConfigureAwait(false);
+                return await Control.Behavior().ConfigureAwait(false);
             }
 
             if (BeforeRun != null)
             {
                 await BeforeRun().ConfigureAwait(false);
             }
 
-            // Randomize ordering...
-            NamedBehavior[] orderedBehaviors;
-            lock (_random)
+
+            var behaviors = new NamedBehavior[0];
+            if (EnsureControlRunsFirst)
             {
-                orderedBehaviors = Behaviors.OrderBy(b => _random.Next()).ToArray();
+
+                behaviors = RandomiseBehavioursOrder(Candidates);
+                behaviors = new[] { Control }.Concat(behaviors).ToArray();
             }
+            else
+            {
+                Candidates.Add(Control);
+                behaviors = RandomiseBehavioursOrder(Candidates);
+            }
+
 
             // Break tasks into batches of "ConcurrentTasks" size
             var observations = new List<Observation<T, TClean>>();
-            foreach (var behaviors in orderedBehaviors.Chunk(ConcurrentTasks))
+            foreach (var behaviorsChunk in behaviors.Chunk(ConcurrentTasks))
             {
                 // Run batch of behaviors simultaneously
-                var tasks = behaviors.Select(b =>
+                var tasks = behaviorsChunk.Select(b =>
                 {
                     return Observation<T, TClean>.New(
                         b.Name,
@@ -95,7 +102,7 @@ public async Task<T> Run()
             }
 
             var controlObservation = observations.FirstOrDefault(o => o.Name == ControlExperimentName);
-            
+
             var result = new Result<T, TClean>(this, observations, controlObservation, Contexts);
 
             try
@@ -115,7 +122,15 @@ public async Task<T> Run()
             if (controlObservation.Thrown) throw controlObservation.Exception;
             return controlObservation.Value;
         }
-
+
+        private NamedBehavior[] RandomiseBehavioursOrder(List<NamedBehavior> behaviors)
+        {
+            lock (_random)
+            {
+                return behaviors.OrderBy(b => _random.Next()).ToArray();
+            }
+        }
+
         /// <summary>
         /// Does <see cref="RunIf"/> allow the experiment to run?
         /// </summary>
@@ -149,17 +164,16 @@ public async Task<bool> IgnoreMismatchedObservation(Observation<T, TClean> contr
                 return false;
             }
         }
-        
+
         /// <summary>
         /// Determine whether or not the experiment should run.
         /// </summary>
         async Task<bool> ShouldExperimentRun()
         {
             try
             {
-                // Only let the experiment run if at least one candidate (> 1 behaviors) is 
-                // included.  The control is always included behaviors count.
-                return Behaviors.Count > 1 && await Enabled().ConfigureAwait(false) && await RunIfAllows().ConfigureAwait(false);
+                // Only let the experiment run if at least one candidate (>= 1 behaviors)
+                return Candidates.Count >= 1 && await Enabled().ConfigureAwait(false) && await RunIfAllows().ConfigureAwait(false);
             }
             catch (Exception ex)
             {

src/Scientist/Internals/ExperimentSettings.cs

@@ -26,5 +26,6 @@ internal class ExperimentSettings<T, TClean>
         public bool ThrowOnMismatches { get; set; }
         public Action<Operation, Exception> Thrown { get; set; }
         public IResultPublisher ResultPublisher { get; set; }
+        public bool EnsureControlRunsFirst { get; set; }
     }
 }

test/Scientist.Test/ExperimentTests/ExperimentAsyncTests.cs

@@ -0,0 +1,43 @@
+using FluentAssertions;
+using GitHub;
+using GitHub.Internals;
+using NSubstitute;
+using System.Linq;
+using System.Threading.Tasks;
+using UnitTests;
+using Xunit;
+
+
+public class ExperimentAsyncTests
+{
+    [Fact]
+    public async Task ExperimentAsync_EnsureControlRunsFirst_ShouldRunControlFirst()
+    {
+        var mock = Substitute.For<IControlCandidateTask<int>>();
+        mock.Control().Returns(Task.FromResult(42));
+        mock.Candidate().Returns(Task.FromResult(42));
+        const string experimentName = nameof(ExperimentAsync_EnsureControlRunsFirst_ShouldRunControlFirst);
+
+        var resultPublisher = new InMemoryResultPublisher();
+        var scientist = new Scientist(resultPublisher);
+
+        var result = await scientist.ExperimentAsync<int>(experimentName, experiment =>
+        {
+            experiment.ThrowOnMismatches = true;
+            experiment.EnsureControlRunsFirst();
+            experiment.Use(mock.Control);
+            experiment.Try("candidate", mock.Candidate);
+        });
+
+        result.Should().Be(42);
+
+        Received.InOrder(() =>
+        {
+            mock.Received().Control();
+            mock.Received().Candidate();
+        });
+
+        resultPublisher.Results<int>(experimentName).First().Matched.Should().BeTrue();
+    }
+}
+

test/Scientist.Test/ExperimentTests/ExperimentTests.cs

@@ -0,0 +1,40 @@
+using GitHub;
+using GitHub.Internals;
+using NSubstitute;
+using System.Linq;
+using UnitTests;
+using Xunit;
+using FluentAssertions;
+
+
+public class ExperimentTests
+{
+    [Fact]
+    public void Experiment_EnsureControlRunsFirst_ShouldRunControlFirst()
+    {
+        var mock = Substitute.For<IControlCandidate<int>>();
+        mock.Control().Returns(42);
+        mock.Candidate().Returns(42);
+        const string experimentName = nameof(Experiment_EnsureControlRunsFirst_ShouldRunControlFirst);
+
+        var resultPublisher = new InMemoryResultPublisher();
+        var scientist = new Scientist(resultPublisher);
+
+        var result = scientist.Experiment<int>(experimentName, experiment =>
+        {
+            experiment.ThrowOnMismatches = true;
+            experiment.EnsureControlRunsFirst();
+            experiment.Use(mock.Control);
+            experiment.Try("candidate", mock.Candidate);
+        });
+
+        result.Should().Be(42);
+        Received.InOrder(() =>
+        {
+            mock.Received().Control();
+            mock.Received().Candidate();
+        });
+        resultPublisher.Results<int>(experimentName).First().Matched.Should().BeTrue();
+    }
+}
+

test/Scientist.Test/Scientist.Test.csproj

@@ -21,8 +21,9 @@
     <None Update="xunit.runner.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="3.1.4">
+    <PackageReference Include="FluentAssertions" Version="8.8.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="3.1.5">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>