Fluent

SECURITY.md

@@ -1,4 +1,4 @@
-# Security Policy
+# Security
 
 ## Supported Versions
 
@@ -9,9 +9,14 @@
 
 ## Reporting a Vulnerability
 
-Security issues and bugs should be reported privately.
+To report a security issue, please use the GitHub Security Advisory [Report a Vulnerability](https://github.com/scientistproject/Scientist.net/security/advisories/new) tab.
 
-> The best approach at the moment is to message one of the contributors on https://gitter.im/scientistproject/community privately
-> If you dont receive a reply within a couple of days please send the message onto another member
+We will:
 
-Please do not open issues for anything you think might have a security implication.
+- Treat the vulnerability as a security issue rather than a simple bug, both in our response and our disclosure. For example, we will explicitly mention that the issue is a security vulnerability in our release notes.
+- Acknowledge receipt of the vulnerability report as quickly as possible, even if no immediate resources are available for investigation.
+- Involve the vulnerability reporter when we verify the impact and veracity of the report.
+- Remediate the issue in a way that we see fit, taking any concerns and advice provided by the vulnerability reporter into careful consideration.
+- Always acknowledge the vulnerability reporter when we credit the discovery.
+- Aim to publish a fix as soon as we can.
+- Ensure that we make the wider ecosystem aware of the issue and its remediation when you disclose the vulnerability.

Scientist.Benchmark/ExperimentBenchmarks.cs

@@ -0,0 +1,40 @@
+using BenchmarkDotNet.Attributes;
+
+namespace Scientist.Benchmark
+{
+    [HtmlExporter]
+    public class ExperimentBenchmarks
+    {
+        readonly Experiment<int> experiment;
+
+        public ExperimentBenchmarks()
+        {
+            experiment = new Experiment<int>(name: "Instance", control: FortyTwo);
+        }
+
+        [Benchmark(Baseline = true)]
+        public int Static()
+        {
+            return new Experiment<int>(nameof(Static), control: FortyTwo)
+                .AddCandidate(FortyTwo)
+                .Run();
+        }
+
+        // Need to name candidates something
+        //[Benchmark]
+        //public int Instance()
+        //{
+        //    return experiment.AddCandidate(FortyTwo).Run();
+        //}
+
+        [Benchmark]
+        public int Multiple()
+        {
+            return new Experiment<int>(nameof(Multiple), control: FortyTwo)
+                .AddCandidate(FortyTwo)
+                .Run();
+        }
+
+        private static int FortyTwo() => 42;
+    }
+}

Scientist.Benchmark/Program.cs

@@ -0,0 +1,17 @@
+// See https://aka.ms/new-console-template for more information
+
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Toolchains.InProcess.Emit;
+using BenchmarkDotNet.Toolchains.InProcess.NoEmit;
+using Scientist.Benchmark;
+
+var config = DefaultConfig.Instance
+    .AddJob(Job
+         .MediumRun
+         .WithLaunchCount(1)
+         .WithToolchain(InProcessNoEmitToolchain.Instance));
+
+var summary = BenchmarkRunner.Run<ExperimentBenchmarks>(config);

Scientist.Benchmark/Scientist.Benchmark.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.12" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Scientist\Scientist.csproj" />
+  </ItemGroup>
+
+</Project>

Scientist.Test/AsyncExperimentTests.cs

@@ -0,0 +1,29 @@
+using FakeItEasy;
+using Scientist.Test.Helpers;
+using System.Threading.Tasks;
+
+namespace Scientist.Test
+{
+    public class AsyncExperiments
+    {
+        [Test]
+        public async Task AsynchronousExperimentAsync()
+        {
+            int actualResult = 42;
+
+            var testMethods = A.Fake<ITestClassAsync<int>>();
+            A.CallTo(() => testMethods.Control()).Returns(Task.FromResult(actualResult));
+            A.CallTo(() => testMethods.Candidate()).Returns(Task.FromResult(actualResult));
+
+            var result = await new Experiment<int>(
+                name: nameof(AsynchronousExperimentAsync),
+                control: testMethods.Control)
+                .AddCandidate(testMethods.Candidate)
+                .ThrowOnMismatch()
+                .RunAsync();
+
+            A.CallTo(() => testMethods.Control()).MustHaveHappened();
+            result.Should().Be(actualResult);
+        }
+    }
+}

Scientist.Test/ComparisonTests.cs

@@ -0,0 +1,60 @@
+using FakeItEasy;
+using Scientist.Test.Helpers;
+using System;
+
+namespace Scientist.Test
+{
+    public class Comparisons
+    {
+        [Test]
+        public void OverrideComparison()
+        {
+            var testMethods = A.Fake<ITestClass<ComplexResult>>();
+
+            var complexResult = A.Fake<ComplexResult>();
+            complexResult.Name = "Tester";
+            complexResult.Count = 10;
+            A.CallTo(() => testMethods.Control()).Returns(complexResult);
+            A.CallTo(() => testMethods.Candidate()).Returns(complexResult);
+
+            var result = new Experiment<ComplexResult>(
+                    name: nameof(OverrideComparison),
+                    control: testMethods.Control
+                )
+                .ThrowOnMismatch()
+                .AddCandidate(testMethods.Candidate)
+                .Compare((a, b) => a.Count == b.Count && a.Name == b.Name)
+                .Run();
+
+            A.CallTo(() => testMethods.Control()).MustHaveHappened();
+            A.CallTo(() => testMethods.Candidate()).MustHaveHappened();
+            result.Should().Be(complexResult);
+        }
+
+        [Test]
+        public void ThrowMismatchOnOverrideComparison()
+        {
+            var testMethods = A.Fake<ITestClass<ComplexResult>>();
+
+            var complexResult = A.Fake<ComplexResult>();
+            complexResult.Name = "Tester";
+            complexResult.Count = 10;
+            A.CallTo(() => testMethods.Control()).Returns(complexResult);
+            A.CallTo(() => testMethods.Candidate()).Returns(complexResult);
+
+            Action act = () => new Experiment<ComplexResult>(
+                    name: nameof(ThrowMismatchOnOverrideComparison),
+                    control: testMethods.Control
+                )
+            .ThrowOnMismatch()
+                .AddCandidate(testMethods.Candidate)
+                .Compare((a, b) => a.Count == 2 && a.Name == b.Name)
+                .Run();
+
+            act.Should().Throw<MismatchException<ComplexResult>>()
+               .WithMessage($"Experiment '{nameof(testMethods.Candidate)}' observations mismatched");
+            A.CallTo(() => testMethods.Control()).MustHaveHappened();
+            A.CallTo(() => testMethods.Candidate()).MustHaveHappened();
+        }
+    }
+}

Scientist.Test/ExceptionTests.cs

@@ -0,0 +1,54 @@
+using FakeItEasy;
+using Scientist.Test.Helpers;
+using System;
+
+namespace Scientist.Test
+{
+    public class Exceptions
+    {
+
+        [Test]
+        public void ThrowOnMismatch()
+        {
+            int actualResult = 42;
+
+            var testMethods = A.Fake<ITestClass<int>>();
+            A.CallTo(() => testMethods.Control()).Returns(actualResult);
+            A.CallTo(() => testMethods.Candidate()).Returns(33);
+
+            Action act = () => new Experiment<int>(
+                name: nameof(ThrowOnMismatch),
+                control: testMethods.Control)
+                .ThrowOnMismatch()
+                .AddCandidate(testMethods.Candidate)
+                .Run();
+
+            act.Should().Throw<MismatchException<int>>()
+                .WithMessage($"Experiment '{nameof(testMethods.Candidate)}' observations mismatched");
+            A.CallTo(() => testMethods.Control()).MustHaveHappened();
+            A.CallTo(() => testMethods.Candidate()).MustHaveHappened();
+        }
+
+
+        [Test]
+        public void ThrowOnMismatchTurnedOffDoesntThrow()
+        {
+            int actualResult = 42;
+
+            var testMethods = A.Fake<ITestClass<int>>();
+            var fakeCandidate = A.Fake<Func<int>>();
+            A.CallTo(() => testMethods.Control()).Returns(actualResult);
+            A.CallTo(() => testMethods.Candidate()).Returns(33);
+
+            Action act = () => new Experiment<int>(
+                name: nameof(ThrowOnMismatchTurnedOffDoesntThrow),
+                control: testMethods.Control)
+                .AddCandidate(testMethods.Candidate)
+                .Run();
+
+            act.Should().NotThrow<MismatchException<int>>();
+            A.CallTo(() => testMethods.Control()).MustHaveHappened();
+            A.CallTo(() => testMethods.Candidate()).MustHaveHappened();
+        }
+    }
+}

Scientist.Test/ExperimentTests.cs

@@ -0,0 +1,70 @@
+using FakeItEasy;
+using Scientist.Test.Helpers;
+using System;
+
+namespace Scientist.Test
+{
+    public class Experiments
+    {
+        [Test]
+        public void WithoutACandidate()
+        {
+            int actualResult = 42;
+
+            var testMethods = A.Fake<ITestClass<int>>();
+            A.CallTo(() => testMethods.Control()).Returns(actualResult);
+            A.CallTo(() => testMethods.Candidate()).Returns(actualResult);
+
+            var result = new Experiment<int>(name: nameof(WithoutACandidate), control: testMethods.Control)
+                .ThrowOnMismatch()
+                .Run();
+
+            A.CallTo(() => testMethods.Control()).MustHaveHappened();
+            result.Should().Be(actualResult);
+        }
+
+        [Test]
+        public void WithACandidate()
+        {
+            int actualResult = 42;
+
+            var testMethods = A.Fake<ITestClass<int>>();
+            A.CallTo(() => testMethods.Control()).Returns(actualResult);
+            A.CallTo(() => testMethods.Candidate()).Returns(actualResult);
+
+            var result = new Experiment<int>(name: nameof(WithACandidate), control: testMethods.Control)
+                .ThrowOnMismatch()
+                .AddCandidate(testMethods.Candidate)
+                .Run();
+
+            A.CallTo(() => testMethods.Control()).MustHaveHappened();
+            A.CallTo(() => testMethods.Candidate()).MustHaveHappened();
+            result.Should().Be(actualResult);
+        }
+
+        [Test]
+        public void WithMultipleCandidates()
+        {
+            int actualResult = 42;
+
+            var testMethods = A.Fake<ITestClass<int>>();
+            var fakeCandidate = A.Fake<Func<int>>();
+            A.CallTo(() => testMethods.Control()).Returns(actualResult);
+            A.CallTo(() => testMethods.Candidate()).Returns(actualResult);
+            A.CallTo(() => fakeCandidate()).Returns(actualResult);
+
+            var result = new Experiment<int>(
+                name: nameof(WithMultipleCandidates),
+                control: testMethods.Control)
+                 .ThrowOnMismatch()
+                .AddCandidate(testMethods.Candidate)
+                .AddCandidate(name: "Candidate 2", candidate: fakeCandidate)
+                .Run();
+
+            A.CallTo(() => testMethods.Control()).MustHaveHappened();
+            A.CallTo(() => testMethods.Candidate()).MustHaveHappened();
+            A.CallTo(() => fakeCandidate()).MustHaveHappened();
+            result.Should().Be(actualResult);
+        }
+    }
+}

Scientist.Test/GlobalUsings.cs

@@ -0,0 +1,2 @@
+global using NUnit.Framework;
+global using FluentAssertions;

Scientist.Test/Helpers/ComplexResult.cs

@@ -0,0 +1,22 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Scientist.Test.Helpers
+{
+    public class ComplexResult : IEquatable<ComplexResult>
+    {
+        public int Count { get; set; }
+        public string Name { get; set; }
+
+        public bool Equals(ComplexResult other)
+        {
+            if (other == null)
+                return false;
+
+            return Count == other.Count && Name == other.Name;
+        }
+    }
+}

Scientist.Test/Helpers/ITestClass.cs

@@ -0,0 +1,8 @@
+namespace Scientist.Test.Helpers
+{
+    public interface ITestClass<T>
+    {
+        T Control();
+        T Candidate();
+    }
+}

Scientist.Test/Helpers/ITestClassAsync.cs

@@ -0,0 +1,10 @@
+using System.Threading.Tasks;
+
+namespace Scientist.Test.Helpers
+{
+    public interface ITestClassAsync<T>
+    {
+        Task<T> Control();
+        Task<T> Candidate();
+    }
+}

Scientist.Test/Helpers/TestHelpers.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Scientist.Test.Helpers
+{
+    public static class TestHelpers
+    {
+        public static IEnumerable<Results<T>> Results<T>(/*string experimentName) => ResultPublisher.Results<T, T>(experimentName);*/
+    }
+}