Remote

docs/remote.md

@@ -0,0 +1,31 @@
+```mermaid
+
+sequenceDiagram
+    [User]->>RemoteFileTrialStore: create()
+    RemoteFileTrialStore->>ExperiturServer: create()
+    ExperiturServer->>FileTrialStore: create()
+    FileTrialStore-->>ExperiturServer: trial_id: str
+    ExperiturServer-->>RemoteFileTrialStore: trial_id: str
+    RemoteFileTrialStore-->>[User]: trial_id: str
+
+    [User]->>RemoteFileTrialStore: set_data(trial_id: str, trial_data: dict)
+    RemoteFileTrialStore->>ExperiturServer: set_data(trial_id: str, trial_data: dict)
+    ExperiturServer->>FileTrialStore: set_data(trial_id: str, trial_data: dict)
+
+    [User]->>RemoteFileTrialStore: get_data(trial_id: str)
+    RemoteFileTrialStore->>ExperiturServer: get_data(trial_id: str)
+    ExperiturServer->>FileTrialStore: get_data(trial_id: str)
+    FileTrialStore-->>ExperiturServer: trial_data: dict
+    ExperiturServer-->>RemoteFileTrialStore: trial_data: dict
+    RemoteFileTrialStore-->>[User]: trial_data: dict
+
+    [User]->>RemoteFileTrialStore: filter(...)
+    RemoteFileTrialStore->>ExperiturServer: filter(...)
+    ExperiturServer->>FileTrialStore: filter(...)
+    FileTrialStore->>FileTrialStore: __iter__()
+    FileTrialStore-->>ExperiturServer: trial_datas: List[dict]
+    ExperiturServer-->>RemoteFileTrialStore: trial_datas: List[dict]
+    RemoteFileTrialStore-->>[User]: trial_datas: List[dict]
+
+
+```

experiments/optimization.py

@@ -0,0 +1,116 @@
+import math
+import operator
+
+import click
+import matplotlib.pyplot as plt
+import pandas as pd
+import skopt
+from scipy.stats import uniform
+from skopt.plots import plot_convergence, plot_objective
+from skopt.space import Categorical, Dimension, Integer, Real
+from skopt.utils import dimensions_aslist, point_asdict, point_aslist
+
+from experitur import Experiment, Trial
+from experitur.parameters import Grid, Multi, SKOpt
+from experitur.parameters.skopt import convert_trial
+
+
+def rosenbrock(a, b, x, y):
+    return (a - x) ** 2 + b * (y - x ** 2) ** 2
+
+
+@Grid({"a": [1], "b": [100]})
+@SKOpt({"x": Real(-10, 10), "y": Real(-10, 10)}, "z", 200)
+@Grid({"repetition": [1, 2, 3]})
+@Experiment(active=False,)
+def exp(trial: Trial):
+    z = trial.call(rosenbrock)
+
+    print(z)
+
+    return {"z": z}
+
+
+def hyper(x, y):
+    return x ** 2 + y ** 2
+
+
+@Experiment(
+    parameters=SKOpt({"x": Real(-10, 10), "y": Real(-10, 10)}, "z", 100), active=False
+)
+def exp2(trial: Trial):
+    z = trial.call(hyper)
+    return {"z": z}
+
+
+def fun(x, y, z):
+    return x ** 2 + math.sin(y) + z
+
+
+@Experiment(
+    parameters=SKOpt(
+        {"x": Real(-10, 10), "y": Real(-10, 10), "z": Real(-10, 10)}, "res", 100
+    )
+)
+def exp3(trial: Trial):
+    res = trial.call(fun)
+    return {"res": res}
+
+
+def as_dimension(d):
+    if isinstance(d, Dimension):
+        return d
+    if isinstance(d, list):
+        return Categorical(d)
+    raise ValueError(f"Unexpected dimension: {d!r}")
+
+
+@click.argument("objective")
+@click.option("--plot-objective", "plot_objective_", is_flag=True)
+@click.option("--plot-convergence", "plot_convergence_", is_flag=True)
+@exp3.command(target="experiment")
+def show(
+    experiment: Experiment, objective, plot_objective_=False, plot_convergence_=False
+):
+    """
+    Show information about optimization.
+
+    Run `experitur do optimization.py show exp3 res` to show information about the optimization that takes place in `exp3` regarding objective `res`.
+    """
+
+    search_space = {
+        n: as_dimension(d)
+        for n, d in experiment.parameter_generator.varying_parameters.items()
+    }
+
+    trials = experiment.ctx.store.match(experiment=experiment)
+
+    results = [
+        convert_trial(trial, search_space, objective)
+        for trial in sorted(trials.values(), key=lambda trial: trial.data["time_start"])
+    ]
+
+    if not results:
+        print("No results.")
+        return
+
+    X, Y = zip(*results)
+
+    optimizer = skopt.Optimizer(dimensions_aslist(search_space))
+
+    optimize_result = optimizer.tell(X, Y)
+
+    print(
+        f"Current optimium {optimize_result.fun} at {point_asdict(search_space, optimize_result.x)}"
+    )
+
+    if plot_objective_:
+        plot_objective(optimize_result, levels=20)
+        plt.show()
+    elif plot_convergence_:
+        plot_convergence(optimize_result)
+        plt.show()
+    else:
+        print(
+            "Use --plot-convergence to plot the convergence trace or --plot-objective to show the pairwise dependence plot of the objective function."
+        )

experitur/core/context.py

@@ -87,11 +87,6 @@ def __init__(
         else:
             self.wdir = wdir
 
-        # Import here to break dependency cycle
-        from experitur.core.trial_store import FileTrialStore
-
-        self.store = FileTrialStore(self)
-
         # Configuration
         if config is None:
             self.config = self._default_config.copy()
@@ -103,6 +98,10 @@ def __init__(
     def _register_experiment(self, experiment):
         self.registered_experiments.append(experiment)
 
+    def create_trial(self, trial_data, experiment: Experiment) -> TrialData:
+        trial_id = self.store.create()
+        return
+
     def run(self, experiments=None):
         """
         Run the specified experiments or all.

experitur/core/experiment.py

@@ -265,7 +265,7 @@ def run(self):
 
             if self.ctx.config["skip_existing"]:
                 # Check, if a trial with this parameter set already exists
-                existing = self.ctx.store.match(
+                existing = self.ctx.trials.filter(
                     func=self.func,
                     parameters=trial_configuration.get("parameters", {}),
                 )

experitur/core/trial.py

@@ -2,24 +2,25 @@
 import inspect
 import itertools
 from collections import OrderedDict, defaultdict
-from collections.abc import Collection
+from collections.abc import Collection, Sequence
 from typing import (
     TYPE_CHECKING,
     Any,
     Callable,
     Iterable,
     List,
     Mapping,
+    Optional,
     Tuple,
     TypeVar,
     Union,
 )
 
-
 from experitur.core.logger import YAMLLogger
 
 if TYPE_CHECKING:  # pragma: no cover
     from experitur.core.experiment import Experiment
+    from experitur.core.trial_store import TrialStore
 
 T = TypeVar("T")
 
@@ -382,7 +383,7 @@ def log(self, values, **kwargs):
         self._logger.log(values)
 
 
-class TrialCollection(Collection):
+class TrialCollectionBase(Sequence):
     _missing = object()
 
     def __init__(self, trials: List[Trial]):
@@ -403,7 +404,7 @@ def pop(self, index=-1):
     @property
     def independent_parameters(self):
         independent_parameters = set()
-        for t in self.trials:
+        for t in self:
             independent_parameters.update(
                 getattr(t, "experiment", {}).get("independent_parameters", [])
             )
@@ -414,7 +415,7 @@ def varying_parameters(self):
         """Independent parameters that vary in this trial collection."""
         independent_parameters = self.independent_parameters
         parameter_values = defaultdict(set)
-        for t in self.trials:
+        for t in self:
             for p in independent_parameters:
                 try:
                     v = t[p]
@@ -425,28 +426,71 @@ def varying_parameters(self):
 
         return set(p for p in independent_parameters if len(parameter_values[p]) > 1)
 
-    def to_pandas(self):
-        import pandas as pd
+    def to_pandas(self, failed=True):
+        try:
+            from pandas import json_normalize
+        except ImportError:
+            try:
+                from pandas.io.json import json_normalize
+            except ImportError:
+                raise RuntimeError("pandas is not available.")
+
+        return json_normalize(
+            [t.data for t in self if failed or not t.is_failed], max_level=1
+        ).set_index("id")
+
+    @abstractmethod
+    def one(self):
+        pass
 
-        return pd.json_normalize([t.data for t in self.trials], max_level=1).set_index(
-            "id"
-        )
+    def filter(
+        self,
+        filter_fn: Optional[Callable[[TrialData], bool]],
+        func=None,
+        parameters=None,
+        experiment=None,
+    ) -> "TrialCollection":
+        """
+        Return a filtered version of this trial collection.
+
+        Args:
+            func (callable): A function that receives a TrialData instance and returns True if the trial should be kept.
+
+        Returns:
+            A new trial collection.
+        """
+
+        if func is not None:
+            raise NotImplementedError()
+
+        if parameters is not None:
+            raise NotImplementedError()
+
+        if experiment is not None:
+            raise NotImplementedError()
+
+        return TrialCollection(list(filter(filter_fn, self)))
 
     def one(self):
-        if len(self.trials) != 1:
+        if len(self) != 1:
             raise ValueError("No individual trial.")
 
-        return self.trials[0]
+        return self[0]
 
     def filter(self, fn: Callable[[Trial], bool]) -> "TrialCollection":
         """
         Return a filtered version of this trial collection.
 
         Args:
-            fn (callable): A function that receives a TrialData instance and returns True if the trial should be kept.
+            filter_fn (callable): A function that receives the trial data dictionary and returns True if the trial should be kept.
 
         Returns:
             A new trial collection.
         """
 
-        return TrialCollection(list(filter(fn, self.trials)))
+        return TrialCollection(
+            self._store.match(
+                func=func, experiment=experiment, resolved_parameters=parameters
+            )
+        ).filter(filter_fn)
+