[WIP] BOEnv

dacbench/benchmarks/__init__.py

@@ -51,3 +51,15 @@
         "Theory Benchmark not installed. If you want to use this benchmark, "
         "please follow the installation guide."
     )
+
+dacbo_spec = importlib.util.find_spec("dacboenv")
+found = dacbo_spec is not None
+if found:
+    from dacbench.benchmarks.dacbo_benchmark import DACBOBenchmark
+
+    __all__.append("DACBOBenchmark")
+else:
+    warnings.warn(  # noqa: B028
+        "DACBOEnv not installed. If you want to use this benchmark, "
+        "please follow the installation guide."
+    )

dacbench/benchmarks/dacbo_benchmark.py

@@ -0,0 +1,124 @@
+"""DACBOEnv Benchmark."""
+
+from __future__ import annotations
+
+from importlib.resources import files
+from itertools import product
+from pathlib import Path
+
+import dacboenv
+import numpy as np
+import yaml
+from dacboenv.env.action import AcqParameterActionSpace
+from omegaconf import OmegaConf
+
+from dacbench.abstract_benchmark import AbstractBenchmark, objdict
+from dacbench.envs.dacbo import DACBOEnv
+
+
+def load_default_optimizer():
+    """Handles dacboenv configs to configure WEI as default."""
+    dacboenv_path = files("dacboenv")
+    base = OmegaConf.load(dacboenv_path / "configs/env/opt/base.yaml")
+    base.dacboenv.optimizer_cfg.smac_cfg.smac_kwargs = None
+    override = OmegaConf.load(
+        dacboenv_path / "configs/env/action/wei_alpha_continuous.yaml"
+    )
+    cfg = OmegaConf.merge(base, override)
+    cfg = OmegaConf.create({"optimizer": cfg.dacboenv.optimizer_cfg})
+
+    def replace_refs(node):
+        if isinstance(node, str):
+            return node.replace("dacboenv.optimizer_cfg", "optimizer")
+        if isinstance(node, dict):
+            return {k: replace_refs(v) for k, v in node.items()}
+        if isinstance(node, list):
+            return [replace_refs(v) for v in node]
+        return node
+
+    cfg = OmegaConf.create(replace_refs(OmegaConf.to_container(cfg, resolve=False)))
+    cfg.outdir = "runs/SMAC-DACBO/${benchmark_id}/${task_id}/${seed}"
+
+    return cfg
+
+
+INFO = {
+    "identifier": "DACBO",
+    "name": "DACBO",
+    "reward": f"""Default: [symlogregret]. Other options: {[
+        rew.name for rew in dacboenv.env.reward.ALL_REWARDS
+    ]}""",
+    "state_description": f"""Default: {[
+            "ubr_difference",
+            "acq_value_EI",
+            "acq_value_PI",
+            "previous_param",
+        ]}. Other options: {[
+        obs.name for obs in dacboenv.env.observation.ALL_OBSERVATIONS
+    ]}""",
+}
+
+DACBO_DEFAULTS = objdict(
+    {
+        "reward_range": [-np.inf, np.inf],
+        "seed": 0,
+        "instance_set_path": "bbob_2_default.yaml",
+        "optimizer_cfg": load_default_optimizer(),
+        "observation_keys": [
+            "ubr_difference",
+            "acq_value_EI",
+            "acq_value_PI",
+            "previous_param",
+        ],
+        "action_space_class": AcqParameterActionSpace,
+        "action_space_kwargs": {"bounds": [0, 1], "adjustment_type": "continuous"},
+        "reward_keys": ["symlogregret"],
+        "benchmark_info": INFO,
+    }
+)
+
+
+class DACBOBenchmark(AbstractBenchmark):
+    """DACBOEnv benchmark."""
+
+    def __init__(self, config_path=None, config=None):
+        """Init DACBOEnv benchmark."""
+        super().__init__(config_path, config)
+
+        if not self.config:
+            self.config = objdict(DACBO_DEFAULTS.copy())
+
+        for key in DACBO_DEFAULTS:
+            if key not in self.config:
+                self.config[key] = DACBO_DEFAULTS[key]
+
+    def get_environment(self):
+        """Returns the internal env."""
+        return DACBOEnv(self.config)
+
+    def read_instance_set(self):
+        """Reads the instance set."""
+        assert self.config.instance_set_path
+        if Path(self.config.instance_set_path).is_file():
+            path = self.config.instance_set_path
+        else:
+            path = (
+                Path(__file__).resolve().parent
+                / "../instance_sets/dacbo/"
+                / self.config.instance_set_path
+            )
+
+        with open(path) as f:
+            instance_data = yaml.safe_load(f)
+        print(instance_data)
+        self.config["task_ids"] = instance_data["task_ids"]
+        self.config["inner_seeds"] = instance_data.get("inner_seeds", None)
+        self.config["instance_set"] = dict(
+            enumerate(
+                product(
+                    instance_data.get("inner_seeds", None), instance_data["task_ids"]
+                )
+            )
+        )  # Not used. Instance selection is handled by the internal env
+
+        assert len(self.config["instance_set"]) > 0, "ERROR: empty instance set"

dacbench/envs/dacbo.py

@@ -0,0 +1,34 @@
+"""DACBO Env."""
+
+from __future__ import annotations
+
+import numpy as np
+from dacboenv.dacboenv import DACBOEnv as DEnv
+
+from dacbench.abstract_env import AbstractEnv
+
+
+class DACBOEnv(AbstractEnv):
+    """DACBO env."""
+
+    def __init__(self, config):
+        """Init DACBO env."""
+        self._env = DEnv(**config)
+        self.reset()
+        config[
+            "cutoff"
+        ] = np.inf  # Not used. DACBO handles BO runs (i.e. episodes) internally
+        config["observation_space"] = self._env.observation_space
+        config["action_space"] = self._env.action_space
+        super().__init__(config)
+
+    def step(self, action):
+        """Takes one env step."""
+        state, reward, terminated, truncated, info = self._env.step(action)
+        if truncated:  # Reset BO loop, select new instance
+            self._env.reset()
+        return state, reward, terminated, truncated, info
+
+    def reset(self, seed=None):
+        """Resets the internal env."""
+        return self._env.reset()

dacbench/instance_sets/dacbo/bbob_2_default.yaml

@@ -0,0 +1,25 @@
+task_ids:
+  - "bbob/2/1/0"
+  - "bbob/2/2/0"
+  - "bbob/2/3/0"
+  - "bbob/2/4/0"
+  - "bbob/2/5/0"
+  - "bbob/2/6/0"
+  - "bbob/2/7/0"
+  - "bbob/2/8/0"
+  - "bbob/2/9/0"
+  - "bbob/2/10/0"
+  - "bbob/2/11/0"
+  - "bbob/2/12/0"
+  - "bbob/2/13/0"
+  - "bbob/2/14/0"
+  - "bbob/2/15/0"
+  - "bbob/2/16/0"
+  - "bbob/2/17/0"
+  - "bbob/2/18/0"
+  - "bbob/2/19/0"
+  - "bbob/2/20/0"
+inner_seeds:
+  - 1
+  - 2
+  - 3