From ff8aa42481f97500b8770b396814220d242201f0 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 23 Jan 2025 16:48:02 +0000 Subject: [PATCH 01/11] service types docs --- .../mlos_bench/services/types/__init__.py | 48 ++++++++++++++++++- .../mlos_bench/services/types/bound_method.py | 8 +++- .../services/types/config_loader_type.py | 7 ++- .../services/types/remote_config_type.py | 2 +- 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/mlos_bench/mlos_bench/services/types/__init__.py b/mlos_bench/mlos_bench/services/types/__init__.py index e2d0cb55b5a..65e1ec0d1b3 100644 --- a/mlos_bench/mlos_bench/services/types/__init__.py +++ b/mlos_bench/mlos_bench/services/types/__init__.py @@ -2,8 +2,52 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Service types for implementing declaring Service behavior for Environments to use in -mlos_bench. +"""Service types (i.e., :py:class:`Protocol`s) for declaring implementation +:py:class:`~mlos_bench.services.base_service.Service` behavior for +:py:mod:`~mlos_bench.environments` to use in :py:mod:`mlos_bench`. + +Overview +-------- +Service loading in ``mlos_bench`` uses a +[mix-in](https://en.wikipedia.org/wiki/Mixin#In_Python) approach to combine the +functionality of multiple classes specified at runtime through config files into +a single class that the :py:mod:`~mlos_bench.environments` can use to invoke the +actions that they were configured to perform (e.g., provisioning a VM, deploying +a network, running a script, etc.). + +Since Services are loaded at runtime and can be swapped out by referencing a +different set of ``--services`` config files via the :py:mod:`mlos_bench.run` +CLI option, this can make it difficult to do type and config checking. + +To address this we define ``@runtime_checkable`` decorated +[`Protocols`](https://peps.python.org/pep-0544/) ("interfaces" in other +languages) to declare the expected behavior of the ``Services`` that are loaded +at runtime in the ``Environments`` that use them. + +For example, the :py:class:`.SupportsFileShareOps` Protocol declares the +expected behavior of a Service that can +:py:meth:`~.SupportsFileShareOps.download` and +:py:meth:`~.SupportsFileShareOps.upload` files to and from a remote file share. + +But we can have more than one Service that implements that Protocol (e.g., one +for Azure, one for AWS, one for a remote SSH server, etc.). + +This allows us to define the expected behavior of the Service that the +Environment will need, but not the specific implementation details. + +It also allows users to define Environment configs that are more reusable so +that we can swap out the Service implementations at runtime without having to +change the Environment config. + +That way we can run Experiments on more than one platform rather easily. + +See the classes below for an overview of the types of Services that are +currently available for Environments. + +Notes +----- +If you find that there are missing types or that you need to add a new Service +type, please `submit a PR `_ to add it here. """ from mlos_bench.services.types.authenticator_type import SupportsAuth diff --git a/mlos_bench/mlos_bench/services/types/bound_method.py b/mlos_bench/mlos_bench/services/types/bound_method.py index 8e6179cffe3..a4c89dafbb9 100644 --- a/mlos_bench/mlos_bench/services/types/bound_method.py +++ b/mlos_bench/mlos_bench/services/types/bound_method.py @@ -2,7 +2,13 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Protocol representing a bound method.""" +"""Protocol representing a bound method. + +Notes +----- +Mostly just used for type checking of +:py:class:`~mlos_bench.services.base_service.Service` mix-ins. +""" from typing import Any, Protocol, runtime_checkable diff --git a/mlos_bench/mlos_bench/services/types/config_loader_type.py b/mlos_bench/mlos_bench/services/types/config_loader_type.py index 94d52ed41a3..49442ab632c 100644 --- a/mlos_bench/mlos_bench/services/types/config_loader_type.py +++ b/mlos_bench/mlos_bench/services/types/config_loader_type.py @@ -2,7 +2,12 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Protocol interface for helper functions to lookup and load configs.""" +"""Protocol interface for helper functions to lookup and load configs. + +See Also +-------- +:py:class:`~mlos_bench.services.config_persistence_service.ConfigPersistenceService` +""" from __future__ import annotations diff --git a/mlos_bench/mlos_bench/services/types/remote_config_type.py b/mlos_bench/mlos_bench/services/types/remote_config_type.py index 09a1e8c20fe..777655eb21f 100644 --- a/mlos_bench/mlos_bench/services/types/remote_config_type.py +++ b/mlos_bench/mlos_bench/services/types/remote_config_type.py @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Protocol interface for configuring cloud services.""" +"""Protocol interface for configuring cloud (Saas) services.""" from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable From 092fc287700369e07309d737957879bddfadbc96 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 23 Jan 2025 16:48:29 +0000 Subject: [PATCH 02/11] docformat --- mlos_bench/mlos_bench/services/types/__init__.py | 3 ++- mlos_bench/mlos_bench/services/types/bound_method.py | 3 ++- mlos_bench/mlos_bench/services/types/config_loader_type.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mlos_bench/mlos_bench/services/types/__init__.py b/mlos_bench/mlos_bench/services/types/__init__.py index 65e1ec0d1b3..53f4ae96be9 100644 --- a/mlos_bench/mlos_bench/services/types/__init__.py +++ b/mlos_bench/mlos_bench/services/types/__init__.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Service types (i.e., :py:class:`Protocol`s) for declaring implementation +""" +Service types (i.e., :py:class:`Protocol`s) for declaring implementation :py:class:`~mlos_bench.services.base_service.Service` behavior for :py:mod:`~mlos_bench.environments` to use in :py:mod:`mlos_bench`. diff --git a/mlos_bench/mlos_bench/services/types/bound_method.py b/mlos_bench/mlos_bench/services/types/bound_method.py index a4c89dafbb9..10039064aa7 100644 --- a/mlos_bench/mlos_bench/services/types/bound_method.py +++ b/mlos_bench/mlos_bench/services/types/bound_method.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Protocol representing a bound method. +""" +Protocol representing a bound method. Notes ----- diff --git a/mlos_bench/mlos_bench/services/types/config_loader_type.py b/mlos_bench/mlos_bench/services/types/config_loader_type.py index 49442ab632c..5a0b5da4fb5 100644 --- a/mlos_bench/mlos_bench/services/types/config_loader_type.py +++ b/mlos_bench/mlos_bench/services/types/config_loader_type.py @@ -2,7 +2,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Protocol interface for helper functions to lookup and load configs. +""" +Protocol interface for helper functions to lookup and load configs. See Also -------- From 04acaecd5ba1084366317cdb6b7f5705a7a7411c Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 23 Jan 2025 17:00:35 +0000 Subject: [PATCH 03/11] doc fixups --- mlos_bench/mlos_bench/services/types/__init__.py | 10 +++++----- .../mlos_bench/services/types/config_loader_type.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mlos_bench/mlos_bench/services/types/__init__.py b/mlos_bench/mlos_bench/services/types/__init__.py index 53f4ae96be9..32c443b15c1 100644 --- a/mlos_bench/mlos_bench/services/types/__init__.py +++ b/mlos_bench/mlos_bench/services/types/__init__.py @@ -3,25 +3,25 @@ # Licensed under the MIT License. # """ -Service types (i.e., :py:class:`Protocol`s) for declaring implementation +Service types (i.e., :external:py:class:`~typing.Protocol`) for declaring implementation :py:class:`~mlos_bench.services.base_service.Service` behavior for :py:mod:`~mlos_bench.environments` to use in :py:mod:`mlos_bench`. Overview -------- Service loading in ``mlos_bench`` uses a -[mix-in](https://en.wikipedia.org/wiki/Mixin#In_Python) approach to combine the +`mix-in `_ approach to combine the functionality of multiple classes specified at runtime through config files into a single class that the :py:mod:`~mlos_bench.environments` can use to invoke the actions that they were configured to perform (e.g., provisioning a VM, deploying a network, running a script, etc.). Since Services are loaded at runtime and can be swapped out by referencing a -different set of ``--services`` config files via the :py:mod:`mlos_bench.run` +different set of config files via the ``--services`` :py:mod:`mlos_bench.run` CLI option, this can make it difficult to do type and config checking. -To address this we define ``@runtime_checkable`` decorated -[`Protocols`](https://peps.python.org/pep-0544/) ("interfaces" in other +To address this we define :external:py:func:`~typing.runtime_checkable` decorated +`Protocols `_ (e.g., *interfaces* in other languages) to declare the expected behavior of the ``Services`` that are loaded at runtime in the ``Environments`` that use them. diff --git a/mlos_bench/mlos_bench/services/types/config_loader_type.py b/mlos_bench/mlos_bench/services/types/config_loader_type.py index 5a0b5da4fb5..e33e129d4e7 100644 --- a/mlos_bench/mlos_bench/services/types/config_loader_type.py +++ b/mlos_bench/mlos_bench/services/types/config_loader_type.py @@ -7,7 +7,7 @@ See Also -------- -:py:class:`~mlos_bench.services.config_persistence_service.ConfigPersistenceService` +:py:class:`~mlos_bench.services.config_persistence.ConfigPersistenceService` """ from __future__ import annotations From df5afc46bffd24c012200c3141c335059ac76522 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 23 Jan 2025 17:17:52 +0000 Subject: [PATCH 04/11] basic example and some docs --- .../mlos_bench/services/config_persistence.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/mlos_bench/mlos_bench/services/config_persistence.py b/mlos_bench/mlos_bench/services/config_persistence.py index 5ecf68a042d..18ff525c878 100644 --- a/mlos_bench/mlos_bench/services/config_persistence.py +++ b/mlos_bench/mlos_bench/services/config_persistence.py @@ -7,9 +7,33 @@ benchmark :py:class:`.Environment`, :py:mod:`~mlos_bench.tunables`, :py:class:`.Service` functions, etc from JSON configuration files and strings. +Typically the :py:class:`.ConfigPersistenceService` is provided automatically by +the :py:mod:`mlos_bench.launcher`. + +It's ``config_path`` parameter is a list of directories to search for the +configuration files referenced in other JSON config files or +:py:mod:`mlos_bench.run` ``--cli-options``. + +That value can itself be adjusted with the ``--config-path`` CLI option or set +in a ``--config`` CLI options file. + +Regardless of the values there, the service will always search the included +config files from the :py:mod:`mlos_bench` package. + See Also -------- mlos_bench.config : Overview of the configuration system. +mlos_bench.run : CLI options for the ``mlos_bench`` command. + +Examples +-------- +>>> from importlib.resources import files +>>> from os.path import abspath, samefile +>>> expected_file = abspath(files("mlos_bench.config").joinpath( +... "optimizers/mlos_core_default_opt.jsonc")) +>>> service = ConfigPersistenceService(config={"config_path": ["./config"]}) +>>> resolved_file = service.resolve_path("optimizers/mlos_core_default_opt.jsonc") +>>> assert samefile(resolved_file, expected_file) """ import logging From 8df1ce8dbb29188000c93115f7ac9ab3ac22df94 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 23 Jan 2025 21:51:56 +0000 Subject: [PATCH 05/11] todo comments --- mlos_bench/mlos_bench/services/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mlos_bench/mlos_bench/services/__init__.py b/mlos_bench/mlos_bench/services/__init__.py index 65ffc8e8d80..5e249c4a3dc 100644 --- a/mlos_bench/mlos_bench/services/__init__.py +++ b/mlos_bench/mlos_bench/services/__init__.py @@ -6,6 +6,18 @@ Services for implementing Environments for mlos_bench. TODO: Improve documentation here. + +Overview +-------- +TODO: Explain Service mix-ins and how they get used with Environments. + +Config +------ +TODO: Explain how to configure Services. + +See Also +-------- +TODO: Provide references to different related classes. """ from mlos_bench.services.base_fileshare import FileShareService From ea6d3d42b4dd32c16a23d248cb0df06ce429440f Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 23 Jan 2025 21:52:07 +0000 Subject: [PATCH 06/11] Expanded example docs --- .../mlos_bench/services/config_persistence.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/mlos_bench/mlos_bench/services/config_persistence.py b/mlos_bench/mlos_bench/services/config_persistence.py index 18ff525c878..e3f237230c2 100644 --- a/mlos_bench/mlos_bench/services/config_persistence.py +++ b/mlos_bench/mlos_bench/services/config_persistence.py @@ -27,13 +27,31 @@ Examples -------- +>>> # Create a new instance of the ConfigPersistenceService. +>>> # This will search for config files in the current directory's config subdir. +>>> # It will also search in the built-in config files that come with the package. +>>> # And the current working directory. +>>> service = ConfigPersistenceService(config={"config_path": ["./config"]}) + +>>> # One of the things the ConfigLoaderType does is find and load config files +>>> # referenced in other JSON files. +>>> # For instance: +>>> config_file_path = "optimizers/mlos_core_default_opt.jsonc" +>>> resolved_file = service.resolve_path(config_file_path) + +>>> # The resolved file should be the same as the expected file. +>>> # That is, the resolved file should be the same as the built-in file. >>> from importlib.resources import files >>> from os.path import abspath, samefile ->>> expected_file = abspath(files("mlos_bench.config").joinpath( -... "optimizers/mlos_core_default_opt.jsonc")) ->>> service = ConfigPersistenceService(config={"config_path": ["./config"]}) ->>> resolved_file = service.resolve_path("optimizers/mlos_core_default_opt.jsonc") +>>> expected_file = abspath(files("mlos_bench.config").joinpath(config_file_path)) >>> assert samefile(resolved_file, expected_file) + +>>> # Create an Optimizer from that file. +>>> from mlos_bench.config.schemas.config_schemas import ConfigSchema +>>> config = service.load_config(config_file_path, schema_type=ConfigSchema.OPTIMIZER) +>>> optimizer = service.build_optimizer(tunables=TunableGroups(), service=service, config=config) +>>> from mlos_bench.optimizers.base_optimizer import Optimizer +>>> assert isinstance(optimizer, Optimizer) """ import logging From 03522686a4900be3179296ba0946960686c0275d Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 23 Jan 2025 21:56:55 +0000 Subject: [PATCH 07/11] fix headers --- mlos_bench/mlos_bench/config/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlos_bench/mlos_bench/config/__init__.py b/mlos_bench/mlos_bench/config/__init__.py index 508d09c7a40..8067bd21421 100644 --- a/mlos_bench/mlos_bench/config/__init__.py +++ b/mlos_bench/mlos_bench/config/__init__.py @@ -218,7 +218,7 @@ provision a numbered VM per worker). Tunable Configs -^^^^^^^^^^^^^^^ ++++++++++++++++ There are two forms of tunable configs: @@ -282,7 +282,7 @@ module. Class Configs -^^^^^^^^^^^^^ ++++++++++++++ Class style configs include most anything else and roughly take this form: From 17d8751a093d425cc87a1e5ec45b7b1e0cbbff8d Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 23 Jan 2025 21:57:33 +0000 Subject: [PATCH 08/11] Revert "fix headers" This reverts commit 03522686a4900be3179296ba0946960686c0275d. --- mlos_bench/mlos_bench/config/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlos_bench/mlos_bench/config/__init__.py b/mlos_bench/mlos_bench/config/__init__.py index 8067bd21421..508d09c7a40 100644 --- a/mlos_bench/mlos_bench/config/__init__.py +++ b/mlos_bench/mlos_bench/config/__init__.py @@ -218,7 +218,7 @@ provision a numbered VM per worker). Tunable Configs -+++++++++++++++ +^^^^^^^^^^^^^^^ There are two forms of tunable configs: @@ -282,7 +282,7 @@ module. Class Configs -+++++++++++++ +^^^^^^^^^^^^^ Class style configs include most anything else and roughly take this form: From 766d2edcfdbb7e0081051689cbb5923e9c63235c Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 23 Jan 2025 22:03:50 +0000 Subject: [PATCH 09/11] fixups --- mlos_bench/mlos_bench/config/__init__.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/mlos_bench/mlos_bench/config/__init__.py b/mlos_bench/mlos_bench/config/__init__.py index 508d09c7a40..455f94ed02d 100644 --- a/mlos_bench/mlos_bench/config/__init__.py +++ b/mlos_bench/mlos_bench/config/__init__.py @@ -178,7 +178,7 @@ for some examples of CLI configs. Globals and Variable Substitution -+++++++++++++++++++++++++++++++++ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :py:attr:`Globals ` are basically just key-value variables that can be used in other configs using @@ -208,14 +208,15 @@ Here is a list of some well known variables that are provided or required by the system and may be used in the config files: -- ``$experiment_id``: A unique identifier for the ``Experiment``. +- ``$experiment_id`` : A unique identifier for the ``Experiment``. Typically provided in globals. -- ``$trial_id``: A unique identifier for the ``Trial`` currently being executed. +- ``$trial_id`` : A unique identifier for the ``Trial`` currently being executed. This can be useful in the configs for :py:mod:`mlos_bench.environments` for instance (e.g., when writing scripts). -- ``$trial_runner_id``: A unique identifier for the ``TrialRunner``. - This can be useful when running multiple trials in parallel (e.g., to - provision a numbered VM per worker). +- ``$trial_runner_id`` : A unique identifier for the + :py:class:`~mlos_bench.schedulers.trial_runner.TrialRunner`. This can be + useful when running multiple trials in parallel (e.g., to provision a + numbered VM per worker). Tunable Configs ^^^^^^^^^^^^^^^ @@ -311,7 +312,7 @@ expected structure of the ``config`` section. In certain cases (e.g., script command execution) the variable substitution rules -take on slightly different behavior +take on slightly different behavior. See various documentation in :py:mod:`mlos_bench.environments` for more details. Config Processing From b70b71e23a42322d3fca7ebe0e713c06389355d2 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 23 Jan 2025 22:09:33 +0000 Subject: [PATCH 10/11] rendering --- mlos_bench/mlos_bench/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index 519ee74ba63..a8d9d4017bf 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -158,9 +158,11 @@ ... --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc \ ... --globals experiment_test_local.jsonc \ ... --tunable_values tunable-values/tunable-values-local.jsonc" + >>> print(f"Here's the shell command you'd actually run:\n# {cmd}") Here's the shell command you'd actually run: # mlos_bench --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc --globals experiment_test_local.jsonc --tunable_values tunable-values/tunable-values-local.jsonc + >>> # Now we run the command and check the output. >>> result = run(cmd, shell=True, capture_output=True, text=True, check=True) >>> assert result.returncode == 0 From 684dd8da86c8e19072e642b7ab832e16c5f7feb3 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 23 Jan 2025 22:10:32 +0000 Subject: [PATCH 11/11] again --- mlos_bench/mlos_bench/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index a8d9d4017bf..681faed56b5 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -150,7 +150,6 @@ The entry point for these configs can be found `here `_. ->>> from subprocess import run >>> # Note: we show the command wrapped in python here for testing purposes. >>> # Alternatively replace test-cli-local-env-bench.jsonc with >>> # test-cli-local-env-opt.jsonc for one that does an optimization loop. @@ -164,6 +163,7 @@ # mlos_bench --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc --globals experiment_test_local.jsonc --tunable_values tunable-values/tunable-values-local.jsonc >>> # Now we run the command and check the output. +>>> from subprocess import run >>> result = run(cmd, shell=True, capture_output=True, text=True, check=True) >>> assert result.returncode == 0 >>> lines = result.stderr.splitlines()