diff --git a/.pylintrc b/.pylintrc index e80e57c..fae5f05 100644 --- a/.pylintrc +++ b/.pylintrc @@ -444,6 +444,7 @@ disable=bad-inline-option, too-many-arguments, too-many-branches, too-many-instance-attributes, + too-many-lines, too-many-locals, too-many-positional-arguments, too-many-public-methods, diff --git a/edq/cli/config/__init__.py b/edq/cli/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/edq/cli/config/list.py b/edq/cli/config/list.py new file mode 100644 index 0000000..d630627 --- /dev/null +++ b/edq/cli/config/list.py @@ -0,0 +1,65 @@ +""" +List current configuration options. +""" + +import argparse +import sys + +import edq.core.argparser + +CONFIG_FIELD_SEPARATOR: str = "\t" + +def run_cli(args: argparse.Namespace) -> int: + """ Run the CLI. """ + + rows = [] + + for (key, value) in args._config.items(): + row = [key, str(value)] + if (args.show_origin): + config_source_obj = args._config_sources.get(key) + + origin = config_source_obj.path + if (origin is None): + origin = config_source_obj.label + + row.append(origin) + + rows.append(CONFIG_FIELD_SEPARATOR.join(row)) + + rows.sort() + + if (not args.skip_header): + header = ["Key", "Value"] + if (args.show_origin): + header.append("Origin") + + rows.insert(0, (CONFIG_FIELD_SEPARATOR.join(header))) + + print("\n".join(rows)) + return 0 + +def main() -> int: + """ Get a parser, parse the args, and call run. """ + + return run_cli(_get_parser().parse_args()) + +def _get_parser() -> edq.core.argparser.Parser: + """ Get a parser and add addition flags. """ + + parser = edq.core.argparser.get_default_parser(__doc__.strip()) + + parser.add_argument("--show-origin", dest = 'show_origin', + action = 'store_true', + help = "Display where each configuration's value was obtained from.", + ) + + parser.add_argument("--skip-header", dest = 'skip_header', + action = 'store_true', + help = 'Skip headers when displaying configs.', + ) + + return parser + +if (__name__ == '__main__'): + sys.exit(main()) diff --git a/edq/cli/version.py b/edq/cli/version.py index afa7115..2c93172 100644 --- a/edq/cli/version.py +++ b/edq/cli/version.py @@ -16,6 +16,7 @@ def run_cli(args: argparse.Namespace) -> int: def main() -> int: """ Get a parser, parse the args, and call run. """ + return run_cli(_get_parser().parse_args()) def _get_parser() -> edq.core.argparser.Parser: diff --git a/edq/core/argparser.py b/edq/core/argparser.py index f14ab20..ae8fb87 100644 --- a/edq/core/argparser.py +++ b/edq/core/argparser.py @@ -10,6 +10,7 @@ import argparse import typing +import edq.core.config import edq.core.log @typing.runtime_checkable @@ -108,5 +109,6 @@ def get_default_parser(description: str) -> Parser: parser = Parser(description = description) parser.register_callbacks('log', edq.core.log.set_cli_args, edq.core.log.init_from_args) + parser.register_callbacks('config', edq.core.config.set_cli_args, edq.core.config.load_config_into_args) return parser diff --git a/edq/core/config.py b/edq/core/config.py index 0881165..39256fd 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -9,11 +9,15 @@ CONFIG_SOURCE_GLOBAL: str = "" CONFIG_SOURCE_LOCAL: str = "" -CONFIG_SOURCE_CLI: str = "" -CONFIG_SOURCE_CLI_BARE: str = "" +CONFIG_SOURCE_CLI_FILE: str = "" +CONFIG_SOURCE_CLI: str = "" CONFIG_PATHS_KEY: str = 'config_paths' +CONFIGS_KEY: str = 'configs' +GLOBAL_CONFIG_KEY: str = 'global_config_path' +IGNORE_CONFIGS_KEY: str = 'ignore_configs' DEFAULT_CONFIG_FILENAME: str = "edq-config.json" +DEFAULT_GLOBAL_CONFIG_PATH: str = platformdirs.user_config_dir(DEFAULT_CONFIG_FILENAME) class ConfigSource: """ A class for storing config source information. """ @@ -23,7 +27,7 @@ def __init__(self, label: str, path: typing.Union[str, None] = None) -> None: """ The label identifying the config (see CONFIG_SOURCE_* constants). """ self.path = path - """ The path of where the config was soruced from. """ + """ The path of where the config was sourced from. """ def __eq__(self, other: object) -> bool: if (not isinstance(other, ConfigSource)): @@ -37,8 +41,6 @@ def __str__(self) -> str: def get_tiered_config( config_file_name: str = DEFAULT_CONFIG_FILENAME, legacy_config_file_name: typing.Union[str, None] = None, - global_config_path: typing.Union[str, None] = None, - skip_keys: typing.Union[list, None] = None, cli_arguments: typing.Union[dict, argparse.Namespace, None] = None, local_config_root_cutoff: typing.Union[str, None] = None, ) -> typing.Tuple[typing.Dict[str, str], typing.Dict[str, ConfigSource]]: @@ -47,12 +49,6 @@ def get_tiered_config( Returns a configuration dictionary with the values based on tiering rules and a source dictionary mapping each key to its origin. """ - if (global_config_path is None): - global_config_path = platformdirs.user_config_dir(config_file_name) - - if (skip_keys is None): - skip_keys = [CONFIG_PATHS_KEY] - if (cli_arguments is None): cli_arguments = {} @@ -63,6 +59,8 @@ def get_tiered_config( if (isinstance(cli_arguments, argparse.Namespace)): cli_arguments = vars(cli_arguments) + global_config_path = cli_arguments.get(GLOBAL_CONFIG_KEY, platformdirs.user_config_dir(config_file_name)) + # Check the global user config file. if (os.path.isfile(global_config_path)): _load_config_file(global_config_path, config, sources, CONFIG_SOURCE_GLOBAL) @@ -79,20 +77,32 @@ def get_tiered_config( # Check the config file specified on the command-line. config_paths = cli_arguments.get(CONFIG_PATHS_KEY, []) - if (config_paths is not None): - for path in config_paths: - _load_config_file(path, config, sources, CONFIG_SOURCE_CLI) + for path in config_paths: + _load_config_file(path, config, sources, CONFIG_SOURCE_CLI_FILE) - # Finally, any command-line options. - for (key, value) in cli_arguments.items(): - if (key in skip_keys): - continue + # Check the command-line config options. + cli_configs = cli_arguments.get(CONFIGS_KEY, []) + for cli_config in cli_configs: + if ("=" not in cli_config): + raise ValueError( + f"Invalid configuration option '{cli_config}'." + + " Configuration options must be provided in the format `=` when passed via the CLI." + ) - if ((value is None) or (value == '')): - continue + (key, value) = cli_config.split("=", maxsplit = 1) + + key = key.strip() + if (key == ""): + raise ValueError(f"Found an empty configuration option key associated with the value '{value}'.") config[key] = value - sources[key] = ConfigSource(label = CONFIG_SOURCE_CLI_BARE) + sources[key] = ConfigSource(label = CONFIG_SOURCE_CLI) + + # Finally, ignore any configs that is specified from CLI command. + cli_ignore_configs = cli_arguments.get(IGNORE_CONFIGS_KEY, []) + for ignore_config in cli_ignore_configs: + config.pop(ignore_config, None) + sources.pop(ignore_config, None) return config, sources @@ -106,6 +116,10 @@ def _load_config_file( config_path = os.path.abspath(config_path) for (key, value) in edq.util.json.load_path(config_path).items(): + key = key.strip() + if (key == ""): + raise ValueError(f"Found an empty configuration option key associated with the value '{value}'.") + config[key] = value sources[key] = ConfigSource(label = source_label, path = config_path) @@ -175,3 +189,55 @@ def _get_ancestor_config_file_path( current_directory = parent_dir return None + +def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, typing.Any]) -> None: + """ + Set common CLI arguments for configuration. + """ + + parser.add_argument('--config-global', dest = GLOBAL_CONFIG_KEY, + action = 'store', type = str, default = DEFAULT_GLOBAL_CONFIG_PATH, + help = 'Set the default global config file path (default: %(default)s).', + ) + + parser.add_argument('--config-file', dest = CONFIG_PATHS_KEY, + action = 'append', type = str, default = [], + help = ('Load config options from a JSON file.' + + ' This flag can be specified multiple times.' + + ' Files are applied in the order provided and later files override earlier ones.' + + ' Will override options form both global and local config files.') + ) + + parser.add_argument('--config', dest = CONFIGS_KEY, + action = 'append', type = str, default = [], + help = ('Set a configuration option from the command-line.' + + ' Specify options as = pairs.' + + ' This flag can be specified multiple times.' + + ' The options are applied in the order provided and later options override earlier ones.' + + ' Will override options form all config files.') + ) + + parser.add_argument('--ignore-config-option', dest = IGNORE_CONFIGS_KEY, + action = 'append', type = str, default = [], + help = ('Ignore any config option with the specified key.' + + ' The system-provided default value will be used for that option if one exists.' + + ' This flag can be specified multiple times.' + + ' Ignored options are processed last.') + ) + +def load_config_into_args( + parser: argparse.ArgumentParser, + args: argparse.Namespace, + extra_state: typing.Dict[str, typing.Any], + ) -> None: + """ + Take in args from a parser that was passed to set_cli_args(), + and get the tired configuration with the appropriate parameters, and attache it to args. + """ + + (config_dict, sources_dict) = get_tiered_config( + cli_arguments = args, + ) + + setattr(args, "_config", config_dict) + setattr(args, "_config_sources", sources_dict) diff --git a/edq/core/config_test.py b/edq/core/config_test.py index 916ca2d..4e5c314 100644 --- a/edq/core/config_test.py +++ b/edq/core/config_test.py @@ -18,10 +18,14 @@ def creat_test_dir(temp_dir_prefix: str) -> str: ├── empty │   └── edq-config.json ├── empty-dir + ├── empty-key + │   └── edq-config.json ├── global │   └── edq-config.json ├── malformed │   └── edq-config.json + ├── multiple-options + │   └── edq-config.json ├── nested │   ├── config.json │   ├── edq-config.json @@ -41,11 +45,22 @@ def creat_test_dir(temp_dir_prefix: str) -> str: empty_config_dir_path = os.path.join(temp_dir, "empty") edq.util.dirent.mkdir(empty_config_dir_path) - edq.util.json.dump_path({}, os.path.join(empty_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME)) + edq.util.json.dump_path({}, os.path.join(empty_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME)) + + multiple_option_config_dir_path = os.path.join(temp_dir, "multiple-options") + edq.util.dirent.mkdir(multiple_option_config_dir_path) + edq.util.json.dump_path( + {"user": "user@test.edulinq.org", "pass": "password1234"}, + os.path.join(multiple_option_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME) + ) + + empty_key_config_dir_path = os.path.join(temp_dir, "empty-key") + edq.util.dirent.mkdir(empty_key_config_dir_path) + edq.util.json.dump_path({"": "user@test.edulinq.org"}, os.path.join(empty_key_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME)) - custome_name_config_dir_path = os.path.join(temp_dir, "custom-name") - edq.util.dirent.mkdir(custome_name_config_dir_path) - edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(custome_name_config_dir_path, "custom-edq-config.json")) + custom_name_config_dir_path = os.path.join(temp_dir, "custom-name") + edq.util.dirent.mkdir(custom_name_config_dir_path) + edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(custom_name_config_dir_path, "custom-edq-config.json")) edq.util.dirent.mkdir(os.path.join(temp_dir, "dir-config", "edq-config.json")) edq.util.dirent.mkdir(os.path.join(temp_dir, "empty-dir")) @@ -112,7 +127,9 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, }, { "user": "user@test.edulinq.org", @@ -130,18 +147,35 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "empty", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "empty", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, }, {}, {}, None, ), + # Empty Key Config JSON + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "empty-key", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, + }, + {}, + {}, + "Found an empty configuration option key associated with the value 'user@test.edulinq.org'.", + ), + # Directory Config JSON ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "dir-config", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "dir-config", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, }, {}, {}, @@ -152,7 +186,9 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "empty-dir", "non-existent-config.json"), + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "empty-dir", "non-existent-config.json"), + }, }, {}, {}, @@ -163,13 +199,61 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "malformed", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "malformed", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, }, {}, {}, "Failed to read JSON file", ), + # Ignore Config Option + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "multiple-options", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.IGNORE_CONFIGS_KEY: [ + "pass", + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_GLOBAL, + path = os.path.join(temp_dir, "multiple-options", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + + # Ignore Non-Existing Config Option + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.IGNORE_CONFIGS_KEY: [ + "non-existing-option", + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_GLOBAL, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + # Local Config # Default config file in current directory. @@ -260,6 +344,15 @@ def test_get_tiered_config_base(self): None, ), + # Empty Key Config JSON + ( + "empty-key", + {}, + {}, + {}, + "Found an empty configuration option key associated with the value 'user@test.edulinq.org'.", + ), + # Directory Config JSON ( "dir-config", @@ -278,6 +371,51 @@ def test_get_tiered_config_base(self): "Failed to read JSON file", ), + # Ignore Config Option + ( + "multiple-options", + { + "cli_arguments":{ + edq.core.config.IGNORE_CONFIGS_KEY: [ + "pass", + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, + path = os.path.join(temp_dir, "multiple-options", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + + # Ignore Non-Existing Config Option + ( + "simple", + { + "cli_arguments":{ + edq.core.config.IGNORE_CONFIGS_KEY: [ + "non-existing-option", + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + + # All 3 local config locations present at the same time. ( os.path.join("nested", "nest1", "nest2b"), @@ -315,11 +453,11 @@ def test_get_tiered_config_base(self): }, { "user": edq.core.config.ConfigSource( - label = edq.core.config.CONFIG_SOURCE_CLI, + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ), "server": edq.core.config.ConfigSource( - label = edq.core.config.CONFIG_SOURCE_CLI, + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, path = os.path.join(temp_dir, "nested", edq.core.config.DEFAULT_CONFIG_FILENAME), ), }, @@ -342,7 +480,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.core.config.ConfigSource( - label = edq.core.config.CONFIG_SOURCE_CLI, + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ), }, @@ -364,6 +502,21 @@ def test_get_tiered_config_base(self): None, ), + # Empty Key Config JSON + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "empty-key", edq.core.config.DEFAULT_CONFIG_FILENAME), + ], + }, + }, + {}, + {}, + "Found an empty configuration option key associated with the value 'user@test.edulinq.org'.", + ), + # Directory Config JSON ( "empty-dir", @@ -409,42 +562,187 @@ def test_get_tiered_config_base(self): "Failed to read JSON file", ), - # CLI Bare Options: + # Ignore Config Option + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "multiple-options", edq.core.config.DEFAULT_CONFIG_FILENAME), + ], + edq.core.config.IGNORE_CONFIGS_KEY: [ + "pass", + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, + path = os.path.join(temp_dir, "multiple-options", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + + # Ignore Non-Existing Config Option + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ], + edq.core.config.IGNORE_CONFIGS_KEY: [ + "non-existing-option", + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + + + # CLI Options: # CLI arguments only (direct key: value). ( "empty-dir", { "cli_arguments": { - "user": "user@test.edulinq.org", + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], }, }, { "user": "user@test.edulinq.org", }, { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), - # Skip keys functionally. + # Empty Config Key ( "empty-dir", { "cli_arguments": { - "user": "user@test.edulinq.org", - "pass": "user", + edq.core.config.CONFIGS_KEY: [ + "=user@test.edulinq.org", + ], + }, + }, + {}, + {}, + "Found an empty configuration option key associated with the value 'user@test.edulinq.org'.", + ), + + # Empty Config Value + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIGS_KEY: [ + "user=", + ], + }, + }, + { + "user": "", + }, + { + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), + }, + None, + ), + + # Separator In Config Value + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIGS_KEY: [ + "pass=password=1234", + ], + }, + }, + { + "pass": "password=1234", + }, + { + "pass": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), + }, + None, + ), + + # Invalid Config Option Format + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIGS_KEY: [ + "useruser@test.edulinq.org", + ], + }, + }, + {}, + {}, + "Invalid configuration option 'useruser@test.edulinq.org'.", + ), + + # Ignore Config Option + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + "pass=password1234", + ], + edq.core.config.IGNORE_CONFIGS_KEY:[ + "pass", + ], }, - "skip_keys": [ - "pass", - ], }, { "user": "user@test.edulinq.org", }, { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), + }, + None, + ), + + # Ignore Non-Existing Config Option + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], + edq.core.config.IGNORE_CONFIGS_KEY:[ + "non-existing-option", + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), @@ -455,7 +753,9 @@ def test_get_tiered_config_base(self): ( "simple", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, }, { "user": "user@test.edulinq.org", @@ -473,11 +773,11 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ], + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, }, { @@ -485,7 +785,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.core.config.ConfigSource( - label = edq.core.config.CONFIG_SOURCE_CLI, + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ), }, @@ -496,16 +796,18 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { - "user": "user@test.edulinq.org", + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, }, { "user": "user@test.edulinq.org", }, { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), @@ -525,7 +827,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.core.config.ConfigSource( - label = edq.core.config.CONFIG_SOURCE_CLI, + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ), }, @@ -537,14 +839,16 @@ def test_get_tiered_config_base(self): "simple", { "cli_arguments": { - "user": "user@test.edulinq.org", + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], }, }, { "user": "user@test.edulinq.org", }, { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), @@ -554,7 +858,9 @@ def test_get_tiered_config_base(self): "empty-dir", { "cli_arguments": { - "user": "user@test.edulinq.org", + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ], @@ -564,7 +870,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), @@ -573,19 +879,21 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { - "user": "user@test.edulinq.org", + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ], + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, }, { "user": "user@test.edulinq.org", }, { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), @@ -594,16 +902,18 @@ def test_get_tiered_config_base(self): ( "simple", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { - "user": "user@test.edulinq.org", + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, }, { "user": "user@test.edulinq.org", }, { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), @@ -612,11 +922,11 @@ def test_get_tiered_config_base(self): ( "simple", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ], + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, }, { @@ -624,7 +934,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.core.config.ConfigSource( - label = edq.core.config.CONFIG_SOURCE_CLI, + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ), }, @@ -636,7 +946,9 @@ def test_get_tiered_config_base(self): "simple", { "cli_arguments": { - "user": "user@test.edulinq.org", + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ], @@ -646,7 +958,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), @@ -655,29 +967,26 @@ def test_get_tiered_config_base(self): ( "simple", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ], - "pass": "user", - "server": "http://test.edulinq.org", + edq.core.config.CONFIGS_KEY: [ + "pass=password1234", + ], + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, - "skip_keys": [ - "server", - edq.core.config.CONFIG_PATHS_KEY, - ], }, { "user": "user@test.edulinq.org", - "pass": "user", + "pass": "password1234", }, { "user": edq.core.config.ConfigSource( - label = edq.core.config.CONFIG_SOURCE_CLI, + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ), - "pass": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "pass": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), @@ -687,9 +996,17 @@ def test_get_tiered_config_base(self): (test_work_dir, extra_args, expected_config, expected_source, error_substring) = test_case with self.subTest(msg = f"Case {i} ('{test_work_dir}'):"): - global_config = extra_args.get("global_config_path", None) - if (global_config is None): - extra_args["global_config_path"] = os.path.join(temp_dir, "empty", edq.core.config.CONFIG_PATHS_KEY) + cli_args = extra_args.get("cli_arguments", None) + if cli_args is None: + extra_args["cli_arguments"] = { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "empty", edq.core.config.DEFAULT_CONFIG_FILENAME) + } + else: + cli_global_config_path = cli_args.get(edq.core.config.GLOBAL_CONFIG_KEY, None) + if cli_global_config_path is None: + extra_args["cli_arguments"][edq.core.config.GLOBAL_CONFIG_KEY] = os.path.join( + temp_dir, "empty", edq.core.config.DEFAULT_CONFIG_FILENAME + ) cutoff = extra_args.get("local_config_root_cutoff", None) if (cutoff is None): diff --git a/edq/testing/cli.py b/edq/testing/cli.py index f0ef34a..a8350af 100644 --- a/edq/testing/cli.py +++ b/edq/testing/cli.py @@ -30,6 +30,7 @@ TEST_CASE_SEP: str = '---' DATA_DIR_ID: str = '__DATA_DIR__' +ABS_DATA_DIR_ID: str = '__ABS_DATA_DIR__' TEMP_DIR_ID: str = '__TEMP_DIR__' BASE_DIR_ID: str = '__BASE_DIR__' @@ -175,6 +176,7 @@ def _expand_paths(self, text: str) -> str: (DATA_DIR_ID, self.data_dir), (TEMP_DIR_ID, self.temp_dir), (BASE_DIR_ID, self.base_dir), + (ABS_DATA_DIR_ID, os.path.abspath(self.data_dir)), ] for (key, target_dir) in replacements: diff --git a/edq/testing/testdata/cli/data/configs/empty/edq-config.json b/edq/testing/testdata/cli/data/configs/empty/edq-config.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/edq/testing/testdata/cli/data/configs/empty/edq-config.json @@ -0,0 +1 @@ +{} diff --git a/edq/testing/testdata/cli/data/configs/simple-1/edq-config.json b/edq/testing/testdata/cli/data/configs/simple-1/edq-config.json new file mode 100644 index 0000000..8257876 --- /dev/null +++ b/edq/testing/testdata/cli/data/configs/simple-1/edq-config.json @@ -0,0 +1,4 @@ +{ + "user": "user@test.edulinq.org", +} + diff --git a/edq/testing/testdata/cli/data/configs/simple-2/edq-config.json b/edq/testing/testdata/cli/data/configs/simple-2/edq-config.json new file mode 100644 index 0000000..43df80f --- /dev/null +++ b/edq/testing/testdata/cli/data/configs/simple-2/edq-config.json @@ -0,0 +1,3 @@ +{ + "api-key": "TEST-KEY-1234567890", +} diff --git a/edq/testing/testdata/cli/data/configs/value-number/edq-config.json b/edq/testing/testdata/cli/data/configs/value-number/edq-config.json new file mode 100644 index 0000000..ddf6976 --- /dev/null +++ b/edq/testing/testdata/cli/data/configs/value-number/edq-config.json @@ -0,0 +1,3 @@ +{ + "timeout": 60 +} diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_base.txt b/edq/testing/testdata/cli/tests/config/list/config_list_base.txt new file mode 100644 index 0000000..485defb --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/list/config_list_base.txt @@ -0,0 +1,16 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + "--config-file", "__DATA_DIR__(configs/simple-1/edq-config.json)", + "--config-file", "__DATA_DIR__(configs/simple-2/edq-config.json)", + "--config", "pass=password1234", + "--config", "server=http://test.edulinq.org" + ], +} +--- +Key Value +api-key TEST-KEY-1234567890 +pass password1234 +server http://test.edulinq.org +user user@test.edulinq.org diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_config_value_number.txt b/edq/testing/testdata/cli/tests/config/list/config_list_config_value_number.txt new file mode 100644 index 0000000..dd550e2 --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/list/config_list_config_value_number.txt @@ -0,0 +1,10 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + "--config-file", "__DATA_DIR__(configs/value-number/edq-config.json)", + ], +} +--- +Key Value +timeout 60 diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_ignore_config.txt b/edq/testing/testdata/cli/tests/config/list/config_list_ignore_config.txt new file mode 100644 index 0000000..7afbe36 --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/list/config_list_ignore_config.txt @@ -0,0 +1,14 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--ignore-config-option", "pass", + "--ignore-config-option", "api-key", + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + "--config-file", "__DATA_DIR__(configs/simple-1/edq-config.json)", + "--config-file", "__DATA_DIR__(configs/simple-2/edq-config.json)", + "--config", "pass=password1234", + ], +} +--- +Key Value +user user@test.edulinq.org diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_no_config.txt b/edq/testing/testdata/cli/tests/config/list/config_list_no_config.txt new file mode 100644 index 0000000..b0aa28f --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/list/config_list_no_config.txt @@ -0,0 +1,8 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + ], +} +--- +Key Value diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_show_origin.txt b/edq/testing/testdata/cli/tests/config/list/config_list_show_origin.txt new file mode 100644 index 0000000..be4e4c1 --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/list/config_list_show_origin.txt @@ -0,0 +1,13 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-file", "__DATA_DIR__(configs/simple-1/edq-config.json)", + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + "--config", "pass=password1234", + "--show-origin", + ], +} +--- +Key Value Origin +pass password1234 +user user@test.edulinq.org __ABS_DATA_DIR__(configs/simple-1/edq-config.json) diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_skip_header.txt b/edq/testing/testdata/cli/tests/config/list/config_list_skip_header.txt new file mode 100644 index 0000000..64d3b8a --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/list/config_list_skip_header.txt @@ -0,0 +1,10 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + "--config", "user=user@test.edulinq.org", + "--skip-header", + ], +} +--- +user user@test.edulinq.org