diff --git a/README.md b/README.md index 2062633..06abbf3 100644 --- a/README.md +++ b/README.md @@ -16,3 +16,95 @@ The project and Python dependencies can be installed from source with: ``` pip3 install . ``` + +## Configuration System + +This project provides a configuration system that supplies options (e.g., username, password) to a command-line interface (CLI) tool. +The configuration system follows a tiered order, allowing options to be specified and overridden from both files and command-line options. + +### Configuration Sources + +In addition to CLI options, the configuration system loads options from [JSON](https://en.wikipedia.org/wiki/JSON) files located across multiple directories. +By default, configuration files are named `edq-config.json`. +This value is customizable, but this document will assume the default is used. + +For example, a configuration file containing the `user` and `pass` options might look like this: +```json +{ + "user": "alice", + "pass": "password123" +} +``` + +The table below summarizes the configuration sources in the order they are evaluated. +Values from earlier sources can be overwritten by values from later sources. + +| Source | Description | +| :----- | :---------- | +| Global | Loaded from a file in a user-specific location, which is platform-dependent. | +| Local | Loaded from a file in the current or nearest ancestor directory. | +| CLI File | Loaded from one or more explicitly provided configuration files through the CLI. | +| CLI | Loaded from the command line. | + +The system produces an error if a global or local configuration file is unreadable (but not missing), or if a CLI-specified file is unreadable or missing. + +#### Global Configuration + +Global configuration are options that are user specific and stick with the user between projects, these are well suited for options like login credentials. +The global configuration file defaults to `/edq-config.json`. +The configuration location is chosen according to the [XDG standard](https://en.wikipedia.org/wiki/Freedesktop.org#Base_Directory_Specification) (implemented by [platformdirs](https://github.com/tox-dev/platformdirs)). +Below are examples of user-specific configuration file paths for different operating systems: + - Linux -- `/home//.config/edq-config.json` + - Mac -- `/Users//Library/Application Support/edq-config.json` + - Windows -- `C:\Users\\AppData\Local\edq-config.json` + +The default global configuration location can be changed by passing a path to `--config-global` through the command line. + +Below is an example command for specifying a global configuration path from the CLI: +```sh +python3 -m edq.cli.config.list --config-global ~/.config/custom-config.json +``` + +#### Local Configuration + +Local configuration are options that are specific to a project or directory, like a project's build directory. +Local configuration files are searched in multiple locations, the first file found is used. +The local config search order is: +1. `edq-config.json` in the current directory. +2. A legacy file in the current directory (only if a legacy file is preconfigured). +3. `edq-config.json` in any ancestor directory on the path to root (including root itself). + +#### CLI-Specified Config Files + +CLI config files are options specified on the command line via a file. +These are useful for a common set of options you don’t need every time, such as login credentials for different user. +Any files passed via `--config-file` will be loaded in the order they appear on the command line. +Options from later files override options from previous files. + +Below is an example of a CLI specified configuration paths: +```sh +python3 -m edq.cli.config.list --config-file ./edq-config.json --config-file ~/.secrets/edq-config.json +``` + +#### CLI Configuration + +CLI configurations are options specified directly on the command line, these are useful for quick option overrides without editing config files. +Configuration options are passed to the command line by the `--config` flag in this format `=`. +The provided values overrides the values from configuration files. +Configuration options are structured as key value pairs and keys cannot contain the "=" character. + +Below is an example of specifying a configuration option directly from the CLI: +```sh +python3 -m edq.cli.config.list --config user=alice --config pass=password123 +``` + +#### CLI Config Options + +The table below lists common configuration CLI options available for CLI tools using this library. + +| CLI Option | Description | +| :-------------- | :---------- | +|`--config-global` | Override the global config file location. | +|`--config-file` | Load configuration options from a CLI specified file. | +| `--config` | Provide additional options to a CLI command. | +| `--help` | Display standard help text and the default global configuration file path for the current platform. | diff --git a/edq/core/config.py b/edq/core/config.py new file mode 100644 index 0000000..0881165 --- /dev/null +++ b/edq/core/config.py @@ -0,0 +1,177 @@ +import argparse +import os +import typing + +import platformdirs + +import edq.util.dirent +import edq.util.json + +CONFIG_SOURCE_GLOBAL: str = "" +CONFIG_SOURCE_LOCAL: str = "" +CONFIG_SOURCE_CLI: str = "" +CONFIG_SOURCE_CLI_BARE: str = "" + +CONFIG_PATHS_KEY: str = 'config_paths' +DEFAULT_CONFIG_FILENAME: str = "edq-config.json" + +class ConfigSource: + """ A class for storing config source information. """ + + def __init__(self, label: str, path: typing.Union[str, None] = None) -> None: + self.label = label + """ The label identifying the config (see CONFIG_SOURCE_* constants). """ + + self.path = path + """ The path of where the config was soruced from. """ + + def __eq__(self, other: object) -> bool: + if (not isinstance(other, ConfigSource)): + return False + + return ((self.label == other.label) and (self.path == other.path)) + + def __str__(self) -> str: + return f"({self.label}, {self.path})" + +def get_tiered_config( + config_file_name: str = DEFAULT_CONFIG_FILENAME, + legacy_config_file_name: typing.Union[str, None] = None, + global_config_path: typing.Union[str, None] = None, + skip_keys: typing.Union[list, None] = None, + cli_arguments: typing.Union[dict, argparse.Namespace, None] = None, + local_config_root_cutoff: typing.Union[str, None] = None, + ) -> typing.Tuple[typing.Dict[str, str], typing.Dict[str, ConfigSource]]: + """ + Load all configuration options from files and command-line arguments. + Returns a configuration dictionary with the values based on tiering rules and a source dictionary mapping each key to its origin. + """ + + if (global_config_path is None): + global_config_path = platformdirs.user_config_dir(config_file_name) + + if (skip_keys is None): + skip_keys = [CONFIG_PATHS_KEY] + + if (cli_arguments is None): + cli_arguments = {} + + config: typing.Dict[str, str] = {} + sources: typing.Dict[str, ConfigSource] = {} + + # Ensure CLI arguments are always a dict, even if provided as argparse.Namespace. + if (isinstance(cli_arguments, argparse.Namespace)): + cli_arguments = vars(cli_arguments) + + # Check the global user config file. + if (os.path.isfile(global_config_path)): + _load_config_file(global_config_path, config, sources, CONFIG_SOURCE_GLOBAL) + + # Check the local user config file. + local_config_path = _get_local_config_path( + config_file_name = config_file_name, + legacy_config_file_name = legacy_config_file_name, + local_config_root_cutoff = local_config_root_cutoff, + ) + + if (local_config_path is not None): + _load_config_file(local_config_path, config, sources, CONFIG_SOURCE_LOCAL) + + # Check the config file specified on the command-line. + config_paths = cli_arguments.get(CONFIG_PATHS_KEY, []) + if (config_paths is not None): + for path in config_paths: + _load_config_file(path, config, sources, CONFIG_SOURCE_CLI) + + # Finally, any command-line options. + for (key, value) in cli_arguments.items(): + if (key in skip_keys): + continue + + if ((value is None) or (value == '')): + continue + + config[key] = value + sources[key] = ConfigSource(label = CONFIG_SOURCE_CLI_BARE) + + return config, sources + +def _load_config_file( + config_path: str, + config: typing.Dict[str, str], + sources: typing.Dict[str, ConfigSource], + source_label: str, + ) -> None: + """ Loads config variables and the source from the given config JSON file. """ + + config_path = os.path.abspath(config_path) + for (key, value) in edq.util.json.load_path(config_path).items(): + config[key] = value + sources[key] = ConfigSource(label = source_label, path = config_path) + +def _get_local_config_path( + config_file_name: str, + legacy_config_file_name: typing.Union[str, None] = None, + local_config_root_cutoff: typing.Union[str, None] = None, + ) -> typing.Union[str, None]: + """ + Search for a config file in hierarchical order. + Begins with the provided config file name, + optionally checks the legacy config file name if specified, + then continues up the directory tree looking for the provided config file name. + Returns the path to the first config file found. + + If no config file is found, returns None. + + The cutoff parameter limits the search depth, preventing detection of config file in higher-level directories during testing. + """ + + # Provided config file is in current directory. + if (os.path.isfile(config_file_name)): + return os.path.abspath(config_file_name) + + # Provided legacy config file is in current directory. + if (legacy_config_file_name is not None): + if (os.path.isfile(legacy_config_file_name)): + return os.path.abspath(legacy_config_file_name) + + # Provided config file is found in an ancestor directory up to the root or cutoff limit. + parent_dir = os.path.dirname(os.getcwd()) + return _get_ancestor_config_file_path( + parent_dir, + config_file_name = config_file_name, + local_config_root_cutoff = local_config_root_cutoff, + ) + +def _get_ancestor_config_file_path( + current_directory: str, + config_file_name: str, + local_config_root_cutoff: typing.Union[str, None] = None, + ) -> typing.Union[str, None]: + """ + Search through the parent directories (until root or a given cutoff directory(inclusive)) for a config file. + Stops at the first occurrence of the specified config file along the path to root. + Returns the path if a config file is found. + Otherwise, returns None. + """ + + if (local_config_root_cutoff is not None): + local_config_root_cutoff = os.path.abspath(local_config_root_cutoff) + + current_directory = os.path.abspath(current_directory) + for _ in range(edq.util.dirent.DEPTH_LIMIT): + config_file_path = os.path.join(current_directory, config_file_name) + if (os.path.isfile(config_file_path)): + return config_file_path + + # Check if current directory is root. + parent_dir = os.path.dirname(current_directory) + if (parent_dir == current_directory): + break + + if (local_config_root_cutoff == current_directory): + break + + current_directory = parent_dir + + return None diff --git a/edq/core/config_test.py b/edq/core/config_test.py new file mode 100644 index 0000000..916ca2d --- /dev/null +++ b/edq/core/config_test.py @@ -0,0 +1,721 @@ +import os + +import edq.testing.unittest +import edq.core.config +import edq.util.dirent +import edq.util.json + +def creat_test_dir(temp_dir_prefix: str) -> str: + """ + Creat a temp dir and populate it with dirents for testing. + + This test data directory is laid out as: + . + ├── custom-name + │   └── custom-edq-config.json + ├── dir-config + │   └── edq-config.json + ├── empty + │   └── edq-config.json + ├── empty-dir + ├── global + │   └── edq-config.json + ├── malformed + │   └── edq-config.json + ├── nested + │   ├── config.json + │   ├── edq-config.json + │   └── nest1 + │   ├── nest2a + │   └── nest2b + │   └── edq-config.json + ├── old-name + │   ├── config.json + │   └── nest1 + │   └── nest2 + └── simple + └── edq-config.json + """ + + temp_dir = edq.util.dirent.get_temp_dir(prefix = temp_dir_prefix) + + empty_config_dir_path = os.path.join(temp_dir, "empty") + edq.util.dirent.mkdir(empty_config_dir_path) + edq.util.json.dump_path({}, os.path.join(empty_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME)) + + custome_name_config_dir_path = os.path.join(temp_dir, "custom-name") + edq.util.dirent.mkdir(custome_name_config_dir_path) + edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(custome_name_config_dir_path, "custom-edq-config.json")) + + edq.util.dirent.mkdir(os.path.join(temp_dir, "dir-config", "edq-config.json")) + edq.util.dirent.mkdir(os.path.join(temp_dir, "empty-dir")) + + global_config_dir_path = os.path.join(temp_dir, "global") + edq.util.dirent.mkdir(global_config_dir_path) + edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(global_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME)) + + old_name_config_dir_path = os.path.join(temp_dir, "old-name") + edq.util.dirent.mkdir(os.path.join(old_name_config_dir_path, "nest1", "nest2")) + edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(old_name_config_dir_path, "config.json")) + + nested_dir_path = os.path.join(temp_dir, "nested") + edq.util.dirent.mkdir(os.path.join(nested_dir_path, "nest1", "nest2a")) + edq.util.dirent.mkdir(os.path.join(nested_dir_path, "nest1", "nest2b")) + + edq.util.json.dump_path({"server": "http://test.edulinq.org"}, os.path.join(nested_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME)) + edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(nested_dir_path, "config.json")) + edq.util.json.dump_path( + {"user": "user@test.edulinq.org"}, + os.path.join(nested_dir_path, "nest1", "nest2b", edq.core.config.DEFAULT_CONFIG_FILENAME), + ) + + simple_config_dir_path = os.path.join(temp_dir, "simple") + edq.util.dirent.mkdir(simple_config_dir_path) + edq.util.dirent.write_file( + os.path.join(simple_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME), + '{"user": "user@test.edulinq.org",}', + ) + + malformed_config_dir_path = os.path.join(temp_dir, "malformed") + edq.util.dirent.mkdir(malformed_config_dir_path) + edq.util.dirent.write_file( + os.path.join(malformed_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME), + "{user: user@test.edulinq.org}", + ) + + return temp_dir + +class TestConfig(edq.testing.unittest.BaseTest): + """ Test basic operations on configs. """ + + def test_get_tiered_config_base(self): + """ + Test that configuration files are loaded correctly from the file system with the expected tier. + """ + + temp_dir = creat_test_dir(temp_dir_prefix = "edq-test-config-get-tiered-config-") + + # [(work directory, extra arguments, expected config, expected source, error substring), ...] + test_cases = [ + # No Config + ( + "empty-dir", + {}, + {}, + {}, + None, + ), + + # Global Config + + # Custom Global Config Path + ( + "empty-dir", + { + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_GLOBAL, + path = os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + + # Empty Config JSON + ( + "empty-dir", + { + "global_config_path": os.path.join(temp_dir, "empty", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, + {}, + {}, + None, + ), + + # Directory Config JSON + ( + "empty-dir", + { + "global_config_path": os.path.join(temp_dir, "dir-config", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, + {}, + {}, + None, + ), + + # Non-Existent Config JSON + ( + "empty-dir", + { + "global_config_path": os.path.join(temp_dir, "empty-dir", "non-existent-config.json"), + }, + {}, + {}, + None, + ), + + # Malformed Config JSON + ( + "empty-dir", + { + "global_config_path": os.path.join(temp_dir, "malformed", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, + {}, + {}, + "Failed to read JSON file", + ), + + # Local Config + + # Default config file in current directory. + ( + "simple", + {}, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + + # Custom config file in current directory. + ( + "custom-name", + { + "config_file_name": "custom-edq-config.json", + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, + path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), + ), + }, + None, + ), + + # Legacy config file in current directory. + ( + "old-name", + { + "legacy_config_file_name": "config.json", + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, + path = os.path.join(temp_dir, "old-name", "config.json"), + ), + }, + None, + ), + + # Default config file in an ancestor directory. + ( + os.path.join("nested", "nest1", "nest2a"), + {}, + { + "server": "http://test.edulinq.org", + }, + { + "server": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, + path = os.path.join(temp_dir, "nested", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + + # Legacy config file in an ancestor directory. + ( + os.path.join("old-name", "nest1", "nest2"), + { + "legacy_config_file_name": "config.json", + }, + {}, + {}, + None, + ), + + # Empty Config JSON + ( + "empty", + {}, + {}, + {}, + None, + ), + + # Directory Config JSON + ( + "dir-config", + {}, + {}, + {}, + None, + ), + + # Malformed Config JSON + ( + "malformed", + {}, + {}, + {}, + "Failed to read JSON file", + ), + + # All 3 local config locations present at the same time. + ( + os.path.join("nested", "nest1", "nest2b"), + { + "legacy_config_file_name": "config.json", + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, + path = os.path.join(temp_dir, "nested", "nest1", "nest2b", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + + # CLI Provided Config + + # Distinct Keys + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + os.path.join(temp_dir, "nested", edq.core.config.DEFAULT_CONFIG_FILENAME), + ], + }, + }, + { + "user": "user@test.edulinq.org", + "server": "http://test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + "server": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI, + path = os.path.join(temp_dir, "nested", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + + # Overwriting Keys + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), + os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + + # Empty Config JSON + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "empty", edq.core.config.DEFAULT_CONFIG_FILENAME), + ], + }, + }, + {}, + {}, + None, + ), + + # Directory Config JSON + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "dir-config", edq.core.config.DEFAULT_CONFIG_FILENAME), + ], + }, + }, + {}, + {}, + "IsADirectoryError", + ), + + # Non-Existent Config JSON + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "empty-dir", "non-existent-config.json"), + ], + }, + }, + {}, + {}, + "FileNotFoundError", + ), + + # Malformed Config JSON + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "malformed", edq.core.config.DEFAULT_CONFIG_FILENAME), + ], + }, + }, + {}, + {}, + "Failed to read JSON file", + ), + + # CLI Bare Options: + + # CLI arguments only (direct key: value). + ( + "empty-dir", + { + "cli_arguments": { + "user": "user@test.edulinq.org", + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + }, + None, + ), + + # Skip keys functionally. + ( + "empty-dir", + { + "cli_arguments": { + "user": "user@test.edulinq.org", + "pass": "user", + }, + "skip_keys": [ + "pass", + ], + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + }, + None, + ), + + # Combinations + + # Global Config + Local Config + ( + "simple", + { + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + + # Global Config + CLI Provided Config + ( + "empty-dir", + { + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + + # Global + CLI Bare Options + ( + "empty-dir", + { + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + "user": "user@test.edulinq.org", + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + }, + None, + ), + + # Local Config + CLI Provided Config + ( + "simple", + { + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI, + path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), + ), + }, + None, + ), + + # Local Config + CLI Bare Options + ( + "simple", + { + "cli_arguments": { + "user": "user@test.edulinq.org", + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + }, + None, + ), + + # CLI Provided Config + CLI Bare Options + ( + "empty-dir", + { + "cli_arguments": { + "user": "user@test.edulinq.org", + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + }, + None, + ), + + # Global Config + CLI Provided Config + CLI Bare Options + ( + "empty-dir", + { + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + "user": "user@test.edulinq.org", + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + }, + None, + ), + + # Global Config + Local Config + CLI Bare Options + ( + "simple", + { + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + "user": "user@test.edulinq.org", + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + }, + None, + ), + + # Global Config + Local Config + CLI Provided Config + ( + "simple", + { + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI, + path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), + ), + }, + None, + ), + + # Local Config + CLI Provided Config + CLI Bare Options + ( + "simple", + { + "cli_arguments": { + "user": "user@test.edulinq.org", + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + }, + None, + ), + + # Global Config + Local Config + CLI Provided Config + CLI Bare Options + ( + "simple", + { + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), + ], + "pass": "user", + "server": "http://test.edulinq.org", + }, + "skip_keys": [ + "server", + edq.core.config.CONFIG_PATHS_KEY, + ], + }, + { + "user": "user@test.edulinq.org", + "pass": "user", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI, + path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), + ), + "pass": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + }, + None, + ), + ] + + for (i, test_case) in enumerate(test_cases): + (test_work_dir, extra_args, expected_config, expected_source, error_substring) = test_case + + with self.subTest(msg = f"Case {i} ('{test_work_dir}'):"): + global_config = extra_args.get("global_config_path", None) + if (global_config is None): + extra_args["global_config_path"] = os.path.join(temp_dir, "empty", edq.core.config.CONFIG_PATHS_KEY) + + cutoff = extra_args.get("local_config_root_cutoff", None) + if (cutoff is None): + extra_args["local_config_root_cutoff"] = temp_dir + + previous_work_directory = os.getcwd() + initial_work_directory = os.path.join(temp_dir, test_work_dir) + os.chdir(initial_work_directory) + + try: + (actual_config, actual_sources) = edq.core.config.get_tiered_config(**extra_args) + except Exception as ex: + error_string = self.format_error_string(ex) + + if (error_substring is None): + self.fail(f"Unexpected error: '{error_string}'.") + + self.assertIn(error_substring, error_string, 'Error is not as expected.') + + continue + + finally: + os.chdir(previous_work_directory) + + if (error_substring is not None): + self.fail(f"Did not get expected error: '{error_substring}'.") + + self.assertJSONDictEqual(expected_config, actual_config) + self.assertJSONDictEqual(expected_source, actual_sources) diff --git a/edq/testing/unittest.py b/edq/testing/unittest.py index 4acf2c9..02b06b2 100644 --- a/edq/testing/unittest.py +++ b/edq/testing/unittest.py @@ -14,12 +14,11 @@ class BaseTest(unittest.TestCase): maxDiff = None """ Don't limit the size of diffs. """ - def assertJSONDictEqual(self, a: typing.Any, b: typing.Any) -> None: # pylint: disable=invalid-name + def assertJSONDictEqual(self, a: typing.Any, b: typing.Any, message: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name """ Like unittest.TestCase.assertDictEqual(), but will try to convert each comparison argument to a dict if it is not already, - and uses an assertion message containing the full JSON representation of the arguments. - + and uses a default assertion message containing the full JSON representation of the arguments. """ if (not isinstance(a, dict)): @@ -37,17 +36,23 @@ def assertJSONDictEqual(self, a: typing.Any, b: typing.Any) -> None: # pylint: a_json = edq.util.json.dumps(a, indent = 4) b_json = edq.util.json.dumps(b, indent = 4) - super().assertDictEqual(a, b, FORMAT_STR % (a_json, b_json)) + if (message is None): + message = FORMAT_STR % (a_json, b_json) + + super().assertDictEqual(a, b, msg = message) - def assertJSONListEqual(self, a: typing.List[typing.Any], b: typing.List[typing.Any]) -> None: # pylint: disable=invalid-name + def assertJSONListEqual(self, a: typing.List[typing.Any], b: typing.List[typing.Any], message: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name """ - Call assertListEqual(), but supply a message containing the full JSON representation of the arguments. + Call assertDictEqual(), but supply a default message containing the full JSON representation of the arguments. """ a_json = edq.util.json.dumps(a, indent = 4) b_json = edq.util.json.dumps(b, indent = 4) - super().assertListEqual(a, b, FORMAT_STR % (a_json, b_json)) + if (message is None): + message = FORMAT_STR % (a_json, b_json) + + super().assertListEqual(a, b, msg = message) def format_error_string(self, ex: typing.Union[BaseException, None]) -> str: """ diff --git a/edq/util/dirent.py b/edq/util/dirent.py index 206816c..f4a9d0c 100644 --- a/edq/util/dirent.py +++ b/edq/util/dirent.py @@ -41,6 +41,8 @@ def get_temp_path(prefix: str = '', suffix: str = '', rm: bool = True) -> str: while ((path is None) or exists(path)): path = os.path.join(tempfile.gettempdir(), prefix + str(uuid.uuid4()) + suffix) + path = os.path.realpath(path) + if (rm): atexit.register(remove, path) diff --git a/edq/util/json.py b/edq/util/json.py index 0f93819..4470b95 100644 --- a/edq/util/json.py +++ b/edq/util/json.py @@ -7,6 +7,7 @@ import abc import enum import json +import os import typing import json5 @@ -107,6 +108,9 @@ def load_path( otherwise use JSON5. """ + if (os.path.isdir(path)): + raise IsADirectoryError(f"Cannot open JSON file, expected a file but got a directory at '{path}'.") + try: with open(path, 'r', encoding = encoding) as file: return load(file, strict = strict, **kwargs)