From 67f376fa44ee38ed423c4635bcea872a8cccb73b Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 15 Aug 2025 15:44:36 -0700 Subject: [PATCH 01/71] Tests passing need to work on other checks. --- edq/testing/unittest.py | 13 + edq/util/config.py | 122 +++++++++ edq/util/config_test.py | 250 ++++++++++++++++++ edq/util/dirent.py | 2 + .../testdata/configs/empty-dir/.gitignore | 2 + .../testdata/configs/empty/autograder.json | 2 + .../testdata/configs/global/autograder.json | 3 + .../testdata/configs/nested/autograder.json | 3 + .../configs/nested/nest1/nest2a/.gitignore | 2 + .../nested/nest1/nest2b/autograder.json | 3 + .../configs/nested/nest1/nest2b/config.json | 3 + .../testdata/configs/old-name/config.json | 3 + .../testdata/configs/simple/autograder.json | 3 + 13 files changed, 411 insertions(+) create mode 100644 edq/util/config.py create mode 100644 edq/util/config_test.py create mode 100644 edq/util/testdata/configs/empty-dir/.gitignore create mode 100644 edq/util/testdata/configs/empty/autograder.json create mode 100644 edq/util/testdata/configs/global/autograder.json create mode 100644 edq/util/testdata/configs/nested/autograder.json create mode 100644 edq/util/testdata/configs/nested/nest1/nest2a/.gitignore create mode 100644 edq/util/testdata/configs/nested/nest1/nest2b/autograder.json create mode 100644 edq/util/testdata/configs/nested/nest1/nest2b/config.json create mode 100644 edq/util/testdata/configs/old-name/config.json create mode 100644 edq/util/testdata/configs/simple/autograder.json diff --git a/edq/testing/unittest.py b/edq/testing/unittest.py index b2dda96..3f2ed9f 100644 --- a/edq/testing/unittest.py +++ b/edq/testing/unittest.py @@ -3,6 +3,7 @@ import edq.util.json import edq.util.reflection +import edq.util.json FORMAT_STR: str = "\n--- Expected ---\n%s\n--- Actual ---\n%s\n---\n" @@ -34,6 +35,18 @@ def assertJSONListEqual(self, a: list, b: list) -> None: # pylint: disable=inva super().assertListEqual(a, b, FORMAT_STR % (a_json, b_json)) + def assertListEqual(self, a, b): + a_json = edq.util.json.dumps(a, indent = 4) + b_json = edq.util.json.dumps(b, indent = 4) + + super().assertListEqual(a, b, FORMAT_STR % (a_json, b_json)) + + def assertDictEqual(self, a, b): + a_json = edq.util.json.dumps(a, indent = 4) + b_json = edq.util.json.dumps(b, indent = 4) + + super().assertDictEqual(a, b, FORMAT_STR % (a_json, b_json)) + def format_error_string(self, ex: typing.Union[BaseException, None]) -> str: """ Format an error string from an exception so it can be checked for testing. diff --git a/edq/util/config.py b/edq/util/config.py new file mode 100644 index 0000000..b18f895 --- /dev/null +++ b/edq/util/config.py @@ -0,0 +1,122 @@ +import argparse + +import os +import platformdirs + +import edq.util.dirent +import edq.util.json + +CONFIG_PATHS_KEY = 'config_paths' +LEGACY_CONFIG_FILENAME = 'config.json' +DEFAULT_CONFIG_FILENAME = 'autograder.json' +DEFAULT_GLOBAL_CONFIG_PATH = platformdirs.user_config_dir(DEFAULT_CONFIG_FILENAME) +CONFIG_TYPE_DELIMITER = "::" + +def get_tiered_config( + cli_arguments, + skip_keys = [CONFIG_PATHS_KEY], + global_config_path = DEFAULT_GLOBAL_CONFIG_PATH, + local_config_root_cutoff = None): + """ + Get all the tiered configuration options (from files and CLI). + If |show_sources| is True, then an addition dict will be returned that shows each key, + and where that key came from. + """ + + config = {} + sources = {} + + if (isinstance(cli_arguments, argparse.Namespace)): + cli_arguments = vars(cli_arguments) + + # Check the global user config file. + if (os.path.isfile(global_config_path)): + _load_config_file(global_config_path, config, sources, "") + + # Check the local user config file. + local_config_path = _get_local_config_path(local_config_root_cutoff = local_config_root_cutoff) + if (local_config_path is not None): + _load_config_file(local_config_path, config, sources, "") + + # Check the config file specified on the command-line. + config_paths = cli_arguments.get(CONFIG_PATHS_KEY, []) + if (config_paths is not None): + for path in config_paths: + _load_config_file(path, config, sources, "") + + # Finally, any command-line options. + for (key, value) in cli_arguments.items(): + if (key in skip_keys): + continue + + if ((value is None) or (value == '')): + continue + + config[key] = value + sources[key] = "" + + return config, sources + +def _load_config_file(config_path, config, sources, source_label): + with open(config_path, 'r') as file: + for (key, value) in edq.util.json.load(file).items(): + config[key] = value + sources[key] = f"{source_label}{CONFIG_TYPE_DELIMITER}{os.path.abspath(config_path)}" + +def _get_local_config_path(local_config_root_cutoff = None): + """ + Searches for a configuration file in a hierarchical order, + starting with DEFAULT_CONFIG_FILENAME, then LEGACY_CONFIG_FILENAME, + and continuing up the directory tree looking for DEFAULT_CONFIG_FILENAME. + Returns the path to the first configuration file found. + + If no configuration file is found, returns None. + The cutoff limits config search depth. + This helps to prevent detection of a config file in higher directories during testing. + """ + + # The case where DEFAULT_CONFIG_FILENAME file in current directory. + if (os.path.isfile(DEFAULT_CONFIG_FILENAME)): + return os.path.abspath(DEFAULT_CONFIG_FILENAME) + + # The case where LEGACY_CONFIG_FILENAME file in current directory. + if (os.path.isfile(LEGACY_CONFIG_FILENAME)): + return os.path.abspath(LEGACY_CONFIG_FILENAME) + + # The case where a DEFAULT_CONFIG_FILENAME file located in any ancestor directory on the path to root. + parent_dir = os.path.dirname(os.getcwd()) + return _get_ancestor_config_file_path( + parent_dir, + local_config_root_cutoff = local_config_root_cutoff + ) + +def _get_ancestor_config_file_path( + current_directory, + config_file = DEFAULT_CONFIG_FILENAME, + local_config_root_cutoff = None): + """ + Search through the parent directories (until root or a given cutoff directory(inclusive)) for a configuration file. + Stops at the first occurrence of the specified config file (default: DEFAULT_CONFIG_FILENAME) along the path to root. + Returns the path if a configuration file is found. + Otherwise, returns None. + """ + + current_directory = os.path.abspath(current_directory) + if (local_config_root_cutoff is not None): + local_config_root_cutoff = os.path.abspath(local_config_root_cutoff) + + for _ in range(edq.util.dirent.DEPTH_LIMIT): + config_file_path = os.path.join(current_directory, config_file) + if (os.path.isfile(config_file_path)): + return config_file_path + + if (local_config_root_cutoff == current_directory): + break + + parent_dir = os.path.dirname(current_directory) + if (parent_dir == current_directory): + break + + current_directory = parent_dir + + return None diff --git a/edq/util/config_test.py b/edq/util/config_test.py new file mode 100644 index 0000000..32bb765 --- /dev/null +++ b/edq/util/config_test.py @@ -0,0 +1,250 @@ +import os + +import edq.testing.unittest +import edq.util.config +import edq.util.dirent + +THIS_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) +CONFIGS_DIR = os.path.join(THIS_DIR, "testdata", "configs") + +class TestConfig(edq.testing.unittest.BaseTest): + def test_base(self): + # [(work directory, expected config, expected source, {skip keys , cli arguments, config global}), ...] + test_cases = [ + ( + "simple", + { + "user": "user@test.edulinq.org" + }, + { + "user": f"::{os.path.join('TEMP_DIR', 'simple', 'autograder.json')}" + }, + {} + ), + ( + "old-name", + { + "user": "user@test.edulinq.org" + }, + { + "user": f"::{os.path.join('TEMP_DIR', 'old-name', 'config.json')}" + }, + {} + ), + ( + os.path.join("nested", "nest1", "nest2a"), + { + "server": "http://test.edulinq.org" + }, + { + "server": f"::{os.path.join('TEMP_DIR', 'nested', 'autograder.json')}" + }, + {} + ), + ( + os.path.join("nested", "nest1", "nest2b"), + { + "user": "user@test.edulinq.org" + }, + { + "user": f"::{os.path.join('TEMP_DIR', 'nested', 'nest1', 'nest2b', 'autograder.json')}" + }, + {} + ), + ( + "empty-dir", + { + "user": "user@test.edulinq.org" + }, + { + "user": f"::{os.path.join('TEMP_DIR', 'global', 'autograder.json')}" + }, + { + "global_config_path": os.path.join("global", "autograder.json") + } + ), + ( + "empty-dir", + { + "user": "user@test.edulinq.org" + }, + { + "user": "" + }, + { + "cli_args": { + "user": "user@test.edulinq.org" + } + } + ), + ( + "empty-dir", + { + "user": "user@test.edulinq.org" + }, + { + "user": "" + }, + { + "cli_args": { + "user": "user@test.edulinq.org", + "pass": "user" + }, + "skip_keys": [ + "pass" + ] + } + ), + ( + "empty-dir", + { + "user": "user@test.edulinq.org" + }, + { + "user": f"::{os.path.join('TEMP_DIR', 'simple', 'autograder.json')}" + }, + { + "cli_args": { + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join("global", "autograder.json"), + os.path.join("simple", "autograder.json") + ] + } + } + ), + ( + "empty-dir", + { + "user": "user@test.edulinq.org", + "server": "http://test.edulinq.org" + }, + { + "user": f"::{os.path.join('TEMP_DIR', 'simple', 'autograder.json')}", + "server": f"::{os.path.join('TEMP_DIR', 'nested', 'autograder.json')}" + }, + { + "cli_args": { + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join("nested", "autograder.json"), + os.path.join("simple", "autograder.json") + ] + } + } + ), + ( + "simple", + { + "user": "user@test.edulinq.org" + }, + { + "user": f"::{os.path.join('TEMP_DIR', 'simple', 'autograder.json')}" + }, + { + "global_config_path": os.path.join("global", "autograder.json") + } + ), + ( + "empty-dir", + { + "user": "user@test.edulinq.org" + }, + { + "user": f"::{os.path.join('TEMP_DIR', 'simple', 'autograder.json')}" + }, + { + "cli_args": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("simple", "autograder.json")] + }, + "global_config_path": os.path.join("global", "autograder.json") + } + ), + ( + "simple", + { + "user": "user@test.edulinq.org" + }, + { + "user": f"::{os.path.join('TEMP_DIR', 'old-name', 'config.json')}" + }, + { + "cli_args": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("old-name", "config.json")] + }, + } + ), + ( + "simple", + { + "user": "user@test.edulinq.org", + "pass": "user" + }, + { + "user": f"::{os.path.join('TEMP_DIR', 'old-name', 'config.json')}", + "pass": "" + }, + { + "cli_args": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("old-name", "config.json")], + "pass": "user", + "server": "http://test.edulinq.org" + }, + "skip_keys": [ + "server", + edq.util.config.CONFIG_PATHS_KEY + ], + "global_config_path": os.path.join("global", "autograder.json") + } + ) + ] + + for test_case in test_cases: + (test_work_dir, expected_config, expected_source, extra_args) = test_case + + self._evaluate_test_config( + test_work_dir, expected_config, expected_source, **extra_args + ) + + def _evaluate_test_config( + self, test_work_dir, expected_config, expected_source, + skip_keys = [edq.util.config.CONFIG_PATHS_KEY], + cli_args = {}, global_config_path = None): + """ + Prepares testing environment and normalizes cli config paths, + global config path and expected source paths. Evaluates the given expected and + source configs with actual get_tiered_config() output. + """ + + temp_dir = edq.util.dirent.get_temp_dir(prefix = 'autograder-test-config-') + + global_config = os.path.join(temp_dir) + if (global_config_path is not None): + global_config = os.path.join(temp_dir, global_config_path) + + abs_config_paths = [] + config_paths = cli_args.get(edq.util.config.CONFIG_PATHS_KEY, None) + if (config_paths is not None): + for rel_config_path in config_paths: + abs_config_paths.append(os.path.join(temp_dir, rel_config_path)) + cli_args[edq.util.config.CONFIG_PATHS_KEY] = abs_config_paths + + edq.util.dirent.copy_contents(CONFIGS_DIR, temp_dir) + + previous_work_directory = os.getcwd() + initial_work_directory = os.path.join(temp_dir, test_work_dir) + os.chdir(initial_work_directory) + + try: + (actual_configs, actual_sources) = edq.util.config.get_tiered_config( + cli_arguments = cli_args, + global_config_path = global_config, + local_config_root_cutoff = temp_dir, + skip_keys = skip_keys + ) + finally: + os.chdir(previous_work_directory) + + for (key, value) in actual_sources.items(): + actual_sources[key] = value.replace(temp_dir, "TEMP_DIR") + + self.assertDictEqual(expected_config, actual_configs) + self.assertDictEqual(expected_source, actual_sources) diff --git a/edq/util/dirent.py b/edq/util/dirent.py index 6cab7d3..cf0c8f6 100644 --- a/edq/util/dirent.py +++ b/edq/util/dirent.py @@ -41,6 +41,8 @@ def get_temp_path(prefix: str = '', suffix: str = '', rm: bool = True) -> str: while ((path is None) or exists(path)): path = os.path.join(tempfile.gettempdir(), prefix + str(uuid.uuid4()) + suffix) + path = os.path.realpath(path) + if (rm): atexit.register(remove, path) diff --git a/edq/util/testdata/configs/empty-dir/.gitignore b/edq/util/testdata/configs/empty-dir/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/edq/util/testdata/configs/empty-dir/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/edq/util/testdata/configs/empty/autograder.json b/edq/util/testdata/configs/empty/autograder.json new file mode 100644 index 0000000..2c63c08 --- /dev/null +++ b/edq/util/testdata/configs/empty/autograder.json @@ -0,0 +1,2 @@ +{ +} diff --git a/edq/util/testdata/configs/global/autograder.json b/edq/util/testdata/configs/global/autograder.json new file mode 100644 index 0000000..98b5beb --- /dev/null +++ b/edq/util/testdata/configs/global/autograder.json @@ -0,0 +1,3 @@ +{ + "user": "user@test.edulinq.org" +} diff --git a/edq/util/testdata/configs/nested/autograder.json b/edq/util/testdata/configs/nested/autograder.json new file mode 100644 index 0000000..d53f13d --- /dev/null +++ b/edq/util/testdata/configs/nested/autograder.json @@ -0,0 +1,3 @@ +{ + "server": "http://test.edulinq.org" +} diff --git a/edq/util/testdata/configs/nested/nest1/nest2a/.gitignore b/edq/util/testdata/configs/nested/nest1/nest2a/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/edq/util/testdata/configs/nested/nest1/nest2a/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/edq/util/testdata/configs/nested/nest1/nest2b/autograder.json b/edq/util/testdata/configs/nested/nest1/nest2b/autograder.json new file mode 100644 index 0000000..98b5beb --- /dev/null +++ b/edq/util/testdata/configs/nested/nest1/nest2b/autograder.json @@ -0,0 +1,3 @@ +{ + "user": "user@test.edulinq.org" +} diff --git a/edq/util/testdata/configs/nested/nest1/nest2b/config.json b/edq/util/testdata/configs/nested/nest1/nest2b/config.json new file mode 100644 index 0000000..669b7b6 --- /dev/null +++ b/edq/util/testdata/configs/nested/nest1/nest2b/config.json @@ -0,0 +1,3 @@ +{ + "pass": "123456" +} diff --git a/edq/util/testdata/configs/old-name/config.json b/edq/util/testdata/configs/old-name/config.json new file mode 100644 index 0000000..98b5beb --- /dev/null +++ b/edq/util/testdata/configs/old-name/config.json @@ -0,0 +1,3 @@ +{ + "user": "user@test.edulinq.org" +} diff --git a/edq/util/testdata/configs/simple/autograder.json b/edq/util/testdata/configs/simple/autograder.json new file mode 100644 index 0000000..98b5beb --- /dev/null +++ b/edq/util/testdata/configs/simple/autograder.json @@ -0,0 +1,3 @@ +{ + "user": "user@test.edulinq.org" +} From 0ff308ae8433111af5a6f96b30ff179020e065a2 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 15 Aug 2025 18:43:37 -0700 Subject: [PATCH 02/71] Worked on linting and typing. --- edq/testing/unittest.py | 37 ++++++++++++++++++++++-------- edq/util/config.py | 50 ++++++++++++++++++++++++++--------------- edq/util/config_test.py | 14 ++++++++++-- 3 files changed, 72 insertions(+), 29 deletions(-) diff --git a/edq/testing/unittest.py b/edq/testing/unittest.py index 3f2ed9f..c227069 100644 --- a/edq/testing/unittest.py +++ b/edq/testing/unittest.py @@ -1,9 +1,10 @@ +import collections import typing import unittest import edq.util.json import edq.util.reflection -import edq.util.json + FORMAT_STR: str = "\n--- Expected ---\n%s\n--- Actual ---\n%s\n---\n" @@ -35,17 +36,35 @@ def assertJSONListEqual(self, a: list, b: list) -> None: # pylint: disable=inva super().assertListEqual(a, b, FORMAT_STR % (a_json, b_json)) - def assertListEqual(self, a, b): - a_json = edq.util.json.dumps(a, indent = 4) - b_json = edq.util.json.dumps(b, indent = 4) + def assertListEqual(self, list1: list, list2: list, msg: typing.Any = None) -> None: + """ + Assert two lists are equal, showing JSON-formatted output when they differ. + """ - super().assertListEqual(a, b, FORMAT_STR % (a_json, b_json)) + list1_json = edq.util.json.dumps(list1, indent = 4) + list2_json = edq.util.json.dumps(list2, indent = 4) - def assertDictEqual(self, a, b): - a_json = edq.util.json.dumps(a, indent = 4) - b_json = edq.util.json.dumps(b, indent = 4) + if(msg is None): + msg = FORMAT_STR % (list1_json, list2_json) - super().assertDictEqual(a, b, FORMAT_STR % (a_json, b_json)) + super().assertListEqual(list1, list2, msg = msg) + + def assertDictEqual(self, + d1: collections.abc.Mapping[typing.Any, object], + d2: collections.abc.Mapping[typing.Any, object], + msg: typing.Any = None + ) -> None: + """ + Assert two dicts are equal, showing JSON-formatted output when they differ. + """ + + d1_json = edq.util.json.dumps(d1, indent = 4) + d2_json = edq.util.json.dumps(d2, indent = 4) + + if(msg is None): + msg = FORMAT_STR % (d1_json, d2_json) + + super().assertDictEqual(d1, d2, msg = msg) def format_error_string(self, ex: typing.Union[BaseException, None]) -> str: """ diff --git a/edq/util/config.py b/edq/util/config.py index b18f895..28756ae 100644 --- a/edq/util/config.py +++ b/edq/util/config.py @@ -1,30 +1,35 @@ import argparse - import os +import typing + import platformdirs import edq.util.dirent import edq.util.json -CONFIG_PATHS_KEY = 'config_paths' -LEGACY_CONFIG_FILENAME = 'config.json' -DEFAULT_CONFIG_FILENAME = 'autograder.json' -DEFAULT_GLOBAL_CONFIG_PATH = platformdirs.user_config_dir(DEFAULT_CONFIG_FILENAME) -CONFIG_TYPE_DELIMITER = "::" +CONFIG_PATHS_KEY: str = 'config_paths' +LEGACY_CONFIG_FILENAME: str = 'config.json' +DEFAULT_CONFIG_FILENAME: str = 'autograder.json' +DEFAULT_GLOBAL_CONFIG_PATH: str = platformdirs.user_config_dir(DEFAULT_CONFIG_FILENAME) +CONFIG_TYPE_DELIMITER: str = "::" def get_tiered_config( - cli_arguments, - skip_keys = [CONFIG_PATHS_KEY], - global_config_path = DEFAULT_GLOBAL_CONFIG_PATH, - local_config_root_cutoff = None): + cli_arguments: typing.Union[dict, argparse.Namespace], + skip_keys: typing.Union[list, None] = None, + global_config_path: str = DEFAULT_GLOBAL_CONFIG_PATH, + local_config_root_cutoff: typing.Union[str, None] = None + )-> typing.Tuple[typing.Dict[str, str], typing.Dict[str, str]]: """ Get all the tiered configuration options (from files and CLI). If |show_sources| is True, then an addition dict will be returned that shows each key, and where that key came from. """ - config = {} - sources = {} + config: typing.Dict[str, str] = {} + sources: typing.Dict[str, str] = {} + + if (skip_keys is None): + skip_keys = [CONFIG_PATHS_KEY] if (isinstance(cli_arguments, argparse.Namespace)): cli_arguments = vars(cli_arguments) @@ -57,13 +62,21 @@ def get_tiered_config( return config, sources -def _load_config_file(config_path, config, sources, source_label): - with open(config_path, 'r') as file: +def _load_config_file( + config_path: str, + config: typing.Dict[str, str], + sources: typing.Dict[str, str], + source_label: str, + encoding: str = edq.util.dirent.DEFAULT_ENCODING + )-> None: + """ Loads configs and the source from the given config JSON file. """ + + with open(config_path, 'r', encoding = encoding) as file: for (key, value) in edq.util.json.load(file).items(): config[key] = value sources[key] = f"{source_label}{CONFIG_TYPE_DELIMITER}{os.path.abspath(config_path)}" -def _get_local_config_path(local_config_root_cutoff = None): +def _get_local_config_path(local_config_root_cutoff: typing.Union[str, None] = None) -> typing.Union[str, None]: """ Searches for a configuration file in a hierarchical order, starting with DEFAULT_CONFIG_FILENAME, then LEGACY_CONFIG_FILENAME, @@ -91,9 +104,10 @@ def _get_local_config_path(local_config_root_cutoff = None): ) def _get_ancestor_config_file_path( - current_directory, - config_file = DEFAULT_CONFIG_FILENAME, - local_config_root_cutoff = None): + current_directory: str, + config_file: str = DEFAULT_CONFIG_FILENAME, + local_config_root_cutoff: typing.Union[str, None] = None + )-> typing.Union[str, None]: """ Search through the parent directories (until root or a given cutoff directory(inclusive)) for a configuration file. Stops at the first occurrence of the specified config file (default: DEFAULT_CONFIG_FILENAME) along the path to root. diff --git a/edq/util/config_test.py b/edq/util/config_test.py index 32bb765..71011dc 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -8,7 +8,11 @@ CONFIGS_DIR = os.path.join(THIS_DIR, "testdata", "configs") class TestConfig(edq.testing.unittest.BaseTest): + """ Test basic operations on configs. """ + def test_base(self): + """ Test that configs are loaded correctly from the file system with the correct tier. """ + # [(work directory, expected config, expected source, {skip keys , cli arguments, config global}), ...] test_cases = [ ( @@ -206,14 +210,20 @@ def test_base(self): def _evaluate_test_config( self, test_work_dir, expected_config, expected_source, - skip_keys = [edq.util.config.CONFIG_PATHS_KEY], - cli_args = {}, global_config_path = None): + skip_keys = None, + cli_args = None, global_config_path = None): """ Prepares testing environment and normalizes cli config paths, global config path and expected source paths. Evaluates the given expected and source configs with actual get_tiered_config() output. """ + if (cli_args is None): + cli_args = {} + + if (skip_keys is None): + skip_keys = [edq.util.config.CONFIG_PATHS_KEY] + temp_dir = edq.util.dirent.get_temp_dir(prefix = 'autograder-test-config-') global_config = os.path.join(temp_dir) From b6148cf6ce1e3091940a9eb84880cad831de2681 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 15 Aug 2025 18:48:24 -0700 Subject: [PATCH 03/71] Changed the Mapping import. --- edq/testing/unittest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/edq/testing/unittest.py b/edq/testing/unittest.py index c227069..bacc9ab 100644 --- a/edq/testing/unittest.py +++ b/edq/testing/unittest.py @@ -1,4 +1,3 @@ -import collections import typing import unittest @@ -50,8 +49,8 @@ def assertListEqual(self, list1: list, list2: list, msg: typing.Any = None) -> N super().assertListEqual(list1, list2, msg = msg) def assertDictEqual(self, - d1: collections.abc.Mapping[typing.Any, object], - d2: collections.abc.Mapping[typing.Any, object], + d1: typing.Mapping[typing.Any, object], + d2: typing.Mapping[typing.Any, object], msg: typing.Any = None ) -> None: """ From d4556b95d17909e80b04879a8a56a0170f192dc7 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 15 Aug 2025 19:46:39 -0700 Subject: [PATCH 04/71] Added an option to specify the config file name. --- edq/util/config.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/edq/util/config.py b/edq/util/config.py index 28756ae..efe0417 100644 --- a/edq/util/config.py +++ b/edq/util/config.py @@ -17,7 +17,8 @@ def get_tiered_config( cli_arguments: typing.Union[dict, argparse.Namespace], skip_keys: typing.Union[list, None] = None, global_config_path: str = DEFAULT_GLOBAL_CONFIG_PATH, - local_config_root_cutoff: typing.Union[str, None] = None + local_config_root_cutoff: typing.Union[str, None] = None, + config_file_name: str = DEFAULT_CONFIG_FILENAME )-> typing.Tuple[typing.Dict[str, str], typing.Dict[str, str]]: """ Get all the tiered configuration options (from files and CLI). @@ -39,7 +40,7 @@ def get_tiered_config( _load_config_file(global_config_path, config, sources, "") # Check the local user config file. - local_config_path = _get_local_config_path(local_config_root_cutoff = local_config_root_cutoff) + local_config_path = _get_local_config_path(config_file_name = config_file_name, local_config_root_cutoff = local_config_root_cutoff) if (local_config_path is not None): _load_config_file(local_config_path, config, sources, "") @@ -76,7 +77,7 @@ def _load_config_file( config[key] = value sources[key] = f"{source_label}{CONFIG_TYPE_DELIMITER}{os.path.abspath(config_path)}" -def _get_local_config_path(local_config_root_cutoff: typing.Union[str, None] = None) -> typing.Union[str, None]: +def _get_local_config_path(config_file_name: str, local_config_root_cutoff: typing.Union[str, None] = None) -> typing.Union[str, None]: """ Searches for a configuration file in a hierarchical order, starting with DEFAULT_CONFIG_FILENAME, then LEGACY_CONFIG_FILENAME, @@ -89,8 +90,8 @@ def _get_local_config_path(local_config_root_cutoff: typing.Union[str, None] = N """ # The case where DEFAULT_CONFIG_FILENAME file in current directory. - if (os.path.isfile(DEFAULT_CONFIG_FILENAME)): - return os.path.abspath(DEFAULT_CONFIG_FILENAME) + if (os.path.isfile(config_file_name)): + return os.path.abspath(config_file_name) # The case where LEGACY_CONFIG_FILENAME file in current directory. if (os.path.isfile(LEGACY_CONFIG_FILENAME)): @@ -100,12 +101,13 @@ def _get_local_config_path(local_config_root_cutoff: typing.Union[str, None] = N parent_dir = os.path.dirname(os.getcwd()) return _get_ancestor_config_file_path( parent_dir, + config_file_name = config_file_name, local_config_root_cutoff = local_config_root_cutoff ) def _get_ancestor_config_file_path( current_directory: str, - config_file: str = DEFAULT_CONFIG_FILENAME, + config_file_name: str, local_config_root_cutoff: typing.Union[str, None] = None )-> typing.Union[str, None]: """ @@ -120,7 +122,7 @@ def _get_ancestor_config_file_path( local_config_root_cutoff = os.path.abspath(local_config_root_cutoff) for _ in range(edq.util.dirent.DEPTH_LIMIT): - config_file_path = os.path.join(current_directory, config_file) + config_file_path = os.path.join(current_directory, config_file_name) if (os.path.isfile(config_file_path)): return config_file_path From cca8ea6f30037f356ca99cea1244a5e6815917eb Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 15 Aug 2025 23:35:54 -0700 Subject: [PATCH 05/71] Reordered some checks. --- edq/testing/unittest.py | 1 - edq/util/config.py | 8 ++++---- edq/util/config_test.py | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/edq/testing/unittest.py b/edq/testing/unittest.py index bacc9ab..bdd95a7 100644 --- a/edq/testing/unittest.py +++ b/edq/testing/unittest.py @@ -4,7 +4,6 @@ import edq.util.json import edq.util.reflection - FORMAT_STR: str = "\n--- Expected ---\n%s\n--- Actual ---\n%s\n---\n" class BaseTest(unittest.TestCase): diff --git a/edq/util/config.py b/edq/util/config.py index efe0417..1fd50de 100644 --- a/edq/util/config.py +++ b/edq/util/config.py @@ -26,12 +26,12 @@ def get_tiered_config( and where that key came from. """ - config: typing.Dict[str, str] = {} - sources: typing.Dict[str, str] = {} - if (skip_keys is None): skip_keys = [CONFIG_PATHS_KEY] + config: typing.Dict[str, str] = {} + sources: typing.Dict[str, str] = {} + if (isinstance(cli_arguments, argparse.Namespace)): cli_arguments = vars(cli_arguments) @@ -117,10 +117,10 @@ def _get_ancestor_config_file_path( Otherwise, returns None. """ - current_directory = os.path.abspath(current_directory) if (local_config_root_cutoff is not None): local_config_root_cutoff = os.path.abspath(local_config_root_cutoff) + current_directory = os.path.abspath(current_directory) for _ in range(edq.util.dirent.DEPTH_LIMIT): config_file_path = os.path.join(current_directory, config_file_name) if (os.path.isfile(config_file_path)): diff --git a/edq/util/config_test.py b/edq/util/config_test.py index 71011dc..f0f8e6c 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -209,8 +209,8 @@ def test_base(self): ) def _evaluate_test_config( - self, test_work_dir, expected_config, expected_source, - skip_keys = None, + self, test_work_dir, expected_config, + expected_source, skip_keys = None, cli_args = None, global_config_path = None): """ Prepares testing environment and normalizes cli config paths, From 588f452908ca5b33f7900eaaec8dc905979d8726 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sat, 16 Aug 2025 00:56:16 -0700 Subject: [PATCH 06/71] Generalizes the use case to edq. --- edq/util/config.py | 4 +- edq/util/config_test.py | 59 ++++++++++++------- .../new-edq-config.json} | 0 .../{autograder.json => edq-config.json} | 0 .../edq-config.json} | 0 .../{autograder.json => edq-config.json} | 0 .../nest1/nest2b/edq-config.json} | 0 .../testdata/configs/simple/edq-config.json | 3 + 8 files changed, 43 insertions(+), 23 deletions(-) rename edq/util/testdata/configs/{global/autograder.json => custom-name/new-edq-config.json} (100%) rename edq/util/testdata/configs/empty/{autograder.json => edq-config.json} (100%) rename edq/util/testdata/configs/{nested/nest1/nest2b/autograder.json => global/edq-config.json} (100%) rename edq/util/testdata/configs/nested/{autograder.json => edq-config.json} (100%) rename edq/util/testdata/configs/{simple/autograder.json => nested/nest1/nest2b/edq-config.json} (100%) create mode 100644 edq/util/testdata/configs/simple/edq-config.json diff --git a/edq/util/config.py b/edq/util/config.py index 1fd50de..c33e1ed 100644 --- a/edq/util/config.py +++ b/edq/util/config.py @@ -9,16 +9,16 @@ CONFIG_PATHS_KEY: str = 'config_paths' LEGACY_CONFIG_FILENAME: str = 'config.json' -DEFAULT_CONFIG_FILENAME: str = 'autograder.json' +DEFAULT_CONFIG_FILENAME: str = 'edq-config.json' DEFAULT_GLOBAL_CONFIG_PATH: str = platformdirs.user_config_dir(DEFAULT_CONFIG_FILENAME) CONFIG_TYPE_DELIMITER: str = "::" def get_tiered_config( cli_arguments: typing.Union[dict, argparse.Namespace], + config_file_name: str, skip_keys: typing.Union[list, None] = None, global_config_path: str = DEFAULT_GLOBAL_CONFIG_PATH, local_config_root_cutoff: typing.Union[str, None] = None, - config_file_name: str = DEFAULT_CONFIG_FILENAME )-> typing.Tuple[typing.Dict[str, str], typing.Dict[str, str]]: """ Get all the tiered configuration options (from files and CLI). diff --git a/edq/util/config_test.py b/edq/util/config_test.py index f0f8e6c..64bcb2c 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -15,13 +15,25 @@ def test_base(self): # [(work directory, expected config, expected source, {skip keys , cli arguments, config global}), ...] test_cases = [ + ( + "custom-name", + { + "user": "user@test.edulinq.org" + }, + { + "user": f"::{os.path.join('TEMP_DIR', 'custom-name', 'new-edq-config.json')}" + }, + { + "config_file_name": "new-edq-config.json" + } + ), ( "simple", { "user": "user@test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'simple', 'autograder.json')}" + "user": f"::{os.path.join('TEMP_DIR', 'simple', 'edq-config.json')}" }, {} ), @@ -41,7 +53,7 @@ def test_base(self): "server": "http://test.edulinq.org" }, { - "server": f"::{os.path.join('TEMP_DIR', 'nested', 'autograder.json')}" + "server": f"::{os.path.join('TEMP_DIR', 'nested', 'edq-config.json')}" }, {} ), @@ -51,7 +63,7 @@ def test_base(self): "user": "user@test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'nested', 'nest1', 'nest2b', 'autograder.json')}" + "user": f"::{os.path.join('TEMP_DIR', 'nested', 'nest1', 'nest2b', 'edq-config.json')}" }, {} ), @@ -61,10 +73,10 @@ def test_base(self): "user": "user@test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'global', 'autograder.json')}" + "user": f"::{os.path.join('TEMP_DIR', 'global', 'edq-config.json')}" }, { - "global_config_path": os.path.join("global", "autograder.json") + "global_config_path": os.path.join("global", "edq-config.json") } ), ( @@ -105,13 +117,13 @@ def test_base(self): "user": "user@test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'simple', 'autograder.json')}" + "user": f"::{os.path.join('TEMP_DIR', 'simple', 'edq-config.json')}" }, { "cli_args": { edq.util.config.CONFIG_PATHS_KEY: [ - os.path.join("global", "autograder.json"), - os.path.join("simple", "autograder.json") + os.path.join("global", "edq-config.json"), + os.path.join("simple", "edq-config.json") ] } } @@ -123,14 +135,14 @@ def test_base(self): "server": "http://test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'simple', 'autograder.json')}", - "server": f"::{os.path.join('TEMP_DIR', 'nested', 'autograder.json')}" + "user": f"::{os.path.join('TEMP_DIR', 'simple', 'edq-config.json')}", + "server": f"::{os.path.join('TEMP_DIR', 'nested', 'edq-config.json')}" }, { "cli_args": { edq.util.config.CONFIG_PATHS_KEY: [ - os.path.join("nested", "autograder.json"), - os.path.join("simple", "autograder.json") + os.path.join("nested", "edq-config.json"), + os.path.join("simple", "edq-config.json") ] } } @@ -141,10 +153,10 @@ def test_base(self): "user": "user@test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'simple', 'autograder.json')}" + "user": f"::{os.path.join('TEMP_DIR', 'simple', 'edq-config.json')}" }, { - "global_config_path": os.path.join("global", "autograder.json") + "global_config_path": os.path.join("global", "edq-config.json") } ), ( @@ -153,13 +165,13 @@ def test_base(self): "user": "user@test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'simple', 'autograder.json')}" + "user": f"::{os.path.join('TEMP_DIR', 'simple', 'edq-config.json')}" }, { "cli_args": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("simple", "autograder.json")] + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("simple", "edq-config.json")] }, - "global_config_path": os.path.join("global", "autograder.json") + "global_config_path": os.path.join("global", "edq-config.json") } ), ( @@ -196,7 +208,7 @@ def test_base(self): "server", edq.util.config.CONFIG_PATHS_KEY ], - "global_config_path": os.path.join("global", "autograder.json") + "global_config_path": os.path.join("global", "edq-config.json") } ) ] @@ -211,7 +223,8 @@ def test_base(self): def _evaluate_test_config( self, test_work_dir, expected_config, expected_source, skip_keys = None, - cli_args = None, global_config_path = None): + cli_args = None, global_config_path = None, + config_file_name = None): """ Prepares testing environment and normalizes cli config paths, global config path and expected source paths. Evaluates the given expected and @@ -224,7 +237,10 @@ def _evaluate_test_config( if (skip_keys is None): skip_keys = [edq.util.config.CONFIG_PATHS_KEY] - temp_dir = edq.util.dirent.get_temp_dir(prefix = 'autograder-test-config-') + if (config_file_name is None): + config_file_name = edq.util.config.DEFAULT_CONFIG_FILENAME + + temp_dir = edq.util.dirent.get_temp_dir(prefix = 'edq-test-config-') global_config = os.path.join(temp_dir) if (global_config_path is not None): @@ -248,7 +264,8 @@ def _evaluate_test_config( cli_arguments = cli_args, global_config_path = global_config, local_config_root_cutoff = temp_dir, - skip_keys = skip_keys + skip_keys = skip_keys, + config_file_name = config_file_name ) finally: os.chdir(previous_work_directory) diff --git a/edq/util/testdata/configs/global/autograder.json b/edq/util/testdata/configs/custom-name/new-edq-config.json similarity index 100% rename from edq/util/testdata/configs/global/autograder.json rename to edq/util/testdata/configs/custom-name/new-edq-config.json diff --git a/edq/util/testdata/configs/empty/autograder.json b/edq/util/testdata/configs/empty/edq-config.json similarity index 100% rename from edq/util/testdata/configs/empty/autograder.json rename to edq/util/testdata/configs/empty/edq-config.json diff --git a/edq/util/testdata/configs/nested/nest1/nest2b/autograder.json b/edq/util/testdata/configs/global/edq-config.json similarity index 100% rename from edq/util/testdata/configs/nested/nest1/nest2b/autograder.json rename to edq/util/testdata/configs/global/edq-config.json diff --git a/edq/util/testdata/configs/nested/autograder.json b/edq/util/testdata/configs/nested/edq-config.json similarity index 100% rename from edq/util/testdata/configs/nested/autograder.json rename to edq/util/testdata/configs/nested/edq-config.json diff --git a/edq/util/testdata/configs/simple/autograder.json b/edq/util/testdata/configs/nested/nest1/nest2b/edq-config.json similarity index 100% rename from edq/util/testdata/configs/simple/autograder.json rename to edq/util/testdata/configs/nested/nest1/nest2b/edq-config.json diff --git a/edq/util/testdata/configs/simple/edq-config.json b/edq/util/testdata/configs/simple/edq-config.json new file mode 100644 index 0000000..98b5beb --- /dev/null +++ b/edq/util/testdata/configs/simple/edq-config.json @@ -0,0 +1,3 @@ +{ + "user": "user@test.edulinq.org" +} From f34c44fbca328d4fc7b0cf79c0fc8d6b507d12ee Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sat, 16 Aug 2025 21:13:41 -0700 Subject: [PATCH 07/71] Revised unittest. --- edq/testing/unittest.py | 39 ++++++++------------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/edq/testing/unittest.py b/edq/testing/unittest.py index bdd95a7..8770f71 100644 --- a/edq/testing/unittest.py +++ b/edq/testing/unittest.py @@ -14,7 +14,7 @@ class BaseTest(unittest.TestCase): maxDiff = None """ Don't limit the size of diffs. """ - def assertJSONDictEqual(self, a: dict, b: dict) -> None: # pylint: disable=invalid-name + def assertJSONDictEqual(self, a: dict, b: dict, msg: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name """ Call assertDictEqual(), but supply a message containing the full JSON representation of the arguments. """ @@ -22,9 +22,12 @@ def assertJSONDictEqual(self, a: dict, b: dict) -> None: # pylint: disable=inva a_json = edq.util.json.dumps(a, indent = 4) b_json = edq.util.json.dumps(b, indent = 4) - super().assertDictEqual(a, b, FORMAT_STR % (a_json, b_json)) + if(msg is None): + msg = FORMAT_STR % (a_json, b_json) + + super().assertDictEqual(a, b, msg = msg) - def assertJSONListEqual(self, a: list, b: list) -> None: # pylint: disable=invalid-name + def assertJSONListEqual(self, a: list, b: list, msg: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name """ Call assertListEqual(), but supply a message containing the full JSON representation of the arguments. """ @@ -32,37 +35,11 @@ def assertJSONListEqual(self, a: list, b: list) -> None: # pylint: disable=inva a_json = edq.util.json.dumps(a, indent = 4) b_json = edq.util.json.dumps(b, indent = 4) - super().assertListEqual(a, b, FORMAT_STR % (a_json, b_json)) - - def assertListEqual(self, list1: list, list2: list, msg: typing.Any = None) -> None: - """ - Assert two lists are equal, showing JSON-formatted output when they differ. - """ - - list1_json = edq.util.json.dumps(list1, indent = 4) - list2_json = edq.util.json.dumps(list2, indent = 4) - if(msg is None): - msg = FORMAT_STR % (list1_json, list2_json) + msg = FORMAT_STR % (a_json, b_json) - super().assertListEqual(list1, list2, msg = msg) - - def assertDictEqual(self, - d1: typing.Mapping[typing.Any, object], - d2: typing.Mapping[typing.Any, object], - msg: typing.Any = None - ) -> None: - """ - Assert two dicts are equal, showing JSON-formatted output when they differ. - """ - - d1_json = edq.util.json.dumps(d1, indent = 4) - d2_json = edq.util.json.dumps(d2, indent = 4) - - if(msg is None): - msg = FORMAT_STR % (d1_json, d2_json) + super().assertListEqual(a, b, FORMAT_STR % (a_json, b_json)) - super().assertDictEqual(d1, d2, msg = msg) def format_error_string(self, ex: typing.Union[BaseException, None]) -> str: """ From bf9808346e32d9dee0c160e23fae6ee32babc3ba Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sun, 17 Aug 2025 16:01:15 -0700 Subject: [PATCH 08/71] Revised config and config tests. --- edq/util/config.py | 111 ++++++++++----- edq/util/config_test.py | 289 +++++++++++++++++++++++----------------- 2 files changed, 245 insertions(+), 155 deletions(-) diff --git a/edq/util/config.py b/edq/util/config.py index c33e1ed..3ffcdb9 100644 --- a/edq/util/config.py +++ b/edq/util/config.py @@ -8,29 +8,60 @@ import edq.util.json CONFIG_PATHS_KEY: str = 'config_paths' -LEGACY_CONFIG_FILENAME: str = 'config.json' -DEFAULT_CONFIG_FILENAME: str = 'edq-config.json' -DEFAULT_GLOBAL_CONFIG_PATH: str = platformdirs.user_config_dir(DEFAULT_CONFIG_FILENAME) -CONFIG_TYPE_DELIMITER: str = "::" +DEFAULT_CONFIG_FILENAME = "edq-config.json" + +class ConfigSource: + """ A class for storing config source in a structured way. """ + + def __init__(self, label, path = None): + self.label = label + self.path = path + + def to_dict(self): + """ Return a dict that can be used to represent this object. """ + + return vars(self) + + def __eq__(self, other: object) -> bool: + """ Check for equality. This check uses to_dict() and compares the results. """ + + # Note the hard type check (done so we can keep this method general). + if (type(self) != type(other)): # pylint: disable=unidiomatic-typecheck + return False + + return bool(self.to_dict() == other.to_dict()) # type: ignore[attr-defined] + + def __str__(self) -> str: + return f"label is {self.label}, path is {self.path}" + + def __repr__(self) -> str: + return f"ConfigSource({self.to_dict()!r})" def get_tiered_config( - cli_arguments: typing.Union[dict, argparse.Namespace], - config_file_name: str, + config_file_name: str = DEFAULT_CONFIG_FILENAME , + legacy_config_file_name: typing.Union[str, None] = None, + global_config_path: typing.Union[str, None] = None, skip_keys: typing.Union[list, None] = None, - global_config_path: str = DEFAULT_GLOBAL_CONFIG_PATH, + cli_arguments: typing.Union[dict, argparse.Namespace, None] = None, local_config_root_cutoff: typing.Union[str, None] = None, - )-> typing.Tuple[typing.Dict[str, str], typing.Dict[str, str]]: + )-> typing.Tuple[typing.Dict[str, str], typing.Dict[str, ConfigSource]]: """ Get all the tiered configuration options (from files and CLI). If |show_sources| is True, then an addition dict will be returned that shows each key, and where that key came from. """ + if (global_config_path is None): + global_config_path = platformdirs.user_config_dir(config_file_name) + + if (cli_arguments is None): + cli_arguments = {} + if (skip_keys is None): skip_keys = [CONFIG_PATHS_KEY] config: typing.Dict[str, str] = {} - sources: typing.Dict[str, str] = {} + sources: typing.Dict[str, ConfigSource] = {} if (isinstance(cli_arguments, argparse.Namespace)): cli_arguments = vars(cli_arguments) @@ -40,7 +71,12 @@ def get_tiered_config( _load_config_file(global_config_path, config, sources, "") # Check the local user config file. - local_config_path = _get_local_config_path(config_file_name = config_file_name, local_config_root_cutoff = local_config_root_cutoff) + local_config_path = _get_local_config_path( + config_file_name = config_file_name, + legacy_config_file_name = legacy_config_file_name, + local_config_root_cutoff = local_config_root_cutoff + ) + if (local_config_path is not None): _load_config_file(local_config_path, config, sources, "") @@ -59,45 +95,50 @@ def get_tiered_config( continue config[key] = value - sources[key] = "" + sources[key] = ConfigSource(label = "") return config, sources def _load_config_file( config_path: str, config: typing.Dict[str, str], - sources: typing.Dict[str, str], - source_label: str, - encoding: str = edq.util.dirent.DEFAULT_ENCODING + sources: typing.Dict[str, ConfigSource], + source_label: str )-> None: """ Loads configs and the source from the given config JSON file. """ - with open(config_path, 'r', encoding = encoding) as file: - for (key, value) in edq.util.json.load(file).items(): - config[key] = value - sources[key] = f"{source_label}{CONFIG_TYPE_DELIMITER}{os.path.abspath(config_path)}" + for (key, value) in edq.util.json.load_path(config_path).items(): + config[key] = value + sources[key] = ConfigSource(label = source_label, path = os.path.abspath(config_path)) -def _get_local_config_path(config_file_name: str, local_config_root_cutoff: typing.Union[str, None] = None) -> typing.Union[str, None]: +def _get_local_config_path( + config_file_name: str, + legacy_config_file_name: typing.Union[str, None] = None, + local_config_root_cutoff: typing.Union[str, None] = None + ) -> typing.Union[str, None]: """ - Searches for a configuration file in a hierarchical order, - starting with DEFAULT_CONFIG_FILENAME, then LEGACY_CONFIG_FILENAME, - and continuing up the directory tree looking for DEFAULT_CONFIG_FILENAME. - Returns the path to the first configuration file found. - - If no configuration file is found, returns None. - The cutoff limits config search depth. - This helps to prevent detection of a config file in higher directories during testing. + Searches for a config file in hierarchical order. + Begins with the provided config file name, + optionally checks the legacy config file name if specified, + then continues up the directory tree looking for the provided config file name. + Returns the path to the first config file found. + + If no config file is found, returns None. + + The cutoff parameter limits the search depth, preventing detection of + config file in higher-level directories during testing. """ - # The case where DEFAULT_CONFIG_FILENAME file in current directory. + # The case where provided config file in current directory. if (os.path.isfile(config_file_name)): return os.path.abspath(config_file_name) - # The case where LEGACY_CONFIG_FILENAME file in current directory. - if (os.path.isfile(LEGACY_CONFIG_FILENAME)): - return os.path.abspath(LEGACY_CONFIG_FILENAME) + # The case where provided legacy config file in current directory. + if (legacy_config_file_name is not None): + if (os.path.isfile(legacy_config_file_name )): + return os.path.abspath(legacy_config_file_name ) - # The case where a DEFAULT_CONFIG_FILENAME file located in any ancestor directory on the path to root. + # The case where a provided config file located in any ancestor directory on the path to root. parent_dir = os.path.dirname(os.getcwd()) return _get_ancestor_config_file_path( parent_dir, @@ -111,9 +152,9 @@ def _get_ancestor_config_file_path( local_config_root_cutoff: typing.Union[str, None] = None )-> typing.Union[str, None]: """ - Search through the parent directories (until root or a given cutoff directory(inclusive)) for a configuration file. - Stops at the first occurrence of the specified config file (default: DEFAULT_CONFIG_FILENAME) along the path to root. - Returns the path if a configuration file is found. + Search through the parent directories (until root or a given cutoff directory(inclusive)) for a config file. + Stops at the first occurrence of the specified config file along the path to root. + Returns the path if a config file is found. Otherwise, returns None. """ diff --git a/edq/util/config_test.py b/edq/util/config_test.py index 64bcb2c..c1f57c1 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -10,197 +10,265 @@ class TestConfig(edq.testing.unittest.BaseTest): """ Test basic operations on configs. """ - def test_base(self): + def test_get_tiered_config_base(self): """ Test that configs are loaded correctly from the file system with the correct tier. """ - # [(work directory, expected config, expected source, {skip keys , cli arguments, config global}), ...] + # [(work directory, expected config, expected source, extra arguments), ...] test_cases = [ ( - "custom-name", + # Global Config: Custom global config path. + "empty-dir", { "user": "user@test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'custom-name', 'new-edq-config.json')}" + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "global", "edq-config.json") + ) }, { - "config_file_name": "new-edq-config.json" + "global_config_path": os.path.join("TEMP_DIR", "global", "edq-config.json"), + "local_config_root_cutoff": "TEMP_DIR" } ), ( - "simple", + # Local Config: Custom config file in current directory. + "custom-name", { "user": "user@test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'simple', 'edq-config.json')}" + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "custom-name", "new-edq-config.json" ) + ) }, - {} + { + "config_file_name": "new-edq-config.json", + "local_config_root_cutoff": "TEMP_DIR" + } ), ( - "old-name", + # Local Config: Default config file in current directory. + "simple", { "user": "user@test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'old-name', 'config.json')}" + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "simple", "edq-config.json") + ) }, - {} + { + "local_config_root_cutoff": "TEMP_DIR" + } ), ( - os.path.join("nested", "nest1", "nest2a"), + # Local Config: Legacy config file in current directory. + "old-name", { - "server": "http://test.edulinq.org" + "user": "user@test.edulinq.org" }, { - "server": f"::{os.path.join('TEMP_DIR', 'nested', 'edq-config.json')}" + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "old-name", "config.json") + ) }, - {} + { + "legacy_config_file_name": "config.json", + "local_config_root_cutoff": "TEMP_DIR" + } ), ( - os.path.join("nested", "nest1", "nest2b"), + # Local Config: Default config file in an ancestor directory. + os.path.join("nested", "nest1", "nest2a"), { - "user": "user@test.edulinq.org" + "server": "http://test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'nested', 'nest1', 'nest2b', 'edq-config.json')}" + "server": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "nested", "edq-config.json") + ) }, - {} + { + "local_config_root_cutoff": "TEMP_DIR" + } ), ( - "empty-dir", + # Local Config: All variations. + os.path.join("nested", "nest1", "nest2b"), { "user": "user@test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'global', 'edq-config.json')}" + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "nested", "nest1", "nest2b", "edq-config.json") + ) }, - { - "global_config_path": os.path.join("global", "edq-config.json") + { "legacy_config_file_name": "config.json", + "local_config_root_cutoff": "TEMP_DIR" } ), ( + # CLI Provided Config: Distinct keys. "empty-dir", { - "user": "user@test.edulinq.org" + "user": "user@test.edulinq.org", + "server": "http://test.edulinq.org" }, { - "user": "" + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "simple", "edq-config.json") + ), + "server": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "nested", "edq-config.json") + ) }, { - "cli_args": { - "user": "user@test.edulinq.org" - } + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join("TEMP_DIR", "simple", "edq-config.json"), + os.path.join("TEMP_DIR", "nested", "edq-config.json") + ] + }, + "local_config_root_cutoff": "TEMP_DIR" } ), ( + # CLI Provided Config: Overriding keys. "empty-dir", { "user": "user@test.edulinq.org" }, { - "user": "" + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "simple", "edq-config.json") + ) }, { - "cli_args": { - "user": "user@test.edulinq.org", - "pass": "user" + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join("TEMP_DIR", "custom-name", "new-edq-config.json"), + os.path.join("TEMP_DIR", "simple", "edq-config.json") + ] }, - "skip_keys": [ - "pass" - ] + "local_config_root_cutoff": "TEMP_DIR" } ), ( + # CLI Bare Options: CLI arguments only (direct key: value). "empty-dir", { "user": "user@test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'simple', 'edq-config.json')}" + "user": edq.util.config.ConfigSource(label = "") }, { - "cli_args": { - edq.util.config.CONFIG_PATHS_KEY: [ - os.path.join("global", "edq-config.json"), - os.path.join("simple", "edq-config.json") - ] - } + "cli_arguments": { + "user": "user@test.edulinq.org" + }, + "local_config_root_cutoff": "TEMP_DIR" } ), ( + # CLI Bare Options: Skip keys functionally. "empty-dir", { - "user": "user@test.edulinq.org", - "server": "http://test.edulinq.org" + "user": "user@test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'simple', 'edq-config.json')}", - "server": f"::{os.path.join('TEMP_DIR', 'nested', 'edq-config.json')}" + "user": edq.util.config.ConfigSource(label = "") }, { - "cli_args": { - edq.util.config.CONFIG_PATHS_KEY: [ - os.path.join("nested", "edq-config.json"), - os.path.join("simple", "edq-config.json") - ] - } + "cli_arguments": { + "user": "user@test.edulinq.org", + "pass": "user" + }, + "skip_keys": [ + "pass" + ], + "local_config_root_cutoff": "TEMP_DIR" } ), ( + # Combination: Local Config + Global Config "simple", { "user": "user@test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'simple', 'edq-config.json')}" + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "simple", "edq-config.json") + ) }, { - "global_config_path": os.path.join("global", "edq-config.json") + "global_config_path": os.path.join("global", "edq-config.json"), + "local_config_root_cutoff": "TEMP_DIR" } ), ( - "empty-dir", + # Combination: CLI Provided Config + Local Config + "simple", { "user": "user@test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'simple', 'edq-config.json')}" + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "old-name", "config.json") + ) }, { - "cli_args": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("simple", "edq-config.json")] + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")] }, - "global_config_path": os.path.join("global", "edq-config.json") + "local_config_root_cutoff": "TEMP_DIR" } ), ( - "simple", + # Combination: CLI Bare Options + CLI Provided Config + "empty-dir", { "user": "user@test.edulinq.org" }, { - "user": f"::{os.path.join('TEMP_DIR', 'old-name', 'config.json')}" + "user": edq.util.config.ConfigSource(label = "") }, { - "cli_args": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("old-name", "config.json")] + "cli_arguments": { + "user": "user@test.edulinq.org", + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "simple", "edq-config.json")] }, + "local_config_root_cutoff": "TEMP_DIR" } ), ( - "simple", + # Combination: CLI Bare Options + CLI Provided Config + Local Config + Global Config + os.path.join("nested", "nest1", "nest2b"), { "user": "user@test.edulinq.org", "pass": "user" }, { - "user": f"::{os.path.join('TEMP_DIR', 'old-name', 'config.json')}", - "pass": "" + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "old-name", "config.json") + ), + "pass": edq.util.config.ConfigSource(label = "") }, { - "cli_args": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("old-name", "config.json")], + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")], "pass": "user", "server": "http://test.edulinq.org" }, @@ -208,70 +276,51 @@ def test_base(self): "server", edq.util.config.CONFIG_PATHS_KEY ], - "global_config_path": os.path.join("global", "edq-config.json") + "global_config_path": os.path.join("TEMP_DIR", "global", "edq-config.json"), + "local_config_root_cutoff": "TEMP_DIR" } ) ] - for test_case in test_cases: + for (i, test_case) in enumerate(test_cases): (test_work_dir, expected_config, expected_source, extra_args) = test_case - self._evaluate_test_config( - test_work_dir, expected_config, expected_source, **extra_args - ) - - def _evaluate_test_config( - self, test_work_dir, expected_config, - expected_source, skip_keys = None, - cli_args = None, global_config_path = None, - config_file_name = None): - """ - Prepares testing environment and normalizes cli config paths, - global config path and expected source paths. Evaluates the given expected and - source configs with actual get_tiered_config() output. - """ - - if (cli_args is None): - cli_args = {} + with self.subTest(msg = f"Case {i} ('{test_work_dir}'):"): + temp_dir = edq.util.dirent.get_temp_dir(prefix = "edq-test-config-") - if (skip_keys is None): - skip_keys = [edq.util.config.CONFIG_PATHS_KEY] + _replace_placeholders_dict(extra_args, "TEMP_DIR", temp_dir) - if (config_file_name is None): - config_file_name = edq.util.config.DEFAULT_CONFIG_FILENAME + cli_arguments = extra_args.get("cli_arguments", None) + if (cli_arguments is not None): + config_paths = cli_arguments.get(edq.util.config.CONFIG_PATHS_KEY, None) + if (config_paths is not None): + _replace_placeholders_list(config_paths, "TEMP_DIR", temp_dir) - temp_dir = edq.util.dirent.get_temp_dir(prefix = 'edq-test-config-') + edq.util.dirent.copy_contents(CONFIGS_DIR, temp_dir) - global_config = os.path.join(temp_dir) - if (global_config_path is not None): - global_config = os.path.join(temp_dir, global_config_path) + previous_work_directory = os.getcwd() + initial_work_directory = os.path.join(temp_dir, test_work_dir) + os.chdir(initial_work_directory) - abs_config_paths = [] - config_paths = cli_args.get(edq.util.config.CONFIG_PATHS_KEY, None) - if (config_paths is not None): - for rel_config_path in config_paths: - abs_config_paths.append(os.path.join(temp_dir, rel_config_path)) - cli_args[edq.util.config.CONFIG_PATHS_KEY] = abs_config_paths + try: + (actual_configs, actual_sources) = edq.util.config.get_tiered_config(**extra_args) + finally: + os.chdir(previous_work_directory) - edq.util.dirent.copy_contents(CONFIGS_DIR, temp_dir) + for (key, value) in actual_sources.items(): + if value.path is not None: + value.path = value.path.replace(temp_dir, "TEMP_DIR") + actual_sources[key] = value - previous_work_directory = os.getcwd() - initial_work_directory = os.path.join(temp_dir, test_work_dir) - os.chdir(initial_work_directory) - - try: - (actual_configs, actual_sources) = edq.util.config.get_tiered_config( - cli_arguments = cli_args, - global_config_path = global_config, - local_config_root_cutoff = temp_dir, - skip_keys = skip_keys, - config_file_name = config_file_name - ) - finally: - os.chdir(previous_work_directory) + self.assertJSONDictEqual(expected_config, actual_configs) + self.assertJSONDictEqual(expected_source, actual_sources) - for (key, value) in actual_sources.items(): - actual_sources[key] = value.replace(temp_dir, "TEMP_DIR") +def _replace_placeholders_dict(data, old, new): + for (key, value) in data.items(): + if (isinstance(value, str)): + if (old in value): + data[key] = value.replace(old, new) - self.assertDictEqual(expected_config, actual_configs) - self.assertDictEqual(expected_source, actual_sources) +def _replace_placeholders_list(data, old, new): + for (i, path) in enumerate(data): + data[i] = path.replace(old, new) From e550e8bfa0e030ccfd0e0f0dd62e7f0f1478e153 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sun, 17 Aug 2025 16:16:46 -0700 Subject: [PATCH 09/71] pointed to defualt config file name in tests. --- edq/util/config_test.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/edq/util/config_test.py b/edq/util/config_test.py index c1f57c1..35b66d1 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -24,11 +24,11 @@ def test_get_tiered_config_base(self): { "user": edq.util.config.ConfigSource( label = "", - path = os.path.join("TEMP_DIR", "global", "edq-config.json") + path = os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME) ) }, { - "global_config_path": os.path.join("TEMP_DIR", "global", "edq-config.json"), + "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME ), "local_config_root_cutoff": "TEMP_DIR" } ), @@ -58,7 +58,7 @@ def test_get_tiered_config_base(self): { "user": edq.util.config.ConfigSource( label = "", - path = os.path.join("TEMP_DIR", "simple", "edq-config.json") + path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ) }, { @@ -91,7 +91,7 @@ def test_get_tiered_config_base(self): { "server": edq.util.config.ConfigSource( label = "", - path = os.path.join("TEMP_DIR", "nested", "edq-config.json") + path = os.path.join("TEMP_DIR", "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) ) }, { @@ -107,7 +107,7 @@ def test_get_tiered_config_base(self): { "user": edq.util.config.ConfigSource( label = "", - path = os.path.join("TEMP_DIR", "nested", "nest1", "nest2b", "edq-config.json") + path = os.path.join("TEMP_DIR", "nested", "nest1", "nest2b", edq.util.config.DEFAULT_CONFIG_FILENAME) ) }, { "legacy_config_file_name": "config.json", @@ -124,18 +124,18 @@ def test_get_tiered_config_base(self): { "user": edq.util.config.ConfigSource( label = "", - path = os.path.join("TEMP_DIR", "simple", "edq-config.json") + path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ), "server": edq.util.config.ConfigSource( label = "", - path = os.path.join("TEMP_DIR", "nested", "edq-config.json") + path = os.path.join("TEMP_DIR", "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) ) }, { "cli_arguments": { edq.util.config.CONFIG_PATHS_KEY: [ - os.path.join("TEMP_DIR", "simple", "edq-config.json"), - os.path.join("TEMP_DIR", "nested", "edq-config.json") + os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), + os.path.join("TEMP_DIR", "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) ] }, "local_config_root_cutoff": "TEMP_DIR" @@ -150,14 +150,14 @@ def test_get_tiered_config_base(self): { "user": edq.util.config.ConfigSource( label = "", - path = os.path.join("TEMP_DIR", "simple", "edq-config.json") + path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ) }, { "cli_arguments": { edq.util.config.CONFIG_PATHS_KEY: [ os.path.join("TEMP_DIR", "custom-name", "new-edq-config.json"), - os.path.join("TEMP_DIR", "simple", "edq-config.json") + os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ] }, "local_config_root_cutoff": "TEMP_DIR" @@ -208,11 +208,11 @@ def test_get_tiered_config_base(self): { "user": edq.util.config.ConfigSource( label = "", - path = os.path.join("TEMP_DIR", "simple", "edq-config.json") + path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ) }, { - "global_config_path": os.path.join("global", "edq-config.json"), + "global_config_path": os.path.join("global", edq.util.config.DEFAULT_CONFIG_FILENAME), "local_config_root_cutoff": "TEMP_DIR" } ), @@ -247,7 +247,7 @@ def test_get_tiered_config_base(self): { "cli_arguments": { "user": "user@test.edulinq.org", - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "simple", "edq-config.json")] + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)] }, "local_config_root_cutoff": "TEMP_DIR" } @@ -276,7 +276,7 @@ def test_get_tiered_config_base(self): "server", edq.util.config.CONFIG_PATHS_KEY ], - "global_config_path": os.path.join("TEMP_DIR", "global", "edq-config.json"), + "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), "local_config_root_cutoff": "TEMP_DIR" } ) From aa14f7f16e31a4a8595d8bc8640f11d05759a7dc Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Mon, 18 Aug 2025 09:11:30 -0700 Subject: [PATCH 10/71] Revised the first pass. --- edq/testing/unittest.py | 5 ++- edq/util/config.py | 32 +++++++++---------- edq/util/config_test.py | 18 +++++------ .../configs/nested/nest1/nest2b/config.json | 2 +- 4 files changed, 27 insertions(+), 30 deletions(-) diff --git a/edq/testing/unittest.py b/edq/testing/unittest.py index 5f3c7de..69c5dbf 100644 --- a/edq/testing/unittest.py +++ b/edq/testing/unittest.py @@ -16,7 +16,7 @@ class BaseTest(unittest.TestCase): def assertJSONDictEqual(self, a: typing.Dict[str, typing.Any], b: typing.Dict[str, typing.Any], msg: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name """ - Call assertDictEqual(), but supply a message containing the full JSON representation of the arguments. + Call assertDictEqual(). If no message is provided, defaults to a message containing the full JSON representation of the arguments. """ a_json = edq.util.json.dumps(a, indent = 4) @@ -29,7 +29,7 @@ def assertJSONDictEqual(self, a: typing.Dict[str, typing.Any], b: typing.Dict[st def assertJSONListEqual(self, a: typing.List[typing.Any], b: typing.List[typing.Any], msg: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name """ - Call assertListEqual(), but supply a message containing the full JSON representation of the arguments. + Call assertListEqual(), If no message is provided, defaults to a message containing the full JSON representation of the arguments. """ a_json = edq.util.json.dumps(a, indent = 4) @@ -40,7 +40,6 @@ def assertJSONListEqual(self, a: typing.List[typing.Any], b: typing.List[typing. super().assertListEqual(a, b, FORMAT_STR % (a_json, b_json)) - def format_error_string(self, ex: typing.Union[BaseException, None]) -> str: """ Format an error string from an exception so it can be checked for testing. diff --git a/edq/util/config.py b/edq/util/config.py index 3ffcdb9..d5d9a81 100644 --- a/edq/util/config.py +++ b/edq/util/config.py @@ -25,7 +25,6 @@ def to_dict(self): def __eq__(self, other: object) -> bool: """ Check for equality. This check uses to_dict() and compares the results. """ - # Note the hard type check (done so we can keep this method general). if (type(self) != type(other)): # pylint: disable=unidiomatic-typecheck return False @@ -38,7 +37,7 @@ def __repr__(self) -> str: return f"ConfigSource({self.to_dict()!r})" def get_tiered_config( - config_file_name: str = DEFAULT_CONFIG_FILENAME , + config_file_name: str = DEFAULT_CONFIG_FILENAME, legacy_config_file_name: typing.Union[str, None] = None, global_config_path: typing.Union[str, None] = None, skip_keys: typing.Union[list, None] = None, @@ -46,20 +45,19 @@ def get_tiered_config( local_config_root_cutoff: typing.Union[str, None] = None, )-> typing.Tuple[typing.Dict[str, str], typing.Dict[str, ConfigSource]]: """ - Get all the tiered configuration options (from files and CLI). - If |show_sources| is True, then an addition dict will be returned that shows each key, - and where that key came from. + Load all tiered configuration options from files and command-line arguments. + Returns a configuration dictionary with the values based on tiering rules and a source dictionary mapping each key to its origin. """ if (global_config_path is None): global_config_path = platformdirs.user_config_dir(config_file_name) - if (cli_arguments is None): - cli_arguments = {} - if (skip_keys is None): skip_keys = [CONFIG_PATHS_KEY] + if (cli_arguments is None): + cli_arguments = {} + config: typing.Dict[str, str] = {} sources: typing.Dict[str, ConfigSource] = {} @@ -105,7 +103,7 @@ def _load_config_file( sources: typing.Dict[str, ConfigSource], source_label: str )-> None: - """ Loads configs and the source from the given config JSON file. """ + """ Loads config variables and the source from the given config JSON file. """ for (key, value) in edq.util.json.load_path(config_path).items(): config[key] = value @@ -129,16 +127,16 @@ def _get_local_config_path( config file in higher-level directories during testing. """ - # The case where provided config file in current directory. + # The case where provided config file is in current directory. if (os.path.isfile(config_file_name)): return os.path.abspath(config_file_name) - # The case where provided legacy config file in current directory. + # The case where provided legacy config file is in current directory. if (legacy_config_file_name is not None): - if (os.path.isfile(legacy_config_file_name )): - return os.path.abspath(legacy_config_file_name ) + if (os.path.isfile(legacy_config_file_name)): + return os.path.abspath(legacy_config_file_name) - # The case where a provided config file located in any ancestor directory on the path to root. + # The case where a provided config file is located in any ancestor directory on the path to root. parent_dir = os.path.dirname(os.getcwd()) return _get_ancestor_config_file_path( parent_dir, @@ -167,13 +165,13 @@ def _get_ancestor_config_file_path( if (os.path.isfile(config_file_path)): return config_file_path - if (local_config_root_cutoff == current_directory): - break - parent_dir = os.path.dirname(current_directory) if (parent_dir == current_directory): break + if (local_config_root_cutoff == current_directory): + break + current_directory = parent_dir return None diff --git a/edq/util/config_test.py b/edq/util/config_test.py index 35b66d1..55393b9 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -303,24 +303,24 @@ def test_get_tiered_config_base(self): os.chdir(initial_work_directory) try: - (actual_configs, actual_sources) = edq.util.config.get_tiered_config(**extra_args) + (actual_config, actual_sources) = edq.util.config.get_tiered_config(**extra_args) finally: os.chdir(previous_work_directory) for (key, value) in actual_sources.items(): - if value.path is not None: + if (value.path is not None): value.path = value.path.replace(temp_dir, "TEMP_DIR") actual_sources[key] = value - self.assertJSONDictEqual(expected_config, actual_configs) + self.assertJSONDictEqual(expected_config, actual_config) self.assertJSONDictEqual(expected_source, actual_sources) -def _replace_placeholders_dict(data, old, new): - for (key, value) in data.items(): +def _replace_placeholders_dict(data_dict, old, new): + for (key, value) in data_dict.items(): if (isinstance(value, str)): if (old in value): - data[key] = value.replace(old, new) + data_dict[key] = value.replace(old, new) -def _replace_placeholders_list(data, old, new): - for (i, path) in enumerate(data): - data[i] = path.replace(old, new) +def _replace_placeholders_list(data_list, old, new): + for (i, path) in enumerate(data_list): + data_list[i] = path.replace(old, new) diff --git a/edq/util/testdata/configs/nested/nest1/nest2b/config.json b/edq/util/testdata/configs/nested/nest1/nest2b/config.json index 669b7b6..0a5bcec 100644 --- a/edq/util/testdata/configs/nested/nest1/nest2b/config.json +++ b/edq/util/testdata/configs/nested/nest1/nest2b/config.json @@ -1,3 +1,3 @@ { - "pass": "123456" + "pass": "user" } From 3b228555226098f0fdccfdc302e8abd61a76222d Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Mon, 18 Aug 2025 09:20:33 -0700 Subject: [PATCH 11/71] Passed in the msg for unittesting. --- edq/testing/unittest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edq/testing/unittest.py b/edq/testing/unittest.py index 69c5dbf..2881ff0 100644 --- a/edq/testing/unittest.py +++ b/edq/testing/unittest.py @@ -25,7 +25,7 @@ def assertJSONDictEqual(self, a: typing.Dict[str, typing.Any], b: typing.Dict[st if(msg is None): msg = FORMAT_STR % (a_json, b_json) - super().assertDictEqual(a, b, FORMAT_STR % (a_json, b_json)) + super().assertDictEqual(a, b, msg = msg) def assertJSONListEqual(self, a: typing.List[typing.Any], b: typing.List[typing.Any], msg: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name """ @@ -38,7 +38,7 @@ def assertJSONListEqual(self, a: typing.List[typing.Any], b: typing.List[typing. if(msg is None): msg = FORMAT_STR % (a_json, b_json) - super().assertListEqual(a, b, FORMAT_STR % (a_json, b_json)) + super().assertListEqual(a, b, msg = msg) def format_error_string(self, ex: typing.Union[BaseException, None]) -> str: """ From 9f0be06cb2386db7f1982c1b93a452b558d5bded Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Mon, 18 Aug 2025 13:56:20 -0700 Subject: [PATCH 12/71] Reviewd the PR, need to work on testing comments. --- edq/testing/unittest.py | 10 +-- edq/util/config.py | 34 ++++----- edq/util/config_test.py | 149 ++++++++++++++++++++++------------------ 3 files changed, 102 insertions(+), 91 deletions(-) diff --git a/edq/testing/unittest.py b/edq/testing/unittest.py index 2881ff0..f6c6e3d 100644 --- a/edq/testing/unittest.py +++ b/edq/testing/unittest.py @@ -16,26 +16,28 @@ class BaseTest(unittest.TestCase): def assertJSONDictEqual(self, a: typing.Dict[str, typing.Any], b: typing.Dict[str, typing.Any], msg: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name """ - Call assertDictEqual(). If no message is provided, defaults to a message containing the full JSON representation of the arguments. + Call assertDictEqual(), but supply a message containing the full JSON representation of the arguments. + If a custom message is provided, it will override the JSON representation. """ a_json = edq.util.json.dumps(a, indent = 4) b_json = edq.util.json.dumps(b, indent = 4) - if(msg is None): + if (msg is None): msg = FORMAT_STR % (a_json, b_json) super().assertDictEqual(a, b, msg = msg) def assertJSONListEqual(self, a: typing.List[typing.Any], b: typing.List[typing.Any], msg: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name """ - Call assertListEqual(), If no message is provided, defaults to a message containing the full JSON representation of the arguments. + Call assertDictEqual(), but supply a message containing the full JSON representation of the arguments. + If a custom message is provided, it will override the JSON representation. """ a_json = edq.util.json.dumps(a, indent = 4) b_json = edq.util.json.dumps(b, indent = 4) - if(msg is None): + if (msg is None): msg = FORMAT_STR % (a_json, b_json) super().assertListEqual(a, b, msg = msg) diff --git a/edq/util/config.py b/edq/util/config.py index d5d9a81..59c03f1 100644 --- a/edq/util/config.py +++ b/edq/util/config.py @@ -11,30 +11,24 @@ DEFAULT_CONFIG_FILENAME = "edq-config.json" class ConfigSource: - """ A class for storing config source in a structured way. """ + """ A class for storing config source information in a structured way. """ - def __init__(self, label, path = None): + def __init__(self, label: str, path: typing.Union[str, None] = None): self.label = label - self.path = path - - def to_dict(self): - """ Return a dict that can be used to represent this object. """ + """ Label of a config. """ - return vars(self) + self.path = path + """ Source path of a config. """ def __eq__(self, other: object) -> bool: - """ Check for equality. This check uses to_dict() and compares the results. """ + """ Check for equality. """ - if (type(self) != type(other)): # pylint: disable=unidiomatic-typecheck - return False - - return bool(self.to_dict() == other.to_dict()) # type: ignore[attr-defined] + return ((self.label == other.label) and (self.path == other.path)) # type: ignore[attr-defined] def __str__(self) -> str: - return f"label is {self.label}, path is {self.path}" + """ Return a human-readable string representation of ConfigSource object. """ - def __repr__(self) -> str: - return f"ConfigSource({self.to_dict()!r})" + return f"({self.label}, {self.path})" def get_tiered_config( config_file_name: str = DEFAULT_CONFIG_FILENAME, @@ -43,7 +37,7 @@ def get_tiered_config( skip_keys: typing.Union[list, None] = None, cli_arguments: typing.Union[dict, argparse.Namespace, None] = None, local_config_root_cutoff: typing.Union[str, None] = None, - )-> typing.Tuple[typing.Dict[str, str], typing.Dict[str, ConfigSource]]: + ) -> typing.Tuple[typing.Dict[str, str], typing.Dict[str, ConfigSource]]: """ Load all tiered configuration options from files and command-line arguments. Returns a configuration dictionary with the values based on tiering rules and a source dictionary mapping each key to its origin. @@ -61,6 +55,7 @@ def get_tiered_config( config: typing.Dict[str, str] = {} sources: typing.Dict[str, ConfigSource] = {} + # Ensure CLI arguments are always a dict, even if provided as argparse.Namespace. if (isinstance(cli_arguments, argparse.Namespace)): cli_arguments = vars(cli_arguments) @@ -102,7 +97,7 @@ def _load_config_file( config: typing.Dict[str, str], sources: typing.Dict[str, ConfigSource], source_label: str - )-> None: + ) -> None: """ Loads config variables and the source from the given config JSON file. """ for (key, value) in edq.util.json.load_path(config_path).items(): @@ -115,7 +110,7 @@ def _get_local_config_path( local_config_root_cutoff: typing.Union[str, None] = None ) -> typing.Union[str, None]: """ - Searches for a config file in hierarchical order. + Search for a config file in hierarchical order. Begins with the provided config file name, optionally checks the legacy config file name if specified, then continues up the directory tree looking for the provided config file name. @@ -148,7 +143,7 @@ def _get_ancestor_config_file_path( current_directory: str, config_file_name: str, local_config_root_cutoff: typing.Union[str, None] = None - )-> typing.Union[str, None]: + ) -> typing.Union[str, None]: """ Search through the parent directories (until root or a given cutoff directory(inclusive)) for a config file. Stops at the first occurrence of the specified config file along the path to root. @@ -165,6 +160,7 @@ def _get_ancestor_config_file_path( if (os.path.isfile(config_file_path)): return config_file_path + # Check if current directory is root. parent_dir = os.path.dirname(current_directory) if (parent_dir == current_directory): break diff --git a/edq/util/config_test.py b/edq/util/config_test.py index 55393b9..ca0c78f 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -11,98 +11,101 @@ class TestConfig(edq.testing.unittest.BaseTest): """ Test basic operations on configs. """ def test_get_tiered_config_base(self): - """ Test that configs are loaded correctly from the file system with the correct tier. """ + """ + Test that configuration files are loaded correctly from the file system with the expected tier. + The placeholder 'TEMP_DIR' is overwritten during testing with the actual path to the directory. + """ # [(work directory, expected config, expected source, extra arguments), ...] test_cases = [ + # Global Config: Custom global config path. ( - # Global Config: Custom global config path. "empty-dir", { - "user": "user@test.edulinq.org" + "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = "", path = os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME) - ) + ), }, { - "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME ), - "local_config_root_cutoff": "TEMP_DIR" + "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), } ), + + # Local Config + + # Custom config file in current directory. ( - # Local Config: Custom config file in current directory. "custom-name", { - "user": "user@test.edulinq.org" + "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = "", - path = os.path.join("TEMP_DIR", "custom-name", "new-edq-config.json" ) - ) + path = os.path.join("TEMP_DIR", "custom-name", "new-edq-config.json") + ), }, { "config_file_name": "new-edq-config.json", - "local_config_root_cutoff": "TEMP_DIR" } ), + + # Default config file in current directory. ( - # Local Config: Default config file in current directory. "simple", { - "user": "user@test.edulinq.org" + "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = "", path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) - ) + ), }, - { - "local_config_root_cutoff": "TEMP_DIR" - } + {} ), + + # Legacy config file in current directory. ( - # Local Config: Legacy config file in current directory. "old-name", { - "user": "user@test.edulinq.org" + "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = "", path = os.path.join("TEMP_DIR", "old-name", "config.json") - ) + ), }, { "legacy_config_file_name": "config.json", - "local_config_root_cutoff": "TEMP_DIR" } ), + + # Default config file in an ancestor directory. ( - # Local Config: Default config file in an ancestor directory. os.path.join("nested", "nest1", "nest2a"), { - "server": "http://test.edulinq.org" + "server": "http://test.edulinq.org", }, { "server": edq.util.config.ConfigSource( label = "", path = os.path.join("TEMP_DIR", "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) - ) + ), }, - { - "local_config_root_cutoff": "TEMP_DIR" - } + {} ), + ( - # Local Config: All variations. + # All Variations os.path.join("nested", "nest1", "nest2b"), { - "user": "user@test.edulinq.org" + "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( @@ -110,16 +113,19 @@ def test_get_tiered_config_base(self): path = os.path.join("TEMP_DIR", "nested", "nest1", "nest2b", edq.util.config.DEFAULT_CONFIG_FILENAME) ) }, - { "legacy_config_file_name": "config.json", - "local_config_root_cutoff": "TEMP_DIR" + { + "legacy_config_file_name": "config.json", } ), + + # CLI Provided Config + + # Distinct Keys ( - # CLI Provided Config: Distinct keys. "empty-dir", { "user": "user@test.edulinq.org", - "server": "http://test.edulinq.org" + "server": "http://test.edulinq.org", }, { "user": edq.util.config.ConfigSource( @@ -129,64 +135,66 @@ def test_get_tiered_config_base(self): "server": edq.util.config.ConfigSource( label = "", path = os.path.join("TEMP_DIR", "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) - ) + ), }, { "cli_arguments": { edq.util.config.CONFIG_PATHS_KEY: [ os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), os.path.join("TEMP_DIR", "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) - ] + ], }, - "local_config_root_cutoff": "TEMP_DIR" } ), + + # Overriding Keys ( - # CLI Provided Config: Overriding keys. "empty-dir", { - "user": "user@test.edulinq.org" + "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = "", path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) - ) + ), }, { "cli_arguments": { edq.util.config.CONFIG_PATHS_KEY: [ os.path.join("TEMP_DIR", "custom-name", "new-edq-config.json"), os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) - ] + ], }, - "local_config_root_cutoff": "TEMP_DIR" } ), + + # CLI Bare Options: + + # CLI arguments only (direct key: value). ( - # CLI Bare Options: CLI arguments only (direct key: value). "empty-dir", { - "user": "user@test.edulinq.org" + "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = "") + "user": edq.util.config.ConfigSource(label = ""), }, { "cli_arguments": { "user": "user@test.edulinq.org" }, - "local_config_root_cutoff": "TEMP_DIR" } ), + + # Skip keys functionally. ( - # CLI Bare Options: Skip keys functionally. "empty-dir", { - "user": "user@test.edulinq.org" + "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = "") + "user": edq.util.config.ConfigSource(label = ""), }, { "cli_arguments": { @@ -196,90 +204,91 @@ def test_get_tiered_config_base(self): "skip_keys": [ "pass" ], - "local_config_root_cutoff": "TEMP_DIR" } ), + + # Combinations + + # Local Config + Global Config ( - # Combination: Local Config + Global Config "simple", { - "user": "user@test.edulinq.org" + "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = "", path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) - ) + ), }, { "global_config_path": os.path.join("global", edq.util.config.DEFAULT_CONFIG_FILENAME), - "local_config_root_cutoff": "TEMP_DIR" } ), + + # CLI Provided Config + Local Config ( - # Combination: CLI Provided Config + Local Config "simple", { - "user": "user@test.edulinq.org" + "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = "", path = os.path.join("TEMP_DIR", "old-name", "config.json") - ) + ), }, { "cli_arguments": { edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")] }, - "local_config_root_cutoff": "TEMP_DIR" } ), + + # CLI Bare Options + CLI Provided Config ( - # Combination: CLI Bare Options + CLI Provided Config "empty-dir", { - "user": "user@test.edulinq.org" + "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = "") + "user": edq.util.config.ConfigSource(label = ""), }, { "cli_arguments": { "user": "user@test.edulinq.org", edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)] }, - "local_config_root_cutoff": "TEMP_DIR" } ), + + # CLI Bare Options + CLI Provided Config + Local Config + Global Config ( - # Combination: CLI Bare Options + CLI Provided Config + Local Config + Global Config os.path.join("nested", "nest1", "nest2b"), { "user": "user@test.edulinq.org", - "pass": "user" + "pass": "user", }, { "user": edq.util.config.ConfigSource( label = "", path = os.path.join("TEMP_DIR", "old-name", "config.json") ), - "pass": edq.util.config.ConfigSource(label = "") + "pass": edq.util.config.ConfigSource(label = ""), }, { "cli_arguments": { edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")], "pass": "user", - "server": "http://test.edulinq.org" + "server": "http://test.edulinq.org", }, "skip_keys": [ "server", - edq.util.config.CONFIG_PATHS_KEY + edq.util.config.CONFIG_PATHS_KEY, ], "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), - "local_config_root_cutoff": "TEMP_DIR" } - ) + ), ] for (i, test_case) in enumerate(test_cases): @@ -296,6 +305,10 @@ def test_get_tiered_config_base(self): if (config_paths is not None): _replace_placeholders_list(config_paths, "TEMP_DIR", temp_dir) + cutoff = extra_args.get("local_config_root_cutoff", None) + if (cutoff is None): + extra_args["local_config_root_cutoff"] = "TEMP_DIR" + edq.util.dirent.copy_contents(CONFIGS_DIR, temp_dir) previous_work_directory = os.getcwd() From a9efa0bc50c563be816195ec1e87f883faf882ff Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Wed, 20 Aug 2025 11:58:04 -0700 Subject: [PATCH 13/71] Reviewed testing. --- edq/cli/config/list.py | 48 ++++ edq/testing/unittest.py | 4 +- edq/util/config.py | 10 +- edq/util/config_test.py | 224 ++++++++++++++++-- .../configs/malformatted/edq-config.json | 3 + .../testdata/configs/nested/nest1/config.json | 3 + .../configs/old-name/nest1/nest2/.gitignore | 2 + 7 files changed, 268 insertions(+), 26 deletions(-) create mode 100644 edq/cli/config/list.py create mode 100644 edq/util/testdata/configs/malformatted/edq-config.json create mode 100644 edq/util/testdata/configs/nested/nest1/config.json create mode 100644 edq/util/testdata/configs/old-name/nest1/nest2/.gitignore diff --git a/edq/cli/config/list.py b/edq/cli/config/list.py new file mode 100644 index 0000000..2e999f1 --- /dev/null +++ b/edq/cli/config/list.py @@ -0,0 +1,48 @@ +import sys + +import edq.util.config + +DESCRIPTION = "List your current configuration options." + +def run(args): + (config, sources) = edq.util.config.get_tiered_config( + cli_arguments = args, + skip_keys = [ + 'show_origin', 'verbose', + edq.util.config.CONFIG_PATHS_KEY, 'global_config_path', + ], + global_config_path = args.global_config_path + ) + + config_list = [] + for (key, value) in config.items(): + config_str = f"{key}\t{value}" + if (args.show_origin): + raw_source = sources.get(key) + source_path = raw_source.split(edq.util.config.CONFIG_TYPE_DELIMITER)[1] + config_str += f"\t{source_path}" + + config_list.append(config_str) + + print("\n".join(config_list)) + return 0 + +def _get_parser(): + parser = edq.util.config.get_argument_parser( + description = DESCRIPTION, + skip_server = True) + + parser.add_argument("--show-origin", dest = 'show_origin', + action = 'store_true', help = "Shows where each configuration's value was obtained from.") + + parser.add_argument("--global-config", dest = 'global_config_path', + action = 'store', type = str, default = edq.util.config.DEFAULT_GLOBAL_CONFIG_PATH, + help = 'Path to the global configuration file (default: %(default)s).') + + return parser + +def main(): + return run(_get_parser().parse_args()) + +if (__name__ == '__main__'): + sys.exit(main()) diff --git a/edq/testing/unittest.py b/edq/testing/unittest.py index f6c6e3d..4418bc5 100644 --- a/edq/testing/unittest.py +++ b/edq/testing/unittest.py @@ -17,7 +17,7 @@ class BaseTest(unittest.TestCase): def assertJSONDictEqual(self, a: typing.Dict[str, typing.Any], b: typing.Dict[str, typing.Any], msg: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name """ Call assertDictEqual(), but supply a message containing the full JSON representation of the arguments. - If a custom message is provided, it will override the JSON representation. + If a custom message is provided, it will replace the default JSON-based message. """ a_json = edq.util.json.dumps(a, indent = 4) @@ -31,7 +31,7 @@ def assertJSONDictEqual(self, a: typing.Dict[str, typing.Any], b: typing.Dict[st def assertJSONListEqual(self, a: typing.List[typing.Any], b: typing.List[typing.Any], msg: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name """ Call assertDictEqual(), but supply a message containing the full JSON representation of the arguments. - If a custom message is provided, it will override the JSON representation. + If a custom message is provided, it will replace the default JSON-based message. """ a_json = edq.util.json.dumps(a, indent = 4) diff --git a/edq/util/config.py b/edq/util/config.py index 59c03f1..9198721 100644 --- a/edq/util/config.py +++ b/edq/util/config.py @@ -15,18 +15,18 @@ class ConfigSource: def __init__(self, label: str, path: typing.Union[str, None] = None): self.label = label - """ Label of a config. """ + """ Label of a config.""" self.path = path - """ Source path of a config. """ + """ Path of a config's source. """ def __eq__(self, other: object) -> bool: - """ Check for equality. """ + """ Check for equality between ConfigSource objects. """ return ((self.label == other.label) and (self.path == other.path)) # type: ignore[attr-defined] def __str__(self) -> str: - """ Return a human-readable string representation of ConfigSource object. """ + """ Return a human-readable string representation of the ConfigSource object. """ return f"({self.label}, {self.path})" @@ -131,7 +131,7 @@ def _get_local_config_path( if (os.path.isfile(legacy_config_file_name)): return os.path.abspath(legacy_config_file_name) - # The case where a provided config file is located in any ancestor directory on the path to root. + # Case where the provided config file is found in an ancestor directory up to the root or cutoff limit. parent_dir = os.path.dirname(os.getcwd()) return _get_ancestor_config_file_path( parent_dir, diff --git a/edq/util/config_test.py b/edq/util/config_test.py index ca0c78f..c96d004 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -32,7 +32,8 @@ def test_get_tiered_config_base(self): }, { "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), - } + }, + None ), # Local Config @@ -51,7 +52,8 @@ def test_get_tiered_config_base(self): }, { "config_file_name": "new-edq-config.json", - } + }, + None ), # Default config file in current directory. @@ -66,7 +68,8 @@ def test_get_tiered_config_base(self): path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ), }, - {} + {}, + None ), # Legacy config file in current directory. @@ -83,7 +86,8 @@ def test_get_tiered_config_base(self): }, { "legacy_config_file_name": "config.json", - } + }, + None ), # Default config file in an ancestor directory. @@ -98,11 +102,21 @@ def test_get_tiered_config_base(self): path = os.path.join("TEMP_DIR", "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) ), }, - {} + {}, + None + ), + + # Legacy config file in an ancestor directory. + ( + os.path.join("old-name", "nest1", "nest2"), + {}, + {}, + {}, + None ), + # All Variations ( - # All Variations os.path.join("nested", "nest1", "nest2b"), { "user": "user@test.edulinq.org", @@ -115,7 +129,8 @@ def test_get_tiered_config_base(self): }, { "legacy_config_file_name": "config.json", - } + }, + None ), # CLI Provided Config @@ -144,7 +159,8 @@ def test_get_tiered_config_base(self): os.path.join("TEMP_DIR", "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) ], }, - } + }, + None ), # Overriding Keys @@ -166,7 +182,8 @@ def test_get_tiered_config_base(self): os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ], }, - } + }, + None ), # CLI Bare Options: @@ -184,7 +201,8 @@ def test_get_tiered_config_base(self): "cli_arguments": { "user": "user@test.edulinq.org" }, - } + }, + None ), # Skip keys functionally. @@ -204,7 +222,8 @@ def test_get_tiered_config_base(self): "skip_keys": [ "pass" ], - } + }, + None ), # Combinations @@ -223,7 +242,8 @@ def test_get_tiered_config_base(self): }, { "global_config_path": os.path.join("global", edq.util.config.DEFAULT_CONFIG_FILENAME), - } + }, + None ), # CLI Provided Config + Local Config @@ -242,7 +262,8 @@ def test_get_tiered_config_base(self): "cli_arguments": { edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")] }, - } + }, + None ), # CLI Bare Options + CLI Provided Config @@ -259,7 +280,8 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)] }, - } + }, + None ), # CLI Bare Options + CLI Provided Config + Local Config + Global Config @@ -287,17 +309,162 @@ def test_get_tiered_config_base(self): edq.util.config.CONFIG_PATHS_KEY, ], "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), - } + }, + None + ), + + # Edge Cases + + # No Config + ( + "empty-dir", + {}, + {}, + {}, + None + ), + + # Global Config: Empty Config JSON + ( + "empty-dir", + {}, + {}, + { + "global_config_path": os.path.join("TEMP_DIR", "empty", edq.util.config.DEFAULT_CONFIG_FILENAME), + }, + None + ), + + # Global Config: Directory Config JSON + ( + "empty-dir", + {}, + {}, + { + "global_config_path": os.path.join("TEMP_DIR", "dir-config", edq.util.config.DEFAULT_CONFIG_FILENAME), + }, + None + ), + + # Global Config: Non-Existent Config JSON + ( + "empty-dir", + {}, + {}, + { + "global_config_path": os.path.join("TEMP_DIR", "empty-dir", "non-existent-config.json"), + }, + None + ), + + # Global Config: Malformatted Config JSON + ( + "empty-dir", + { + "user": "user@test.edulinq.org" + }, + { + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME) + ), + }, + { + "global_config_path": os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME), + }, + None + ), + + # Local Config: Empty Config JSON + ( + "empty", + {}, + {}, + {}, + None + ), + + # Local Config: Malformatted Config JSON + ( + "malformatted", + { + "user": "user@test.edulinq.org" + }, + { + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME) + ), + }, + {}, + None + ), + + # CLI Provided Config: Empty Config JSON + ( + "empty-dir", + {}, + {}, + { + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "empty", edq.util.config.DEFAULT_CONFIG_FILENAME)], + }, + }, + None + ), + + # CLI Provided Config: Directory Config JSON + ( + "empty-dir", + {}, + {}, + { + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "dir-config", edq.util.config.DEFAULT_CONFIG_FILENAME)], + }, + }, + "IsADirectoryError" + ), + + # CLI Provided Config: Non-Existent Config JSON + ( + "empty-dir", + {}, + {}, + { + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "empty-dir", "non-existent-config.json")], + }, + }, + "FileNotFoundError" + ), + + # CLI Provided Config: Malformatted Config JSON + ( + "empty-dir", + { + "user": "user@test.edulinq.org" + }, + { + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME) + ), + }, + { + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME)], + }, + }, + None ), ] for (i, test_case) in enumerate(test_cases): - (test_work_dir, expected_config, expected_source, extra_args) = test_case + (test_work_dir, expected_config, expected_source, extra_args, error_substring) = test_case with self.subTest(msg = f"Case {i} ('{test_work_dir}'):"): - temp_dir = edq.util.dirent.get_temp_dir(prefix = "edq-test-config-") - - _replace_placeholders_dict(extra_args, "TEMP_DIR", temp_dir) + temp_dir = edq.util.dirent.get_temp_dir(prefix = "edq-test-config-get-tiered-config-") cli_arguments = extra_args.get("cli_arguments", None) if (cli_arguments is not None): @@ -309,6 +476,12 @@ def test_get_tiered_config_base(self): if (cutoff is None): extra_args["local_config_root_cutoff"] = "TEMP_DIR" + global_config = extra_args.get("global_config_path", None) + if (global_config is None): + extra_args["global_config_path"] = os.path.join("TEMP_DIR", "empty", edq.util.config.CONFIG_PATHS_KEY) + + _replace_placeholders_dict(extra_args, "TEMP_DIR", temp_dir) + edq.util.dirent.copy_contents(CONFIGS_DIR, temp_dir) previous_work_directory = os.getcwd() @@ -317,9 +490,22 @@ def test_get_tiered_config_base(self): try: (actual_config, actual_sources) = edq.util.config.get_tiered_config(**extra_args) + except Exception as ex: + error_string = self.format_error_string(ex) + + if (error_substring is None): + self.fail(f"Unexpected error: '{error_string}'.") + + self.assertIn(error_substring, error_string, 'Error is not as expected.') + + continue + finally: os.chdir(previous_work_directory) + if (error_substring is not None): + self.fail(f"Did not get expected error: '{error_substring}'.") + for (key, value) in actual_sources.items(): if (value.path is not None): value.path = value.path.replace(temp_dir, "TEMP_DIR") diff --git a/edq/util/testdata/configs/malformatted/edq-config.json b/edq/util/testdata/configs/malformatted/edq-config.json new file mode 100644 index 0000000..f619907 --- /dev/null +++ b/edq/util/testdata/configs/malformatted/edq-config.json @@ -0,0 +1,3 @@ +{ + "user": "user@test.edulinq.org", +} diff --git a/edq/util/testdata/configs/nested/nest1/config.json b/edq/util/testdata/configs/nested/nest1/config.json new file mode 100644 index 0000000..d53f13d --- /dev/null +++ b/edq/util/testdata/configs/nested/nest1/config.json @@ -0,0 +1,3 @@ +{ + "server": "http://test.edulinq.org" +} diff --git a/edq/util/testdata/configs/old-name/nest1/nest2/.gitignore b/edq/util/testdata/configs/old-name/nest1/nest2/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/edq/util/testdata/configs/old-name/nest1/nest2/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore From ee3d21be8fb8d4668493d1007822a8e53f273546 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Wed, 20 Aug 2025 13:54:40 -0700 Subject: [PATCH 14/71] Revised questions in mind and added missing config directory. --- edq/testing/unittest.py | 7 ++----- edq/util/config.py | 5 ++--- .../testdata/configs/dir-config/edq-config.json/.gitignore | 2 ++ 3 files changed, 6 insertions(+), 8 deletions(-) create mode 100644 edq/util/testdata/configs/dir-config/edq-config.json/.gitignore diff --git a/edq/testing/unittest.py b/edq/testing/unittest.py index 54996a8..61ed237 100644 --- a/edq/testing/unittest.py +++ b/edq/testing/unittest.py @@ -18,9 +18,7 @@ def assertJSONDictEqual(self, a: typing.Any, b: typing.Any, msg: typing.Union[st """ Like unittest.TestCase.assertDictEqual(), but will try to convert each comparison argument to a dict if it is not already, - and uses an assertion message containing the full JSON representation of the arguments. - - If a custom message is provided, it will replace the default JSON-based message. + and uses an default assertion message containing the full JSON representation of the arguments. """ if (not isinstance(a, dict)): @@ -45,8 +43,7 @@ def assertJSONDictEqual(self, a: typing.Any, b: typing.Any, msg: typing.Union[st def assertJSONListEqual(self, a: typing.List[typing.Any], b: typing.List[typing.Any], msg: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name """ - Call assertDictEqual(), but supply a message containing the full JSON representation of the arguments. - If a custom message is provided, it will replace the default JSON-based message. + Call assertDictEqual(), but supply a default message containing the full JSON representation of the arguments. """ a_json = edq.util.json.dumps(a, indent = 4) diff --git a/edq/util/config.py b/edq/util/config.py index 9198721..aeade3c 100644 --- a/edq/util/config.py +++ b/edq/util/config.py @@ -21,13 +21,12 @@ def __init__(self, label: str, path: typing.Union[str, None] = None): """ Path of a config's source. """ def __eq__(self, other: object) -> bool: - """ Check for equality between ConfigSource objects. """ + if (not isinstance(other, ConfigSource)): + raise TypeError(f"Cannot compare ConfigSource with '{type(other)}'") return ((self.label == other.label) and (self.path == other.path)) # type: ignore[attr-defined] def __str__(self) -> str: - """ Return a human-readable string representation of the ConfigSource object. """ - return f"({self.label}, {self.path})" def get_tiered_config( diff --git a/edq/util/testdata/configs/dir-config/edq-config.json/.gitignore b/edq/util/testdata/configs/dir-config/edq-config.json/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/edq/util/testdata/configs/dir-config/edq-config.json/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore From c06b6fcffa9e80aa199085d846ebbc56e175f1e5 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Wed, 20 Aug 2025 14:54:22 -0700 Subject: [PATCH 15/71] Testing Windows permission error on directories when loading JSON. --- edq/util/json.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/edq/util/json.py b/edq/util/json.py index 0f93819..1d4a00d 100644 --- a/edq/util/json.py +++ b/edq/util/json.py @@ -7,6 +7,7 @@ import abc import enum import json +import os import typing import json5 @@ -107,6 +108,8 @@ def load_path( otherwise use JSON5. """ + if (os.path.isdir(path)): + raise IsADirectoryError(f"{path} is a directory, not a file") try: with open(path, 'r', encoding = encoding) as file: return load(file, strict = strict, **kwargs) From 0f61041aee524952fd032d80702fa24334e0252c Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Wed, 20 Aug 2025 20:20:06 -0700 Subject: [PATCH 16/71] Changed the way equals returns for ConfigSource object. --- edq/util/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edq/util/config.py b/edq/util/config.py index aeade3c..5be8d15 100644 --- a/edq/util/config.py +++ b/edq/util/config.py @@ -22,7 +22,7 @@ def __init__(self, label: str, path: typing.Union[str, None] = None): def __eq__(self, other: object) -> bool: if (not isinstance(other, ConfigSource)): - raise TypeError(f"Cannot compare ConfigSource with '{type(other)}'") + return False return ((self.label == other.label) and (self.path == other.path)) # type: ignore[attr-defined] From 4cc295bdf9bd7125ef371b019739d1af5bd2c73d Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Wed, 20 Aug 2025 21:29:01 -0700 Subject: [PATCH 17/71] Tested all combinations of 4 config labels. --- edq/util/config_test.py | 158 ++++++++++++++++-- .../testdata/configs/simple/edq-config.json | 2 +- 2 files changed, 149 insertions(+), 11 deletions(-) diff --git a/edq/util/config_test.py b/edq/util/config_test.py index c96d004..966f0e3 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -18,6 +18,15 @@ def test_get_tiered_config_base(self): # [(work directory, expected config, expected source, extra arguments), ...] test_cases = [ + # No Config + ( + "empty-dir", + {}, + {}, + {}, + None + ), + # Global Config: Custom global config path. ( "empty-dir", @@ -241,11 +250,70 @@ def test_get_tiered_config_base(self): ), }, { - "global_config_path": os.path.join("global", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + }, + None + ), + + # Global + CLI Bare Options + ( + "empty-dir", + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.util.config.ConfigSource(label = ""), + }, + { + "cli_arguments": { + "user": "user@test.edulinq.org", + }, + "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + + }, + None + ), + + # CLI Bare Options + Local + ( + "simple", + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.util.config.ConfigSource(label = ""), + }, + { + "cli_arguments": { + "user": "user@test.edulinq.org" + }, + }, + None + ), + + + # CLI Provided Config + Global Config + ( + "empty-dir", + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) + ), + }, + { + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)] + }, + "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, None ), + # CLI Provided Config + Local Config ( "simple", @@ -284,6 +352,85 @@ def test_get_tiered_config_base(self): None ), + # CLI Bare Options + CLI Provided Config + Global Config + ( + "empty-dir", + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.util.config.ConfigSource(label = ""), + }, + { + "cli_arguments": { + "user": "user@test.edulinq.org", + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)] + }, + "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + }, + None + ), + + # CLI Bare Options + CLI Provided Config + Local Config + ( + "simple", + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.util.config.ConfigSource(label = ""), + }, + { + "cli_arguments": { + "user": "user@test.edulinq.org", + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")], + }, + }, + None + ), + + # CLI Bare Options + Local Config + Global Config + ( + "simple", + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.util.config.ConfigSource(label = ""), + }, + { + "cli_arguments": { + "user": "user@test.edulinq.org", + }, + "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + }, + None + ), + + # CLI Provided Config + Local Config + Global Config + ( + "simple", + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "old-name", "config.json") + ), + }, + { + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")], + }, + "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + }, + None + ), + + + + # CLI Bare Options + CLI Provided Config + Local Config + Global Config ( os.path.join("nested", "nest1", "nest2b"), @@ -315,15 +462,6 @@ def test_get_tiered_config_base(self): # Edge Cases - # No Config - ( - "empty-dir", - {}, - {}, - {}, - None - ), - # Global Config: Empty Config JSON ( "empty-dir", diff --git a/edq/util/testdata/configs/simple/edq-config.json b/edq/util/testdata/configs/simple/edq-config.json index 98b5beb..f619907 100644 --- a/edq/util/testdata/configs/simple/edq-config.json +++ b/edq/util/testdata/configs/simple/edq-config.json @@ -1,3 +1,3 @@ { - "user": "user@test.edulinq.org" + "user": "user@test.edulinq.org", } From 9d255952ebbef44477354172deb5217be379d40a Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Thu, 21 Aug 2025 00:10:47 -0700 Subject: [PATCH 18/71] Revised the structure of tests, need to go over them. --- edq/util/config_test.py | 369 +++++++++--------- ...edq-config.json => custom-edq-config.json} | 0 .../configs/malformatted/edq-config.json | 2 +- .../testdata/configs/nested/nest1/config.json | 3 - 4 files changed, 181 insertions(+), 193 deletions(-) rename edq/util/testdata/configs/custom-name/{new-edq-config.json => custom-edq-config.json} (100%) delete mode 100644 edq/util/testdata/configs/nested/nest1/config.json diff --git a/edq/util/config_test.py b/edq/util/config_test.py index 966f0e3..d540f75 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -27,7 +27,9 @@ def test_get_tiered_config_base(self): None ), - # Global Config: Custom global config path. + # Global Config + + # Custom global config path. ( "empty-dir", { @@ -45,26 +47,52 @@ def test_get_tiered_config_base(self): None ), - # Local Config - - # Custom config file in current directory. + # Empty Config JSON ( - "custom-name", + "empty-dir", + {}, + {}, { - "user": "user@test.edulinq.org", + "global_config_path": os.path.join("TEMP_DIR", "empty", edq.util.config.DEFAULT_CONFIG_FILENAME), }, + None + ), + + # Directory Config JSON + ( + "empty-dir", + {}, + {}, { - "user": edq.util.config.ConfigSource( - label = "", - path = os.path.join("TEMP_DIR", "custom-name", "new-edq-config.json") - ), + "global_config_path": os.path.join("TEMP_DIR", "dir-config", edq.util.config.DEFAULT_CONFIG_FILENAME), }, + None + ), + + # Non-Existent Config JSON + ( + "empty-dir", + {}, + {}, { - "config_file_name": "new-edq-config.json", + "global_config_path": os.path.join("TEMP_DIR", "empty-dir", "non-existent-config.json"), }, None ), + # Malformatted Config JSON + ( + "empty-dir", + {}, + {}, + { + "global_config_path": os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME), + }, + "Failed to read JSON file" + ), + + # Local Config + # Default config file in current directory. ( "simple", @@ -81,6 +109,24 @@ def test_get_tiered_config_base(self): None ), + # Custom config file in current directory. + ( + "custom-name", + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json") + ), + }, + { + "config_file_name": "custom-edq-config.json", + }, + None + ), + # Legacy config file in current directory. ( "old-name", @@ -124,7 +170,34 @@ def test_get_tiered_config_base(self): None ), - # All Variations + # Empty Config JSON + ( + "empty", + {}, + {}, + {}, + None + ), + + # Directory Config JSON + ( + "dir-config", + {}, + {}, + {}, + None + ), + + # Malformatted Config JSON + ( + "malformatted", + {}, + {}, + {}, + "Failed to read JSON file" + ), + + # All 3 local config locations present at the same time. ( os.path.join("nested", "nest1", "nest2b"), { @@ -187,7 +260,7 @@ def test_get_tiered_config_base(self): { "cli_arguments": { edq.util.config.CONFIG_PATHS_KEY: [ - os.path.join("TEMP_DIR", "custom-name", "new-edq-config.json"), + os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json"), os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ], }, @@ -195,6 +268,65 @@ def test_get_tiered_config_base(self): None ), + # Empty Config JSON + ( + "empty-dir", + {}, + {}, + { + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "empty", edq.util.config.DEFAULT_CONFIG_FILENAME)], + }, + }, + None + ), + + # Directory Config JSON + ( + "empty-dir", + {}, + {}, + { + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "dir-config", edq.util.config.DEFAULT_CONFIG_FILENAME)], + }, + }, + "IsADirectoryError" + ), + + # Non-Existent Config JSON + ( + "empty-dir", + {}, + {}, + { + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "empty-dir", "non-existent-config.json")], + }, + }, + "FileNotFoundError" + ), + + # Malformatted Config JSON + ( + "empty-dir", + { + "user": "user@test.edulinq.org" + }, + { + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME) + ), + }, + { + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME)], + }, + }, + "Failed to read JSON file" + ), + # CLI Bare Options: # CLI arguments only (direct key: value). @@ -237,7 +369,7 @@ def test_get_tiered_config_base(self): # Combinations - # Local Config + Global Config + # Global Config + Local Config ( "simple", { @@ -255,28 +387,30 @@ def test_get_tiered_config_base(self): None ), - # Global + CLI Bare Options + # Global Config + CLI Provided Config ( "empty-dir", { "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = ""), + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) + ), }, { "cli_arguments": { - "user": "user@test.edulinq.org", + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)] }, "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), - }, None ), - # CLI Bare Options + Local + # Global + CLI Bare Options ( - "simple", + "empty-dir", { "user": "user@test.edulinq.org", }, @@ -285,56 +419,52 @@ def test_get_tiered_config_base(self): }, { "cli_arguments": { - "user": "user@test.edulinq.org" + "user": "user@test.edulinq.org", }, + "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + }, None ), - - # CLI Provided Config + Global Config + # Local Config + CLI Provided Config ( - "empty-dir", + "simple", { "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = "", - path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) + path = os.path.join("TEMP_DIR", "old-name", "config.json") ), }, { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)] + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")] }, - "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, None ), - - # CLI Provided Config + Local Config + # Local Config + CLI Bare Options ( "simple", { "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource( - label = "", - path = os.path.join("TEMP_DIR", "old-name", "config.json") - ), + "user": edq.util.config.ConfigSource(label = ""), }, { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")] + "user": "user@test.edulinq.org" }, }, None ), - # CLI Bare Options + CLI Provided Config + # CLI Provided Config + CLI Bare Options ( "empty-dir", { @@ -352,7 +482,8 @@ def test_get_tiered_config_base(self): None ), - # CLI Bare Options + CLI Provided Config + Global Config + # Global Config + CLI Provided Config + CLI Bare Options + ( "empty-dir", { @@ -371,7 +502,7 @@ def test_get_tiered_config_base(self): None ), - # CLI Bare Options + CLI Provided Config + Local Config + # Global Config + Local Config + CLI Bare Options ( "simple", { @@ -382,56 +513,53 @@ def test_get_tiered_config_base(self): }, { "cli_arguments": { - "user": "user@test.edulinq.org", - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")], + "user": "user@test.edulinq.org", }, + "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, None ), - # CLI Bare Options + Local Config + Global Config + # Global Config + Local Config + CLI Provided Config ( "simple", { "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = ""), + "user": edq.util.config.ConfigSource( + label = "", + path = os.path.join("TEMP_DIR", "old-name", "config.json") + ), }, { "cli_arguments": { - "user": "user@test.edulinq.org", + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")], }, "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, None ), - # CLI Provided Config + Local Config + Global Config + # Local Config + CLI Provided Config + CLI Bare Options ( "simple", { "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource( - label = "", - path = os.path.join("TEMP_DIR", "old-name", "config.json") - ), + "user": edq.util.config.ConfigSource(label = ""), }, { "cli_arguments": { + "user": "user@test.edulinq.org", edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")], }, - "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, None ), - - - - # CLI Bare Options + CLI Provided Config + Local Config + Global Config + # Global Config + Local Config + CLI Provided Config + CLI Bare Options ( os.path.join("nested", "nest1", "nest2b"), { @@ -459,143 +587,6 @@ def test_get_tiered_config_base(self): }, None ), - - # Edge Cases - - # Global Config: Empty Config JSON - ( - "empty-dir", - {}, - {}, - { - "global_config_path": os.path.join("TEMP_DIR", "empty", edq.util.config.DEFAULT_CONFIG_FILENAME), - }, - None - ), - - # Global Config: Directory Config JSON - ( - "empty-dir", - {}, - {}, - { - "global_config_path": os.path.join("TEMP_DIR", "dir-config", edq.util.config.DEFAULT_CONFIG_FILENAME), - }, - None - ), - - # Global Config: Non-Existent Config JSON - ( - "empty-dir", - {}, - {}, - { - "global_config_path": os.path.join("TEMP_DIR", "empty-dir", "non-existent-config.json"), - }, - None - ), - - # Global Config: Malformatted Config JSON - ( - "empty-dir", - { - "user": "user@test.edulinq.org" - }, - { - "user": edq.util.config.ConfigSource( - label = "", - path = os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME) - ), - }, - { - "global_config_path": os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME), - }, - None - ), - - # Local Config: Empty Config JSON - ( - "empty", - {}, - {}, - {}, - None - ), - - # Local Config: Malformatted Config JSON - ( - "malformatted", - { - "user": "user@test.edulinq.org" - }, - { - "user": edq.util.config.ConfigSource( - label = "", - path = os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME) - ), - }, - {}, - None - ), - - # CLI Provided Config: Empty Config JSON - ( - "empty-dir", - {}, - {}, - { - "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "empty", edq.util.config.DEFAULT_CONFIG_FILENAME)], - }, - }, - None - ), - - # CLI Provided Config: Directory Config JSON - ( - "empty-dir", - {}, - {}, - { - "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "dir-config", edq.util.config.DEFAULT_CONFIG_FILENAME)], - }, - }, - "IsADirectoryError" - ), - - # CLI Provided Config: Non-Existent Config JSON - ( - "empty-dir", - {}, - {}, - { - "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "empty-dir", "non-existent-config.json")], - }, - }, - "FileNotFoundError" - ), - - # CLI Provided Config: Malformatted Config JSON - ( - "empty-dir", - { - "user": "user@test.edulinq.org" - }, - { - "user": edq.util.config.ConfigSource( - label = "", - path = os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME) - ), - }, - { - "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME)], - }, - }, - None - ), ] for (i, test_case) in enumerate(test_cases): diff --git a/edq/util/testdata/configs/custom-name/new-edq-config.json b/edq/util/testdata/configs/custom-name/custom-edq-config.json similarity index 100% rename from edq/util/testdata/configs/custom-name/new-edq-config.json rename to edq/util/testdata/configs/custom-name/custom-edq-config.json diff --git a/edq/util/testdata/configs/malformatted/edq-config.json b/edq/util/testdata/configs/malformatted/edq-config.json index f619907..bd5cacc 100644 --- a/edq/util/testdata/configs/malformatted/edq-config.json +++ b/edq/util/testdata/configs/malformatted/edq-config.json @@ -1,3 +1,3 @@ { - "user": "user@test.edulinq.org", + user user@test.edulinq.org } diff --git a/edq/util/testdata/configs/nested/nest1/config.json b/edq/util/testdata/configs/nested/nest1/config.json deleted file mode 100644 index d53f13d..0000000 --- a/edq/util/testdata/configs/nested/nest1/config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "server": "http://test.edulinq.org" -} From bcaa6093b22b25690954c49b63dc743f0105bee5 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Thu, 21 Aug 2025 10:18:14 -0700 Subject: [PATCH 19/71] Revised testcases and added file tree creation when testing. --- edq/util/config_test.py | 120 ++++++++++++++---- .../custom-name/custom-edq-config.json | 3 - .../dir-config/edq-config.json/.gitignore | 2 - .../testdata/configs/empty-dir/.gitignore | 2 - .../testdata/configs/empty/edq-config.json | 2 - .../testdata/configs/global/edq-config.json | 3 - .../configs/malformatted/edq-config.json | 3 - .../testdata/configs/nested/edq-config.json | 3 - .../configs/nested/nest1/nest2a/.gitignore | 2 - .../configs/nested/nest1/nest2b/config.json | 3 - .../nested/nest1/nest2b/edq-config.json | 3 - .../testdata/configs/old-name/config.json | 3 - .../configs/old-name/nest1/nest2/.gitignore | 2 - .../testdata/configs/simple/edq-config.json | 3 - 14 files changed, 97 insertions(+), 57 deletions(-) delete mode 100644 edq/util/testdata/configs/custom-name/custom-edq-config.json delete mode 100644 edq/util/testdata/configs/dir-config/edq-config.json/.gitignore delete mode 100644 edq/util/testdata/configs/empty-dir/.gitignore delete mode 100644 edq/util/testdata/configs/empty/edq-config.json delete mode 100644 edq/util/testdata/configs/global/edq-config.json delete mode 100644 edq/util/testdata/configs/malformatted/edq-config.json delete mode 100644 edq/util/testdata/configs/nested/edq-config.json delete mode 100644 edq/util/testdata/configs/nested/nest1/nest2a/.gitignore delete mode 100644 edq/util/testdata/configs/nested/nest1/nest2b/config.json delete mode 100644 edq/util/testdata/configs/nested/nest1/nest2b/edq-config.json delete mode 100644 edq/util/testdata/configs/old-name/config.json delete mode 100644 edq/util/testdata/configs/old-name/nest1/nest2/.gitignore delete mode 100644 edq/util/testdata/configs/simple/edq-config.json diff --git a/edq/util/config_test.py b/edq/util/config_test.py index d540f75..f6dcbf8 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -3,10 +3,93 @@ import edq.testing.unittest import edq.util.config import edq.util.dirent +import edq.util.json THIS_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) CONFIGS_DIR = os.path.join(THIS_DIR, "testdata", "configs") +def creat_test_dir(temp_dir_prefix: str) -> str: + """ + Creat a temp dir and populate it with dirents for testing. + + This test data directory is laid out as: + + . + ├── custom-name + | └── custom-edq-config.json + ├── dir-config + │   └── edq-config.json + ├── empty + │   └── edq-config.json + ├── empty-dir + ├── global + │   └── edq-config.json + ├── malformatted + │   └── edq-config.json + ├── nested + │   ├── edq-config.json + │   └── nest1 + │   ├── nest2a + │   └── nest2b + │   ├── config.json + │   └── edq-config.json + ├── old-name + │   ├── config.json + │   └── nest1 + │   └── nest2 + └── simple + └── edq-config.json + """ + + temp_dir = edq.util.dirent.get_temp_dir(prefix = temp_dir_prefix) + + empty_config_dir_path = os.path.join(temp_dir, "empty") + edq.util.dirent.mkdir(empty_config_dir_path) + edq.util.json.dump_path({}, os.path.join(empty_config_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME)) + + custome_name_config_dir_path = os.path.join(temp_dir, "custom-name") + edq.util.dirent.mkdir(custome_name_config_dir_path) + edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(custome_name_config_dir_path, "custom-edq-config.json")) + + edq.util.dirent.mkdir(os.path.join(temp_dir, "dir-config", "edq-config.json")) + edq.util.dirent.mkdir(os.path.join(temp_dir, "empty-dir")) + + global_config_dir_path = os.path.join(temp_dir, "global") + edq.util.dirent.mkdir(global_config_dir_path) + edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(global_config_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME)) + + old_name_config_dir_path = os.path.join(temp_dir, "old-name") + edq.util.dirent.mkdir(os.path.join(old_name_config_dir_path, "nest1", "nest2")) + edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(old_name_config_dir_path, "config.json")) + + nested_dir_path = os.path.join(temp_dir, "nested") + edq.util.dirent.mkdir(os.path.join(nested_dir_path, "nest1", "nest2a")) + edq.util.dirent.mkdir(os.path.join(nested_dir_path, "nest1", "nest2b")) + + edq.util.json.dump_path({"server": "http://test.edulinq.org"}, os.path.join(nested_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME)) + edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(nested_dir_path, "nest1", "nest2b","config.json")) + edq.util.json.dump_path( + {"user": "user@test.edulinq.org"}, + os.path.join(nested_dir_path, "nest1", "nest2b", edq.util.config.DEFAULT_CONFIG_FILENAME) + ) + + # Malformatted JSONs + simple_config_dir_path = os.path.join(temp_dir, "simple") + edq.util.dirent.mkdir(simple_config_dir_path) + edq.util.dirent.write_file( + os.path.join(simple_config_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME), + '{\n\t"user": "user@test.edulinq.org",\n}' + ) + + malformatted_config_dir_path = os.path.join(temp_dir, "malformatted") + edq.util.dirent.mkdir(malformatted_config_dir_path) + edq.util.dirent.write_file( + os.path.join(malformatted_config_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME), + "{\n\tuser: user@test.edulinq.org\n}" + ) + + return temp_dir + class TestConfig(edq.testing.unittest.BaseTest): """ Test basic operations on configs. """ @@ -16,7 +99,7 @@ def test_get_tiered_config_base(self): The placeholder 'TEMP_DIR' is overwritten during testing with the actual path to the directory. """ - # [(work directory, expected config, expected source, extra arguments), ...] + # [] test_cases = [ # No Config ( @@ -245,7 +328,7 @@ def test_get_tiered_config_base(self): None ), - # Overriding Keys + # Overwriting Keys ( "empty-dir", { @@ -310,15 +393,8 @@ def test_get_tiered_config_base(self): # Malformatted Config JSON ( "empty-dir", - { - "user": "user@test.edulinq.org" - }, - { - "user": edq.util.config.ConfigSource( - label = "", - path = os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME) - ), - }, + {}, + {}, { "cli_arguments": { edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME)], @@ -436,12 +512,12 @@ def test_get_tiered_config_base(self): { "user": edq.util.config.ConfigSource( label = "", - path = os.path.join("TEMP_DIR", "old-name", "config.json") + path = os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json") ), }, { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")] + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json")] }, }, None @@ -529,12 +605,12 @@ def test_get_tiered_config_base(self): { "user": edq.util.config.ConfigSource( label = "", - path = os.path.join("TEMP_DIR", "old-name", "config.json") + path = os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json") ), }, { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")], + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json")], }, "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, @@ -552,8 +628,8 @@ def test_get_tiered_config_base(self): }, { "cli_arguments": { - "user": "user@test.edulinq.org", - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")], + "user": "user@test.edulinq.org", + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json")], }, }, None @@ -561,7 +637,7 @@ def test_get_tiered_config_base(self): # Global Config + Local Config + CLI Provided Config + CLI Bare Options ( - os.path.join("nested", "nest1", "nest2b"), + "simple", { "user": "user@test.edulinq.org", "pass": "user", @@ -569,13 +645,13 @@ def test_get_tiered_config_base(self): { "user": edq.util.config.ConfigSource( label = "", - path = os.path.join("TEMP_DIR", "old-name", "config.json") + path = os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json") ), "pass": edq.util.config.ConfigSource(label = ""), }, { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "old-name", "config.json")], + edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json")], "pass": "user", "server": "http://test.edulinq.org", }, @@ -593,7 +669,7 @@ def test_get_tiered_config_base(self): (test_work_dir, expected_config, expected_source, extra_args, error_substring) = test_case with self.subTest(msg = f"Case {i} ('{test_work_dir}'):"): - temp_dir = edq.util.dirent.get_temp_dir(prefix = "edq-test-config-get-tiered-config-") + temp_dir = creat_test_dir(temp_dir_prefix = "edq-test-config-get-tiered-config-") cli_arguments = extra_args.get("cli_arguments", None) if (cli_arguments is not None): @@ -611,8 +687,6 @@ def test_get_tiered_config_base(self): _replace_placeholders_dict(extra_args, "TEMP_DIR", temp_dir) - edq.util.dirent.copy_contents(CONFIGS_DIR, temp_dir) - previous_work_directory = os.getcwd() initial_work_directory = os.path.join(temp_dir, test_work_dir) os.chdir(initial_work_directory) diff --git a/edq/util/testdata/configs/custom-name/custom-edq-config.json b/edq/util/testdata/configs/custom-name/custom-edq-config.json deleted file mode 100644 index 98b5beb..0000000 --- a/edq/util/testdata/configs/custom-name/custom-edq-config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "user": "user@test.edulinq.org" -} diff --git a/edq/util/testdata/configs/dir-config/edq-config.json/.gitignore b/edq/util/testdata/configs/dir-config/edq-config.json/.gitignore deleted file mode 100644 index d6b7ef3..0000000 --- a/edq/util/testdata/configs/dir-config/edq-config.json/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/edq/util/testdata/configs/empty-dir/.gitignore b/edq/util/testdata/configs/empty-dir/.gitignore deleted file mode 100644 index d6b7ef3..0000000 --- a/edq/util/testdata/configs/empty-dir/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/edq/util/testdata/configs/empty/edq-config.json b/edq/util/testdata/configs/empty/edq-config.json deleted file mode 100644 index 2c63c08..0000000 --- a/edq/util/testdata/configs/empty/edq-config.json +++ /dev/null @@ -1,2 +0,0 @@ -{ -} diff --git a/edq/util/testdata/configs/global/edq-config.json b/edq/util/testdata/configs/global/edq-config.json deleted file mode 100644 index 98b5beb..0000000 --- a/edq/util/testdata/configs/global/edq-config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "user": "user@test.edulinq.org" -} diff --git a/edq/util/testdata/configs/malformatted/edq-config.json b/edq/util/testdata/configs/malformatted/edq-config.json deleted file mode 100644 index bd5cacc..0000000 --- a/edq/util/testdata/configs/malformatted/edq-config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - user user@test.edulinq.org -} diff --git a/edq/util/testdata/configs/nested/edq-config.json b/edq/util/testdata/configs/nested/edq-config.json deleted file mode 100644 index d53f13d..0000000 --- a/edq/util/testdata/configs/nested/edq-config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "server": "http://test.edulinq.org" -} diff --git a/edq/util/testdata/configs/nested/nest1/nest2a/.gitignore b/edq/util/testdata/configs/nested/nest1/nest2a/.gitignore deleted file mode 100644 index d6b7ef3..0000000 --- a/edq/util/testdata/configs/nested/nest1/nest2a/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/edq/util/testdata/configs/nested/nest1/nest2b/config.json b/edq/util/testdata/configs/nested/nest1/nest2b/config.json deleted file mode 100644 index 0a5bcec..0000000 --- a/edq/util/testdata/configs/nested/nest1/nest2b/config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "pass": "user" -} diff --git a/edq/util/testdata/configs/nested/nest1/nest2b/edq-config.json b/edq/util/testdata/configs/nested/nest1/nest2b/edq-config.json deleted file mode 100644 index 98b5beb..0000000 --- a/edq/util/testdata/configs/nested/nest1/nest2b/edq-config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "user": "user@test.edulinq.org" -} diff --git a/edq/util/testdata/configs/old-name/config.json b/edq/util/testdata/configs/old-name/config.json deleted file mode 100644 index 98b5beb..0000000 --- a/edq/util/testdata/configs/old-name/config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "user": "user@test.edulinq.org" -} diff --git a/edq/util/testdata/configs/old-name/nest1/nest2/.gitignore b/edq/util/testdata/configs/old-name/nest1/nest2/.gitignore deleted file mode 100644 index d6b7ef3..0000000 --- a/edq/util/testdata/configs/old-name/nest1/nest2/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/edq/util/testdata/configs/simple/edq-config.json b/edq/util/testdata/configs/simple/edq-config.json deleted file mode 100644 index f619907..0000000 --- a/edq/util/testdata/configs/simple/edq-config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "user": "user@test.edulinq.org", -} From 05b7966cc31e892bbddfea12a55966389eea8711 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Thu, 21 Aug 2025 10:24:15 -0700 Subject: [PATCH 20/71] Removing files that need to be in the next PR. --- edq/cli/config/list.py | 48 ------------------------------------------ 1 file changed, 48 deletions(-) delete mode 100644 edq/cli/config/list.py diff --git a/edq/cli/config/list.py b/edq/cli/config/list.py deleted file mode 100644 index 2e999f1..0000000 --- a/edq/cli/config/list.py +++ /dev/null @@ -1,48 +0,0 @@ -import sys - -import edq.util.config - -DESCRIPTION = "List your current configuration options." - -def run(args): - (config, sources) = edq.util.config.get_tiered_config( - cli_arguments = args, - skip_keys = [ - 'show_origin', 'verbose', - edq.util.config.CONFIG_PATHS_KEY, 'global_config_path', - ], - global_config_path = args.global_config_path - ) - - config_list = [] - for (key, value) in config.items(): - config_str = f"{key}\t{value}" - if (args.show_origin): - raw_source = sources.get(key) - source_path = raw_source.split(edq.util.config.CONFIG_TYPE_DELIMITER)[1] - config_str += f"\t{source_path}" - - config_list.append(config_str) - - print("\n".join(config_list)) - return 0 - -def _get_parser(): - parser = edq.util.config.get_argument_parser( - description = DESCRIPTION, - skip_server = True) - - parser.add_argument("--show-origin", dest = 'show_origin', - action = 'store_true', help = "Shows where each configuration's value was obtained from.") - - parser.add_argument("--global-config", dest = 'global_config_path', - action = 'store', type = str, default = edq.util.config.DEFAULT_GLOBAL_CONFIG_PATH, - help = 'Path to the global configuration file (default: %(default)s).') - - return parser - -def main(): - return run(_get_parser().parse_args()) - -if (__name__ == '__main__'): - sys.exit(main()) From 24f240e5a6a3396f6aa453e6c8fc3ddfa8a2ea20 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Thu, 21 Aug 2025 10:58:09 -0700 Subject: [PATCH 21/71] Reviewed previous PR comments. --- edq/util/config.py | 3 +-- edq/util/config_test.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/edq/util/config.py b/edq/util/config.py index 5be8d15..736585a 100644 --- a/edq/util/config.py +++ b/edq/util/config.py @@ -117,8 +117,7 @@ def _get_local_config_path( If no config file is found, returns None. - The cutoff parameter limits the search depth, preventing detection of - config file in higher-level directories during testing. + The cutoff parameter limits the search depth, preventing detection of config file in higher-level directories during testing. """ # The case where provided config file is in current directory. diff --git a/edq/util/config_test.py b/edq/util/config_test.py index f6dcbf8..9490fd7 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -99,7 +99,7 @@ def test_get_tiered_config_base(self): The placeholder 'TEMP_DIR' is overwritten during testing with the actual path to the directory. """ - # [] + # [(work directory, expected config, expected source, extra arguments, error substring), ...] test_cases = [ # No Config ( From cdeee745c7e5e128b99ac9825e5db55ac4ff2cfe Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Thu, 21 Aug 2025 17:44:04 -0700 Subject: [PATCH 22/71] Revised the second pass, need to work on testing and README. --- edq/testing/unittest.py | 2 +- edq/util/config.py | 36 +++++++++++++++----------- edq/util/config_test.py | 57 +++++++++++++++++++++-------------------- edq/util/json.py | 3 ++- 4 files changed, 53 insertions(+), 45 deletions(-) diff --git a/edq/testing/unittest.py b/edq/testing/unittest.py index 61ed237..446c342 100644 --- a/edq/testing/unittest.py +++ b/edq/testing/unittest.py @@ -18,7 +18,7 @@ def assertJSONDictEqual(self, a: typing.Any, b: typing.Any, msg: typing.Union[st """ Like unittest.TestCase.assertDictEqual(), but will try to convert each comparison argument to a dict if it is not already, - and uses an default assertion message containing the full JSON representation of the arguments. + and uses a default assertion message containing the full JSON representation of the arguments. """ if (not isinstance(a, dict)): diff --git a/edq/util/config.py b/edq/util/config.py index 736585a..06de57c 100644 --- a/edq/util/config.py +++ b/edq/util/config.py @@ -7,24 +7,29 @@ import edq.util.dirent import edq.util.json +CONFIG_SOURCE_LOCAL: str = "" +CONFIG_SOURCE_GLOBAL: str = "" +CONFIG_SOURCE_CLI: str = "" +CONFIG_SOURCE_CLI_BARE: str = "" + CONFIG_PATHS_KEY: str = 'config_paths' DEFAULT_CONFIG_FILENAME = "edq-config.json" class ConfigSource: - """ A class for storing config source information in a structured way. """ + """ A class for storing config source information. """ - def __init__(self, label: str, path: typing.Union[str, None] = None): + def __init__(self, label: str, path: typing.Union[str, None] = None) -> None: self.label = label - """ Label of a config.""" + """ The label identifying the config (see CONFIG_SOURCE_* constants). """ self.path = path - """ Path of a config's source. """ + """ The path of where the config was soruced from. """ def __eq__(self, other: object) -> bool: if (not isinstance(other, ConfigSource)): return False - return ((self.label == other.label) and (self.path == other.path)) # type: ignore[attr-defined] + return ((self.label == other.label) and (self.path == other.path)) def __str__(self) -> str: return f"({self.label}, {self.path})" @@ -38,7 +43,7 @@ def get_tiered_config( local_config_root_cutoff: typing.Union[str, None] = None, ) -> typing.Tuple[typing.Dict[str, str], typing.Dict[str, ConfigSource]]: """ - Load all tiered configuration options from files and command-line arguments. + Load all configuration options from files and command-line arguments. Returns a configuration dictionary with the values based on tiering rules and a source dictionary mapping each key to its origin. """ @@ -60,23 +65,23 @@ def get_tiered_config( # Check the global user config file. if (os.path.isfile(global_config_path)): - _load_config_file(global_config_path, config, sources, "") + _load_config_file(global_config_path, config, sources, CONFIG_SOURCE_GLOBAL) # Check the local user config file. local_config_path = _get_local_config_path( config_file_name = config_file_name, legacy_config_file_name = legacy_config_file_name, - local_config_root_cutoff = local_config_root_cutoff + local_config_root_cutoff = local_config_root_cutoff, ) if (local_config_path is not None): - _load_config_file(local_config_path, config, sources, "") + _load_config_file(local_config_path, config, sources, CONFIG_SOURCE_LOCAL) # Check the config file specified on the command-line. config_paths = cli_arguments.get(CONFIG_PATHS_KEY, []) if (config_paths is not None): for path in config_paths: - _load_config_file(path, config, sources, "") + _load_config_file(path, config, sources, CONFIG_SOURCE_CLI) # Finally, any command-line options. for (key, value) in cli_arguments.items(): @@ -87,7 +92,7 @@ def get_tiered_config( continue config[key] = value - sources[key] = ConfigSource(label = "") + sources[key] = ConfigSource(label = CONFIG_SOURCE_CLI_BARE) return config, sources @@ -99,9 +104,10 @@ def _load_config_file( ) -> None: """ Loads config variables and the source from the given config JSON file. """ + config_path = os.path.abspath(config_path) for (key, value) in edq.util.json.load_path(config_path).items(): config[key] = value - sources[key] = ConfigSource(label = source_label, path = os.path.abspath(config_path)) + sources[key] = ConfigSource(label = source_label, path = config_path) def _get_local_config_path( config_file_name: str, @@ -120,16 +126,16 @@ def _get_local_config_path( The cutoff parameter limits the search depth, preventing detection of config file in higher-level directories during testing. """ - # The case where provided config file is in current directory. + # Provided config file is in current directory. if (os.path.isfile(config_file_name)): return os.path.abspath(config_file_name) - # The case where provided legacy config file is in current directory. + # Provided legacy config file is in current directory. if (legacy_config_file_name is not None): if (os.path.isfile(legacy_config_file_name)): return os.path.abspath(legacy_config_file_name) - # Case where the provided config file is found in an ancestor directory up to the root or cutoff limit. + # Provided config file is found in an ancestor directory up to the root or cutoff limit. parent_dir = os.path.dirname(os.getcwd()) return _get_ancestor_config_file_path( parent_dir, diff --git a/edq/util/config_test.py b/edq/util/config_test.py index 9490fd7..7fe8462 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -13,7 +13,6 @@ def creat_test_dir(temp_dir_prefix: str) -> str: Creat a temp dir and populate it with dirents for testing. This test data directory is laid out as: - . ├── custom-name | └── custom-edq-config.json @@ -78,14 +77,14 @@ def creat_test_dir(temp_dir_prefix: str) -> str: edq.util.dirent.mkdir(simple_config_dir_path) edq.util.dirent.write_file( os.path.join(simple_config_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME), - '{\n\t"user": "user@test.edulinq.org",\n}' + '{\n"user": "user@test.edulinq.org",\n}' ) malformatted_config_dir_path = os.path.join(temp_dir, "malformatted") edq.util.dirent.mkdir(malformatted_config_dir_path) edq.util.dirent.write_file( os.path.join(malformatted_config_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME), - "{\n\tuser: user@test.edulinq.org\n}" + "{\nuser: user@test.edulinq.org\n}" ) return temp_dir @@ -112,7 +111,7 @@ def test_get_tiered_config_base(self): # Global Config - # Custom global config path. + # Custom Global Config Path ( "empty-dir", { @@ -120,7 +119,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.util.config.ConfigSource( - label = "", + label = edq.util.config.CONFIG_SOURCE_GLOBAL, path = os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME) ), }, @@ -184,7 +183,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.util.config.ConfigSource( - label = "", + label = edq.util.config.CONFIG_SOURCE_LOCAL, path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ), }, @@ -200,7 +199,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.util.config.ConfigSource( - label = "", + label = edq.util.config.CONFIG_SOURCE_LOCAL, path = os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json") ), }, @@ -218,7 +217,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.util.config.ConfigSource( - label = "", + label = edq.util.config.CONFIG_SOURCE_LOCAL, path = os.path.join("TEMP_DIR", "old-name", "config.json") ), }, @@ -236,7 +235,7 @@ def test_get_tiered_config_base(self): }, { "server": edq.util.config.ConfigSource( - label = "", + label = edq.util.config.CONFIG_SOURCE_LOCAL, path = os.path.join("TEMP_DIR", "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) ), }, @@ -249,7 +248,9 @@ def test_get_tiered_config_base(self): os.path.join("old-name", "nest1", "nest2"), {}, {}, - {}, + { + "legacy_config_file_name": "config.json", + }, None ), @@ -288,7 +289,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.util.config.ConfigSource( - label = "", + label = edq.util.config.CONFIG_SOURCE_LOCAL, path = os.path.join("TEMP_DIR", "nested", "nest1", "nest2b", edq.util.config.DEFAULT_CONFIG_FILENAME) ) }, @@ -309,11 +310,11 @@ def test_get_tiered_config_base(self): }, { "user": edq.util.config.ConfigSource( - label = "", + label = edq.util.config.CONFIG_SOURCE_CLI, path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ), "server": edq.util.config.ConfigSource( - label = "", + label = edq.util.config.CONFIG_SOURCE_CLI, path = os.path.join("TEMP_DIR", "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) ), }, @@ -336,7 +337,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.util.config.ConfigSource( - label = "", + label = edq.util.config.CONFIG_SOURCE_CLI, path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ), }, @@ -412,7 +413,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = ""), + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, { "cli_arguments": { @@ -429,7 +430,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = ""), + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, { "cli_arguments": { @@ -453,7 +454,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.util.config.ConfigSource( - label = "", + label = edq.util.config.CONFIG_SOURCE_LOCAL, path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ), }, @@ -471,7 +472,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.util.config.ConfigSource( - label = "", + label = edq.util.config.CONFIG_SOURCE_CLI, path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ), }, @@ -491,7 +492,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = ""), + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, { "cli_arguments": { @@ -511,7 +512,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.util.config.ConfigSource( - label = "", + label = edq.util.config.CONFIG_SOURCE_CLI, path = os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json") ), }, @@ -530,7 +531,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = ""), + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, { "cli_arguments": { @@ -547,7 +548,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = ""), + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, { "cli_arguments": { @@ -566,7 +567,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = ""), + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, { "cli_arguments": { @@ -585,7 +586,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = ""), + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, { "cli_arguments": { @@ -604,7 +605,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.util.config.ConfigSource( - label = "", + label = edq.util.config.CONFIG_SOURCE_CLI, path = os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json") ), }, @@ -624,7 +625,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = ""), + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, { "cli_arguments": { @@ -644,10 +645,10 @@ def test_get_tiered_config_base(self): }, { "user": edq.util.config.ConfigSource( - label = "", + label = edq.util.config.CONFIG_SOURCE_CLI, path = os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json") ), - "pass": edq.util.config.ConfigSource(label = ""), + "pass": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, { "cli_arguments": { diff --git a/edq/util/json.py b/edq/util/json.py index 1d4a00d..4470b95 100644 --- a/edq/util/json.py +++ b/edq/util/json.py @@ -109,7 +109,8 @@ def load_path( """ if (os.path.isdir(path)): - raise IsADirectoryError(f"{path} is a directory, not a file") + raise IsADirectoryError(f"Cannot open JSON file, expected a file but got a directory at '{path}'.") + try: with open(path, 'r', encoding = encoding) as file: return load(file, strict = strict, **kwargs) From 67634eec39f7ccf710f438757d50df46859c9014 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Thu, 21 Aug 2025 23:14:07 -0700 Subject: [PATCH 23/71] Changed the order of the test structure. Made a single temp_dir for all test cases. --- edq/util/config_test.py | 415 +++++++++++++++++++--------------------- 1 file changed, 195 insertions(+), 220 deletions(-) diff --git a/edq/util/config_test.py b/edq/util/config_test.py index 7fe8462..7d63892 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -23,7 +23,7 @@ def creat_test_dir(temp_dir_prefix: str) -> str: ├── empty-dir ├── global │   └── edq-config.json - ├── malformatted + ├── malformed │   └── edq-config.json ├── nested │   ├── edq-config.json @@ -72,7 +72,7 @@ def creat_test_dir(temp_dir_prefix: str) -> str: os.path.join(nested_dir_path, "nest1", "nest2b", edq.util.config.DEFAULT_CONFIG_FILENAME) ) - # Malformatted JSONs + # Malformed JSONs simple_config_dir_path = os.path.join(temp_dir, "simple") edq.util.dirent.mkdir(simple_config_dir_path) edq.util.dirent.write_file( @@ -80,10 +80,10 @@ def creat_test_dir(temp_dir_prefix: str) -> str: '{\n"user": "user@test.edulinq.org",\n}' ) - malformatted_config_dir_path = os.path.join(temp_dir, "malformatted") - edq.util.dirent.mkdir(malformatted_config_dir_path) + malformed_config_dir_path = os.path.join(temp_dir, "malformed") + edq.util.dirent.mkdir(malformed_config_dir_path) edq.util.dirent.write_file( - os.path.join(malformatted_config_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME), + os.path.join(malformed_config_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME), "{\nuser: user@test.edulinq.org\n}" ) @@ -98,6 +98,8 @@ def test_get_tiered_config_base(self): The placeholder 'TEMP_DIR' is overwritten during testing with the actual path to the directory. """ + temp_dir = creat_test_dir(temp_dir_prefix = "edq-test-config-get-tiered-config-") + # [(work directory, expected config, expected source, extra arguments, error substring), ...] test_cases = [ # No Config @@ -114,63 +116,63 @@ def test_get_tiered_config_base(self): # Custom Global Config Path ( "empty-dir", + { + "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + }, { "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_GLOBAL, - path = os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME) + path = os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), ), }, - { - "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), - }, - None + None, ), # Empty Config JSON ( "empty-dir", - {}, - {}, { - "global_config_path": os.path.join("TEMP_DIR", "empty", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join(temp_dir, "empty", edq.util.config.DEFAULT_CONFIG_FILENAME), }, - None + {}, + {}, + None, ), # Directory Config JSON ( "empty-dir", - {}, - {}, { - "global_config_path": os.path.join("TEMP_DIR", "dir-config", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join(temp_dir, "dir-config", edq.util.config.DEFAULT_CONFIG_FILENAME), }, - None + {}, + {}, + None, ), # Non-Existent Config JSON ( "empty-dir", - {}, - {}, { - "global_config_path": os.path.join("TEMP_DIR", "empty-dir", "non-existent-config.json"), + "global_config_path": os.path.join(temp_dir, "empty-dir", "non-existent-config.json"), }, - None + {}, + {}, + None, ), - # Malformatted Config JSON + # Malformed Config JSON ( "empty-dir", - {}, - {}, { - "global_config_path": os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join(temp_dir, "malformed", edq.util.config.DEFAULT_CONFIG_FILENAME), }, - "Failed to read JSON file" + {}, + {}, + "Failed to read JSON file", ), # Local Config @@ -178,79 +180,79 @@ def test_get_tiered_config_base(self): # Default config file in current directory. ( "simple", + {}, { "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_LOCAL, - path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) + path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), ), }, - {}, - None + None, ), # Custom config file in current directory. ( "custom-name", + { + "config_file_name": "custom-edq-config.json", + }, { "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_LOCAL, - path = os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json") + path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ), }, - { - "config_file_name": "custom-edq-config.json", - }, - None + None, ), # Legacy config file in current directory. ( "old-name", + { + "legacy_config_file_name": "config.json", + }, { "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_LOCAL, - path = os.path.join("TEMP_DIR", "old-name", "config.json") + path = os.path.join(temp_dir, "old-name", "config.json"), ), }, - { - "legacy_config_file_name": "config.json", - }, - None + None, ), # Default config file in an ancestor directory. ( os.path.join("nested", "nest1", "nest2a"), + {}, { "server": "http://test.edulinq.org", }, { "server": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_LOCAL, - path = os.path.join("TEMP_DIR", "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) + path = os.path.join(temp_dir, "nested", edq.util.config.DEFAULT_CONFIG_FILENAME), ), }, - {}, - None + None, ), # Legacy config file in an ancestor directory. ( os.path.join("old-name", "nest1", "nest2"), - {}, - {}, { "legacy_config_file_name": "config.json", }, + {}, + {}, None ), @@ -260,7 +262,7 @@ def test_get_tiered_config_base(self): {}, {}, {}, - None + None, ), # Directory Config JSON @@ -269,41 +271,49 @@ def test_get_tiered_config_base(self): {}, {}, {}, - None + None, ), - # Malformatted Config JSON + # Malformed Config JSON ( - "malformatted", + "malformed", {}, {}, {}, - "Failed to read JSON file" + "Failed to read JSON file", ), # All 3 local config locations present at the same time. ( os.path.join("nested", "nest1", "nest2b"), + { + "legacy_config_file_name": "config.json", + }, { "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_LOCAL, - path = os.path.join("TEMP_DIR", "nested", "nest1", "nest2b", edq.util.config.DEFAULT_CONFIG_FILENAME) + path = os.path.join(temp_dir, "nested", "nest1", "nest2b", edq.util.config.DEFAULT_CONFIG_FILENAME) ) }, - { - "legacy_config_file_name": "config.json", - }, - None + None, ), - # CLI Provided Config + ## CLI Provided Config # Distinct Keys ( "empty-dir", + { + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), + os.path.join(temp_dir, "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) + ], + }, + }, { "user": "user@test.edulinq.org", "server": "http://test.edulinq.org", @@ -311,97 +321,89 @@ def test_get_tiered_config_base(self): { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_CLI, - path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) + path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ), "server": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_CLI, - path = os.path.join("TEMP_DIR", "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) + path = os.path.join(temp_dir, "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) ), }, + None, + ), + + # Overwriting Keys + ( + "empty-dir", { "cli_arguments": { edq.util.config.CONFIG_PATHS_KEY: [ - os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), - os.path.join("TEMP_DIR", "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) + os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), + os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), ], }, }, - None - ), - - # Overwriting Keys - ( - "empty-dir", { "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_CLI, - path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) + path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ), }, - { - "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [ - os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json"), - os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) - ], - }, - }, - None + None, ), # Empty Config JSON ( "empty-dir", - {}, - {}, { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "empty", edq.util.config.DEFAULT_CONFIG_FILENAME)], + edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "empty", edq.util.config.DEFAULT_CONFIG_FILENAME)], }, }, + {}, + {}, None ), # Directory Config JSON ( "empty-dir", - {}, - {}, { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "dir-config", edq.util.config.DEFAULT_CONFIG_FILENAME)], + edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "dir-config", edq.util.config.DEFAULT_CONFIG_FILENAME)], }, }, + {}, + {}, "IsADirectoryError" ), # Non-Existent Config JSON ( "empty-dir", - {}, - {}, { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "empty-dir", "non-existent-config.json")], + edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "empty-dir", "non-existent-config.json")], }, }, - "FileNotFoundError" + {}, + {}, + "FileNotFoundError", ), - # Malformatted Config JSON + # Malformed Config JSON ( "empty-dir", - {}, - {}, { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "malformatted", edq.util.config.DEFAULT_CONFIG_FILENAME)], + edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "malformed", edq.util.config.DEFAULT_CONFIG_FILENAME)], }, }, - "Failed to read JSON file" + {}, + {}, + "Failed to read JSON file", ), # CLI Bare Options: @@ -410,38 +412,38 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "user": "user@test.edulinq.org", + "cli_arguments": { + "user": "user@test.edulinq.org", + }, }, { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "user": "user@test.edulinq.org", }, { - "cli_arguments": { - "user": "user@test.edulinq.org" - }, + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, - None + None, ), # Skip keys functionally. ( "empty-dir", - { - "user": "user@test.edulinq.org", - }, - { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), - }, { "cli_arguments": { "user": "user@test.edulinq.org", - "pass": "user" + "pass": "user", }, "skip_keys": [ - "pass" + "pass", ], }, - None + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + }, + None, ), # Combinations @@ -449,210 +451,197 @@ def test_get_tiered_config_base(self): # Global Config + Local Config ( "simple", + { + "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + }, { "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_LOCAL, - path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) + path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) ), }, - { - "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), - }, - None + None, ), # Global Config + CLI Provided Config ( "empty-dir", + { + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)], + }, + "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + }, { "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_CLI, - path = os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) + path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), ), }, - { - "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)] - }, - "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), - }, - None + None, ), # Global + CLI Bare Options ( "empty-dir", { - "user": "user@test.edulinq.org", + "cli_arguments": { + "user": "user@test.edulinq.org", + }, + "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "user": "user@test.edulinq.org", }, { - "cli_arguments": { - "user": "user@test.edulinq.org", - }, - "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), - + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, - None + None, ), # Local Config + CLI Provided Config ( "simple", + { + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "custom-name", "custom-edq-config.json")], + }, + }, { "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_CLI, - path = os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json") + path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ), }, - { - "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json")] - }, - }, - None + None, ), # Local Config + CLI Bare Options ( "simple", { - "user": "user@test.edulinq.org", + "cli_arguments": { + "user": "user@test.edulinq.org", + }, }, { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "user": "user@test.edulinq.org", }, { - "cli_arguments": { - "user": "user@test.edulinq.org" - }, + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, - None + None, ), # CLI Provided Config + CLI Bare Options ( "empty-dir", { - "user": "user@test.edulinq.org", + "cli_arguments": { + "user": "user@test.edulinq.org", + edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)], + }, }, { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "user": "user@test.edulinq.org", }, { - "cli_arguments": { - "user": "user@test.edulinq.org", - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)] - }, + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, - None + None, ), # Global Config + CLI Provided Config + CLI Bare Options - ( "empty-dir", { - "user": "user@test.edulinq.org", + "cli_arguments": { + "user": "user@test.edulinq.org", + edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)], + }, + "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "user": "user@test.edulinq.org", }, { - "cli_arguments": { - "user": "user@test.edulinq.org", - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)] - }, - "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, - None + None, ), # Global Config + Local Config + CLI Bare Options ( "simple", { - "user": "user@test.edulinq.org", + "cli_arguments": { + "user": "user@test.edulinq.org", + }, + "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "user": "user@test.edulinq.org", }, { - "cli_arguments": { - "user": "user@test.edulinq.org", - }, - "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, - None + None, ), # Global Config + Local Config + CLI Provided Config ( "simple", + { + "cli_arguments": { + edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "custom-name", "custom-edq-config.json")], + }, + "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + }, { "user": "user@test.edulinq.org", }, { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_CLI, - path = os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json") + path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json") ), }, - { - "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json")], - }, - "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), - }, - None + None, ), # Local Config + CLI Provided Config + CLI Bare Options ( "simple", { - "user": "user@test.edulinq.org", + "cli_arguments": { + "user": "user@test.edulinq.org", + edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "custom-name", "custom-edq-config.json")], + }, }, { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "user": "user@test.edulinq.org", }, { - "cli_arguments": { - "user": "user@test.edulinq.org", - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json")], - }, + "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), }, - None + None, ), # Global Config + Local Config + CLI Provided Config + CLI Bare Options ( "simple", - { - "user": "user@test.edulinq.org", - "pass": "user", - }, - { - "user": edq.util.config.ConfigSource( - label = edq.util.config.CONFIG_SOURCE_CLI, - path = os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json") - ), - "pass": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), - }, { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join("TEMP_DIR", "custom-name", "custom-edq-config.json")], + edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "custom-name", "custom-edq-config.json")], "pass": "user", "server": "http://test.edulinq.org", }, @@ -660,33 +649,34 @@ def test_get_tiered_config_base(self): "server", edq.util.config.CONFIG_PATHS_KEY, ], - "global_config_path": os.path.join("TEMP_DIR", "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, - None + { + "user": "user@test.edulinq.org", + "pass": "user", + }, + { + "user": edq.util.config.ConfigSource( + label = edq.util.config.CONFIG_SOURCE_CLI, + path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), + ), + "pass": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + }, + None, ), ] for (i, test_case) in enumerate(test_cases): - (test_work_dir, expected_config, expected_source, extra_args, error_substring) = test_case + (test_work_dir, extra_args, expected_config, expected_source, error_substring) = test_case with self.subTest(msg = f"Case {i} ('{test_work_dir}'):"): - temp_dir = creat_test_dir(temp_dir_prefix = "edq-test-config-get-tiered-config-") - - cli_arguments = extra_args.get("cli_arguments", None) - if (cli_arguments is not None): - config_paths = cli_arguments.get(edq.util.config.CONFIG_PATHS_KEY, None) - if (config_paths is not None): - _replace_placeholders_list(config_paths, "TEMP_DIR", temp_dir) - cutoff = extra_args.get("local_config_root_cutoff", None) if (cutoff is None): - extra_args["local_config_root_cutoff"] = "TEMP_DIR" + extra_args["local_config_root_cutoff"] = temp_dir global_config = extra_args.get("global_config_path", None) if (global_config is None): - extra_args["global_config_path"] = os.path.join("TEMP_DIR", "empty", edq.util.config.CONFIG_PATHS_KEY) - - _replace_placeholders_dict(extra_args, "TEMP_DIR", temp_dir) + extra_args["global_config_path"] = os.path.join(temp_dir, "empty", edq.util.config.CONFIG_PATHS_KEY) previous_work_directory = os.getcwd() initial_work_directory = os.path.join(temp_dir, test_work_dir) @@ -710,20 +700,5 @@ def test_get_tiered_config_base(self): if (error_substring is not None): self.fail(f"Did not get expected error: '{error_substring}'.") - for (key, value) in actual_sources.items(): - if (value.path is not None): - value.path = value.path.replace(temp_dir, "TEMP_DIR") - actual_sources[key] = value - self.assertJSONDictEqual(expected_config, actual_config) self.assertJSONDictEqual(expected_source, actual_sources) - -def _replace_placeholders_dict(data_dict, old, new): - for (key, value) in data_dict.items(): - if (isinstance(value, str)): - if (old in value): - data_dict[key] = value.replace(old, new) - -def _replace_placeholders_list(data_list, old, new): - for (i, path) in enumerate(data_list): - data_list[i] = path.replace(old, new) From f2ef098048f0e25fc5adbeae9b5cd45f1500b704 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 22 Aug 2025 08:06:51 -0700 Subject: [PATCH 24/71] Added config on README --- README.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/README.md b/README.md index 2062633..2dd0181 100644 --- a/README.md +++ b/README.md @@ -16,3 +16,42 @@ The project and Python dependencies can be installed from source with: ``` pip3 install . ``` + +## Configuration + +Many EduLinq tools share a common configuration system. +This system provides a consistent way to supply options to the CLI tool being used. +While each CLI tool may require additional options, the configuration loading process is the same across the EduLinq ecosystem. + +By default, the config file is named `edq-config.json`. +This is customizable and may differ depending on the tool being used. + +### Configuration Sources + +Configuration options can come from several places, with later sources overriding earlier ones: + +#### Global Configuration +- **Path:** `/edq-config.json` +- This is the "proper" place to store user-related configuration, according to [platformdirs](https://github.com/tox-dev/platformdirs). +- Best suited for login credentials or persistent user preferences. +- Run any CLI tool with `--help` to see the exact path on your platform. + +#### Local Configuration +If an `edq-config.json` exists in the current working directory, it will be loaded. + +Local configuration files can be found in different locations. +The first file found will be used, and other locations will not be searched. + +**Search order for a local config file:** +1. `./edq-config.json` +2. `./legacy-config.json` + *(applies only if a legacy file name, such as `legacy-config.json`, is explicitly passed when using the `get-tiered-config` utility)* +3. An `edq-config.json` file located in any ancestor directory on the path to root (or up to a cutoff limit if one is specified). + +#### CLI-Specified Config Files +Any files passed via `--config` will be loaded in the order they appear on the command line. +Latter files will override options from previous ones. + +#### Bare CLI Options +Options passed directly on the command line (e.g., `--user`, `--token`, `--server`). +These always override every other configuration source. From 707a3515e31b429067ae3eb9b0f6d7a2a71f1a85 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 22 Aug 2025 08:09:29 -0700 Subject: [PATCH 25/71] Got rid of points for Global config. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2dd0181..28b61ee 100644 --- a/README.md +++ b/README.md @@ -31,10 +31,10 @@ This is customizable and may differ depending on the tool being used. Configuration options can come from several places, with later sources overriding earlier ones: #### Global Configuration -- **Path:** `/edq-config.json` -- This is the "proper" place to store user-related configuration, according to [platformdirs](https://github.com/tox-dev/platformdirs). -- Best suited for login credentials or persistent user preferences. -- Run any CLI tool with `--help` to see the exact path on your platform. +**Path:** `/edq-config.json` +This is the "proper" place to store user-related configuration, according to [platformdirs](https://github.com/tox-dev/platformdirs). +Best suited for login credentials or persistent user preferences. +Run any CLI tool with `--help` to see the exact path on your platform. #### Local Configuration If an `edq-config.json` exists in the current working directory, it will be loaded. From 683140b617005c0b45073723fe14478d7c3a56f9 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 22 Aug 2025 14:26:01 -0700 Subject: [PATCH 26/71] Polished of the README. --- README.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 28b61ee..7e4bd82 100644 --- a/README.md +++ b/README.md @@ -26,32 +26,40 @@ While each CLI tool may require additional options, the configuration loading pr By default, the config file is named `edq-config.json`. This is customizable and may differ depending on the tool being used. +For the purposes of this documentation we are going to use the default config file name. + ### Configuration Sources -Configuration options can come from several places, with later sources overriding earlier ones: +Configuration options can come from several places, with later sources overwriting earlier ones: #### Global Configuration -**Path:** `/edq-config.json` -This is the "proper" place to store user-related configuration, according to [platformdirs](https://github.com/tox-dev/platformdirs). -Best suited for login credentials or persistent user preferences. + +The default place a global config is looked for is `/edq-config.json`. +This is considered to be the "proper" place to store user-related configuration, according to [platformdirs](https://github.com/tox-dev/platformdi). +The location where a global config will be looked for can be changed by passing a path to --global-config trough the command line. +This type of config is best suited for login credentials or persistent user preferences. Run any CLI tool with `--help` to see the exact path on your platform. #### Local Configuration + If an `edq-config.json` exists in the current working directory, it will be loaded. Local configuration files can be found in different locations. The first file found will be used, and other locations will not be searched. -**Search order for a local config file:** +##### Search order for a local config file: + 1. `./edq-config.json` 2. `./legacy-config.json` - *(applies only if a legacy file name, such as `legacy-config.json`, is explicitly passed when using the `get-tiered-config` utility)* + *(applies only if the config system is set to support a custome legacy file name)* 3. An `edq-config.json` file located in any ancestor directory on the path to root (or up to a cutoff limit if one is specified). #### CLI-Specified Config Files + Any files passed via `--config` will be loaded in the order they appear on the command line. Latter files will override options from previous ones. #### Bare CLI Options + Options passed directly on the command line (e.g., `--user`, `--token`, `--server`). These always override every other configuration source. From 97d16d9b956f35b439712ca0d06bfbd98b4ccabc Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 22 Aug 2025 14:29:04 -0700 Subject: [PATCH 27/71] Fixed the url for platformdir. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7e4bd82..c2b1d49 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Configuration options can come from several places, with later sources overwriti #### Global Configuration The default place a global config is looked for is `/edq-config.json`. -This is considered to be the "proper" place to store user-related configuration, according to [platformdirs](https://github.com/tox-dev/platformdi). +This is considered to be the "proper" place to store user-related configuration, according to [platformdirs](https://github.com/tox-dev/platformdir). The location where a global config will be looked for can be changed by passing a path to --global-config trough the command line. This type of config is best suited for login credentials or persistent user preferences. Run any CLI tool with `--help` to see the exact path on your platform. From cf20ac66422f9946a74055ba01261251342d73d9 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 22 Aug 2025 14:30:48 -0700 Subject: [PATCH 28/71] Fixed the url for platformdir. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c2b1d49..055a0cf 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Configuration options can come from several places, with later sources overwriti #### Global Configuration The default place a global config is looked for is `/edq-config.json`. -This is considered to be the "proper" place to store user-related configuration, according to [platformdirs](https://github.com/tox-dev/platformdir). +This is considered to be the "proper" place to store user-related configuration, according to [platformdirs](https://github.com/tox-dev/platformdirs). The location where a global config will be looked for can be changed by passing a path to --global-config trough the command line. This type of config is best suited for login credentials or persistent user preferences. Run any CLI tool with `--help` to see the exact path on your platform. From 5edb54a97b5b48560dd7dfe31a5db45e82a018b4 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 22 Aug 2025 14:46:00 -0700 Subject: [PATCH 29/71] Made the local config description better. --- README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 055a0cf..1a7822f 100644 --- a/README.md +++ b/README.md @@ -36,22 +36,19 @@ Configuration options can come from several places, with later sources overwriti The default place a global config is looked for is `/edq-config.json`. This is considered to be the "proper" place to store user-related configuration, according to [platformdirs](https://github.com/tox-dev/platformdirs). -The location where a global config will be looked for can be changed by passing a path to --global-config trough the command line. +The location where a global config will be looked for can be changed by passing a path to `--global-config` trough the command line. This type of config is best suited for login credentials or persistent user preferences. Run any CLI tool with `--help` to see the exact path on your platform. #### Local Configuration -If an `edq-config.json` exists in the current working directory, it will be loaded. - Local configuration files can be found in different locations. The first file found will be used, and other locations will not be searched. ##### Search order for a local config file: -1. `./edq-config.json` -2. `./legacy-config.json` - *(applies only if the config system is set to support a custome legacy file name)* +1. An `edq-config.json` in the current directory. +2. If the config system is set to support a custom legacy file, it will look for the custom file in the current directory. 3. An `edq-config.json` file located in any ancestor directory on the path to root (or up to a cutoff limit if one is specified). #### CLI-Specified Config Files From aed3ffb695160fd77a151c1af34bc0fe24ccd05b Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 22 Aug 2025 14:49:48 -0700 Subject: [PATCH 30/71] Made the local config description better. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1a7822f..b8ed36a 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ The first file found will be used, and other locations will not be searched. ##### Search order for a local config file: 1. An `edq-config.json` in the current directory. -2. If the config system is set to support a custom legacy file, it will look for the custom file in the current directory. +2. A custom legacy file in the current directory. If the config system is set to support a custom legacy file. 3. An `edq-config.json` file located in any ancestor directory on the path to root (or up to a cutoff limit if one is specified). #### CLI-Specified Config Files From 7c905b523357554ea097a7cc3202f8a440a67cbc Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sun, 24 Aug 2025 08:42:18 -0700 Subject: [PATCH 31/71] Revised for 3rd pass --- README.md | 10 ++-- edq/util/config_test.py | 107 ++++++++++++++++++++++++---------------- 2 files changed, 69 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index b8ed36a..f92e85e 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ pip3 install . ## Configuration Many EduLinq tools share a common configuration system. -This system provides a consistent way to supply options to the CLI tool being used. +This system provides a consistent way to supply configuration options to the CLI tool being used. While each CLI tool may require additional options, the configuration loading process is the same across the EduLinq ecosystem. By default, the config file is named `edq-config.json`. @@ -30,13 +30,13 @@ For the purposes of this documentation we are going to use the default config fi ### Configuration Sources -Configuration options can come from several places, with later sources overwriting earlier ones: +Configuration options can come from several places, with later sources overwriting earlier ones. #### Global Configuration The default place a global config is looked for is `/edq-config.json`. This is considered to be the "proper" place to store user-related configuration, according to [platformdirs](https://github.com/tox-dev/platformdirs). -The location where a global config will be looked for can be changed by passing a path to `--global-config` trough the command line. +The location where a global config will be looked for can be changed by passing a path to `--global-config` through the command line. This type of config is best suited for login credentials or persistent user preferences. Run any CLI tool with `--help` to see the exact path on your platform. @@ -48,13 +48,13 @@ The first file found will be used, and other locations will not be searched. ##### Search order for a local config file: 1. An `edq-config.json` in the current directory. -2. A custom legacy file in the current directory. If the config system is set to support a custom legacy file. +2. A legacy file in the current directory. If the config system is set to support a specified legacy file. 3. An `edq-config.json` file located in any ancestor directory on the path to root (or up to a cutoff limit if one is specified). #### CLI-Specified Config Files Any files passed via `--config` will be loaded in the order they appear on the command line. -Latter files will override options from previous ones. +Later files will override options from previous ones. #### Bare CLI Options diff --git a/edq/util/config_test.py b/edq/util/config_test.py index 7d63892..6508d35 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -15,7 +15,7 @@ def creat_test_dir(temp_dir_prefix: str) -> str: This test data directory is laid out as: . ├── custom-name - | └── custom-edq-config.json + │   └── custom-edq-config.json ├── dir-config │   └── edq-config.json ├── empty @@ -26,11 +26,11 @@ def creat_test_dir(temp_dir_prefix: str) -> str: ├── malformed │   └── edq-config.json ├── nested + │   ├── config.json │   ├── edq-config.json │   └── nest1 │   ├── nest2a │   └── nest2b - │   ├── config.json │   └── edq-config.json ├── old-name │   ├── config.json @@ -66,10 +66,10 @@ def creat_test_dir(temp_dir_prefix: str) -> str: edq.util.dirent.mkdir(os.path.join(nested_dir_path, "nest1", "nest2b")) edq.util.json.dump_path({"server": "http://test.edulinq.org"}, os.path.join(nested_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME)) - edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(nested_dir_path, "nest1", "nest2b","config.json")) + edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(nested_dir_path, "config.json")) edq.util.json.dump_path( {"user": "user@test.edulinq.org"}, - os.path.join(nested_dir_path, "nest1", "nest2b", edq.util.config.DEFAULT_CONFIG_FILENAME) + os.path.join(nested_dir_path, "nest1", "nest2b", edq.util.config.DEFAULT_CONFIG_FILENAME), ) # Malformed JSONs @@ -77,14 +77,14 @@ def creat_test_dir(temp_dir_prefix: str) -> str: edq.util.dirent.mkdir(simple_config_dir_path) edq.util.dirent.write_file( os.path.join(simple_config_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME), - '{\n"user": "user@test.edulinq.org",\n}' + '{"user": "user@test.edulinq.org",}', ) malformed_config_dir_path = os.path.join(temp_dir, "malformed") edq.util.dirent.mkdir(malformed_config_dir_path) edq.util.dirent.write_file( os.path.join(malformed_config_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME), - "{\nuser: user@test.edulinq.org\n}" + "{user: user@test.edulinq.org}", ) return temp_dir @@ -95,12 +95,11 @@ class TestConfig(edq.testing.unittest.BaseTest): def test_get_tiered_config_base(self): """ Test that configuration files are loaded correctly from the file system with the expected tier. - The placeholder 'TEMP_DIR' is overwritten during testing with the actual path to the directory. """ temp_dir = creat_test_dir(temp_dir_prefix = "edq-test-config-get-tiered-config-") - # [(work directory, expected config, expected source, extra arguments, error substring), ...] + # [(work directory, extra arguments, expected config, expected source, error substring), ...] test_cases = [ # No Config ( @@ -108,7 +107,7 @@ def test_get_tiered_config_base(self): {}, {}, {}, - None + None, ), # Global Config @@ -253,7 +252,7 @@ def test_get_tiered_config_base(self): }, {}, {}, - None + None, ), # Empty Config JSON @@ -295,13 +294,13 @@ def test_get_tiered_config_base(self): { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_LOCAL, - path = os.path.join(temp_dir, "nested", "nest1", "nest2b", edq.util.config.DEFAULT_CONFIG_FILENAME) - ) + path = os.path.join(temp_dir, "nested", "nest1", "nest2b", edq.util.config.DEFAULT_CONFIG_FILENAME), + ), }, None, ), - ## CLI Provided Config + # CLI Provided Config # Distinct Keys ( @@ -310,7 +309,7 @@ def test_get_tiered_config_base(self): "cli_arguments": { edq.util.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), - os.path.join(temp_dir, "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) + os.path.join(temp_dir, "nested", edq.util.config.DEFAULT_CONFIG_FILENAME), ], }, }, @@ -321,11 +320,11 @@ def test_get_tiered_config_base(self): { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_CLI, - path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) + path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), ), "server": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_CLI, - path = os.path.join(temp_dir, "nested", edq.util.config.DEFAULT_CONFIG_FILENAME) + path = os.path.join(temp_dir, "nested", edq.util.config.DEFAULT_CONFIG_FILENAME), ), }, None, @@ -348,7 +347,7 @@ def test_get_tiered_config_base(self): { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_CLI, - path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) + path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), ), }, None, @@ -359,12 +358,14 @@ def test_get_tiered_config_base(self): "empty-dir", { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "empty", edq.util.config.DEFAULT_CONFIG_FILENAME)], + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "empty", edq.util.config.DEFAULT_CONFIG_FILENAME), + ], }, }, {}, {}, - None + None, ), # Directory Config JSON @@ -372,12 +373,14 @@ def test_get_tiered_config_base(self): "empty-dir", { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "dir-config", edq.util.config.DEFAULT_CONFIG_FILENAME)], + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "dir-config", edq.util.config.DEFAULT_CONFIG_FILENAME), + ], }, }, {}, {}, - "IsADirectoryError" + "IsADirectoryError", ), # Non-Existent Config JSON @@ -385,7 +388,9 @@ def test_get_tiered_config_base(self): "empty-dir", { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "empty-dir", "non-existent-config.json")], + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "empty-dir", "non-existent-config.json"), + ], }, }, {}, @@ -398,7 +403,9 @@ def test_get_tiered_config_base(self): "empty-dir", { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "malformed", edq.util.config.DEFAULT_CONFIG_FILENAME)], + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "malformed", edq.util.config.DEFAULT_CONFIG_FILENAME), + ], }, }, {}, @@ -460,7 +467,7 @@ def test_get_tiered_config_base(self): { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_LOCAL, - path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME) + path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), ), }, None, @@ -470,10 +477,12 @@ def test_get_tiered_config_base(self): ( "empty-dir", { + "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)], + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), + ], }, - "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, { "user": "user@test.edulinq.org", @@ -491,10 +500,10 @@ def test_get_tiered_config_base(self): ( "empty-dir", { + "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { "user": "user@test.edulinq.org", }, - "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, { "user": "user@test.edulinq.org", @@ -510,7 +519,9 @@ def test_get_tiered_config_base(self): "simple", { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "custom-name", "custom-edq-config.json")], + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), + ], }, }, { @@ -548,7 +559,9 @@ def test_get_tiered_config_base(self): { "cli_arguments": { "user": "user@test.edulinq.org", - edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)], + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), + ], }, }, { @@ -564,11 +577,13 @@ def test_get_tiered_config_base(self): ( "empty-dir", { + "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { - "user": "user@test.edulinq.org", - edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME)], + "user": "user@test.edulinq.org", + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), + ], }, - "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, { "user": "user@test.edulinq.org", @@ -583,10 +598,10 @@ def test_get_tiered_config_base(self): ( "simple", { + "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { "user": "user@test.edulinq.org", }, - "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, { "user": "user@test.edulinq.org", @@ -601,10 +616,12 @@ def test_get_tiered_config_base(self): ( "simple", { + "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "custom-name", "custom-edq-config.json")], + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), + ], }, - "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, { "user": "user@test.edulinq.org", @@ -612,7 +629,7 @@ def test_get_tiered_config_base(self): { "user": edq.util.config.ConfigSource( label = edq.util.config.CONFIG_SOURCE_CLI, - path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json") + path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ), }, None, @@ -624,7 +641,9 @@ def test_get_tiered_config_base(self): { "cli_arguments": { "user": "user@test.edulinq.org", - edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "custom-name", "custom-edq-config.json")], + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), + ], }, }, { @@ -640,8 +659,11 @@ def test_get_tiered_config_base(self): ( "simple", { + "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [os.path.join(temp_dir, "custom-name", "custom-edq-config.json")], + edq.util.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), + ], "pass": "user", "server": "http://test.edulinq.org", }, @@ -649,7 +671,6 @@ def test_get_tiered_config_base(self): "server", edq.util.config.CONFIG_PATHS_KEY, ], - "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), }, { "user": "user@test.edulinq.org", @@ -670,14 +691,14 @@ def test_get_tiered_config_base(self): (test_work_dir, extra_args, expected_config, expected_source, error_substring) = test_case with self.subTest(msg = f"Case {i} ('{test_work_dir}'):"): - cutoff = extra_args.get("local_config_root_cutoff", None) - if (cutoff is None): - extra_args["local_config_root_cutoff"] = temp_dir - global_config = extra_args.get("global_config_path", None) if (global_config is None): extra_args["global_config_path"] = os.path.join(temp_dir, "empty", edq.util.config.CONFIG_PATHS_KEY) + cutoff = extra_args.get("local_config_root_cutoff", None) + if (cutoff is None): + extra_args["local_config_root_cutoff"] = temp_dir + previous_work_directory = os.getcwd() initial_work_directory = os.path.join(temp_dir, test_work_dir) os.chdir(initial_work_directory) From 078eaee8e5293ff91f3bbc39257173751a9abe4f Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sun, 24 Aug 2025 10:53:11 -0700 Subject: [PATCH 32/71] Corrected inconsistencies. --- edq/util/config.py | 12 ++++++------ edq/util/config_test.py | 3 --- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/edq/util/config.py b/edq/util/config.py index 06de57c..0881165 100644 --- a/edq/util/config.py +++ b/edq/util/config.py @@ -7,13 +7,13 @@ import edq.util.dirent import edq.util.json -CONFIG_SOURCE_LOCAL: str = "" CONFIG_SOURCE_GLOBAL: str = "" +CONFIG_SOURCE_LOCAL: str = "" CONFIG_SOURCE_CLI: str = "" CONFIG_SOURCE_CLI_BARE: str = "" CONFIG_PATHS_KEY: str = 'config_paths' -DEFAULT_CONFIG_FILENAME = "edq-config.json" +DEFAULT_CONFIG_FILENAME: str = "edq-config.json" class ConfigSource: """ A class for storing config source information. """ @@ -100,7 +100,7 @@ def _load_config_file( config_path: str, config: typing.Dict[str, str], sources: typing.Dict[str, ConfigSource], - source_label: str + source_label: str, ) -> None: """ Loads config variables and the source from the given config JSON file. """ @@ -112,7 +112,7 @@ def _load_config_file( def _get_local_config_path( config_file_name: str, legacy_config_file_name: typing.Union[str, None] = None, - local_config_root_cutoff: typing.Union[str, None] = None + local_config_root_cutoff: typing.Union[str, None] = None, ) -> typing.Union[str, None]: """ Search for a config file in hierarchical order. @@ -140,13 +140,13 @@ def _get_local_config_path( return _get_ancestor_config_file_path( parent_dir, config_file_name = config_file_name, - local_config_root_cutoff = local_config_root_cutoff + local_config_root_cutoff = local_config_root_cutoff, ) def _get_ancestor_config_file_path( current_directory: str, config_file_name: str, - local_config_root_cutoff: typing.Union[str, None] = None + local_config_root_cutoff: typing.Union[str, None] = None, ) -> typing.Union[str, None]: """ Search through the parent directories (until root or a given cutoff directory(inclusive)) for a config file. diff --git a/edq/util/config_test.py b/edq/util/config_test.py index 6508d35..a69bdb5 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -5,9 +5,6 @@ import edq.util.dirent import edq.util.json -THIS_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) -CONFIGS_DIR = os.path.join(THIS_DIR, "testdata", "configs") - def creat_test_dir(temp_dir_prefix: str) -> str: """ Creat a temp dir and populate it with dirents for testing. From c6c7a0adf9da0f64bb203f7d42a17d44cad3d48c Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Thu, 28 Aug 2025 07:35:39 -0700 Subject: [PATCH 33/71] Revised 4th pass. --- README.md | 60 ++++++++++++++++++++++++++++++++--------- edq/testing/unittest.py | 16 +++++------ edq/util/config_test.py | 1 - 3 files changed, 55 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index f92e85e..ca5c675 100644 --- a/README.md +++ b/README.md @@ -17,24 +17,59 @@ The project and Python dependencies can be installed from source with: pip3 install . ``` -## Configuration +## Configuration System -Many EduLinq tools share a common configuration system. -This system provides a consistent way to supply configuration options to the CLI tool being used. -While each CLI tool may require additional options, the configuration loading process is the same across the EduLinq ecosystem. +This system provides a consistent method for supplying configuration options to a CLI tool. +While each CLI tool may require its own additional options, this system standardizes the configuration loading process. -By default, the config file is named `edq-config.json`. -This is customizable and may differ depending on the tool being used. +By default, the configuration file is named `edq-config.json`. +This name can be customized by specifying a different file name when calling the configuration system. -For the purposes of this documentation we are going to use the default config file name. +For the purposes of this documentation, the default file name is used. ### Configuration Sources Configuration options can come from several places, with later sources overwriting earlier ones. + + + + + + + + + + + + + + + + + + + + + +
SourcesDescription
Global + The default global configuration file, edq-config.json, is located in the platform-specific user configuration directory. + This follows the recommendation from platformdirs. + You can change its location with the --global-config option. +
Local + Local config files can be found in different locations. + Once a match is found, the search stops. + Local config is resolved in this order: edq-config.json in current directory, a supported legacy file in the current directory, then the nearest ancestor edq-config.json. +
CLI + Config files passed with --config-file are loaded sequentially, and options in later files override those in earlier ones. +
CLI Bear + Command-line options (e.g., --user, --token, --server) override all other configuration sources, unless specific keys are configured to be ignored. +
+ + #### Global Configuration -The default place a global config is looked for is `/edq-config.json`. +The default location a global config is looked for is `/edq-config.json`. This is considered to be the "proper" place to store user-related configuration, according to [platformdirs](https://github.com/tox-dev/platformdirs). The location where a global config will be looked for can be changed by passing a path to `--global-config` through the command line. This type of config is best suited for login credentials or persistent user preferences. @@ -44,19 +79,18 @@ Run any CLI tool with `--help` to see the exact path on your platform. Local configuration files can be found in different locations. The first file found will be used, and other locations will not be searched. - -##### Search order for a local config file: - +Local config search order is as follows: 1. An `edq-config.json` in the current directory. 2. A legacy file in the current directory. If the config system is set to support a specified legacy file. -3. An `edq-config.json` file located in any ancestor directory on the path to root (or up to a cutoff limit if one is specified). +3. An `edq-config.json` file located in any ancestor directory on the path to root (or up to a cutoff limit if one is specified to the config system). #### CLI-Specified Config Files -Any files passed via `--config` will be loaded in the order they appear on the command line. +Any files passed via `--config-file` will be loaded in the order they appear on the command line. Later files will override options from previous ones. #### Bare CLI Options Options passed directly on the command line (e.g., `--user`, `--token`, `--server`). These always override every other configuration source. +The configuration system can be set to ignore specific CLI keys when applying overrides, if needed. diff --git a/edq/testing/unittest.py b/edq/testing/unittest.py index 446c342..02b06b2 100644 --- a/edq/testing/unittest.py +++ b/edq/testing/unittest.py @@ -14,7 +14,7 @@ class BaseTest(unittest.TestCase): maxDiff = None """ Don't limit the size of diffs. """ - def assertJSONDictEqual(self, a: typing.Any, b: typing.Any, msg: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name + def assertJSONDictEqual(self, a: typing.Any, b: typing.Any, message: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name """ Like unittest.TestCase.assertDictEqual(), but will try to convert each comparison argument to a dict if it is not already, @@ -36,12 +36,12 @@ def assertJSONDictEqual(self, a: typing.Any, b: typing.Any, msg: typing.Union[st a_json = edq.util.json.dumps(a, indent = 4) b_json = edq.util.json.dumps(b, indent = 4) - if (msg is None): - msg = FORMAT_STR % (a_json, b_json) + if (message is None): + message = FORMAT_STR % (a_json, b_json) - super().assertDictEqual(a, b, msg = msg) + super().assertDictEqual(a, b, msg = message) - def assertJSONListEqual(self, a: typing.List[typing.Any], b: typing.List[typing.Any], msg: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name + def assertJSONListEqual(self, a: typing.List[typing.Any], b: typing.List[typing.Any], message: typing.Union[str, None] = None) -> None: # pylint: disable=invalid-name """ Call assertDictEqual(), but supply a default message containing the full JSON representation of the arguments. """ @@ -49,10 +49,10 @@ def assertJSONListEqual(self, a: typing.List[typing.Any], b: typing.List[typing. a_json = edq.util.json.dumps(a, indent = 4) b_json = edq.util.json.dumps(b, indent = 4) - if (msg is None): - msg = FORMAT_STR % (a_json, b_json) + if (message is None): + message = FORMAT_STR % (a_json, b_json) - super().assertListEqual(a, b, msg = msg) + super().assertListEqual(a, b, msg = message) def format_error_string(self, ex: typing.Union[BaseException, None]) -> str: """ diff --git a/edq/util/config_test.py b/edq/util/config_test.py index a69bdb5..ba1c4b1 100644 --- a/edq/util/config_test.py +++ b/edq/util/config_test.py @@ -69,7 +69,6 @@ def creat_test_dir(temp_dir_prefix: str) -> str: os.path.join(nested_dir_path, "nest1", "nest2b", edq.util.config.DEFAULT_CONFIG_FILENAME), ) - # Malformed JSONs simple_config_dir_path = os.path.join(temp_dir, "simple") edq.util.dirent.mkdir(simple_config_dir_path) edq.util.dirent.write_file( From 53d8bf451add70e3eacf08539edd3e2ecec6b18a Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Thu, 28 Aug 2025 14:47:46 -0700 Subject: [PATCH 34/71] Revised the README, converted the table format. --- README.md | 74 ++++++++++++++++--------------------------------------- 1 file changed, 21 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index ca5c675..fa22089 100644 --- a/README.md +++ b/README.md @@ -19,70 +19,38 @@ pip3 install . ## Configuration System -This system provides a consistent method for supplying configuration options to a CLI tool. -While each CLI tool may require its own additional options, this system standardizes the configuration loading process. - -By default, the configuration file is named `edq-config.json`. -This name can be customized by specifying a different file name when calling the configuration system. - -For the purposes of this documentation, the default file name is used. +This project provides a configuration system that supplies configuration options to a command line interface (CLI) tool. +The configuration file is named `edq-config.json` by default. Specify a different file name when calling the configuration system to change it. +For this documentation, the default file name (`edq-config.json`) is used. ### Configuration Sources -Configuration options can come from several places, with later sources overwriting earlier ones. - - - - - - - - - - - - - - - - - - - - - - -
SourcesDescription
Global - The default global configuration file, edq-config.json, is located in the platform-specific user configuration directory. - This follows the recommendation from platformdirs. - You can change its location with the --global-config option. -
Local - Local config files can be found in different locations. - Once a match is found, the search stops. - Local config is resolved in this order: edq-config.json in current directory, a supported legacy file in the current directory, then the nearest ancestor edq-config.json. -
CLI - Config files passed with --config-file are loaded sequentially, and options in later files override those in earlier ones. -
CLI Bear - Command-line options (e.g., --user, --token, --server) override all other configuration sources, unless specific keys are configured to be ignored. -
+Configuration options can come from several places. Later sources overwrite earlier ones. The table below explains the sources: +| Sources | Description | +| :-------: | :----------- | +| Global | The default global configuration file, `edq-config.json`, lives in the platform-specific user configuration directory, as recommended by [platformdirs](https://github.com/tox-dev/platformdirs). Use the `--global-config` option to change its location.| +| Local | The search stops when it finds a match. The system resolves local config in this order: `edq-config.json` in the current directory, a supported legacy file in the current directory, then the nearest ancestor `edq-config.json`.| +| CLI | When multiple config files are passed with `--config-file`, the system loads them sequentially. Options in later files override those in earlier ones.| +| CLI Bear | Command-line options (e.g., `--user`, `--token`, `--server`) override all other configuration sources unless specific keys are configured to be ignored.| #### Global Configuration -The default location a global config is looked for is `/edq-config.json`. -This is considered to be the "proper" place to store user-related configuration, according to [platformdirs](https://github.com/tox-dev/platformdirs). -The location where a global config will be looked for can be changed by passing a path to `--global-config` through the command line. +The global config file defaults to `/edq-config.json`. +According to [platformdirs](https://github.com/tox-dev/platformdirs), this location serves as the proper place to store user-related configuration. +You can change the global config location by passing a path to `--global-config` in the command line. This type of config is best suited for login credentials or persistent user preferences. -Run any CLI tool with `--help` to see the exact path on your platform. +Run any CLI tool with `--help` to see the exact path for your platform. +A global config may look like: #### Local Configuration -Local configuration files can be found in different locations. +Local configuration files exist in different locations. The first file found will be used, and other locations will not be searched. -Local config search order is as follows: -1. An `edq-config.json` in the current directory. -2. A legacy file in the current directory. If the config system is set to support a specified legacy file. -3. An `edq-config.json` file located in any ancestor directory on the path to root (or up to a cutoff limit if one is specified to the config system). +Local config search order: +1. `edq-config.json` in the current directory. +2. A legacy file in the current directory (only if the config system supports a specified legacy file). +3. `edq-config.json` in any ancestor directory on the path to root (or up to a cutoff limit, if specified in the config system). #### CLI-Specified Config Files @@ -93,4 +61,4 @@ Later files will override options from previous ones. Options passed directly on the command line (e.g., `--user`, `--token`, `--server`). These always override every other configuration source. -The configuration system can be set to ignore specific CLI keys when applying overrides, if needed. +The configuration system can be set to ignore specific CLI keys when applying overrides, if needed. \ No newline at end of file From 22797dbbb28f9f947eb87ae7c5f479ce5ad43189 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Thu, 28 Aug 2025 17:54:39 -0700 Subject: [PATCH 35/71] Got rid of a inconsistency. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index fa22089..66288c6 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,6 @@ According to [platformdirs](https://github.com/tox-dev/platformdirs), this locat You can change the global config location by passing a path to `--global-config` in the command line. This type of config is best suited for login credentials or persistent user preferences. Run any CLI tool with `--help` to see the exact path for your platform. -A global config may look like: #### Local Configuration From c7537297ca7ce4a630b78675ae6d70f68ed0b15e Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Thu, 28 Aug 2025 19:29:17 -0700 Subject: [PATCH 36/71] 2nd revision of README --- README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 66288c6..7ac86d3 100644 --- a/README.md +++ b/README.md @@ -20,19 +20,21 @@ pip3 install . ## Configuration System This project provides a configuration system that supplies configuration options to a command line interface (CLI) tool. -The configuration file is named `edq-config.json` by default. Specify a different file name when calling the configuration system to change it. +Specify a different file name when calling the configuration system to change it. For this documentation, the default file name (`edq-config.json`) is used. ### Configuration Sources -Configuration options can come from several places. Later sources overwrite earlier ones. The table below explains the sources: +Configuration options can be set in multiple places. +If the same option is set in more than one place, the value from the later source overrides the earlier ones. +The table below shows the order in which sources are applied, from top to bottom. | Sources | Description | | :-------: | :----------- | -| Global | The default global configuration file, `edq-config.json`, lives in the platform-specific user configuration directory, as recommended by [platformdirs](https://github.com/tox-dev/platformdirs). Use the `--global-config` option to change its location.| -| Local | The search stops when it finds a match. The system resolves local config in this order: `edq-config.json` in the current directory, a supported legacy file in the current directory, then the nearest ancestor `edq-config.json`.| -| CLI | When multiple config files are passed with `--config-file`, the system loads them sequentially. Options in later files override those in earlier ones.| -| CLI Bear | Command-line options (e.g., `--user`, `--token`, `--server`) override all other configuration sources unless specific keys are configured to be ignored.| +| Global | Global config file defaults to a platform-specific user location and can be changed with the `--global-config` option.| +| Local | Local configuration is loaded from the first matching file found, starting in the current directory and moving up to ancestor directories.| +| CLI | Files passed with `--config-file` are loaded in order, with later files overriding earlier ones.| +| CLI Bear | Command-line options override all other configuration sources.| #### Global Configuration @@ -40,7 +42,7 @@ The global config file defaults to `/edq According to [platformdirs](https://github.com/tox-dev/platformdirs), this location serves as the proper place to store user-related configuration. You can change the global config location by passing a path to `--global-config` in the command line. This type of config is best suited for login credentials or persistent user preferences. -Run any CLI tool with `--help` to see the exact path for your platform. +Run any CLI tool with `--help` to see the exact path for the current platform under the flag `--config-file`. #### Local Configuration From 629f110956753a9ccaa2adae2dce23a38789b7db Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Thu, 28 Aug 2025 21:47:04 -0700 Subject: [PATCH 37/71] Revised it again. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7ac86d3..308fd34 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ pip3 install . ## Configuration System This project provides a configuration system that supplies configuration options to a command line interface (CLI) tool. -Specify a different file name when calling the configuration system to change it. For this documentation, the default file name (`edq-config.json`) is used. +Specify a different file name when calling the configuration system to change the default. ### Configuration Sources @@ -51,7 +51,7 @@ The first file found will be used, and other locations will not be searched. Local config search order: 1. `edq-config.json` in the current directory. 2. A legacy file in the current directory (only if the config system supports a specified legacy file). -3. `edq-config.json` in any ancestor directory on the path to root (or up to a cutoff limit, if specified in the config system). +3. `edq-config.json` in any ancestor directory on the path to root (or up to a cutoff limit, if specified to the config system). #### CLI-Specified Config Files @@ -62,4 +62,4 @@ Later files will override options from previous ones. Options passed directly on the command line (e.g., `--user`, `--token`, `--server`). These always override every other configuration source. -The configuration system can be set to ignore specific CLI keys when applying overrides, if needed. \ No newline at end of file +The configuration system can be set to ignore specific CLI options when applying overrides, if needed. \ No newline at end of file From f223a92478ebd9c2e496e42aba2804b17fea0954 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 29 Aug 2025 19:07:04 -0700 Subject: [PATCH 38/71] Added error description. --- README.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 308fd34..91622fa 100644 --- a/README.md +++ b/README.md @@ -19,30 +19,34 @@ pip3 install . ## Configuration System -This project provides a configuration system that supplies configuration options to a command line interface (CLI) tool. -For this documentation, the default file name (`edq-config.json`) is used. -Specify a different file name when calling the configuration system to change the default. +This project provides a configuration system that supplies configuration options to a command-line interface (CLI) tool from multiple sources. ### Configuration Sources -Configuration options can be set in multiple places. -If the same option is set in more than one place, the value from the later source overrides the earlier ones. +The configuration system reads options from multiple files located in different directories. +By default the configuration system searches for files named `edq-config.json`, this can be customizable. +For this documentation, the default file name (`edq-config.json`) is used. +Specify a different file name when calling the configuration system to change the default. + +If the same configuration option is set in more than one place, the value from the later source overrides the earlier ones. The table below shows the order in which sources are applied, from top to bottom. | Sources | Description | -| :-------: | :----------- | +| :------:| :---------- | | Global | Global config file defaults to a platform-specific user location and can be changed with the `--global-config` option.| | Local | Local configuration is loaded from the first matching file found, starting in the current directory and moving up to ancestor directories.| | CLI | Files passed with `--config-file` are loaded in order, with later files overriding earlier ones.| | CLI Bear | Command-line options override all other configuration sources.| +The system produces an error if a global or local config file is unreadable (but not missing), or if a CLI-specified file is unreadable or missing. + #### Global Configuration The global config file defaults to `/edq-config.json`. According to [platformdirs](https://github.com/tox-dev/platformdirs), this location serves as the proper place to store user-related configuration. -You can change the global config location by passing a path to `--global-config` in the command line. +You can change the global config location by passing a path to `--global-config` to the command line. This type of config is best suited for login credentials or persistent user preferences. -Run any CLI tool with `--help` to see the exact path for the current platform under the flag `--config-file`. +Run any CLI tool with `--help` to see the exact path for the current platform under the flag `--global-config`. #### Local Configuration @@ -50,7 +54,7 @@ Local configuration files exist in different locations. The first file found will be used, and other locations will not be searched. Local config search order: 1. `edq-config.json` in the current directory. -2. A legacy file in the current directory (only if the config system supports a specified legacy file). +2. A legacy file in the current directory (only if a specified legacy file is passed to the config system). 3. `edq-config.json` in any ancestor directory on the path to root (or up to a cutoff limit, if specified to the config system). #### CLI-Specified Config Files From 418b6cba33c6cb2eb4378a5d550b08266bf24221 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sat, 30 Aug 2025 19:51:24 -0700 Subject: [PATCH 39/71] Added examples to the README. --- README.md | 55 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 91622fa..0c54f87 100644 --- a/README.md +++ b/README.md @@ -19,51 +19,74 @@ pip3 install . ## Configuration System -This project provides a configuration system that supplies configuration options to a command-line interface (CLI) tool from multiple sources. +This project provides a configuration system that supplies configuration options to a command-line interface (CLI) tool. ### Configuration Sources -The configuration system reads options from multiple files located in different directories. -By default the configuration system searches for files named `edq-config.json`, this can be customizable. -For this documentation, the default file name (`edq-config.json`) is used. -Specify a different file name when calling the configuration system to change the default. +The configuration system reads options from multiple JSON files located in different directories. +These configuration files are not generated by default. +They are located through the configuration system’s search process. +By default the configuration system searches for files named `edq-config.json`, this is customizable. + +For example, a configuration file containing the `user` and `token` options might look like this: +``` +{ + "user": "edq-user", + "token": "1234567890" +} +``` -If the same configuration option is set in more than one place, the value from the later source overrides the earlier ones. The table below shows the order in which sources are applied, from top to bottom. +If the same configuration option is set in more than one place, the value from the later source overrides the earlier ones. | Sources | Description | | :------:| :---------- | -| Global | Global config file defaults to a platform-specific user location and can be changed with the `--global-config` option.| +| Global | Global configuration file defaults to a platform-specific user location and can be changed with the `--global-config` option.| | Local | Local configuration is loaded from the first matching file found, starting in the current directory and moving up to ancestor directories.| | CLI | Files passed with `--config-file` are loaded in order, with later files overriding earlier ones.| | CLI Bear | Command-line options override all other configuration sources.| -The system produces an error if a global or local config file is unreadable (but not missing), or if a CLI-specified file is unreadable or missing. +The system produces an error if a global or local configuration file is unreadable (but not missing), or if a CLI-specified file is unreadable or missing. #### Global Configuration -The global config file defaults to `/edq-config.json`. +The global configuration file defaults to `/edq-config.json`. According to [platformdirs](https://github.com/tox-dev/platformdirs), this location serves as the proper place to store user-related configuration. -You can change the global config location by passing a path to `--global-config` to the command line. -This type of config is best suited for login credentials or persistent user preferences. -Run any CLI tool with `--help` to see the exact path for the current platform under the flag `--global-config`. +You can change the global configuration location by passing a path to `--global-config` to the command line. +This type of configuration is best suited for login credentials or persistent user preferences. +Run any CLI command with `--help` to see the exact path for the current platform under the flag `--global-config`. + +Here is an example of specifying a global config path: +``` +cli-tool --global-config /path/to/file/.json +``` #### Local Configuration -Local configuration files exist in different locations. +Local configuration files are searched for in multiple locations. The first file found will be used, and other locations will not be searched. Local config search order: 1. `edq-config.json` in the current directory. -2. A legacy file in the current directory (only if a specified legacy file is passed to the config system). -3. `edq-config.json` in any ancestor directory on the path to root (or up to a cutoff limit, if specified to the config system). +2. A legacy file in the current directory (only if a specified legacy file is passed to the configuration system). +3. `edq-config.json` in any ancestor directory on the path to root (or up to a cutoff limit, if specified to the configuration system). #### CLI-Specified Config Files Any files passed via `--config-file` will be loaded in the order they appear on the command line. Later files will override options from previous ones. +Here is an example of specifying a CLI-specified configuration path: +``` +cli-tool --config-file /path/to/file/.json +``` + #### Bare CLI Options Options passed directly on the command line (e.g., `--user`, `--token`, `--server`). These always override every other configuration source. -The configuration system can be set to ignore specific CLI options when applying overrides, if needed. \ No newline at end of file +The configuration system can be set to ignore specific CLI options when applying overrides, if needed. + +Here is an example of specifying a config option directly from the CLI: +``` +cli-tool --user=edq-user +``` \ No newline at end of file From 869d963f38cf851f9a2c931a33d2cc986383970a Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sun, 31 Aug 2025 14:58:11 -0700 Subject: [PATCH 40/71] Made clarifications. --- README.md | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 0c54f87..62fa8c6 100644 --- a/README.md +++ b/README.md @@ -19,14 +19,14 @@ pip3 install . ## Configuration System -This project provides a configuration system that supplies configuration options to a command-line interface (CLI) tool. +This project provides a configuration system that supplies options (e.g., `user`, `token`) to a command-line interface (CLI) tool. +The configuration system follows a tiered order, incorporating both files and command-line options. ### Configuration Sources -The configuration system reads options from multiple JSON files located in different directories. -These configuration files are not generated by default. -They are located through the configuration system’s search process. -By default the configuration system searches for files named `edq-config.json`, this is customizable. +The configuration system loads options from JSON files located across multiple directories. +It searches for these files when invoked, they are not created automatically. +By default, it looks for files named `edq-config.json`, this is customizable. For example, a configuration file containing the `user` and `token` options might look like this: ``` @@ -36,14 +36,15 @@ For example, a configuration file containing the `user` and `token` options migh } ``` -The table below shows the order in which sources are applied, from top to bottom. -If the same configuration option is set in more than one place, the value from the later source overrides the earlier ones. +The table below lists configuration sources in the order they are evaluated, from the first source at the top row to the last source at the bottom row. +All sources are processed in the order show in the table: if an option appears only in one source, it is included as is. +If the same option appears in multiple sources, the value from the later source in the table overrides the earlier one. -| Sources | Description | -| :------:| :---------- | -| Global | Global configuration file defaults to a platform-specific user location and can be changed with the `--global-config` option.| -| Local | Local configuration is loaded from the first matching file found, starting in the current directory and moving up to ancestor directories.| -| CLI | Files passed with `--config-file` are loaded in order, with later files overriding earlier ones.| +| Sources | Description | +| :-----: | :---------- | +| Global | Global configuration file defaults to a platform-specific user location and can be changed with the `--global-config` option.| +| Local | Local configuration is loaded from the first matching file found, starting in the current directory and moving up to ancestor directories.| +| CLI | Files passed with `--config-file` are loaded in order, with later files overriding earlier ones.| | CLI Bear | Command-line options override all other configuration sources.| The system produces an error if a global or local configuration file is unreadable (but not missing), or if a CLI-specified file is unreadable or missing. @@ -52,30 +53,30 @@ The system produces an error if a global or local configuration file is unreadab The global configuration file defaults to `/edq-config.json`. According to [platformdirs](https://github.com/tox-dev/platformdirs), this location serves as the proper place to store user-related configuration. -You can change the global configuration location by passing a path to `--global-config` to the command line. +The global configuration location is changed by passing a path to `--global-config` to the command line. This type of configuration is best suited for login credentials or persistent user preferences. Run any CLI command with `--help` to see the exact path for the current platform under the flag `--global-config`. -Here is an example of specifying a global config path: +Below is an example of specifying a global config path: ``` cli-tool --global-config /path/to/file/.json ``` #### Local Configuration -Local configuration files are searched for in multiple locations. -The first file found will be used, and other locations will not be searched. +Local configuration files are searched in multiple locations, first file found is used. +Once a file is located, the search for local configuration stops. Local config search order: 1. `edq-config.json` in the current directory. 2. A legacy file in the current directory (only if a specified legacy file is passed to the configuration system). -3. `edq-config.json` in any ancestor directory on the path to root (or up to a cutoff limit, if specified to the configuration system). +3. `edq-config.json` in any ancestor directory on the path to root (or up to a cutoff directory limit, if specified to the configuration system). #### CLI-Specified Config Files Any files passed via `--config-file` will be loaded in the order they appear on the command line. Later files will override options from previous ones. -Here is an example of specifying a CLI-specified configuration path: +Below is an example of a CLI specified configuration path: ``` cli-tool --config-file /path/to/file/.json ``` @@ -84,9 +85,9 @@ cli-tool --config-file /path/to/file/.json Options passed directly on the command line (e.g., `--user`, `--token`, `--server`). These always override every other configuration source. -The configuration system can be set to ignore specific CLI options when applying overrides, if needed. +The configuration system skips specific CLI options when applying overrides if certain keys are passed to it. -Here is an example of specifying a config option directly from the CLI: +Below is an example of specifying a config option directly from the CLI: ``` -cli-tool --user=edq-user +cli-tool --user=edq-user --token=12345 ``` \ No newline at end of file From 033d119035fda918511f8f519b42e1b3aab0e292 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sun, 31 Aug 2025 15:20:09 -0700 Subject: [PATCH 41/71] Revised the overriding on skip keys. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 62fa8c6..4eeddf0 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ For example, a configuration file containing the `user` and `token` options migh ``` The table below lists configuration sources in the order they are evaluated, from the first source at the top row to the last source at the bottom row. -All sources are processed in the order show in the table: if an option appears only in one source, it is included as is. +All sources are processed in the order shown in the table: if an option appears only in one source, it is included as is. If the same option appears in multiple sources, the value from the later source in the table overrides the earlier one. | Sources | Description | @@ -68,7 +68,7 @@ Local configuration files are searched in multiple locations, first file found i Once a file is located, the search for local configuration stops. Local config search order: 1. `edq-config.json` in the current directory. -2. A legacy file in the current directory (only if a specified legacy file is passed to the configuration system). +2. A legacy file in the current directory (only if a legacy file is passed to the configuration system). 3. `edq-config.json` in any ancestor directory on the path to root (or up to a cutoff directory limit, if specified to the configuration system). #### CLI-Specified Config Files @@ -78,14 +78,14 @@ Later files will override options from previous ones. Below is an example of a CLI specified configuration path: ``` -cli-tool --config-file /path/to/file/.json +cli-tool --config-file .json --config-file ~/.secrets/.json ``` #### Bare CLI Options Options passed directly on the command line (e.g., `--user`, `--token`, `--server`). These always override every other configuration source. -The configuration system skips specific CLI options when applying overrides if certain keys are passed to it. +The configuration system ignores certain CLI options when applying overrides if specific skip-keys are provided. Below is an example of specifying a config option directly from the CLI: ``` From 51c139b40681335ce66490d80290be1e6eba1022 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Mon, 1 Sep 2025 18:12:16 -0700 Subject: [PATCH 42/71] Moved config from util to core. --- README.md | 45 ++++---- edq/{util => core}/config.py | 0 edq/{util => core}/config_test.py | 178 +++++++++++++++--------------- 3 files changed, 108 insertions(+), 115 deletions(-) rename edq/{util => core}/config.py (100%) rename edq/{util => core}/config_test.py (78%) diff --git a/README.md b/README.md index 4eeddf0..d27db17 100644 --- a/README.md +++ b/README.md @@ -19,14 +19,14 @@ pip3 install . ## Configuration System -This project provides a configuration system that supplies options (e.g., `user`, `token`) to a command-line interface (CLI) tool. -The configuration system follows a tiered order, incorporating both files and command-line options. +This project provides a configuration system that supplies options (e.g., user name, password) to a command-line interface ([CLI](https://en.wikipedia.org/wiki/Command-line_interface)) tool. +The configuration system follows a tiered order, allowing options to be specified and overridden from both files and command-line options. ### Configuration Sources -The configuration system loads options from JSON files located across multiple directories. -It searches for these files when invoked, they are not created automatically. -By default, it looks for files named `edq-config.json`, this is customizable. +In addition to CLI options, the configuration system loads options from [JSON](https://en.wikipedia.org/wiki/JSON) files located across multiple directories. +By default, config files are named `edq-config.json`. +This value is customizable, but this document will assume the default is used. For example, a configuration file containing the `user` and `token` options might look like this: ``` @@ -36,40 +36,34 @@ For example, a configuration file containing the `user` and `token` options migh } ``` -The table below lists configuration sources in the order they are evaluated, from the first source at the top row to the last source at the bottom row. +The table below lists the configuration sources in the order they are evaluated. All sources are processed in the order shown in the table: if an option appears only in one source, it is included as is. If the same option appears in multiple sources, the value from the later source in the table overrides the earlier one. -| Sources | Description | +| Source | Description | | :-----: | :---------- | -| Global | Global configuration file defaults to a platform-specific user location and can be changed with the `--global-config` option.| -| Local | Local configuration is loaded from the first matching file found, starting in the current directory and moving up to ancestor directories.| -| CLI | Files passed with `--config-file` are loaded in order, with later files overriding earlier ones.| -| CLI Bear | Command-line options override all other configuration sources.| +| Global | Global configuration file defaults to a platform-specific user location and can be changed with the `--global-config` option. | +| Local | Local configuration is loaded from the first matching file found, starting in the current directory and moving up to ancestor directories until root. | +| CLI File | Files passed with `--config-file` are loaded in order, with later files overriding earlier ones. | +| CLI | Command-line options override all other configuration sources. | The system produces an error if a global or local configuration file is unreadable (but not missing), or if a CLI-specified file is unreadable or missing. #### Global Configuration The global configuration file defaults to `/edq-config.json`. -According to [platformdirs](https://github.com/tox-dev/platformdirs), this location serves as the proper place to store user-related configuration. -The global configuration location is changed by passing a path to `--global-config` to the command line. +The configuration location is chosen according to the [XDG standard](https://en.wikipedia.org/wiki/Freedesktop.org#Base_Directory_Specification) (implemented by [platformdirs](https://github.com/tox-dev/platformdirs)). +The global configuration location can be changed by passing a path to `--global-config` to the command line. This type of configuration is best suited for login credentials or persistent user preferences. Run any CLI command with `--help` to see the exact path for the current platform under the flag `--global-config`. -Below is an example of specifying a global config path: -``` -cli-tool --global-config /path/to/file/.json -``` - #### Local Configuration -Local configuration files are searched in multiple locations, first file found is used. -Once a file is located, the search for local configuration stops. -Local config search order: +Local configuration files are searched in multiple locations, the first file found is used. +The Local config search order is: 1. `edq-config.json` in the current directory. -2. A legacy file in the current directory (only if a legacy file is passed to the configuration system). -3. `edq-config.json` in any ancestor directory on the path to root (or up to a cutoff directory limit, if specified to the configuration system). +2. A legacy file in the current directory (only if a legacy file is preconfigured). +3. `edq-config.json` in any ancestor directory on the path to root . #### CLI-Specified Config Files @@ -78,16 +72,15 @@ Later files will override options from previous ones. Below is an example of a CLI specified configuration path: ``` -cli-tool --config-file .json --config-file ~/.secrets/.json +python3 -m edq.cli.config.list --config-file .json --config-file ~/.secrets/.json ``` #### Bare CLI Options Options passed directly on the command line (e.g., `--user`, `--token`, `--server`). These always override every other configuration source. -The configuration system ignores certain CLI options when applying overrides if specific skip-keys are provided. Below is an example of specifying a config option directly from the CLI: ``` -cli-tool --user=edq-user --token=12345 +python3 -m edq.cli.config.list --user=edq-user --token=12345 ``` \ No newline at end of file diff --git a/edq/util/config.py b/edq/core/config.py similarity index 100% rename from edq/util/config.py rename to edq/core/config.py diff --git a/edq/util/config_test.py b/edq/core/config_test.py similarity index 78% rename from edq/util/config_test.py rename to edq/core/config_test.py index ba1c4b1..916ca2d 100644 --- a/edq/util/config_test.py +++ b/edq/core/config_test.py @@ -1,7 +1,7 @@ import os import edq.testing.unittest -import edq.util.config +import edq.core.config import edq.util.dirent import edq.util.json @@ -41,7 +41,7 @@ def creat_test_dir(temp_dir_prefix: str) -> str: empty_config_dir_path = os.path.join(temp_dir, "empty") edq.util.dirent.mkdir(empty_config_dir_path) - edq.util.json.dump_path({}, os.path.join(empty_config_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME)) + edq.util.json.dump_path({}, os.path.join(empty_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME)) custome_name_config_dir_path = os.path.join(temp_dir, "custom-name") edq.util.dirent.mkdir(custome_name_config_dir_path) @@ -52,7 +52,7 @@ def creat_test_dir(temp_dir_prefix: str) -> str: global_config_dir_path = os.path.join(temp_dir, "global") edq.util.dirent.mkdir(global_config_dir_path) - edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(global_config_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME)) + edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(global_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME)) old_name_config_dir_path = os.path.join(temp_dir, "old-name") edq.util.dirent.mkdir(os.path.join(old_name_config_dir_path, "nest1", "nest2")) @@ -62,24 +62,24 @@ def creat_test_dir(temp_dir_prefix: str) -> str: edq.util.dirent.mkdir(os.path.join(nested_dir_path, "nest1", "nest2a")) edq.util.dirent.mkdir(os.path.join(nested_dir_path, "nest1", "nest2b")) - edq.util.json.dump_path({"server": "http://test.edulinq.org"}, os.path.join(nested_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME)) + edq.util.json.dump_path({"server": "http://test.edulinq.org"}, os.path.join(nested_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME)) edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(nested_dir_path, "config.json")) edq.util.json.dump_path( {"user": "user@test.edulinq.org"}, - os.path.join(nested_dir_path, "nest1", "nest2b", edq.util.config.DEFAULT_CONFIG_FILENAME), + os.path.join(nested_dir_path, "nest1", "nest2b", edq.core.config.DEFAULT_CONFIG_FILENAME), ) simple_config_dir_path = os.path.join(temp_dir, "simple") edq.util.dirent.mkdir(simple_config_dir_path) edq.util.dirent.write_file( - os.path.join(simple_config_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME), + os.path.join(simple_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME), '{"user": "user@test.edulinq.org",}', ) malformed_config_dir_path = os.path.join(temp_dir, "malformed") edq.util.dirent.mkdir(malformed_config_dir_path) edq.util.dirent.write_file( - os.path.join(malformed_config_dir_path, edq.util.config.DEFAULT_CONFIG_FILENAME), + os.path.join(malformed_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME), "{user: user@test.edulinq.org}", ) @@ -112,15 +112,15 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, { "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource( - label = edq.util.config.CONFIG_SOURCE_GLOBAL, - path = os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_GLOBAL, + path = os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), ), }, None, @@ -130,7 +130,7 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "empty", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join(temp_dir, "empty", edq.core.config.DEFAULT_CONFIG_FILENAME), }, {}, {}, @@ -141,7 +141,7 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "dir-config", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join(temp_dir, "dir-config", edq.core.config.DEFAULT_CONFIG_FILENAME), }, {}, {}, @@ -163,7 +163,7 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "malformed", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join(temp_dir, "malformed", edq.core.config.DEFAULT_CONFIG_FILENAME), }, {}, {}, @@ -180,9 +180,9 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource( - label = edq.util.config.CONFIG_SOURCE_LOCAL, - path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ), }, None, @@ -198,8 +198,8 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource( - label = edq.util.config.CONFIG_SOURCE_LOCAL, + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ), }, @@ -216,8 +216,8 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource( - label = edq.util.config.CONFIG_SOURCE_LOCAL, + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, path = os.path.join(temp_dir, "old-name", "config.json"), ), }, @@ -232,9 +232,9 @@ def test_get_tiered_config_base(self): "server": "http://test.edulinq.org", }, { - "server": edq.util.config.ConfigSource( - label = edq.util.config.CONFIG_SOURCE_LOCAL, - path = os.path.join(temp_dir, "nested", edq.util.config.DEFAULT_CONFIG_FILENAME), + "server": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, + path = os.path.join(temp_dir, "nested", edq.core.config.DEFAULT_CONFIG_FILENAME), ), }, None, @@ -288,9 +288,9 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource( - label = edq.util.config.CONFIG_SOURCE_LOCAL, - path = os.path.join(temp_dir, "nested", "nest1", "nest2b", edq.util.config.DEFAULT_CONFIG_FILENAME), + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, + path = os.path.join(temp_dir, "nested", "nest1", "nest2b", edq.core.config.DEFAULT_CONFIG_FILENAME), ), }, None, @@ -303,9 +303,9 @@ def test_get_tiered_config_base(self): "empty-dir", { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [ - os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), - os.path.join(temp_dir, "nested", edq.util.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + os.path.join(temp_dir, "nested", edq.core.config.DEFAULT_CONFIG_FILENAME), ], }, }, @@ -314,13 +314,13 @@ def test_get_tiered_config_base(self): "server": "http://test.edulinq.org", }, { - "user": edq.util.config.ConfigSource( - label = edq.util.config.CONFIG_SOURCE_CLI, - path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ), - "server": edq.util.config.ConfigSource( - label = edq.util.config.CONFIG_SOURCE_CLI, - path = os.path.join(temp_dir, "nested", edq.util.config.DEFAULT_CONFIG_FILENAME), + "server": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI, + path = os.path.join(temp_dir, "nested", edq.core.config.DEFAULT_CONFIG_FILENAME), ), }, None, @@ -331,9 +331,9 @@ def test_get_tiered_config_base(self): "empty-dir", { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [ + edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), - os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), + os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ], }, }, @@ -341,9 +341,9 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource( - label = edq.util.config.CONFIG_SOURCE_CLI, - path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ), }, None, @@ -354,8 +354,8 @@ def test_get_tiered_config_base(self): "empty-dir", { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [ - os.path.join(temp_dir, "empty", edq.util.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "empty", edq.core.config.DEFAULT_CONFIG_FILENAME), ], }, }, @@ -369,8 +369,8 @@ def test_get_tiered_config_base(self): "empty-dir", { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [ - os.path.join(temp_dir, "dir-config", edq.util.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "dir-config", edq.core.config.DEFAULT_CONFIG_FILENAME), ], }, }, @@ -384,7 +384,7 @@ def test_get_tiered_config_base(self): "empty-dir", { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [ + edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "empty-dir", "non-existent-config.json"), ], }, @@ -399,8 +399,8 @@ def test_get_tiered_config_base(self): "empty-dir", { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [ - os.path.join(temp_dir, "malformed", edq.util.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "malformed", edq.core.config.DEFAULT_CONFIG_FILENAME), ], }, }, @@ -423,7 +423,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), }, None, ), @@ -444,7 +444,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), }, None, ), @@ -455,15 +455,15 @@ def test_get_tiered_config_base(self): ( "simple", { - "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, { "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource( - label = edq.util.config.CONFIG_SOURCE_LOCAL, - path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ), }, None, @@ -473,10 +473,10 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [ - os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ], }, }, @@ -484,9 +484,9 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource( - label = edq.util.config.CONFIG_SOURCE_CLI, - path = os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ), }, None, @@ -496,7 +496,7 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { "user": "user@test.edulinq.org", }, @@ -505,7 +505,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), }, None, ), @@ -515,7 +515,7 @@ def test_get_tiered_config_base(self): "simple", { "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [ + edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ], }, @@ -524,8 +524,8 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource( - label = edq.util.config.CONFIG_SOURCE_CLI, + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI, path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ), }, @@ -544,7 +544,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), }, None, ), @@ -555,8 +555,8 @@ def test_get_tiered_config_base(self): { "cli_arguments": { "user": "user@test.edulinq.org", - edq.util.config.CONFIG_PATHS_KEY: [ - os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ], }, }, @@ -564,7 +564,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), }, None, ), @@ -573,11 +573,11 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { "user": "user@test.edulinq.org", - edq.util.config.CONFIG_PATHS_KEY: [ - os.path.join(temp_dir, "simple", edq.util.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ], }, }, @@ -585,7 +585,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), }, None, ), @@ -594,7 +594,7 @@ def test_get_tiered_config_base(self): ( "simple", { - "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { "user": "user@test.edulinq.org", }, @@ -603,7 +603,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), }, None, ), @@ -612,9 +612,9 @@ def test_get_tiered_config_base(self): ( "simple", { - "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [ + edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ], }, @@ -623,8 +623,8 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource( - label = edq.util.config.CONFIG_SOURCE_CLI, + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI, path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ), }, @@ -637,7 +637,7 @@ def test_get_tiered_config_base(self): { "cli_arguments": { "user": "user@test.edulinq.org", - edq.util.config.CONFIG_PATHS_KEY: [ + edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ], }, @@ -646,7 +646,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), }, None, ), @@ -655,9 +655,9 @@ def test_get_tiered_config_base(self): ( "simple", { - "global_config_path": os.path.join(temp_dir, "global", edq.util.config.DEFAULT_CONFIG_FILENAME), + "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { - edq.util.config.CONFIG_PATHS_KEY: [ + edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ], "pass": "user", @@ -665,7 +665,7 @@ def test_get_tiered_config_base(self): }, "skip_keys": [ "server", - edq.util.config.CONFIG_PATHS_KEY, + edq.core.config.CONFIG_PATHS_KEY, ], }, { @@ -673,11 +673,11 @@ def test_get_tiered_config_base(self): "pass": "user", }, { - "user": edq.util.config.ConfigSource( - label = edq.util.config.CONFIG_SOURCE_CLI, + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI, path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ), - "pass": edq.util.config.ConfigSource(label = edq.util.config.CONFIG_SOURCE_CLI_BARE), + "pass": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), }, None, ), @@ -689,7 +689,7 @@ def test_get_tiered_config_base(self): with self.subTest(msg = f"Case {i} ('{test_work_dir}'):"): global_config = extra_args.get("global_config_path", None) if (global_config is None): - extra_args["global_config_path"] = os.path.join(temp_dir, "empty", edq.util.config.CONFIG_PATHS_KEY) + extra_args["global_config_path"] = os.path.join(temp_dir, "empty", edq.core.config.CONFIG_PATHS_KEY) cutoff = extra_args.get("local_config_root_cutoff", None) if (cutoff is None): @@ -700,7 +700,7 @@ def test_get_tiered_config_base(self): os.chdir(initial_work_directory) try: - (actual_config, actual_sources) = edq.util.config.get_tiered_config(**extra_args) + (actual_config, actual_sources) = edq.core.config.get_tiered_config(**extra_args) except Exception as ex: error_string = self.format_error_string(ex) From b15cbd02b41699025b96c84e483bf6546e4ce729 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Mon, 1 Sep 2025 20:59:59 -0700 Subject: [PATCH 43/71] Revised the README with 5th pass. --- README.md | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d27db17..45fd726 100644 --- a/README.md +++ b/README.md @@ -37,14 +37,14 @@ For example, a configuration file containing the `user` and `token` options migh ``` The table below lists the configuration sources in the order they are evaluated. -All sources are processed in the order shown in the table: if an option appears only in one source, it is included as is. -If the same option appears in multiple sources, the value from the later source in the table overrides the earlier one. +All sources are processed in the order shown in the table. +When there are multiple sources, the value from the later source in the table overrides the earlier one. | Source | Description | | :-----: | :---------- | -| Global | Global configuration file defaults to a platform-specific user location and can be changed with the `--global-config` option. | +| Global | The global configuration file path is a platform-specific user location by default. | | Local | Local configuration is loaded from the first matching file found, starting in the current directory and moving up to ancestor directories until root. | -| CLI File | Files passed with `--config-file` are loaded in order, with later files overriding earlier ones. | +| CLI File | Files passed with `--file` are loaded in order, with later files overriding earlier ones. | | CLI | Command-line options override all other configuration sources. | The system produces an error if a global or local configuration file is unreadable (but not missing), or if a CLI-specified file is unreadable or missing. @@ -53,9 +53,8 @@ The system produces an error if a global or local configuration file is unreadab The global configuration file defaults to `/edq-config.json`. The configuration location is chosen according to the [XDG standard](https://en.wikipedia.org/wiki/Freedesktop.org#Base_Directory_Specification) (implemented by [platformdirs](https://github.com/tox-dev/platformdirs)). -The global configuration location can be changed by passing a path to `--global-config` to the command line. -This type of configuration is best suited for login credentials or persistent user preferences. -Run any CLI command with `--help` to see the exact path for the current platform under the flag `--global-config`. +The default global configuration location can be changed by passing a path to `--global` through the command line. +This type of configuration is best suited for options that follow the user across multiple projects. #### Local Configuration @@ -67,7 +66,7 @@ The Local config search order is: #### CLI-Specified Config Files -Any files passed via `--config-file` will be loaded in the order they appear on the command line. +Any files passed via `--file` will be loaded in the order they appear on the command line. Later files will override options from previous ones. Below is an example of a CLI specified configuration path: @@ -75,12 +74,26 @@ Below is an example of a CLI specified configuration path: python3 -m edq.cli.config.list --config-file .json --config-file ~/.secrets/.json ``` -#### Bare CLI Options +#### CLI Configuration -Options passed directly on the command line (e.g., `--user`, `--token`, `--server`). -These always override every other configuration source. +Configuration options are structured as `key` and `value` pairs. +Keys cannot contain the "=" character. +Configuration options are passed to the command line by the `--config`/`-c` flag in this format `-c =`. +The provided value overrides the key’s value from configuration files. Below is an example of specifying a config option directly from the CLI: ``` -python3 -m edq.cli.config.list --user=edq-user --token=12345 -``` \ No newline at end of file +python3 -m edq.cli.config.list -c user=edq-user -c token=12345 +``` + +#### CLI Config Options + +The table below lists all the default configuration CLI options available. + +| CLI Option | Description | +| :------------: | :---------- | +|`--global` | For writing options: writes to the default global file path, or to the specified file if provided. For reading options: loads global configuration from the default global file path, or from the specified file if provided. When `--help` is used, the exact default global file path for the current platform will be displayed under this flag. | +| `--local` | For writing options: writes to the first local file found (check local config section for [search order.](#local-configuration)). If none exists, creates an `edq-config.json` in the current directory and writes to it. For reading options: loads local configuration as described in [local configuration section](#local-configuration) | +|`--file` | For writing options: writes to the specified file. For reading options: loads CLI file config options from the specified file. | +| `--config`/`-c`| For providing additional CLI configuration parameters when running any config command. | +| `--help` | Displays a help message with detailed descriptions of each option. | From 568830f720e00b4ffb7bdf1c0f0353fbd1876489 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Mon, 1 Sep 2025 21:17:45 -0700 Subject: [PATCH 44/71] Added 'only' to local and global config description. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 45fd726..94b6384 100644 --- a/README.md +++ b/README.md @@ -92,8 +92,8 @@ The table below lists all the default configuration CLI options available. | CLI Option | Description | | :------------: | :---------- | -|`--global` | For writing options: writes to the default global file path, or to the specified file if provided. For reading options: loads global configuration from the default global file path, or from the specified file if provided. When `--help` is used, the exact default global file path for the current platform will be displayed under this flag. | -| `--local` | For writing options: writes to the first local file found (check local config section for [search order.](#local-configuration)). If none exists, creates an `edq-config.json` in the current directory and writes to it. For reading options: loads local configuration as described in [local configuration section](#local-configuration) | +|`--global` | For writing options: writes to the default global file path, or to the specified file if provided. For reading options: loads only global configuration from the default global file path, or from the specified file if provided. When `--help` is used, the exact default global file path for the current platform will be displayed under this flag. | +| `--local` | For writing options: writes to the first local file found (check local config section for [search order.](#local-configuration)). If none exists, creates an `edq-config.json` in the current directory and writes to it. For reading options: loads only local configuration as described in [local configuration section](#local-configuration) | |`--file` | For writing options: writes to the specified file. For reading options: loads CLI file config options from the specified file. | | `--config`/`-c`| For providing additional CLI configuration parameters when running any config command. | | `--help` | Displays a help message with detailed descriptions of each option. | From 1aed3d714a9c85c1f0f1fb2680ca74695f9f9373 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Wed, 3 Sep 2025 15:17:04 -0700 Subject: [PATCH 45/71] Adding CLI for config list. --- edq/cli/config/__init__.py | 0 edq/cli/config/list.py | 24 ++++++++++++++++ edq/core/argparser.py | 2 ++ edq/core/config.py | 58 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+) create mode 100644 edq/cli/config/__init__.py create mode 100644 edq/cli/config/list.py diff --git a/edq/cli/config/__init__.py b/edq/cli/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/edq/cli/config/list.py b/edq/cli/config/list.py new file mode 100644 index 0000000..ef16023 --- /dev/null +++ b/edq/cli/config/list.py @@ -0,0 +1,24 @@ +import sys + +import edq.core.argparser + +DESCRIPTION = "List your current configuration options." + +def run(args): + config_list = [] + for (key, value) in args._config.items(): + config_str = f"{key}\t{value}" + if (args.show_origin): + config_source_obj = args._config_sources.get(key) + config_str += f"\t{config_source_obj.path}" + + config_list.append(config_str) + + print("\n".join(config_list)) + return 0 + +def main(): + return run(edq.core.argparser.get_default_parser(DESCRIPTION).parse_args()) + +if (__name__ == '__main__'): + sys.exit(main()) \ No newline at end of file diff --git a/edq/core/argparser.py b/edq/core/argparser.py index f14ab20..1f5b121 100644 --- a/edq/core/argparser.py +++ b/edq/core/argparser.py @@ -11,6 +11,7 @@ import typing import edq.core.log +import edq.core.config @typing.runtime_checkable class PreParseFunction(typing.Protocol): @@ -108,5 +109,6 @@ def get_default_parser(description: str) -> Parser: parser = Parser(description = description) parser.register_callbacks('log', edq.core.log.set_cli_args, edq.core.log.init_from_args) + parser.register_callbacks('config', edq.core.config.set_cli_args, edq.core.config.config_from_parsed_args) return parser diff --git a/edq/core/config.py b/edq/core/config.py index 0881165..522d0ca 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -14,6 +14,7 @@ CONFIG_PATHS_KEY: str = 'config_paths' DEFAULT_CONFIG_FILENAME: str = "edq-config.json" +DEFAULT_GLOBAL_CONFIG_PATH: str = platformdirs.user_config_dir(DEFAULT_CONFIG_FILENAME) class ConfigSource: """ A class for storing config source information. """ @@ -175,3 +176,60 @@ def _get_ancestor_config_file_path( current_directory = parent_dir return None + +def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, typing.Any]) -> None: + """ + Set common CLI arguments for configuration. + """ + + parser.add_argument('--config-file', dest = 'config_paths', + action = 'append', type = str, default = None, + help = "A JSON config file with your submission/authentication details. " + + "Can be specified multiple times with later values overriding earlier ones. " + + "Config values can be specified in multiple places" + + "(with later values overriding earlier values): " + + f"First '{DEFAULT_GLOBAL_CONFIG_PATH}' or a global config specified with --config-global, " + + f"then '{DEFAULT_CONFIG_FILENAME}' in the current directory or the path to root. " + + "Then any files specified using --config-file in the order they were specified, " + + "and finally any variables specified directly on the command line either directly as a dedicated CLI flag (like --user) " + + "or as a configuration key-value pair to the CLI’s --config option." + ) + + parser.add_argument("--show-origin", dest = 'show_origin', + action = 'store_true', help = "Shows where each configuration's value was obtained from.") + + parser.add_argument('--config-global', dest = 'global_config_path', + action = 'store', type = str, default = DEFAULT_GLOBAL_CONFIG_PATH , + help = 'Path to the global configuration file (default: %(default)s).') + + parser.add_argument('--config', dest = 'config', + action = 'append', type = str, default = None, + help = "Specify configuration options as key=value pairs. " + + "Multiple options can be specified, values set here override values from config files." + ) + +def config_from_parsed_args( + parser: argparse.ArgumentParser, + args: argparse.Namespace, + extra_state: typing.Dict[str, typing.Any]) -> None: + """ + Take in args from a parser that was passed to set_cli_args(), + and gets the tired configurations with the appropriate parameters, and attaches it to args + """ + + (config_dict, sources_dict) = get_tiered_config( + global_config_path = args.global_config_path, + cli_arguments = args, + skip_keys = [ + 'local', + 'show_origin', + edq.core.config.CONFIG_PATHS_KEY, + 'global_config_path', + 'log_level', + 'quiet', + 'debug' + ] + ) + + setattr(args, "_config", config_dict) + setattr(args, "_config_sources", sources_dict) From 7e73dd72d4667fbd835ded8ecae66f640387631a Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 5 Sep 2025 17:13:45 -0700 Subject: [PATCH 46/71] Renamed the functions for list cli. --- edq/cli/config/list.py | 18 ++++++++++++++---- edq/cli/version.py | 1 + 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/edq/cli/config/list.py b/edq/cli/config/list.py index ef16023..818910c 100644 --- a/edq/cli/config/list.py +++ b/edq/cli/config/list.py @@ -1,10 +1,13 @@ +import argparse import sys import edq.core.argparser DESCRIPTION = "List your current configuration options." -def run(args): +def run_cli(args: argparse.Namespace) -> int: + """ Run the CLI. """ + config_list = [] for (key, value) in args._config.items(): config_str = f"{key}\t{value}" @@ -17,8 +20,15 @@ def run(args): print("\n".join(config_list)) return 0 -def main(): - return run(edq.core.argparser.get_default_parser(DESCRIPTION).parse_args()) +def main() -> int: + """ Get a parser, parse the args, and call run. """ + + return run_cli(_get_parser().parse_args()) + +def _get_parser() -> edq.core.argparser.Parser: + """ Get the parser. """ + + return edq.core.argparser.get_default_parser(DESCRIPTION) if (__name__ == '__main__'): - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/edq/cli/version.py b/edq/cli/version.py index afa7115..2c93172 100644 --- a/edq/cli/version.py +++ b/edq/cli/version.py @@ -16,6 +16,7 @@ def run_cli(args: argparse.Namespace) -> int: def main() -> int: """ Get a parser, parse the args, and call run. """ + return run_cli(_get_parser().parse_args()) def _get_parser() -> edq.core.argparser.Parser: From 6273d49d4269456618b9409c3c01fe37415dc133 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sat, 6 Sep 2025 13:46:49 -0700 Subject: [PATCH 47/71] Adding cli tests. --- edq/cli/config/list.py | 11 +++++- edq/core/config.py | 37 ++++++++----------- edq/testing/cli.py | 2 + .../testdata/cli/data/config/empty.json | 1 + .../testdata/cli/data/config/simple.json | 3 ++ .../testdata/cli/tests/config_list_base.txt | 9 +++++ .../cli/tests/config_list_sources.txt | 10 +++++ 7 files changed, 50 insertions(+), 23 deletions(-) create mode 100644 edq/testing/testdata/cli/data/config/empty.json create mode 100644 edq/testing/testdata/cli/data/config/simple.json create mode 100644 edq/testing/testdata/cli/tests/config_list_base.txt create mode 100644 edq/testing/testdata/cli/tests/config_list_sources.txt diff --git a/edq/cli/config/list.py b/edq/cli/config/list.py index 818910c..8c2ec79 100644 --- a/edq/cli/config/list.py +++ b/edq/cli/config/list.py @@ -26,9 +26,16 @@ def main() -> int: return run_cli(_get_parser().parse_args()) def _get_parser() -> edq.core.argparser.Parser: - """ Get the parser. """ + """ Get the parser and add addition flags. """ - return edq.core.argparser.get_default_parser(DESCRIPTION) + parser = edq.core.argparser.get_default_parser(DESCRIPTION) + + parser.add_argument("--show-origin", dest = 'show_origin', + action = 'store_true', + help = "Display where each configuration's value was obtained from.", + ) + + return parser if (__name__ == '__main__'): sys.exit(main()) diff --git a/edq/core/config.py b/edq/core/config.py index 56424dd..d98b70a 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -182,39 +182,33 @@ def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, Set common CLI arguments for configuration. """ + parser.add_argument('--config-global', dest = 'global_config_path', + action = 'store', type = str, default = DEFAULT_GLOBAL_CONFIG_PATH, + help = 'Override the default global config file path (default: %(default)s).', + ) + parser.add_argument('--config-file', dest = 'config_paths', action = 'append', type = str, default = None, - help = "A JSON config file with your submission/authentication details. " - + "Can be specified multiple times with later values overriding earlier ones. " - + "Config values can be specified in multiple places" - + "(with later values overriding earlier values): " - + f"First '{DEFAULT_GLOBAL_CONFIG_PATH}' or a global config specified with --config-global, " - + f"then '{DEFAULT_CONFIG_FILENAME}' in the current directory or the path to root. " - + "Then any files specified using --config-file in the order they were specified, " - + "and finally any variables specified directly on the command line either directly as a dedicated CLI flag (like --user) " - + "or as a configuration key-value pair to the CLI’s --config option." + help = "Load config options from a JSON file. " + + "When this flag is given multiple times files are applied in the order provided, and later files override earlier ones. " + + "This will override options form both global and local configs." ) - parser.add_argument("--show-origin", dest = 'show_origin', - action = 'store_true', help = "Shows where each configuration's value was obtained from.") - - parser.add_argument('--config-global', dest = 'global_config_path', - action = 'store', type = str, default = DEFAULT_GLOBAL_CONFIG_PATH, - help = 'Path to the global configuration file (default: %(default)s).') - parser.add_argument('--config', dest = 'config', action = 'append', type = str, default = None, - help = "Specify configuration options as key=value pairs. " - + "Multiple options can be specified, values set here override values from config files." + help = "Provide additional options to a CLI command. " + + "Specify configuration options as = pairs. " + + "This will override options form all config files." ) def config_from_parsed_args( parser: argparse.ArgumentParser, args: argparse.Namespace, - extra_state: typing.Dict[str, typing.Any]) -> None: + extra_state: typing.Dict[str, typing.Any] + ) -> None: """ Take in args from a parser that was passed to set_cli_args(), - and gets the tired configurations with the appropriate parameters, and attaches it to args + and gets the tired configurations with the appropriate parameters, and attaches it to args. """ (config_dict, sources_dict) = get_tiered_config( @@ -227,7 +221,8 @@ def config_from_parsed_args( 'global_config_path', 'log_level', 'quiet', - 'debug' + 'debug', + 'config', ] ) diff --git a/edq/testing/cli.py b/edq/testing/cli.py index f0ef34a..44f72e2 100644 --- a/edq/testing/cli.py +++ b/edq/testing/cli.py @@ -30,6 +30,7 @@ TEST_CASE_SEP: str = '---' DATA_DIR_ID: str = '__DATA_DIR__' +ABS_DATA_DIR_ID: str = '__ABS_DATA_DIR__' TEMP_DIR_ID: str = '__TEMP_DIR__' BASE_DIR_ID: str = '__BASE_DIR__' @@ -175,6 +176,7 @@ def _expand_paths(self, text: str) -> str: (DATA_DIR_ID, self.data_dir), (TEMP_DIR_ID, self.temp_dir), (BASE_DIR_ID, self.base_dir), + (ABS_DATA_DIR_ID, os.path.abspath(self.data_dir)) ] for (key, target_dir) in replacements: diff --git a/edq/testing/testdata/cli/data/config/empty.json b/edq/testing/testdata/cli/data/config/empty.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/edq/testing/testdata/cli/data/config/empty.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/edq/testing/testdata/cli/data/config/simple.json b/edq/testing/testdata/cli/data/config/simple.json new file mode 100644 index 0000000..f5ef5fd --- /dev/null +++ b/edq/testing/testdata/cli/data/config/simple.json @@ -0,0 +1,3 @@ +{ + "user": "user@test.edulinq.org", +} \ No newline at end of file diff --git a/edq/testing/testdata/cli/tests/config_list_base.txt b/edq/testing/testdata/cli/tests/config_list_base.txt new file mode 100644 index 0000000..ac07611 --- /dev/null +++ b/edq/testing/testdata/cli/tests/config_list_base.txt @@ -0,0 +1,9 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-file", "__DATA_DIR__(config/simple.json)", + "--config-global", "__DATA_DIR__(config/empty.json)" + ] +} +--- +user user@test.edulinq.org diff --git a/edq/testing/testdata/cli/tests/config_list_sources.txt b/edq/testing/testdata/cli/tests/config_list_sources.txt new file mode 100644 index 0000000..8a673da --- /dev/null +++ b/edq/testing/testdata/cli/tests/config_list_sources.txt @@ -0,0 +1,10 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-file", "__DATA_DIR__(config/simple.json)", + "--config-global", "__DATA_DIR__(config/empty.json)", + "--show-origin", + ] +} +--- +user user@test.edulinq.org __ABS_DATA_DIR__(config/simple.json) From 2ecf78bbef3ecc6421977e298624cc68007124b8 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sat, 6 Sep 2025 13:48:36 -0700 Subject: [PATCH 48/71] Fixed import order. --- edq/core/argparser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edq/core/argparser.py b/edq/core/argparser.py index 1f5b121..0e9e2ce 100644 --- a/edq/core/argparser.py +++ b/edq/core/argparser.py @@ -10,8 +10,8 @@ import argparse import typing -import edq.core.log import edq.core.config +import edq.core.log @typing.runtime_checkable class PreParseFunction(typing.Protocol): From 1428e59b3ab2fe55c945f6d2d4d892e4808e2dd0 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sat, 6 Sep 2025 14:57:50 -0700 Subject: [PATCH 49/71] Added --config flag functionality to get_tierd_config(). --- edq/core/config.py | 34 ++++++++---------------- edq/core/config_test.py | 58 +++++++++++++++++------------------------ 2 files changed, 35 insertions(+), 57 deletions(-) diff --git a/edq/core/config.py b/edq/core/config.py index d98b70a..6788a23 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -13,6 +13,7 @@ CONFIG_SOURCE_CLI_BARE: str = "" CONFIG_PATHS_KEY: str = 'config_paths' +CONFIGS_KEY: str = 'config' DEFAULT_CONFIG_FILENAME: str = "edq-config.json" DEFAULT_GLOBAL_CONFIG_PATH: str = platformdirs.user_config_dir(DEFAULT_CONFIG_FILENAME) @@ -39,7 +40,6 @@ def get_tiered_config( config_file_name: str = DEFAULT_CONFIG_FILENAME, legacy_config_file_name: typing.Union[str, None] = None, global_config_path: typing.Union[str, None] = None, - skip_keys: typing.Union[list, None] = None, cli_arguments: typing.Union[dict, argparse.Namespace, None] = None, local_config_root_cutoff: typing.Union[str, None] = None, ) -> typing.Tuple[typing.Dict[str, str], typing.Dict[str, ConfigSource]]: @@ -51,9 +51,6 @@ def get_tiered_config( if (global_config_path is None): global_config_path = platformdirs.user_config_dir(config_file_name) - if (skip_keys is None): - skip_keys = [CONFIG_PATHS_KEY] - if (cli_arguments is None): cli_arguments = {} @@ -79,21 +76,22 @@ def get_tiered_config( _load_config_file(local_config_path, config, sources, CONFIG_SOURCE_LOCAL) # Check the config file specified on the command-line. - config_paths = cli_arguments.get(CONFIG_PATHS_KEY, []) + config_paths = cli_arguments.get(CONFIG_PATHS_KEY, None) if (config_paths is not None): for path in config_paths: _load_config_file(path, config, sources, CONFIG_SOURCE_CLI) - # Finally, any command-line options. - for (key, value) in cli_arguments.items(): - if (key in skip_keys): - continue + # Finally, any command-line config options. + cli_configs = cli_arguments.get(CONFIGS_KEY, None) + if (cli_configs is not None): + for cli_config in cli_configs: + (key, value) = cli_config.split("=") - if ((value is None) or (value == '')): - continue + if ((value is None) or (value == '')): + continue - config[key] = value - sources[key] = ConfigSource(label = CONFIG_SOURCE_CLI_BARE) + config[key] = value + sources[key] = ConfigSource(label = CONFIG_SOURCE_CLI_BARE) return config, sources @@ -214,16 +212,6 @@ def config_from_parsed_args( (config_dict, sources_dict) = get_tiered_config( global_config_path = args.global_config_path, cli_arguments = args, - skip_keys = [ - 'local', - 'show_origin', - edq.core.config.CONFIG_PATHS_KEY, - 'global_config_path', - 'log_level', - 'quiet', - 'debug', - 'config', - ] ) setattr(args, "_config", config_dict) diff --git a/edq/core/config_test.py b/edq/core/config_test.py index 916ca2d..ea177c0 100644 --- a/edq/core/config_test.py +++ b/edq/core/config_test.py @@ -416,29 +416,10 @@ def test_get_tiered_config_base(self): "empty-dir", { "cli_arguments": { - "user": "user@test.edulinq.org", - }, - }, - { - "user": "user@test.edulinq.org", - }, - { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), - }, - None, - ), - - # Skip keys functionally. - ( - "empty-dir", - { - "cli_arguments": { - "user": "user@test.edulinq.org", - "pass": "user", + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], }, - "skip_keys": [ - "pass", - ], }, { "user": "user@test.edulinq.org", @@ -498,7 +479,9 @@ def test_get_tiered_config_base(self): { "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { - "user": "user@test.edulinq.org", + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], }, }, { @@ -537,7 +520,9 @@ def test_get_tiered_config_base(self): "simple", { "cli_arguments": { - "user": "user@test.edulinq.org", + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], }, }, { @@ -554,7 +539,9 @@ def test_get_tiered_config_base(self): "empty-dir", { "cli_arguments": { - "user": "user@test.edulinq.org", + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ], @@ -575,7 +562,9 @@ def test_get_tiered_config_base(self): { "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { - "user": "user@test.edulinq.org", + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ], @@ -596,7 +585,9 @@ def test_get_tiered_config_base(self): { "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { - "user": "user@test.edulinq.org", + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], }, }, { @@ -636,7 +627,9 @@ def test_get_tiered_config_base(self): "simple", { "cli_arguments": { - "user": "user@test.edulinq.org", + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ], @@ -660,13 +653,10 @@ def test_get_tiered_config_base(self): edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ], - "pass": "user", - "server": "http://test.edulinq.org", + edq.core.config.CONFIGS_KEY: [ + "pass=user", + ], }, - "skip_keys": [ - "server", - edq.core.config.CONFIG_PATHS_KEY, - ], }, { "user": "user@test.edulinq.org", From e57b66224f714f8730be3deda8e91d6f11c82dd4 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sat, 6 Sep 2025 15:21:40 -0700 Subject: [PATCH 50/71] Added more CLI tests for config list. --- .../empty.json => configs/empty/edq-config.json} | 0 .../simple.json => configs/simple/edq-config.json} | 0 .../testdata/cli/tests/config/config_list_base.txt | 9 +++++++++ .../cli/tests/config/config_list_cli_config.txt | 9 +++++++++ .../cli/tests/config/config_list_sources.txt | 12 ++++++++++++ edq/testing/testdata/cli/tests/config_list_base.txt | 9 --------- .../testdata/cli/tests/config_list_sources.txt | 10 ---------- 7 files changed, 30 insertions(+), 19 deletions(-) rename edq/testing/testdata/cli/data/{config/empty.json => configs/empty/edq-config.json} (100%) rename edq/testing/testdata/cli/data/{config/simple.json => configs/simple/edq-config.json} (100%) create mode 100644 edq/testing/testdata/cli/tests/config/config_list_base.txt create mode 100644 edq/testing/testdata/cli/tests/config/config_list_cli_config.txt create mode 100644 edq/testing/testdata/cli/tests/config/config_list_sources.txt delete mode 100644 edq/testing/testdata/cli/tests/config_list_base.txt delete mode 100644 edq/testing/testdata/cli/tests/config_list_sources.txt diff --git a/edq/testing/testdata/cli/data/config/empty.json b/edq/testing/testdata/cli/data/configs/empty/edq-config.json similarity index 100% rename from edq/testing/testdata/cli/data/config/empty.json rename to edq/testing/testdata/cli/data/configs/empty/edq-config.json diff --git a/edq/testing/testdata/cli/data/config/simple.json b/edq/testing/testdata/cli/data/configs/simple/edq-config.json similarity index 100% rename from edq/testing/testdata/cli/data/config/simple.json rename to edq/testing/testdata/cli/data/configs/simple/edq-config.json diff --git a/edq/testing/testdata/cli/tests/config/config_list_base.txt b/edq/testing/testdata/cli/tests/config/config_list_base.txt new file mode 100644 index 0000000..7744cc8 --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/config_list_base.txt @@ -0,0 +1,9 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-file", "__DATA_DIR__(configs/simple/edq-config.json)", + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + ] +} +--- +user user@test.edulinq.org diff --git a/edq/testing/testdata/cli/tests/config/config_list_cli_config.txt b/edq/testing/testdata/cli/tests/config/config_list_cli_config.txt new file mode 100644 index 0000000..71a9a08 --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/config_list_cli_config.txt @@ -0,0 +1,9 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config", "user=user@test.edulinq.org", + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + ] +} +--- +user user@test.edulinq.org diff --git a/edq/testing/testdata/cli/tests/config/config_list_sources.txt b/edq/testing/testdata/cli/tests/config/config_list_sources.txt new file mode 100644 index 0000000..4899bb2 --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/config_list_sources.txt @@ -0,0 +1,12 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-file", "__DATA_DIR__(configs/simple/edq-config.json)", + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + "--config", "pass=pass123", + "--show-origin", + ] +} +--- +user user@test.edulinq.org __ABS_DATA_DIR__(configs/simple/edq-config.json) +pass pass123 None diff --git a/edq/testing/testdata/cli/tests/config_list_base.txt b/edq/testing/testdata/cli/tests/config_list_base.txt deleted file mode 100644 index ac07611..0000000 --- a/edq/testing/testdata/cli/tests/config_list_base.txt +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cli": "edq.cli.config.list", - "arguments": [ - "--config-file", "__DATA_DIR__(config/simple.json)", - "--config-global", "__DATA_DIR__(config/empty.json)" - ] -} ---- -user user@test.edulinq.org diff --git a/edq/testing/testdata/cli/tests/config_list_sources.txt b/edq/testing/testdata/cli/tests/config_list_sources.txt deleted file mode 100644 index 8a673da..0000000 --- a/edq/testing/testdata/cli/tests/config_list_sources.txt +++ /dev/null @@ -1,10 +0,0 @@ -{ - "cli": "edq.cli.config.list", - "arguments": [ - "--config-file", "__DATA_DIR__(config/simple.json)", - "--config-global", "__DATA_DIR__(config/empty.json)", - "--show-origin", - ] -} ---- -user user@test.edulinq.org __ABS_DATA_DIR__(config/simple.json) From 39dabdb346b56400e2008eb3d118afb521fcf280 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sat, 6 Sep 2025 17:12:10 -0700 Subject: [PATCH 51/71] Revised for PR. --- edq/core/argparser.py | 2 +- edq/core/config.py | 15 ++++++++------- .../cli/data/configs/empty/edq-config.json | 2 +- .../cli/data/configs/simple/edq-config.json | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/edq/core/argparser.py b/edq/core/argparser.py index 0e9e2ce..60bf81a 100644 --- a/edq/core/argparser.py +++ b/edq/core/argparser.py @@ -109,6 +109,6 @@ def get_default_parser(description: str) -> Parser: parser = Parser(description = description) parser.register_callbacks('log', edq.core.log.set_cli_args, edq.core.log.init_from_args) - parser.register_callbacks('config', edq.core.config.set_cli_args, edq.core.config.config_from_parsed_args) + parser.register_callbacks('config', edq.core.config.set_cli_args, edq.core.config.attach_config_to_args) return parser diff --git a/edq/core/config.py b/edq/core/config.py index 6788a23..29551ba 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -76,13 +76,13 @@ def get_tiered_config( _load_config_file(local_config_path, config, sources, CONFIG_SOURCE_LOCAL) # Check the config file specified on the command-line. - config_paths = cli_arguments.get(CONFIG_PATHS_KEY, None) + config_paths = cli_arguments.get(CONFIG_PATHS_KEY, []) if (config_paths is not None): for path in config_paths: _load_config_file(path, config, sources, CONFIG_SOURCE_CLI) # Finally, any command-line config options. - cli_configs = cli_arguments.get(CONFIGS_KEY, None) + cli_configs = cli_arguments.get(CONFIGS_KEY, []) if (cli_configs is not None): for cli_config in cli_configs: (key, value) = cli_config.split("=") @@ -188,25 +188,26 @@ def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, parser.add_argument('--config-file', dest = 'config_paths', action = 'append', type = str, default = None, help = "Load config options from a JSON file. " - + "When this flag is given multiple times files are applied in the order provided, and later files override earlier ones. " + + "This flag can be specified multiple times. " + + "Files are applied in the order provided and later files override earlier ones. " + "This will override options form both global and local configs." ) parser.add_argument('--config', dest = 'config', action = 'append', type = str, default = None, - help = "Provide additional options to a CLI command. " - + "Specify configuration options as = pairs. " + help = "Provide configuration options to a CLI command. " + + "Specify options as = pairs. " + "This will override options form all config files." ) -def config_from_parsed_args( +def attach_config_to_args( parser: argparse.ArgumentParser, args: argparse.Namespace, extra_state: typing.Dict[str, typing.Any] ) -> None: """ Take in args from a parser that was passed to set_cli_args(), - and gets the tired configurations with the appropriate parameters, and attaches it to args. + and get the tired configuration with the appropriate parameters, and attache it to args. """ (config_dict, sources_dict) = get_tiered_config( diff --git a/edq/testing/testdata/cli/data/configs/empty/edq-config.json b/edq/testing/testdata/cli/data/configs/empty/edq-config.json index 9e26dfe..0967ef4 100644 --- a/edq/testing/testdata/cli/data/configs/empty/edq-config.json +++ b/edq/testing/testdata/cli/data/configs/empty/edq-config.json @@ -1 +1 @@ -{} \ No newline at end of file +{} diff --git a/edq/testing/testdata/cli/data/configs/simple/edq-config.json b/edq/testing/testdata/cli/data/configs/simple/edq-config.json index f5ef5fd..f619907 100644 --- a/edq/testing/testdata/cli/data/configs/simple/edq-config.json +++ b/edq/testing/testdata/cli/data/configs/simple/edq-config.json @@ -1,3 +1,3 @@ { "user": "user@test.edulinq.org", -} \ No newline at end of file +} From 815f904221255bcbf0671d1dfadf580c2bc51274 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sat, 6 Sep 2025 17:29:50 -0700 Subject: [PATCH 52/71] Revised it one last time before the PR. --- edq/cli/config/list.py | 2 +- edq/core/config.py | 10 +++---- edq/core/config_test.py | 30 +++++++++---------- edq/testing/cli.py | 2 +- .../cli/tests/config/config_list_base.txt | 2 +- .../tests/config/config_list_cli_config.txt | 2 +- .../cli/tests/config/config_list_sources.txt | 2 +- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/edq/cli/config/list.py b/edq/cli/config/list.py index 8c2ec79..3635cfb 100644 --- a/edq/cli/config/list.py +++ b/edq/cli/config/list.py @@ -3,7 +3,7 @@ import edq.core.argparser -DESCRIPTION = "List your current configuration options." +DESCRIPTION = "List current configuration options." def run_cli(args: argparse.Namespace) -> int: """ Run the CLI. """ diff --git a/edq/core/config.py b/edq/core/config.py index 29551ba..1a32ac4 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -9,8 +9,8 @@ CONFIG_SOURCE_GLOBAL: str = "" CONFIG_SOURCE_LOCAL: str = "" -CONFIG_SOURCE_CLI: str = "" -CONFIG_SOURCE_CLI_BARE: str = "" +CONFIG_SOURCE_CLI_FILE: str = "" +CONFIG_SOURCE_CLI: str = "" CONFIG_PATHS_KEY: str = 'config_paths' CONFIGS_KEY: str = 'config' @@ -79,7 +79,7 @@ def get_tiered_config( config_paths = cli_arguments.get(CONFIG_PATHS_KEY, []) if (config_paths is not None): for path in config_paths: - _load_config_file(path, config, sources, CONFIG_SOURCE_CLI) + _load_config_file(path, config, sources, CONFIG_SOURCE_CLI_FILE) # Finally, any command-line config options. cli_configs = cli_arguments.get(CONFIGS_KEY, []) @@ -91,7 +91,7 @@ def get_tiered_config( continue config[key] = value - sources[key] = ConfigSource(label = CONFIG_SOURCE_CLI_BARE) + sources[key] = ConfigSource(label = CONFIG_SOURCE_CLI) return config, sources @@ -203,7 +203,7 @@ def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, def attach_config_to_args( parser: argparse.ArgumentParser, args: argparse.Namespace, - extra_state: typing.Dict[str, typing.Any] + extra_state: typing.Dict[str, typing.Any], ) -> None: """ Take in args from a parser that was passed to set_cli_args(), diff --git a/edq/core/config_test.py b/edq/core/config_test.py index ea177c0..faf5487 100644 --- a/edq/core/config_test.py +++ b/edq/core/config_test.py @@ -315,11 +315,11 @@ def test_get_tiered_config_base(self): }, { "user": edq.core.config.ConfigSource( - label = edq.core.config.CONFIG_SOURCE_CLI, + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ), "server": edq.core.config.ConfigSource( - label = edq.core.config.CONFIG_SOURCE_CLI, + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, path = os.path.join(temp_dir, "nested", edq.core.config.DEFAULT_CONFIG_FILENAME), ), }, @@ -342,7 +342,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.core.config.ConfigSource( - label = edq.core.config.CONFIG_SOURCE_CLI, + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ), }, @@ -425,7 +425,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), @@ -466,7 +466,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.core.config.ConfigSource( - label = edq.core.config.CONFIG_SOURCE_CLI, + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ), }, @@ -488,7 +488,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), @@ -508,7 +508,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.core.config.ConfigSource( - label = edq.core.config.CONFIG_SOURCE_CLI, + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ), }, @@ -529,7 +529,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), @@ -551,7 +551,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), @@ -574,7 +574,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), @@ -594,7 +594,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), @@ -615,7 +615,7 @@ def test_get_tiered_config_base(self): }, { "user": edq.core.config.ConfigSource( - label = edq.core.config.CONFIG_SOURCE_CLI, + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ), }, @@ -639,7 +639,7 @@ def test_get_tiered_config_base(self): "user": "user@test.edulinq.org", }, { - "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), @@ -664,10 +664,10 @@ def test_get_tiered_config_base(self): }, { "user": edq.core.config.ConfigSource( - label = edq.core.config.CONFIG_SOURCE_CLI, + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, path = os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ), - "pass": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI_BARE), + "pass": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), }, None, ), diff --git a/edq/testing/cli.py b/edq/testing/cli.py index 44f72e2..a8350af 100644 --- a/edq/testing/cli.py +++ b/edq/testing/cli.py @@ -176,7 +176,7 @@ def _expand_paths(self, text: str) -> str: (DATA_DIR_ID, self.data_dir), (TEMP_DIR_ID, self.temp_dir), (BASE_DIR_ID, self.base_dir), - (ABS_DATA_DIR_ID, os.path.abspath(self.data_dir)) + (ABS_DATA_DIR_ID, os.path.abspath(self.data_dir)), ] for (key, target_dir) in replacements: diff --git a/edq/testing/testdata/cli/tests/config/config_list_base.txt b/edq/testing/testdata/cli/tests/config/config_list_base.txt index 7744cc8..81ab63b 100644 --- a/edq/testing/testdata/cli/tests/config/config_list_base.txt +++ b/edq/testing/testdata/cli/tests/config/config_list_base.txt @@ -3,7 +3,7 @@ "arguments": [ "--config-file", "__DATA_DIR__(configs/simple/edq-config.json)", "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", - ] + ], } --- user user@test.edulinq.org diff --git a/edq/testing/testdata/cli/tests/config/config_list_cli_config.txt b/edq/testing/testdata/cli/tests/config/config_list_cli_config.txt index 71a9a08..a30f548 100644 --- a/edq/testing/testdata/cli/tests/config/config_list_cli_config.txt +++ b/edq/testing/testdata/cli/tests/config/config_list_cli_config.txt @@ -3,7 +3,7 @@ "arguments": [ "--config", "user=user@test.edulinq.org", "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", - ] + ], } --- user user@test.edulinq.org diff --git a/edq/testing/testdata/cli/tests/config/config_list_sources.txt b/edq/testing/testdata/cli/tests/config/config_list_sources.txt index 4899bb2..07b7b2d 100644 --- a/edq/testing/testdata/cli/tests/config/config_list_sources.txt +++ b/edq/testing/testdata/cli/tests/config/config_list_sources.txt @@ -5,7 +5,7 @@ "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", "--config", "pass=pass123", "--show-origin", - ] + ], } --- user user@test.edulinq.org __ABS_DATA_DIR__(configs/simple/edq-config.json) From e07f88707976b28659eda95565baac05d7b27994 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sat, 6 Sep 2025 17:49:07 -0700 Subject: [PATCH 53/71] Corrected an inaccruacy on a comment. --- edq/cli/config/list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edq/cli/config/list.py b/edq/cli/config/list.py index 3635cfb..673c58c 100644 --- a/edq/cli/config/list.py +++ b/edq/cli/config/list.py @@ -26,7 +26,7 @@ def main() -> int: return run_cli(_get_parser().parse_args()) def _get_parser() -> edq.core.argparser.Parser: - """ Get the parser and add addition flags. """ + """ Get a parser and add addition flags. """ parser = edq.core.argparser.get_default_parser(DESCRIPTION) From 77476bf30276d8db1ed884e78aff38056444f416 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Mon, 8 Sep 2025 06:20:58 -0700 Subject: [PATCH 54/71] Made CLI flags descriptions better. --- edq/core/config.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/edq/core/config.py b/edq/core/config.py index 1a32ac4..39e28ad 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -12,7 +12,7 @@ CONFIG_SOURCE_CLI_FILE: str = "" CONFIG_SOURCE_CLI: str = "" -CONFIG_PATHS_KEY: str = 'config_paths' +CONFIG_PATHS_KEY: str = 'config_path' CONFIGS_KEY: str = 'config' DEFAULT_CONFIG_FILENAME: str = "edq-config.json" DEFAULT_GLOBAL_CONFIG_PATH: str = platformdirs.user_config_dir(DEFAULT_CONFIG_FILENAME) @@ -185,18 +185,20 @@ def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, help = 'Override the default global config file path (default: %(default)s).', ) - parser.add_argument('--config-file', dest = 'config_paths', + parser.add_argument('--config-file', dest = CONFIG_PATHS_KEY, action = 'append', type = str, default = None, help = "Load config options from a JSON file. " + "This flag can be specified multiple times. " + "Files are applied in the order provided and later files override earlier ones. " - + "This will override options form both global and local configs." + + "This will override options form both global and local config files." ) - parser.add_argument('--config', dest = 'config', + parser.add_argument('--config', dest = CONFIGS_KEY, action = 'append', type = str, default = None, - help = "Provide configuration options to a CLI command. " + help = "Load configuration options from the CLI command. " + "Specify options as = pairs. " + + "This flag can be specified multiple times. " + + "The options are applied in the order provided and later options override earlier ones. " + "This will override options form all config files." ) From 8257ca8b9461ceb70f6467976a7ddf8545537138 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Tue, 9 Sep 2025 20:01:12 -0700 Subject: [PATCH 55/71] Reviewed 1st pass. --- edq/cli/config/list.py | 28 +++++++++--- edq/core/config.py | 43 ++++++++++--------- .../cli/tests/config/config_list_base.txt | 5 ++- .../cli/tests/config/config_list_empty.txt | 8 ++++ ...config.txt => config_list_skip_header.txt} | 3 +- .../cli/tests/config/config_list_sources.txt | 3 +- 6 files changed, 60 insertions(+), 30 deletions(-) create mode 100644 edq/testing/testdata/cli/tests/config/config_list_empty.txt rename edq/testing/testdata/cli/tests/config/{config_list_cli_config.txt => config_list_skip_header.txt} (89%) diff --git a/edq/cli/config/list.py b/edq/cli/config/list.py index 673c58c..fe53cbb 100644 --- a/edq/cli/config/list.py +++ b/edq/cli/config/list.py @@ -1,21 +1,32 @@ +""" +List current configuration options. +""" + import argparse import sys import edq.core.argparser -DESCRIPTION = "List current configuration options." - def run_cli(args: argparse.Namespace) -> int: """ Run the CLI. """ config_list = [] for (key, value) in args._config.items(): - config_str = f"{key}\t{value}" + config = [key, value] if (args.show_origin): config_source_obj = args._config_sources.get(key) - config_str += f"\t{config_source_obj.path}" - config_list.append(config_str) + if (config_source_obj.path is None): + config.append(config_source_obj.label) + else: + config.append(config_source_obj.path) + + config_list.append("\t".join(config)) + + if (not args.skip_header): + config_list.insert(0, "Key\tValue") + if (args.show_origin): + config_list[0] = config_list[0] + "\tOrigin" print("\n".join(config_list)) return 0 @@ -28,13 +39,18 @@ def main() -> int: def _get_parser() -> edq.core.argparser.Parser: """ Get a parser and add addition flags. """ - parser = edq.core.argparser.get_default_parser(DESCRIPTION) + parser = edq.core.argparser.get_default_parser(__doc__.strip()) parser.add_argument("--show-origin", dest = 'show_origin', action = 'store_true', help = "Display where each configuration's value was obtained from.", ) + parser.add_argument("--skip-header", dest = 'skip_header', + action = 'store_true', + help = "Skip headers when displaying configs.", + ) + return parser if (__name__ == '__main__'): diff --git a/edq/core/config.py b/edq/core/config.py index 39e28ad..b03c26b 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -77,21 +77,22 @@ def get_tiered_config( # Check the config file specified on the command-line. config_paths = cli_arguments.get(CONFIG_PATHS_KEY, []) - if (config_paths is not None): - for path in config_paths: - _load_config_file(path, config, sources, CONFIG_SOURCE_CLI_FILE) + for path in config_paths: + _load_config_file(path, config, sources, CONFIG_SOURCE_CLI_FILE) # Finally, any command-line config options. cli_configs = cli_arguments.get(CONFIGS_KEY, []) - if (cli_configs is not None): - for cli_config in cli_configs: - (key, value) = cli_config.split("=") + for cli_config in cli_configs: + if ("=" not in cli_config): + raise ValueError("The provided config option does not match the expected format.") - if ((value is None) or (value == '')): - continue + (key, value) = cli_config.split("=", maxsplit = 1) - config[key] = value - sources[key] = ConfigSource(label = CONFIG_SOURCE_CLI) + if (value == ''): + continue + + config[key] = value + sources[key] = ConfigSource(label = CONFIG_SOURCE_CLI) return config, sources @@ -186,20 +187,20 @@ def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, ) parser.add_argument('--config-file', dest = CONFIG_PATHS_KEY, - action = 'append', type = str, default = None, - help = "Load config options from a JSON file. " - + "This flag can be specified multiple times. " - + "Files are applied in the order provided and later files override earlier ones. " - + "This will override options form both global and local config files." + action = 'append', type = str, default = [], + help = ("Load config options from a JSON file." + + " This flag can be specified multiple times." + + " Files are applied in the order provided and later files override earlier ones." + + " This will override options form both global and local config files.") ) parser.add_argument('--config', dest = CONFIGS_KEY, - action = 'append', type = str, default = None, - help = "Load configuration options from the CLI command. " - + "Specify options as = pairs. " - + "This flag can be specified multiple times. " - + "The options are applied in the order provided and later options override earlier ones. " - + "This will override options form all config files." + action = 'append', type = str, default = [], + help = ("Load configuration options from the CLI command." + + " Specify options as = pairs. " + + " This flag can be specified multiple times." + + " The options are applied in the order provided and later options override earlier ones." + + " This will override options form all config files.") ) def attach_config_to_args( diff --git a/edq/testing/testdata/cli/tests/config/config_list_base.txt b/edq/testing/testdata/cli/tests/config/config_list_base.txt index 81ab63b..38da9e7 100644 --- a/edq/testing/testdata/cli/tests/config/config_list_base.txt +++ b/edq/testing/testdata/cli/tests/config/config_list_base.txt @@ -1,9 +1,12 @@ { "cli": "edq.cli.config.list", "arguments": [ - "--config-file", "__DATA_DIR__(configs/simple/edq-config.json)", "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + "--config-file", "__DATA_DIR__(configs/simple/edq-config.json)", + "--config", "pass=pass123", ], } --- +Key Value user user@test.edulinq.org +pass pass123 diff --git a/edq/testing/testdata/cli/tests/config/config_list_empty.txt b/edq/testing/testdata/cli/tests/config/config_list_empty.txt new file mode 100644 index 0000000..b371d86 --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/config_list_empty.txt @@ -0,0 +1,8 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + ], +} +--- +Key Value diff --git a/edq/testing/testdata/cli/tests/config/config_list_cli_config.txt b/edq/testing/testdata/cli/tests/config/config_list_skip_header.txt similarity index 89% rename from edq/testing/testdata/cli/tests/config/config_list_cli_config.txt rename to edq/testing/testdata/cli/tests/config/config_list_skip_header.txt index a30f548..64d3b8a 100644 --- a/edq/testing/testdata/cli/tests/config/config_list_cli_config.txt +++ b/edq/testing/testdata/cli/tests/config/config_list_skip_header.txt @@ -1,8 +1,9 @@ { "cli": "edq.cli.config.list", "arguments": [ - "--config", "user=user@test.edulinq.org", "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + "--config", "user=user@test.edulinq.org", + "--skip-header", ], } --- diff --git a/edq/testing/testdata/cli/tests/config/config_list_sources.txt b/edq/testing/testdata/cli/tests/config/config_list_sources.txt index 07b7b2d..962e352 100644 --- a/edq/testing/testdata/cli/tests/config/config_list_sources.txt +++ b/edq/testing/testdata/cli/tests/config/config_list_sources.txt @@ -8,5 +8,6 @@ ], } --- +Key Value Origin user user@test.edulinq.org __ABS_DATA_DIR__(configs/simple/edq-config.json) -pass pass123 None +pass pass123 From d4495bbe529c1b426e54b3608b96164b59a0031c Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Wed, 10 Sep 2025 08:07:23 -0700 Subject: [PATCH 56/71] Reviwed the 1st pass one last time. --- edq/cli/config/list.py | 30 ++++++++++++++++-------------- edq/core/config.py | 18 +++++++++--------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/edq/cli/config/list.py b/edq/cli/config/list.py index fe53cbb..d029d72 100644 --- a/edq/cli/config/list.py +++ b/edq/cli/config/list.py @@ -10,25 +10,27 @@ def run_cli(args: argparse.Namespace) -> int: """ Run the CLI. """ - config_list = [] + configs_list = [] + + if (not args.skip_header): + header = "Key\tValue" + if (args.show_origin): + header = header + "\tOrigin" + configs_list.append(header) + for (key, value) in args._config.items(): - config = [key, value] + config_list = [key, value] if (args.show_origin): config_source_obj = args._config_sources.get(key) - if (config_source_obj.path is None): - config.append(config_source_obj.label) - else: - config.append(config_source_obj.path) + origin = config_source_obj.path + if (origin is None): + origin = config_source_obj.label + config_list.append(origin) - config_list.append("\t".join(config)) - - if (not args.skip_header): - config_list.insert(0, "Key\tValue") - if (args.show_origin): - config_list[0] = config_list[0] + "\tOrigin" + configs_list.append("\t".join(config_list)) - print("\n".join(config_list)) + print("\n".join(configs_list)) return 0 def main() -> int: @@ -48,7 +50,7 @@ def _get_parser() -> edq.core.argparser.Parser: parser.add_argument("--skip-header", dest = 'skip_header', action = 'store_true', - help = "Skip headers when displaying configs.", + help = 'Skip headers when displaying configs.', ) return parser diff --git a/edq/core/config.py b/edq/core/config.py index b03c26b..b411baa 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -188,19 +188,19 @@ def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, parser.add_argument('--config-file', dest = CONFIG_PATHS_KEY, action = 'append', type = str, default = [], - help = ("Load config options from a JSON file." - + " This flag can be specified multiple times." - + " Files are applied in the order provided and later files override earlier ones." - + " This will override options form both global and local config files.") + help = ('Load config options from a JSON file.' + + ' This flag can be specified multiple times.' + + ' Files are applied in the order provided and later files override earlier ones.' + + ' This will override options form both global and local config files.') ) parser.add_argument('--config', dest = CONFIGS_KEY, action = 'append', type = str, default = [], - help = ("Load configuration options from the CLI command." - + " Specify options as = pairs. " - + " This flag can be specified multiple times." - + " The options are applied in the order provided and later options override earlier ones." - + " This will override options form all config files.") + help = ('Load configuration options from the CLI command.' + + ' Specify options as = pairs. ' + + ' This flag can be specified multiple times.' + + ' The options are applied in the order provided and later options override earlier ones.' + + ' This will override options form all config files.') ) def attach_config_to_args( From cdf293c99efbdeaf844ad993182f8c6e23d2293a Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Wed, 10 Sep 2025 08:16:08 -0700 Subject: [PATCH 57/71] Changed the name of the post parsing function for config. --- edq/core/argparser.py | 2 +- edq/core/config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/edq/core/argparser.py b/edq/core/argparser.py index 60bf81a..ae8fb87 100644 --- a/edq/core/argparser.py +++ b/edq/core/argparser.py @@ -109,6 +109,6 @@ def get_default_parser(description: str) -> Parser: parser = Parser(description = description) parser.register_callbacks('log', edq.core.log.set_cli_args, edq.core.log.init_from_args) - parser.register_callbacks('config', edq.core.config.set_cli_args, edq.core.config.attach_config_to_args) + parser.register_callbacks('config', edq.core.config.set_cli_args, edq.core.config.load_config_into_args) return parser diff --git a/edq/core/config.py b/edq/core/config.py index b411baa..f91e71e 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -203,7 +203,7 @@ def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, + ' This will override options form all config files.') ) -def attach_config_to_args( +def load_config_into_args( parser: argparse.ArgumentParser, args: argparse.Namespace, extra_state: typing.Dict[str, typing.Any], From 61d69de23004269a2dd73223da0d0eaa22bad26f Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Wed, 10 Sep 2025 12:43:08 -0700 Subject: [PATCH 58/71] Revised 2nd pass. --- edq/cli/config/list.py | 20 ++++++++++++------- edq/core/config.py | 9 ++++++--- .../cli/tests/config/config_list_base.txt | 2 +- .../config_list_empty_config_key_error.txt | 10 ++++++++++ .../config/config_list_empty_config_value.txt | 10 ++++++++++ .../config_list_incorrect_config_error.txt | 10 ++++++++++ .../cli/tests/config/config_list_sources.txt | 2 +- 7 files changed, 51 insertions(+), 12 deletions(-) create mode 100644 edq/testing/testdata/cli/tests/config/config_list_empty_config_key_error.txt create mode 100644 edq/testing/testdata/cli/tests/config/config_list_empty_config_value.txt create mode 100644 edq/testing/testdata/cli/tests/config/config_list_incorrect_config_error.txt diff --git a/edq/cli/config/list.py b/edq/cli/config/list.py index d029d72..e474956 100644 --- a/edq/cli/config/list.py +++ b/edq/cli/config/list.py @@ -7,17 +7,13 @@ import edq.core.argparser +CONFIG_FIELD_SEPARATOR: str = "\t" + def run_cli(args: argparse.Namespace) -> int: """ Run the CLI. """ configs_list = [] - if (not args.skip_header): - header = "Key\tValue" - if (args.show_origin): - header = header + "\tOrigin" - configs_list.append(header) - for (key, value) in args._config.items(): config_list = [key, value] if (args.show_origin): @@ -26,9 +22,19 @@ def run_cli(args: argparse.Namespace) -> int: origin = config_source_obj.path if (origin is None): origin = config_source_obj.label + config_list.append(origin) - configs_list.append("\t".join(config_list)) + configs_list.append(CONFIG_FIELD_SEPARATOR.join(config_list)) + + configs_list.sort() + + if (not args.skip_header): + header = ["Key", "Value"] + if (args.show_origin): + header.append("Origin") + + configs_list.insert(0, (CONFIG_FIELD_SEPARATOR.join(header))) print("\n".join(configs_list)) return 0 diff --git a/edq/core/config.py b/edq/core/config.py index f91e71e..2b5f840 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -84,12 +84,15 @@ def get_tiered_config( cli_configs = cli_arguments.get(CONFIGS_KEY, []) for cli_config in cli_configs: if ("=" not in cli_config): - raise ValueError("The provided config option does not match the expected format.") + raise ValueError(f"The provided '{cli_config}' config option does not match the expected format.") (key, value) = cli_config.split("=", maxsplit = 1) - if (value == ''): - continue + key = key.strip() + value = value.strip() + + if (key == ""): + raise ValueError(f"The provided '{cli_config}' config option has an empty key.") config[key] = value sources[key] = ConfigSource(label = CONFIG_SOURCE_CLI) diff --git a/edq/testing/testdata/cli/tests/config/config_list_base.txt b/edq/testing/testdata/cli/tests/config/config_list_base.txt index 38da9e7..8739178 100644 --- a/edq/testing/testdata/cli/tests/config/config_list_base.txt +++ b/edq/testing/testdata/cli/tests/config/config_list_base.txt @@ -8,5 +8,5 @@ } --- Key Value -user user@test.edulinq.org pass pass123 +user user@test.edulinq.org diff --git a/edq/testing/testdata/cli/tests/config/config_list_empty_config_key_error.txt b/edq/testing/testdata/cli/tests/config/config_list_empty_config_key_error.txt new file mode 100644 index 0000000..4208176 --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/config_list_empty_config_key_error.txt @@ -0,0 +1,10 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + "--config", "=pass123", + ], + "error": "True" +} +--- +builtins.ValueError: The provided '=pass123' config option has an empty key. diff --git a/edq/testing/testdata/cli/tests/config/config_list_empty_config_value.txt b/edq/testing/testdata/cli/tests/config/config_list_empty_config_value.txt new file mode 100644 index 0000000..c55aa14 --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/config_list_empty_config_value.txt @@ -0,0 +1,10 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + "--config", "pass=", + ], +} +--- +Key Value +pass diff --git a/edq/testing/testdata/cli/tests/config/config_list_incorrect_config_error.txt b/edq/testing/testdata/cli/tests/config/config_list_incorrect_config_error.txt new file mode 100644 index 0000000..3c4d237 --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/config_list_incorrect_config_error.txt @@ -0,0 +1,10 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + "--config", "passpass123", + ], + "error": "True" +} +--- +builtins.ValueError: The provided 'passpass123' config option does not match the expected format. diff --git a/edq/testing/testdata/cli/tests/config/config_list_sources.txt b/edq/testing/testdata/cli/tests/config/config_list_sources.txt index 962e352..7a61737 100644 --- a/edq/testing/testdata/cli/tests/config/config_list_sources.txt +++ b/edq/testing/testdata/cli/tests/config/config_list_sources.txt @@ -9,5 +9,5 @@ } --- Key Value Origin -user user@test.edulinq.org __ABS_DATA_DIR__(configs/simple/edq-config.json) pass pass123 +user user@test.edulinq.org __ABS_DATA_DIR__(configs/simple/edq-config.json) From 04636f7e8f47328a97393dded35ed18997391bde Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Wed, 10 Sep 2025 14:28:05 -0700 Subject: [PATCH 59/71] Added more test cases. --- ..._error.txt => config_list_invalid_config_error.txt} | 0 ...config_list_empty.txt => config_list_no_config.txt} | 0 .../tests/config/config_list_seperator_in_value.txt | 10 ++++++++++ 3 files changed, 10 insertions(+) rename edq/testing/testdata/cli/tests/config/{config_list_incorrect_config_error.txt => config_list_invalid_config_error.txt} (100%) rename edq/testing/testdata/cli/tests/config/{config_list_empty.txt => config_list_no_config.txt} (100%) create mode 100644 edq/testing/testdata/cli/tests/config/config_list_seperator_in_value.txt diff --git a/edq/testing/testdata/cli/tests/config/config_list_incorrect_config_error.txt b/edq/testing/testdata/cli/tests/config/config_list_invalid_config_error.txt similarity index 100% rename from edq/testing/testdata/cli/tests/config/config_list_incorrect_config_error.txt rename to edq/testing/testdata/cli/tests/config/config_list_invalid_config_error.txt diff --git a/edq/testing/testdata/cli/tests/config/config_list_empty.txt b/edq/testing/testdata/cli/tests/config/config_list_no_config.txt similarity index 100% rename from edq/testing/testdata/cli/tests/config/config_list_empty.txt rename to edq/testing/testdata/cli/tests/config/config_list_no_config.txt diff --git a/edq/testing/testdata/cli/tests/config/config_list_seperator_in_value.txt b/edq/testing/testdata/cli/tests/config/config_list_seperator_in_value.txt new file mode 100644 index 0000000..b0c8e08 --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/config_list_seperator_in_value.txt @@ -0,0 +1,10 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + "--config", "pass=pass123=", + ], +} +--- +Key Value +pass pass123= From bf08974900674355cf956cb63e2f6d3a20d4be07 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Wed, 10 Sep 2025 14:30:25 -0700 Subject: [PATCH 60/71] Moved config list test under list directory. --- .../testdata/cli/tests/config/{ => list}/config_list_base.txt | 0 .../config/{ => list}/config_list_empty_config_key_error.txt | 0 .../tests/config/{ => list}/config_list_empty_config_value.txt | 0 .../tests/config/{ => list}/config_list_invalid_config_error.txt | 0 .../cli/tests/config/{ => list}/config_list_no_config.txt | 0 .../tests/config/{ => list}/config_list_seperator_in_value.txt | 0 .../cli/tests/config/{ => list}/config_list_skip_header.txt | 0 .../testdata/cli/tests/config/{ => list}/config_list_sources.txt | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename edq/testing/testdata/cli/tests/config/{ => list}/config_list_base.txt (100%) rename edq/testing/testdata/cli/tests/config/{ => list}/config_list_empty_config_key_error.txt (100%) rename edq/testing/testdata/cli/tests/config/{ => list}/config_list_empty_config_value.txt (100%) rename edq/testing/testdata/cli/tests/config/{ => list}/config_list_invalid_config_error.txt (100%) rename edq/testing/testdata/cli/tests/config/{ => list}/config_list_no_config.txt (100%) rename edq/testing/testdata/cli/tests/config/{ => list}/config_list_seperator_in_value.txt (100%) rename edq/testing/testdata/cli/tests/config/{ => list}/config_list_skip_header.txt (100%) rename edq/testing/testdata/cli/tests/config/{ => list}/config_list_sources.txt (100%) diff --git a/edq/testing/testdata/cli/tests/config/config_list_base.txt b/edq/testing/testdata/cli/tests/config/list/config_list_base.txt similarity index 100% rename from edq/testing/testdata/cli/tests/config/config_list_base.txt rename to edq/testing/testdata/cli/tests/config/list/config_list_base.txt diff --git a/edq/testing/testdata/cli/tests/config/config_list_empty_config_key_error.txt b/edq/testing/testdata/cli/tests/config/list/config_list_empty_config_key_error.txt similarity index 100% rename from edq/testing/testdata/cli/tests/config/config_list_empty_config_key_error.txt rename to edq/testing/testdata/cli/tests/config/list/config_list_empty_config_key_error.txt diff --git a/edq/testing/testdata/cli/tests/config/config_list_empty_config_value.txt b/edq/testing/testdata/cli/tests/config/list/config_list_empty_config_value.txt similarity index 100% rename from edq/testing/testdata/cli/tests/config/config_list_empty_config_value.txt rename to edq/testing/testdata/cli/tests/config/list/config_list_empty_config_value.txt diff --git a/edq/testing/testdata/cli/tests/config/config_list_invalid_config_error.txt b/edq/testing/testdata/cli/tests/config/list/config_list_invalid_config_error.txt similarity index 100% rename from edq/testing/testdata/cli/tests/config/config_list_invalid_config_error.txt rename to edq/testing/testdata/cli/tests/config/list/config_list_invalid_config_error.txt diff --git a/edq/testing/testdata/cli/tests/config/config_list_no_config.txt b/edq/testing/testdata/cli/tests/config/list/config_list_no_config.txt similarity index 100% rename from edq/testing/testdata/cli/tests/config/config_list_no_config.txt rename to edq/testing/testdata/cli/tests/config/list/config_list_no_config.txt diff --git a/edq/testing/testdata/cli/tests/config/config_list_seperator_in_value.txt b/edq/testing/testdata/cli/tests/config/list/config_list_seperator_in_value.txt similarity index 100% rename from edq/testing/testdata/cli/tests/config/config_list_seperator_in_value.txt rename to edq/testing/testdata/cli/tests/config/list/config_list_seperator_in_value.txt diff --git a/edq/testing/testdata/cli/tests/config/config_list_skip_header.txt b/edq/testing/testdata/cli/tests/config/list/config_list_skip_header.txt similarity index 100% rename from edq/testing/testdata/cli/tests/config/config_list_skip_header.txt rename to edq/testing/testdata/cli/tests/config/list/config_list_skip_header.txt diff --git a/edq/testing/testdata/cli/tests/config/config_list_sources.txt b/edq/testing/testdata/cli/tests/config/list/config_list_sources.txt similarity index 100% rename from edq/testing/testdata/cli/tests/config/config_list_sources.txt rename to edq/testing/testdata/cli/tests/config/list/config_list_sources.txt From 8af6f7e59fbb1ce0e162e58970a5f2752c53892d Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Wed, 10 Sep 2025 15:24:12 -0700 Subject: [PATCH 61/71] Moved some test from cli test to config test side. --- edq/core/config_test.py | 74 ++++++++++++++++++- .../config_list_empty_config_key_error.txt | 10 --- .../list/config_list_empty_config_value.txt | 10 --- .../list/config_list_invalid_config_error.txt | 10 --- .../list/config_list_seperator_in_value.txt | 10 --- 5 files changed, 71 insertions(+), 43 deletions(-) delete mode 100644 edq/testing/testdata/cli/tests/config/list/config_list_empty_config_key_error.txt delete mode 100644 edq/testing/testdata/cli/tests/config/list/config_list_empty_config_value.txt delete mode 100644 edq/testing/testdata/cli/tests/config/list/config_list_invalid_config_error.txt delete mode 100644 edq/testing/testdata/cli/tests/config/list/config_list_seperator_in_value.txt diff --git a/edq/core/config_test.py b/edq/core/config_test.py index faf5487..b779c67 100644 --- a/edq/core/config_test.py +++ b/edq/core/config_test.py @@ -409,7 +409,7 @@ def test_get_tiered_config_base(self): "Failed to read JSON file", ), - # CLI Bare Options: + # CLI Options: # CLI arguments only (direct key: value). ( @@ -430,6 +430,74 @@ def test_get_tiered_config_base(self): None, ), + # Empty Config Key + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIGS_KEY: [ + "=user@test.edulinq.org", + ], + }, + }, + {}, + {}, + "The provided '=user@test.edulinq.org' config option has an empty key." + ), + + # Empty Config Value + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIGS_KEY: [ + "user=", + ], + }, + }, + { + "user": "", + }, + { + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), + }, + None, + ), + + # Separator In Config Value + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIGS_KEY: [ + "pass=password=1234", + ], + }, + }, + { + "pass": "password=1234", + }, + { + "pass": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), + }, + None, + ), + + # Invalid Config Option Format + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIGS_KEY: [ + "useruser@test.edulinq.org", + ], + }, + }, + {}, + {}, + "The provided 'useruser@test.edulinq.org' config option does not match the expected format", + ), + # Combinations # Global Config + Local Config @@ -654,13 +722,13 @@ def test_get_tiered_config_base(self): os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ], edq.core.config.CONFIGS_KEY: [ - "pass=user", + "pass=password1234", ], }, }, { "user": "user@test.edulinq.org", - "pass": "user", + "pass": "password1234", }, { "user": edq.core.config.ConfigSource( diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_empty_config_key_error.txt b/edq/testing/testdata/cli/tests/config/list/config_list_empty_config_key_error.txt deleted file mode 100644 index 4208176..0000000 --- a/edq/testing/testdata/cli/tests/config/list/config_list_empty_config_key_error.txt +++ /dev/null @@ -1,10 +0,0 @@ -{ - "cli": "edq.cli.config.list", - "arguments": [ - "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", - "--config", "=pass123", - ], - "error": "True" -} ---- -builtins.ValueError: The provided '=pass123' config option has an empty key. diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_empty_config_value.txt b/edq/testing/testdata/cli/tests/config/list/config_list_empty_config_value.txt deleted file mode 100644 index c55aa14..0000000 --- a/edq/testing/testdata/cli/tests/config/list/config_list_empty_config_value.txt +++ /dev/null @@ -1,10 +0,0 @@ -{ - "cli": "edq.cli.config.list", - "arguments": [ - "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", - "--config", "pass=", - ], -} ---- -Key Value -pass diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_invalid_config_error.txt b/edq/testing/testdata/cli/tests/config/list/config_list_invalid_config_error.txt deleted file mode 100644 index 3c4d237..0000000 --- a/edq/testing/testdata/cli/tests/config/list/config_list_invalid_config_error.txt +++ /dev/null @@ -1,10 +0,0 @@ -{ - "cli": "edq.cli.config.list", - "arguments": [ - "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", - "--config", "passpass123", - ], - "error": "True" -} ---- -builtins.ValueError: The provided 'passpass123' config option does not match the expected format. diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_seperator_in_value.txt b/edq/testing/testdata/cli/tests/config/list/config_list_seperator_in_value.txt deleted file mode 100644 index b0c8e08..0000000 --- a/edq/testing/testdata/cli/tests/config/list/config_list_seperator_in_value.txt +++ /dev/null @@ -1,10 +0,0 @@ -{ - "cli": "edq.cli.config.list", - "arguments": [ - "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", - "--config", "pass=pass123=", - ], -} ---- -Key Value -pass pass123= From f8986f8e001c30ab253b92830edfbfb3a788bf2e Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Wed, 10 Sep 2025 15:26:23 -0700 Subject: [PATCH 62/71] Made password test case consisten on cli side. --- .../testdata/cli/tests/config/list/config_list_base.txt | 4 ++-- .../testdata/cli/tests/config/list/config_list_sources.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_base.txt b/edq/testing/testdata/cli/tests/config/list/config_list_base.txt index 8739178..0951f0f 100644 --- a/edq/testing/testdata/cli/tests/config/list/config_list_base.txt +++ b/edq/testing/testdata/cli/tests/config/list/config_list_base.txt @@ -3,10 +3,10 @@ "arguments": [ "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", "--config-file", "__DATA_DIR__(configs/simple/edq-config.json)", - "--config", "pass=pass123", + "--config", "pass=password1234", ], } --- Key Value -pass pass123 +pass password1234 user user@test.edulinq.org diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_sources.txt b/edq/testing/testdata/cli/tests/config/list/config_list_sources.txt index 7a61737..09b55dc 100644 --- a/edq/testing/testdata/cli/tests/config/list/config_list_sources.txt +++ b/edq/testing/testdata/cli/tests/config/list/config_list_sources.txt @@ -3,11 +3,11 @@ "arguments": [ "--config-file", "__DATA_DIR__(configs/simple/edq-config.json)", "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", - "--config", "pass=pass123", + "--config", "pass=password1234", "--show-origin", ], } --- Key Value Origin -pass pass123 +pass password1234 user user@test.edulinq.org __ABS_DATA_DIR__(configs/simple/edq-config.json) From 648b4be9983a5944d561fc88ce1b57014fdb379c Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Wed, 24 Sep 2025 09:27:30 -0700 Subject: [PATCH 63/71] Added ignore-config flag. --- edq/core/config.py | 35 ++++++++++--- edq/core/config_test.py | 49 +++++++++++++++++-- .../tests/config/list/config_list_base.txt | 2 +- .../config/list/config_list_ignore_config.txt | 10 ++++ ...ources.txt => config_list_show_origin.txt} | 2 +- .../config/list/config_list_skip_header.txt | 2 +- 6 files changed, 85 insertions(+), 15 deletions(-) create mode 100644 edq/testing/testdata/cli/tests/config/list/config_list_ignore_config.txt rename edq/testing/testdata/cli/tests/config/list/{config_list_sources.txt => config_list_show_origin.txt} (88%) diff --git a/edq/core/config.py b/edq/core/config.py index 2b5f840..ae03be6 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -12,8 +12,9 @@ CONFIG_SOURCE_CLI_FILE: str = "" CONFIG_SOURCE_CLI: str = "" -CONFIG_PATHS_KEY: str = 'config_path' -CONFIGS_KEY: str = 'config' +CONFIG_PATHS_KEY: str = 'config_paths' +CONFIGS_KEY: str = 'configs' +IGNORE_CONFIGS_KEY: str = 'ignore_configs' DEFAULT_CONFIG_FILENAME: str = "edq-config.json" DEFAULT_GLOBAL_CONFIG_PATH: str = platformdirs.user_config_dir(DEFAULT_CONFIG_FILENAME) @@ -25,7 +26,7 @@ def __init__(self, label: str, path: typing.Union[str, None] = None) -> None: """ The label identifying the config (see CONFIG_SOURCE_* constants). """ self.path = path - """ The path of where the config was soruced from. """ + """ The path of where the config was sourced from. """ def __eq__(self, other: object) -> bool: if (not isinstance(other, ConfigSource)): @@ -80,7 +81,7 @@ def get_tiered_config( for path in config_paths: _load_config_file(path, config, sources, CONFIG_SOURCE_CLI_FILE) - # Finally, any command-line config options. + # Check the command-line config options. cli_configs = cli_arguments.get(CONFIGS_KEY, []) for cli_config in cli_configs: if ("=" not in cli_config): @@ -89,7 +90,6 @@ def get_tiered_config( (key, value) = cli_config.split("=", maxsplit = 1) key = key.strip() - value = value.strip() if (key == ""): raise ValueError(f"The provided '{cli_config}' config option has an empty key.") @@ -97,6 +97,12 @@ def get_tiered_config( config[key] = value sources[key] = ConfigSource(label = CONFIG_SOURCE_CLI) + # Finally, ignore any configs that is specified from CLI command. + cli_ignore_configs = cli_arguments.get(IGNORE_CONFIGS_KEY, []) + for ignore_config in cli_ignore_configs: + config.pop(ignore_config, None) + sources.pop(ignore_config, None) + return config, sources def _load_config_file( @@ -109,6 +115,12 @@ def _load_config_file( config_path = os.path.abspath(config_path) for (key, value) in edq.util.json.load_path(config_path).items(): + + key = key.strip() + + if (key == ""): + raise ValueError(f"The provided '{key}: {value}' config option has an empty key.") + config[key] = value sources[key] = ConfigSource(label = source_label, path = config_path) @@ -197,15 +209,22 @@ def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, + ' This will override options form both global and local config files.') ) - parser.add_argument('--config', dest = CONFIGS_KEY, + parser.add_argument('--config-option', dest = CONFIGS_KEY, action = 'append', type = str, default = [], - help = ('Load configuration options from the CLI command.' - + ' Specify options as = pairs. ' + help = ('Load configuration option from the CLI command.' + + ' Specify options as = pairs.' + ' This flag can be specified multiple times.' + ' The options are applied in the order provided and later options override earlier ones.' + ' This will override options form all config files.') ) + parser.add_argument('--ignore-config-option', dest = IGNORE_CONFIGS_KEY, + action = 'append', type = str, default = [], + help = ('Ignore configuration options from the CLI command.' + + ' This will ignore specified config options from both files and CLI.' + + ' This flag can be specified multiple times.') + ) + def load_config_into_args( parser: argparse.ArgumentParser, args: argparse.Namespace, diff --git a/edq/core/config_test.py b/edq/core/config_test.py index b779c67..4949c02 100644 --- a/edq/core/config_test.py +++ b/edq/core/config_test.py @@ -18,6 +18,8 @@ def creat_test_dir(temp_dir_prefix: str) -> str: ├── empty │   └── edq-config.json ├── empty-dir + ├── empty-key + │   └── edq-config.json ├── global │   └── edq-config.json ├── malformed @@ -41,11 +43,15 @@ def creat_test_dir(temp_dir_prefix: str) -> str: empty_config_dir_path = os.path.join(temp_dir, "empty") edq.util.dirent.mkdir(empty_config_dir_path) - edq.util.json.dump_path({}, os.path.join(empty_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME)) + edq.util.json.dump_path({}, os.path.join(empty_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME)) + + empty_key_config_dir_path = os.path.join(temp_dir, "empty-key") + edq.util.dirent.mkdir(empty_key_config_dir_path) + edq.util.json.dump_path({"": "user@test.edulinq.org"}, os.path.join(empty_key_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME)) - custome_name_config_dir_path = os.path.join(temp_dir, "custom-name") - edq.util.dirent.mkdir(custome_name_config_dir_path) - edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(custome_name_config_dir_path, "custom-edq-config.json")) + custom_name_config_dir_path = os.path.join(temp_dir, "custom-name") + edq.util.dirent.mkdir(custom_name_config_dir_path) + edq.util.json.dump_path({"user": "user@test.edulinq.org"}, os.path.join(custom_name_config_dir_path, "custom-edq-config.json")) edq.util.dirent.mkdir(os.path.join(temp_dir, "dir-config", "edq-config.json")) edq.util.dirent.mkdir(os.path.join(temp_dir, "empty-dir")) @@ -137,6 +143,17 @@ def test_get_tiered_config_base(self): None, ), + # Empty Key Config JSON + ( + "empty-dir", + { + "global_config_path": os.path.join(temp_dir, "empty-key", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, + {}, + {}, + "The provided ': user@test.edulinq.org' config option has an empty key.", + ), + # Directory Config JSON ( "empty-dir", @@ -260,6 +277,15 @@ def test_get_tiered_config_base(self): None, ), + # Empty Key Config JSON + ( + "empty-key", + {}, + {}, + {}, + "The provided ': user@test.edulinq.org' config option has an empty key.", + ), + # Directory Config JSON ( "dir-config", @@ -364,6 +390,21 @@ def test_get_tiered_config_base(self): None, ), + # Empty Key Config JSON + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "empty-key", edq.core.config.DEFAULT_CONFIG_FILENAME), + ], + }, + }, + {}, + {}, + "The provided ': user@test.edulinq.org' config option has an empty key.", + ), + # Directory Config JSON ( "empty-dir", diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_base.txt b/edq/testing/testdata/cli/tests/config/list/config_list_base.txt index 0951f0f..ac2312c 100644 --- a/edq/testing/testdata/cli/tests/config/list/config_list_base.txt +++ b/edq/testing/testdata/cli/tests/config/list/config_list_base.txt @@ -3,7 +3,7 @@ "arguments": [ "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", "--config-file", "__DATA_DIR__(configs/simple/edq-config.json)", - "--config", "pass=password1234", + "--config-option", "pass=password1234", ], } --- diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_ignore_config.txt b/edq/testing/testdata/cli/tests/config/list/config_list_ignore_config.txt new file mode 100644 index 0000000..30db65e --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/list/config_list_ignore_config.txt @@ -0,0 +1,10 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + "--config-file", "__DATA_DIR__(configs/simple/edq-config.json)", + "--ignore-config-option", "user", + ], +} +--- +Key Value diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_sources.txt b/edq/testing/testdata/cli/tests/config/list/config_list_show_origin.txt similarity index 88% rename from edq/testing/testdata/cli/tests/config/list/config_list_sources.txt rename to edq/testing/testdata/cli/tests/config/list/config_list_show_origin.txt index 09b55dc..e10d6b6 100644 --- a/edq/testing/testdata/cli/tests/config/list/config_list_sources.txt +++ b/edq/testing/testdata/cli/tests/config/list/config_list_show_origin.txt @@ -3,7 +3,7 @@ "arguments": [ "--config-file", "__DATA_DIR__(configs/simple/edq-config.json)", "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", - "--config", "pass=password1234", + "--config-option", "pass=password1234", "--show-origin", ], } diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_skip_header.txt b/edq/testing/testdata/cli/tests/config/list/config_list_skip_header.txt index 64d3b8a..d289905 100644 --- a/edq/testing/testdata/cli/tests/config/list/config_list_skip_header.txt +++ b/edq/testing/testdata/cli/tests/config/list/config_list_skip_header.txt @@ -2,7 +2,7 @@ "cli": "edq.cli.config.list", "arguments": [ "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", - "--config", "user=user@test.edulinq.org", + "--config-option", "user=user@test.edulinq.org", "--skip-header", ], } From 4cdc0503ad2f1bf750c203f50905683a57d2b536 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 26 Sep 2025 11:25:30 -0700 Subject: [PATCH 64/71] Fixed issues with helpand error messages. --- edq/core/config.py | 22 +++++----- edq/core/config_test.py | 40 +++++++++---------- .../config/list/config_list_ignore_config.txt | 4 +- 3 files changed, 35 insertions(+), 31 deletions(-) diff --git a/edq/core/config.py b/edq/core/config.py index ae03be6..0fee9a5 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -14,6 +14,7 @@ CONFIG_PATHS_KEY: str = 'config_paths' CONFIGS_KEY: str = 'configs' +GLOBAL_CONFIG_KEY: str = 'global_config_path' IGNORE_CONFIGS_KEY: str = 'ignore_configs' DEFAULT_CONFIG_FILENAME: str = "edq-config.json" DEFAULT_GLOBAL_CONFIG_PATH: str = platformdirs.user_config_dir(DEFAULT_CONFIG_FILENAME) @@ -85,14 +86,16 @@ def get_tiered_config( cli_configs = cli_arguments.get(CONFIGS_KEY, []) for cli_config in cli_configs: if ("=" not in cli_config): - raise ValueError(f"The provided '{cli_config}' config option does not match the expected format.") + raise ValueError( + f"Invalid configuration option '{cli_config}'." + + " Configuration options must be provided in the format `=` when passed via the CLI." + ) (key, value) = cli_config.split("=", maxsplit = 1) key = key.strip() - if (key == ""): - raise ValueError(f"The provided '{cli_config}' config option has an empty key.") + raise ValueError(f"Found an empty configuration option key associated with the value '{value}'.") config[key] = value sources[key] = ConfigSource(label = CONFIG_SOURCE_CLI) @@ -115,11 +118,9 @@ def _load_config_file( config_path = os.path.abspath(config_path) for (key, value) in edq.util.json.load_path(config_path).items(): - key = key.strip() - if (key == ""): - raise ValueError(f"The provided '{key}: {value}' config option has an empty key.") + raise ValueError(f"Found an empty configuration option key associated with the value '{value}'.") config[key] = value sources[key] = ConfigSource(label = source_label, path = config_path) @@ -196,7 +197,7 @@ def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, Set common CLI arguments for configuration. """ - parser.add_argument('--config-global', dest = 'global_config_path', + parser.add_argument('--config-global', dest = GLOBAL_CONFIG_KEY, action = 'store', type = str, default = DEFAULT_GLOBAL_CONFIG_PATH, help = 'Override the default global config file path (default: %(default)s).', ) @@ -220,9 +221,10 @@ def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, parser.add_argument('--ignore-config-option', dest = IGNORE_CONFIGS_KEY, action = 'append', type = str, default = [], - help = ('Ignore configuration options from the CLI command.' - + ' This will ignore specified config options from both files and CLI.' - + ' This flag can be specified multiple times.') + help = ('Ignore any specified values for a config option.' + + ' The default value will be used for that option if one exists.' + + ' This flag can be specified multiple times.' + + ' Ignoring options happens last, so the specified option will be ignored regardless of where other flags appear in the CLI command.') ) def load_config_into_args( diff --git a/edq/core/config_test.py b/edq/core/config_test.py index 4949c02..f3f0b6b 100644 --- a/edq/core/config_test.py +++ b/edq/core/config_test.py @@ -118,7 +118,7 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, { "user": "user@test.edulinq.org", @@ -136,7 +136,7 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "empty", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "empty", edq.core.config.DEFAULT_CONFIG_FILENAME), }, {}, {}, @@ -147,18 +147,18 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "empty-key", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "empty-key", edq.core.config.DEFAULT_CONFIG_FILENAME), }, {}, {}, - "The provided ': user@test.edulinq.org' config option has an empty key.", + "Found an empty configuration option key associated with the value 'user@test.edulinq.org'.", ), # Directory Config JSON ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "dir-config", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "dir-config", edq.core.config.DEFAULT_CONFIG_FILENAME), }, {}, {}, @@ -169,7 +169,7 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "empty-dir", "non-existent-config.json"), + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "empty-dir", "non-existent-config.json"), }, {}, {}, @@ -180,7 +180,7 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "malformed", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "malformed", edq.core.config.DEFAULT_CONFIG_FILENAME), }, {}, {}, @@ -283,7 +283,7 @@ def test_get_tiered_config_base(self): {}, {}, {}, - "The provided ': user@test.edulinq.org' config option has an empty key.", + "Found an empty configuration option key associated with the value 'user@test.edulinq.org'.", ), # Directory Config JSON @@ -402,7 +402,7 @@ def test_get_tiered_config_base(self): }, {}, {}, - "The provided ': user@test.edulinq.org' config option has an empty key.", + "Found an empty configuration option key associated with the value 'user@test.edulinq.org'.", ), # Directory Config JSON @@ -483,7 +483,7 @@ def test_get_tiered_config_base(self): }, {}, {}, - "The provided '=user@test.edulinq.org' config option has an empty key." + "Found an empty configuration option key associated with the value 'user@test.edulinq.org'." ), # Empty Config Value @@ -536,7 +536,7 @@ def test_get_tiered_config_base(self): }, {}, {}, - "The provided 'useruser@test.edulinq.org' config option does not match the expected format", + "Invalid configuration option 'useruser@test.edulinq.org'.", ), # Combinations @@ -545,7 +545,7 @@ def test_get_tiered_config_base(self): ( "simple", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, { "user": "user@test.edulinq.org", @@ -563,7 +563,7 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), @@ -586,7 +586,7 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { edq.core.config.CONFIGS_KEY: [ "user=user@test.edulinq.org", @@ -669,7 +669,7 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { edq.core.config.CONFIGS_KEY: [ "user=user@test.edulinq.org", @@ -692,7 +692,7 @@ def test_get_tiered_config_base(self): ( "simple", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { edq.core.config.CONFIGS_KEY: [ "user=user@test.edulinq.org", @@ -712,7 +712,7 @@ def test_get_tiered_config_base(self): ( "simple", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), @@ -757,7 +757,7 @@ def test_get_tiered_config_base(self): ( "simple", { - "global_config_path": os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), @@ -786,9 +786,9 @@ def test_get_tiered_config_base(self): (test_work_dir, extra_args, expected_config, expected_source, error_substring) = test_case with self.subTest(msg = f"Case {i} ('{test_work_dir}'):"): - global_config = extra_args.get("global_config_path", None) + global_config = extra_args.get(edq.core.config.GLOBAL_CONFIG_KEY, None) if (global_config is None): - extra_args["global_config_path"] = os.path.join(temp_dir, "empty", edq.core.config.CONFIG_PATHS_KEY) + extra_args[edq.core.config.GLOBAL_CONFIG_KEY] = os.path.join(temp_dir, "empty", edq.core.config.CONFIG_PATHS_KEY) cutoff = extra_args.get("local_config_root_cutoff", None) if (cutoff is None): diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_ignore_config.txt b/edq/testing/testdata/cli/tests/config/list/config_list_ignore_config.txt index 30db65e..f66eddf 100644 --- a/edq/testing/testdata/cli/tests/config/list/config_list_ignore_config.txt +++ b/edq/testing/testdata/cli/tests/config/list/config_list_ignore_config.txt @@ -1,10 +1,12 @@ { "cli": "edq.cli.config.list", "arguments": [ + "--ignore-config-option", "pass", "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", "--config-file", "__DATA_DIR__(configs/simple/edq-config.json)", - "--ignore-config-option", "user", + "--config-option", "pass=password1234", ], } --- Key Value +user user@test.edulinq.org From fdb1abac150272030c202fa5d5cdf2888358d769 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Fri, 26 Sep 2025 12:43:01 -0700 Subject: [PATCH 65/71] Embedded global config path in to cli arguments. --- edq/core/config.py | 7 ++---- edq/core/config_test.py | 54 +++++++++++++++++++++++++++++------------ 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/edq/core/config.py b/edq/core/config.py index 0fee9a5..797c54a 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -41,7 +41,6 @@ def __str__(self) -> str: def get_tiered_config( config_file_name: str = DEFAULT_CONFIG_FILENAME, legacy_config_file_name: typing.Union[str, None] = None, - global_config_path: typing.Union[str, None] = None, cli_arguments: typing.Union[dict, argparse.Namespace, None] = None, local_config_root_cutoff: typing.Union[str, None] = None, ) -> typing.Tuple[typing.Dict[str, str], typing.Dict[str, ConfigSource]]: @@ -50,9 +49,6 @@ def get_tiered_config( Returns a configuration dictionary with the values based on tiering rules and a source dictionary mapping each key to its origin. """ - if (global_config_path is None): - global_config_path = platformdirs.user_config_dir(config_file_name) - if (cli_arguments is None): cli_arguments = {} @@ -63,6 +59,8 @@ def get_tiered_config( if (isinstance(cli_arguments, argparse.Namespace)): cli_arguments = vars(cli_arguments) + global_config_path = cli_arguments.get(GLOBAL_CONFIG_KEY, platformdirs.user_config_dir(config_file_name)) + # Check the global user config file. if (os.path.isfile(global_config_path)): _load_config_file(global_config_path, config, sources, CONFIG_SOURCE_GLOBAL) @@ -238,7 +236,6 @@ def load_config_into_args( """ (config_dict, sources_dict) = get_tiered_config( - global_config_path = args.global_config_path, cli_arguments = args, ) diff --git a/edq/core/config_test.py b/edq/core/config_test.py index f3f0b6b..2c2a741 100644 --- a/edq/core/config_test.py +++ b/edq/core/config_test.py @@ -118,7 +118,9 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, }, { "user": "user@test.edulinq.org", @@ -136,7 +138,9 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "empty", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "empty", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, }, {}, {}, @@ -147,7 +151,9 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "empty-key", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "empty-key", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, }, {}, {}, @@ -158,7 +164,9 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "dir-config", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "dir-config", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, }, {}, {}, @@ -169,7 +177,9 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "empty-dir", "non-existent-config.json"), + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "empty-dir", "non-existent-config.json"), + }, }, {}, {}, @@ -180,7 +190,9 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "malformed", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "malformed", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, }, {}, {}, @@ -545,7 +557,9 @@ def test_get_tiered_config_base(self): ( "simple", { - edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), + }, }, { "user": "user@test.edulinq.org", @@ -563,11 +577,11 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ], + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, }, { @@ -586,11 +600,11 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { edq.core.config.CONFIGS_KEY: [ "user=user@test.edulinq.org", ], + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, }, { @@ -669,7 +683,6 @@ def test_get_tiered_config_base(self): ( "empty-dir", { - edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { edq.core.config.CONFIGS_KEY: [ "user=user@test.edulinq.org", @@ -677,6 +690,7 @@ def test_get_tiered_config_base(self): edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), ], + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, }, { @@ -692,11 +706,11 @@ def test_get_tiered_config_base(self): ( "simple", { - edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { edq.core.config.CONFIGS_KEY: [ "user=user@test.edulinq.org", ], + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, }, { @@ -712,11 +726,11 @@ def test_get_tiered_config_base(self): ( "simple", { - edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), ], + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, }, { @@ -757,7 +771,6 @@ def test_get_tiered_config_base(self): ( "simple", { - edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), "cli_arguments": { edq.core.config.CONFIG_PATHS_KEY: [ os.path.join(temp_dir, "custom-name", "custom-edq-config.json"), @@ -765,6 +778,7 @@ def test_get_tiered_config_base(self): edq.core.config.CONFIGS_KEY: [ "pass=password1234", ], + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "global", edq.core.config.DEFAULT_CONFIG_FILENAME), }, }, { @@ -786,9 +800,17 @@ def test_get_tiered_config_base(self): (test_work_dir, extra_args, expected_config, expected_source, error_substring) = test_case with self.subTest(msg = f"Case {i} ('{test_work_dir}'):"): - global_config = extra_args.get(edq.core.config.GLOBAL_CONFIG_KEY, None) - if (global_config is None): - extra_args[edq.core.config.GLOBAL_CONFIG_KEY] = os.path.join(temp_dir, "empty", edq.core.config.CONFIG_PATHS_KEY) + cli_args = extra_args.get("cli_arguments", None) + if cli_args is None: + extra_args["cli_arguments"] = { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "empty", edq.core.config.DEFAULT_CONFIG_FILENAME) + } + else: + cli_global_config_path = cli_args.get(edq.core.config.GLOBAL_CONFIG_KEY, None) + if cli_global_config_path is None: + extra_args["cli_arguments"][edq.core.config.GLOBAL_CONFIG_KEY] = os.path.join( + temp_dir, "empty", edq.core.config.DEFAULT_CONFIG_FILENAME + ) cutoff = extra_args.get("local_config_root_cutoff", None) if (cutoff is None): From 9303bf1218640e37a00118f755a52c74d6329ffe Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sat, 27 Sep 2025 06:12:09 -0700 Subject: [PATCH 66/71] Added tests and updated the help message for ignore config. --- edq/core/config.py | 4 +- edq/core/config_test.py | 103 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/edq/core/config.py b/edq/core/config.py index 797c54a..3b094de 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -219,10 +219,10 @@ def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, parser.add_argument('--ignore-config-option', dest = IGNORE_CONFIGS_KEY, action = 'append', type = str, default = [], - help = ('Ignore any specified values for a config option.' + help = ('Ignore any config option with the specified values.' + ' The default value will be used for that option if one exists.' + ' This flag can be specified multiple times.' - + ' Ignoring options happens last, so the specified option will be ignored regardless of where other flags appear in the CLI command.') + + ' Ignoring options are processed last.') ) def load_config_into_args( diff --git a/edq/core/config_test.py b/edq/core/config_test.py index 2c2a741..4c92d66 100644 --- a/edq/core/config_test.py +++ b/edq/core/config_test.py @@ -24,6 +24,8 @@ def creat_test_dir(temp_dir_prefix: str) -> str: │   └── edq-config.json ├── malformed │   └── edq-config.json + ├── multiple-options + │   └── edq-config.json ├── nested │   ├── config.json │   ├── edq-config.json @@ -45,6 +47,13 @@ def creat_test_dir(temp_dir_prefix: str) -> str: edq.util.dirent.mkdir(empty_config_dir_path) edq.util.json.dump_path({}, os.path.join(empty_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME)) + multiple_option_config_dir_path = os.path.join(temp_dir, "multiple-options") + edq.util.dirent.mkdir(multiple_option_config_dir_path) + edq.util.json.dump_path( + {"user": "user@test.edulinq.org", "pass": "password1234"}, + os.path.join(multiple_option_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME) + ) + empty_key_config_dir_path = os.path.join(temp_dir, "empty-key") edq.util.dirent.mkdir(empty_key_config_dir_path) edq.util.json.dump_path({"": "user@test.edulinq.org"}, os.path.join(empty_key_config_dir_path, edq.core.config.DEFAULT_CONFIG_FILENAME)) @@ -199,6 +208,29 @@ def test_get_tiered_config_base(self): "Failed to read JSON file", ), + # Ignore Config Option + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "multiple-options", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.IGNORE_CONFIGS_KEY: [ + "pass" + ] + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_GLOBAL, + path = os.path.join(temp_dir, "multiple-options", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + # Local Config # Default config file in current directory. @@ -316,6 +348,28 @@ def test_get_tiered_config_base(self): "Failed to read JSON file", ), + # Ignore Config Option + ( + "multiple-options", + { + "cli_arguments":{ + edq.core.config.IGNORE_CONFIGS_KEY: [ + "pass" + ] + } + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, + path = os.path.join(temp_dir, "multiple-options", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + # All 3 local config locations present at the same time. ( os.path.join("nested", "nest1", "nest2b"), @@ -462,6 +516,32 @@ def test_get_tiered_config_base(self): "Failed to read JSON file", ), + # Ignore Config Option + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "multiple-options", edq.core.config.DEFAULT_CONFIG_FILENAME), + ], + edq.core.config.IGNORE_CONFIGS_KEY: [ + "pass", + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, + path = os.path.join(temp_dir, "multiple-options", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + + # CLI Options: # CLI arguments only (direct key: value). @@ -551,6 +631,29 @@ def test_get_tiered_config_base(self): "Invalid configuration option 'useruser@test.edulinq.org'.", ), + # Ignore Config Option + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + "pass=password1234" + ], + edq.core.config.IGNORE_CONFIGS_KEY:[ + "pass", + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), + }, + None, + ), + # Combinations # Global Config + Local Config From 3052836268dc1674523b7ff330c86ab575615e2d Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sat, 27 Sep 2025 06:28:09 -0700 Subject: [PATCH 67/71] Added non-existing key test for ignore config. --- edq/core/config_test.py | 107 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 7 deletions(-) diff --git a/edq/core/config_test.py b/edq/core/config_test.py index 4c92d66..4e5c314 100644 --- a/edq/core/config_test.py +++ b/edq/core/config_test.py @@ -215,8 +215,8 @@ def test_get_tiered_config_base(self): "cli_arguments": { edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "multiple-options", edq.core.config.DEFAULT_CONFIG_FILENAME), edq.core.config.IGNORE_CONFIGS_KEY: [ - "pass" - ] + "pass", + ], }, }, { @@ -231,6 +231,29 @@ def test_get_tiered_config_base(self): None, ), + # Ignore Non-Existing Config Option + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.GLOBAL_CONFIG_KEY: os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + edq.core.config.IGNORE_CONFIGS_KEY: [ + "non-existing-option", + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_GLOBAL, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + # Local Config # Default config file in current directory. @@ -354,9 +377,9 @@ def test_get_tiered_config_base(self): { "cli_arguments":{ edq.core.config.IGNORE_CONFIGS_KEY: [ - "pass" - ] - } + "pass", + ], + }, }, { "user": "user@test.edulinq.org", @@ -370,6 +393,29 @@ def test_get_tiered_config_base(self): None, ), + # Ignore Non-Existing Config Option + ( + "simple", + { + "cli_arguments":{ + edq.core.config.IGNORE_CONFIGS_KEY: [ + "non-existing-option", + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_LOCAL, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + + # All 3 local config locations present at the same time. ( os.path.join("nested", "nest1", "nest2b"), @@ -541,6 +587,31 @@ def test_get_tiered_config_base(self): None, ), + # Ignore Non-Existing Config Option + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIG_PATHS_KEY: [ + os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ], + edq.core.config.IGNORE_CONFIGS_KEY: [ + "non-existing-option", + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource( + label = edq.core.config.CONFIG_SOURCE_CLI_FILE, + path = os.path.join(temp_dir, "simple", edq.core.config.DEFAULT_CONFIG_FILENAME), + ), + }, + None, + ), + # CLI Options: @@ -575,7 +646,7 @@ def test_get_tiered_config_base(self): }, {}, {}, - "Found an empty configuration option key associated with the value 'user@test.edulinq.org'." + "Found an empty configuration option key associated with the value 'user@test.edulinq.org'.", ), # Empty Config Value @@ -638,7 +709,7 @@ def test_get_tiered_config_base(self): "cli_arguments": { edq.core.config.CONFIGS_KEY: [ "user=user@test.edulinq.org", - "pass=password1234" + "pass=password1234", ], edq.core.config.IGNORE_CONFIGS_KEY:[ "pass", @@ -654,6 +725,28 @@ def test_get_tiered_config_base(self): None, ), + # Ignore Non-Existing Config Option + ( + "empty-dir", + { + "cli_arguments": { + edq.core.config.CONFIGS_KEY: [ + "user=user@test.edulinq.org", + ], + edq.core.config.IGNORE_CONFIGS_KEY:[ + "non-existing-option", + ], + }, + }, + { + "user": "user@test.edulinq.org", + }, + { + "user": edq.core.config.ConfigSource(label = edq.core.config.CONFIG_SOURCE_CLI), + }, + None, + ), + # Combinations # Global Config + Local Config From 39012398b91447923036cd48c30e1bda3aa1de36 Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sat, 27 Sep 2025 06:31:00 -0700 Subject: [PATCH 68/71] Got rid of extra space. --- .../testdata/cli/tests/config/list/config_list_no_config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_no_config.txt b/edq/testing/testdata/cli/tests/config/list/config_list_no_config.txt index b371d86..b0aa28f 100644 --- a/edq/testing/testdata/cli/tests/config/list/config_list_no_config.txt +++ b/edq/testing/testdata/cli/tests/config/list/config_list_no_config.txt @@ -1,6 +1,6 @@ { "cli": "edq.cli.config.list", - "arguments": [ + "arguments": [ "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", ], } From 234d4a6d410c461d858ba18e42d7a7ddb694c1ea Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sat, 27 Sep 2025 06:37:04 -0700 Subject: [PATCH 69/71] Increaed the max module lines for format. --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index e80e57c..ce49843 100644 --- a/.pylintrc +++ b/.pylintrc @@ -348,7 +348,7 @@ indent-string=' ' max-line-length=150 # Maximum number of lines in a module. -max-module-lines=1000 +max-module-lines=1050 # Allow the body of a class to be on the same line as the declaration if body # contains single statement. From a00427a79f01efe80a76f7cc35b765746ac5b98a Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sun, 28 Sep 2025 07:16:29 -0700 Subject: [PATCH 70/71] Done with 3rd pass. --- .pylintrc | 3 ++- edq/cli/config/list.py | 14 +++++++------- edq/core/config.py | 16 ++++++++-------- .../configs/{simple => simple-1}/edq-config.json | 1 + .../cli/data/configs/simple-2/edq-config.json | 3 +++ .../data/configs/value-number/edq-config.json | 3 +++ .../cli/tests/config/list/config_list_base.txt | 8 ++++++-- .../list/config_list_config_value_number.txt | 10 ++++++++++ .../config/list/config_list_ignore_config.txt | 6 ++++-- .../config/list/config_list_show_origin.txt | 6 +++--- .../config/list/config_list_skip_header.txt | 2 +- 11 files changed, 48 insertions(+), 24 deletions(-) rename edq/testing/testdata/cli/data/configs/{simple => simple-1}/edq-config.json (97%) create mode 100644 edq/testing/testdata/cli/data/configs/simple-2/edq-config.json create mode 100644 edq/testing/testdata/cli/data/configs/value-number/edq-config.json create mode 100644 edq/testing/testdata/cli/tests/config/list/config_list_config_value_number.txt diff --git a/.pylintrc b/.pylintrc index ce49843..fae5f05 100644 --- a/.pylintrc +++ b/.pylintrc @@ -348,7 +348,7 @@ indent-string=' ' max-line-length=150 # Maximum number of lines in a module. -max-module-lines=1050 +max-module-lines=1000 # Allow the body of a class to be on the same line as the declaration if body # contains single statement. @@ -444,6 +444,7 @@ disable=bad-inline-option, too-many-arguments, too-many-branches, too-many-instance-attributes, + too-many-lines, too-many-locals, too-many-positional-arguments, too-many-public-methods, diff --git a/edq/cli/config/list.py b/edq/cli/config/list.py index e474956..665bade 100644 --- a/edq/cli/config/list.py +++ b/edq/cli/config/list.py @@ -12,10 +12,10 @@ def run_cli(args: argparse.Namespace) -> int: """ Run the CLI. """ - configs_list = [] + rows = [] for (key, value) in args._config.items(): - config_list = [key, value] + row = [key, f"{value}"] if (args.show_origin): config_source_obj = args._config_sources.get(key) @@ -23,20 +23,20 @@ def run_cli(args: argparse.Namespace) -> int: if (origin is None): origin = config_source_obj.label - config_list.append(origin) + row.append(origin) - configs_list.append(CONFIG_FIELD_SEPARATOR.join(config_list)) + rows.append(CONFIG_FIELD_SEPARATOR.join(row)) - configs_list.sort() + rows.sort() if (not args.skip_header): header = ["Key", "Value"] if (args.show_origin): header.append("Origin") - configs_list.insert(0, (CONFIG_FIELD_SEPARATOR.join(header))) + rows.insert(0, (CONFIG_FIELD_SEPARATOR.join(header))) - print("\n".join(configs_list)) + print("\n".join(rows)) return 0 def main() -> int: diff --git a/edq/core/config.py b/edq/core/config.py index 3b094de..39256fd 100644 --- a/edq/core/config.py +++ b/edq/core/config.py @@ -197,7 +197,7 @@ def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, parser.add_argument('--config-global', dest = GLOBAL_CONFIG_KEY, action = 'store', type = str, default = DEFAULT_GLOBAL_CONFIG_PATH, - help = 'Override the default global config file path (default: %(default)s).', + help = 'Set the default global config file path (default: %(default)s).', ) parser.add_argument('--config-file', dest = CONFIG_PATHS_KEY, @@ -205,24 +205,24 @@ def set_cli_args(parser: argparse.ArgumentParser, extra_state: typing.Dict[str, help = ('Load config options from a JSON file.' + ' This flag can be specified multiple times.' + ' Files are applied in the order provided and later files override earlier ones.' - + ' This will override options form both global and local config files.') + + ' Will override options form both global and local config files.') ) - parser.add_argument('--config-option', dest = CONFIGS_KEY, + parser.add_argument('--config', dest = CONFIGS_KEY, action = 'append', type = str, default = [], - help = ('Load configuration option from the CLI command.' + help = ('Set a configuration option from the command-line.' + ' Specify options as = pairs.' + ' This flag can be specified multiple times.' + ' The options are applied in the order provided and later options override earlier ones.' - + ' This will override options form all config files.') + + ' Will override options form all config files.') ) parser.add_argument('--ignore-config-option', dest = IGNORE_CONFIGS_KEY, action = 'append', type = str, default = [], - help = ('Ignore any config option with the specified values.' - + ' The default value will be used for that option if one exists.' + help = ('Ignore any config option with the specified key.' + + ' The system-provided default value will be used for that option if one exists.' + ' This flag can be specified multiple times.' - + ' Ignoring options are processed last.') + + ' Ignored options are processed last.') ) def load_config_into_args( diff --git a/edq/testing/testdata/cli/data/configs/simple/edq-config.json b/edq/testing/testdata/cli/data/configs/simple-1/edq-config.json similarity index 97% rename from edq/testing/testdata/cli/data/configs/simple/edq-config.json rename to edq/testing/testdata/cli/data/configs/simple-1/edq-config.json index f619907..8257876 100644 --- a/edq/testing/testdata/cli/data/configs/simple/edq-config.json +++ b/edq/testing/testdata/cli/data/configs/simple-1/edq-config.json @@ -1,3 +1,4 @@ { "user": "user@test.edulinq.org", } + diff --git a/edq/testing/testdata/cli/data/configs/simple-2/edq-config.json b/edq/testing/testdata/cli/data/configs/simple-2/edq-config.json new file mode 100644 index 0000000..43df80f --- /dev/null +++ b/edq/testing/testdata/cli/data/configs/simple-2/edq-config.json @@ -0,0 +1,3 @@ +{ + "api-key": "TEST-KEY-1234567890", +} diff --git a/edq/testing/testdata/cli/data/configs/value-number/edq-config.json b/edq/testing/testdata/cli/data/configs/value-number/edq-config.json new file mode 100644 index 0000000..250fb55 --- /dev/null +++ b/edq/testing/testdata/cli/data/configs/value-number/edq-config.json @@ -0,0 +1,3 @@ +{ + "pass": 1234567890 +} diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_base.txt b/edq/testing/testdata/cli/tests/config/list/config_list_base.txt index ac2312c..485defb 100644 --- a/edq/testing/testdata/cli/tests/config/list/config_list_base.txt +++ b/edq/testing/testdata/cli/tests/config/list/config_list_base.txt @@ -2,11 +2,15 @@ "cli": "edq.cli.config.list", "arguments": [ "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", - "--config-file", "__DATA_DIR__(configs/simple/edq-config.json)", - "--config-option", "pass=password1234", + "--config-file", "__DATA_DIR__(configs/simple-1/edq-config.json)", + "--config-file", "__DATA_DIR__(configs/simple-2/edq-config.json)", + "--config", "pass=password1234", + "--config", "server=http://test.edulinq.org" ], } --- Key Value +api-key TEST-KEY-1234567890 pass password1234 +server http://test.edulinq.org user user@test.edulinq.org diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_config_value_number.txt b/edq/testing/testdata/cli/tests/config/list/config_list_config_value_number.txt new file mode 100644 index 0000000..aa03c66 --- /dev/null +++ b/edq/testing/testdata/cli/tests/config/list/config_list_config_value_number.txt @@ -0,0 +1,10 @@ +{ + "cli": "edq.cli.config.list", + "arguments": [ + "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", + "--config-file", "__DATA_DIR__(configs/value-number/edq-config.json)", + ], +} +--- +Key Value +pass 1234567890 diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_ignore_config.txt b/edq/testing/testdata/cli/tests/config/list/config_list_ignore_config.txt index f66eddf..7afbe36 100644 --- a/edq/testing/testdata/cli/tests/config/list/config_list_ignore_config.txt +++ b/edq/testing/testdata/cli/tests/config/list/config_list_ignore_config.txt @@ -2,9 +2,11 @@ "cli": "edq.cli.config.list", "arguments": [ "--ignore-config-option", "pass", + "--ignore-config-option", "api-key", "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", - "--config-file", "__DATA_DIR__(configs/simple/edq-config.json)", - "--config-option", "pass=password1234", + "--config-file", "__DATA_DIR__(configs/simple-1/edq-config.json)", + "--config-file", "__DATA_DIR__(configs/simple-2/edq-config.json)", + "--config", "pass=password1234", ], } --- diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_show_origin.txt b/edq/testing/testdata/cli/tests/config/list/config_list_show_origin.txt index e10d6b6..be4e4c1 100644 --- a/edq/testing/testdata/cli/tests/config/list/config_list_show_origin.txt +++ b/edq/testing/testdata/cli/tests/config/list/config_list_show_origin.txt @@ -1,13 +1,13 @@ { "cli": "edq.cli.config.list", "arguments": [ - "--config-file", "__DATA_DIR__(configs/simple/edq-config.json)", + "--config-file", "__DATA_DIR__(configs/simple-1/edq-config.json)", "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", - "--config-option", "pass=password1234", + "--config", "pass=password1234", "--show-origin", ], } --- Key Value Origin pass password1234 -user user@test.edulinq.org __ABS_DATA_DIR__(configs/simple/edq-config.json) +user user@test.edulinq.org __ABS_DATA_DIR__(configs/simple-1/edq-config.json) diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_skip_header.txt b/edq/testing/testdata/cli/tests/config/list/config_list_skip_header.txt index d289905..64d3b8a 100644 --- a/edq/testing/testdata/cli/tests/config/list/config_list_skip_header.txt +++ b/edq/testing/testdata/cli/tests/config/list/config_list_skip_header.txt @@ -2,7 +2,7 @@ "cli": "edq.cli.config.list", "arguments": [ "--config-global", "__DATA_DIR__(configs/empty/edq-config.json)", - "--config-option", "user=user@test.edulinq.org", + "--config", "user=user@test.edulinq.org", "--skip-header", ], } From 2241faabd4fe49f5b24df98f69ef762bc438b48e Mon Sep 17 00:00:00 2001 From: BatuhanSA Date: Sun, 28 Sep 2025 07:53:32 -0700 Subject: [PATCH 71/71] Simplified str conversion, chnaged the numeric value test. --- edq/cli/config/list.py | 2 +- .../testdata/cli/data/configs/value-number/edq-config.json | 2 +- .../cli/tests/config/list/config_list_config_value_number.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/edq/cli/config/list.py b/edq/cli/config/list.py index 665bade..d630627 100644 --- a/edq/cli/config/list.py +++ b/edq/cli/config/list.py @@ -15,7 +15,7 @@ def run_cli(args: argparse.Namespace) -> int: rows = [] for (key, value) in args._config.items(): - row = [key, f"{value}"] + row = [key, str(value)] if (args.show_origin): config_source_obj = args._config_sources.get(key) diff --git a/edq/testing/testdata/cli/data/configs/value-number/edq-config.json b/edq/testing/testdata/cli/data/configs/value-number/edq-config.json index 250fb55..ddf6976 100644 --- a/edq/testing/testdata/cli/data/configs/value-number/edq-config.json +++ b/edq/testing/testdata/cli/data/configs/value-number/edq-config.json @@ -1,3 +1,3 @@ { - "pass": 1234567890 + "timeout": 60 } diff --git a/edq/testing/testdata/cli/tests/config/list/config_list_config_value_number.txt b/edq/testing/testdata/cli/tests/config/list/config_list_config_value_number.txt index aa03c66..dd550e2 100644 --- a/edq/testing/testdata/cli/tests/config/list/config_list_config_value_number.txt +++ b/edq/testing/testdata/cli/tests/config/list/config_list_config_value_number.txt @@ -7,4 +7,4 @@ } --- Key Value -pass 1234567890 +timeout 60