From bca72d23e96b9a88b3b5b4841075dede0fcfab65 Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Wed, 20 Dec 2023 15:37:49 -0700 Subject: [PATCH 1/4] [WIP] initial changes for _sync_grains only affecting masterless minions --- salt/minion.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/salt/minion.py b/salt/minion.py index 12c9a86ba2a8..95588496c996 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -113,6 +113,24 @@ # 6. Handle publications +def _sync_grains(opts): + # if local client (masterless minion), need sync of custom grains + # as they may be used in pillar compilation + # in addition, with masterless minion some opts may not be filled + # at this point of syncing,for example sometimes does not contain + # extmod_whitelist and extmod_blacklist hence set those to defaults, + # empty dict, if not part of opts, as ref'd in + # salt.utils.extmod sync function + if "local" == opts.get("file_client", "remote"): + if opts.get("extmod_whitelist", None) is None: + opts["extmod_whitelist"] = {} + + if opts.get("extmod_blacklist", None) is None: + opts["extmod_blacklist"] = {} + + salt.utils.extmods.sync(opts, "grains", force_local=True) + + def resolve_dns(opts, fallback=True): """ Resolves the master_ip and master_uri options From 7b7b3c4fef6c2bd738639c4127e719014f792b21 Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Thu, 4 Jan 2024 14:44:37 -0700 Subject: [PATCH 2/4] Added changelog --- changelog/65027.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/65027.fixed.md diff --git a/changelog/65027.fixed.md b/changelog/65027.fixed.md new file mode 100644 index 000000000000..a2692b4a83db --- /dev/null +++ b/changelog/65027.fixed.md @@ -0,0 +1 @@ +Ensure sync from _grains occurs before attempting pillar compilation in case custom grain used in pillar file and salt-minion in masterless mode From a54d4e6b9b4cfe1d70a27e669cc0e9fa23ba9dcc Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Thu, 4 Jan 2024 15:11:54 -0700 Subject: [PATCH 3/4] Added code to handle masterless sync of salt-minion, tests work-in-progress --- salt/fileclient.py | 7 +- salt/minion.py | 1 + salt/utils/extmods.py | 6 +- .../pytests/integration/cli/test_salt_call.py | 131 ++++++++++++++++++ 4 files changed, 142 insertions(+), 3 deletions(-) diff --git a/salt/fileclient.py b/salt/fileclient.py index 32c5cd0d948d..4c60d93fcab3 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -46,12 +46,15 @@ MAX_FILENAME_LENGTH = 255 -def get_file_client(opts, pillar=False): +def get_file_client(opts, pillar=False, force_local=False): """ Read in the ``file_client`` option and return the correct type of file server """ - client = opts.get("file_client", "remote") + if force_local: + client = "local" + else: + client = opts.get("file_client", "remote") if pillar and client == "local": client = "pillar" diff --git a/salt/minion.py b/salt/minion.py index 95588496c996..04787ad93798 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -938,6 +938,7 @@ def __init__(self, opts, context=None): # Late setup of the opts grains, so we can log from the grains module import salt.loader + _sync_grains(opts) opts["grains"] = salt.loader.grains(opts) super().__init__(opts) diff --git a/salt/utils/extmods.py b/salt/utils/extmods.py index f1b8a8264483..e46ce13611cf 100644 --- a/salt/utils/extmods.py +++ b/salt/utils/extmods.py @@ -39,6 +39,7 @@ def sync( saltenv=None, extmod_whitelist=None, extmod_blacklist=None, + force_local=False, ): """ Sync custom modules into the extension_modules directory @@ -82,7 +83,10 @@ def sync( "Cannot create cache module directory %s. Check permissions.", mod_dir, ) - with salt.fileclient.get_file_client(opts) as fileclient: + ## DGM with salt.fileclient.get_file_client(opts) as fileclient: + with salt.fileclient.get_file_client( + opts, pillar=False, force_local=force_local + ) as fileclient: for sub_env in saltenv: log.info("Syncing %s for environment '%s'", form, sub_env) cache = [] diff --git a/tests/pytests/integration/cli/test_salt_call.py b/tests/pytests/integration/cli/test_salt_call.py index 1d770c0ffbed..ce3b4ad5d0b8 100644 --- a/tests/pytests/integration/cli/test_salt_call.py +++ b/tests/pytests/integration/cli/test_salt_call.py @@ -429,3 +429,134 @@ def test_local_salt_call_no_function_no_retcode(salt_call_cli): assert "test" in ret.data assert ret.data["test"] == "'test' is not available." assert "test.echo" in ret.data + + +def test_state_highstate_custom_grains_masterless_mode( + salt_master, salt_minion_factory +): + """ + This test ensure that custom grains in salt://_grains are loaded before pillar compilation + to ensure that any use of custom grains in pillar files are available when in masterless mode, + this implies that a sync of grains occurs before loading the regular + /etc/salt/grains or configuration file grains, as well as the usual grains. + Note: cannot use salt_minion and salt_call_cli, since these will be loaded before + the pillar and custom_grains files are written, hence using salt_minion_factory. + """ + pillar_top_sls = """ + base: + '*': + - defaults + """ + + pillar_defaults_sls = """ + mypillar: "{{ grains['custom_grain'] }}" + """ + + salt_top_sls = """ + base: + '*': + - test + """ + + salt_test_sls = """ + "donothing": + test.nop: [] + """ + + salt_custom_grains_py = """ + def main(): + return {'custom_grain': 'test_value'} + """ + + ## DGM TBD need to get masterless mode + assert salt_master.is_running() + with salt_minion_factory.started(): + salt_minion = salt_minion_factory + salt_call_cli = salt_minion_factory.salt_call_cli() + with salt_minion.pillar_tree.base.temp_file( + "top.sls", pillar_top_sls + ), salt_minion.pillar_tree.base.temp_file( + "defaults.sls", pillar_defaults_sls + ), salt_minion.state_tree.base.temp_file( + "top.sls", salt_top_sls + ), salt_minion.state_tree.base.temp_file( + "test.sls", salt_test_sls + ), salt_minion.state_tree.base.temp_file( + "_grains/custom_grain.py", salt_custom_grains_py + ): + ret = salt_call_cli.run("state.highstate") + assert ret.returncode == 0 + ret = salt_call_cli.run("pillar.items") + assert ret.returncode == 0 + assert ret.data + pillar_items = ret.data + assert "mypillar" in pillar_items + assert pillar_items["mypillar"] == "test_value" + + +def test_state_highstate_custom_grains_master_mode(salt_master, salt_minion_factory): + """ + This test ensure that custom grains in salt://_grains are loaded before pillar compilation + to ensure that any use of custom grains in pillar files are unaffected by changes for + sync custom grains for the salt-minion in masterless mode, + """ + pillar_top_sls = """ + base: + '*': + - defaults + """ + + pillar_defaults_sls = """ + mypillar: "{{ grains['custom_grain'] }}" + """ + + salt_top_sls = """ + base: + '*': + - test + """ + + salt_test_sls = """ + "donothing": + test.nop: [] + """ + + salt_custom_grains_py = """ + def main(): + return {'custom_grain': 'test_value'} + """ + + ## DGM TBD need to ensure master mode + assert salt_master.is_running() + with salt_minion_factory.started(): + salt_minion = salt_minion_factory + salt_call_cli = salt_minion_factory.salt_call_cli() + with salt_minion.pillar_tree.base.temp_file( + "top.sls", pillar_top_sls + ), salt_minion.pillar_tree.base.temp_file( + "defaults.sls", pillar_defaults_sls + ), salt_minion.state_tree.base.temp_file( + "top.sls", salt_top_sls + ), salt_minion.state_tree.base.temp_file( + "test.sls", salt_test_sls + ), salt_minion.state_tree.base.temp_file( + "_grains/custom_grain.py", salt_custom_grains_py + ): + ret = salt_call_cli.run("state.highstate") + assert ret.returncode == 0 + ret = salt_call_cli.run("pillar.items") + assert ret.returncode == 0 + assert ret.data + pillar_items = ret.data + assert "mypillar" not in pillar_items + + +def test_salt_call_versions(salt_call_cli, caplog): + """ + Call test.versions without '--local' to test grains + are sync'd without any missing keys in opts + """ + with caplog.at_level(logging.DEBUG): + ret = salt_call_cli.run("test.versions") + assert ret.returncode == 0 + assert "Failed to sync grains module: 'master_uri'" not in caplog.messages From 63ca6f4a57210ec58e1fe8f8817d0a7f0b40849a Mon Sep 17 00:00:00 2001 From: David Murphy < dmurphy@saltstack.com> Date: Tue, 7 May 2024 15:03:08 -0600 Subject: [PATCH 4/4] Initial refactoring of tests for testing masterless minion and sync of custom pillar grains --- salt/utils/extmods.py | 1 - .../pytests/integration/cli/test_salt_call.py | 54 ++----------------- 2 files changed, 3 insertions(+), 52 deletions(-) diff --git a/salt/utils/extmods.py b/salt/utils/extmods.py index e46ce13611cf..8601cfeedbc3 100644 --- a/salt/utils/extmods.py +++ b/salt/utils/extmods.py @@ -83,7 +83,6 @@ def sync( "Cannot create cache module directory %s. Check permissions.", mod_dir, ) - ## DGM with salt.fileclient.get_file_client(opts) as fileclient: with salt.fileclient.get_file_client( opts, pillar=False, force_local=force_local ) as fileclient: diff --git a/tests/pytests/integration/cli/test_salt_call.py b/tests/pytests/integration/cli/test_salt_call.py index ce3b4ad5d0b8..d964d11e66cb 100644 --- a/tests/pytests/integration/cli/test_salt_call.py +++ b/tests/pytests/integration/cli/test_salt_call.py @@ -468,7 +468,6 @@ def main(): return {'custom_grain': 'test_value'} """ - ## DGM TBD need to get masterless mode assert salt_master.is_running() with salt_minion_factory.started(): salt_minion = salt_minion_factory @@ -484,7 +483,8 @@ def main(): ), salt_minion.state_tree.base.temp_file( "_grains/custom_grain.py", salt_custom_grains_py ): - ret = salt_call_cli.run("state.highstate") + ## need to try masterless mode + ret = salt_call_cli.run("--local", "state.highstate") assert ret.returncode == 0 ret = salt_call_cli.run("pillar.items") assert ret.returncode == 0 @@ -493,55 +493,7 @@ def main(): assert "mypillar" in pillar_items assert pillar_items["mypillar"] == "test_value" - -def test_state_highstate_custom_grains_master_mode(salt_master, salt_minion_factory): - """ - This test ensure that custom grains in salt://_grains are loaded before pillar compilation - to ensure that any use of custom grains in pillar files are unaffected by changes for - sync custom grains for the salt-minion in masterless mode, - """ - pillar_top_sls = """ - base: - '*': - - defaults - """ - - pillar_defaults_sls = """ - mypillar: "{{ grains['custom_grain'] }}" - """ - - salt_top_sls = """ - base: - '*': - - test - """ - - salt_test_sls = """ - "donothing": - test.nop: [] - """ - - salt_custom_grains_py = """ - def main(): - return {'custom_grain': 'test_value'} - """ - - ## DGM TBD need to ensure master mode - assert salt_master.is_running() - with salt_minion_factory.started(): - salt_minion = salt_minion_factory - salt_call_cli = salt_minion_factory.salt_call_cli() - with salt_minion.pillar_tree.base.temp_file( - "top.sls", pillar_top_sls - ), salt_minion.pillar_tree.base.temp_file( - "defaults.sls", pillar_defaults_sls - ), salt_minion.state_tree.base.temp_file( - "top.sls", salt_top_sls - ), salt_minion.state_tree.base.temp_file( - "test.sls", salt_test_sls - ), salt_minion.state_tree.base.temp_file( - "_grains/custom_grain.py", salt_custom_grains_py - ): + ## need to try with master mode ret = salt_call_cli.run("state.highstate") assert ret.returncode == 0 ret = salt_call_cli.run("pillar.items")