diff --git a/changelog/65027.fixed.md b/changelog/65027.fixed.md new file mode 100644 index 000000000000..a2692b4a83db --- /dev/null +++ b/changelog/65027.fixed.md @@ -0,0 +1 @@ +Ensure sync from _grains occurs before attempting pillar compilation in case custom grain used in pillar file and salt-minion in masterless mode diff --git a/salt/fileclient.py b/salt/fileclient.py index 32c5cd0d948d..4c60d93fcab3 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -46,12 +46,15 @@ MAX_FILENAME_LENGTH = 255 -def get_file_client(opts, pillar=False): +def get_file_client(opts, pillar=False, force_local=False): """ Read in the ``file_client`` option and return the correct type of file server """ - client = opts.get("file_client", "remote") + if force_local: + client = "local" + else: + client = opts.get("file_client", "remote") if pillar and client == "local": client = "pillar" diff --git a/salt/minion.py b/salt/minion.py index 12c9a86ba2a8..04787ad93798 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -113,6 +113,24 @@ # 6. Handle publications +def _sync_grains(opts): + # if local client (masterless minion), need sync of custom grains + # as they may be used in pillar compilation + # in addition, with masterless minion some opts may not be filled + # at this point of syncing,for example sometimes does not contain + # extmod_whitelist and extmod_blacklist hence set those to defaults, + # empty dict, if not part of opts, as ref'd in + # salt.utils.extmod sync function + if "local" == opts.get("file_client", "remote"): + if opts.get("extmod_whitelist", None) is None: + opts["extmod_whitelist"] = {} + + if opts.get("extmod_blacklist", None) is None: + opts["extmod_blacklist"] = {} + + salt.utils.extmods.sync(opts, "grains", force_local=True) + + def resolve_dns(opts, fallback=True): """ Resolves the master_ip and master_uri options @@ -920,6 +938,7 @@ def __init__(self, opts, context=None): # Late setup of the opts grains, so we can log from the grains module import salt.loader + _sync_grains(opts) opts["grains"] = salt.loader.grains(opts) super().__init__(opts) diff --git a/salt/utils/extmods.py b/salt/utils/extmods.py index f1b8a8264483..8601cfeedbc3 100644 --- a/salt/utils/extmods.py +++ b/salt/utils/extmods.py @@ -39,6 +39,7 @@ def sync( saltenv=None, extmod_whitelist=None, extmod_blacklist=None, + force_local=False, ): """ Sync custom modules into the extension_modules directory @@ -82,7 +83,9 @@ def sync( "Cannot create cache module directory %s. Check permissions.", mod_dir, ) - with salt.fileclient.get_file_client(opts) as fileclient: + with salt.fileclient.get_file_client( + opts, pillar=False, force_local=force_local + ) as fileclient: for sub_env in saltenv: log.info("Syncing %s for environment '%s'", form, sub_env) cache = [] diff --git a/tests/pytests/integration/cli/test_salt_call.py b/tests/pytests/integration/cli/test_salt_call.py index 1d770c0ffbed..d964d11e66cb 100644 --- a/tests/pytests/integration/cli/test_salt_call.py +++ b/tests/pytests/integration/cli/test_salt_call.py @@ -429,3 +429,86 @@ def test_local_salt_call_no_function_no_retcode(salt_call_cli): assert "test" in ret.data assert ret.data["test"] == "'test' is not available." assert "test.echo" in ret.data + + +def test_state_highstate_custom_grains_masterless_mode( + salt_master, salt_minion_factory +): + """ + This test ensure that custom grains in salt://_grains are loaded before pillar compilation + to ensure that any use of custom grains in pillar files are available when in masterless mode, + this implies that a sync of grains occurs before loading the regular + /etc/salt/grains or configuration file grains, as well as the usual grains. + Note: cannot use salt_minion and salt_call_cli, since these will be loaded before + the pillar and custom_grains files are written, hence using salt_minion_factory. + """ + pillar_top_sls = """ + base: + '*': + - defaults + """ + + pillar_defaults_sls = """ + mypillar: "{{ grains['custom_grain'] }}" + """ + + salt_top_sls = """ + base: + '*': + - test + """ + + salt_test_sls = """ + "donothing": + test.nop: [] + """ + + salt_custom_grains_py = """ + def main(): + return {'custom_grain': 'test_value'} + """ + + assert salt_master.is_running() + with salt_minion_factory.started(): + salt_minion = salt_minion_factory + salt_call_cli = salt_minion_factory.salt_call_cli() + with salt_minion.pillar_tree.base.temp_file( + "top.sls", pillar_top_sls + ), salt_minion.pillar_tree.base.temp_file( + "defaults.sls", pillar_defaults_sls + ), salt_minion.state_tree.base.temp_file( + "top.sls", salt_top_sls + ), salt_minion.state_tree.base.temp_file( + "test.sls", salt_test_sls + ), salt_minion.state_tree.base.temp_file( + "_grains/custom_grain.py", salt_custom_grains_py + ): + ## need to try masterless mode + ret = salt_call_cli.run("--local", "state.highstate") + assert ret.returncode == 0 + ret = salt_call_cli.run("pillar.items") + assert ret.returncode == 0 + assert ret.data + pillar_items = ret.data + assert "mypillar" in pillar_items + assert pillar_items["mypillar"] == "test_value" + + ## need to try with master mode + ret = salt_call_cli.run("state.highstate") + assert ret.returncode == 0 + ret = salt_call_cli.run("pillar.items") + assert ret.returncode == 0 + assert ret.data + pillar_items = ret.data + assert "mypillar" not in pillar_items + + +def test_salt_call_versions(salt_call_cli, caplog): + """ + Call test.versions without '--local' to test grains + are sync'd without any missing keys in opts + """ + with caplog.at_level(logging.DEBUG): + ret = salt_call_cli.run("test.versions") + assert ret.returncode == 0 + assert "Failed to sync grains module: 'master_uri'" not in caplog.messages