diff --git a/crmsh/bootstrap.py b/crmsh/bootstrap.py index e11a96d367..73e04583c1 100644 --- a/crmsh/bootstrap.py +++ b/crmsh/bootstrap.py @@ -23,6 +23,7 @@ import readline import shutil import typing +import shlex import yaml import socket @@ -2158,11 +2159,6 @@ def update_nodeid(nodeid, node=None): shell = sh.cluster_shell() - if is_qdevice_configured and not _context.use_ssh_agent: - # trigger init_qnetd_remote on init node - cmd = f"crm cluster init qnetd_remote {utils.this_node()} -y" - shell.get_stdout_or_raise_error(cmd, seed_host) - shutil.copy(corosync.conf(), COROSYNC_CONF_ORIG) # check if use IPv6 @@ -2300,6 +2296,10 @@ def update_nodeid(nodeid, node=None): sync_files_to_disk() if is_qdevice_configured: + if not _context.use_ssh_agent: + # trigger init_qnetd_remote on init node + cmd = f"crm cluster init qnetd_remote {utils.this_node()} -y" + shell.get_stdout_or_raise_error(cmd, seed_host) start_qdevice_on_join_node(seed_host) else: ServiceManager(sh.ClusterShellAdaptorForLocalShell(sh.LocalShell())).disable_service("corosync-qdevice.service") @@ -2347,10 +2347,7 @@ def start_qdevice_on_join_node(seed_host): corosync.add_nodelist_from_cmaptool() sync_file(corosync.conf()) invoke("crm corosync reload") - if utils.is_qdevice_tls_on(): - qnetd_addr = corosync.get_value("quorum.device.net.host") - qdevice_inst = qdevice.QDevice(qnetd_addr, cluster_node=seed_host) - qdevice_inst.certificate_process_on_join() + retrieve_data(seed_host, [qdevice.QDevice.qdevice_path], "qdevice") ServiceManager(sh.ClusterShellAdaptorForLocalShell(sh.LocalShell())).start_service("corosync-qdevice.service", enable=True) @@ -2689,7 +2686,7 @@ def bootstrap_join(context): _context.skip_csync2 = not service_manager.service_is_active(CSYNC2_SERVICE, cluster_node) if _context.skip_csync2: service_manager.stop_service(CSYNC2_SERVICE, disable=True) - retrieve_all_config_files(cluster_node) + retrieve_data(cluster_node) logger.warning("csync2 is not initiated yet. Before using csync2 for the first time, please run \"crm cluster init csync2 -y\" on any one node. Note, this may take a while.") else: join_csync2(cluster_node, remote_user) @@ -3138,14 +3135,18 @@ def adjust_properties(): adjust_priority_fencing_delay(is_2node_wo_qdevice) -def retrieve_all_config_files(cluster_node): - """ - Retrieve config files from cluster_node if exists - """ - with logger_utils.status_long("Retrieve all config files"): - cmd = 'cpio -o << EOF\n{}\nEOF\n'.format( - '\n'.join((f for f in FILES_TO_SYNC if f != CSYNC2_KEY and f != CSYNC2_CFG)) - ) +def retrieve_data(from_node, data_list=None, data_type=None): + if not data_list: + data_list = [f for f in FILES_TO_SYNC if f != CSYNC2_KEY and f != CSYNC2_CFG] + find_args = ' '.join(shlex.quote(f) for f in data_list) + cmd = f'find {find_args} -print | cpio -o' + + if data_type: + msg = f"Retrieving {data_type} configuration files from {from_node}" + else: + msg = f"Retrieving all configuration files from {from_node}" + + with logger_utils.status_long(msg): pipe_outlet, pipe_inlet = os.pipe() try: child = subprocess.Popen(['cpio', '-iu'], stdin=pipe_outlet, stderr=subprocess.DEVNULL) @@ -3155,15 +3156,17 @@ def retrieve_all_config_files(cluster_node): finally: os.close(pipe_outlet) try: - result = sh.cluster_shell().subprocess_run_without_input(cluster_node, None, cmd, stdout=pipe_inlet, stderr=subprocess.DEVNULL) + result = sh.cluster_shell().subprocess_run_without_input( + from_node, None, cmd, stdout=pipe_inlet, stderr=subprocess.DEVNULL + ) finally: os.close(pipe_inlet) rc = child.wait() # Some errors may happen here, since all files in FILES_TO_SYNC may not exist. if result is None or result.returncode == 255: - utils.fatal("Failed to create ssh connect to {}".format(cluster_node)) + utils.fatal(f"Failed to create ssh connection to {from_node}") if rc != 0: - utils.fatal("Failed to retrieve config files from {}".format(cluster_node)) + utils.fatal(f"Failed to retrieve config files from {from_node}") def sync_file(path): diff --git a/crmsh/qdevice.py b/crmsh/qdevice.py index d8fed8dc46..7f119bd485 100644 --- a/crmsh/qdevice.py +++ b/crmsh/qdevice.py @@ -101,7 +101,6 @@ class QDevice(object): """Class to manage qdevice configuration and services Call `certificate_process_on_init` to generate all of CA, server, and client certs. - Call `certificate_process_on_join` to generate a single client cert for the local node. """ qnetd_service = "corosync-qnetd.service" @@ -113,7 +112,7 @@ class QDevice(object): qdevice_db_path = "/etc/corosync/qdevice/net/nssdb" def __init__(self, qnetd_addr, port=5403, algo="ffsplit", tie_breaker="lowest", - tls="on", ssh_user=None, cluster_node=None, cmds=None, mode=None, cluster_name=None, is_stage=False): + tls="on", ssh_user=None, cmds=None, mode=None, cluster_name=None, is_stage=False): """ Init function """ @@ -123,7 +122,6 @@ def __init__(self, qnetd_addr, port=5403, algo="ffsplit", tie_breaker="lowest", self.tie_breaker = tie_breaker self.tls = tls self.ssh_user = ssh_user - self.cluster_node = cluster_node self.cmds = cmds self.mode = mode self.cluster_name = cluster_name @@ -145,13 +143,6 @@ def qnetd_cacert_on_local(self): """ return "{}/{}/{}".format(self.qdevice_path, self.qnetd_addr, self.qnetd_cacert_filename) - @property - def qnetd_cacert_on_cluster(self): - """ - Return path of qnetd-cacert.crt on cluster node - """ - return "{}/{}/{}".format(self.qdevice_path, self.cluster_node, self.qnetd_cacert_filename) - @property def qdevice_crq_on_qnetd(self): """ @@ -187,13 +178,6 @@ def qdevice_p12_on_local(self): """ return "{}/nssdb/{}".format(self.qdevice_path, self.qdevice_p12_filename) - @property - def qdevice_p12_on_cluster(self): - """ - Return path of qdevice-net-node.p12 on cluster node - """ - return "{}/{}/{}".format(self.qdevice_path, self.cluster_node, self.qdevice_p12_filename) - @staticmethod def check_qnetd_addr(qnetd_addr): qnetd_ip = None @@ -455,65 +439,6 @@ def certificate_process_on_init(self): ]): step(lambda s, cmd=None: self.log_only_to_file(f'Step {i+1}: {s}', cmd)) - def fetch_qnetd_crt_from_cluster(self): - """ - Certificate process for join - Step 1 - Fetch QNetd CA certificate(qnetd-cacert.crt) from init node - """ - if os.path.exists(self.qnetd_cacert_on_cluster): - return - - desc = "Step 1: Fetch {} from {}".format(self.qnetd_cacert_filename, self.cluster_node) - QDevice.log_only_to_file(desc) - crmsh.parallax.parallax_slurp([self.cluster_node], self.qdevice_path, self.qnetd_cacert_on_local) - - def init_db_on_local(self): - """ - Certificate process for join - Step 2 - Initialize database by running - /usr/sbin/corosync-qdevice-net-certutil -i -c qnetd-cacert.crt - """ - if os.path.exists(self.qdevice_db_path): - utils.rmdir_r(self.qdevice_db_path) - - cmd = "corosync-qdevice-net-certutil -i -c {}".format(self.qnetd_cacert_on_cluster) - QDevice.log_only_to_file("Step 2: Initialize database on local", cmd) - sh.cluster_shell().get_stdout_or_raise_error(cmd) - - def fetch_p12_from_cluster(self): - """ - Certificate process for join - Step 3 - Fetch p12 key file from init node - """ - if os.path.exists(self.qdevice_p12_on_cluster): - return - - desc = "Step 3: Fetch {} from {}".format(self.qdevice_p12_filename, self.cluster_node) - QDevice.log_only_to_file(desc) - crmsh.parallax.parallax_slurp([self.cluster_node], self.qdevice_path, self.qdevice_p12_on_local) - - def import_p12_on_local(self): - """ - Certificate process for join - Step 4 - Import cluster certificate and key - """ - cmd = "corosync-qdevice-net-certutil -m -c {}".format(self.qdevice_p12_on_cluster) - QDevice.log_only_to_file("Step 4: Import cluster certificate and key", cmd) - sh.cluster_shell().get_stdout_or_raise_error(cmd) - - def certificate_process_on_join(self): - """ - The qdevice certificate process on join node - """ - self.fetch_qnetd_crt_from_cluster() - self.init_db_on_local() - self.fetch_p12_from_cluster() - self.import_p12_on_local() - def write_qdevice_config(self): """ Write qdevice attributes to config file diff --git a/test/features/qdevice_setup_remove.feature b/test/features/qdevice_setup_remove.feature index dbd0aa1bc0..d4f961662e 100644 --- a/test/features/qdevice_setup_remove.feature +++ b/test/features/qdevice_setup_remove.feature @@ -131,6 +131,17 @@ Feature: corosync qdevice/qnetd setup/remove process And Service "corosync-qdevice" is "started" on "hanode1" And Show status from qnetd + @clean + Scenario: Re-join cluster (bsc#1254243) + When Run "crm cluster init --qnetd-hostname=qnetd-node -y" on "hanode1" + When Run "crm cluster join -c hanode1 -y" on "hanode2" + Then Service "corosync-qdevice" is "started" on "hanode1" + And Service "corosync-qdevice" is "started" on "hanode2" + When Run "crm cluster remove hanode1 -y" on "hanode2" + Then Service "corosync-qdevice" is "stopped" on "hanode1" + When Run "crm cluster join -c hanode2 -y" on "hanode1" + Then Service "corosync-qdevice" is "started" on "hanode1" + @skip_non_root @clean Scenario: Passwordless for root, not for sudoer (bsc#1209193) diff --git a/test/unittests/test_bootstrap.py b/test/unittests/test_bootstrap.py index cdd8d599a9..4041e495f7 100644 --- a/test/unittests/test_bootstrap.py +++ b/test/unittests/test_bootstrap.py @@ -1275,9 +1275,7 @@ def test_remove_qdevice_reload(self, mock_qdevice_configured, mock_confirm, mock mock_remove_db.assert_called_once_with() @mock.patch('crmsh.service_manager.ServiceManager.start_service') - @mock.patch('crmsh.qdevice.QDevice') - @mock.patch('crmsh.corosync.get_value') - @mock.patch('crmsh.utils.is_qdevice_tls_on') + @mock.patch('crmsh.bootstrap.retrieve_data') @mock.patch('crmsh.bootstrap.invoke') @mock.patch('crmsh.bootstrap.sync_file') @mock.patch('crmsh.corosync.conf') @@ -1285,15 +1283,9 @@ def test_remove_qdevice_reload(self, mock_qdevice_configured, mock_confirm, mock @mock.patch('crmsh.corosync.is_unicast') @mock.patch('crmsh.log.LoggerUtils.status_long') def test_start_qdevice_on_join_node(self, mock_status_long, mock_is_unicast, mock_add_nodelist, - mock_conf, mock_csync2_update, mock_invoke, mock_qdevice_tls, - mock_get_value, mock_qdevice, mock_start_service): + mock_conf, mock_csync2_update, mock_invoke, mock_retrieve_data, mock_start_service): mock_is_unicast.return_value = False - mock_qdevice_tls.return_value = True mock_conf.return_value = "corosync.conf" - mock_get_value.return_value = "10.10.10.123" - mock_qdevice_inst = mock.Mock() - mock_qdevice.return_value = mock_qdevice_inst - mock_qdevice_inst.certificate_process_on_join = mock.Mock() bootstrap.start_qdevice_on_join_node("node2") @@ -1303,10 +1295,6 @@ def test_start_qdevice_on_join_node(self, mock_status_long, mock_is_unicast, moc mock_conf.assert_called_once_with() mock_csync2_update.assert_called_once_with("corosync.conf") mock_invoke.assert_called_once_with("crm corosync reload") - mock_qdevice_tls.assert_called_once_with() - mock_get_value.assert_called_once_with("quorum.device.net.host") - mock_qdevice.assert_called_once_with("10.10.10.123", cluster_node="node2") - mock_qdevice_inst.certificate_process_on_join.assert_called_once_with() mock_start_service.assert_called_once_with("corosync-qdevice.service", enable=True) @mock.patch('crmsh.sh.ShellUtils.get_stdout_stderr') diff --git a/test/unittests/test_qdevice.py b/test/unittests/test_qdevice.py index 41664f9ab2..9fff09844a 100644 --- a/test/unittests/test_qdevice.py +++ b/test/unittests/test_qdevice.py @@ -154,7 +154,6 @@ def setUp(self): self.qdevice_with_hostname = qdevice.QDevice("node.qnetd") self.qdevice_with_invalid_port = qdevice.QDevice("10.10.10.123", port=100) self.qdevice_with_invalid_tie_breaker = qdevice.QDevice("10.10.10.123", tie_breaker="wrong") - self.qdevice_with_ip_cluster_node = qdevice.QDevice("10.10.10.123", cluster_node="node1.com") self.qdevice_with_invalid_cmds_relative_path = qdevice.QDevice("10.10.10.123", cmds="ls") self.qdevice_with_invalid_cmds_not_exist = qdevice.QDevice("10.10.10.123", cmds="/not_exist") self.qdevice_with_cluster_name = qdevice.QDevice("10.10.10.123", cluster_name="hacluster1") @@ -175,10 +174,6 @@ def test_qnetd_cacert_on_local(self): res = self.qdevice_with_ip.qnetd_cacert_on_local self.assertEqual(res, "/etc/corosync/qdevice/net/10.10.10.123/qnetd-cacert.crt") - def test_qnetd_cacert_on_cluster(self): - res = self.qdevice_with_ip_cluster_node.qnetd_cacert_on_cluster - self.assertEqual(res, "/etc/corosync/qdevice/net/node1.com/qnetd-cacert.crt") - def test_qdevice_crq_on_qnetd(self): res = self.qdevice_with_cluster_name.qdevice_crq_on_qnetd self.assertEqual(res, "/etc/corosync/qnetd/nssdb/qdevice-net-node.crq.hacluster1") @@ -203,10 +198,6 @@ def test_qdevice_p12_on_local(self): res = self.qdevice_with_ip.qdevice_p12_on_local self.assertEqual(res, "/etc/corosync/qdevice/net/nssdb/qdevice-net-node.p12") - def test_qdevice_p12_on_cluster(self): - res = self.qdevice_with_ip_cluster_node.qdevice_p12_on_cluster - self.assertEqual(res, "/etc/corosync/qdevice/net/node1.com/qdevice-net-node.p12") - @mock.patch('crmsh.utils.InterfacesInfo.ip_in_local') @mock.patch('crmsh.utils.node_reachable_check') @mock.patch('socket.getaddrinfo') @@ -629,119 +620,6 @@ def test_certificate_process_on_init(self, mock_fetch_qnetd_crt_from_qnetd, mock_copy_p12_to_cluster.assert_called_once() mock_import_p12_on_cluster.assert_called_once() - @mock.patch("crmsh.qdevice.QDevice.log_only_to_file") - @mock.patch("os.path.exists") - @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_cluster", new_callable=mock.PropertyMock) - @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_local", new_callable=mock.PropertyMock) - @mock.patch("crmsh.parallax.parallax_slurp") - def test_fetch_qnetd_crt_from_cluster_exist(self, mock_parallax_slurp, mock_qnetd_cacert_local, - mock_qnetd_cacert_cluster, mock_exists, mock_log): - mock_exists.return_value = True - mock_qnetd_cacert_cluster.return_value = "/etc/corosync/qdevice/net/node1.com/qnetd-cacert.crt" - - self.qdevice_with_ip_cluster_node.fetch_qnetd_crt_from_cluster() - - mock_log.assert_not_called() - mock_exists.assert_called_once_with(mock_qnetd_cacert_cluster.return_value) - mock_qnetd_cacert_cluster.assert_called_once_with() - mock_qnetd_cacert_local.assert_not_called() - mock_parallax_slurp.assert_not_called() - - @mock.patch("crmsh.qdevice.QDevice.log_only_to_file") - @mock.patch("os.path.exists") - @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_cluster", new_callable=mock.PropertyMock) - @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_local", new_callable=mock.PropertyMock) - @mock.patch("crmsh.parallax.parallax_slurp") - def test_fetch_qnetd_crt_from_cluster(self, mock_parallax_slurp, mock_qnetd_cacert_local, - mock_qnetd_cacert_cluster, mock_exists, mock_log): - mock_exists.return_value = False - mock_qnetd_cacert_cluster.return_value = "/etc/corosync/qdevice/net/node1.com/qnetd-cacert.crt" - mock_qnetd_cacert_local.return_value = "/etc/corosync/qdevice/net/10.10.10.123/qnetd-cacert.crt" - - self.qdevice_with_ip_cluster_node.fetch_qnetd_crt_from_cluster() - - mock_log.assert_called_once_with("Step 1: Fetch qnetd-cacert.crt from node1.com") - mock_exists.assert_called_once_with(mock_qnetd_cacert_cluster.return_value) - mock_qnetd_cacert_cluster.assert_called_once_with() - mock_qnetd_cacert_local.assert_called_once_with() - mock_parallax_slurp.assert_called_once_with(["node1.com"], "/etc/corosync/qdevice/net", mock_qnetd_cacert_local.return_value) - - @mock.patch("crmsh.qdevice.QDevice.log_only_to_file") - @mock.patch("crmsh.sh.ClusterShell.get_stdout_or_raise_error") - @mock.patch("crmsh.qdevice.QDevice.qnetd_cacert_on_cluster", new_callable=mock.PropertyMock) - def test_init_db_on_local(self, mock_qnetd_cacert_cluster, mock_stdout_stderr, mock_log): - mock_qnetd_cacert_cluster.return_value = "/etc/corosync/qdevice/net/node1.com/qnetd-cacert.crt" - mock_stdout_stderr.return_value = (0, None, None) - - self.qdevice_with_ip_cluster_node.init_db_on_local() - - mock_log.assert_called_once_with("Step 2: Initialize database on local", - 'corosync-qdevice-net-certutil -i -c /etc/corosync/qdevice/net/node1.com/qnetd-cacert.crt') - mock_qnetd_cacert_cluster.assert_called_once_with() - mock_stdout_stderr.assert_called_once_with("corosync-qdevice-net-certutil -i -c {}".format(mock_qnetd_cacert_cluster.return_value)) - - @mock.patch("crmsh.qdevice.QDevice.log_only_to_file") - @mock.patch("os.path.exists") - @mock.patch("crmsh.qdevice.QDevice.qdevice_p12_on_cluster", new_callable=mock.PropertyMock) - @mock.patch("crmsh.qdevice.QDevice.qdevice_p12_on_local", new_callable=mock.PropertyMock) - @mock.patch("crmsh.parallax.parallax_slurp") - def test_fetch_p12_from_cluster_exist(self, mock_parallax_slurp, mock_p12_on_local, - mock_p12_on_cluster, mock_exists, mock_log): - mock_exists.return_value = True - mock_p12_on_cluster.return_value = "/etc/corosync/qdevice/net/node1.com/qdevice-net-node.p12" - - self.qdevice_with_ip_cluster_node.fetch_p12_from_cluster() - - mock_log.assert_not_called() - mock_exists.assert_called_once_with(mock_p12_on_cluster.return_value) - mock_p12_on_cluster.assert_called_once_with() - mock_p12_on_local.assert_not_called() - mock_parallax_slurp.assert_not_called() - - @mock.patch("crmsh.qdevice.QDevice.log_only_to_file") - @mock.patch("os.path.exists") - @mock.patch("crmsh.qdevice.QDevice.qdevice_p12_on_cluster", new_callable=mock.PropertyMock) - @mock.patch("crmsh.qdevice.QDevice.qdevice_p12_on_local", new_callable=mock.PropertyMock) - @mock.patch("crmsh.parallax.parallax_slurp") - def test_fetch_p12_from_cluster(self, mock_parallax_slurp, mock_p12_on_local, - mock_p12_on_cluster, mock_exists, mock_log): - mock_exists.return_value = False - mock_p12_on_cluster.return_value = "/etc/corosync/qdevice/net/node1.com/qdevice-net-node.p12" - mock_p12_on_local.return_value = "/etc/corosync/qdevice/net/nssdb/qdevice-net-node.p12" - - self.qdevice_with_ip_cluster_node.fetch_p12_from_cluster() - - mock_log.assert_called_once_with("Step 3: Fetch qdevice-net-node.p12 from node1.com") - mock_exists.assert_called_once_with(mock_p12_on_cluster.return_value) - mock_p12_on_cluster.assert_called_once_with() - mock_p12_on_local.assert_called_once_with() - mock_parallax_slurp.assert_called_once_with(["node1.com"], '/etc/corosync/qdevice/net', mock_p12_on_local.return_value) - - @mock.patch("crmsh.qdevice.QDevice.log_only_to_file") - @mock.patch("crmsh.sh.ClusterShell.get_stdout_or_raise_error") - @mock.patch("crmsh.qdevice.QDevice.qdevice_p12_on_cluster", new_callable=mock.PropertyMock) - def test_import_p12_on_local(self, mock_p12_on_cluster, mock_stdout_stderr, mock_log): - mock_p12_on_cluster.return_value = "/etc/corosync/qdevice/net/node1.com/qdevice-net-node.p12" - - self.qdevice_with_ip_cluster_node.import_p12_on_local() - - mock_log.assert_called_once_with("Step 4: Import cluster certificate and key", - 'corosync-qdevice-net-certutil -m -c /etc/corosync/qdevice/net/node1.com/qdevice-net-node.p12') - mock_p12_on_cluster.assert_called_once_with() - mock_stdout_stderr.assert_called_once_with("corosync-qdevice-net-certutil -m -c {}".format(mock_p12_on_cluster.return_value)) - - @mock.patch("crmsh.qdevice.QDevice.import_p12_on_local") - @mock.patch("crmsh.qdevice.QDevice.fetch_p12_from_cluster") - @mock.patch("crmsh.qdevice.QDevice.init_db_on_local") - @mock.patch("crmsh.qdevice.QDevice.fetch_qnetd_crt_from_cluster") - def test_certificate_process_on_join(self, mock_fetch_qnetd_crt_from_cluster, mock_init_db_on_local, - mock_fetch_p12_from_cluster, mock_import_p12_on_local): - self.qdevice_with_ip.certificate_process_on_join() - mock_fetch_qnetd_crt_from_cluster.assert_called_once_with() - mock_init_db_on_local.assert_called_once_with() - mock_fetch_p12_from_cluster.assert_called_once_with() - mock_import_p12_on_local.assert_called_once_with() - @mock.patch("crmsh.utils.str2file") @mock.patch("crmsh.corosync.make_section") @mock.patch("crmsh.corosync.Parser")