From 1e1455d162d0528059113c5d8acab16ec65d99aa Mon Sep 17 00:00:00 2001 From: githubzilla Date: Tue, 4 Nov 2025 20:15:03 +0800 Subject: [PATCH 01/12] Update cluster config --- storage/eloq/ha_eloq.cc | 244 +++++++++++++++++++++++----------------- 1 file changed, 139 insertions(+), 105 deletions(-) diff --git a/storage/eloq/ha_eloq.cc b/storage/eloq/ha_eloq.cc index feadfe2461e..eb4e78a0bbf 100644 --- a/storage/eloq/ha_eloq.cc +++ b/storage/eloq/ha_eloq.cc @@ -2568,6 +2568,59 @@ static int eloq_init_func(void *p) std::string store_keyspace_name(eloq_keyspace_name); + if (opt_bootstrap) + { + // When execute mysql_install_db.sh, eloq should run in solo mode. + std::vector solo_config; + solo_config.emplace_back(0, local_ip, local_port); + ng_configs.try_emplace(0, std::move(solo_config)); + } + else if (!txservice::ReadClusterConfigFile(cluster_config_file, ng_configs, + cluster_config_version)) + { + + // Read cluster topology from general config file in this case + auto parse_res= txservice::ParseNgConfig( + eloq_ip_list, eloq_standby_ip_list, eloq_voter_ip_list, ng_configs, + eloq_node_group_replica_num, 0); + if (!parse_res) + { + LOG(ERROR) << "Failed to extract cluster configs from ip_port_list."; + DBUG_RETURN(eloq_init_abort()); + } + } + + bool found= false; + uint32_t native_ng_id= 0; + // check whether this node is in cluster. + for (auto &pair : ng_configs) + { + auto &ng_nodes= pair.second; + for (size_t i= 0; i < ng_nodes.size(); i++) + { + if (ng_nodes[i].host_name_ == local_ip && + ng_nodes[i].port_ == local_port) + { + node_id= ng_nodes[i].node_id_; + found= true; + if (ng_nodes[i].is_candidate_) + { + // found native_ng_id. + native_ng_id= pair.first; + break; + } + } + } + } + + if (!found) + { + sql_print_error("!!!!!!!! Current node does not belong to any node " + "group, EloqDB " + "startup is terminated !!!!!!!!"); + DBUG_RETURN(eloq_init_abort()); + } + switch (eloq_kv_storage) { #if defined(DATA_STORE_TYPE_DYNAMODB) @@ -2624,58 +2677,59 @@ static int eloq_init_func(void *p) sql_print_error("Failed to create dir: %s", dss_data_path.c_str()); DBUG_RETURN(eloq_init_abort()); } - std::string dss_config_file_path= eloq_dss_config_file_path; - if (dss_config_file_path.empty()) - { - dss_config_file_path= dss_data_path + "/dss_config.ini"; - } - EloqDS::DataStoreServiceClusterManager ds_config; - if (!ds_config.Load(dss_config_file_path)) - { - if (!ds_peer_node.empty()) - { - ds_config.SetThisNode(local_ip, local_port + 7); - // Fetch ds topology from peer node - if (!EloqDS::DataStoreService::FetchConfigFromPeer(ds_peer_node, - ds_config)) - { - sql_print_error("Failed to fetch config from peer node: %s", - ds_peer_node.c_str()); - DBUG_RETURN(eloq_init_abort()); - } - - // Save the fetched config to the local file - if (!ds_config.Save(dss_config_file_path)) - { - sql_print_error("Failed to save config to file: %s", - dss_config_file_path.c_str()); - DBUG_RETURN(eloq_init_abort()); - } - } - else if (opt_bootstrap || is_single_node) - { - // Initialize the data store service config - ds_config.Initialize(local_ip, local_port + 7); - if (!ds_config.Save(dss_config_file_path)) - { - sql_print_error("Failed to save config to file: %s", - dss_config_file_path.c_str()); - DBUG_RETURN(eloq_init_abort()); - } - } - else - { - sql_print_error("Failed to load data store service config file: %s", - dss_config_file_path.c_str()); - DBUG_RETURN(eloq_init_abort()); - } - } - else - { - sql_print_information("EloqDataStoreService loaded config file %s", - dss_config_file_path.c_str()); - } + // std::string dss_config_file_path= eloq_dss_config_file_path; + // if (dss_config_file_path.empty()) + // { + // dss_config_file_path= dss_data_path + "/dss_config.ini"; + // } + // + // EloqDS::DataStoreServiceClusterManager ds_config; + // if (!ds_config.Load(dss_config_file_path)) + // { + // if (!ds_peer_node.empty()) + // { + // ds_config.SetThisNode(local_ip, local_port + 7); + // // Fetch ds topology from peer node + // if (!EloqDS::DataStoreService::FetchConfigFromPeer(ds_peer_node, + // ds_config)) + // { + // sql_print_error("Failed to fetch config from peer node: %s", + // ds_peer_node.c_str()); + // DBUG_RETURN(eloq_init_abort()); + // } + // + // // Save the fetched config to the local file + // if (!ds_config.Save(dss_config_file_path)) + // { + // sql_print_error("Failed to save config to file: %s", + // dss_config_file_path.c_str()); + // DBUG_RETURN(eloq_init_abort()); + // } + // } + // else if (opt_bootstrap || is_single_node) + // { + // // Initialize the data store service config + // ds_config.Initialize(local_ip, local_port + 7); + // if (!ds_config.Save(dss_config_file_path)) + // { + // sql_print_error("Failed to save config to file: %s", + // dss_config_file_path.c_str()); + // DBUG_RETURN(eloq_init_abort()); + // } + // } + // else + // { + // sql_print_error("Failed to load data store service config file: %s", + // dss_config_file_path.c_str()); + // DBUG_RETURN(eloq_init_abort()); + // } + // } + // else + // { + // sql_print_information("EloqDataStoreService loaded config file %s", + // dss_config_file_path.c_str()); + // } #if defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_GCS) @@ -2721,14 +2775,46 @@ static int eloq_init_func(void *p) std::move(eloq_store_config)); #endif + std::string dss_config_file_path= ""; + EloqDS::DataStoreServiceClusterManager ds_config; + uint32_t dss_leader_id= EloqDS::UNKNOWN_DSS_LEADER_NODE_ID; + + // use tx node id as the dss node id + // since they are deployed together + uint32_t dss_node_id= node_id; + if (opt_bootstrap || is_single_node) + { + dss_leader_id= node_id; + } + + if (!ds_peer_node.empty()) + { + ds_config.SetThisNode( + local_ip, + EloqDS::DataStoreServiceClient::TxPort2DssPort(local_port)); + // Fetch ds topology from peer node + if (!EloqDS::DataStoreService::FetchConfigFromPeer(eloq_dss_peer_node, + ds_config)) + { + LOG(ERROR) << "Failed to fetch config from peer node: " + << eloq_dss_peer_node; + return false; + } + } + else + { + EloqDS::DataStoreServiceClient::TxConfigsToDssClusterConfig( + dss_node_id, native_ng_id, ng_configs, dss_leader_id, ds_config); + } + data_store_service_= std::make_unique( ds_config, dss_config_file_path, dss_data_path + "/DSMigrateLog", std::move(ds_factory)); // setup local data store service, the data store service will start // data store if needed. - bool ret= - data_store_service_->StartService((opt_bootstrap || is_single_node)); + bool ret= data_store_service_->StartService( + (opt_bootstrap || is_single_node), dss_leader_id, dss_node_id); if (!ret) { sql_print_error("Failed to start data store service"); @@ -2754,58 +2840,6 @@ static int eloq_init_func(void *p) default: DBUG_RETURN(eloq_init_abort()); } - if (opt_bootstrap) - { - // When execute mysql_install_db.sh, eloq should run in solo mode. - std::vector solo_config; - solo_config.emplace_back(0, local_ip, local_port); - ng_configs.try_emplace(0, std::move(solo_config)); - } - else if (!txservice::ReadClusterConfigFile(cluster_config_file, ng_configs, - cluster_config_version)) - { - - // Read cluster topology from general config file in this case - auto parse_res= txservice::ParseNgConfig( - eloq_ip_list, eloq_standby_ip_list, eloq_voter_ip_list, ng_configs, - eloq_node_group_replica_num, 0); - if (!parse_res) - { - LOG(ERROR) << "Failed to extract cluster configs from ip_port_list."; - DBUG_RETURN(eloq_init_abort()); - } - } - - bool found= false; - uint32_t native_ng_id= 0; - // check whether this node is in cluster. - for (auto &pair : ng_configs) - { - auto &ng_nodes= pair.second; - for (size_t i= 0; i < ng_nodes.size(); i++) - { - if (ng_nodes[i].host_name_ == local_ip && - ng_nodes[i].port_ == local_port) - { - node_id= ng_nodes[i].node_id_; - found= true; - if (ng_nodes[i].is_candidate_) - { - // found native_ng_id. - native_ng_id= pair.first; - break; - } - } - } - } - - if (!found) - { - sql_print_error("!!!!!!!! Current node does not belong to any node " - "group, EloqDB " - "startup is terminated !!!!!!!!"); - DBUG_RETURN(eloq_init_abort()); - } // Set max rpc message size as 512mb. GFLAGS_NAMESPACE::SetCommandLineOption("max_body_size", "536870912"); From dcb78d42b9341548917b5c78f3e057d7fc77778f Mon Sep 17 00:00:00 2001 From: githubzilla Date: Tue, 4 Nov 2025 20:15:15 +0800 Subject: [PATCH 02/12] Update store_handler --- storage/eloq/store_handler | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/eloq/store_handler b/storage/eloq/store_handler index 62e17d3b8c8..8d8d1b3a9d0 160000 --- a/storage/eloq/store_handler +++ b/storage/eloq/store_handler @@ -1 +1 @@ -Subproject commit 62e17d3b8c8c4095d094b66351fdd7dec3a4f4d4 +Subproject commit 8d8d1b3a9d08cc53b07910e2f80cf0002df2a2be From 7a8edb8dd48170a9a501df017d098e1aa4c3d48d Mon Sep 17 00:00:00 2001 From: githubzilla Date: Tue, 4 Nov 2025 20:15:22 +0800 Subject: [PATCH 03/12] Update tx_service --- storage/eloq/tx_service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/eloq/tx_service b/storage/eloq/tx_service index f3f89e6e6d9..5cd5be35382 160000 --- a/storage/eloq/tx_service +++ b/storage/eloq/tx_service @@ -1 +1 @@ -Subproject commit f3f89e6e6d9a4c8dca6ceed4b6058848ab256597 +Subproject commit 5cd5be353827a58263b21e1118e37209c00c4cad From 72566dd0d7a1fa7ededdb87eebbc3bee3ddd4cbe Mon Sep 17 00:00:00 2001 From: githubzilla Date: Wed, 5 Nov 2025 17:55:20 +0800 Subject: [PATCH 04/12] set create_if_missing when dss is local rocksdb --- storage/eloq/ha_eloq.cc | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/storage/eloq/ha_eloq.cc b/storage/eloq/ha_eloq.cc index eb4e78a0bbf..14d832bf03d 100644 --- a/storage/eloq/ha_eloq.cc +++ b/storage/eloq/ha_eloq.cc @@ -2803,6 +2803,13 @@ static int eloq_init_func(void *p) } else { + if (ng_configs.size() > 1) + { + sql_print_error("EloqDS multi-node cluster must specify " + "eloq_dss_peer_node."); + DBUG_RETURN(eloq_init_abort()); + } + EloqDS::DataStoreServiceClient::TxConfigsToDssClusterConfig( dss_node_id, native_ng_id, ng_configs, dss_leader_id, ds_config); } @@ -2813,8 +2820,18 @@ static int eloq_init_func(void *p) // setup local data store service, the data store service will start // data store if needed. - bool ret= data_store_service_->StartService( - (opt_bootstrap || is_single_node), dss_leader_id, dss_node_id); + bool ret= true; +#if defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) + // For non shared storage like rocksdb, + // we always set create_if_missing to true + // since non conflicts will happen under + // multi-node deployment. + ret= data_store_service_->StartService(true, dss_leader_id, dss_node_id); +#else + ret= data_store_service_->StartService((opt_bootstrap || is_single_node), + dss_leader_id, dss_node_id); +#endif + if (!ret) { sql_print_error("Failed to start data store service"); From 32f13d2382b907eb62d459d0b4d93df1df83c657 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Wed, 5 Nov 2025 17:55:29 +0800 Subject: [PATCH 05/12] Update store_handler --- storage/eloq/store_handler | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/eloq/store_handler b/storage/eloq/store_handler index 8d8d1b3a9d0..bb4d347a1c9 160000 --- a/storage/eloq/store_handler +++ b/storage/eloq/store_handler @@ -1 +1 @@ -Subproject commit 8d8d1b3a9d08cc53b07910e2f80cf0002df2a2be +Subproject commit bb4d347a1c9de9d1f3cb3174b4eb2636849cbca8 From 3106573e40674d859a3fb1552090ad35beadeb4d Mon Sep 17 00:00:00 2001 From: githubzilla Date: Wed, 5 Nov 2025 19:12:31 +0800 Subject: [PATCH 06/12] Update store_handler --- storage/eloq/store_handler | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/eloq/store_handler b/storage/eloq/store_handler index bb4d347a1c9..0f1c1243db2 160000 --- a/storage/eloq/store_handler +++ b/storage/eloq/store_handler @@ -1 +1 @@ -Subproject commit bb4d347a1c9de9d1f3cb3174b4eb2636849cbca8 +Subproject commit 0f1c1243db27802ff96087b1bc868b37010f8511 From 63781245b7cea8b2e131c11d5660d2614844b4b8 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Fri, 7 Nov 2025 12:46:45 +0800 Subject: [PATCH 07/12] Remove commented code --- storage/eloq/ha_eloq.cc | 53 ----------------------------------------- 1 file changed, 53 deletions(-) diff --git a/storage/eloq/ha_eloq.cc b/storage/eloq/ha_eloq.cc index 14d832bf03d..1671a029b25 100644 --- a/storage/eloq/ha_eloq.cc +++ b/storage/eloq/ha_eloq.cc @@ -2678,59 +2678,6 @@ static int eloq_init_func(void *p) DBUG_RETURN(eloq_init_abort()); } - // std::string dss_config_file_path= eloq_dss_config_file_path; - // if (dss_config_file_path.empty()) - // { - // dss_config_file_path= dss_data_path + "/dss_config.ini"; - // } - // - // EloqDS::DataStoreServiceClusterManager ds_config; - // if (!ds_config.Load(dss_config_file_path)) - // { - // if (!ds_peer_node.empty()) - // { - // ds_config.SetThisNode(local_ip, local_port + 7); - // // Fetch ds topology from peer node - // if (!EloqDS::DataStoreService::FetchConfigFromPeer(ds_peer_node, - // ds_config)) - // { - // sql_print_error("Failed to fetch config from peer node: %s", - // ds_peer_node.c_str()); - // DBUG_RETURN(eloq_init_abort()); - // } - // - // // Save the fetched config to the local file - // if (!ds_config.Save(dss_config_file_path)) - // { - // sql_print_error("Failed to save config to file: %s", - // dss_config_file_path.c_str()); - // DBUG_RETURN(eloq_init_abort()); - // } - // } - // else if (opt_bootstrap || is_single_node) - // { - // // Initialize the data store service config - // ds_config.Initialize(local_ip, local_port + 7); - // if (!ds_config.Save(dss_config_file_path)) - // { - // sql_print_error("Failed to save config to file: %s", - // dss_config_file_path.c_str()); - // DBUG_RETURN(eloq_init_abort()); - // } - // } - // else - // { - // sql_print_error("Failed to load data store service config file: %s", - // dss_config_file_path.c_str()); - // DBUG_RETURN(eloq_init_abort()); - // } - // } - // else - // { - // sql_print_information("EloqDataStoreService loaded config file %s", - // dss_config_file_path.c_str()); - // } - #if defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_S3) || \ defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB_CLOUD_GCS) // std::string ds_rocksdb_config_file_path= From 81fba596208a66494055b3b62135320d1f1d2ee0 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Fri, 7 Nov 2025 15:01:17 +0800 Subject: [PATCH 08/12] Fix comments --- storage/eloq/ha_eloq.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/eloq/ha_eloq.cc b/storage/eloq/ha_eloq.cc index 1671a029b25..9f644d492fe 100644 --- a/storage/eloq/ha_eloq.cc +++ b/storage/eloq/ha_eloq.cc @@ -2743,9 +2743,9 @@ static int eloq_init_func(void *p) if (!EloqDS::DataStoreService::FetchConfigFromPeer(eloq_dss_peer_node, ds_config)) { - LOG(ERROR) << "Failed to fetch config from peer node: " - << eloq_dss_peer_node; - return false; + sql_print_error("Failed to fetch DSS config from peer node: %s", + eloq_dss_peer_node); + DBUG_RETURN(eloq_init_abort()); } } else From 115bd7984da825d0bbe3b90adc0e2c77e71bf759 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Fri, 7 Nov 2025 15:01:31 +0800 Subject: [PATCH 09/12] Update tx --- storage/eloq/tx_service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/eloq/tx_service b/storage/eloq/tx_service index 5cd5be35382..ec8fd7cebd7 160000 --- a/storage/eloq/tx_service +++ b/storage/eloq/tx_service @@ -1 +1 @@ -Subproject commit 5cd5be353827a58263b21e1118e37209c00c4cad +Subproject commit ec8fd7cebd7d4030a49c5f75fd9ba567efdd9807 From 8db1113dd2ffad8baa4bff267dbf63dfa0fcdcf4 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Fri, 7 Nov 2025 19:19:48 +0800 Subject: [PATCH 10/12] Fix ww conflict in test --- .../mono_multi/t/fault_inject_for_upsert_table.test | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/storage/eloq/mysql-test/mono_multi/t/fault_inject_for_upsert_table.test b/storage/eloq/mysql-test/mono_multi/t/fault_inject_for_upsert_table.test index 9c5840781b4..bfb37618837 100644 --- a/storage/eloq/mysql-test/mono_multi/t/fault_inject_for_upsert_table.test +++ b/storage/eloq/mysql-test/mono_multi/t/fault_inject_for_upsert_table.test @@ -54,6 +54,10 @@ drop table t1; --error ER_NO_SUCH_TABLE select * from t1; SET SESSION debug_dbug="-d,eloq;kv_upsert_table;node_id=1"; +# Sleep for replay work finished to avoid ww conflict +--disable_result_log +--sleep 3 +--enable_result_log create table t1(i int primary key, j int) engine= eloq; select * from t1; drop table t1; From f3e8a7cb7ba568d947795348b9ebc9b1ed21ef3e Mon Sep 17 00:00:00 2001 From: githubzilla Date: Fri, 7 Nov 2025 19:19:56 +0800 Subject: [PATCH 11/12] Update store_handler --- storage/eloq/store_handler | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/eloq/store_handler b/storage/eloq/store_handler index 0f1c1243db2..ce0e9317fd4 160000 --- a/storage/eloq/store_handler +++ b/storage/eloq/store_handler @@ -1 +1 @@ -Subproject commit 0f1c1243db27802ff96087b1bc868b37010f8511 +Subproject commit ce0e9317fd486a0e5a1342228ca4f2805d492daf From eceb14c59fdea923db322a5bca321ab31f5c53c5 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Fri, 7 Nov 2025 19:20:03 +0800 Subject: [PATCH 12/12] Update tx --- storage/eloq/tx_service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/eloq/tx_service b/storage/eloq/tx_service index ec8fd7cebd7..307387086a3 160000 --- a/storage/eloq/tx_service +++ b/storage/eloq/tx_service @@ -1 +1 @@ -Subproject commit ec8fd7cebd7d4030a49c5f75fd9ba567efdd9807 +Subproject commit 307387086a3e05d9da8c31dc6b818923361b076b