From 368729300c843a985bd8fb0f2c1ff7250de72b61 Mon Sep 17 00:00:00 2001 From: lzxddz Date: Tue, 11 Nov 2025 19:11:53 +0800 Subject: [PATCH 1/2] Support dss multi shards dss client add arg bind_data_shard_with_ng update mono_multi test cases for the node of key located on changed --- storage/eloq/ha_eloq.cc | 110 ++++++++++++++---- .../r/fault_inject_for_upsert_1.result | 2 +- .../mono_multi/r/multi_nodes.result | 2 +- .../t/fault_inject_for_upsert_1.test | 4 +- .../mysql-test/mono_multi/t/multi_nodes.test | 6 +- 5 files changed, 97 insertions(+), 27 deletions(-) diff --git a/storage/eloq/ha_eloq.cc b/storage/eloq/ha_eloq.cc index b8e30d3f40e..4ed299ccf09 100644 --- a/storage/eloq/ha_eloq.cc +++ b/storage/eloq/ha_eloq.cc @@ -2298,7 +2298,11 @@ Aws::SDKOptions aws_options; static int aws_init() { DBUG_ENTER("aws_init"); +#if !defined(DBUG_OFF) + aws_options.loggingOptions.logLevel= Aws::Utils::Logging::LogLevel::Debug; +#else aws_options.loggingOptions.logLevel= Aws::Utils::Logging::LogLevel::Info; +#endif Aws::InitAPI(aws_options); DBUG_RETURN(0); } @@ -2660,7 +2664,6 @@ static int eloq_init_func(void *p) #elif ELOQDS case KV_ELOQDS: { - bool is_single_node= true; std::string ds_peer_node= eloq_dss_peer_node; std::string ds_branch_name= eloq_dss_branch_name; std::string dss_data_path= mysql_real_data_home_ptr; @@ -2724,15 +2727,7 @@ static int eloq_init_func(void *p) std::string dss_config_file_path= ""; EloqDS::DataStoreServiceClusterManager ds_config; - uint32_t dss_leader_id= EloqDS::UNKNOWN_DSS_LEADER_NODE_ID; - - // use tx node id as the dss node id - // since they are deployed together - uint32_t dss_node_id= node_id; - if (opt_bootstrap || is_single_node) - { - dss_leader_id= node_id; - } + bool is_single_node= false; if (!ds_peer_node.empty()) { @@ -2747,18 +2742,88 @@ static int eloq_init_func(void *p) eloq_dss_peer_node); DBUG_RETURN(eloq_init_abort()); } + is_single_node= + (ng_configs.size() == 1 && ng_configs.begin()->second.size() == 1); } else { - if (ng_configs.size() > 1) + // Bind dss leaders with TxService ng leaders. + std::unordered_map ng_leaders; + + if (opt_bootstrap) { - sql_print_error("EloqDS multi-node cluster must specify " - "eloq_dss_peer_node."); - DBUG_RETURN(eloq_init_abort()); + // Must parse all node groups to generate data store shards configs. + std::unordered_map> tmp_ng_configs; + uint64_t tmp_cluster_version= 2; + if (!txservice::ReadClusterConfigFile( + cluster_config_file, tmp_ng_configs, tmp_cluster_version)) + { + + // Read cluster topology from general config file in this case + auto parse_res= txservice::ParseNgConfig( + eloq_ip_list, eloq_standby_ip_list, eloq_voter_ip_list, + tmp_ng_configs, eloq_node_group_replica_num, 0); + if (!parse_res) + { + LOG(ERROR) + << "Failed to extract cluster configs from ip_port_list."; + DBUG_RETURN(eloq_init_abort()); + } + } + + bool found= false; + uint32_t tmp_node_id; + // check whether this node is in cluster. + for (auto &pair : tmp_ng_configs) + { + auto &ng_nodes= pair.second; + for (size_t i= 0; i < ng_nodes.size(); i++) + { + if (ng_nodes[i].host_name_ == local_ip && + ng_nodes[i].port_ == local_port) + { + tmp_node_id= ng_nodes[i].node_id_; + found= true; + break; + } + } + if (found) + { + break; + } + } + if (!found) + { + sql_print_error("!!!!!!!! Current node does not belong to any node " + "group, EloqDB " + "startup is terminated !!!!!!!!"); + DBUG_RETURN(eloq_init_abort()); + } + + // For bootstrap, start all data store shards in current node. + for (auto &ng : tmp_ng_configs) + { + ng_leaders.emplace(ng.first, tmp_node_id); + } + + EloqDS::DataStoreServiceClient::TxConfigsToDssClusterConfig( + tmp_node_id, tmp_ng_configs, ng_leaders, ds_config); } + else + { + is_single_node= + (ng_configs.size() == 1 && ng_configs.begin()->second.size() == 1); + if (is_single_node) + { + for (const auto &[ng_id, ng_config] : ng_configs) + { + ng_leaders.emplace(ng_id, node_id); + } + } - EloqDS::DataStoreServiceClient::TxConfigsToDssClusterConfig( - dss_node_id, native_ng_id, ng_configs, dss_leader_id, ds_config); + EloqDS::DataStoreServiceClient::TxConfigsToDssClusterConfig( + node_id, ng_configs, ng_leaders, ds_config); + } } data_store_service_= std::make_unique( @@ -2773,10 +2838,9 @@ static int eloq_init_func(void *p) // we always set create_if_missing to true // since non conflicts will happen under // multi-node deployment. - ret= data_store_service_->StartService(true, dss_leader_id, dss_node_id); + ret= data_store_service_->StartService(true); #else - ret= data_store_service_->StartService((opt_bootstrap || is_single_node), - dss_leader_id, dss_node_id); + ret= data_store_service_->StartService((opt_bootstrap || is_single_node)); #endif if (!ret) @@ -2787,7 +2851,13 @@ static int eloq_init_func(void *p) // setup data store service client storage_hd= std::make_unique( - catalog_factory, ds_config, data_store_service_.get()); +#if defined(DATA_STORE_TYPE_ELOQDSS_ROCKSDB) + true, +#else + (opt_bootstrap || is_single_node), +#endif + catalog_factory, ds_config, ds_peer_node.empty(), + data_store_service_.get()); if (!storage_hd->Connect()) { diff --git a/storage/eloq/mysql-test/mono_multi/r/fault_inject_for_upsert_1.result b/storage/eloq/mysql-test/mono_multi/r/fault_inject_for_upsert_1.result index d93a70e6fe7..9daca6ce8b8 100644 --- a/storage/eloq/mysql-test/mono_multi/r/fault_inject_for_upsert_1.result +++ b/storage/eloq/mysql-test/mono_multi/r/fault_inject_for_upsert_1.result @@ -6,7 +6,7 @@ insert into t2 values(1,1); select * from t2; i j 1 1 -SET SESSION debug_dbug="+d,eloq;remote_acquire_before_sendmessage;node_id=1;action=panic"; +SET SESSION debug_dbug="+d,eloq;remote_acquire_before_sendmessage;node_id=2;action=panic"; insert into t2 values(2,2); ERROR HY000: Eloq: failed to commit transaction. try restarting transaction. Transaction failed due to internal request timeout. select * from t2; diff --git a/storage/eloq/mysql-test/mono_multi/r/multi_nodes.result b/storage/eloq/mysql-test/mono_multi/r/multi_nodes.result index fc962320c4b..1ab7ac68402 100644 --- a/storage/eloq/mysql-test/mono_multi/r/multi_nodes.result +++ b/storage/eloq/mysql-test/mono_multi/r/multi_nodes.result @@ -21,7 +21,7 @@ i j 1 2 Begin; update t2 set j=3; -connection serv_2; +connection serv_3; # restart connection serv_1; commit; diff --git a/storage/eloq/mysql-test/mono_multi/t/fault_inject_for_upsert_1.test b/storage/eloq/mysql-test/mono_multi/t/fault_inject_for_upsert_1.test index aa4869068a8..1a43b56e133 100644 --- a/storage/eloq/mysql-test/mono_multi/t/fault_inject_for_upsert_1.test +++ b/storage/eloq/mysql-test/mono_multi/t/fault_inject_for_upsert_1.test @@ -16,9 +16,9 @@ insert into t2 values(1,1); select * from t2; # The row(2,2) will be located on server3(ng2). -SET SESSION debug_dbug="+d,eloq;remote_acquire_before_sendmessage;node_id=1;action=panic"; +SET SESSION debug_dbug="+d,eloq;remote_acquire_before_sendmessage;node_id=2;action=panic"; # Write file to make mysql-test-run.pl expect crash ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.3.expect # insert transaction will timeout when waiting for the acquire write response. --error 199 diff --git a/storage/eloq/mysql-test/mono_multi/t/multi_nodes.test b/storage/eloq/mysql-test/mono_multi/t/multi_nodes.test index 3aee30e34ad..09b1dc484e3 100644 --- a/storage/eloq/mysql-test/mono_multi/t/multi_nodes.test +++ b/storage/eloq/mysql-test/mono_multi/t/multi_nodes.test @@ -38,10 +38,10 @@ select * from t2; Begin; update t2 set j=3; -# The row(1,1) is located on server-2, then restart it to trigger changing node term. -# Here will restart server 2. It will decide which node to restart +# The row(1,1) is located on server-3, then restart it to trigger changing node term. +# Here will restart server-3. It will decide which node to restart # according to the current connection. ---connection serv_2 +--connection serv_3 --source include/restart_mysqld.inc --connection serv_1 From 522adc80f86e0af7797258452b00bb60ac9532ff Mon Sep 17 00:00:00 2001 From: lzxddz Date: Fri, 21 Nov 2025 11:31:38 +0800 Subject: [PATCH 2/2] update submodule --- storage/eloq/store_handler | 2 +- storage/eloq/tx_service | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/eloq/store_handler b/storage/eloq/store_handler index 979e1594765..9fc13524deb 160000 --- a/storage/eloq/store_handler +++ b/storage/eloq/store_handler @@ -1 +1 @@ -Subproject commit 979e159476542b1f0f7e8519d91457529ed0ad22 +Subproject commit 9fc13524deb51390850482456d2a0b524a1e83ba diff --git a/storage/eloq/tx_service b/storage/eloq/tx_service index ab3c1a2e094..7e38a11f4f8 160000 --- a/storage/eloq/tx_service +++ b/storage/eloq/tx_service @@ -1 +1 @@ -Subproject commit ab3c1a2e094fffc3b72d1942c38717f2e8a68271 +Subproject commit 7e38a11f4f8dafe6e6c38fa7f10c83c48412699c