From a3f8173a5c7d4159ca285bc6c31e53a03bb024b3 Mon Sep 17 00:00:00 2001 From: yi-xmu Date: Thu, 15 Jan 2026 19:10:45 +0800 Subject: [PATCH 1/7] allow checkpoint run on candidate leader node fix release metadata readlocc fix update last sync ts update debug log debug log and debug print ccpage function debug log for range split datasyncscan and debug print ccmap function disable debug log fix slice size mismatch --- tx_service/include/cc/catalog_cc_map.h | 42 ++-- tx_service/include/cc/cc_handler.h | 17 +- tx_service/include/cc/cc_map.h | 2 + tx_service/include/cc/cc_page_clean_guard.h | 22 ++ tx_service/include/cc/cc_req_misc.h | 5 +- tx_service/include/cc/cc_request.h | 212 ++++++++++++++++-- tx_service/include/cc/cc_shard.h | 4 + tx_service/include/cc/cluster_config_cc_map.h | 9 +- tx_service/include/cc/local_cc_handler.h | 38 ++-- tx_service/include/cc/local_cc_shards.h | 1 + tx_service/include/cc/range_bucket_cc_map.h | 30 +-- tx_service/include/cc/range_cc_map.h | 26 ++- tx_service/include/cc/template_cc_map.h | 183 ++++++++++----- tx_service/include/data_sync_task.h | 9 +- tx_service/include/proto/cc_request.proto | 1 + tx_service/include/remote/remote_cc_handler.h | 1 + tx_service/include/sequences/sequences.h | 1 + tx_service/include/sharder.h | 5 +- tx_service/include/tx_execution.h | 8 +- tx_service/include/tx_request.h | 18 +- tx_service/include/tx_util.h | 11 +- tx_service/src/cc/cc_req_misc.cpp | 86 ++++++- tx_service/src/cc/cc_shard.cpp | 37 +++ tx_service/src/cc/local_cc_handler.cpp | 75 +++++-- tx_service/src/cc/local_cc_shards.cpp | 154 +++++++++++-- tx_service/src/checkpointer.cpp | 14 +- tx_service/src/data_sync_task.cpp | 3 +- tx_service/src/remote/remote_cc_handler.cpp | 2 + tx_service/src/remote/remote_cc_request.cpp | 1 + tx_service/src/sequences/sequences.cpp | 35 ++- tx_service/src/sharder.cpp | 8 +- tx_service/src/tx_execution.cpp | 140 ++++++++++-- tx_service/src/tx_operation.cpp | 66 +++++- 33 files changed, 1002 insertions(+), 264 deletions(-) diff --git a/tx_service/include/cc/catalog_cc_map.h b/tx_service/include/cc/catalog_cc_map.h index 302c81a2..9670f967 100644 --- a/tx_service/include/cc/catalog_cc_map.h +++ b/tx_service/include/cc/catalog_cc_map.h @@ -90,12 +90,7 @@ class CatalogCcMap uint32_t ng_id = req.NodeGroupId(); if (shard_->IsNative(ng_id) && req.CcOp() == CcOperation::ReadForWrite) { - int64_t ng_term = Sharder::Instance().LeaderTerm(ng_id); CcHandlerResult *hd_res = req.Result(); - if (ng_term < 0) - { - return hd_res->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); - } const CatalogKey *catalog_key = nullptr; if (req.Key() != nullptr) @@ -199,19 +194,17 @@ class CatalogCcMap } }); - int64_t ng_term = Sharder::Instance().LeaderTerm(req.NodeGroupId()); + int64_t ng_term = req.NodeGroupTerm(); + assert(ng_term > 0); CODE_FAULT_INJECTOR("term_CatalogCcMap_Execute_PostWriteAllCc", { LOG(INFO) << "FaultInject term_CatalogCcMap_Execute_PostWriteAllCc"; ng_term = -1; FaultInject::Instance().InjectFault( "term_CatalogCcMap_Execute_PostWriteAllCc", "remove"); - }); - if (ng_term < 0) - { req.Result()->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); return true; - } + }); const CatalogKey *table_key = nullptr; if (req.Key() != nullptr) @@ -1148,28 +1141,31 @@ class CatalogCcMap assert(req.IsLocal()); uint32_t ng_id = req.NodeGroupId(); - int64_t ng_term = Sharder::Instance().LeaderTerm(ng_id); - ng_term = std::max(ng_term, Sharder::Instance().StandbyNodeTerm()); - - if (req.IsInRecovering()) + int64_t ng_term = req.NodeGroupTerm(); + if (ng_term < 0) { - ng_term = ng_term > 0 - ? ng_term - : Sharder::Instance().CandidateLeaderTerm(ng_id); + if (req.AllowRunOnCandidate()) + { + ng_term = Sharder::Instance().CandidateLeaderTerm(ng_id); + } + if (ng_term < 0) + { + ng_term = Sharder::Instance().LeaderTerm(ng_id); + int64_t standby_node_term = + Sharder::Instance().StandbyNodeTerm(); + ng_term = std::max(ng_term, standby_node_term); + } } + assert(ng_term > 0); CODE_FAULT_INJECTOR("term_CatalogCcMap_Execute_ReadCc", { - LOG(INFO) << "FaultInject term_CatalogCcMap_Execute_ReadCc"; + LOG(INFO) << "FaultInject term_CatalogCcMap_Execute_ReadCc"; ng_term = -1; FaultInject::Instance().InjectFault( "term_CatalogCcMap_Execute_ReadCc", "remove"); - }); - - if (ng_term < 0) - { req.Result()->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); return true; - } + }); const CatalogKey *table_key = static_cast(req.Key()); diff --git a/tx_service/include/cc/cc_handler.h b/tx_service/include/cc/cc_handler.h index 3d4640b8..67418635 100644 --- a/tx_service/include/cc/cc_handler.h +++ b/tx_service/include/cc/cc_handler.h @@ -99,7 +99,8 @@ class CcHandler uint32_t hd_res_idx, CcProtocol proto, IsolationLevel iso_level, - bool abort_if_oom) = 0; + bool abort_if_oom, + bool allow_run_on_candidate) = 0; /** * @brief Acquires write locks for the input key in all shards. This method @@ -166,7 +167,8 @@ class CcHandler const TxRecord *record, OperationType operation_type, uint32_t key_shard_code, - CcHandlerResult &hres) = 0; + CcHandlerResult &hres, + bool allow_run_on_candidate) = 0; /** * @briefPost-processes a read/scan key. Post-processing clears the read @@ -202,7 +204,8 @@ class CcHandler CcHandlerResult &hres, bool is_local = false, bool need_remote_resp = true, - PostReadType post_read_type = PostReadType::Release) = 0; + PostReadType post_read_type = PostReadType::Release, + bool allow_run_on_candidate = false) = 0; /** * @brief Reads the input key and returns the key's record. The request puts @@ -239,6 +242,7 @@ class CcHandler CcProtocol proto = CcProtocol::OCC, bool is_for_write = false, bool is_covering_keys = false, + bool allow_run_on_candidate = false, bool point_read_on_miss = false, int32_t partition_id = -1, bool abort_if_oom = false) = 0; @@ -297,7 +301,7 @@ class CcHandler IsolationLevel iso_level = IsolationLevel::RepeatableRead, CcProtocol proto = CcProtocol::Locking, bool is_for_write = false, - bool is_recovring = false, + bool allow_run_on_candidate = false, bool execute_immediately = true) = 0; virtual bool ReadLocal( @@ -313,7 +317,7 @@ class CcHandler IsolationLevel iso_level = IsolationLevel::RepeatableRead, CcProtocol proto = CcProtocol::Locking, bool is_for_write = false, - bool is_recovring = false) = 0; + bool allow_run_on_candidate = false) = 0; virtual void ScanOpen( const TableName &table_name, @@ -404,7 +408,8 @@ class CcHandler virtual void NewTxn(CcHandlerResult &hres, IsolationLevel iso_level, NodeGroupId tx_ng_id, - uint32_t log_group_id) = 0; + uint32_t log_group_id, + bool allow_run_on_candidate) = 0; /// /// Sets the commit timestamp of the input tx. diff --git a/tx_service/include/cc/cc_map.h b/tx_service/include/cc/cc_map.h index 0d1434b6..7823a5bd 100644 --- a/tx_service/include/cc/cc_map.h +++ b/tx_service/include/cc/cc_map.h @@ -224,6 +224,8 @@ class CcMap KickoutCcEntryCc *kickout_cc = nullptr, bool *is_success = nullptr) = 0; virtual void Clean() = 0; + virtual void PrintCcPage(LruPage *lru_page) = 0; + virtual void PrintAllCcPages() = 0; virtual void CleanEntry(LruEntry *entry, LruPage *page) = 0; diff --git a/tx_service/include/cc/cc_page_clean_guard.h b/tx_service/include/cc/cc_page_clean_guard.h index 39c1c316..6e9ca9d3 100644 --- a/tx_service/include/cc/cc_page_clean_guard.h +++ b/tx_service/include/cc/cc_page_clean_guard.h @@ -396,9 +396,31 @@ struct CcPageCleanGuardWithoutKickoutCc // map. At this time, we cannot evict this cce because the read // request will still access the old range. assert(*dirty_range_ts > 0); + // LOG(INFO) << "CanBeCleaned: cce: 0x" << std::hex << (void *) cce + // << ", dirty range ts: " << std::dec << *dirty_range_ts + // << ", range last sync ts: " << *range_last_sync_ts + // << ", ckpt ts: " << cce->CkptTs() + // << ", is free: " << std::boolalpha << cce->IsFree() + // << ", being ckpt: " << std::boolalpha + // << cce->GetBeingCkpt() << ", can not be cleaned."; return {false, false}; } + // if (!cce->IsFree() || cce->GetBeingCkpt()) + // { + // LOG(INFO) << "CanBeCleaned: cce: 0x" << std::hex << (void *) cce + // << ", is free: " << std::boolalpha << cce->IsFree() + // << ", being ckpt: " << std::boolalpha + // << cce->GetBeingCkpt() << ", can not be cleaned."; + // } + // else if (cce->IsFree() && !cce->GetBeingCkpt()) + // { + // LOG(INFO) << "CanBeCleaned: cce: 0x" << std::hex << (void *) cce + // << ", is free: " << std::boolalpha << cce->IsFree() + // << ", being ckpt: " << std::boolalpha + // << cce->GetBeingCkpt() << ", can be cleaned."; + // } + return {(cce->IsFree() && !cce->GetBeingCkpt()), false}; } diff --git a/tx_service/include/cc/cc_req_misc.h b/tx_service/include/cc/cc_req_misc.h index ed9ea8b9..504d1c08 100644 --- a/tx_service/include/cc/cc_req_misc.h +++ b/tx_service/include/cc/cc_req_misc.h @@ -437,7 +437,8 @@ struct FillStoreSliceCc : public CcRequestBase { assert(err_code != CcErrorCode::NO_ERROR); DLOG(ERROR) << "Abort this FillStoreSliceCc request with error: " - << CcErrorMessage(err_code); + << CcErrorMessage(err_code) + << ", table name: " << table_name_->StringView(); bool finish_all = SetError(err_code); // Recycle request if (finish_all) @@ -989,6 +990,8 @@ struct UpdateCceCkptTsCc : public CcRequestBase UpdateCceCkptTsCc(const UpdateCceCkptTsCc &) = delete; UpdateCceCkptTsCc &operator=(const UpdateCceCkptTsCc &) = delete; + bool ValidTermCheck() const; + bool Execute(CcShard &ccs) override; void SetFinished() diff --git a/tx_service/include/cc/cc_request.h b/tx_service/include/cc/cc_request.h index 97e93fae..7502bad9 100644 --- a/tx_service/include/cc/cc_request.h +++ b/tx_service/include/cc/cc_request.h @@ -232,6 +232,12 @@ struct TemplatedCcRequest : public CcRequestBase // is marked as errored. if (init_res.error != CcErrorCode::NO_ERROR) { + LOG(INFO) + << "AbortCcRequest, tx:" << Txn() + << ", table name: " << table_name_->StringView() + << ", error code: " + << static_cast(init_res.error) + << ", with result: 0x" << std::hex << res_; AbortCcRequest(init_res.error); } // The req is aborted or will be re-enqueued. @@ -249,6 +255,11 @@ struct TemplatedCcRequest : public CcRequestBase { ccm_ = ccm; } + // LOG(INFO) << "Execute CcRequest, tx:" << Txn() + // << ", table name: " << table_name_->StringView() + // << ", table type: " + // << static_cast(table_name_->Type()) + // << ", txn: " << Txn(); assert(ccm != nullptr); assert(ccs.core_id_ == ccm->shard_->core_id_); return ccm->Execute(*typed_req); @@ -283,6 +294,11 @@ struct TemplatedCcRequest : public CcRequestBase return node_group_id_; } + int64_t NodeGroupTerm() const + { + return ng_term_; + } + int64_t TxTerm() const { return tx_term_; @@ -313,6 +329,10 @@ struct TemplatedCcRequest : public CcRequestBase void AbortCcRequest(CcErrorCode err_code) override { assert(err_code != CcErrorCode::NO_ERROR); + LOG(INFO) << "AbortCcRequest, tx:" << Txn() + << ", table name: " << table_name_->StringView() + << ", error code: " << static_cast(err_code) + << ", with result: 0x" << std::hex << res_; bool finished = res_->SetError(err_code); if (finished) @@ -366,6 +386,34 @@ struct AcquireCc AcquireCc(const AcquireCc &rhs) = delete; AcquireCc(AcquireCc &&rhs) = delete; + bool ValidTermCheck() override + { + int64_t cc_ng_term = -1; + if (allow_run_on_candidate_) + { + cc_ng_term = + Sharder::Instance().CandidateLeaderTerm(node_group_id_); + } + if (cc_ng_term < 0) + { + cc_ng_term = Sharder::Instance().LeaderTerm(node_group_id_); + } + + if (ng_term_ < 0) + { + ng_term_ = cc_ng_term; + } + + if (cc_ng_term < 0 || cc_ng_term != ng_term_) + { + return false; + } + else + { + return true; + } + } + void Reset(const TableName *tname, const uint64_t schema_version, const TxKey *key, @@ -378,7 +426,8 @@ struct AcquireCc uint32_t hd_res_idx, CcProtocol proto, IsolationLevel iso_level, - bool abort_if_oom) + bool abort_if_oom, + bool allow_run_on_candidate) { uint32_t ng_id = Sharder::Instance().ShardToCcNodeGroup(key_shard_code); TemplatedCcRequest>::Reset( @@ -395,6 +444,7 @@ struct AcquireCc is_local_ = true; block_by_lock_ = false; abort_if_oom_ = abort_if_oom; + allow_run_on_candidate_ = allow_run_on_candidate; } void Reset(const TableName *tname, @@ -512,6 +562,7 @@ struct AcquireCc bool is_local_{true}; bool block_by_lock_{false}; bool abort_if_oom_{false}; + bool allow_run_on_candidate_{false}; }; struct AcquireAllCc : public TemplatedCcRequest @@ -526,6 +577,28 @@ struct AcquireAllCc : public TemplatedCcRequest AcquireAllCc(const AcquireAllCc &rhs) = delete; AcquireAllCc(AcquireAllCc &&rhs) = delete; + bool ValidTermCheck() override + { + int64_t cc_ng_term = Sharder::Instance().LeaderTerm(node_group_id_); + int64_t candidate_ng_term = + Sharder::Instance().CandidateLeaderTerm(node_group_id_); + cc_ng_term = std::max(cc_ng_term, candidate_ng_term); + + if (ng_term_ < 0) + { + ng_term_ = cc_ng_term; + } + + if (cc_ng_term < 0 || cc_ng_term != ng_term_) + { + return false; + } + else + { + return true; + } + } + void Reset(const TableName *tname, const TxKey *key, uint32_t node_group_id, @@ -689,7 +762,17 @@ struct PostWriteCc : public TemplatedCcRequest bool ValidTermCheck() override { - int64_t cc_ng_term = Sharder::Instance().LeaderTerm(node_group_id_); + int64_t cc_ng_term = -1; + if (allow_run_on_candidate_) + { + cc_ng_term = + Sharder::Instance().CandidateLeaderTerm(node_group_id_); + } + if (cc_ng_term < 0) + { + cc_ng_term = Sharder::Instance().LeaderTerm(node_group_id_); + } + if (cce_addr_ != nullptr) { if (cce_addr_->Term() != cc_ng_term) @@ -740,7 +823,8 @@ struct PostWriteCc : public TemplatedCcRequest const TxRecord *rec, OperationType operation_type, uint32_t key_shard_code, - CcHandlerResult *res) + CcHandlerResult *res, + bool allow_run_on_candidate) { TemplatedCcRequest::Reset( nullptr, res, addr->NodeGroupId(), tx_number, tx_term); @@ -754,6 +838,7 @@ struct PostWriteCc : public TemplatedCcRequest is_remote_ = false; ccm_ = nullptr; is_initial_insert_ = false; + allow_run_on_candidate_ = allow_run_on_candidate; } void Reset(const TxKey *key, @@ -788,6 +873,7 @@ struct PostWriteCc : public TemplatedCcRequest is_remote_ = false; ccm_ = nullptr; is_initial_insert_ = initial_insertion; + allow_run_on_candidate_ = false; } void Reset(const CcEntryAddr *addr, @@ -811,6 +897,7 @@ struct PostWriteCc : public TemplatedCcRequest is_remote_ = true; ccm_ = nullptr; is_initial_insert_ = false; + allow_run_on_candidate_ = false; } void Reset(const TableName *table_name, @@ -845,6 +932,7 @@ struct PostWriteCc : public TemplatedCcRequest is_remote_ = true; ccm_ = nullptr; is_initial_insert_ = initial_insertion; + allow_run_on_candidate_ = false; } const CcEntryAddr *CceAddr() const @@ -909,6 +997,7 @@ struct PostWriteCc : public TemplatedCcRequest const void *key_; const std::string *key_str_; }; + bool allow_run_on_candidate_{false}; }; struct PostWriteAllCc @@ -919,6 +1008,28 @@ struct PostWriteAllCc PostWriteAllCc(const PostWriteAllCc &rhs) = delete; PostWriteAllCc(PostWriteAllCc &&rhs) = delete; + bool ValidTermCheck() override + { + int64_t cc_ng_term = Sharder::Instance().LeaderTerm(node_group_id_); + int64_t candidate_ng_term = + Sharder::Instance().CandidateLeaderTerm(node_group_id_); + cc_ng_term = std::max(cc_ng_term, candidate_ng_term); + + if (ng_term_ < 0) + { + ng_term_ = cc_ng_term; + } + + if (cc_ng_term < 0 || cc_ng_term != ng_term_) + { + return false; + } + else + { + return true; + } + } + void Reset(const TableName *tname, const TxKey *key, uint32_t node_group_id, @@ -1117,9 +1228,29 @@ struct PostReadCc : public TemplatedCcRequest bool ValidTermCheck() override { - int64_t cc_ng_term = Sharder::Instance().LeaderTerm(node_group_id_); - int64_t standby_node_term = Sharder::Instance().StandbyNodeTerm(); - cc_ng_term = std::max(cc_ng_term, standby_node_term); + int64_t cc_ng_term = -1; + if (allow_run_on_candidate_) + { + cc_ng_term = + Sharder::Instance().CandidateLeaderTerm(node_group_id_); + } + + if (cc_ng_term < 0) + { + cc_ng_term = Sharder::Instance().LeaderTerm(node_group_id_); + int64_t standby_node_term = Sharder::Instance().StandbyNodeTerm(); + cc_ng_term = std::max(cc_ng_term, standby_node_term); + } + + if (ng_term_ < 0) + { + ng_term_ = cc_ng_term; + } + + if (cc_ng_term < 0 || cc_ng_term != ng_term_) + { + return false; + } assert(cce_addr_ != nullptr); if (cce_addr_->Term() != cc_ng_term) @@ -1152,7 +1283,8 @@ struct PostReadCc : public TemplatedCcRequest uint64_t key_ts, uint64_t gap_ts, PostReadType post_read_type, - CcHandlerResult *res) + CcHandlerResult *res, + bool allow_run_on_candidate = false) { TemplatedCcRequest::Reset( nullptr, res, addr->NodeGroupId(), tx_number, tx_term); @@ -1163,6 +1295,7 @@ struct PostReadCc : public TemplatedCcRequest gap_ts_ = gap_ts; post_read_type_ = post_read_type; ccm_ = nullptr; + allow_run_on_candidate_ = allow_run_on_candidate; } const CcEntryAddr *CceAddr() const @@ -1196,6 +1329,7 @@ struct PostReadCc : public TemplatedCcRequest uint64_t key_ts_; uint64_t gap_ts_; PostReadType post_read_type_; + bool allow_run_on_candidate_{false}; }; struct ReadCc : public TemplatedCcRequest @@ -1233,7 +1367,7 @@ struct ReadCc : public TemplatedCcRequest bool ValidTermCheck() override { int64_t cc_ng_term = -1; - if (is_in_recovering_) + if (allow_run_on_candidate_) { cc_ng_term = Sharder::Instance().CandidateLeaderTerm(node_group_id_); @@ -1250,6 +1384,8 @@ struct ReadCc : public TemplatedCcRequest { if (tmp_cce_addr.Term() != cc_ng_term) { + LOG(INFO) << "ReadCc not leader, table name: " + << table_name_->StringView() << ", txn: " << Txn(); return false; } @@ -1278,6 +1414,8 @@ struct ReadCc : public TemplatedCcRequest if (cc_ng_term < 0 || cc_ng_term != ng_term_) { + LOG(INFO) << "ReadCc not leader, table name: " + << table_name_->StringView() << ", txn: " << Txn(); return false; } else @@ -1304,7 +1442,7 @@ struct ReadCc : public TemplatedCcRequest bool is_for_write = false, bool is_covering_keys = false, std::vector *archives = nullptr, - bool is_in_recovering = false, + bool allow_run_on_candidate = false, bool point_read_on_miss = false, int32_t partition_id = -1, bool abort_if_oom = false) @@ -1326,7 +1464,7 @@ struct ReadCc : public TemplatedCcRequest cce_ptr_ = nullptr; archives_ = archives; is_local_ = true; - is_in_recovering_ = is_in_recovering; + allow_run_on_candidate_ = allow_run_on_candidate; is_covering_keys_ = is_covering_keys; point_read_on_cache_miss_ = point_read_on_miss; blk_type_ = NotBlocked; @@ -1359,6 +1497,7 @@ struct ReadCc : public TemplatedCcRequest bool is_for_write = false, bool is_covering_keys = false, std::vector *archives = nullptr, + bool allow_run_on_candidate = false, bool point_read_on_miss = false, int32_t partition_id = -1, bool abort_if_oom = false) @@ -1380,7 +1519,7 @@ struct ReadCc : public TemplatedCcRequest cce_ptr_ = nullptr; archives_ = archives; is_local_ = false; - is_in_recovering_ = false; + allow_run_on_candidate_ = allow_run_on_candidate; is_covering_keys_ = is_covering_keys; point_read_on_cache_miss_ = point_read_on_miss; blk_type_ = NotBlocked; @@ -1434,7 +1573,7 @@ struct ReadCc : public TemplatedCcRequest cce_ptr_ = nullptr; archives_ = archives; is_local_ = true; - is_in_recovering_ = false; + allow_run_on_candidate_ = false; is_covering_keys_ = is_covering_keys; point_read_on_cache_miss_ = point_read_on_miss; blk_type_ = NotBlocked; @@ -1533,9 +1672,9 @@ struct ReadCc : public TemplatedCcRequest return is_local_; } - bool IsInRecovering() const + bool AllowRunOnCandidate() const { - return is_in_recovering_; + return allow_run_on_candidate_; } bool IsCoveringKeys() const @@ -1610,8 +1749,9 @@ struct ReadCc : public TemplatedCcRequest LruEntry *cce_ptr_{nullptr}; bool is_local_{true}; - // Is issued in a recovering process - bool is_in_recovering_{false}; + // True if this ccrequest is allowed to run on candidate node, such as + // issued in a recovering process + bool allow_run_on_candidate_{false}; // Reserved for unique sk read bool is_covering_keys_{false}; bool point_read_on_cache_miss_{false}; @@ -3783,8 +3923,11 @@ struct HashPartitionDataSyncScanCc : public CcRequestBase bool ValidTermCheck() { int64_t cc_ng_term = Sharder::Instance().LeaderTerm(node_group_id_); + int64_t candidate_ng_term = + Sharder::Instance().CandidateLeaderTerm(node_group_id_); int64_t standby_node_term = Sharder::Instance().StandbyNodeTerm(); - int64_t current_term = std::max(cc_ng_term, standby_node_term); + int64_t current_term = + std::max({cc_ng_term, standby_node_term, candidate_ng_term}); if (node_group_term_ < 0) { @@ -3840,6 +3983,14 @@ struct HashPartitionDataSyncScanCc : public CcRequestBase return false; } + int64_t GetNodeGroupLeaderTerm() const + { + int64_t leader_term = Sharder::Instance().LeaderTerm(node_group_id_); + int64_t candidate_term = + Sharder::Instance().CandidateLeaderTerm(node_group_id_); + return std::max(leader_term, candidate_term); + } + bool IsDrained() const { return pause_pos_.second; @@ -4105,8 +4256,11 @@ struct RangePartitionDataSyncScanCc : public CcRequestBase bool ValidTermCheck() { int64_t cc_ng_term = Sharder::Instance().LeaderTerm(node_group_id_); + int64_t candidate_ng_term = + Sharder::Instance().CandidateLeaderTerm(node_group_id_); int64_t standby_node_term = Sharder::Instance().StandbyNodeTerm(); - int64_t current_term = std::max(cc_ng_term, standby_node_term); + int64_t current_term = + std::max({cc_ng_term, standby_node_term, candidate_ng_term}); if (node_group_term_ < 0) { @@ -4133,7 +4287,6 @@ struct RangePartitionDataSyncScanCc : public CcRequestBase SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); return false; } - scan_count_++; CcMap *ccm = ccs.GetCcm(*table_name_, node_group_id_); if (ccm == nullptr) { @@ -4427,14 +4580,17 @@ struct RangePartitionDataSyncScanCc : public CcRequestBase return last_data_sync_ts_; } + bool RunOnCandidateNode() const + { + return true; + } + std::vector accumulated_scan_cnt_; std::vector accumulated_flush_data_size_; // std::vector is not safe to use in multi-threaded environment, std::vector scan_heap_is_full_{0}; - size_t scan_count_{0}; - private: struct SliceCoordinator { @@ -6024,6 +6180,9 @@ struct KickoutCcEntryCc : public TemplatedCcRequest bool Execute(CcShard &ccs) override { int64_t ng_term = Sharder::Instance().LeaderTerm(node_group_id_); + int64_t candidate_ng_term = + Sharder::Instance().CandidateLeaderTerm(node_group_id_); + ng_term = std::max(ng_term, candidate_ng_term); if (ng_term < 0 && clean_type_ == CleanType::CleanDeletedData) { // Purge deleted data is the only type of kickout cc that will @@ -6036,7 +6195,7 @@ struct KickoutCcEntryCc : public TemplatedCcRequest if (ng_term < 0) { - return SetError(CcErrorCode::TX_NODE_NOT_LEADER); + return SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); } if (clean_type_ == CleanType::CleanCcm) @@ -6672,6 +6831,9 @@ struct UpdateKeyCacheCc : public CcRequestBase bool Execute(CcShard &ccs) override { int64_t ng_term = Sharder::Instance().LeaderTerm(node_group_id_); + int64_t candidate_ng_term = + Sharder::Instance().CandidateLeaderTerm(node_group_id_); + ng_term = std::max(ng_term, candidate_ng_term); if (ng_term < 0 || ng_term != ng_term_) { return SetFinish(); @@ -8783,6 +8945,9 @@ struct ScanSliceDeltaSizeCcForRangePartition : public CcRequestBase bool ValidTermCheck() const { int64_t cc_ng_term = Sharder::Instance().LeaderTerm(node_group_id_); + int64_t candidate_ng_term = + Sharder::Instance().CandidateLeaderTerm(node_group_id_); + cc_ng_term = std::max(cc_ng_term, candidate_ng_term); assert(node_group_term_ > 0); return cc_ng_term >= 0 && cc_ng_term == node_group_term_; @@ -9240,6 +9405,9 @@ struct SampleSubRangeKeysCc : public CcRequestBase bool ValidTermCheck() { int64_t cc_ng_term = Sharder::Instance().LeaderTerm(node_group_id_); + int64_t candidate_ng_term = + Sharder::Instance().CandidateLeaderTerm(node_group_id_); + cc_ng_term = std::max(cc_ng_term, candidate_ng_term); assert(node_group_term_ > 0); if (cc_ng_term < 0 || cc_ng_term != node_group_term_) diff --git a/tx_service/include/cc/cc_shard.h b/tx_service/include/cc/cc_shard.h index ee977e68..842ddadd 100644 --- a/tx_service/include/cc/cc_shard.h +++ b/tx_service/include/cc/cc_shard.h @@ -600,6 +600,9 @@ class CcShard uint64_t min_ts = UINT64_MAX; int64_t cc_ng_term = Sharder::Instance().LeaderTerm(cc_ng_id); + int64_t candidate_ng_term = + Sharder::Instance().CandidateLeaderTerm(cc_ng_id); + cc_ng_term = std::max(cc_ng_term, candidate_ng_term); if (cc_ng_term < 0) { cc_ng_term = Sharder::Instance().StandbyNodeTerm(); @@ -1122,6 +1125,7 @@ class CcShard void OnDirtyDataFlushed() { + LOG(INFO) << "OnDirtyDataFlushed on shard: " << core_id_; ResetCleanStart(); if (WaitListSizeForMemory() > 0) { diff --git a/tx_service/include/cc/cluster_config_cc_map.h b/tx_service/include/cc/cluster_config_cc_map.h index 3e55acb9..9af4be24 100644 --- a/tx_service/include/cc/cluster_config_cc_map.h +++ b/tx_service/include/cc/cluster_config_cc_map.h @@ -67,13 +67,8 @@ class ClusterConfigCcMap hd_res->ClearRefCnt(); AcquireAllResult &acquire_all_result = hd_res->Value(); uint32_t ng_id = req.NodeGroupId(); - int64_t ng_term = Sharder::Instance().LeaderTerm(ng_id); - - if (ng_term < 0) - { - hd_res->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); - return true; - } + int64_t ng_term = req.NodeGroupTerm(); + assert(ng_term > 0); LockType acquired_lock = LockType::NoLock; CcErrorCode err_code = CcErrorCode::NO_ERROR; diff --git a/tx_service/include/cc/local_cc_handler.h b/tx_service/include/cc/local_cc_handler.h index eae6ba46..8513a85f 100644 --- a/tx_service/include/cc/local_cc_handler.h +++ b/tx_service/include/cc/local_cc_handler.h @@ -55,7 +55,8 @@ class LocalCcHandler : public CcHandler uint32_t hd_res_idx, CcProtocol proto, IsolationLevel iso_level, - bool abort_if_oom) override; + bool abort_if_oom, + bool allow_run_on_candidate) override; void AcquireWriteAll(const TableName &table_name, const TxKey &key, @@ -103,20 +104,21 @@ class LocalCcHandler : public CcHandler const TxRecord *record, OperationType operation_type, uint32_t key_shard_code, - CcHandlerResult &hres) override; + CcHandlerResult &hres, + bool allow_run_on_candidate) override; - CcReqStatus PostRead( - uint64_t tx_number, - int64_t tx_term, - uint16_t command_id, - uint64_t key_ts, - uint64_t gap_ts, - uint64_t commit_ts, - const CcEntryAddr &ccentry_addr, - CcHandlerResult &hres, - bool is_local = false, - bool need_remote_resp = true, - PostReadType post_read_type = PostReadType::Release) override; + CcReqStatus PostRead(uint64_t tx_number, + int64_t tx_term, + uint16_t command_id, + uint64_t key_ts, + uint64_t gap_ts, + uint64_t commit_ts, + const CcEntryAddr &ccentry_addr, + CcHandlerResult &hres, + bool is_local = false, + bool need_remote_resp = true, + PostReadType post_read_type = PostReadType::Release, + bool allow_run_on_candidate = false) override; /// /// Starts concurrency control for the input key and returns the key's @@ -146,6 +148,7 @@ class LocalCcHandler : public CcHandler CcProtocol proto = CcProtocol::OCC, bool is_for_write = false, bool is_covering_keys = false, + bool allow_run_on_candidate = false, bool point_read_on_miss = false, int32_t partition_id = -1, bool abort_if_oom = false) override; @@ -171,7 +174,7 @@ class LocalCcHandler : public CcHandler IsolationLevel iso_level = IsolationLevel::RepeatableRead, CcProtocol proto = CcProtocol::Locking, bool is_for_write = false, - bool is_recovering = false, + bool allow_run_on_candidate = false, bool execute_immediately = true) override; std::tuple @@ -194,7 +197,7 @@ class LocalCcHandler : public CcHandler IsolationLevel iso_level = IsolationLevel::RepeatableRead, CcProtocol proto = CcProtocol::Locking, bool is_for_write = false, - bool is_recovring = false) override; + bool allow_run_on_candidate = false) override; void ScanOpen(const TableName &table_name, const uint64_t schema_version, @@ -281,7 +284,8 @@ class LocalCcHandler : public CcHandler void NewTxn(CcHandlerResult &hres, IsolationLevel iso_level, NodeGroupId tx_ng_id, - uint32_t log_group_id) override; + uint32_t log_group_id, + bool allow_run_on_candidate) override; /// /// Sets the commit timestamp of the input tx. diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index 961bee52..c0cca56d 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -2449,6 +2449,7 @@ class LocalCcShards bool GetNextRangePartitionId(const TableName &tablename, const TableSchema *table_schema, + NodeGroupId ng_id, uint32_t range_cnt, int32_t &out_next_partition_id); diff --git a/tx_service/include/cc/range_bucket_cc_map.h b/tx_service/include/cc/range_bucket_cc_map.h index 6805aa5f..0f01a77d 100644 --- a/tx_service/include/cc/range_bucket_cc_map.h +++ b/tx_service/include/cc/range_bucket_cc_map.h @@ -89,22 +89,23 @@ class RangeBucketCcMap TX_TRACE_DUMP(&req); assert(req.IsLocal()); - uint32_t ng_id = req.NodeGroupId(); - int64_t ng_term = Sharder::Instance().LeaderTerm(ng_id); - if (req.IsInRecovering()) - { - ng_term = Sharder::Instance().CandidateLeaderTerm(ng_id); - } - else - { - ng_term = std::max(ng_term, Sharder::Instance().StandbyNodeTerm()); - } - + int64_t ng_term = req.NodeGroupTerm(); if (ng_term < 0) { - req.Result()->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); - return true; + if (req.AllowRunOnCandidate()) + { + ng_term = + Sharder::Instance().CandidateLeaderTerm(req.NodeGroupId()); + } + if (ng_term < 0) + { + ng_term = Sharder::Instance().LeaderTerm(req.NodeGroupId()); + int64_t standby_node_term = + Sharder::Instance().StandbyNodeTerm(); + ng_term = std::max(ng_term, standby_node_term); + } } + assert(ng_term > 0); const RangeBucketKey *bucket_key = static_cast(req.Key()); @@ -177,6 +178,9 @@ class RangeBucketCcMap { case CcErrorCode::NO_ERROR: { + // LOG(INFO) << "RangeBucketCcMap::Execute: NO_ERROR, ng_term: " + // << ng_term << ", txn: " << req.Txn() << ", lock type: " + // << static_cast(acquired_lock); CcEntryAddr &cce_addr = hd_result->Value().cce_addr_; cce_addr.SetCceLock( reinterpret_cast(cce->GetKeyGapLockAndExtraData()), diff --git a/tx_service/include/cc/range_cc_map.h b/tx_service/include/cc/range_cc_map.h index 5b39fd71..88bd2d28 100644 --- a/tx_service/include/cc/range_cc_map.h +++ b/tx_service/include/cc/range_cc_map.h @@ -217,12 +217,23 @@ class RangeCcMap : public TemplateCcMap assert(this->table_name_ == *req.GetTableName()); CcHandlerResult *hd_result = req.Result(); - int64_t ng_term = Sharder::Instance().LeaderTerm(req.NodeGroupId()); + int64_t ng_term = req.NodeGroupTerm(); if (ng_term < 0) { - hd_result->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); - return true; + if (req.AllowRunOnCandidate()) + { + ng_term = + Sharder::Instance().CandidateLeaderTerm(req.NodeGroupId()); + } + if (ng_term < 0) + { + ng_term = Sharder::Instance().LeaderTerm(req.NodeGroupId()); + int64_t standby_node_term = + Sharder::Instance().StandbyNodeTerm(); + ng_term = std::max(ng_term, standby_node_term); + } } + assert(ng_term > 0); // For range cc maps, we assume that all of a table's ranges are loaded // into memory for caching when the range cc map is initialized. There @@ -399,13 +410,6 @@ class RangeCcMap : public TemplateCcMap */ bool Execute(PostWriteAllCc &req) override { - int64_t ng_term = Sharder::Instance().LeaderTerm(req.NodeGroupId()); - if (ng_term < 0) - { - req.Result()->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); - return false; - } - // When the commit ts is 0 or the commit type is DowngradeLock, the // request commits nothing and only removes the write intents/locks // acquired earlier. @@ -610,6 +614,8 @@ class RangeCcMap : public TemplateCcMap if (txservice_enable_key_cache && this->table_name_.IsBase()) { + int64_t ng_term = req.NodeGroupTerm(); + assert(ng_term > 0); // try to init the key cache for new range if it // lands on this ng for (auto new_range : new_range_entries) diff --git a/tx_service/include/cc/template_cc_map.h b/tx_service/include/cc/template_cc_map.h index f2a00630..c886bbad 100644 --- a/tx_service/include/cc/template_cc_map.h +++ b/tx_service/include/cc/template_cc_map.h @@ -199,16 +199,14 @@ class TemplateCcMap : public CcMap const KeyT *target_key = nullptr; KeyT decoded_key; - int64_t ng_term = Sharder::Instance().LeaderTerm(req.NodeGroupId()); + int64_t ng_term = req.NodeGroupTerm(); + assert(ng_term > 0); CODE_FAULT_INJECTOR("term_TemplateCcMap_Execute_AcquireCc", { - LOG(INFO) << "FaultInject term_TemplateCcMap_Execute_AcquireCc"; + LOG(INFO) << "FaultInject term_TemplateCcMap_Execute_AcquireCc"; ng_term = -1; - }); - if (ng_term < 0) - { hd_res->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); return true; - } + }); if (req.SchemaVersion() != 0 && req.SchemaVersion() != schema_ts_) { @@ -511,14 +509,7 @@ class TemplateCcMap : public CcMap req.Result()->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); return true; } - }); - - if (!Sharder::Instance().CheckLeaderTerm(cce_addr->NodeGroupId(), - cce_addr->Term())) - { - req.Result()->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); - return true; - } + }) const ValueT *commit_val = static_cast(req.Payload()); TxNumber txn = req.Txn(); @@ -702,11 +693,8 @@ class TemplateCcMap : public CcMap bool will_insert = false; uint32_t ng_id = req.NodeGroupId(); - int64_t ng_term = Sharder::Instance().LeaderTerm(ng_id); - if (ng_term < 0) - { - return hd_res->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); - } + int64_t ng_term = req.NodeGroupTerm(); + assert(ng_term > 0); uint16_t tx_core_id = ((req.Txn() >> 32L) & 0x3FF) % shard_->core_cnt_; @@ -996,13 +984,6 @@ class TemplateCcMap : public CcMap return false; } - int64_t ng_term = Sharder::Instance().LeaderTerm(req.NodeGroupId()); - if (ng_term < 0) - { - req.Result()->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); - return true; - } - const KeyT *target_key = nullptr; if (req.Key() != nullptr) { @@ -1241,19 +1222,6 @@ class TemplateCcMap : public CcMap } }); - int64_t standby_node_term = Sharder::Instance().StandbyNodeTerm(); - - if (!Sharder::Instance().CheckLeaderTerm(cce_addr.NodeGroupId(), - cce_addr.Term()) && - (standby_node_term < 0 || standby_node_term != cce_addr.Term())) - { - LOG(INFO) << "PostReadCc, node_group(#" << cce_addr.NodeGroupId() - << ") term < 0, tx:" << req.Txn() - << " ,cce: " << cce_addr.ExtractCce(); - hd_res->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); - return true; - } - uint64_t key_ts = req.KeyTs(); uint64_t gap_ts = req.GapTs(); uint64_t commit_ts = req.CommitTs(); @@ -1264,6 +1232,12 @@ class TemplateCcMap : public CcMap // accessed. recycle_lock = table_name_.Type() != TableType::Catalog; + // LOG(INFO) << "PostReadCc::Execute: ng_term: " << req.NodeGroupTerm() + // << ", txn: " << txn << ", table: " << + // table_name_.StringView() + // << ", table type: " + // << static_cast(table_name_.Type()); + // FIXME(lzx): Now, we don't backfill for "Unkown" entry when scanning. // So, Validate operation fails if another tx backfilled it. Temporary // fix is that we don't validate for "Unkown" status results. @@ -1419,15 +1393,20 @@ class TemplateCcMap : public CcMap }); uint32_t ng_id = req.NodeGroupId(); - int64_t ng_term = -1; - if (req.IsInRecovering()) - { - ng_term = Sharder::Instance().CandidateLeaderTerm(ng_id); - } - else + int64_t ng_term = req.NodeGroupTerm(); + if (ng_term < 0) { - ng_term = Sharder::Instance().LeaderTerm(ng_id); - ng_term = std::max(ng_term, Sharder::Instance().StandbyNodeTerm()); + if (req.AllowRunOnCandidate()) + { + ng_term = Sharder::Instance().CandidateLeaderTerm(ng_id); + } + if (ng_term < 0) + { + ng_term = Sharder::Instance().LeaderTerm(ng_id); + int64_t standby_node_term = + Sharder::Instance().StandbyNodeTerm(); + ng_term = std::max(ng_term, standby_node_term); + } } if (ng_term < 0) @@ -1441,6 +1420,8 @@ class TemplateCcMap : public CcMap if (req.SchemaVersion() != 0 && req.SchemaVersion() != schema_ts_) { hd_res->SetError(CcErrorCode::REQUESTED_TABLE_SCHEMA_MISMATCH); + LOG(INFO) << "ReadCc, schema version mismatch, tx:" << req.Txn() + << ", table name: " << table_name_.StringView(); return true; } @@ -1602,6 +1583,14 @@ class TemplateCcMap : public CcMap if (Type() == TableType::Primary || Type() == TableType::UniqueSecondary) { + // LOG(INFO) + // << "ReadCc PinRangeSlice with non force + // load." + // << ", shard: " << shard_->core_id_ + // << ", table_name: " << + // table_name_.StringView() + // << ", partition id: " << req.PartitionId() + // << ", txn: " << req.Txn(); RangeSliceOpStatus pin_status; RangeSliceId slice_id = shard_->local_shards_.PinRangeSlice( @@ -1642,6 +1631,12 @@ class TemplateCcMap : public CcMap req.SetCacheHitMissCollected(); } + // LOG(INFO) + // << "ReadCc PinRangeSlice done, pin_status: " + // << static_cast(pin_status) + // << ", shard: " << shard_->core_id_ + // << ", table_name: " << + // table_name_.StringView(); if (pin_status == RangeSliceOpStatus::Successful || pin_status == RangeSliceOpStatus::KeyNotExists) { @@ -1841,6 +1836,14 @@ class TemplateCcMap : public CcMap // But the cc map is full and cannot allocates a new entry. if (cce == nullptr) { + // LOG(INFO) + // << "ReadCc, cce is nullptr, tx:" << req.Txn() + // << ", table name: " << table_name_.StringView() + // << ", key: " << look_key->ToString() + // << ", txn: " << req.Txn() + // << ", is for write: " << std::boolalpha + // << req.IsForWrite() + // << ", shard: " << shard_->core_id_; shard_->EnqueueWaitListIfMemoryFull(&req); return false; } @@ -5756,6 +5759,23 @@ class TemplateCcMap : public CcMap // round ckpt. need_export = false; } + else if (req.RunOnCandidateNode()) + { + // Just skip the cce in this round ckpt. + // if (req.export_base_table_item_ && + // table_name_.StringView() == "./sbtest/history" && + // shard_->core_id_ == 0) + // { + // LOG(INFO) + // << "DataSync for range split, start key: " + // << req.start_key_->ToString() + // << ", end key: " << req.end_key_->ToString() + // << ", table name: " << table_name_.StringView() + // << ", with cce: " << std::hex << (void *) (cce) + // << std::dec << ", txn: " << req.Txn(); + // } + need_export = false; + } else { LOG(ERROR) @@ -5786,7 +5806,13 @@ class TemplateCcMap : public CcMap req.accumulated_flush_data_size_[shard_->core_id_] += flush_size; - + // if (shard_->core_id_ == 0) + // { + // LOG(INFO) << "Exported the cce: 0x" << std::hex + // << (void *) cce << ", on shard: 0" + // << ", table name: " << + // table_name_.StringView(); + // } if (export_result.second) { is_scan_mem_full = true; @@ -5903,15 +5929,6 @@ class TemplateCcMap : public CcMap return false; } - int64_t ng_term = Sharder::Instance().LeaderTerm(req.NodeGroupId()); - int64_t standby_node_term = Sharder::Instance().StandbyNodeTerm(); - int64_t current_term = std::max(ng_term, standby_node_term); - - if (current_term < 0 || current_term != req.node_group_term_) - { - req.SetError(CcErrorCode::TX_NODE_NOT_LEADER); - return false; - } Iterator it; Iterator end_it = End(); @@ -6083,6 +6100,10 @@ class TemplateCcMap : public CcMap << ", data_sync_ts_: " << req.data_sync_ts_; replay_cmds_notnull = true; + int64_t current_term = req.NodeGroupTerm(); + assert(current_term > 0); + int64_t leader_term = req.GetNodeGroupLeaderTerm(); + // The fetch record may failed when the // cce is touch at 1st place, so the record status can // be Unknown. If the data is owned by this ng, fetch @@ -6107,12 +6128,12 @@ class TemplateCcMap : public CcMap TxKey(key), cce, cc_ng_id_, - ng_term, + current_term, nullptr, part_id); } } - else if (ng_term > 0) + else if (leader_term > 0) { // After node escalate to leader, and we've loaded // from kv, there should be no gap in the buffered @@ -6949,6 +6970,10 @@ class TemplateCcMap : public CcMap if (cce == nullptr) { + // LOG(INFO) << "TemplateCcMap::Execute ReplayLogCc, cce is " + // "nullptr, enqueue wait list." + // << ", table name: " << table_name_.StringView() + // << ", on shard: " << shard_->core_id_; // Since we're not holding any range lock that would // block data sync during replay, just keep retrying // until we have free space in cc map. @@ -8132,6 +8157,16 @@ class TemplateCcMap : public CcMap if (ccp->last_dirty_commit_ts_ <= req.LastDataSyncTs()) { + if (cce->NeedCkpt()) + { + LOG(INFO) << "ScanSliceDeltaSizeCcForRangePartition: cce " + "need ckpt." + << ", ccpage last dirty commit ts: " + << ccp->last_dirty_commit_ts_ + << ", last data sync ts: " << req.LastDataSyncTs() + << ", on shard: " << shard_->core_id_; + assert(false); + } assert(!cce->NeedCkpt()); // Skip the pages that have no updates since last data sync. if (ccp->next_page_ == PagePosInf()) @@ -8724,6 +8759,36 @@ class TemplateCcMap : public CcMap return {free_cnt, next_page}; } + void PrintCcPage(LruPage *lru_page) override + { + CcPage *page = + static_cast< + CcPage *>( + lru_page); + for (auto &cce : page->entries_) + { + LOG(INFO) << "cce: 0x" << std::hex << (void *) (cce.get()) + << ", is free: " << std::boolalpha << cce->IsFree() + << ", being ckpt: " << std::boolalpha + << cce->GetBeingCkpt() + << ", commit ts: " << cce->CommitTs() + << ", payload status: " + << static_cast(cce->PayloadStatus()) + << ", ckpt ts: " << cce->CkptTs() + << ", for table: " << table_name_.StringView(); + } + } + + void PrintAllCcPages() override + { + for (auto it = ccmp_.begin(); it != ccmp_.end(); it++) + { + CcPage &page = + *it->second; + PrintCcPage(&page); + } + } + void Clean() override { size_t total_freed = 0; diff --git a/tx_service/include/data_sync_task.h b/tx_service/include/data_sync_task.h index 70972f35..3196505f 100644 --- a/tx_service/include/data_sync_task.h +++ b/tx_service/include/data_sync_task.h @@ -138,7 +138,8 @@ struct DataSyncTask is_dirty_(is_dirty), sync_ts_adjustable_(need_adjust_ts), task_res_(hres), - need_update_ckpt_ts_(true) + need_update_ckpt_ts_(true), + sync_on_leader_(true) { } @@ -182,6 +183,11 @@ struct DataSyncTask sync_ts_adjustable_ = false; } + void SetSyncOnLeader(bool sync_on_leader) + { + sync_on_leader_ = sync_on_leader; + } + const TableName table_name_; int32_t id_; uint64_t range_version_; @@ -238,6 +244,7 @@ struct DataSyncTask cce_entries_; bool need_update_ckpt_ts_{true}; + bool sync_on_leader_{true}; }; struct FlushTaskEntry diff --git a/tx_service/include/proto/cc_request.proto b/tx_service/include/proto/cc_request.proto index d02a679f..7ce1f6ee 100644 --- a/tx_service/include/proto/cc_request.proto +++ b/tx_service/include/proto/cc_request.proto @@ -888,6 +888,7 @@ message ReadRequest { uint64 schema_version = 15; int32 partition_id = 16; bool abort_if_oom = 17; + bool allow_run_on_candidate = 18; } message ReadResponse { diff --git a/tx_service/include/remote/remote_cc_handler.h b/tx_service/include/remote/remote_cc_handler.h index 83695f21..8ae9e01b 100644 --- a/tx_service/include/remote/remote_cc_handler.h +++ b/tx_service/include/remote/remote_cc_handler.h @@ -130,6 +130,7 @@ class RemoteCcHandler CcProtocol proto = CcProtocol::OCC, bool is_for_write = false, bool is_covering_keys = false, + bool allow_run_on_candidate = false, bool point_read_on_miss = false, int32_t partition_id = -1, bool abort_if_oom = false); diff --git a/tx_service/include/sequences/sequences.h b/tx_service/include/sequences/sequences.h index c459d85b..b5416059 100644 --- a/tx_service/include/sequences/sequences.h +++ b/tx_service/include/sequences/sequences.h @@ -121,6 +121,7 @@ class Sequences int16_t thd_group_id); static bool ApplyIdOfTableRangePartition(const TableName &table, + NodeGroupId ng_id, int64_t desired_vals, int64_t &first_reserved_id, int64_t &reserved_vals, diff --git a/tx_service/include/sharder.h b/tx_service/include/sharder.h index 4586505d..80b33e33 100644 --- a/tx_service/include/sharder.h +++ b/tx_service/include/sharder.h @@ -320,10 +320,13 @@ class Sharder * * @param ng_id The cc node group ID. * @param term The expected leader term. + * @param check_candidate Whether to check the candidate leader term. * @return true, if the current leader is the expected one. * @return false, otherwise. */ - bool CheckLeaderTerm(uint32_t ng_id, int64_t term) const; + bool CheckLeaderTerm(uint32_t ng_id, + int64_t term, + bool check_candidate = false) const; void SetLeaderTerm(NodeGroupId ng_id, int64_t term) { diff --git a/tx_service/include/tx_execution.h b/tx_service/include/tx_execution.h index dc1f81f9..75d2a16c 100644 --- a/tx_service/include/tx_execution.h +++ b/tx_service/include/tx_execution.h @@ -183,7 +183,8 @@ class TransactionExecution : public LinkedTransaction NodeGroupId tx_ng_id = UINT32_MAX, bool start_now = false, const std::function *yield_func = nullptr, - const std::function *resume_func = nullptr); + const std::function *resume_func = nullptr, + bool allow_run_on_candidate = false); std::unique_ptr init_tx_req_; bool CommitTx(CommitTxRequest &commit_req); std::unique_ptr commit_tx_req_; @@ -268,7 +269,8 @@ class TransactionExecution : public LinkedTransaction bool CheckLeaderTerm() const { NodeGroupId ng_id = TxCcNodeId(); - if (Sharder::Instance().CheckLeaderTerm(ng_id, TxTerm()) || + if (Sharder::Instance().CheckLeaderTerm( + ng_id, TxTerm(), allow_run_on_candidate_) || (TxStatus() == TxnStatus::Recovering && Sharder::Instance().CandidateLeaderTerm(ng_id) >= 0)) { @@ -686,6 +688,8 @@ class TransactionExecution : public LinkedTransaction bool bind_to_ext_proc_{false}; + bool allow_run_on_candidate_{false}; + // Initialization phase. InitTxnOperation init_txn_; diff --git a/tx_service/include/tx_request.h b/tx_service/include/tx_request.h index 579d24da..121bb08b 100644 --- a/tx_service/include/tx_request.h +++ b/tx_service/include/tx_request.h @@ -193,12 +193,14 @@ struct InitTxRequest : public TemplateTxRequest const std::function *resume_fptr = nullptr, TransactionExecution *txm = nullptr, uint32_t tx_ng_id = UINT32_MAX, - uint32_t log_group_id = UINT32_MAX) + uint32_t log_group_id = UINT32_MAX, + bool allow_run_on_candidate = false) : TemplateTxRequest(yield_fptr, resume_fptr, txm), iso_level_(level), protocol_(proto), tx_ng_id_(tx_ng_id), - log_group_id_(log_group_id) + log_group_id_(log_group_id), + allow_run_on_candidate_(allow_run_on_candidate) { } @@ -208,6 +210,7 @@ struct InitTxRequest : public TemplateTxRequest CcProtocol protocol_{CcProtocol::OCC}; uint32_t tx_ng_id_{UINT32_MAX}; uint32_t log_group_id_{UINT32_MAX}; + bool allow_run_on_candidate_{false}; }; struct ReadTxRequest @@ -223,7 +226,6 @@ struct ReadTxRequest bool read_local = false, uint64_t ts = 0, bool is_covering_keys = false, - bool is_recovering = false, bool point_read_on_cache_miss = false, const std::function *yield_fptr = nullptr, const std::function *resume_fptr = nullptr, @@ -239,7 +241,6 @@ struct ReadTxRequest ts_(ts), schema_version_(schema_version), is_covering_keys_(is_covering_keys), - is_recovering_(is_recovering), point_read_on_cache_miss_(point_read_on_cache_miss) { } @@ -253,7 +254,6 @@ struct ReadTxRequest bool read_local = false, uint64_t ts = 0, bool is_covering_keys = false, - bool is_recovering = false, bool point_read_on_cache_miss = false) { tab_name_ = tab_name; @@ -266,7 +266,6 @@ struct ReadTxRequest ts_ = ts; schema_version_ = schema_version; is_covering_keys_ = is_covering_keys; - is_recovering_ = is_recovering; point_read_on_cache_miss_ = point_read_on_cache_miss; } @@ -279,7 +278,6 @@ struct ReadTxRequest bool read_local = false, uint64_t ts = 0, bool is_covering_keys = false, - bool is_recovering = false, bool point_read_on_cache_miss = false) { tab_name_ = tab_name; @@ -292,7 +290,6 @@ struct ReadTxRequest ts_ = ts; schema_version_ = schema_version; is_covering_keys_ = is_covering_keys; - is_recovering_ = is_recovering; point_read_on_cache_miss_ = point_read_on_cache_miss; } @@ -333,11 +330,6 @@ struct ReadTxRequest // For unique_sk point query bool is_covering_keys_; - // If this is a read request for recovering. If true we should - // rely on candidate leader term instead of current term to decide - // if node is valid leader. - bool is_recovering_; - bool point_read_on_cache_miss_; }; diff --git a/tx_service/include/tx_util.h b/tx_service/include/tx_util.h index 5e599b8b..f2d51a93 100644 --- a/tx_service/include/tx_util.h +++ b/tx_service/include/tx_util.h @@ -66,7 +66,8 @@ static inline TransactionExecution *NewTxInit( int16_t group_id = -1, bool start_now = false, const std::function *yield_fptr = nullptr, - const std::function *resume_fptr = nullptr) + const std::function *resume_fptr = nullptr, + bool allow_run_on_candidate = false) { assert(tx_service != nullptr); txservice::TransactionExecution *txm = nullptr; @@ -75,7 +76,13 @@ static inline TransactionExecution *NewTxInit( #else txm = tx_service->NewTx(); #endif - txm->InitTx(level, proto, tx_owner, start_now, yield_fptr, resume_fptr); + txm->InitTx(level, + proto, + tx_owner, + start_now, + yield_fptr, + resume_fptr, + allow_run_on_candidate); return txm; } diff --git a/tx_service/src/cc/cc_req_misc.cpp b/tx_service/src/cc/cc_req_misc.cpp index ead420ac..eee369ca 100644 --- a/tx_service/src/cc/cc_req_misc.cpp +++ b/tx_service/src/cc/cc_req_misc.cpp @@ -886,6 +886,9 @@ bool FetchRecordCc::Execute(CcShard &ccs) { if (req) { + LOG(INFO) << "FetchRecordCc, abort request, tx:" << req->Txn() + << ", table name: " << table_name_.StringView() + << ", key: " << tx_key_.ToString(); req->AbortCcRequest(CcErrorCode::NG_TERM_CHANGED); } } @@ -947,6 +950,11 @@ bool FetchRecordCc::Execute(CcShard &ccs) cce_->GetKeyGapLockAndExtraData()->ReleasePin(); cce_->RecycleKeyLock(ccs); + LOG(INFO) + << "FetchRecordCc, abort request, tx:" << req->Txn() + << ", table name: " << table_name_.StringView() + << ", key: " << tx_key_.ToString() << ", error code: " + << static_cast(error_code_); req->AbortCcRequest(CcErrorCode::DATA_STORE_ERR); } } @@ -1168,8 +1176,36 @@ bool RunOnTxProcessorCc::Execute(CcShard &ccs) return true; } +bool UpdateCceCkptTsCc::ValidTermCheck() const +{ + int64_t ng_term = Sharder::Instance().LeaderTerm(node_group_id_); + int64_t candidate_ng_term = + Sharder::Instance().CandidateLeaderTerm(node_group_id_); + ng_term = std::max(ng_term, candidate_ng_term); + int64_t standby_node_term = Sharder::Instance().StandbyNodeTerm(); + int64_t current_term = std::max(ng_term, standby_node_term); + + if (current_term < 0 || current_term != term_) + { + LOG(INFO) + << "UpdateCceCkptTsCc::ValidTermCheck failed with current term: " + << current_term << ", term_: " << term_; + return false; + } + + return true; +} + bool UpdateCceCkptTsCc::Execute(CcShard &ccs) { + if (!ValidTermCheck()) + { + LOG(INFO) << "UpdateCceCkptTsCc::ValidTermCheck failed on shard: " + << ccs.core_id_; + SetFinished(); + return false; + } + assert(indices_.count(ccs.core_id_) > 0); auto &index = indices_[ccs.core_id_]; @@ -1183,16 +1219,6 @@ bool UpdateCceCkptTsCc::Execute(CcShard &ccs) return false; } - int64_t ng_leader_term = Sharder::Instance().LeaderTerm(node_group_id_); - int64_t standby_node_term = Sharder::Instance().StandbyNodeTerm(); - int64_t current_term = std::max(ng_leader_term, standby_node_term); - - if (current_term < 0 || current_term != term_) - { - SetFinished(); - return false; - } - size_t last_index = std::min(index + SCAN_BATCH_SIZE, records.size()); CcMap *ccm = ccs.GetCcm(table_name_, node_group_id_); @@ -1217,6 +1243,17 @@ bool UpdateCceCkptTsCc::Execute(CcShard &ccs) v_entry->SetCkptTs(ref.commit_ts_); v_entry->ClearBeingCkpt(); ccm->OnEntryFlushed(true, v_entry->IsPersistent()); + // if (ccs.core_id_ == 0) + // { + // LOG(INFO) << "SetCKptTs for versioned cce: 0x" << + // std::hex + // << (void *) (ref.cce_) + // << ", commit ts: " << std::dec << + // ref.commit_ts_ + // << ", post flush size: " << + // ref.post_flush_size_ + // << ", on shard: 0"; + // } } else { @@ -1228,6 +1265,14 @@ bool UpdateCceCkptTsCc::Execute(CcShard &ccs) v_entry->SetCkptTs(ref.commit_ts_); v_entry->ClearBeingCkpt(); ccm->OnEntryFlushed(true, v_entry->IsPersistent()); + if (ccs.core_id_ == 0) + { + LOG(INFO) << "SetCKptTs for non versioned cce: 0x" + << std::hex << (void *) (ref.cce_) + << ", commit ts: " << std::dec << ref.commit_ts_ + << ", post flush size: " << ref.post_flush_size_ + << ", on shard: 0"; + } } } else @@ -1476,6 +1521,13 @@ bool ShardCleanCc::Execute(CcShard &ccs) if (shard_heap->Full(&heap_alloc, &heap_commit) && shard_heap->NeedCleanShard(heap_alloc, heap_commit)) { + // LOG(INFO) << "ShardCleanCc::Execute still full after clean shard, + // " + // "on shard: " + // << ccs.core_id_ << ", heap committed: " << heap_commit + // << ", heap allocated: " << heap_alloc + // << ", need yield: " << std::boolalpha << need_yield + // << ", free count: " << free_count_; if (need_yield) { // Continue to clean in the next run one round. @@ -1507,6 +1559,13 @@ bool ShardCleanCc::Execute(CcShard &ccs) { // Get the free memory, re-run a batch of the waiting ccrequest. bool wait_list_empty = ccs.DequeueWaitListAfterMemoryFree(); + // LOG(INFO) + // << "ShardCleanCc::Execute after clean shard, wait_list_empty: + // " + // << std::boolalpha << wait_list_empty + // << ", heap committed: " << heap_commit + // << ", heap allocated: " << heap_alloc + // << ", on shard: " << ccs.core_id_; if (!wait_list_empty) { ccs.Enqueue(this); @@ -1523,6 +1582,13 @@ bool ShardCleanCc::Execute(CcShard &ccs) // waiting ccrequest if has any waiting request, otherwise, finish // this shard clean ccrequests. bool wait_list_empty = ccs.DequeueWaitListAfterMemoryFree(); + // LOG(INFO) + // << "ShardCleanCc::Execute no need to clean shard, + // wait_list_empty: " + // << std::boolalpha << wait_list_empty + // << ", heap committed: " << heap_commit + // << ", heap allocated: " << heap_alloc + // << ", on shard: " << ccs.core_id_; if (!wait_list_empty) { ccs.Enqueue(this); diff --git a/tx_service/src/cc/cc_shard.cpp b/tx_service/src/cc/cc_shard.cpp index c61cc0dc..d54bf0ab 100644 --- a/tx_service/src/cc/cc_shard.cpp +++ b/tx_service/src/cc/cc_shard.cpp @@ -1071,6 +1071,7 @@ void CcShard::CheckRecoverTx(TxNumber lock_holding_txn, if (Sharder::Instance().LeaderTerm(cc_ng_id) > 0) { + // TODO(ysw): check candidate leader term? LOG(WARNING) << "orphan lock detected, lock holding txn: " << lock_holding_txn << ", try to recover"; Sharder::Instance().RecoverTx(lock_holding_txn, @@ -1159,6 +1160,9 @@ std::pair CcShard::Clean() size_t free_cnt = 0; bool yield = false; + // LOG(INFO) << "Clean started: clean start page addr: 0x" << std::hex + // << (void *) clean_start_ccp_ << ", ccp: 0x" << (void *) ccp + // << std::dec << ", shard: " << core_id_; #ifndef RUNNING_TXSERVICE_CTEST size_t clean_page_cnt = 0, scan_page_cnt = 0; @@ -1202,6 +1206,39 @@ std::pair CcShard::Clean() } #endif clean_start_ccp_ = ccp; + // LOG(INFO) << "Clean finished: free_cnt: " << free_cnt + // << ", yield: " << std::boolalpha << yield + // << ", clean start page addr: 0x" << std::hex + // << (void *) clean_start_ccp_ << ", clean page cnt: " << + // std::dec + // << clean_page_cnt << ", scan page cnt: " << scan_page_cnt + // << ", shard: " << core_id_; + // if (core_id_ == 0 && free_cnt == 0 && !yield) + // { + // LOG(INFO) << "Printing cc pages on lru list on shard: 0"; + // LruPage *loop_ccp = head_ccp_.lru_next_; + // size_t ccp_cnt = 0; + // while (loop_ccp != &tail_ccp_) + // { + // loop_ccp->parent_map_->PrintCcPage(loop_ccp); + // loop_ccp = loop_ccp->lru_next_; + // ++ccp_cnt; + // } + // LOG(INFO) << "Printing cc pages on lru list on shard: " << core_id_ + // << " finished, ccpage count: " << ccp_cnt; + + // for (const auto &[table_name, ccm] : native_ccms_) + // { + // LOG(INFO) << "Table: " << table_name.StringView() + // << ", all records on this ccm: " << ccm->size(); + // if (!table_name.IsMeta()) + // { + // ccm->PrintAllCcPages(); + // } + // LOG(INFO) << "Printing cc pages on table: " + // << table_name.StringView() << " finished"; + // } + // } return {free_cnt, yield}; } diff --git a/tx_service/src/cc/local_cc_handler.cpp b/tx_service/src/cc/local_cc_handler.cpp index 9dd7962d..afb0897b 100644 --- a/tx_service/src/cc/local_cc_handler.cpp +++ b/tx_service/src/cc/local_cc_handler.cpp @@ -62,7 +62,8 @@ void txservice::LocalCcHandler::AcquireWrite( uint32_t hd_res_idx, CcProtocol proto, IsolationLevel iso_level, - bool abort_if_oom) + bool abort_if_oom, + bool allow_run_on_candidate) { uint32_t ng_id = Sharder::Instance().ShardToCcNodeGroup(key_shard_code); AcquireKeyResult &acquire_result = hres.Value()[hd_res_idx]; @@ -92,7 +93,8 @@ void txservice::LocalCcHandler::AcquireWrite( hd_res_idx, proto, iso_level, - abort_if_oom); + abort_if_oom, + allow_run_on_candidate); TX_TRACE_ACTION(this, req); TX_TRACE_DUMP(req); @@ -274,7 +276,8 @@ txservice::CcReqStatus txservice::LocalCcHandler::PostWrite( const TxRecord *record, OperationType operation_type, uint32_t key_shard_code, - CcHandlerResult &hres) + CcHandlerResult &hres, + bool allow_run_on_candidate) { uint32_t ng_id = cce_addr.NodeGroupId(); uint32_t dest_node_id = Sharder::Instance().LeaderNodeId(ng_id); @@ -293,7 +296,8 @@ txservice::CcReqStatus txservice::LocalCcHandler::PostWrite( record, operation_type, key_shard_code, - &hres); + &hres, + allow_run_on_candidate); TX_TRACE_ACTION(this, req); TX_TRACE_DUMP(req); cc_shards_.EnqueueCcRequest(thd_id_, cce_addr.CoreId(), req); @@ -328,7 +332,8 @@ txservice::CcReqStatus txservice::LocalCcHandler::PostRead( CcHandlerResult &hres, bool is_local, bool need_remote_resp, - PostReadType post_read_type) + PostReadType post_read_type, + bool allow_run_on_candidate) { if (IsStandbyTx(tx_term)) { @@ -369,7 +374,8 @@ txservice::CcReqStatus txservice::LocalCcHandler::PostRead( key_ts, gap_ts, post_read_type, - &hres); + &hres, + allow_run_on_candidate); TX_TRACE_ACTION(this, req); TX_TRACE_DUMP(req); if (thd_id_ == cce_addr.CoreId()) @@ -434,6 +440,7 @@ void txservice::LocalCcHandler::Read(const TableName &table_name, CcProtocol proto, bool is_for_write, bool is_covering_keys, + bool allow_run_on_candidate, bool point_read_on_miss, int32_t partition_id, bool abort_if_oom) @@ -469,13 +476,19 @@ void txservice::LocalCcHandler::Read(const TableName &table_name, is_for_write, is_covering_keys, nullptr, - false, + allow_run_on_candidate, point_read_on_miss, partition_id, abort_if_oom); TX_TRACE_ACTION(this, req); TX_TRACE_DUMP(req); cc_shards_.EnqueueCcRequest(thd_id_, key_shard_code, req); + // LOG(INFO) << "Read local request, txn: " << req->Txn() + // << ", table name: " << table_name.StringView() + // << ", table type: " + // << static_cast(table_name.Type()) + // << ", key: " << key.ToString() + // << ", partition id: " << partition_id; } else { @@ -498,6 +511,7 @@ void txservice::LocalCcHandler::Read(const TableName &table_name, proto, is_for_write, is_covering_keys, + allow_run_on_candidate, point_read_on_miss, partition_id, abort_if_oom); @@ -587,7 +601,7 @@ bool txservice::LocalCcHandler::ReadLocal(const TableName &table_name, IsolationLevel iso_level, CcProtocol proto, bool is_for_write, - bool is_recovering, + bool allow_run_on_candidate, bool execute_immediately) { ReadKeyResult &read_result = hres.Value(); @@ -609,14 +623,15 @@ bool txservice::LocalCcHandler::ReadLocal(const TableName &table_name, { ccs = cc_shards_.cc_shards_[thd_id_].get(); } - int64_t term; + int64_t term = -1; uint32_t shard_code = tx_number >> 32L; uint32_t cc_ng_id = shard_code >> 10; - if (is_recovering) + if (allow_run_on_candidate) { term = Sharder::Instance().CandidateLeaderTerm(cc_ng_id); } - else + + if (term < 0) { int64_t ng_leader_term = Sharder::Instance().LeaderTerm(cc_ng_id); int64_t standby_node_term = Sharder::Instance().StandbyNodeTerm(); @@ -632,6 +647,11 @@ bool txservice::LocalCcHandler::ReadLocal(const TableName &table_name, // shard to which the tx is bound, if the native cc node is not the // leader now, returns an error. hres.SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); + LOG(INFO) << "ReadLocal Execute not leader, table name: " + << table_name.StringView() + << ", allow_run_on_candidate: " << std::boolalpha + << allow_run_on_candidate << ", term: " << term + << ", txn: " << tx_number; return true; } @@ -659,7 +679,7 @@ bool txservice::LocalCcHandler::ReadLocal(const TableName &table_name, is_for_write, false, nullptr, - is_recovering); + allow_run_on_candidate); TX_TRACE_ACTION(this, read_req); TX_TRACE_DUMP(read_req); @@ -675,6 +695,13 @@ bool txservice::LocalCcHandler::ReadLocal(const TableName &table_name, { //__catalog table will be preloaded when ccshard constructed finished = ccm->Execute(*read_req); + // LOG(INFO) << "ReadLocal Execute, table name: " + // << table_name.StringView() + // << ", allow_run_on_candidate: " << std::boolalpha + // << allow_run_on_candidate << ", term: " << term + // << ", txn: " << tx_number << ", finished: " << + // std::boolalpha + // << finished; if (finished) { read_req->Free(); @@ -732,7 +759,7 @@ bool txservice::LocalCcHandler::ReadLocal(const TableName &table_name, IsolationLevel iso_level, CcProtocol proto, bool is_for_write, - bool is_recovering) + bool allow_run_on_candidate) { ReadKeyResult &read_result = hres.Value(); read_result.rec_ = &record; @@ -756,7 +783,7 @@ bool txservice::LocalCcHandler::ReadLocal(const TableName &table_name, int64_t term = -1; uint32_t shard_code = tx_number >> 32L; uint32_t cc_ng_id = shard_code >> 10; - if (is_recovering) + if (allow_run_on_candidate) { term = Sharder::Instance().CandidateLeaderTerm(cc_ng_id); } @@ -777,6 +804,11 @@ bool txservice::LocalCcHandler::ReadLocal(const TableName &table_name, // shard to which the tx is bound, if the native cc node is not the // leader now, returns an error. hres.SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); + // LOG(INFO) << "ReadLocal Execute, table name: " + // << table_name.StringView() + // << ", allow_run_on_candidate: " << std::boolalpha + // << allow_run_on_candidate << ", term: " << term + // << ", txn: " << tx_number; return true; } @@ -811,6 +843,13 @@ bool txservice::LocalCcHandler::ReadLocal(const TableName &table_name, { //__catalog table will be preloaded when ccshard constructed finished = ccm->Execute(*read_req); + // LOG(INFO) << "ReadLocal Execute, table name: " + // << table_name.StringView() + // << ", allow_run_on_candidate: " << std::boolalpha + // << allow_run_on_candidate << ", term: " << term + // << ", txn: " << tx_number << ", finished: " << + // std::boolalpha + // << finished; if (finished) { read_req->Free(); @@ -1379,11 +1418,17 @@ void txservice::LocalCcHandler::ScanClose(const TableName &table_name, void txservice::LocalCcHandler::NewTxn(CcHandlerResult &hres, IsolationLevel iso_level, NodeGroupId tx_ng_id, - uint32_t log_group_id) + uint32_t log_group_id, + bool allow_run_on_candidate) { CcShard &ccs = *(cc_shards_.cc_shards_[thd_id_]); int64_t term = Sharder::Instance().LeaderTerm(tx_ng_id); + if (term < 0 && allow_run_on_candidate) + { + term = Sharder::Instance().CandidateLeaderTerm(tx_ng_id); + } + if (term < 0) { term = Sharder::Instance().StandbyNodeTerm(); diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 2c1c9a76..b74eb12d 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -716,8 +716,7 @@ void LocalCcShards::CreateSchemaRecoveryTx( false, true, 0, - false, - true); + false); txm->Execute(&read_req); read_req.Wait(); if (read_req.IsError()) @@ -820,8 +819,7 @@ void LocalCcShards::CreateSplitRangeRecoveryTx( false, true, 0, - false, - true); + false); txm->Execute(&read_req); read_req.Wait(); // Only case we fail here is that leader gone before replay @@ -3201,7 +3199,9 @@ void LocalCcShards::PostProcessFlushTaskEntries( } if (!Sharder::Instance().CheckLeaderTerm( - task->node_group_id_, task->node_group_term_)) + task->node_group_id_, + task->node_group_term_, + true)) { err_code = CcErrorCode::REQUESTED_NODE_NOT_LEADER; } @@ -3251,7 +3251,9 @@ void LocalCcShards::PostProcessFlushTaskEntries( { if (!task->during_split_range_) { - range_entry->UpdateLastDataSyncTS(task->data_sync_ts_); + uint64_t last_sync_ts = + task->sync_on_leader_ ? task->data_sync_ts_ : 0; + range_entry->UpdateLastDataSyncTS(last_sync_ts); range_entry->UnPinStoreRange(); // Commit the data sync txm txservice::CommitTx(entry->data_sync_txm_); @@ -3287,7 +3289,7 @@ void LocalCcShards::PostProcessFlushTaskEntries( } if (!Sharder::Instance().CheckLeaderTerm( - task->node_group_id_, task->node_group_term_)) + task->node_group_id_, task->node_group_term_, true)) { err_code = CcErrorCode::REQUESTED_NODE_NOT_LEADER; } @@ -3390,7 +3392,7 @@ void LocalCcShards::PostProcessRangePartitionDataSyncTask( << task->table_name_.Trace() << ", retrying."; std::this_thread::sleep_for(1s); if (!Sharder::Instance().CheckLeaderTerm( - task->node_group_id_, task->node_group_term_)) + task->node_group_id_, task->node_group_term_, true)) { LOG(ERROR) << "Leader term changed during store slice update"; @@ -3406,7 +3408,9 @@ void LocalCcShards::PostProcessRangePartitionDataSyncTask( { if (!task->during_split_range_) { - range_entry->UpdateLastDataSyncTS(task->data_sync_ts_); + uint64_t last_sync_ts = + task->sync_on_leader_ ? task->data_sync_ts_ : 0; + range_entry->UpdateLastDataSyncTS(last_sync_ts); range_entry->UnPinStoreRange(); // Commit the data sync txm txservice::CommitTx(data_sync_txm); @@ -3461,8 +3465,8 @@ void LocalCcShards::PostProcessRangePartitionDataSyncTask( task->id_); } - if (!Sharder::Instance().CheckLeaderTerm(task->node_group_id_, - task->node_group_term_)) + if (!Sharder::Instance().CheckLeaderTerm( + task->node_group_id_, task->node_group_term_, true)) { err_code = CcErrorCode::REQUESTED_NODE_NOT_LEADER; } @@ -3627,6 +3631,7 @@ void LocalCcShards::DataSyncForRangePartition( // might miss the data that has not been recovered yet. data_sync_task->SetErrorCode( CcErrorCode::REQUESTED_NODE_NOT_LEADER); + data_sync_task->SetSyncOnLeader(false); } defer_unpin = std::shared_ptr( @@ -3638,7 +3643,12 @@ void LocalCcShards::DataSyncForRangePartition( data_sync_txm = txservice::NewTxInit(tx_service_, IsolationLevel::RepeatableRead, CcProtocol::Locking, - ng_id); + ng_id, + -1, + false, + nullptr, + nullptr, + true); if (data_sync_txm == nullptr) { @@ -3700,7 +3710,9 @@ void LocalCcShards::DataSyncForRangePartition( { LOG(ERROR) << "DataSync add read lock on table failed, " "table name: " - << table_key.Name().StringView(); + << table_key.Name().StringView() << ", error code: " + << static_cast(read_req.ErrorCode()) + << ", txn: " << data_sync_txm->TxNumber(); // If read lock acquire failed, retry next time. // Put back into the beginning. @@ -3791,6 +3803,12 @@ void LocalCcShards::DataSyncForRangePartition( return; } + // LOG(INFO) << "DataSyncForRangePartition acquired read lock on catalog + // " + // "and bucket: ng_term: " + // << ng_term << ", txn: " << data_sync_txm->TxNumber() + // << ", table: " << table_name.StringView(); + // Now that we have acquired read lock on catalog and bucket, there // won't be any ddl on this range. Update store_range and check if this // range is still owned by this node group. @@ -3838,6 +3856,15 @@ void LocalCcShards::DataSyncForRangePartition( assert(store_range); last_sync_ts = is_dirty ? 0 : range_entry->GetLastSyncTs(); + // if (table_name.StringView() == "./sbtest/history") + // { + // LOG(INFO) << "DataSync for range split, range id: " << range_id + // << ", last sync ts: " << last_sync_ts + // << ", table name: " << table_name.StringView() + // << ", start key: " << start_tx_key.ToString() + // << ", end key: " << end_tx_key.ToString() + // << ", txn: " << tx_number; + // } } if (table_name.IsBase()) @@ -3906,13 +3933,19 @@ void LocalCcShards::DataSyncForRangePartition( DLOG(INFO) << "No items need to be sync in this round data sync of range#" << range_id << " for table: " << table_name.StringView() - << ". Finish this task directly."; + << ". Finish this task directly." + << ", txn: " + << (!during_split_range ? data_sync_txm->TxNumber() : 0) + << ", during split range: " << std::boolalpha << during_split_range; if (!during_split_range) { txservice::CommitTx(data_sync_txm); // Update the task status for this range. - range_entry->UpdateLastDataSyncTS(data_sync_task->data_sync_ts_); + uint64_t last_sync_ts = data_sync_task->sync_on_leader_ + ? data_sync_task->data_sync_ts_ + : 0; + range_entry->UpdateLastDataSyncTS(last_sync_ts); // Generally, the StoreRange will be pinned only when there are data // items in the range that needs to be ckpted. if (scan_delta_size_cc.StoreRangePtr() != nullptr) @@ -3933,6 +3966,11 @@ void LocalCcShards::DataSyncForRangePartition( } data_sync_task->SetFinish(); data_sync_task->SetScanTaskFinished(); + // LOG(INFO) << "Finished this round data sync of range#" << range_id + // << " for table: " << table_name.StringView() + // << ", txn: " << data_sync_txm->TxNumber() + // << ", during split range: " << std::boolalpha + // << during_split_range; return; } assert(slices_delta_size.size() > 0 || export_base_table_items); @@ -3947,7 +3985,12 @@ void LocalCcShards::DataSyncForRangePartition( // Update slice post ckpt size. UpdateSlicePostCkptSize(store_range, slices_delta_size); - if (!during_split_range) + bool check_range_update = !during_split_range; + CODE_FAULT_INJECTOR("check_range_update_skip", { + DLOG(INFO) << "FaultInject check_range_update_skip"; + check_range_update = false; + }); + if (check_range_update) { // If the task comes from split range transaction, it is assumed that // there will be no further splitting. @@ -3992,6 +4035,14 @@ void LocalCcShards::DataSyncForRangePartition( } } + // if (table_name.StringView() == "./sbtest/history") + // { + // LOG(INFO) << "DataSync for range split, range id: " << range_id + // << ", table name: " << table_name.StringView() + // << ", with slice count with dirty keys: " + // << slices_delta_size.size() << ", txn: " << tx_number; + // } + // 3. Scan records. // The data sync worker thread is the owner of those vectors. @@ -4079,6 +4130,7 @@ void LocalCcShards::DataSyncForRangePartition( data_sync_task->flight_task_cnt_ += 1; } + size_t loop_count = 0; while (!scan_data_drained) { uint32_t core_rand = butil::fast_rand(); @@ -4088,6 +4140,7 @@ void LocalCcShards::DataSyncForRangePartition( EnqueueLowPriorityCcRequestToShard(core_rand % cc_shards_.size(), &scan_cc); scan_cc.Wait(); + ++loop_count; if (scan_cc.IsError()) { @@ -4180,6 +4233,16 @@ void LocalCcShards::DataSyncForRangePartition( GetCatalogFactory(table_name.Engine())->PositiveInfKey(); for (size_t i = 0; i < cc_shards_.size(); ++i) { + // if (table_name.StringView() == "./sbtest/history" && i == 0) + // { + // LOG(INFO) + // << "DataSync for range split, range id: " << range_id + // << ", table name: " << table_name.StringView() + // << ", with scan count: " + // << scan_cc.accumulated_scan_cnt_[i] + // << ", the loop count: " << loop_count + // << ", txn: " << tx_number; + // } for (size_t j = 0; j < scan_cc.accumulated_scan_cnt_[i]; ++j) { auto &rec = scan_cc.DataSyncVec(i)[j]; @@ -4431,6 +4494,12 @@ void LocalCcShards::DataSyncForRangePartition( req_vec.pop_back(); } + // LOG(INFO) << "DataSyncForRangePartition: worker: " << worker_idx + // << " finished scan." + // << ", during range split: " << std::boolalpha + // << during_split_range << ", txn: " + // << (data_sync_txm != nullptr ? data_sync_txm->TxNumber() : 0) + // << ", table: " << table_name.StringView(); PostProcessRangePartitionDataSyncTask(std::move(data_sync_task), data_sync_txm, DataSyncTask::CkptErrorCode::NO_ERROR, @@ -4468,7 +4537,7 @@ void LocalCcShards::PostProcessHashPartitionDataSyncTask( // Make sure that the term has not changed so that catalog entry // is still valid. if (!Sharder::Instance().CheckLeaderTerm( - task->node_group_id_, task->node_group_term_)) + task->node_group_id_, task->node_group_term_, true)) { err_code = CcErrorCode::NG_TERM_CHANGED; } @@ -4492,7 +4561,9 @@ void LocalCcShards::PostProcessHashPartitionDataSyncTask( // term has not changed. catalog entry should not be // nullptr. assert(catalog_entry); - catalog_entry->UpdateLastDataSyncTS(task->data_sync_ts_, + uint64_t last_sync_ts = + task->sync_on_leader_ ? task->data_sync_ts_ : 0; + catalog_entry->UpdateLastDataSyncTS(last_sync_ts, task->id_); } } @@ -4673,6 +4744,7 @@ void LocalCcShards::DataSyncForHashPartition( // might miss the data that has not been recovered yet. data_sync_task->SetErrorCode( CcErrorCode::REQUESTED_NODE_NOT_LEADER); + data_sync_task->SetSyncOnLeader(false); } } @@ -4698,7 +4770,12 @@ void LocalCcShards::DataSyncForHashPartition( txservice::NewTxInit(tx_service_, IsolationLevel::RepeatableRead, CcProtocol::Locking, - ng_id); + ng_id, + -1, + false, + nullptr, + nullptr, + true); if (data_sync_txm == nullptr) { @@ -4790,6 +4867,10 @@ void LocalCcShards::DataSyncForHashPartition( } } + // LOG(INFO) << "DataSyncForHashPartition: ng_term: " << ng_term + // << ", txn: " << data_sync_txm->TxNumber() + // << ", table: " << table_name.StringView(); + // 3. Scan records. { std::lock_guard flight_task_lk( @@ -5451,6 +5532,19 @@ void LocalCcShards::UpdateSlices(const TableName &table_name, paused_subslice_post_ckpt_size < avg_subslice_size ? paused_subslice_post_ckpt_size : 0; + // For very large slices that require multiple rounds of data sync + // scan, each round flushes exported keys to storage and updates the + // checkpoint timestamp, allowing data to be evicted from memory. + // When a scan round finishes, the pinned slice is unpinned. In + // subsequent rounds, the slice is re-pinned, triggering a load + // operation that updates `StoreSlice::size_` with the flushed keys' + // size. This can cause a mismatch with the slice size from previous + // batches. + if (slice_split_keys.size() > 0 && + slice_split_keys.back().cur_size_ != subslice_size) + { + subslice_size = slice_split_keys.back().cur_size_; + } } else { @@ -5613,8 +5707,11 @@ void LocalCcShards::SplitFlushRange( // by data store are always unique. std::vector> new_range_ids; int32_t new_part_id; - if (!GetNextRangePartitionId( - table_name, table_schema.get(), split_keys.size(), new_part_id)) + if (!GetNextRangePartitionId(table_name, + table_schema.get(), + node_group, + split_keys.size(), + new_part_id)) { LOG(ERROR) << "Split range failed due to unable to get next " "partition id. table_name = " @@ -5691,7 +5788,9 @@ void LocalCcShards::SplitFlushRange( return; } - range_entry->UpdateLastDataSyncTS(data_sync_task->data_sync_ts_); + uint64_t last_sync_ts = + data_sync_task->sync_on_leader_ ? data_sync_task->data_sync_ts_ : 0; + range_entry->UpdateLastDataSyncTS(last_sync_ts); range_entry->UnPinStoreRange(); data_sync_task->SetFinish(); @@ -5906,6 +6005,8 @@ void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk) EnqueueToCcShard(core_idx, &reset_cc); } reset_cc.Wait(); + // LOG(INFO) << "FlushData after set dirty data flushed" + // << ", the shard count: " << updated_ckpt_ts_core_ids.size(); auto ckpt_err = succ ? DataSyncTask::CkptErrorCode::NO_ERROR : DataSyncTask::CkptErrorCode::FLUSH_ERROR; @@ -6683,6 +6784,7 @@ void LocalCcShards::ReportHashPartitionCkptHeapUsage() bool LocalCcShards::GetNextRangePartitionId(const TableName &tablename, const TableSchema *table_schema, + NodeGroupId ng_id, uint32_t range_cnt, int32_t &out_next_partition_id) { @@ -6698,8 +6800,12 @@ bool LocalCcShards::GetNextRangePartitionId(const TableName &tablename, } int64_t first_reserved_id = -1; - bool res = Sequences::ApplyIdOfTableRangePartition( - tablename, range_cnt, first_reserved_id, reserved_cnt, key_schema_ts); + bool res = Sequences::ApplyIdOfTableRangePartition(tablename, + ng_id, + range_cnt, + first_reserved_id, + reserved_cnt, + key_schema_ts); if (!res) { diff --git a/tx_service/src/checkpointer.cpp b/tx_service/src/checkpointer.cpp index 7de601aa..6b82acd3 100644 --- a/tx_service/src/checkpointer.cpp +++ b/tx_service/src/checkpointer.cpp @@ -236,11 +236,16 @@ void Checkpointer::Ckpt(bool is_last_ckpt) std::unordered_map tables = local_shards_.GetCatalogTableNameSnapshot(node_group, ckpt_ts); + bool truncate_log = true; + CODE_FAULT_INJECTOR("datasync_status_no_truncate_log", { + DLOG(INFO) << "FaultInject datasync_status_no_truncate_log"; + truncate_log = false; + }); std::shared_ptr status = std::make_shared( node_group, is_standby_node ? standby_node_term : leader_term, - true); + truncate_log); uint64_t last_succ_ckpt_ts = UINT64_MAX; bool can_be_skipped = !is_last_ckpt; @@ -251,8 +256,8 @@ void Checkpointer::Ckpt(bool is_last_ckpt) for (auto it = tables.begin(); it != tables.end(); ++it) { // Check leader term for leader node - if (!is_standby_node && - Sharder::Instance().LeaderTerm(node_group) != leader_term) + if (!is_standby_node && !Sharder::Instance().CheckLeaderTerm( + node_group, leader_term, true)) { break; } @@ -307,7 +312,7 @@ void Checkpointer::Ckpt(bool is_last_ckpt) // Check leadter term for leader node if (!is_standby_node && - Sharder::Instance().LeaderTerm(node_group) != leader_term) + !Sharder::Instance().CheckLeaderTerm(node_group, leader_term, true)) { continue; } @@ -470,6 +475,7 @@ void Checkpointer::Run() */ void Checkpointer::Notify(bool request_ckpt) { + ACTION_FAULT_INJECTOR("panic_notify_checkpointer"); std::unique_lock lk(ckpt_mux_); if (request_ckpt) { diff --git a/tx_service/src/data_sync_task.cpp b/tx_service/src/data_sync_task.cpp index d0028412..c9a00822 100644 --- a/tx_service/src/data_sync_task.cpp +++ b/tx_service/src/data_sync_task.cpp @@ -79,7 +79,8 @@ DataSyncTask::DataSyncTask(const TableName &table_name, range_entry_(range_entry), during_split_range_(true), export_base_table_items_(export_base_table_items), - tx_number_(txn) + tx_number_(txn), + sync_on_leader_(true) { assert(!table_name_.IsHashPartitioned()); if (start_key_.KeyPtr() == diff --git a/tx_service/src/remote/remote_cc_handler.cpp b/tx_service/src/remote/remote_cc_handler.cpp index 848ae8f7..075707c8 100644 --- a/tx_service/src/remote/remote_cc_handler.cpp +++ b/tx_service/src/remote/remote_cc_handler.cpp @@ -326,6 +326,7 @@ void txservice::remote::RemoteCcHandler::Read( CcProtocol proto, bool is_for_write, bool is_covering_keys, + bool allow_run_on_candidate, bool point_read_on_miss, int32_t partition_id, bool abort_if_oom) @@ -381,6 +382,7 @@ void txservice::remote::RemoteCcHandler::Read( read->set_ts(ts); read->set_schema_version(schema_version); read->set_abort_if_oom(abort_if_oom); + read->set_allow_run_on_candidate(allow_run_on_candidate); stream_sender_.SendMessageToNg(dest_ng_id, send_msg, &hres); } diff --git a/tx_service/src/remote/remote_cc_request.cpp b/tx_service/src/remote/remote_cc_request.cpp index 32fbb935..dfa95b25 100644 --- a/tx_service/src/remote/remote_cc_request.cpp +++ b/tx_service/src/remote/remote_cc_request.cpp @@ -462,6 +462,7 @@ void txservice::remote::RemoteRead::Reset(std::unique_ptr input_msg) req.is_for_write(), req.is_covering_keys(), nullptr, + req.allow_run_on_candidate(), req.point_read_on_miss(), req.partition_id(), req.abort_if_oom()); diff --git a/tx_service/src/sequences/sequences.cpp b/tx_service/src/sequences/sequences.cpp index 3154bcb4..cee11d4e 100644 --- a/tx_service/src/sequences/sequences.cpp +++ b/tx_service/src/sequences/sequences.cpp @@ -74,7 +74,6 @@ bool Sequences::DeleteSequenceInternal(const std::string &seq_name, 0, false, false, - false, nullptr, nullptr, txm); @@ -276,7 +275,6 @@ int Sequences::ApplySequenceBatch( 0, false, false, - false, coro_functors.first, coro_functors.second, txm); @@ -303,7 +301,6 @@ int Sequences::ApplySequenceBatch( false, 0, false, - false, true, coro_functors.first, coro_functors.second, @@ -472,7 +469,6 @@ int Sequences::UpdateAutoIncrement( false, 0, false, - false, true, coro_functors.first, coro_functors.second, @@ -548,6 +544,7 @@ int Sequences::UpdateAutoIncrement( } bool Sequences::ApplyIdOfTableRangePartition(const TableName &table, + NodeGroupId ng_id, int64_t desired_vals, int64_t &first_reserved_id, int64_t &reserved_vals, @@ -558,7 +555,13 @@ bool Sequences::ApplyIdOfTableRangePartition(const TableName &table, TransactionExecution *txm = NewTxInit(instance_->tx_service_, txservice::IsolationLevel::RepeatableRead, - txservice::CcProtocol::Locking); + txservice::CcProtocol::Locking, + ng_id, + -1, + false, + nullptr, + nullptr, + true); if (txm == nullptr) { @@ -578,7 +581,6 @@ bool Sequences::ApplyIdOfTableRangePartition(const TableName &table, 0, false, false, - false, nullptr, nullptr, txm); @@ -595,6 +597,10 @@ bool Sequences::ApplyIdOfTableRangePartition(const TableName &table, TxKey mkey = GenKey(seq_name); // std::unique_ptr mrec = std::make_unique(); std::unique_ptr mrec = GenRecord(); + LOG(INFO) << "ApplyIdOfTableRangePartition: mkey: " << mkey.ToString() + << ", table_name: " << table_name_.StringView() + << ", seq_schema_version: " << seq_schema_version_ + << ", txn: " << txm->TxNumber(); // uint64_t schema_version = Sequences::GetTableSchema()->Version(); ReadTxRequest read_req(&table_name_, seq_schema_version_, @@ -605,7 +611,6 @@ bool Sequences::ApplyIdOfTableRangePartition(const TableName &table, false, 0, false, - false, true, nullptr, nullptr, @@ -613,6 +618,9 @@ bool Sequences::ApplyIdOfTableRangePartition(const TableName &table, txm->Execute(&read_req); read_req.Wait(); + LOG(INFO) << "ApplyIdOfTableRangePartition: done mkey: " << mkey.ToString() + << ", error code: " << static_cast(read_req.ErrorCode()) + << ", txn: " << txm->TxNumber(); if (read_req.IsError()) { @@ -661,10 +669,16 @@ bool Sequences::ApplyIdOfTableRangePartition(const TableName &table, if (err != TxErrorCode::NO_ERROR) { txservice::AbortTx(txm); + LOG(INFO) << "ApplyIdOfTableRangePartition: aborted txn: " + << txm->TxNumber(); return false; } auto [success, commit_err] = txservice::CommitTx(txm); + LOG(INFO) << "ApplyIdOfTableRangePartition: committed txn: " + << txm->TxNumber() + << ", error code: " << static_cast(commit_err) + << ", success: " << std::boolalpha << success; assert(success); if (success) { @@ -709,7 +723,6 @@ bool Sequences::InitIdOfTableRangePartition(const TableName &table, 0, false, false, - false, nullptr, nullptr, txm); @@ -736,7 +749,6 @@ bool Sequences::InitIdOfTableRangePartition(const TableName &table, false, 0, false, - false, true, nullptr, nullptr, @@ -798,6 +810,10 @@ bool Sequences::InitIdOfTableRangePartition(const TableName &table, txservice::AbortTx(txm, nullptr, nullptr); return false; } + LOG(INFO) << "InitIdOfTableRangePartition: done mkey: " << mkey.ToString() + << ", error code: " << static_cast(err) + << ", txn: " << txm->TxNumber() + << ", last_range_partition_id: " << last_range_partition_id; auto [success, commit_err] = txservice::CommitTx(txm, nullptr, nullptr); assert(success); @@ -829,7 +845,6 @@ bool Sequences::InitIdOfAutoIncrementColumn(const TableName &table_name) 0, false, false, - false, nullptr, nullptr, txm); diff --git a/tx_service/src/sharder.cpp b/tx_service/src/sharder.cpp index b1e5df45..998b9b33 100644 --- a/tx_service/src/sharder.cpp +++ b/tx_service/src/sharder.cpp @@ -614,9 +614,15 @@ std::shared_ptr Sharder::UpdateCcNodeServiceChannel( } } -bool Sharder::CheckLeaderTerm(uint32_t ng_id, int64_t term) const +bool Sharder::CheckLeaderTerm(uint32_t ng_id, + int64_t term, + bool check_candidate) const { int64_t node_term = LeaderTerm(ng_id); + if (node_term < 0 && check_candidate) + { + node_term = CandidateLeaderTerm(ng_id); + } if (node_term < 0) { diff --git a/tx_service/src/tx_execution.cpp b/tx_service/src/tx_execution.cpp index 24553f56..24e7917f 100644 --- a/tx_service/src/tx_execution.cpp +++ b/tx_service/src/tx_execution.cpp @@ -179,6 +179,7 @@ void TransactionExecution::Reset() tx_req_queue_.Reset(); req_queue_lock_.Unlock(); } + allow_run_on_candidate_ = false; ClearCachedBucketInfos(); lock_cluster_config_op_.Reset(); @@ -244,6 +245,7 @@ void TransactionExecution::SetRecoverTxState(uint64_t txn, tx_number_.store(txn, std::memory_order_relaxed); tx_term_ = tx_term; commit_ts_ = commit_ts; + allow_run_on_candidate_ = true; tx_status_.store(TxnStatus::Recovering, std::memory_order_relaxed); } @@ -551,12 +553,19 @@ void TransactionExecution::InitTx(IsolationLevel iso_level, NodeGroupId tx_ng_id, bool start_now, const std::function *yield_func, - const std::function *resume_func) + const std::function *resume_func, + bool allow_run_on_candidate) { if (start_now) { - InitTxRequest init_tx_req{ - iso_level, protocol, yield_func, resume_func, this, tx_ng_id}; + InitTxRequest init_tx_req{iso_level, + protocol, + yield_func, + resume_func, + this, + tx_ng_id, + UINT32_MAX, + allow_run_on_candidate}; Execute(&init_tx_req); init_tx_req.Wait(); } @@ -567,6 +576,7 @@ void TransactionExecution::InitTx(IsolationLevel iso_level, init_tx_req_->protocol_ = protocol; init_tx_req_->tx_ng_id_ = tx_ng_id; init_tx_req_->txm_ = this; + init_tx_req_->allow_run_on_candidate_ = allow_run_on_candidate; Execute(init_tx_req_.get()); } } @@ -735,6 +745,7 @@ void TransactionExecution::ProcessTxRequest(InitTxRequest &init_txn_req) : init_txn_req.tx_ng_id_; init_txn_.log_group_id_ = init_txn_req.log_group_id_; + allow_run_on_candidate_ = init_txn_req.allow_run_on_candidate_; PushOperation(&init_txn_); Process(init_txn_); } @@ -1743,7 +1754,8 @@ void TransactionExecution::Process(InitTxnOperation &init_txn) cc_handler_->NewTxn(init_txn.hd_result_, iso_level_, init_txn.tx_ng_id_, - init_txn.log_group_id_); + init_txn.log_group_id_, + allow_run_on_candidate_); } void TransactionExecution::PostProcess(InitTxnOperation &init_txn) @@ -1866,7 +1878,7 @@ void TransactionExecution::Process(ReadOperation &read) read.iso_level_, read.protocol_, read.read_tx_req_->is_for_write_, - read.read_tx_req_->is_recovering_); + allow_run_on_candidate_); } else { @@ -1883,7 +1895,7 @@ void TransactionExecution::Process(ReadOperation &read) read.iso_level_, read.protocol_, read.read_tx_req_->is_for_write_, - read.read_tx_req_->is_recovering_); + allow_run_on_candidate_); } if (finished) @@ -1970,6 +1982,12 @@ void TransactionExecution::Process(ReadOperation &read) range_rec_.GetRangeOwnerNg()->BucketOwner(); key_shard_code = range_ng << 10 | residual; partition_id = range_rec_.GetRangeInfo()->PartitionId(); + // LOG(INFO) << "ReadOperation for range partitioned table: + // " + // << table_name.StringView() + // << ", key: " << key.ToString() + // << ", partition id: " << partition_id + // << ", txn: " << tx_number_; } } else @@ -1991,14 +2009,27 @@ void TransactionExecution::Process(ReadOperation &read) partition_id = Sharder::MapKeyHashToHashPartitionId(key_hash); if (bucket_info != nullptr) { + // LOG(INFO) << "ReadOperation for hash partitioned table: " + // << table_name.StringView() + // << ", key: " << key.ToString() + // << ", get bucket info fast success" + // << ", read is running: " << std::boolalpha + // << read.is_running_; // Uses the lower 10 bits of the key's hash code to shard // the key across CPU cores in a cc node. uint32_t residual = key_hash & 0x3FF; NodeGroupId bucket_ng = bucket_info->BucketOwner(); key_shard_code = bucket_ng << 10 | residual; + + // Set the lock range bucket result + lock_range_bucket_result_.SetFinished(); } else if (!lock_range_bucket_result_.IsFinished()) { + // LOG(INFO) + // << "ReadOperation for hash partitioned table: " + // << table_name.StringView() + // << ", key: " << key.ToString() << ", to lock bucket"; read.is_running_ = false; // First read and lock the bucket the key located in through // lock_bucket_op_. @@ -2023,6 +2054,10 @@ void TransactionExecution::Process(ReadOperation &read) } else // lock bucket finished and succeeded { + // LOG(INFO) << "ReadOperation for hash partitioned table: " + // << table_name.StringView() + // << ", key: " << key.ToString() + // << ", lock bucket finished and succeeded"; // If there is an error when getting the key's bucket info, // the error would be caught when forwarding the read // operation, which forces the tx state machine moves to @@ -2060,6 +2095,15 @@ void TransactionExecution::Process(ReadOperation &read) { read_ts = ts; } + // LOG(INFO) << "ReadOperation for table: " << + // table_name.StringView() + // << ", key: " << key.ToString() << ", read ts: " << + // read_ts + // << ", read is running: " << std::boolalpha + // << read.is_running_ << ", txn: " << tx_number_ + // << ", is hash partitioned: " << std::boolalpha + // << table_name.IsHashPartitioned() + // << ", partition id: " << partition_id; cc_handler_->Read(table_name, read.read_tx_req_->schema_version_, @@ -2076,6 +2120,7 @@ void TransactionExecution::Process(ReadOperation &read) read.protocol_, read.read_tx_req_->is_for_write_, is_covering_keys, + allow_run_on_candidate_, read.read_tx_req_->point_read_on_cache_miss_, partition_id, HoldingRangeReadLock()); @@ -2111,6 +2156,17 @@ void TransactionExecution::Process(ReadOperation &read) StartTiming(); } } + // LOG(INFO) << "ReadOperation sent read request for table: " + // << table_name.StringView() + // << ", key: " << read.read_tx_req_->key_->ToString() + // << ", txn: " << tx_number_ + // << ", read is running: " << std::boolalpha + // << read.is_running_ << ", with result: 0x" << std::hex + // << &read.hd_result_ << ". operation: 0x" << std::hex + // << &read << ", op lock range bucket result is finished: + // " + // << std::boolalpha + // << read.lock_range_bucket_result_->IsFinished(); } } else @@ -2309,7 +2365,7 @@ void TransactionExecution::Process(ReadLocalOperation &lock_local) IsolationLevel::RepeatableRead, CcProtocol::Locking, false, - false, + allow_run_on_candidate_, lock_local.execute_immediately_); if (finished) { @@ -3712,6 +3768,7 @@ void TransactionExecution::Commit() { if (tx_term_ < 0) { + // LOG(INFO) << ">>>>Commit tx " << TxNumber() << ", tx_term < 0"; bool_resp_->Finish(false); bool_resp_ = nullptr; @@ -3760,6 +3817,8 @@ void TransactionExecution::Commit() } else { + // LOG(INFO) << ">>>>Commit tx " << TxNumber() << ", push set ts + // op"; PushOperation(&set_ts_); } } @@ -4047,7 +4106,8 @@ void TransactionExecution::Process(AcquireWriteOperation &acquire_write) res_idx++, protocol_, iso_level_, - abort_if_oom); + abort_if_oom, + allow_run_on_candidate_); for (auto &[forward_shard_code, cce_addr] : write_entry.forward_addr_) { @@ -4065,7 +4125,8 @@ void TransactionExecution::Process(AcquireWriteOperation &acquire_write) res_idx++, protocol_, iso_level_, - abort_if_oom); + abort_if_oom, + allow_run_on_candidate_); } } } @@ -4128,6 +4189,10 @@ void TransactionExecution::PostProcess(AcquireWriteOperation &acquire_write) } else if (acquire_write.hd_result_.IsError()) { + DLOG(ERROR) << "AcquireWriteOperation failed for cc error:" + << acquire_write.hd_result_.ErrorMsg() << " ,error code: " + << static_cast(acquire_write.hd_result_.ErrorCode()) + << "; txn: " << TxNumber(); bool_resp_->SetErrorCode( ConvertCcError(acquire_write.hd_result_.ErrorCode())); Abort(); @@ -4358,7 +4423,11 @@ void TransactionExecution::Process(ValidateOperation &validate) 0, commit_ts_, cce_addr, - validate.hd_result_); + validate.hd_result_, + false, + true, + PostReadType::Release, + allow_run_on_candidate_); if (ret == CcReqStatus::SentLocal) { @@ -5297,7 +5366,8 @@ void TransactionExecution::Process(PostProcessOp &post_process) write_entry.rec_.get(), write_entry.op_, write_entry.key_shard_code_, - post_process.hd_result_); + post_process.hd_result_, + allow_run_on_candidate_); update_post_cnt(ret); for (auto &[forward_shard_code, cce_addr] : @@ -5312,7 +5382,8 @@ void TransactionExecution::Process(PostProcessOp &post_process) write_entry.rec_.get(), write_entry.op_, forward_shard_code, - post_process.hd_result_); + post_process.hd_result_, + allow_run_on_candidate_); update_post_cnt(ret); } } @@ -5336,7 +5407,8 @@ void TransactionExecution::Process(PostProcessOp &post_process) nullptr, OperationType::CommitCommands, 0, - post_process.hd_result_); + post_process.hd_result_, + allow_run_on_candidate_); update_post_cnt(ret); if (cmd_set_entry.forward_entry_ != nullptr && @@ -5388,7 +5460,8 @@ void TransactionExecution::Process(PostProcessOp &post_process) nullptr, write_entry.op_, write_entry.key_shard_code_, - post_process.hd_result_); + post_process.hd_result_, + allow_run_on_candidate_); update_post_cnt(ret); } // Keys that were not successfully locked in the cc @@ -5409,7 +5482,8 @@ void TransactionExecution::Process(PostProcessOp &post_process) nullptr, write_entry.op_, forward_shard_code, - post_process.hd_result_); + post_process.hd_result_, + allow_run_on_candidate_); update_post_cnt(ret); } } @@ -5435,7 +5509,8 @@ void TransactionExecution::Process(PostProcessOp &post_process) nullptr, OperationType::CommitCommands, 0, - post_process.hd_result_); + post_process.hd_result_, + allow_run_on_candidate_); update_post_cnt(ret); if (cmd_set_entry.forward_entry_ != nullptr && @@ -5450,7 +5525,8 @@ void TransactionExecution::Process(PostProcessOp &post_process) nullptr, OperationType::CommitCommands, 0, - post_process.hd_result_); + post_process.hd_result_, + allow_run_on_candidate_); update_post_cnt(ret); } } @@ -5473,7 +5549,8 @@ void TransactionExecution::Process(PostProcessOp &post_process) 0, 0, cce_addr, - post_process.hd_result_); + post_process.hd_result_, + allow_run_on_candidate_); update_post_cnt(ret); } } @@ -5749,6 +5826,7 @@ void TransactionExecution::ReleaseMetaDataReadLock( size_t post_local_cnt = 0; for (const auto &[cce_addr, read_entry_pair] : rset) { + // LOG(INFO) << ">>>>ReleaseMetaDataReadLock, tx " << TxNumber(); const ReadSetEntry &read_entry = read_entry_pair.first; CcReqStatus ret = cc_handler_->PostRead(TxNumber(), TxTerm(), @@ -5758,7 +5836,10 @@ void TransactionExecution::ReleaseMetaDataReadLock( commit_ts_, cce_addr, meta_data_hd_result, - true); + true, + true, + PostReadType::Release, + allow_run_on_candidate_); --ref_cnt; if (ret == CcReqStatus::SentLocal) @@ -6342,7 +6423,11 @@ void TransactionExecution::Process(PostReadOperation &post_read_operation) 0, commit_ts_, *post_read_operation.cce_addr_, - post_read_operation.hd_result_); + post_read_operation.hd_result_, + false, + true, + PostReadType::Release, + allow_run_on_candidate_); if (ret == CcReqStatus::Processed) { AdvanceCommand(); @@ -7415,7 +7500,8 @@ void TransactionExecution::Process(CmdForwardAcquireWriteOp &forward_acquire) res_idx++, protocol_, iso_level_, - abort_if_oom); + abort_if_oom, + allow_run_on_candidate_); } } @@ -7740,6 +7826,17 @@ void TransactionExecution::Process(BatchReadOperation &batch_read_op) { partition_id = batch_read_op.range_ids_[idx]; } + // LOG(INFO) << "BatchReadOperation, txn: " << TxNumber() + // << ", table name: " << table_name.StringView() + // << ", table type: " + // << static_cast(table_name.Type()) + // << ", is hash partitioned: " << std::boolalpha + // << table_name.IsHashPartitioned() + // << ", batch read op range ids size: " + // << batch_read_op.range_ids_.size() + // << ", key: " << key.ToString() + // << ", partition id: " << partition_id << ", idx: " << idx; + cc_handler_->Read( table_name, batch_read_op.batch_read_tx_req_->schema_version_, @@ -7757,6 +7854,7 @@ void TransactionExecution::Process(BatchReadOperation &batch_read_op) protocol_, batch_read_op.batch_read_tx_req_->is_for_write_, false, // is_covering_keys + allow_run_on_candidate_, batch_read_op.batch_read_tx_req_->point_read_on_cache_miss_, partition_id, abort_if_oom); diff --git a/tx_service/src/tx_operation.cpp b/tx_service/src/tx_operation.cpp index 663d50c1..2d6ae7d1 100644 --- a/tx_service/src/tx_operation.cpp +++ b/tx_service/src/tx_operation.cpp @@ -144,6 +144,20 @@ void ReadOperation::Forward(TransactionExecution *txm) if (!read_tx_req_->read_local_) { // Just returned from lock_bucket_op_, check lock_bucket_result_. + // LOG(INFO) << "ReadOperation Forward, lock_range_bucket_result_ is + // " + // "not finished" + // << ", tx allow_run_on_candidate: " << std::boolalpha + // << txm->allow_run_on_candidate_ << ", table_name: " + // << read_tx_req_->tab_name_->StringView() + // << ", read key: " << read_tx_req_->key_->ToString() + // << ", txn: " << txm->TxNumber() + // << ", lock range bucket result: " << std::boolalpha + // << lock_range_bucket_result_->IsFinished() + // << ", error code: " + // << static_cast( + // lock_range_bucket_result_->ErrorCode()) + // << ", this: 0x" << std::hex << this; assert(lock_range_bucket_result_->IsFinished()); if (lock_range_bucket_result_->IsError()) { @@ -167,6 +181,11 @@ void ReadOperation::Forward(TransactionExecution *txm) txm->PostProcess(*this); return; } + // LOG(INFO) << "ReadOperation Forward, this: 0x" << std::hex << this + // << ", table name: " << + // read_tx_req_->tab_name_->StringView() + // << ", read key: " << read_tx_req_->key_->ToString() + // << ", txn: " << std::dec << txm->TxNumber(); txm->Process(*this); return; @@ -189,11 +208,26 @@ void ReadOperation::Forward(TransactionExecution *txm) } if (retry_num_ > 0) { + // LOG(INFO) << "ReadOperation ReRunOp, this: 0x" << std::hex + // << this << ", table name: " + // << read_tx_req_->tab_name_->StringView() + // << ", read key: " << read_tx_req_->key_->ToString() + // << ", txn: " << std::dec << txm->TxNumber() + // << ", hd result error code: " + // << static_cast(hd_result_.ErrorCode()) + // << ", retry num: " << retry_num_; hd_result_.Value().Reset(); hd_result_.Reset(); ReRunOp(txm); return; } + // LOG(INFO) << "ReadOperation topostprocess, this: 0x" << std::hex + // << this << ", table name: " + // << read_tx_req_->tab_name_->StringView() + // << ", read key: " << read_tx_req_->key_->ToString() + // << ", txn: " << std::dec << txm->TxNumber() + // << ", hd result error code: " + // << static_cast(hd_result_.ErrorCode()); } txm->PostProcess(*this); @@ -217,6 +251,11 @@ void ReadOperation::Forward(TransactionExecution *txm) { if (!hd_result_.Value().is_local_) { + // LOG(INFO) << "ReadOperation timeout, tx:" << txm->TxNumber() + // << ", table name: " + // << read_tx_req_->tab_name_->StringView() + // << ", read key: " << + // read_tx_req_->key_->ToString(); if (!txm->CheckLeaderTerm()) { hd_result_.SetError(CcErrorCode::TX_NODE_NOT_LEADER); @@ -226,6 +265,11 @@ void ReadOperation::Forward(TransactionExecution *txm) if (cce_addr.Term() > 0) { + // LOG(INFO) + // << "ReadOperation timeout, ack received, tx:" + // << txm->TxNumber() << ", table name: " + // << read_tx_req_->tab_name_->StringView() + // << ", read key: " << read_tx_req_->key_->ToString(); // ack is received, but timeout happens. // Trigger deadlock check. @@ -254,6 +298,11 @@ void ReadOperation::Forward(TransactionExecution *txm) // FIXME(lzx): Is it more appropriate to retry? // If the tx node fails, also force the tx to abort // instantly. + // LOG(INFO) + // << "ReadOperation timeout, no ack received, tx:" + // << txm->TxNumber() << ", table name: " + // << read_tx_req_->tab_name_->StringView() + // << ", read key: " << read_tx_req_->key_->ToString(); bool force_error = hd_result_.ForceError(); if (force_error) { @@ -265,6 +314,11 @@ void ReadOperation::Forward(TransactionExecution *txm) // execution. The tx will be re-executed when the tx // processor visits it in the execution queue. } + // LOG(INFO) << "ReadOperation timeout, force error, tx:" + // << txm->TxNumber() << ", table name: " + // << read_tx_req_->tab_name_->StringView() + // << ", read key: " << + // read_tx_req_->key_->ToString(); } else { @@ -298,6 +352,9 @@ void ReadLocalOperation::Forward(txservice::TransactionExecution *txm) hd_result_->SetError(CcErrorCode::TX_NODE_NOT_LEADER); hd_result_->ForceError(); txm->PostProcess(*this); + // LOG(INFO) << "ReadLocalOperation error for table: " + // << table_name_.StringView() + // << ", key: " << key_->ToString(); } else if (hd_result_->IsError()) { @@ -343,6 +400,11 @@ void ReadLocalOperation::Forward(txservice::TransactionExecution *txm) { // Read local succeeded txm->PostProcess(*this); + // LOG(INFO) << "ReadLocalOperation succeeded for table: " + // << table_name_.StringView() << ", table type: " + // << static_cast(table_name_.Type()) + // << ", txn: " << txm->TxNumber() + // << ", key: " << key_->ToString(); } } else @@ -4221,7 +4283,9 @@ void SplitFlushRangeOp::Forward(TransactionExecution *txm) if (lock_cluster_config_op_.hd_result_->IsError()) { LOG(ERROR) << "Split Flush read cluster config failed, tx_number:" - << txm->TxNumber(); + << txm->TxNumber() << ", error code: " + << static_cast( + lock_cluster_config_op_.hd_result_->ErrorCode()); ForceToFinish(txm); return; } From 232180ee2217fd9f96fdfaf890622aa612b7b938 Mon Sep 17 00:00:00 2001 From: yi-xmu Date: Thu, 29 Jan 2026 19:14:11 +0800 Subject: [PATCH 2/7] remvove debug log --- tx_service/include/cc/cc_map.h | 2 - tx_service/include/cc/cc_page_clean_guard.h | 22 ----- tx_service/include/cc/cc_request.h | 19 ----- tx_service/include/cc/cc_shard.h | 1 - tx_service/include/cc/range_bucket_cc_map.h | 3 - tx_service/include/cc/template_cc_map.h | 94 --------------------- tx_service/src/cc/cc_req_misc.cpp | 48 ----------- tx_service/src/cc/cc_shard.cpp | 36 -------- tx_service/src/cc/local_cc_handler.cpp | 30 ------- tx_service/src/cc/local_cc_shards.cpp | 67 +-------------- tx_service/src/checkpointer.cpp | 8 +- tx_service/src/sequences/sequences.cpp | 17 ---- tx_service/src/tx_execution.cpp | 58 ------------- tx_service/src/tx_operation.cpp | 63 -------------- 14 files changed, 4 insertions(+), 464 deletions(-) diff --git a/tx_service/include/cc/cc_map.h b/tx_service/include/cc/cc_map.h index 7823a5bd..0d1434b6 100644 --- a/tx_service/include/cc/cc_map.h +++ b/tx_service/include/cc/cc_map.h @@ -224,8 +224,6 @@ class CcMap KickoutCcEntryCc *kickout_cc = nullptr, bool *is_success = nullptr) = 0; virtual void Clean() = 0; - virtual void PrintCcPage(LruPage *lru_page) = 0; - virtual void PrintAllCcPages() = 0; virtual void CleanEntry(LruEntry *entry, LruPage *page) = 0; diff --git a/tx_service/include/cc/cc_page_clean_guard.h b/tx_service/include/cc/cc_page_clean_guard.h index 6e9ca9d3..39c1c316 100644 --- a/tx_service/include/cc/cc_page_clean_guard.h +++ b/tx_service/include/cc/cc_page_clean_guard.h @@ -396,31 +396,9 @@ struct CcPageCleanGuardWithoutKickoutCc // map. At this time, we cannot evict this cce because the read // request will still access the old range. assert(*dirty_range_ts > 0); - // LOG(INFO) << "CanBeCleaned: cce: 0x" << std::hex << (void *) cce - // << ", dirty range ts: " << std::dec << *dirty_range_ts - // << ", range last sync ts: " << *range_last_sync_ts - // << ", ckpt ts: " << cce->CkptTs() - // << ", is free: " << std::boolalpha << cce->IsFree() - // << ", being ckpt: " << std::boolalpha - // << cce->GetBeingCkpt() << ", can not be cleaned."; return {false, false}; } - // if (!cce->IsFree() || cce->GetBeingCkpt()) - // { - // LOG(INFO) << "CanBeCleaned: cce: 0x" << std::hex << (void *) cce - // << ", is free: " << std::boolalpha << cce->IsFree() - // << ", being ckpt: " << std::boolalpha - // << cce->GetBeingCkpt() << ", can not be cleaned."; - // } - // else if (cce->IsFree() && !cce->GetBeingCkpt()) - // { - // LOG(INFO) << "CanBeCleaned: cce: 0x" << std::hex << (void *) cce - // << ", is free: " << std::boolalpha << cce->IsFree() - // << ", being ckpt: " << std::boolalpha - // << cce->GetBeingCkpt() << ", can be cleaned."; - // } - return {(cce->IsFree() && !cce->GetBeingCkpt()), false}; } diff --git a/tx_service/include/cc/cc_request.h b/tx_service/include/cc/cc_request.h index 7502bad9..aa3d6695 100644 --- a/tx_service/include/cc/cc_request.h +++ b/tx_service/include/cc/cc_request.h @@ -232,12 +232,6 @@ struct TemplatedCcRequest : public CcRequestBase // is marked as errored. if (init_res.error != CcErrorCode::NO_ERROR) { - LOG(INFO) - << "AbortCcRequest, tx:" << Txn() - << ", table name: " << table_name_->StringView() - << ", error code: " - << static_cast(init_res.error) - << ", with result: 0x" << std::hex << res_; AbortCcRequest(init_res.error); } // The req is aborted or will be re-enqueued. @@ -255,11 +249,6 @@ struct TemplatedCcRequest : public CcRequestBase { ccm_ = ccm; } - // LOG(INFO) << "Execute CcRequest, tx:" << Txn() - // << ", table name: " << table_name_->StringView() - // << ", table type: " - // << static_cast(table_name_->Type()) - // << ", txn: " << Txn(); assert(ccm != nullptr); assert(ccs.core_id_ == ccm->shard_->core_id_); return ccm->Execute(*typed_req); @@ -329,10 +318,6 @@ struct TemplatedCcRequest : public CcRequestBase void AbortCcRequest(CcErrorCode err_code) override { assert(err_code != CcErrorCode::NO_ERROR); - LOG(INFO) << "AbortCcRequest, tx:" << Txn() - << ", table name: " << table_name_->StringView() - << ", error code: " << static_cast(err_code) - << ", with result: 0x" << std::hex << res_; bool finished = res_->SetError(err_code); if (finished) @@ -1384,8 +1369,6 @@ struct ReadCc : public TemplatedCcRequest { if (tmp_cce_addr.Term() != cc_ng_term) { - LOG(INFO) << "ReadCc not leader, table name: " - << table_name_->StringView() << ", txn: " << Txn(); return false; } @@ -1414,8 +1397,6 @@ struct ReadCc : public TemplatedCcRequest if (cc_ng_term < 0 || cc_ng_term != ng_term_) { - LOG(INFO) << "ReadCc not leader, table name: " - << table_name_->StringView() << ", txn: " << Txn(); return false; } else diff --git a/tx_service/include/cc/cc_shard.h b/tx_service/include/cc/cc_shard.h index 842ddadd..39561f0c 100644 --- a/tx_service/include/cc/cc_shard.h +++ b/tx_service/include/cc/cc_shard.h @@ -1125,7 +1125,6 @@ class CcShard void OnDirtyDataFlushed() { - LOG(INFO) << "OnDirtyDataFlushed on shard: " << core_id_; ResetCleanStart(); if (WaitListSizeForMemory() > 0) { diff --git a/tx_service/include/cc/range_bucket_cc_map.h b/tx_service/include/cc/range_bucket_cc_map.h index 0f01a77d..189f5c68 100644 --- a/tx_service/include/cc/range_bucket_cc_map.h +++ b/tx_service/include/cc/range_bucket_cc_map.h @@ -178,9 +178,6 @@ class RangeBucketCcMap { case CcErrorCode::NO_ERROR: { - // LOG(INFO) << "RangeBucketCcMap::Execute: NO_ERROR, ng_term: " - // << ng_term << ", txn: " << req.Txn() << ", lock type: " - // << static_cast(acquired_lock); CcEntryAddr &cce_addr = hd_result->Value().cce_addr_; cce_addr.SetCceLock( reinterpret_cast(cce->GetKeyGapLockAndExtraData()), diff --git a/tx_service/include/cc/template_cc_map.h b/tx_service/include/cc/template_cc_map.h index c886bbad..483d88f4 100644 --- a/tx_service/include/cc/template_cc_map.h +++ b/tx_service/include/cc/template_cc_map.h @@ -1232,12 +1232,6 @@ class TemplateCcMap : public CcMap // accessed. recycle_lock = table_name_.Type() != TableType::Catalog; - // LOG(INFO) << "PostReadCc::Execute: ng_term: " << req.NodeGroupTerm() - // << ", txn: " << txn << ", table: " << - // table_name_.StringView() - // << ", table type: " - // << static_cast(table_name_.Type()); - // FIXME(lzx): Now, we don't backfill for "Unkown" entry when scanning. // So, Validate operation fails if another tx backfilled it. Temporary // fix is that we don't validate for "Unkown" status results. @@ -1420,8 +1414,6 @@ class TemplateCcMap : public CcMap if (req.SchemaVersion() != 0 && req.SchemaVersion() != schema_ts_) { hd_res->SetError(CcErrorCode::REQUESTED_TABLE_SCHEMA_MISMATCH); - LOG(INFO) << "ReadCc, schema version mismatch, tx:" << req.Txn() - << ", table name: " << table_name_.StringView(); return true; } @@ -1583,14 +1575,6 @@ class TemplateCcMap : public CcMap if (Type() == TableType::Primary || Type() == TableType::UniqueSecondary) { - // LOG(INFO) - // << "ReadCc PinRangeSlice with non force - // load." - // << ", shard: " << shard_->core_id_ - // << ", table_name: " << - // table_name_.StringView() - // << ", partition id: " << req.PartitionId() - // << ", txn: " << req.Txn(); RangeSliceOpStatus pin_status; RangeSliceId slice_id = shard_->local_shards_.PinRangeSlice( @@ -1631,12 +1615,6 @@ class TemplateCcMap : public CcMap req.SetCacheHitMissCollected(); } - // LOG(INFO) - // << "ReadCc PinRangeSlice done, pin_status: " - // << static_cast(pin_status) - // << ", shard: " << shard_->core_id_ - // << ", table_name: " << - // table_name_.StringView(); if (pin_status == RangeSliceOpStatus::Successful || pin_status == RangeSliceOpStatus::KeyNotExists) { @@ -1836,14 +1814,6 @@ class TemplateCcMap : public CcMap // But the cc map is full and cannot allocates a new entry. if (cce == nullptr) { - // LOG(INFO) - // << "ReadCc, cce is nullptr, tx:" << req.Txn() - // << ", table name: " << table_name_.StringView() - // << ", key: " << look_key->ToString() - // << ", txn: " << req.Txn() - // << ", is for write: " << std::boolalpha - // << req.IsForWrite() - // << ", shard: " << shard_->core_id_; shard_->EnqueueWaitListIfMemoryFull(&req); return false; } @@ -5761,19 +5731,6 @@ class TemplateCcMap : public CcMap } else if (req.RunOnCandidateNode()) { - // Just skip the cce in this round ckpt. - // if (req.export_base_table_item_ && - // table_name_.StringView() == "./sbtest/history" && - // shard_->core_id_ == 0) - // { - // LOG(INFO) - // << "DataSync for range split, start key: " - // << req.start_key_->ToString() - // << ", end key: " << req.end_key_->ToString() - // << ", table name: " << table_name_.StringView() - // << ", with cce: " << std::hex << (void *) (cce) - // << std::dec << ", txn: " << req.Txn(); - // } need_export = false; } else @@ -5806,13 +5763,6 @@ class TemplateCcMap : public CcMap req.accumulated_flush_data_size_[shard_->core_id_] += flush_size; - // if (shard_->core_id_ == 0) - // { - // LOG(INFO) << "Exported the cce: 0x" << std::hex - // << (void *) cce << ", on shard: 0" - // << ", table name: " << - // table_name_.StringView(); - // } if (export_result.second) { is_scan_mem_full = true; @@ -6970,10 +6920,6 @@ class TemplateCcMap : public CcMap if (cce == nullptr) { - // LOG(INFO) << "TemplateCcMap::Execute ReplayLogCc, cce is " - // "nullptr, enqueue wait list." - // << ", table name: " << table_name_.StringView() - // << ", on shard: " << shard_->core_id_; // Since we're not holding any range lock that would // block data sync during replay, just keep retrying // until we have free space in cc map. @@ -8157,16 +8103,6 @@ class TemplateCcMap : public CcMap if (ccp->last_dirty_commit_ts_ <= req.LastDataSyncTs()) { - if (cce->NeedCkpt()) - { - LOG(INFO) << "ScanSliceDeltaSizeCcForRangePartition: cce " - "need ckpt." - << ", ccpage last dirty commit ts: " - << ccp->last_dirty_commit_ts_ - << ", last data sync ts: " << req.LastDataSyncTs() - << ", on shard: " << shard_->core_id_; - assert(false); - } assert(!cce->NeedCkpt()); // Skip the pages that have no updates since last data sync. if (ccp->next_page_ == PagePosInf()) @@ -8759,36 +8695,6 @@ class TemplateCcMap : public CcMap return {free_cnt, next_page}; } - void PrintCcPage(LruPage *lru_page) override - { - CcPage *page = - static_cast< - CcPage *>( - lru_page); - for (auto &cce : page->entries_) - { - LOG(INFO) << "cce: 0x" << std::hex << (void *) (cce.get()) - << ", is free: " << std::boolalpha << cce->IsFree() - << ", being ckpt: " << std::boolalpha - << cce->GetBeingCkpt() - << ", commit ts: " << cce->CommitTs() - << ", payload status: " - << static_cast(cce->PayloadStatus()) - << ", ckpt ts: " << cce->CkptTs() - << ", for table: " << table_name_.StringView(); - } - } - - void PrintAllCcPages() override - { - for (auto it = ccmp_.begin(); it != ccmp_.end(); it++) - { - CcPage &page = - *it->second; - PrintCcPage(&page); - } - } - void Clean() override { size_t total_freed = 0; diff --git a/tx_service/src/cc/cc_req_misc.cpp b/tx_service/src/cc/cc_req_misc.cpp index eee369ca..669eab80 100644 --- a/tx_service/src/cc/cc_req_misc.cpp +++ b/tx_service/src/cc/cc_req_misc.cpp @@ -886,9 +886,6 @@ bool FetchRecordCc::Execute(CcShard &ccs) { if (req) { - LOG(INFO) << "FetchRecordCc, abort request, tx:" << req->Txn() - << ", table name: " << table_name_.StringView() - << ", key: " << tx_key_.ToString(); req->AbortCcRequest(CcErrorCode::NG_TERM_CHANGED); } } @@ -950,11 +947,6 @@ bool FetchRecordCc::Execute(CcShard &ccs) cce_->GetKeyGapLockAndExtraData()->ReleasePin(); cce_->RecycleKeyLock(ccs); - LOG(INFO) - << "FetchRecordCc, abort request, tx:" << req->Txn() - << ", table name: " << table_name_.StringView() - << ", key: " << tx_key_.ToString() << ", error code: " - << static_cast(error_code_); req->AbortCcRequest(CcErrorCode::DATA_STORE_ERR); } } @@ -1243,17 +1235,6 @@ bool UpdateCceCkptTsCc::Execute(CcShard &ccs) v_entry->SetCkptTs(ref.commit_ts_); v_entry->ClearBeingCkpt(); ccm->OnEntryFlushed(true, v_entry->IsPersistent()); - // if (ccs.core_id_ == 0) - // { - // LOG(INFO) << "SetCKptTs for versioned cce: 0x" << - // std::hex - // << (void *) (ref.cce_) - // << ", commit ts: " << std::dec << - // ref.commit_ts_ - // << ", post flush size: " << - // ref.post_flush_size_ - // << ", on shard: 0"; - // } } else { @@ -1265,14 +1246,6 @@ bool UpdateCceCkptTsCc::Execute(CcShard &ccs) v_entry->SetCkptTs(ref.commit_ts_); v_entry->ClearBeingCkpt(); ccm->OnEntryFlushed(true, v_entry->IsPersistent()); - if (ccs.core_id_ == 0) - { - LOG(INFO) << "SetCKptTs for non versioned cce: 0x" - << std::hex << (void *) (ref.cce_) - << ", commit ts: " << std::dec << ref.commit_ts_ - << ", post flush size: " << ref.post_flush_size_ - << ", on shard: 0"; - } } } else @@ -1521,13 +1494,6 @@ bool ShardCleanCc::Execute(CcShard &ccs) if (shard_heap->Full(&heap_alloc, &heap_commit) && shard_heap->NeedCleanShard(heap_alloc, heap_commit)) { - // LOG(INFO) << "ShardCleanCc::Execute still full after clean shard, - // " - // "on shard: " - // << ccs.core_id_ << ", heap committed: " << heap_commit - // << ", heap allocated: " << heap_alloc - // << ", need yield: " << std::boolalpha << need_yield - // << ", free count: " << free_count_; if (need_yield) { // Continue to clean in the next run one round. @@ -1559,13 +1525,6 @@ bool ShardCleanCc::Execute(CcShard &ccs) { // Get the free memory, re-run a batch of the waiting ccrequest. bool wait_list_empty = ccs.DequeueWaitListAfterMemoryFree(); - // LOG(INFO) - // << "ShardCleanCc::Execute after clean shard, wait_list_empty: - // " - // << std::boolalpha << wait_list_empty - // << ", heap committed: " << heap_commit - // << ", heap allocated: " << heap_alloc - // << ", on shard: " << ccs.core_id_; if (!wait_list_empty) { ccs.Enqueue(this); @@ -1582,13 +1541,6 @@ bool ShardCleanCc::Execute(CcShard &ccs) // waiting ccrequest if has any waiting request, otherwise, finish // this shard clean ccrequests. bool wait_list_empty = ccs.DequeueWaitListAfterMemoryFree(); - // LOG(INFO) - // << "ShardCleanCc::Execute no need to clean shard, - // wait_list_empty: " - // << std::boolalpha << wait_list_empty - // << ", heap committed: " << heap_commit - // << ", heap allocated: " << heap_alloc - // << ", on shard: " << ccs.core_id_; if (!wait_list_empty) { ccs.Enqueue(this); diff --git a/tx_service/src/cc/cc_shard.cpp b/tx_service/src/cc/cc_shard.cpp index d54bf0ab..5bb0f29e 100644 --- a/tx_service/src/cc/cc_shard.cpp +++ b/tx_service/src/cc/cc_shard.cpp @@ -1160,9 +1160,6 @@ std::pair CcShard::Clean() size_t free_cnt = 0; bool yield = false; - // LOG(INFO) << "Clean started: clean start page addr: 0x" << std::hex - // << (void *) clean_start_ccp_ << ", ccp: 0x" << (void *) ccp - // << std::dec << ", shard: " << core_id_; #ifndef RUNNING_TXSERVICE_CTEST size_t clean_page_cnt = 0, scan_page_cnt = 0; @@ -1206,39 +1203,6 @@ std::pair CcShard::Clean() } #endif clean_start_ccp_ = ccp; - // LOG(INFO) << "Clean finished: free_cnt: " << free_cnt - // << ", yield: " << std::boolalpha << yield - // << ", clean start page addr: 0x" << std::hex - // << (void *) clean_start_ccp_ << ", clean page cnt: " << - // std::dec - // << clean_page_cnt << ", scan page cnt: " << scan_page_cnt - // << ", shard: " << core_id_; - // if (core_id_ == 0 && free_cnt == 0 && !yield) - // { - // LOG(INFO) << "Printing cc pages on lru list on shard: 0"; - // LruPage *loop_ccp = head_ccp_.lru_next_; - // size_t ccp_cnt = 0; - // while (loop_ccp != &tail_ccp_) - // { - // loop_ccp->parent_map_->PrintCcPage(loop_ccp); - // loop_ccp = loop_ccp->lru_next_; - // ++ccp_cnt; - // } - // LOG(INFO) << "Printing cc pages on lru list on shard: " << core_id_ - // << " finished, ccpage count: " << ccp_cnt; - - // for (const auto &[table_name, ccm] : native_ccms_) - // { - // LOG(INFO) << "Table: " << table_name.StringView() - // << ", all records on this ccm: " << ccm->size(); - // if (!table_name.IsMeta()) - // { - // ccm->PrintAllCcPages(); - // } - // LOG(INFO) << "Printing cc pages on table: " - // << table_name.StringView() << " finished"; - // } - // } return {free_cnt, yield}; } diff --git a/tx_service/src/cc/local_cc_handler.cpp b/tx_service/src/cc/local_cc_handler.cpp index afb0897b..632ea213 100644 --- a/tx_service/src/cc/local_cc_handler.cpp +++ b/tx_service/src/cc/local_cc_handler.cpp @@ -483,12 +483,6 @@ void txservice::LocalCcHandler::Read(const TableName &table_name, TX_TRACE_ACTION(this, req); TX_TRACE_DUMP(req); cc_shards_.EnqueueCcRequest(thd_id_, key_shard_code, req); - // LOG(INFO) << "Read local request, txn: " << req->Txn() - // << ", table name: " << table_name.StringView() - // << ", table type: " - // << static_cast(table_name.Type()) - // << ", key: " << key.ToString() - // << ", partition id: " << partition_id; } else { @@ -647,11 +641,6 @@ bool txservice::LocalCcHandler::ReadLocal(const TableName &table_name, // shard to which the tx is bound, if the native cc node is not the // leader now, returns an error. hres.SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); - LOG(INFO) << "ReadLocal Execute not leader, table name: " - << table_name.StringView() - << ", allow_run_on_candidate: " << std::boolalpha - << allow_run_on_candidate << ", term: " << term - << ", txn: " << tx_number; return true; } @@ -695,13 +684,6 @@ bool txservice::LocalCcHandler::ReadLocal(const TableName &table_name, { //__catalog table will be preloaded when ccshard constructed finished = ccm->Execute(*read_req); - // LOG(INFO) << "ReadLocal Execute, table name: " - // << table_name.StringView() - // << ", allow_run_on_candidate: " << std::boolalpha - // << allow_run_on_candidate << ", term: " << term - // << ", txn: " << tx_number << ", finished: " << - // std::boolalpha - // << finished; if (finished) { read_req->Free(); @@ -804,11 +786,6 @@ bool txservice::LocalCcHandler::ReadLocal(const TableName &table_name, // shard to which the tx is bound, if the native cc node is not the // leader now, returns an error. hres.SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); - // LOG(INFO) << "ReadLocal Execute, table name: " - // << table_name.StringView() - // << ", allow_run_on_candidate: " << std::boolalpha - // << allow_run_on_candidate << ", term: " << term - // << ", txn: " << tx_number; return true; } @@ -843,13 +820,6 @@ bool txservice::LocalCcHandler::ReadLocal(const TableName &table_name, { //__catalog table will be preloaded when ccshard constructed finished = ccm->Execute(*read_req); - // LOG(INFO) << "ReadLocal Execute, table name: " - // << table_name.StringView() - // << ", allow_run_on_candidate: " << std::boolalpha - // << allow_run_on_candidate << ", term: " << term - // << ", txn: " << tx_number << ", finished: " << - // std::boolalpha - // << finished; if (finished) { read_req->Free(); diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index b74eb12d..15b70d0b 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -3711,8 +3711,7 @@ void LocalCcShards::DataSyncForRangePartition( LOG(ERROR) << "DataSync add read lock on table failed, " "table name: " << table_key.Name().StringView() << ", error code: " - << static_cast(read_req.ErrorCode()) - << ", txn: " << data_sync_txm->TxNumber(); + << static_cast(read_req.ErrorCode()); // If read lock acquire failed, retry next time. // Put back into the beginning. @@ -3803,12 +3802,6 @@ void LocalCcShards::DataSyncForRangePartition( return; } - // LOG(INFO) << "DataSyncForRangePartition acquired read lock on catalog - // " - // "and bucket: ng_term: " - // << ng_term << ", txn: " << data_sync_txm->TxNumber() - // << ", table: " << table_name.StringView(); - // Now that we have acquired read lock on catalog and bucket, there // won't be any ddl on this range. Update store_range and check if this // range is still owned by this node group. @@ -3856,15 +3849,6 @@ void LocalCcShards::DataSyncForRangePartition( assert(store_range); last_sync_ts = is_dirty ? 0 : range_entry->GetLastSyncTs(); - // if (table_name.StringView() == "./sbtest/history") - // { - // LOG(INFO) << "DataSync for range split, range id: " << range_id - // << ", last sync ts: " << last_sync_ts - // << ", table name: " << table_name.StringView() - // << ", start key: " << start_tx_key.ToString() - // << ", end key: " << end_tx_key.ToString() - // << ", txn: " << tx_number; - // } } if (table_name.IsBase()) @@ -3933,10 +3917,7 @@ void LocalCcShards::DataSyncForRangePartition( DLOG(INFO) << "No items need to be sync in this round data sync of range#" << range_id << " for table: " << table_name.StringView() - << ". Finish this task directly." - << ", txn: " - << (!during_split_range ? data_sync_txm->TxNumber() : 0) - << ", during split range: " << std::boolalpha << during_split_range; + << ". Finish this task directly."; if (!during_split_range) { txservice::CommitTx(data_sync_txm); @@ -3966,11 +3947,6 @@ void LocalCcShards::DataSyncForRangePartition( } data_sync_task->SetFinish(); data_sync_task->SetScanTaskFinished(); - // LOG(INFO) << "Finished this round data sync of range#" << range_id - // << " for table: " << table_name.StringView() - // << ", txn: " << data_sync_txm->TxNumber() - // << ", during split range: " << std::boolalpha - // << during_split_range; return; } assert(slices_delta_size.size() > 0 || export_base_table_items); @@ -3985,12 +3961,7 @@ void LocalCcShards::DataSyncForRangePartition( // Update slice post ckpt size. UpdateSlicePostCkptSize(store_range, slices_delta_size); - bool check_range_update = !during_split_range; - CODE_FAULT_INJECTOR("check_range_update_skip", { - DLOG(INFO) << "FaultInject check_range_update_skip"; - check_range_update = false; - }); - if (check_range_update) + if (!during_split_range) { // If the task comes from split range transaction, it is assumed that // there will be no further splitting. @@ -4035,14 +4006,6 @@ void LocalCcShards::DataSyncForRangePartition( } } - // if (table_name.StringView() == "./sbtest/history") - // { - // LOG(INFO) << "DataSync for range split, range id: " << range_id - // << ", table name: " << table_name.StringView() - // << ", with slice count with dirty keys: " - // << slices_delta_size.size() << ", txn: " << tx_number; - // } - // 3. Scan records. // The data sync worker thread is the owner of those vectors. @@ -4130,7 +4093,6 @@ void LocalCcShards::DataSyncForRangePartition( data_sync_task->flight_task_cnt_ += 1; } - size_t loop_count = 0; while (!scan_data_drained) { uint32_t core_rand = butil::fast_rand(); @@ -4140,7 +4102,6 @@ void LocalCcShards::DataSyncForRangePartition( EnqueueLowPriorityCcRequestToShard(core_rand % cc_shards_.size(), &scan_cc); scan_cc.Wait(); - ++loop_count; if (scan_cc.IsError()) { @@ -4233,16 +4194,6 @@ void LocalCcShards::DataSyncForRangePartition( GetCatalogFactory(table_name.Engine())->PositiveInfKey(); for (size_t i = 0; i < cc_shards_.size(); ++i) { - // if (table_name.StringView() == "./sbtest/history" && i == 0) - // { - // LOG(INFO) - // << "DataSync for range split, range id: " << range_id - // << ", table name: " << table_name.StringView() - // << ", with scan count: " - // << scan_cc.accumulated_scan_cnt_[i] - // << ", the loop count: " << loop_count - // << ", txn: " << tx_number; - // } for (size_t j = 0; j < scan_cc.accumulated_scan_cnt_[i]; ++j) { auto &rec = scan_cc.DataSyncVec(i)[j]; @@ -4494,12 +4445,6 @@ void LocalCcShards::DataSyncForRangePartition( req_vec.pop_back(); } - // LOG(INFO) << "DataSyncForRangePartition: worker: " << worker_idx - // << " finished scan." - // << ", during range split: " << std::boolalpha - // << during_split_range << ", txn: " - // << (data_sync_txm != nullptr ? data_sync_txm->TxNumber() : 0) - // << ", table: " << table_name.StringView(); PostProcessRangePartitionDataSyncTask(std::move(data_sync_task), data_sync_txm, DataSyncTask::CkptErrorCode::NO_ERROR, @@ -4867,10 +4812,6 @@ void LocalCcShards::DataSyncForHashPartition( } } - // LOG(INFO) << "DataSyncForHashPartition: ng_term: " << ng_term - // << ", txn: " << data_sync_txm->TxNumber() - // << ", table: " << table_name.StringView(); - // 3. Scan records. { std::lock_guard flight_task_lk( @@ -6005,8 +5946,6 @@ void LocalCcShards::FlushData(std::unique_lock &flush_worker_lk) EnqueueToCcShard(core_idx, &reset_cc); } reset_cc.Wait(); - // LOG(INFO) << "FlushData after set dirty data flushed" - // << ", the shard count: " << updated_ckpt_ts_core_ids.size(); auto ckpt_err = succ ? DataSyncTask::CkptErrorCode::NO_ERROR : DataSyncTask::CkptErrorCode::FLUSH_ERROR; diff --git a/tx_service/src/checkpointer.cpp b/tx_service/src/checkpointer.cpp index 6b82acd3..ce7acde6 100644 --- a/tx_service/src/checkpointer.cpp +++ b/tx_service/src/checkpointer.cpp @@ -236,16 +236,11 @@ void Checkpointer::Ckpt(bool is_last_ckpt) std::unordered_map tables = local_shards_.GetCatalogTableNameSnapshot(node_group, ckpt_ts); - bool truncate_log = true; - CODE_FAULT_INJECTOR("datasync_status_no_truncate_log", { - DLOG(INFO) << "FaultInject datasync_status_no_truncate_log"; - truncate_log = false; - }); std::shared_ptr status = std::make_shared( node_group, is_standby_node ? standby_node_term : leader_term, - truncate_log); + true); uint64_t last_succ_ckpt_ts = UINT64_MAX; bool can_be_skipped = !is_last_ckpt; @@ -475,7 +470,6 @@ void Checkpointer::Run() */ void Checkpointer::Notify(bool request_ckpt) { - ACTION_FAULT_INJECTOR("panic_notify_checkpointer"); std::unique_lock lk(ckpt_mux_); if (request_ckpt) { diff --git a/tx_service/src/sequences/sequences.cpp b/tx_service/src/sequences/sequences.cpp index cee11d4e..d8e01897 100644 --- a/tx_service/src/sequences/sequences.cpp +++ b/tx_service/src/sequences/sequences.cpp @@ -597,10 +597,6 @@ bool Sequences::ApplyIdOfTableRangePartition(const TableName &table, TxKey mkey = GenKey(seq_name); // std::unique_ptr mrec = std::make_unique(); std::unique_ptr mrec = GenRecord(); - LOG(INFO) << "ApplyIdOfTableRangePartition: mkey: " << mkey.ToString() - << ", table_name: " << table_name_.StringView() - << ", seq_schema_version: " << seq_schema_version_ - << ", txn: " << txm->TxNumber(); // uint64_t schema_version = Sequences::GetTableSchema()->Version(); ReadTxRequest read_req(&table_name_, seq_schema_version_, @@ -618,9 +614,6 @@ bool Sequences::ApplyIdOfTableRangePartition(const TableName &table, txm->Execute(&read_req); read_req.Wait(); - LOG(INFO) << "ApplyIdOfTableRangePartition: done mkey: " << mkey.ToString() - << ", error code: " << static_cast(read_req.ErrorCode()) - << ", txn: " << txm->TxNumber(); if (read_req.IsError()) { @@ -669,16 +662,10 @@ bool Sequences::ApplyIdOfTableRangePartition(const TableName &table, if (err != TxErrorCode::NO_ERROR) { txservice::AbortTx(txm); - LOG(INFO) << "ApplyIdOfTableRangePartition: aborted txn: " - << txm->TxNumber(); return false; } auto [success, commit_err] = txservice::CommitTx(txm); - LOG(INFO) << "ApplyIdOfTableRangePartition: committed txn: " - << txm->TxNumber() - << ", error code: " << static_cast(commit_err) - << ", success: " << std::boolalpha << success; assert(success); if (success) { @@ -810,10 +797,6 @@ bool Sequences::InitIdOfTableRangePartition(const TableName &table, txservice::AbortTx(txm, nullptr, nullptr); return false; } - LOG(INFO) << "InitIdOfTableRangePartition: done mkey: " << mkey.ToString() - << ", error code: " << static_cast(err) - << ", txn: " << txm->TxNumber() - << ", last_range_partition_id: " << last_range_partition_id; auto [success, commit_err] = txservice::CommitTx(txm, nullptr, nullptr); assert(success); diff --git a/tx_service/src/tx_execution.cpp b/tx_service/src/tx_execution.cpp index 24e7917f..be14d6b7 100644 --- a/tx_service/src/tx_execution.cpp +++ b/tx_service/src/tx_execution.cpp @@ -1982,12 +1982,6 @@ void TransactionExecution::Process(ReadOperation &read) range_rec_.GetRangeOwnerNg()->BucketOwner(); key_shard_code = range_ng << 10 | residual; partition_id = range_rec_.GetRangeInfo()->PartitionId(); - // LOG(INFO) << "ReadOperation for range partitioned table: - // " - // << table_name.StringView() - // << ", key: " << key.ToString() - // << ", partition id: " << partition_id - // << ", txn: " << tx_number_; } } else @@ -2009,12 +2003,6 @@ void TransactionExecution::Process(ReadOperation &read) partition_id = Sharder::MapKeyHashToHashPartitionId(key_hash); if (bucket_info != nullptr) { - // LOG(INFO) << "ReadOperation for hash partitioned table: " - // << table_name.StringView() - // << ", key: " << key.ToString() - // << ", get bucket info fast success" - // << ", read is running: " << std::boolalpha - // << read.is_running_; // Uses the lower 10 bits of the key's hash code to shard // the key across CPU cores in a cc node. uint32_t residual = key_hash & 0x3FF; @@ -2026,10 +2014,6 @@ void TransactionExecution::Process(ReadOperation &read) } else if (!lock_range_bucket_result_.IsFinished()) { - // LOG(INFO) - // << "ReadOperation for hash partitioned table: " - // << table_name.StringView() - // << ", key: " << key.ToString() << ", to lock bucket"; read.is_running_ = false; // First read and lock the bucket the key located in through // lock_bucket_op_. @@ -2054,10 +2038,6 @@ void TransactionExecution::Process(ReadOperation &read) } else // lock bucket finished and succeeded { - // LOG(INFO) << "ReadOperation for hash partitioned table: " - // << table_name.StringView() - // << ", key: " << key.ToString() - // << ", lock bucket finished and succeeded"; // If there is an error when getting the key's bucket info, // the error would be caught when forwarding the read // operation, which forces the tx state machine moves to @@ -2095,15 +2075,6 @@ void TransactionExecution::Process(ReadOperation &read) { read_ts = ts; } - // LOG(INFO) << "ReadOperation for table: " << - // table_name.StringView() - // << ", key: " << key.ToString() << ", read ts: " << - // read_ts - // << ", read is running: " << std::boolalpha - // << read.is_running_ << ", txn: " << tx_number_ - // << ", is hash partitioned: " << std::boolalpha - // << table_name.IsHashPartitioned() - // << ", partition id: " << partition_id; cc_handler_->Read(table_name, read.read_tx_req_->schema_version_, @@ -2156,17 +2127,6 @@ void TransactionExecution::Process(ReadOperation &read) StartTiming(); } } - // LOG(INFO) << "ReadOperation sent read request for table: " - // << table_name.StringView() - // << ", key: " << read.read_tx_req_->key_->ToString() - // << ", txn: " << tx_number_ - // << ", read is running: " << std::boolalpha - // << read.is_running_ << ", with result: 0x" << std::hex - // << &read.hd_result_ << ". operation: 0x" << std::hex - // << &read << ", op lock range bucket result is finished: - // " - // << std::boolalpha - // << read.lock_range_bucket_result_->IsFinished(); } } else @@ -3768,7 +3728,6 @@ void TransactionExecution::Commit() { if (tx_term_ < 0) { - // LOG(INFO) << ">>>>Commit tx " << TxNumber() << ", tx_term < 0"; bool_resp_->Finish(false); bool_resp_ = nullptr; @@ -3817,8 +3776,6 @@ void TransactionExecution::Commit() } else { - // LOG(INFO) << ">>>>Commit tx " << TxNumber() << ", push set ts - // op"; PushOperation(&set_ts_); } } @@ -4189,10 +4146,6 @@ void TransactionExecution::PostProcess(AcquireWriteOperation &acquire_write) } else if (acquire_write.hd_result_.IsError()) { - DLOG(ERROR) << "AcquireWriteOperation failed for cc error:" - << acquire_write.hd_result_.ErrorMsg() << " ,error code: " - << static_cast(acquire_write.hd_result_.ErrorCode()) - << "; txn: " << TxNumber(); bool_resp_->SetErrorCode( ConvertCcError(acquire_write.hd_result_.ErrorCode())); Abort(); @@ -5826,7 +5779,6 @@ void TransactionExecution::ReleaseMetaDataReadLock( size_t post_local_cnt = 0; for (const auto &[cce_addr, read_entry_pair] : rset) { - // LOG(INFO) << ">>>>ReleaseMetaDataReadLock, tx " << TxNumber(); const ReadSetEntry &read_entry = read_entry_pair.first; CcReqStatus ret = cc_handler_->PostRead(TxNumber(), TxTerm(), @@ -7826,16 +7778,6 @@ void TransactionExecution::Process(BatchReadOperation &batch_read_op) { partition_id = batch_read_op.range_ids_[idx]; } - // LOG(INFO) << "BatchReadOperation, txn: " << TxNumber() - // << ", table name: " << table_name.StringView() - // << ", table type: " - // << static_cast(table_name.Type()) - // << ", is hash partitioned: " << std::boolalpha - // << table_name.IsHashPartitioned() - // << ", batch read op range ids size: " - // << batch_read_op.range_ids_.size() - // << ", key: " << key.ToString() - // << ", partition id: " << partition_id << ", idx: " << idx; cc_handler_->Read( table_name, diff --git a/tx_service/src/tx_operation.cpp b/tx_service/src/tx_operation.cpp index 2d6ae7d1..d991b9a9 100644 --- a/tx_service/src/tx_operation.cpp +++ b/tx_service/src/tx_operation.cpp @@ -143,21 +143,6 @@ void ReadOperation::Forward(TransactionExecution *txm) { if (!read_tx_req_->read_local_) { - // Just returned from lock_bucket_op_, check lock_bucket_result_. - // LOG(INFO) << "ReadOperation Forward, lock_range_bucket_result_ is - // " - // "not finished" - // << ", tx allow_run_on_candidate: " << std::boolalpha - // << txm->allow_run_on_candidate_ << ", table_name: " - // << read_tx_req_->tab_name_->StringView() - // << ", read key: " << read_tx_req_->key_->ToString() - // << ", txn: " << txm->TxNumber() - // << ", lock range bucket result: " << std::boolalpha - // << lock_range_bucket_result_->IsFinished() - // << ", error code: " - // << static_cast( - // lock_range_bucket_result_->ErrorCode()) - // << ", this: 0x" << std::hex << this; assert(lock_range_bucket_result_->IsFinished()); if (lock_range_bucket_result_->IsError()) { @@ -181,11 +166,6 @@ void ReadOperation::Forward(TransactionExecution *txm) txm->PostProcess(*this); return; } - // LOG(INFO) << "ReadOperation Forward, this: 0x" << std::hex << this - // << ", table name: " << - // read_tx_req_->tab_name_->StringView() - // << ", read key: " << read_tx_req_->key_->ToString() - // << ", txn: " << std::dec << txm->TxNumber(); txm->Process(*this); return; @@ -208,26 +188,11 @@ void ReadOperation::Forward(TransactionExecution *txm) } if (retry_num_ > 0) { - // LOG(INFO) << "ReadOperation ReRunOp, this: 0x" << std::hex - // << this << ", table name: " - // << read_tx_req_->tab_name_->StringView() - // << ", read key: " << read_tx_req_->key_->ToString() - // << ", txn: " << std::dec << txm->TxNumber() - // << ", hd result error code: " - // << static_cast(hd_result_.ErrorCode()) - // << ", retry num: " << retry_num_; hd_result_.Value().Reset(); hd_result_.Reset(); ReRunOp(txm); return; } - // LOG(INFO) << "ReadOperation topostprocess, this: 0x" << std::hex - // << this << ", table name: " - // << read_tx_req_->tab_name_->StringView() - // << ", read key: " << read_tx_req_->key_->ToString() - // << ", txn: " << std::dec << txm->TxNumber() - // << ", hd result error code: " - // << static_cast(hd_result_.ErrorCode()); } txm->PostProcess(*this); @@ -251,11 +216,6 @@ void ReadOperation::Forward(TransactionExecution *txm) { if (!hd_result_.Value().is_local_) { - // LOG(INFO) << "ReadOperation timeout, tx:" << txm->TxNumber() - // << ", table name: " - // << read_tx_req_->tab_name_->StringView() - // << ", read key: " << - // read_tx_req_->key_->ToString(); if (!txm->CheckLeaderTerm()) { hd_result_.SetError(CcErrorCode::TX_NODE_NOT_LEADER); @@ -265,11 +225,6 @@ void ReadOperation::Forward(TransactionExecution *txm) if (cce_addr.Term() > 0) { - // LOG(INFO) - // << "ReadOperation timeout, ack received, tx:" - // << txm->TxNumber() << ", table name: " - // << read_tx_req_->tab_name_->StringView() - // << ", read key: " << read_tx_req_->key_->ToString(); // ack is received, but timeout happens. // Trigger deadlock check. @@ -298,11 +253,6 @@ void ReadOperation::Forward(TransactionExecution *txm) // FIXME(lzx): Is it more appropriate to retry? // If the tx node fails, also force the tx to abort // instantly. - // LOG(INFO) - // << "ReadOperation timeout, no ack received, tx:" - // << txm->TxNumber() << ", table name: " - // << read_tx_req_->tab_name_->StringView() - // << ", read key: " << read_tx_req_->key_->ToString(); bool force_error = hd_result_.ForceError(); if (force_error) { @@ -314,11 +264,6 @@ void ReadOperation::Forward(TransactionExecution *txm) // execution. The tx will be re-executed when the tx // processor visits it in the execution queue. } - // LOG(INFO) << "ReadOperation timeout, force error, tx:" - // << txm->TxNumber() << ", table name: " - // << read_tx_req_->tab_name_->StringView() - // << ", read key: " << - // read_tx_req_->key_->ToString(); } else { @@ -352,9 +297,6 @@ void ReadLocalOperation::Forward(txservice::TransactionExecution *txm) hd_result_->SetError(CcErrorCode::TX_NODE_NOT_LEADER); hd_result_->ForceError(); txm->PostProcess(*this); - // LOG(INFO) << "ReadLocalOperation error for table: " - // << table_name_.StringView() - // << ", key: " << key_->ToString(); } else if (hd_result_->IsError()) { @@ -400,11 +342,6 @@ void ReadLocalOperation::Forward(txservice::TransactionExecution *txm) { // Read local succeeded txm->PostProcess(*this); - // LOG(INFO) << "ReadLocalOperation succeeded for table: " - // << table_name_.StringView() << ", table type: " - // << static_cast(table_name_.Type()) - // << ", txn: " << txm->TxNumber() - // << ", key: " << key_->ToString(); } } else From 19943e88970a7d96b0d8c6c7d1b64a471e101da9 Mon Sep 17 00:00:00 2001 From: yi-xmu Date: Mon, 2 Feb 2026 12:33:46 +0800 Subject: [PATCH 3/7] remove useless variable for rangepartitiondatasyncscancc --- tx_service/include/cc/cc_request.h | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tx_service/include/cc/cc_request.h b/tx_service/include/cc/cc_request.h index aa3d6695..b3f46442 100644 --- a/tx_service/include/cc/cc_request.h +++ b/tx_service/include/cc/cc_request.h @@ -4319,7 +4319,7 @@ struct RangePartitionDataSyncScanCc : public CcRequestBase cv_.wait(lk, [this] { return unfinished_cnt_ == 0; }); } - void Reset(OpType op_type = OpType::Normal) + void Reset() { std::lock_guard lk(mux_); unfinished_cnt_ = 1; @@ -4349,7 +4349,6 @@ struct RangePartitionDataSyncScanCc : public CcRequestBase } err_ = CcErrorCode::NO_ERROR; - op_type_ = op_type; slice_coordinator_.Reset(); } @@ -4436,11 +4435,6 @@ struct RangePartitionDataSyncScanCc : public CcRequestBase err_ = CcErrorCode::LOG_NOT_TRUNCATABLE; } - bool IsTerminated() const - { - return op_type_ == OpType::Terminated; - } - StoreRange *StoreRangePtr() const { return store_range_; @@ -4740,8 +4734,6 @@ struct RangePartitionDataSyncScanCc : public CcRequestBase // TODO(xxx) general solution for #1130 const uint64_t schema_version_{0}; - OpType op_type_{OpType::Normal}; - template Date: Mon, 2 Feb 2026 12:47:38 +0800 Subject: [PATCH 4/7] check the node status when execute datasync --- tx_service/include/cc/cc_request.h | 10 +++++++--- tx_service/include/cc/template_cc_map.h | 5 +++++ tx_service/src/cc/local_cc_shards.cpp | 6 +++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/tx_service/include/cc/cc_request.h b/tx_service/include/cc/cc_request.h index b3f46442..bb00b86b 100644 --- a/tx_service/include/cc/cc_request.h +++ b/tx_service/include/cc/cc_request.h @@ -4179,7 +4179,8 @@ struct RangePartitionDataSyncScanCc : public CcRequestBase bool export_base_table_item_only = false, StoreRange *store_range = nullptr, const std::map *old_slices_delta_size = nullptr, - uint64_t schema_version = 0) + uint64_t schema_version = 0, + bool run_on_candidate_node = false) : scan_heap_is_full_(false), table_name_(&table_name), node_group_id_(node_group_id), @@ -4198,7 +4199,8 @@ struct RangePartitionDataSyncScanCc : public CcRequestBase slice_coordinator_(export_base_table_item_, &slices_to_scan_), export_base_table_item_only_(export_base_table_item_only), store_range_(store_range), - schema_version_(schema_version) + schema_version_(schema_version), + run_on_candidate_node_(run_on_candidate_node) { tx_number_ = txn; assert(scan_batch_size_ > DataSyncScanBatchSize); @@ -4557,7 +4559,7 @@ struct RangePartitionDataSyncScanCc : public CcRequestBase bool RunOnCandidateNode() const { - return true; + return run_on_candidate_node_; } std::vector accumulated_scan_cnt_; @@ -4734,6 +4736,8 @@ struct RangePartitionDataSyncScanCc : public CcRequestBase // TODO(xxx) general solution for #1130 const uint64_t schema_version_{0}; + bool run_on_candidate_node_{false}; + template IndexKeySchema(table_name)->SchemaTs(); } + bool run_on_candidate_node = + Sharder::Instance().CandidateLeaderTerm(ng_id) > 0; + // Scan the delta slice size std::map slices_delta_size; ScanSliceDeltaSizeCcForRangePartition scan_delta_size_cc( @@ -4083,7 +4086,8 @@ void LocalCcShards::DataSyncForRangePartition( false, store_range, &slices_delta_size, - schema_version); + schema_version, + run_on_candidate_node); { // DataSync Worker will call PostProcessDataSyncTask() to decrement From 3dba7d344128565748f5224c53d13c505fd75426 Mon Sep 17 00:00:00 2001 From: yi-xmu Date: Mon, 2 Feb 2026 16:20:31 +0800 Subject: [PATCH 5/7] debug log for try update ckptts but cce persistent --- tx_service/include/cc/template_cc_map.h | 22 ++++++++++++++++++++++ tx_service/src/cc/cc_req_misc.cpp | 9 +++++++++ 2 files changed, 31 insertions(+) diff --git a/tx_service/include/cc/template_cc_map.h b/tx_service/include/cc/template_cc_map.h index 9ccdeda4..5de575cd 100644 --- a/tx_service/include/cc/template_cc_map.h +++ b/tx_service/include/cc/template_cc_map.h @@ -5765,6 +5765,28 @@ class TemplateCcMap : public CcMap req.export_base_table_item_only_, export_persisted_key_only, flush_size); + if (cce->IsPersistent()) + { + size_t entry_idx = + req.accumulated_scan_cnt_[shard_->core_id_] - 1; + FlushRecord &ref = + req.DataSyncVec(shard_->core_id_)[entry_idx]; + if (ref.cce_ != nullptr && !req.export_base_table_item_) + { + LOG(ERROR) + << "cce is persistent but cce is not nullptr and " + "export_base_table_item_ is false" + << ". cce addr: " << std::hex << (void *) cce + << ", export persisted key only: " << std::boolalpha + << export_persisted_key_only + << ", export base table term only: " + << std::boolalpha + << req.export_base_table_item_only_ + << ", for table: " << table_name_.StringView() + << ", on shard: " << shard_->core_id_; + assert(false); + } + } req.accumulated_flush_data_size_[shard_->core_id_] += flush_size; diff --git a/tx_service/src/cc/cc_req_misc.cpp b/tx_service/src/cc/cc_req_misc.cpp index 669eab80..830798dc 100644 --- a/tx_service/src/cc/cc_req_misc.cpp +++ b/tx_service/src/cc/cc_req_misc.cpp @@ -1229,6 +1229,15 @@ bool UpdateCceCkptTsCc::Execute(CcShard &ccs) VersionedLruEntry *v_entry = static_cast *>(ref.cce_); + if (v_entry->CommitTs() <= 1 || v_entry->IsPersistent()) + { + LOG(INFO) + << "v_entry->CommitTs() <= 1 || v_entry->IsPersistent()" + << " v_entry->CommitTs(): " << v_entry->CommitTs() + << " v_entry->CkptTs(): " << v_entry->CkptTs() + << " v_entry->IsPersistent(): " << std::boolalpha + << v_entry->IsPersistent(); + } assert(v_entry->CommitTs() > 1 && !v_entry->IsPersistent()); v_entry->entry_info_.SetDataStoreSize(ref.post_flush_size_); From b36ca7374c02d7388cf24bec319a2584dcbbe3f9 Mon Sep 17 00:00:00 2001 From: yi-xmu Date: Mon, 2 Feb 2026 16:21:41 +0800 Subject: [PATCH 6/7] remove debug log for updateckpt ts --- tx_service/include/cc/template_cc_map.h | 22 ---------------------- tx_service/src/cc/cc_req_misc.cpp | 9 --------- 2 files changed, 31 deletions(-) diff --git a/tx_service/include/cc/template_cc_map.h b/tx_service/include/cc/template_cc_map.h index 5de575cd..9ccdeda4 100644 --- a/tx_service/include/cc/template_cc_map.h +++ b/tx_service/include/cc/template_cc_map.h @@ -5765,28 +5765,6 @@ class TemplateCcMap : public CcMap req.export_base_table_item_only_, export_persisted_key_only, flush_size); - if (cce->IsPersistent()) - { - size_t entry_idx = - req.accumulated_scan_cnt_[shard_->core_id_] - 1; - FlushRecord &ref = - req.DataSyncVec(shard_->core_id_)[entry_idx]; - if (ref.cce_ != nullptr && !req.export_base_table_item_) - { - LOG(ERROR) - << "cce is persistent but cce is not nullptr and " - "export_base_table_item_ is false" - << ". cce addr: " << std::hex << (void *) cce - << ", export persisted key only: " << std::boolalpha - << export_persisted_key_only - << ", export base table term only: " - << std::boolalpha - << req.export_base_table_item_only_ - << ", for table: " << table_name_.StringView() - << ", on shard: " << shard_->core_id_; - assert(false); - } - } req.accumulated_flush_data_size_[shard_->core_id_] += flush_size; diff --git a/tx_service/src/cc/cc_req_misc.cpp b/tx_service/src/cc/cc_req_misc.cpp index 830798dc..669eab80 100644 --- a/tx_service/src/cc/cc_req_misc.cpp +++ b/tx_service/src/cc/cc_req_misc.cpp @@ -1229,15 +1229,6 @@ bool UpdateCceCkptTsCc::Execute(CcShard &ccs) VersionedLruEntry *v_entry = static_cast *>(ref.cce_); - if (v_entry->CommitTs() <= 1 || v_entry->IsPersistent()) - { - LOG(INFO) - << "v_entry->CommitTs() <= 1 || v_entry->IsPersistent()" - << " v_entry->CommitTs(): " << v_entry->CommitTs() - << " v_entry->CkptTs(): " << v_entry->CkptTs() - << " v_entry->IsPersistent(): " << std::boolalpha - << v_entry->IsPersistent(); - } assert(v_entry->CommitTs() > 1 && !v_entry->IsPersistent()); v_entry->entry_info_.SetDataStoreSize(ref.post_flush_size_); From 2c5a15bf9222ec276f4ff03a450173d519937d8b Mon Sep 17 00:00:00 2001 From: yi-xmu Date: Mon, 2 Feb 2026 17:25:40 +0800 Subject: [PATCH 7/7] update --- tx_service/include/data_sync_task.h | 8 ++++---- tx_service/src/cc/cc_shard.cpp | 1 - tx_service/src/cc/local_cc_shards.cpp | 23 ++++++++++++----------- tx_service/src/data_sync_task.cpp | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tx_service/include/data_sync_task.h b/tx_service/include/data_sync_task.h index 3196505f..fd6add41 100644 --- a/tx_service/include/data_sync_task.h +++ b/tx_service/include/data_sync_task.h @@ -139,7 +139,7 @@ struct DataSyncTask sync_ts_adjustable_(need_adjust_ts), task_res_(hres), need_update_ckpt_ts_(true), - sync_on_leader_(true) + run_on_leader_node_(true) { } @@ -183,9 +183,9 @@ struct DataSyncTask sync_ts_adjustable_ = false; } - void SetSyncOnLeader(bool sync_on_leader) + void SetRunOnLeaderNode(bool run_on_leader_node) { - sync_on_leader_ = sync_on_leader; + run_on_leader_node_ = run_on_leader_node; } const TableName table_name_; @@ -244,7 +244,7 @@ struct DataSyncTask cce_entries_; bool need_update_ckpt_ts_{true}; - bool sync_on_leader_{true}; + bool run_on_leader_node_{true}; }; struct FlushTaskEntry diff --git a/tx_service/src/cc/cc_shard.cpp b/tx_service/src/cc/cc_shard.cpp index 5bb0f29e..c61cc0dc 100644 --- a/tx_service/src/cc/cc_shard.cpp +++ b/tx_service/src/cc/cc_shard.cpp @@ -1071,7 +1071,6 @@ void CcShard::CheckRecoverTx(TxNumber lock_holding_txn, if (Sharder::Instance().LeaderTerm(cc_ng_id) > 0) { - // TODO(ysw): check candidate leader term? LOG(WARNING) << "orphan lock detected, lock holding txn: " << lock_holding_txn << ", try to recover"; Sharder::Instance().RecoverTx(lock_holding_txn, diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index ddfe1b21..9316fa19 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -3252,7 +3252,7 @@ void LocalCcShards::PostProcessFlushTaskEntries( if (!task->during_split_range_) { uint64_t last_sync_ts = - task->sync_on_leader_ ? task->data_sync_ts_ : 0; + task->run_on_leader_node_ ? task->data_sync_ts_ : 0; range_entry->UpdateLastDataSyncTS(last_sync_ts); range_entry->UnPinStoreRange(); // Commit the data sync txm @@ -3409,7 +3409,7 @@ void LocalCcShards::PostProcessRangePartitionDataSyncTask( if (!task->during_split_range_) { uint64_t last_sync_ts = - task->sync_on_leader_ ? task->data_sync_ts_ : 0; + task->run_on_leader_node_ ? task->data_sync_ts_ : 0; range_entry->UpdateLastDataSyncTS(last_sync_ts); range_entry->UnPinStoreRange(); // Commit the data sync txm @@ -3631,7 +3631,7 @@ void LocalCcShards::DataSyncForRangePartition( // might miss the data that has not been recovered yet. data_sync_task->SetErrorCode( CcErrorCode::REQUESTED_NODE_NOT_LEADER); - data_sync_task->SetSyncOnLeader(false); + data_sync_task->SetRunOnLeaderNode(false); } defer_unpin = std::shared_ptr( @@ -3849,6 +3849,10 @@ void LocalCcShards::DataSyncForRangePartition( assert(store_range); last_sync_ts = is_dirty ? 0 : range_entry->GetLastSyncTs(); + if (Sharder::Instance().CandidateLeaderTerm(ng_id) > 0) + { + data_sync_task->SetRunOnLeaderNode(false); + } } if (table_name.IsBase()) @@ -3860,9 +3864,6 @@ void LocalCcShards::DataSyncForRangePartition( schema_version = table_schema->IndexKeySchema(table_name)->SchemaTs(); } - bool run_on_candidate_node = - Sharder::Instance().CandidateLeaderTerm(ng_id) > 0; - // Scan the delta slice size std::map slices_delta_size; ScanSliceDeltaSizeCcForRangePartition scan_delta_size_cc( @@ -3926,7 +3927,7 @@ void LocalCcShards::DataSyncForRangePartition( txservice::CommitTx(data_sync_txm); // Update the task status for this range. - uint64_t last_sync_ts = data_sync_task->sync_on_leader_ + uint64_t last_sync_ts = data_sync_task->run_on_leader_node_ ? data_sync_task->data_sync_ts_ : 0; range_entry->UpdateLastDataSyncTS(last_sync_ts); @@ -4087,7 +4088,7 @@ void LocalCcShards::DataSyncForRangePartition( store_range, &slices_delta_size, schema_version, - run_on_candidate_node); + !data_sync_task->run_on_leader_node_); { // DataSync Worker will call PostProcessDataSyncTask() to decrement @@ -4511,7 +4512,7 @@ void LocalCcShards::PostProcessHashPartitionDataSyncTask( // nullptr. assert(catalog_entry); uint64_t last_sync_ts = - task->sync_on_leader_ ? task->data_sync_ts_ : 0; + task->run_on_leader_node_ ? task->data_sync_ts_ : 0; catalog_entry->UpdateLastDataSyncTS(last_sync_ts, task->id_); } @@ -4693,7 +4694,7 @@ void LocalCcShards::DataSyncForHashPartition( // might miss the data that has not been recovered yet. data_sync_task->SetErrorCode( CcErrorCode::REQUESTED_NODE_NOT_LEADER); - data_sync_task->SetSyncOnLeader(false); + data_sync_task->SetRunOnLeaderNode(false); } } @@ -5734,7 +5735,7 @@ void LocalCcShards::SplitFlushRange( } uint64_t last_sync_ts = - data_sync_task->sync_on_leader_ ? data_sync_task->data_sync_ts_ : 0; + data_sync_task->run_on_leader_node_ ? data_sync_task->data_sync_ts_ : 0; range_entry->UpdateLastDataSyncTS(last_sync_ts); range_entry->UnPinStoreRange(); diff --git a/tx_service/src/data_sync_task.cpp b/tx_service/src/data_sync_task.cpp index c9a00822..727e5aae 100644 --- a/tx_service/src/data_sync_task.cpp +++ b/tx_service/src/data_sync_task.cpp @@ -80,7 +80,7 @@ DataSyncTask::DataSyncTask(const TableName &table_name, during_split_range_(true), export_base_table_items_(export_base_table_items), tx_number_(txn), - sync_on_leader_(true) + run_on_leader_node_(true) { assert(!table_name_.IsHashPartitioned()); if (start_key_.KeyPtr() ==