From ab7e488cb748056f3664a29a0a42faa0e9df0fbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E8=B6=85?= Date: Fri, 9 May 2025 16:34:50 +0800 Subject: [PATCH 001/110] txn: collect mvcc for error `CommitTsExpired` (#18442) close tikv/tikv#18441 When the secondary commit failed with error `CommitTsExpired`, collect the MVCC info for debugging. Signed-off-by: Chao Wang --- src/storage/errors.rs | 153 ++++++++++++++++++++++-------- src/storage/mvcc/mod.rs | 3 + src/storage/mvcc/txn.rs | 1 + src/storage/txn/actions/commit.rs | 110 ++++++++++++++++++--- 4 files changed, 210 insertions(+), 57 deletions(-) diff --git a/src/storage/errors.rs b/src/storage/errors.rs index 1eb3444d319..60874e45dc3 100644 --- a/src/storage/errors.rs +++ b/src/storage/errors.rs @@ -19,7 +19,7 @@ use crate::storage::{ kv::{self, Error as KvError, ErrorInner as KvErrorInner}, mvcc::{Error as MvccError, ErrorInner as MvccErrorInner}, txn::{self, Error as TxnError, ErrorInner as TxnErrorInner}, - CommandKind, Result, + types, CommandKind, Result, }; #[derive(Debug, Error)] @@ -364,6 +364,34 @@ pub fn extract_committed(err: &Error) -> Option { } } +fn get_or_insert_default_for_key_error_debug_info( + err: &mut kvrpcpb::KeyError, +) -> &mut kvrpcpb::DebugInfo { + let debug_info = &mut err.debug_info; + if debug_info.is_none() { + debug_info.set_default() + } else { + debug_info.as_mut().unwrap() + } +} + +fn add_debug_mvcc_for_key_error( + err: &mut kvrpcpb::KeyError, + key: &[u8], + mvcc_info: Option, +) { + if let Some(mut mvcc) = mvcc_info { + let debug_info = get_or_insert_default_for_key_error_debug_info(err); + // remove the values in default CF to reduce the size of the response. + mvcc.values.clear(); + // set mvcc info to debug_info + let mut mvcc_debug_info = kvrpcpb::MvccDebugInfo::default(); + mvcc_debug_info.set_key(key.to_owned()); + mvcc_debug_info.set_mvcc(mvcc.into_proto()); + debug_info.mvcc_info.push(mvcc_debug_info); + } +} + pub fn extract_key_error(err: &Error) -> kvrpcpb::KeyError { let mut key_error = kvrpcpb::KeyError::default(); match err { @@ -429,20 +457,7 @@ pub fn extract_key_error(err: &Error) -> kvrpcpb::KeyError { let mut txn_lock_not_found = kvrpcpb::TxnLockNotFound::default(); txn_lock_not_found.set_key(key.clone()); key_error.set_txn_lock_not_found(txn_lock_not_found); - // if mvcc_info is present, fill the debug_info for client's further debugging. - if let Some(mvcc_info) = mvcc_info { - // remove the values in default CF to reduce the size of the response. - let mut mvcc_info = mvcc_info.clone(); - mvcc_info.values.clear(); - - let mut mvcc_debug_info = kvrpcpb::MvccDebugInfo::default(); - mvcc_debug_info.set_key(key.clone()); - mvcc_debug_info.set_mvcc(mvcc_info.into_proto()); - - let mut debug_info = kvrpcpb::DebugInfo::default(); - debug_info.set_mvcc_info(vec![mvcc_debug_info].into()); - key_error.set_debug_info(debug_info); - } + add_debug_mvcc_for_key_error(&mut key_error, key, mvcc_info.clone()); } Error(box ErrorInner::Txn(TxnError(box TxnErrorInner::Mvcc(MvccError( box MvccErrorInner::TxnNotFound { start_ts, key }, @@ -475,6 +490,7 @@ pub fn extract_key_error(err: &Error) -> kvrpcpb::KeyError { commit_ts, key, min_commit_ts, + mvcc_info, }, ))))) => { let mut commit_ts_expired = kvrpcpb::CommitTsExpired::default(); @@ -483,6 +499,7 @@ pub fn extract_key_error(err: &Error) -> kvrpcpb::KeyError { commit_ts_expired.set_key(key.to_owned()); commit_ts_expired.set_min_commit_ts(min_commit_ts.into_inner()); key_error.set_commit_ts_expired(commit_ts_expired); + add_debug_mvcc_for_key_error(&mut key_error, key, mvcc_info.clone()); } Error(box ErrorInner::Txn(TxnError(box TxnErrorInner::Mvcc(MvccError( box MvccErrorInner::CommitTsTooLarge { min_commit_ts, .. }, @@ -640,10 +657,8 @@ mod test { assert_eq!(got, expect); } - #[test] - fn test_extract_key_error_txn_lock_not_found() { - let key = b"key".to_vec(); - let mvcc_info = MvccInfo { + fn mock_mvcc_info() -> MvccInfo { + MvccInfo { lock: Some(Lock::new( LockType::Lock, b"k".to_vec(), @@ -660,17 +675,38 @@ mod test { Write::new(WriteType::Lock, 7.into(), None), )], values: vec![(TimeStamp::new(7), b"v".to_vec())], - }; + } + } - let case = Error::from(TxnError::from(MvccError::from( - MvccErrorInner::TxnLockNotFound { - start_ts: TimeStamp::new(123), - commit_ts: TimeStamp::new(456), - key: key.clone(), - mvcc_info: Some(mvcc_info.clone()), - }, - ))); + fn expected_debug_info_from_mvcc(key: Vec, mvcc: MvccInfo) -> kvrpcpb::DebugInfo { + let mut expect_pb_mvcc_info = mvcc.clone().into_proto(); + // should clear the values in default CF to reduce the size of the response. + expect_pb_mvcc_info.values.clear(); + kvrpcpb::DebugInfo { + mvcc_info: vec![kvrpcpb::MvccDebugInfo { + key, + mvcc: Some(expect_pb_mvcc_info).into(), + ..Default::default() + }] + .into(), + ..Default::default() + } + } + #[test] + fn test_extract_key_error_txn_lock_not_found() { + fn mock_txn_lock_not_found_err(has_mvcc: bool) -> kvrpcpb::KeyError { + extract_key_error(&Error::from(TxnError::from(MvccError::from( + MvccErrorInner::TxnLockNotFound { + start_ts: TimeStamp::new(123), + commit_ts: TimeStamp::new(456), + key: b"key".to_vec(), + mvcc_info: has_mvcc.then(|| mock_mvcc_info()), + }, + )))) + } + + let key = b"key".to_vec(); let mut expect = kvrpcpb::KeyError::default(); let mut txn_lock_not_found = kvrpcpb::TxnLockNotFound::default(); txn_lock_not_found.set_key(key.clone()); @@ -687,20 +723,53 @@ mod test { ))), ); expect.set_retryable(expected_retryable_msg); - let mut expect_pb_mvcc_info = mvcc_info.clone().into_proto(); - // should clear the values in default CF to reduce the size of the response. - expect_pb_mvcc_info.values.clear(); - expect.set_debug_info(kvrpcpb::DebugInfo { - mvcc_info: vec![kvrpcpb::MvccDebugInfo { - key: key.clone(), - mvcc: Some(expect_pb_mvcc_info).into(), - ..Default::default() - }] - .into(), - ..Default::default() - }); - let got = extract_key_error(&case); - assert_eq!(got, expect); + // without mvcc + expect.clear_debug_info(); + assert_eq!(mock_txn_lock_not_found_err(false), expect); + + // with mvcc + let mvcc_info = Some(mock_mvcc_info()); + expect.set_debug_info(expected_debug_info_from_mvcc( + key.clone(), + mvcc_info.clone().unwrap(), + )); + assert_eq!(mock_txn_lock_not_found_err(true), expect); + } + + #[test] + fn test_extract_key_error_commit_ts_expired() { + fn mock_commit_ts_expired_err(has_mvcc: bool) -> kvrpcpb::KeyError { + extract_key_error(&Error::from(TxnError::from(MvccError::from( + MvccErrorInner::CommitTsExpired { + start_ts: TimeStamp::new(123), + commit_ts: TimeStamp::new(456), + key: b"key".to_vec(), + min_commit_ts: TimeStamp::new(789), + mvcc_info: has_mvcc.then(|| mock_mvcc_info()), + }, + )))) + } + + let key = b"key".to_vec(); + let mut expect = kvrpcpb::KeyError::default(); + let mut commit_ts_expired = kvrpcpb::CommitTsExpired::default(); + commit_ts_expired.set_key(key.clone()); + commit_ts_expired.set_start_ts(123); + commit_ts_expired.set_attempted_commit_ts(456); + commit_ts_expired.set_min_commit_ts(789); + expect.set_commit_ts_expired(commit_ts_expired); + + // without mvcc + expect.clear_debug_info(); + assert_eq!(mock_commit_ts_expired_err(false), expect); + + // with mvcc + let mvcc = Some(mock_mvcc_info()); + expect.set_debug_info(expected_debug_info_from_mvcc( + key.clone(), + mvcc.clone().unwrap(), + )); + assert_eq!(mock_commit_ts_expired_err(true), expect); } } diff --git a/src/storage/mvcc/mod.rs b/src/storage/mvcc/mod.rs index 064a29da4ce..052c306eaa1 100644 --- a/src/storage/mvcc/mod.rs +++ b/src/storage/mvcc/mod.rs @@ -135,6 +135,7 @@ pub enum ErrorInner { commit_ts: TimeStamp, key: Vec, min_commit_ts: TimeStamp, + mvcc_info: Option, }, #[error("bad format key(version)")] @@ -265,11 +266,13 @@ impl ErrorInner { commit_ts, key, min_commit_ts, + mvcc_info, } => Some(ErrorInner::CommitTsExpired { start_ts: *start_ts, commit_ts: *commit_ts, key: key.clone(), min_commit_ts: *min_commit_ts, + mvcc_info: mvcc_info.clone(), }), ErrorInner::KeyVersion => Some(ErrorInner::KeyVersion), ErrorInner::Committed { diff --git a/src/storage/mvcc/txn.rs b/src/storage/mvcc/txn.rs index 9b0044ff46d..40b2b386f84 100644 --- a/src/storage/mvcc/txn.rs +++ b/src/storage/mvcc/txn.rs @@ -304,6 +304,7 @@ pub(crate) fn make_txn_error( commit_ts: TimeStamp::zero(), key: key.to_raw().unwrap(), min_commit_ts: TimeStamp::zero(), + mvcc_info: None, }, "pessimisticlocknotfound" => ErrorInner::PessimisticLockNotFound { start_ts, diff --git a/src/storage/txn/actions/commit.rs b/src/storage/txn/actions/commit.rs index d6be1e1d44b..92600ddea61 100644 --- a/src/storage/txn/actions/commit.rs +++ b/src/storage/txn/actions/commit.rs @@ -23,22 +23,43 @@ pub fn commit( crate::storage::mvcc::txn::make_txn_error(err, &key, reader.start_ts,).into() )); + let collect_mvcc = |reader: &SnapshotReader| { + collect_mvcc_info_for_debug(reader.reader.snapshot().clone(), &key).map( + |(lock, writes, values)| MvccInfo { + lock, + writes, + values, + }, + ) + }; + let (mut lock, commit) = match reader.load_lock(&key)? { Some(lock) if lock.ts == reader.start_ts => { // A lock with larger min_commit_ts than current commit_ts can't be committed if commit_ts < lock.min_commit_ts { - info!( + // The min_commit_ts can be pushed to a larger value before + // committing the primary. + // When committing the secondary, the commit_ts must be determined and + // should always be greater than the min_commit_ts. + // If not, it is an unexpected case and may be a bug. + let unexpected = matches!(commit_role, Some(CommitRole::Secondary)); + // collect mvcc info when an unexpected case happens. + let mvcc_info = unexpected.then(|| collect_mvcc(reader)).flatten(); + info_or_error!( + !unexpected; "trying to commit with smaller commit_ts than min_commit_ts"; "key" => %key, "start_ts" => reader.start_ts, "commit_ts" => commit_ts, "min_commit_ts" => lock.min_commit_ts, + "mvcc_info" => ?mvcc_info, ); return Err(ErrorInner::CommitTsExpired { start_ts: reader.start_ts, commit_ts, key: key.into_raw()?, min_commit_ts: lock.min_commit_ts, + mvcc_info, } .into()); } @@ -57,7 +78,7 @@ pub fn commit( "commit_ts" => commit_ts, // Though it may not be a bug here, but we still want to collect the mvcc // info here for further debugging if needed. - "mvcc_info" => ?collect_mvcc_info_for_debug(reader.reader.snapshot().clone(), &key), + "mvcc_info" => ?collect_mvcc(reader), ); (lock, false) } else { @@ -75,17 +96,7 @@ pub fn commit( // collect more information for debugging. let unexpected = matches!(commit_role, Some(CommitRole::Secondary)); // only collect the mvcc information if an unexpected case happens. - let mvcc_info = unexpected - .then(|| { - collect_mvcc_info_for_debug(reader.reader.snapshot().clone(), &key) - }) - .flatten() - .map(|(lock, writes, values)| MvccInfo { - lock, - writes, - values, - }); - + let mvcc_info = unexpected.then(|| collect_mvcc(reader)).flatten(); info_or_error!( !unexpected; "txn conflict (lock not found)"; @@ -325,9 +336,78 @@ pub mod tests { uncommitted(100, ts(20, 1)), ); + fn check_commit_ts_expired_err( + err: Error, + expected_key: &[u8], + expected_start_ts: TimeStamp, + expected_commit_ts: TimeStamp, + expected_min_commit_ts: TimeStamp, + has_mvcc: bool, + ) { + assert_matches!(err, Error(box ErrorInner::CommitTsExpired { + start_ts, + commit_ts, + key, + min_commit_ts, + mvcc_info, + }) if { + assert_eq!(key, expected_key.to_vec()); + assert_eq!(start_ts, expected_start_ts); + assert_eq!(commit_ts, expected_commit_ts); + assert_eq!(min_commit_ts, expected_min_commit_ts); + assert_eq!(has_mvcc, mvcc_info.is_some()); + true + }) + } + // The min_commit_ts should be ts(20, 1) - must_err(&mut engine, k, ts(10, 0), ts(15, 0), None); - must_err(&mut engine, k, ts(10, 0), ts(20, 0), None); + check_commit_ts_expired_err( + must_err(&mut engine, k, ts(10, 0), ts(15, 0), None), + k, + ts(10, 0), + ts(15, 0), + ts(20, 1), + // Unknown commit role should not collect mvcc + false, + ); + check_commit_ts_expired_err( + must_err( + &mut engine, + k, + ts(10, 0), + ts(15, 0), + Some(CommitRole::Primary), + ), + k, + ts(10, 0), + ts(15, 0), + ts(20, 1), + // Primary commit should not collect mvcc because it is an expected case. + false, + ); + check_commit_ts_expired_err( + must_err( + &mut engine, + k, + ts(10, 0), + ts(15, 0), + Some(CommitRole::Secondary), + ), + k, + ts(10, 0), + ts(15, 0), + ts(20, 1), + // Secondary commit should not collect mvcc because it may be a bug. + true, + ); + check_commit_ts_expired_err( + must_err(&mut engine, k, ts(10, 0), ts(20, 0), None), + k, + ts(10, 0), + ts(20, 0), + ts(20, 1), + false, + ); must_succeed(&mut engine, k, ts(10, 0), ts(20, 1)); must_prewrite_put_for_large_txn(&mut engine, k, v, k, ts(30, 0), 100, 0); From 4056e44f3713330d0640563a8140bb8871b49c6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E8=B6=85?= Date: Mon, 12 May 2025 22:07:47 +0800 Subject: [PATCH 002/110] txn: return debug info for some error when `commit_role` is unknown (#18448) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit close tikv/tikv#18441 In the previous PR, we only collect mvcc info when `commit_role` is `Secondary` when `commit_ts < min_commit_ts`. However, when resolving a lock, the `commit_role` of `commit` is None and we cannot get any mvcc info when this error happens. This PR does an enhancement and checks whether the resolved key is primary in the lock, if not, it still collect mvcc for further debugging. Signed-off-by: Chao Wang Signed-off-by: 王超 Co-authored-by: cfzjywxk --- src/storage/txn/actions/commit.rs | 55 ++++++++++++++++++------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/src/storage/txn/actions/commit.rs b/src/storage/txn/actions/commit.rs index 92600ddea61..2e93a7ac0d2 100644 --- a/src/storage/txn/actions/commit.rs +++ b/src/storage/txn/actions/commit.rs @@ -37,18 +37,20 @@ pub fn commit( Some(lock) if lock.ts == reader.start_ts => { // A lock with larger min_commit_ts than current commit_ts can't be committed if commit_ts < lock.min_commit_ts { + let primary_key = Key::from_raw(&lock.primary); // The min_commit_ts can be pushed to a larger value before // committing the primary. // When committing the secondary, the commit_ts must be determined and // should always be greater than the min_commit_ts. // If not, it is an unexpected case and may be a bug. - let unexpected = matches!(commit_role, Some(CommitRole::Secondary)); + let unexpected = key != primary_key; // collect mvcc info when an unexpected case happens. let mvcc_info = unexpected.then(|| collect_mvcc(reader)).flatten(); info_or_error!( !unexpected; "trying to commit with smaller commit_ts than min_commit_ts"; "key" => %key, + "primary_key" => %primary_key, "start_ts" => reader.start_ts, "commit_ts" => commit_ts, "min_commit_ts" => lock.min_commit_ts, @@ -158,6 +160,8 @@ pub mod tests { use kvproto::kvrpcpb::Context; #[cfg(test)] use kvproto::kvrpcpb::PrewriteRequestPessimisticAction::*; + #[cfg(test)] + use kvproto::kvrpcpb::{Assertion, AssertionLevel}; use tikv_kv::SnapContext; #[cfg(test)] use txn_types::{LastChange, TimeStamp}; @@ -310,6 +314,7 @@ pub mod tests { let mut engine = TestEngineBuilder::new().build().unwrap(); let (k, v) = (b"k", b"v"); + let (k2, v2) = (b"k2", b"v2"); // Shortcuts let ts = TimeStamp::compose; @@ -324,6 +329,23 @@ pub mod tests { }; must_prewrite_put_for_large_txn(&mut engine, k, v, k, ts(10, 0), 100, 0); + must_prewrite_put_impl( + &mut engine, + k2, + v2, + k, + &None, + ts(10, 0), + SkipPessimisticCheck, + 100, + TimeStamp::default(), + 0, + ts(20, 1), + TimeStamp::default(), + false, + Assertion::None, + AssertionLevel::Off, + ); check_txn_status::tests::must_success( &mut engine, k, @@ -367,46 +389,33 @@ pub mod tests { ts(10, 0), ts(15, 0), ts(20, 1), - // Unknown commit role should not collect mvcc + // The primary key should not collect mvcc because it is an expected case. false, ); check_commit_ts_expired_err( must_err( &mut engine, - k, - ts(10, 0), - ts(15, 0), - Some(CommitRole::Primary), - ), - k, - ts(10, 0), - ts(15, 0), - ts(20, 1), - // Primary commit should not collect mvcc because it is an expected case. - false, - ); - check_commit_ts_expired_err( - must_err( - &mut engine, - k, + k2, ts(10, 0), ts(15, 0), Some(CommitRole::Secondary), ), - k, + k2, ts(10, 0), ts(15, 0), ts(20, 1), - // Secondary commit should not collect mvcc because it may be a bug. + // The secondary key should not collect mvcc because it may be a bug. true, ); check_commit_ts_expired_err( - must_err(&mut engine, k, ts(10, 0), ts(20, 0), None), - k, + must_err(&mut engine, k2, ts(10, 0), ts(20, 0), None), + k2, ts(10, 0), ts(20, 0), ts(20, 1), - false, + // The mvcc info should be collected committing secondary keys as it could be a bug. + // Although the commit role is none, the primary key could be read from the lock. + true, ); must_succeed(&mut engine, k, ts(10, 0), ts(20, 1)); From d241ebe469d2c9ce8f84801517414c55f5ce05a2 Mon Sep 17 00:00:00 2001 From: Zequan <110292382+EricZequan@users.noreply.github.com> Date: Tue, 13 May 2025 15:35:41 +0800 Subject: [PATCH 003/110] Revert pr "copr: add simd for vector distance functions" (#18447) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ref tikv/tikv#17290 Signed-off-by: “EricZequan” --- Cargo.lock | 10 - components/tidb_query_datatype/Cargo.toml | 1 - .../benches/bench_vector_distance.rs | 241 ++++++---- .../data_type/chunked_vec_vector_float32.rs | 47 +- .../tidb_query_datatype/src/codec/datum.rs | 4 +- .../src/codec/mysql/vector.rs | 427 +++++++++++------- components/tidb_query_expr/src/impl_vec.rs | 63 +-- 7 files changed, 453 insertions(+), 340 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 09234656c0c..9c2db0f7b04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6301,15 +6301,6 @@ dependencies = [ "libc 0.2.151", ] -[[package]] -name = "simsimd" -version = "6.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56838e1a0aceaef39fc1327227033efa84a294f54a39b356eefa5c60c885e9f0" -dependencies = [ - "cc", -] - [[package]] name = "siphasher" version = "0.3.3" @@ -7256,7 +7247,6 @@ dependencies = [ "regex", "serde", "serde_json", - "simsimd", "slog", "slog-global", "static_assertions", diff --git a/components/tidb_query_datatype/Cargo.toml b/components/tidb_query_datatype/Cargo.toml index 6c0ebcf82fd..ccfb82750c0 100644 --- a/components/tidb_query_datatype/Cargo.toml +++ b/components/tidb_query_datatype/Cargo.toml @@ -35,7 +35,6 @@ protobuf = "2" regex = "1.1" serde = "1.0" serde_json = "1.0" -simsimd = "6.2.3" slog = { workspace = true } slog-global = { workspace = true } static_assertions = { version = "1.0", features = ["nightly"] } diff --git a/components/tidb_query_datatype/benches/bench_vector_distance.rs b/components/tidb_query_datatype/benches/bench_vector_distance.rs index e2169e2df3b..62cf2b2ad98 100644 --- a/components/tidb_query_datatype/benches/bench_vector_distance.rs +++ b/components/tidb_query_datatype/benches/bench_vector_distance.rs @@ -3,108 +3,189 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use tidb_query_datatype::codec::mysql::VectorFloat32Ref; -static TEST_CASES: &[(&str, &[f32])] = &[ - ("3d", &[1.1, 2.2, 3.3]), - ("10d", &[1.0; 10]), - ("100d", &[1.0; 100]), - ("400d", &[1.0; 400]), - ("784d", &[1.0; 784]), - ("1000d", &[1.0; 1000]), - ("4000d", &[1.0; 4000]), -]; - -fn bench_l1_distance(c: &mut Criterion) { - for (label, va) in TEST_CASES { - let vec_va = VectorFloat32Ref::from_f32(va); - let vec_vb = VectorFloat32Ref::from_f32(va); - - c.bench_function(&format!("l1_distance_{}", label), |b| { - b.iter(|| { - black_box(black_box(vec_va).l1_distance(black_box(vec_vb)).unwrap()); - }); +fn bench_l1_distance_3d(c: &mut Criterion) { + let va: Vec = vec![1.1, 2.2, 3.3]; + let vb: Vec = vec![1.1, 2.2, 3.3]; + let vec_va = VectorFloat32Ref::from_f32(va.as_slice()).unwrap(); + let vec_vb = VectorFloat32Ref::from_f32(vb.as_slice()).unwrap(); + + c.bench_function("l1_distance_3d", |b| { + b.iter(|| { + black_box(black_box(vec_va).l1_distance(black_box(vec_vb)).unwrap()); + }); + }); +} + +fn bench_l1_distance_784d(c: &mut Criterion) { + let va: Vec = vec![1.0; 784]; + let vb: Vec = vec![1.0; 784]; + let vec_va = VectorFloat32Ref::from_f32(va.as_slice()).unwrap(); + let vec_vb = VectorFloat32Ref::from_f32(vb.as_slice()).unwrap(); + + c.bench_function("l1_distance_784d", |b| { + b.iter(|| { + black_box(black_box(vec_va).l1_distance(black_box(vec_vb)).unwrap()); }); - } + }); } -fn bench_l2_squared_distance(c: &mut Criterion) { - for (label, va) in TEST_CASES { - let vec_va = VectorFloat32Ref::from_f32(va); - let vec_vb = VectorFloat32Ref::from_f32(va); - - c.bench_function(&format!("l2_squared_distance_{}", label), |b| { - b.iter(|| { - black_box( - black_box(vec_va) - .l2_squared_distance(black_box(vec_vb)) - .unwrap(), - ); - }); +fn bench_l2_squared_distance_3d(c: &mut Criterion) { + let va: Vec = vec![1.1, 2.2, 3.3]; + let vb: Vec = vec![1.1, 2.2, 3.3]; + let vec_va = VectorFloat32Ref::from_f32(va.as_slice()).unwrap(); + let vec_vb = VectorFloat32Ref::from_f32(vb.as_slice()).unwrap(); + + c.bench_function("l2_squared_distance_3d", |b| { + b.iter(|| { + black_box( + black_box(vec_va) + .l2_squared_distance(black_box(vec_vb)) + .unwrap(), + ); }); - } + }); } -fn bench_l2_distance(c: &mut Criterion) { - for (label, va) in TEST_CASES { - let vec_va = VectorFloat32Ref::from_f32(va); - let vec_vb = VectorFloat32Ref::from_f32(va); +fn bench_l2_squared_distance_784d(c: &mut Criterion) { + let va: Vec = vec![1.0; 784]; + let vb: Vec = vec![1.0; 784]; + let vec_va = VectorFloat32Ref::from_f32(va.as_slice()).unwrap(); + let vec_vb = VectorFloat32Ref::from_f32(vb.as_slice()).unwrap(); - c.bench_function(&format!("bench_l2_distance_{}", label), |b| { - b.iter(|| { - black_box(black_box(vec_va).l2_distance(black_box(vec_vb)).unwrap()); - }); + c.bench_function("l2_squared_distance_784d", |b| { + b.iter(|| { + black_box( + black_box(vec_va) + .l2_squared_distance(black_box(vec_vb)) + .unwrap(), + ); }); - } + }); } -fn bench_inner_product(c: &mut Criterion) { - for (label, va) in TEST_CASES { - let vec_va = VectorFloat32Ref::from_f32(va); - let vec_vb = VectorFloat32Ref::from_f32(va); +fn bench_l2_distance_3d(c: &mut Criterion) { + let va: Vec = vec![1.1, 2.2, 3.3]; + let vb: Vec = vec![1.1, 2.2, 3.3]; + let vec_va = VectorFloat32Ref::from_f32(va.as_slice()).unwrap(); + let vec_vb = VectorFloat32Ref::from_f32(vb.as_slice()).unwrap(); - c.bench_function(&format!("bench_inner_product_{}", label), |b| { - b.iter(|| { - black_box(black_box(vec_va).inner_product(black_box(vec_vb)).unwrap()); - }); + c.bench_function("l2_distance_3d", |b| { + b.iter(|| { + black_box(black_box(vec_va).l2_distance(black_box(vec_vb)).unwrap()); }); - } + }); } -fn bench_cosine_distance(c: &mut Criterion) { - for (label, va) in TEST_CASES { - let vec_va = VectorFloat32Ref::from_f32(va); - let vec_vb = VectorFloat32Ref::from_f32(va); - - c.bench_function(&format!("bench_cosine_distance_{}", label), |b| { - b.iter(|| { - black_box( - black_box(vec_va) - .cosine_distance(black_box(vec_vb)) - .unwrap(), - ); - }); +fn bench_l2_distance_784d(c: &mut Criterion) { + let va: Vec = vec![1.0; 784]; + let vb: Vec = vec![1.0; 784]; + let vec_va = VectorFloat32Ref::from_f32(va.as_slice()).unwrap(); + let vec_vb = VectorFloat32Ref::from_f32(vb.as_slice()).unwrap(); + + c.bench_function("l2_distance_784d", |b| { + b.iter(|| { + black_box(black_box(vec_va).l2_distance(black_box(vec_vb)).unwrap()); + }); + }); +} + +fn bench_inner_product_3d(c: &mut Criterion) { + let va: Vec = vec![1.1, 2.2, 3.3]; + let vb: Vec = vec![1.1, 2.2, 3.3]; + let vec_va = VectorFloat32Ref::from_f32(va.as_slice()).unwrap(); + let vec_vb = VectorFloat32Ref::from_f32(vb.as_slice()).unwrap(); + + c.bench_function("inner_product_3d", |b| { + b.iter(|| { + black_box(black_box(vec_va).inner_product(black_box(vec_vb)).unwrap()); + }); + }); +} + +fn bench_inner_product_784d(c: &mut Criterion) { + let va: Vec = vec![1.0; 784]; + let vb: Vec = vec![1.0; 784]; + let vec_va = VectorFloat32Ref::from_f32(va.as_slice()).unwrap(); + let vec_vb = VectorFloat32Ref::from_f32(vb.as_slice()).unwrap(); + + c.bench_function("inner_product_784d", |b| { + b.iter(|| { + black_box(black_box(vec_va).inner_product(black_box(vec_vb)).unwrap()); }); - } + }); } -fn bench_l2_norm(c: &mut Criterion) { - for (label, va) in TEST_CASES { - let vec_va = VectorFloat32Ref::from_f32(va); +fn bench_cosine_distance_3d(c: &mut Criterion) { + let va: Vec = vec![1.1, 2.2, 3.3]; + let vb: Vec = vec![1.1, 2.2, 3.3]; + let vec_va = VectorFloat32Ref::from_f32(va.as_slice()).unwrap(); + let vec_vb = VectorFloat32Ref::from_f32(vb.as_slice()).unwrap(); + + c.bench_function("cosine_distance_3d", |b| { + b.iter(|| { + black_box( + black_box(vec_va) + .cosine_distance(black_box(vec_vb)) + .unwrap(), + ); + }); + }); +} + +fn bench_cosine_distance_784d(c: &mut Criterion) { + let va: Vec = vec![1.0; 784]; + let vb: Vec = vec![1.0; 784]; + let vec_va = VectorFloat32Ref::from_f32(va.as_slice()).unwrap(); + let vec_vb = VectorFloat32Ref::from_f32(vb.as_slice()).unwrap(); + + c.bench_function("cosine_distance_784d", |b| { + b.iter(|| { + black_box( + black_box(vec_va) + .cosine_distance(black_box(vec_vb)) + .unwrap(), + ); + }); + }); +} + +fn bench_l2_norm_3d(c: &mut Criterion) { + let va: Vec = vec![1.1, 2.2, 3.3]; + + let vec_va = VectorFloat32Ref::from_f32(va.as_slice()).unwrap(); + + c.bench_function("l2_norm_3d", |b| { + b.iter(|| { + black_box(black_box(vec_va).l2_norm()); + }); + }); +} + +fn bench_l2_norm_784d(c: &mut Criterion) { + let va: Vec = vec![1.0; 784]; + + let vec_va = VectorFloat32Ref::from_f32(va.as_slice()).unwrap(); - c.bench_function(&format!("bench_l2_norm_{}", label), |b| { - b.iter(|| { - black_box(black_box(vec_va).l2_norm()).unwrap(); - }); + c.bench_function("l2_norm_784d", |b| { + b.iter(|| { + black_box(black_box(vec_va).l2_norm()); }); - } + }); } criterion_group!( benches, - bench_l1_distance, - bench_l2_squared_distance, - bench_l2_distance, - bench_inner_product, - bench_cosine_distance, - bench_l2_norm, + bench_l1_distance_3d, + bench_l1_distance_784d, + bench_l2_squared_distance_3d, + bench_l2_squared_distance_784d, + bench_l2_distance_3d, + bench_l2_distance_784d, + bench_inner_product_3d, + bench_inner_product_784d, + bench_cosine_distance_3d, + bench_cosine_distance_784d, + bench_l2_norm_3d, + bench_l2_norm_784d, ); criterion_main!(benches); diff --git a/components/tidb_query_datatype/src/codec/data_type/chunked_vec_vector_float32.rs b/components/tidb_query_datatype/src/codec/data_type/chunked_vec_vector_float32.rs index 93886107e75..d065a8ec0ff 100644 --- a/components/tidb_query_datatype/src/codec/data_type/chunked_vec_vector_float32.rs +++ b/components/tidb_query_datatype/src/codec/data_type/chunked_vec_vector_float32.rs @@ -3,7 +3,10 @@ use super::{ bit_vec::BitVec, ChunkRef, ChunkedVec, UnsafeRefInto, VectorFloat32, VectorFloat32Ref, }; -use crate::impl_chunked_vec_common; +use crate::{ + codec::mysql::{VectorFloat32Decoder, VectorFloat32Encoder}, + impl_chunked_vec_common, +}; /// A vector storing `Option` with a compact layout. /// @@ -15,7 +18,7 @@ use crate::impl_chunked_vec_common; /// each element. #[derive(Debug, PartialEq, Clone)] pub struct ChunkedVecVectorFloat32 { - data: Vec, // Only contains the data part, without any length prefix + data: Vec, bitmap: BitVec, length: usize, var_offset: Vec, @@ -26,8 +29,12 @@ impl ChunkedVecVectorFloat32 { pub fn get(&self, idx: usize) -> Option> { assert!(idx < self.len()); if self.bitmap.get(idx) { - let sliced_data = &self.data[self.var_offset[idx]..self.var_offset[idx + 1]]; - Some(VectorFloat32Ref::from_f32(sliced_data)) + let mut sliced_data = &self.data[self.var_offset[idx]..self.var_offset[idx + 1]]; + let v: VectorFloat32Ref<'_> = sliced_data.read_vector_float32_ref().unwrap(); + unsafe { + let v_with_static_lifetime = v.unsafe_into(); + Some(v_with_static_lifetime) + } } else { None } @@ -49,7 +56,7 @@ impl ChunkedVec for ChunkedVecVectorFloat32 { #[inline] fn push_data(&mut self, value: VectorFloat32) { self.bitmap.push(true); - self.data.extend_from_slice(value.as_ref().data()); + self.data.write_vector_float32(value.as_ref()).unwrap(); self.var_offset.push(self.data.len()); self.length += 1; } @@ -126,33 +133,3 @@ impl<'a> UnsafeRefInto<&'static ChunkedVecVectorFloat32> for &'a ChunkedVecVecto std::mem::transmute(self) } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_push_data_and_get() { - let mut chunked_vec = ChunkedVecVectorFloat32::with_capacity(5); - - chunked_vec.push_data(VectorFloat32::copy_from_f32(&[1.1, 2.2, 3.3])); - chunked_vec.push_data(VectorFloat32::copy_from_f32(&[4.4, 5.5])); - // push a null value - chunked_vec.push_null(); - chunked_vec.push_data(VectorFloat32::copy_from_f32(&[6.6, 7.7, 8.8, 9.9])); - - assert_eq!(chunked_vec.len(), 4); - - let vector1 = chunked_vec.get(0).unwrap().to_string(); - assert_eq!(vector1, "[1.1,2.2,3.3]"); - - let vector2 = chunked_vec.get(1).unwrap().to_string(); - assert_eq!(vector2, "[4.4,5.5]"); - - // check if null value is right - assert!(chunked_vec.get(2).is_none()); - - let vector3 = chunked_vec.get(3).unwrap().to_string(); - assert_eq!(vector3, "[6.6,7.7,8.8,9.9]"); - } -} diff --git a/components/tidb_query_datatype/src/codec/datum.rs b/components/tidb_query_datatype/src/codec/datum.rs index 3b6520b8e66..9da537a2c09 100644 --- a/components/tidb_query_datatype/src/codec/datum.rs +++ b/components/tidb_query_datatype/src/codec/datum.rs @@ -1267,8 +1267,8 @@ mod tests { ), ], vec![ - Datum::VectorFloat32(VectorFloat32::copy_from_f32(&[1.0, 2.0, 3.0])), - Datum::VectorFloat32(VectorFloat32::copy_from_f32(&[])), + Datum::VectorFloat32(VectorFloat32::from_f32(vec![1.0, 2.0, 3.0]).unwrap()), + Datum::VectorFloat32(VectorFloat32::from_f32(vec![]).unwrap()), ], ]; for vs in table { diff --git a/components/tidb_query_datatype/src/codec/mysql/vector.rs b/components/tidb_query_datatype/src/codec/mysql/vector.rs index 9ed8209b39e..039638be792 100644 --- a/components/tidb_query_datatype/src/codec/mysql/vector.rs +++ b/components/tidb_query_datatype/src/codec/mysql/vector.rs @@ -1,31 +1,44 @@ // Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0. +use std::cmp::Ordering; + use codec::prelude::*; -use ordered_float::OrderedFloat; -use simsimd::SpatialSimilarity; -use static_assertions::assert_eq_size; use crate::codec::Result; -const LEN_PREFIX_SIZE: usize = std::mem::size_of::(); -const ELEMENT_SIZE: usize = std::mem::size_of::(); +const F32_SIZE: usize = std::mem::size_of::(); -/// Aligned, Owned. -#[derive(Clone, Hash, Eq, PartialEq, Ord, PartialOrd)] +// TODO: Implement generic version +#[derive(Clone, Eq, Hash, PartialEq)] pub struct VectorFloat32 { - pub value: Vec>, // Data must be aligned + // Use Vec instead of Vec to avoid reading from unaligned bytes. For example, + // bytes read from protobuf is usually not aligned by f32. + pub value: Vec, +} + +impl Ord for VectorFloat32 { + fn cmp(&self, other: &Self) -> Ordering { + self.as_ref().cmp(&other.as_ref()) + } +} + +impl PartialOrd for VectorFloat32 { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } } impl VectorFloat32 { - pub fn copy_from_f32(value: &[f32]) -> Self { - let ordered_value: Vec> = - value.iter().map(|v| OrderedFloat(*v)).collect(); - VectorFloat32 { - value: ordered_value, - } + pub fn new(value: Vec) -> Result { + _ = VectorFloat32Ref::new(&value[..])?; + Ok(VectorFloat32 { value }) + } + + pub fn from_f32(value: Vec) -> Result { + let value_u8: &[u8] = bytemuck::cast_slice(&value); + VectorFloat32::new(value_u8.to_owned()) } - #[inline] pub fn as_ref(&self) -> VectorFloat32Ref<'_> { VectorFloat32Ref { value: &self.value } } @@ -43,58 +56,110 @@ impl std::fmt::Debug for VectorFloat32 { } } -/// Aligned, Ref. -#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub struct VectorFloat32Ref<'a> { - value: &'a [OrderedFloat], // Data must be aligned +pub trait VectorFloat32DatumPayloadChunkEncoder: BufferWriter { + fn write_vector_float32_to_chunk_by_datum_payload(&mut self, src_payload: &[u8]) -> Result<()> { + // VectorFloat32's chunk format is same as binary format. + self.write_bytes(src_payload)?; + Ok(()) + } +} + +impl VectorFloat32DatumPayloadChunkEncoder for T {} + +impl VectorFloat32Encoder for T {} + +pub trait VectorFloat32Decoder: NumberDecoder { + // `read_vector_float32_ref` decodes value encoded by `write_vector_float32` + // before. + fn read_vector_float32_ref(&mut self) -> Result> { + if !cfg!(target_endian = "little") { + return Err(box_err!("VectorFloat32 only support Little Endian")); + } + + if self.bytes().is_empty() { + return VectorFloat32Ref::new(&[]); + } + let n = self.read_u32_le()? as usize; + let data_size = n * F32_SIZE; + let data = self.read_bytes(data_size)?; + VectorFloat32Ref::new(data) + } + + // `read_vector_float32` decodes value encoded by `write_vector_float32` before. + fn read_vector_float32(&mut self) -> Result { + let r = self.read_vector_float32_ref()?; + Ok(r.to_owned()) + } } -impl<'a> std::ops::Index for VectorFloat32Ref<'a> { - type Output = f32; +impl VectorFloat32Decoder for T {} + +/// Represents a reference of VectorFloat32 value aiming to reduce memory copy. +#[derive(Clone, Copy, Eq, PartialEq, Hash)] +pub struct VectorFloat32Ref<'a> { + // Use &[u8] instead of &[u32] to allow reading from unaligned bytes. For example, + // bytes read from protobuf is usually not aligned by f32. + value: &'a [u8], +} - #[inline] - fn index(&self, index: usize) -> &Self::Output { - &self.value[index].0 +impl<'a> Ord for VectorFloat32Ref<'a> { + fn cmp(&self, other: &Self) -> Ordering { + let la = self.len(); + let lb = other.len(); + let common_len = std::cmp::min(la, lb); + for i in 0..common_len { + if self.index(i) > other.index(i) { + return Ordering::Greater; + } else if self.index(i) < other.index(i) { + return Ordering::Less; + } + } + la.cmp(&lb) } } -assert_eq_size!(OrderedFloat, f32); +impl<'a> PartialOrd for VectorFloat32Ref<'a> { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} impl<'a> VectorFloat32Ref<'a> { - pub fn from_f32(value: &[f32]) -> VectorFloat32Ref<'_> { - // OrderedFloat is POD, so it is safe. - let ordered_value = unsafe { - std::slice::from_raw_parts(value.as_ptr() as *const OrderedFloat, value.len()) - }; - VectorFloat32Ref { - value: ordered_value, + pub fn new(value: &[u8]) -> Result> { + if value.len() % F32_SIZE != 0 { + return Err(box_err!("Vector length error. Please check the input.")); + } + let check_vec = VectorFloat32Ref { value }; + for i in 0..check_vec.len() { + if check_vec.index(i).is_nan() { + return Err(box_err!("NaN not allowed in vector")); + } + if check_vec.index(i).is_infinite() { + return Err(box_err!("infinite value not allowed in vector")); + } } + Ok(check_vec) } - #[inline] - pub fn data(&self) -> &[f32] { - // OrderedFloat is POD, so it is safe. - unsafe { std::slice::from_raw_parts(self.value.as_ptr() as *const f32, self.value.len()) } + pub fn from_f32(value: &[f32]) -> Result> { + let vec_u8: &[u8] = bytemuck::cast_slice(value); + VectorFloat32Ref::new(vec_u8) } - #[inline] pub fn encoded_len(&self) -> usize { - self.value.len() * ELEMENT_SIZE + LEN_PREFIX_SIZE + self.value.len() + std::mem::size_of::() } - #[inline] pub fn to_owned(&self) -> VectorFloat32 { VectorFloat32 { value: self.value.to_owned(), } } - #[inline] pub fn len(&self) -> usize { - self.value.len() + self.value.len() / F32_SIZE } - #[inline] pub fn is_empty(&self) -> bool { self.len() == 0 } @@ -109,211 +174,233 @@ impl<'a> VectorFloat32Ref<'a> { } Ok(()) } -} -impl std::fmt::Display for VectorFloat32Ref<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "[")?; - for i in 0..self.len() { - if i == 0 { - write!(f, "{}", self[i])?; - } else { - write!(f, ",{}", self[i])?; - } + fn index(&self, idx: usize) -> f32 { + if idx > self.len() { + panic!( + "Index out of bounds: index = {}, length = {}", + idx, + self.len() + ); + } + let byte_index: usize = idx * F32_SIZE; + unsafe { + let float_ptr = self.value.as_ptr().add(byte_index) as *const f32; + float_ptr.read_unaligned() } - write!(f, "]")?; - Ok(()) - } -} - -impl std::fmt::Debug for VectorFloat32Ref<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self) } -} -impl ToString for VectorFloat32Ref<'_> { - fn to_string(&self) -> String { - format!("{}", self) + // An unsafe function to get the 'f32' value without boundary check. + // it will check the bounding in debug model and remove the check in + // release. + unsafe fn index_unchecked(&self, idx: usize) -> f32 { + #[cfg(debug_assertions)] + { + if idx > self.len() { + panic!( + "Index out of bounds: index = {}, length = {}", + idx, + self.len() + ); + } + } + let byte_index: usize = idx * 4; + let float_ptr = self.value.as_ptr().add(byte_index) as *const f32; + float_ptr.read_unaligned() } -} -// Vector distance and functions -impl<'a> VectorFloat32Ref<'a> { - #[inline] pub fn l2_squared_distance(&self, b: VectorFloat32Ref<'a>) -> Result { - match SpatialSimilarity::sqeuclidean(self.data(), b.data()) { - Some(l2_distance) => Ok(l2_distance), - None => Err(box_err!("Vectors must be of the same length")), + self.check_dims(b)?; + let mut distance: f32 = 0.0; + + for i in 0..self.len() { + let diff = unsafe { self.index_unchecked(i) - b.index_unchecked(i) }; + distance += diff * diff; } + + Ok(distance as f64) } - #[inline] pub fn l2_distance(&self, b: VectorFloat32Ref<'a>) -> Result { Ok(self.l2_squared_distance(b)?.sqrt()) } - #[inline] pub fn inner_product(&self, b: VectorFloat32Ref<'a>) -> Result { - match SpatialSimilarity::dot(self.data(), b.data()) { - Some(inner_product) => Ok(inner_product), - None => Err(box_err!("Vectors must be of the same length")), + self.check_dims(b)?; + let mut distance: f32 = 0.0; + for i in 0..self.len() { + distance += unsafe { self.index_unchecked(i) * b.index_unchecked(i) }; } + + Ok(distance as f64) } - #[inline] pub fn cosine_distance(&self, b: VectorFloat32Ref<'a>) -> Result { - match SpatialSimilarity::cosine(self.data(), b.data()) { - Some(cosine_similarity) => Ok(cosine_similarity), - None => Err(box_err!("Vectors must be of the same length")), + self.check_dims(b)?; + let mut distance: f32 = 0.0; + let mut norma: f32 = 0.0; + let mut normb: f32 = 0.0; + for i in 0..self.len() { + unsafe { + distance += self.index_unchecked(i) * b.index_unchecked(i); + norma += self.index_unchecked(i) * self.index_unchecked(i); + normb += b.index_unchecked(i) * b.index_unchecked(i); + } + } + + let similarity = (distance as f64) / ((norma as f64) * (normb as f64)).sqrt(); + if similarity.is_nan() { + // Divide by zero + return Ok(std::f64::NAN); } + let similarity = similarity.clamp(-1.0, 1.0); + Ok(1.0 - similarity) } pub fn l1_distance(&self, b: VectorFloat32Ref<'a>) -> Result { self.check_dims(b)?; let mut distance: f32 = 0.0; for i in 0..self.len() { - let diff = self[i] - b[i]; + let diff = unsafe { self.index_unchecked(i) - b.index_unchecked(i) }; distance += diff.abs(); } Ok(distance as f64) } - #[inline] - pub fn l2_norm(&self) -> Result { + pub fn l2_norm(&self) -> f64 { // Note: We align the impl with pgvector: Only l2_norm use double // precision during calculation. - match SpatialSimilarity::dot(self.data(), self.data()) { - Some(norm) => Ok(norm.sqrt()), - None => Err(box_err!("Vectors must be of the same length")), + let mut norm: f64 = 0.0; + for i in 0..self.len() { + let v = unsafe { self.index_unchecked(i) as f64 }; + norm += v * v; } - } -} -pub trait VectorFloat32DatumPayloadChunkEncoder: BufferWriter { - fn write_vector_float32_to_chunk_by_datum_payload(&mut self, src_payload: &[u8]) -> Result<()> { - // VectorFloat32's chunk format is same as binary format. - self.write_bytes(src_payload)?; - Ok(()) + norm.sqrt() } } -impl VectorFloat32DatumPayloadChunkEncoder for T {} - -impl VectorFloat32Encoder for T {} - -/// Unaligned, Ref. -#[derive(Clone, Copy, Debug)] -pub struct VectorFloat32RefUnaligned<'a> { - value: &'a [u8], // Data could be notaligned. Does not contain length prefix. -} - -impl<'a> VectorFloat32RefUnaligned<'a> { - #[inline] - pub fn len(&self) -> usize { - self.value.len() / ELEMENT_SIZE - } +pub trait VectorFloat32Encoder: NumberEncoder { + fn write_vector_float32(&mut self, data: VectorFloat32Ref<'_>) -> Result<()> { + if !cfg!(target_endian = "little") { + return Err(box_err!("VectorFloat32 only support Little Endian")); + } - #[inline] - pub fn is_empty(&self) -> bool { - self.value.len() == 0 - } + self.write_u32_le(data.len() as u32)?; + self.write_bytes(data.value)?; - /// It's safe to convert a unaligned ref to an aligned owned value. We will - /// copy data with proper alignment. - pub fn to_owned(&self) -> VectorFloat32 { - let mut aligned_data: Vec> = Vec::new(); - aligned_data.resize(self.len(), OrderedFloat(0.0f32)); - unsafe { - std::ptr::copy_nonoverlapping( - self.value.as_ptr(), // Aligned by u8 - aligned_data.as_mut_ptr().cast(), // Aligned by f32 → u8 - self.value.len(), - ) - } - VectorFloat32 { - value: aligned_data, - } + Ok(()) } } -pub trait VectorFloat32Decoder: NumberDecoder { - // `read_vector_float32_ref` decodes value encoded by `write_vector_float32` - // before. - fn read_vector_float32_ref(&mut self) -> Result> { - if !cfg!(target_endian = "little") { - return Err(box_err!("VectorFloat32 only support Little Endian")); - } - - if self.bytes().is_empty() { - return Ok(VectorFloat32RefUnaligned { value: &[] }); +impl std::fmt::Display for VectorFloat32Ref<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[")?; + let mut is_first = true; + for i in 0..self.len() { + if is_first { + write!(f, "{}", self.index(i))?; + is_first = false; + } else { + write!(f, ",{}", self.index(i))?; + } } - let n = self.read_u32_le()? as usize; - let data_size = n * ELEMENT_SIZE; - let data = self.read_bytes(data_size)?; - Ok(VectorFloat32RefUnaligned { value: data }) + write!(f, "]")?; + Ok(()) } +} - // `read_vector_float32` decodes value encoded by `write_vector_float32` before. - fn read_vector_float32(&mut self) -> Result { - let r = self.read_vector_float32_ref()?; - Ok(r.to_owned()) +impl std::fmt::Debug for VectorFloat32Ref<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self) } } -impl VectorFloat32Decoder for T {} - -pub trait VectorFloat32Encoder: NumberEncoder { - fn write_vector_float32(&mut self, data: VectorFloat32Ref<'_>) -> Result<()> { - if !cfg!(target_endian = "little") { - return Err(box_err!("VectorFloat32 only support Little Endian")); - } - self.write_u32_le(data.len() as u32)?; - self.write_bytes(bytemuck::cast_slice(data.data()))?; - Ok(()) +impl ToString for VectorFloat32Ref<'_> { + fn to_string(&self) -> String { + format!("{}", self) } } #[cfg(test)] mod tests { - use super::*; + #[test] + fn test_nan_inf() { + let v = VectorFloat32::from_f32(vec![1.0, std::f32::NAN]); + v.unwrap_err(); + + let v = VectorFloat32::from_f32(vec![1.0, std::f32::INFINITY]); + v.unwrap_err(); + + let v = VectorFloat32::from_f32(vec![1.0, std::f32::NEG_INFINITY]); + v.unwrap_err(); + + let v = VectorFloat32Ref::from_f32(&[1.0, std::f32::NAN]); + v.unwrap_err(); + + let v = VectorFloat32Ref::from_f32(&[1.0, std::f32::INFINITY]); + v.unwrap_err(); + + let v = VectorFloat32Ref::from_f32(&[1.0, std::f32::NEG_INFINITY]); + v.unwrap_err(); + } + #[test] fn test_to_string() { - let v = VectorFloat32::copy_from_f32(&[1.0, 2.0]); + let v = VectorFloat32::from_f32(vec![1.0, 2.0]).unwrap(); assert_eq!("[1,2]", v.to_string()); - let v = VectorFloat32::copy_from_f32(&[1.1, 2.2]); + let v = VectorFloat32::from_f32(vec![1.1, 2.2]).unwrap(); assert_eq!("[1.1,2.2]", v.to_string()); - let v = VectorFloat32::copy_from_f32(&[]); + let v = VectorFloat32::from_f32(vec![]).unwrap(); assert_eq!("[]", v.to_string()); } + #[test] + fn test_input_length() { + let buf: Vec = vec![ + 0xcd, 0xcc, 0x8c, 0x3f, // Element 1 = 0x3f8ccccd + 0xcd, 0xcc, 0x0c, 0x40, // Element 2 = 0x400ccccd + 0xcd, 0xcc, 0x0c, + ]; + let v = VectorFloat32Ref::new(&buf[..]); + v.unwrap_err(); + + let buf: Vec = vec![ + 0xcd, 0xcc, 0x8c, 0x3f, // Element 1 = 0x3f8ccccd + 0xcd, 0xcc, 0x0c, 0x40, // Element 2 = 0x400ccccd + 0xcd, 0xcc, 0x0c, 0x40, + ]; + let v = VectorFloat32Ref::new(&buf[..]); + v.unwrap(); + } + #[test] fn test_compare() { - let v1 = VectorFloat32::copy_from_f32(&[1.0, 2.0]); - let v2 = VectorFloat32::copy_from_f32(&[1.1, 2.2]); + let v1 = VectorFloat32::from_f32(vec![1.0, 2.0]).unwrap(); + let v2 = VectorFloat32::from_f32(vec![1.1, 2.2]).unwrap(); assert!(v1 < v2); - let v3 = VectorFloat32::copy_from_f32(&[1.0, 2.0]); + let v3 = VectorFloat32::from_f32(vec![1.0, 2.0]).unwrap(); assert!(v1 == v3); - let v4 = VectorFloat32::copy_from_f32(&[0.3, 0.4]); + let v4 = VectorFloat32::from_f32(vec![0.3, 0.4]).unwrap(); assert!(v1 > v4); - let v4 = VectorFloat32::copy_from_f32(&[1.0]); + let v4 = VectorFloat32::from_f32(vec![1.0]).unwrap(); assert!(v1 > v4); - let v5 = VectorFloat32::copy_from_f32(&[1.0, 2.0, 0.5]); + let v5 = VectorFloat32::from_f32(vec![1.0, 2.0, 0.5]).unwrap(); assert!(v1 < v5); } #[test] fn test_encode() { - let v = VectorFloat32::copy_from_f32(&[1.1, 2.2]); + let v = VectorFloat32::from_f32(vec![1.1, 2.2]).unwrap(); let mut encoded = Vec::new(); encoded.write_vector_float32(v.as_ref()).unwrap(); @@ -342,7 +429,7 @@ mod tests { let v = buf_slice.read_vector_float32_ref().unwrap(); assert_eq!(v.len(), 2); - assert_eq!(v.to_owned().as_ref().to_string(), "[1.1,2.2]"); + assert_eq!(v.to_string(), "[1.1,2.2]"); assert_eq!(buf_slice.len(), 1); assert_eq!(buf_slice, &[0xff]); @@ -354,11 +441,9 @@ mod tests { buf_slice = &[]; let v = buf_slice.read_vector_float32_ref().unwrap(); assert_eq!(v.len(), 0); - assert_eq!(v.to_owned().as_ref().to_string(), "[]"); + assert_eq!(v.to_string(), "[]"); let mut encode_buf = Vec::new(); - encode_buf - .write_vector_float32(v.to_owned().as_ref()) - .unwrap(); + encode_buf.write_vector_float32(v).unwrap(); assert_eq!(encode_buf, vec![0x00, 0x00, 0x00, 0x00]); } } diff --git a/components/tidb_query_expr/src/impl_vec.rs b/components/tidb_query_expr/src/impl_vec.rs index 1c7dcd3d43b..1ebe423e2ae 100644 --- a/components/tidb_query_expr/src/impl_vec.rs +++ b/components/tidb_query_expr/src/impl_vec.rs @@ -48,14 +48,11 @@ fn vec_cosine_distance(a: VectorFloat32Ref, b: VectorFloat32Ref) -> Result Result> { // TiKV does not support NaN. This turns NaN into null - Ok(Real::new(a.l2_norm()?).ok()) + Ok(Real::new(a.l2_norm()).ok()) } -// note: The distance calculated by the simd instruction will -// be different depending on the platform. The following is based on x86. #[cfg(test)] mod tests { - use tipb::ScalarFuncSig; use super::*; @@ -72,7 +69,7 @@ mod tests { (vec![1.0, 2.0, 3.0], Some(3)), ]; for (arg, expected_output) in cases { - let arg = VectorFloat32::copy_from_f32(&arg); + let arg = VectorFloat32::from_f32(arg).unwrap(); let output: Option = RpnFnScalarEvaluator::new() .push_param(arg) .evaluate(ScalarFuncSig::VecDimsSig) @@ -90,7 +87,7 @@ mod tests { ]; for (arg, expected_output) in cases { - let arg = VectorFloat32::copy_from_f32(&arg); + let arg = VectorFloat32::from_f32(arg).unwrap(); let output: Option = RpnFnScalarEvaluator::new() .push_param(arg) .evaluate(ScalarFuncSig::VecL2NormSig) @@ -108,8 +105,8 @@ mod tests { (Some(vec![1.0, 2.0]), None, None), ]; for (arg1, arg2, expected_output) in ok_cases { - let arg1 = arg1.map(|v| VectorFloat32::copy_from_f32(v.as_slice())); - let arg2 = arg2.map(|v| VectorFloat32::copy_from_f32(v.as_slice())); + let arg1 = arg1.map(|v| VectorFloat32::from_f32(v).unwrap()); + let arg2 = arg2.map(|v| VectorFloat32::from_f32(v).unwrap()); let output: Option = RpnFnScalarEvaluator::new() .push_param(arg1) .push_param(arg2) @@ -120,8 +117,8 @@ mod tests { let err_cases = vec![(vec![1.0, 2.0], vec![3.0])]; for (arg1, arg2) in err_cases { - let arg1 = VectorFloat32::copy_from_f32(&arg1); - let arg2 = VectorFloat32::copy_from_f32(&arg2); + let arg1 = VectorFloat32::from_f32(arg1).unwrap(); + let arg2 = VectorFloat32::from_f32(arg2).unwrap(); let output: Result> = RpnFnScalarEvaluator::new() .push_param(arg1) .push_param(arg2) @@ -138,8 +135,8 @@ mod tests { (Some(vec![1.0, 2.0]), None, None), ]; for (arg1, arg2, expected_output) in ok_cases { - let arg1 = arg1.map(|v| VectorFloat32::copy_from_f32(v.as_slice())); - let arg2 = arg2.map(|v| VectorFloat32::copy_from_f32(v.as_slice())); + let arg1 = arg1.map(|v| VectorFloat32::from_f32(v).unwrap()); + let arg2 = arg2.map(|v| VectorFloat32::from_f32(v).unwrap()); let output: Option = RpnFnScalarEvaluator::new() .push_param(arg1) .push_param(arg2) @@ -150,8 +147,8 @@ mod tests { let err_cases = vec![(vec![1.0, 2.0], vec![3.0])]; for (arg1, arg2) in err_cases { - let arg1 = VectorFloat32::copy_from_f32(&arg1); - let arg2 = VectorFloat32::copy_from_f32(&arg2); + let arg1 = VectorFloat32::from_f32(arg1).unwrap(); + let arg2 = VectorFloat32::from_f32(arg2).unwrap(); let output: Result> = RpnFnScalarEvaluator::new() .push_param(arg1) .push_param(arg2) @@ -164,46 +161,30 @@ mod tests { fn test_cosine_distance() { let ok_cases = vec![ (Some(vec![1.0, 2.0]), Some(vec![2.0, 4.0]), Some(0.0)), - (Some(vec![1.0, 2.0]), Some(vec![0.0, 0.0]), Some(1.0)), + (Some(vec![1.0, 2.0]), Some(vec![0.0, 0.0]), None), // NaN turns to NULL (Some(vec![1.0, 1.0]), Some(vec![1.0, 1.0]), Some(0.0)), (Some(vec![1.0, 0.0]), Some(vec![0.0, 2.0]), Some(1.0)), (Some(vec![1.0, 1.0]), Some(vec![-1.0, -1.0]), Some(2.0)), (Some(vec![1.0, 1.0]), Some(vec![1.1, 1.1]), Some(0.0)), (Some(vec![1.0, 1.0]), Some(vec![-1.1, -1.1]), Some(2.0)), - (Some(vec![3e38]), Some(vec![3e38]), Some(0.0)), + (Some(vec![3e38]), Some(vec![3e38]), None), // NaN turns to NULL (Some(vec![1.0, 2.0]), None, None), ]; for (arg1, arg2, expected_output) in ok_cases { - let arg1 = arg1.map(|v| VectorFloat32::copy_from_f32(v.as_slice())); - let arg2 = arg2.map(|v| VectorFloat32::copy_from_f32(v.as_slice())); + let arg1 = arg1.map(|v| VectorFloat32::from_f32(v).unwrap()); + let arg2 = arg2.map(|v| VectorFloat32::from_f32(v).unwrap()); let output: Option = RpnFnScalarEvaluator::new() .push_param(arg1) .push_param(arg2) .evaluate(ScalarFuncSig::VecCosineDistanceSig) .unwrap(); - match (output, expected_output.map(|x| Real::new(x).unwrap())) { - (Some(output_val), Some(expected_val)) => { - let diff = (output_val - expected_val).abs(); - assert!( - diff < 1e-6, - "assertion failed: |{} - {}| = {} > 1e-6", - output_val, - expected_val, - diff - ); - } - (None, None) => {} - _ => panic!( - "Mismatched variants: output {:?}, expected {:?}", - output, expected_output - ), - } + assert_eq!(output, expected_output.map(|x| Real::new(x).unwrap())); } let err_cases = vec![(vec![1.0, 2.0], vec![3.0])]; for (arg1, arg2) in err_cases { - let arg1 = VectorFloat32::copy_from_f32(&arg1); - let arg2 = VectorFloat32::copy_from_f32(&arg2); + let arg1 = VectorFloat32::from_f32(arg1).unwrap(); + let arg2 = VectorFloat32::from_f32(arg2).unwrap(); let output: Result> = RpnFnScalarEvaluator::new() .push_param(arg1) .push_param(arg2) @@ -221,8 +202,8 @@ mod tests { (Some(vec![1.0, 2.0]), None, None), ]; for (arg1, arg2, expected_output) in ok_cases { - let arg1 = arg1.map(|v| VectorFloat32::copy_from_f32(v.as_slice())); - let arg2 = arg2.map(|v| VectorFloat32::copy_from_f32(v.as_slice())); + let arg1 = arg1.map(|v| VectorFloat32::from_f32(v).unwrap()); + let arg2 = arg2.map(|v| VectorFloat32::from_f32(v).unwrap()); let output: Option = RpnFnScalarEvaluator::new() .push_param(arg1) .push_param(arg2) @@ -233,8 +214,8 @@ mod tests { let err_cases = vec![(vec![1.0, 2.0], vec![3.0])]; for (arg1, arg2) in err_cases { - let arg1 = VectorFloat32::copy_from_f32(&arg1); - let arg2 = VectorFloat32::copy_from_f32(&arg2); + let arg1 = VectorFloat32::from_f32(arg1).unwrap(); + let arg2 = VectorFloat32::from_f32(arg2).unwrap(); let output: Result> = RpnFnScalarEvaluator::new() .push_param(arg1) .push_param(arg2) From 079d2a92ae594bc933e35de29b7485478f8fb132 Mon Sep 17 00:00:00 2001 From: glorv Date: Thu, 15 May 2025 10:45:59 +0800 Subject: [PATCH 004/110] *: Update rust toolchain to nightly-2025-04-03 (#18416) ref tikv/tikv#17465 Following #17605, another attempt to update rust toolchain Changes: - Language - After https://github.com/rust-lang/rust/pull/134258, we can't manually impl both `ToString` and `fmt::Display`, so this PR add a new trait `ToStringValue` to work around types type produces different result between ToString and Display. - Clippy - `Option::map_or(false, ...)` --> `Option::is_some_and(...)` - `Option::map_or(true, ...)` --> `Option::is_none_or(...)` - `(a + b - 1 )/ b` --> `a.div_ceil(b)` - `io::Error::new(ErrorKind::Other, ...)` --> ` io::Error::other(...)` - `Slice::group_by` --> `Slice::chunk_by` - `Result::map_err(|e| {...; e})` --> `Result::inspect_err(|e| { ... })` - `Map::get(&key).is_{some, none}()` --> `Map::contains_key()` - Formatter - The import order now follows ascii order, e.g. before is "use crate::{a, b, c, A, B, C}", after is "use crate::{A, B, C, a, b, c}". Most changes are due to this. - List in rust doc should be properly aligned. - cargo-deny - `vulnerability`, `notice` and `unsound` can't be config in version 2, and `unmaintained` can't be allowed anymore(but support setting `workspace` to allow indirected pkgs). So replacing some unmaintained packages with suggested alternatives. (See: https://github.com/tikv/tikv/pull/18416) Signed-off-by: glorv --- Cargo.lock | 197 +++++++----------- cmd/tikv-ctl/src/cmd.rs | 6 +- cmd/tikv-ctl/src/executor.rs | 22 +- cmd/tikv-ctl/src/fork_readonly_tikv.rs | 2 +- cmd/tikv-ctl/src/main.rs | 23 +- cmd/tikv-ctl/src/util.rs | 2 +- cmd/tikv-server/src/main.rs | 4 +- components/api_version/src/api_v1ttl.rs | 2 +- components/api_version/src/api_v2.rs | 3 +- components/api_version/src/lib.rs | 2 +- .../backup-stream/src/checkpoint_manager.rs | 12 +- components/backup-stream/src/endpoint.rs | 13 +- components/backup-stream/src/errors.rs | 19 +- components/backup-stream/src/event_loader.rs | 12 +- components/backup-stream/src/lib.rs | 1 - .../backup-stream/src/metadata/store/pd.rs | 6 +- .../src/metadata/store/slash_etc.rs | 4 +- components/backup-stream/src/metadata/test.rs | 6 +- components/backup-stream/src/observer.rs | 5 +- components/backup-stream/src/router.rs | 23 +- components/backup-stream/src/service.rs | 3 +- .../backup-stream/src/subscription_manager.rs | 26 +-- .../backup-stream/src/subscription_track.rs | 5 +- components/backup-stream/src/tempfiles.rs | 19 +- components/backup-stream/src/utils.rs | 21 +- .../backup-stream/tests/failpoints/mod.rs | 14 +- .../backup-stream/tests/integration/mod.rs | 4 +- components/backup-stream/tests/suite.rs | 17 +- components/backup/src/disk_snap.rs | 6 +- components/backup/src/endpoint.rs | 17 +- components/backup/src/lib.rs | 2 +- components/backup/src/service.rs | 10 +- components/backup/src/softlimit.rs | 4 +- components/backup/src/utils.rs | 2 +- components/backup/src/writer.rs | 23 +- .../batch-system/benches/batch-system.rs | 2 +- components/batch-system/src/batch.rs | 8 +- components/batch-system/src/fsm.rs | 3 +- components/batch-system/src/lib.rs | 4 +- components/batch-system/src/mailbox.rs | 2 +- components/batch-system/src/router.rs | 4 +- components/batch-system/src/scheduler.rs | 2 +- components/batch-system/src/test_runner.rs | 2 +- components/batch-system/tests/cases/batch.rs | 2 +- components/batch-system/tests/cases/router.rs | 2 +- components/causal_ts/src/lib.rs | 4 +- components/causal_ts/src/tso.rs | 27 ++- components/cdc/benches/cdc_event.rs | 2 +- components/cdc/src/channel.rs | 13 +- components/cdc/src/delegate.rs | 14 +- components/cdc/src/endpoint.rs | 30 +-- components/cdc/src/initializer.rs | 38 ++-- components/cdc/src/lib.rs | 2 +- components/cdc/src/observer.rs | 4 +- components/cdc/src/old_value.rs | 19 +- components/cdc/src/service.rs | 14 +- .../cdc/tests/failpoints/test_endpoint.rs | 10 +- .../cdc/tests/failpoints/test_memory_quota.rs | 4 +- .../cdc/tests/failpoints/test_observe.rs | 9 +- .../cdc/tests/failpoints/test_register.rs | 4 +- .../cdc/tests/failpoints/test_resolve.rs | 4 +- components/cdc/tests/integrations/test_cdc.rs | 10 +- .../tests/integrations/test_flow_control.rs | 4 +- components/cdc/tests/mod.rs | 6 +- components/cloud/aws/src/kms.rs | 9 +- components/cloud/aws/src/s3.rs | 37 ++-- components/cloud/aws/src/util.rs | 6 +- components/cloud/azure/Cargo.toml | 2 +- components/cloud/azure/src/azblob.rs | 25 ++- components/cloud/azure/src/kms.rs | 11 +- .../certificate_credentials.rs | 7 +- components/cloud/gcp/src/client.rs | 9 +- components/cloud/gcp/src/gcs.rs | 21 +- components/cloud/gcp/src/kms.rs | 5 +- components/cloud/gcp/src/lib.rs | 4 +- components/cloud/src/error.rs | 4 +- components/cloud/src/lib.rs | 2 +- components/codec/src/buffer.rs | 10 +- components/codec/src/byte.rs | 2 +- components/codec/src/number.rs | 15 +- .../src/compaction/collector.rs | 2 +- .../compact-log-backup/src/compaction/exec.rs | 8 +- .../compact-log-backup/src/compaction/meta.rs | 8 +- .../src/exec_hooks/checkpoint.rs | 2 +- .../src/exec_hooks/consistency.rs | 4 +- .../src/exec_hooks/observability.rs | 2 +- .../src/exec_hooks/save_meta.rs | 4 +- .../compact-log-backup/src/execute/hooking.rs | 2 +- .../compact-log-backup/src/execute/mod.rs | 7 +- .../compact-log-backup/src/execute/test.rs | 6 +- components/compact-log-backup/src/source.rs | 6 +- .../compact-log-backup/src/statistic.rs | 3 +- components/compact-log-backup/src/storage.rs | 8 +- .../compact-log-backup/src/test_util.rs | 7 +- components/compact-log-backup/src/util.rs | 2 +- .../benches/update_max_ts.rs | 2 +- .../concurrency_manager/src/key_handle.rs | 5 +- components/concurrency_manager/src/lib.rs | 7 +- .../concurrency_manager/tests/memory_usage.rs | 2 +- .../coprocessor_plugin_api/src/allocator.rs | 1 + components/crossbeam-skiplist/src/lib.rs | 7 +- components/crypto/Cargo.toml | 3 + components/crypto/build.rs | 2 +- components/encryption/export/examples/ecli.rs | 4 +- components/encryption/export/src/lib.rs | 8 +- components/encryption/src/config.rs | 6 +- .../encryption/src/encrypted_file/mod.rs | 4 +- components/encryption/src/errors.rs | 4 +- components/encryption/src/file_dict_file.rs | 8 +- components/encryption/src/io.rs | 41 ++-- components/encryption/src/lib.rs | 7 +- components/encryption/src/manager/mod.rs | 26 +-- components/encryption/src/master_key/kms.rs | 4 +- components/encryption/src/master_key/mem.rs | 2 +- components/encryption/src/master_key/mod.rs | 16 +- components/engine_panic/src/db_vector.rs | 2 +- components/engine_rocks/src/checkpoint.rs | 4 +- components/engine_rocks/src/compact.rs | 2 +- .../engine_rocks/src/compact_listener.rs | 7 +- components/engine_rocks/src/config.rs | 8 +- components/engine_rocks/src/db_vector.rs | 2 +- .../engine_rocks/src/decode_properties.rs | 2 +- components/engine_rocks/src/engine.rs | 14 +- .../engine_rocks/src/engine_iterator.rs | 2 +- components/engine_rocks/src/event_listener.rs | 15 +- components/engine_rocks/src/file_system.rs | 6 +- components/engine_rocks/src/flow_listener.rs | 5 +- components/engine_rocks/src/import.rs | 6 +- components/engine_rocks/src/lib.rs | 5 +- components/engine_rocks/src/misc.rs | 13 +- .../engine_rocks/src/mvcc_properties.rs | 2 +- .../engine_rocks/src/perf_context_impl.rs | 6 +- components/engine_rocks/src/properties.rs | 9 +- components/engine_rocks/src/raft_engine.rs | 8 +- .../engine_rocks/src/range_properties.rs | 4 +- components/engine_rocks/src/raw.rs | 7 +- components/engine_rocks/src/rocks_metrics.rs | 4 +- components/engine_rocks/src/snapshot.rs | 6 +- components/engine_rocks/src/sst.rs | 10 +- .../engine_rocks/src/table_properties.rs | 7 +- components/engine_rocks/src/ttl_properties.rs | 2 +- components/engine_rocks/src/util.rs | 14 +- components/engine_rocks/src/write_batch.rs | 6 +- .../engine_rocks_helper/src/sst_recovery.rs | 2 +- components/engine_test/src/lib.rs | 7 +- components/engine_tirocks/src/db_vector.rs | 2 +- components/engine_tirocks/src/write_batch.rs | 2 +- components/engine_traits/src/cf_defs.rs | 2 +- components/engine_traits/src/cf_options.rs | 2 +- components/engine_traits/src/compact.rs | 2 +- components/engine_traits/src/file_system.rs | 2 +- components/engine_traits/src/flush.rs | 4 +- components/engine_traits/src/import.rs | 2 +- components/engine_traits/src/lib.rs | 4 +- components/engine_traits/src/misc.rs | 4 +- components/engine_traits/src/raft_engine.rs | 4 +- .../engine_traits/src/range_properties.rs | 2 +- components/engine_traits/src/snapshot.rs | 2 +- components/engine_traits/src/sst.rs | 2 +- components/engine_traits/src/tablet.rs | 2 +- .../src/basic_read_write.rs | 2 +- .../engine_traits_tests/src/cf_names.rs | 2 +- .../engine_traits_tests/src/checkpoint.rs | 2 +- components/engine_traits_tests/src/ctor.rs | 2 +- .../engine_traits_tests/src/iterator.rs | 2 +- .../src/read_consistency.rs | 2 +- .../src/scenario_writes.rs | 4 +- .../engine_traits_tests/src/snapshot_basic.rs | 2 +- components/external_storage/examples/scli.rs | 48 ++--- components/external_storage/src/export.rs | 8 +- components/external_storage/src/hdfs.rs | 23 +- components/external_storage/src/lib.rs | 46 ++-- components/external_storage/src/local.rs | 3 +- components/external_storage/src/locking.rs | 4 +- components/file_system/src/file.rs | 8 +- .../file_system/src/io_stats/biosnoop.rs | 12 +- components/file_system/src/io_stats/proc.rs | 2 +- components/file_system/src/lib.rs | 17 +- components/file_system/src/metrics_manager.rs | 4 +- components/file_system/src/rate_limiter.rs | 7 +- components/health_controller/src/lib.rs | 6 +- components/health_controller/src/reporters.rs | 2 +- components/health_controller/src/types.rs | 2 +- components/hybrid_engine/src/db_vector.rs | 2 +- components/hybrid_engine/src/engine.rs | 6 +- components/hybrid_engine/src/metrics.rs | 2 +- .../src/observer/load_eviction.rs | 2 +- .../hybrid_engine/src/observer/snapshot.rs | 2 +- .../src/observer/test_write_batch.rs | 4 +- .../hybrid_engine/src/observer/write_batch.rs | 6 +- components/hybrid_engine/src/snapshot.rs | 10 +- components/hybrid_engine/src/util.rs | 6 +- .../in_memory_engine/benches/load_region.rs | 8 +- components/in_memory_engine/src/background.rs | 32 ++- components/in_memory_engine/src/config.rs | 2 +- .../in_memory_engine/src/cross_check.rs | 18 +- components/in_memory_engine/src/engine.rs | 24 +-- components/in_memory_engine/src/keys.rs | 2 +- components/in_memory_engine/src/lib.rs | 4 +- .../in_memory_engine/src/memory_controller.rs | 6 +- components/in_memory_engine/src/metrics.rs | 2 +- components/in_memory_engine/src/prop_test.rs | 8 +- components/in_memory_engine/src/read.rs | 43 ++-- .../in_memory_engine/src/region_label.rs | 4 +- .../in_memory_engine/src/region_manager.rs | 2 +- .../in_memory_engine/src/region_stats.rs | 12 +- components/in_memory_engine/src/statistics.rs | 2 +- components/in_memory_engine/src/test_util.rs | 4 +- .../in_memory_engine/src/write_batch.rs | 22 +- .../tests/failpoints/test_memory_engine.rs | 16 +- components/keys/src/lib.rs | 2 +- components/log_wrappers/src/lib.rs | 6 +- components/memory_trace_macros/src/lib.rs | 2 +- components/online_config/src/lib.rs | 14 +- components/panic_hook/src/lib.rs | 8 +- components/pd_client/src/client.rs | 19 +- components/pd_client/src/client_v2.rs | 11 +- components/pd_client/src/config.rs | 2 +- components/pd_client/src/feature_gate.rs | 2 +- components/pd_client/src/lib.rs | 6 +- components/pd_client/src/tso.rs | 2 +- components/pd_client/src/util.rs | 10 +- components/profiler/src/lib.rs | 1 + components/raft_log_engine/src/engine.rs | 7 +- .../raft_log_engine/src/perf_context.rs | 2 +- components/raftstore-v2/src/batch/mod.rs | 2 +- components/raftstore-v2/src/batch/store.rs | 31 ++- components/raftstore-v2/src/bootstrap.rs | 4 +- components/raftstore-v2/src/fsm/apply.rs | 4 +- components/raftstore-v2/src/fsm/peer.rs | 8 +- components/raftstore-v2/src/fsm/store.rs | 9 +- components/raftstore-v2/src/lib.rs | 8 +- .../raftstore-v2/src/operation/bucket.rs | 2 +- .../operation/command/admin/compact_log.rs | 8 +- .../operation/command/admin/conf_change.rs | 4 +- .../src/operation/command/admin/flashback.rs | 4 +- .../operation/command/admin/merge/commit.rs | 10 +- .../src/operation/command/admin/merge/mod.rs | 4 +- .../operation/command/admin/merge/prepare.rs | 12 +- .../operation/command/admin/merge/rollback.rs | 4 +- .../src/operation/command/admin/mod.rs | 18 +- .../src/operation/command/admin/split.rs | 18 +- .../command/admin/transfer_leader.rs | 8 +- .../src/operation/command/control.rs | 6 +- .../raftstore-v2/src/operation/command/mod.rs | 19 +- .../src/operation/command/write/ingest.rs | 4 +- .../src/operation/command/write/mod.rs | 19 +- components/raftstore-v2/src/operation/life.rs | 12 +- components/raftstore-v2/src/operation/misc.rs | 6 +- components/raftstore-v2/src/operation/mod.rs | 17 +- components/raftstore-v2/src/operation/pd.rs | 2 +- .../src/operation/query/capture.rs | 16 +- .../raftstore-v2/src/operation/query/lease.rs | 17 +- .../raftstore-v2/src/operation/query/local.rs | 39 ++-- .../raftstore-v2/src/operation/query/mod.rs | 10 +- .../src/operation/query/replica.rs | 6 +- .../src/operation/ready/apply_trace.rs | 14 +- .../src/operation/ready/async_writer.rs | 8 +- .../raftstore-v2/src/operation/ready/mod.rs | 20 +- .../src/operation/ready/snapshot.rs | 21 +- .../raftstore-v2/src/operation/txn_ext.rs | 8 +- .../src/operation/unsafe_recovery/create.rs | 6 +- .../src/operation/unsafe_recovery/demote.rs | 4 +- .../operation/unsafe_recovery/force_leader.rs | 4 +- .../src/operation/unsafe_recovery/report.rs | 2 +- components/raftstore-v2/src/raft/apply.rs | 10 +- components/raftstore-v2/src/raft/peer.rs | 12 +- components/raftstore-v2/src/raft/storage.rs | 24 +-- components/raftstore-v2/src/router/imp.rs | 7 +- components/raftstore-v2/src/router/message.rs | 6 +- components/raftstore-v2/src/router/mod.rs | 6 +- .../src/router/response_channel.rs | 13 +- .../src/worker/cleanup/compact.rs | 8 +- components/raftstore-v2/src/worker/pd/misc.rs | 6 +- components/raftstore-v2/src/worker/pd/mod.rs | 12 +- .../raftstore-v2/src/worker/pd/region.rs | 4 +- .../raftstore-v2/src/worker/pd/slowness.rs | 2 +- .../raftstore-v2/src/worker/pd/split.rs | 6 +- .../raftstore-v2/src/worker/pd/store.rs | 6 +- .../raftstore-v2/src/worker/refresh_config.rs | 6 +- components/raftstore-v2/src/worker/tablet.rs | 8 +- .../tests/failpoints/test_basic_write.rs | 6 +- .../tests/failpoints/test_bucket.rs | 2 +- .../tests/failpoints/test_life.rs | 4 +- .../tests/failpoints/test_merge.rs | 4 +- .../tests/failpoints/test_pd_heartbeat.rs | 2 +- .../tests/failpoints/test_split.rs | 6 +- .../tests/integrations/cluster.rs | 19 +- .../tests/integrations/test_basic_write.rs | 6 +- .../tests/integrations/test_conf_change.rs | 6 +- .../tests/integrations/test_life.rs | 6 +- .../tests/integrations/test_merge.rs | 2 +- .../tests/integrations/test_pd_heartbeat.rs | 10 +- .../tests/integrations/test_read.rs | 4 +- .../tests/integrations/test_split.rs | 4 +- .../tests/integrations/test_trace_apply.rs | 6 +- .../integrations/test_transfer_leader.rs | 4 +- components/raftstore/Cargo.toml | 2 +- .../raftstore/src/compacted_event_sender.rs | 2 +- .../raftstore/src/coprocessor/config.rs | 2 +- .../src/coprocessor/consistency_check.rs | 4 +- .../raftstore/src/coprocessor/metrics.rs | 2 +- components/raftstore/src/coprocessor/mod.rs | 8 +- .../src/coprocessor/read_write/write_batch.rs | 2 +- .../src/coprocessor/region_info_accessor.rs | 21 +- .../src/coprocessor/split_check/half.rs | 8 +- .../src/coprocessor/split_check/keys.rs | 13 +- .../src/coprocessor/split_check/mod.rs | 8 +- .../src/coprocessor/split_check/size.rs | 16 +- .../src/coprocessor/split_check/table.rs | 6 +- .../src/coprocessor/split_observer.rs | 4 +- components/raftstore/src/errors.rs | 2 +- components/raftstore/src/lib.rs | 4 - components/raftstore/src/router.rs | 6 +- .../raftstore/src/store/async_io/read.rs | 6 +- .../raftstore/src/store/async_io/write.rs | 13 +- .../src/store/async_io/write_router.rs | 6 +- .../src/store/async_io/write_tests.rs | 8 +- components/raftstore/src/store/bootstrap.rs | 6 +- .../raftstore/src/store/compaction_guard.rs | 12 +- components/raftstore/src/store/config.rs | 10 +- .../raftstore/src/store/entry_storage.rs | 24 +-- components/raftstore/src/store/fsm/apply.rs | 75 +++---- components/raftstore/src/store/fsm/life.rs | 2 +- components/raftstore/src/store/fsm/metrics.rs | 4 +- components/raftstore/src/store/fsm/mod.rs | 12 +- components/raftstore/src/store/fsm/peer.rs | 67 +++--- components/raftstore/src/store/fsm/store.rs | 58 +++--- .../raftstore/src/store/local_metrics.rs | 2 +- components/raftstore/src/store/mod.rs | 50 ++--- components/raftstore/src/store/msg.rs | 10 +- components/raftstore/src/store/peer.rs | 69 +++--- .../raftstore/src/store/peer_storage.rs | 26 ++- components/raftstore/src/store/read_queue.rs | 9 +- .../raftstore/src/store/region_snapshot.rs | 14 +- .../raftstore/src/store/replication_mode.rs | 2 +- .../raftstore/src/store/simple_write.rs | 2 +- components/raftstore/src/store/snap.rs | 75 ++++--- components/raftstore/src/store/snap/io.rs | 7 +- .../raftstore/src/store/snapshot_backup.rs | 10 +- components/raftstore/src/store/transport.rs | 8 +- components/raftstore/src/store/txn_ext.rs | 2 +- .../raftstore/src/store/unsafe_recovery.rs | 5 +- components/raftstore/src/store/util.rs | 30 ++- .../src/store/worker/cleanup_snapshot.rs | 2 +- .../raftstore/src/store/worker/compact.rs | 8 +- .../src/store/worker/consistency_check.rs | 10 +- .../raftstore/src/store/worker/disk_check.rs | 2 +- components/raftstore/src/store/worker/mod.rs | 16 +- components/raftstore/src/store/worker/pd.rs | 23 +- .../raftstore/src/store/worker/raftlog_gc.rs | 2 +- components/raftstore/src/store/worker/read.rs | 19 +- .../src/store/worker/refresh_config.rs | 6 +- .../raftstore/src/store/worker/region.rs | 19 +- .../raftstore/src/store/worker/snap_gen.rs | 7 +- .../raftstore/src/store/worker/split_check.rs | 8 +- .../src/store/worker/split_config.rs | 2 +- .../src/store/worker/split_controller.rs | 8 +- components/resolved_ts/src/advance.rs | 8 +- components/resolved_ts/src/cmd.rs | 10 +- components/resolved_ts/src/endpoint.rs | 13 +- components/resolved_ts/src/observer.rs | 2 +- components/resolved_ts/src/resolver.rs | 10 +- components/resolved_ts/src/scanner.rs | 6 +- .../resolved_ts/tests/failpoints/mod.rs | 2 +- .../resolved_ts/tests/integrations/mod.rs | 2 +- components/resolved_ts/tests/mod.rs | 2 +- components/resource_control/src/channel.rs | 2 +- components/resource_control/src/future.rs | 2 +- components/resource_control/src/lib.rs | 6 +- .../resource_control/src/resource_group.rs | 8 +- components/resource_control/src/service.rs | 12 +- components/resource_control/src/worker.rs | 6 +- components/resource_metering/src/config.rs | 4 +- components/resource_metering/src/lib.rs | 13 +- components/resource_metering/src/model.rs | 2 +- .../src/recorder/collector_reg.rs | 2 +- .../src/recorder/localstorage.rs | 4 +- .../resource_metering/src/recorder/mod.rs | 8 +- .../src/recorder/sub_recorder/cpu.rs | 4 +- .../src/recorder/sub_recorder/mod.rs | 2 +- .../src/recorder/sub_recorder/summary.rs | 4 +- .../src/reporter/collector_impl.rs | 2 +- .../resource_metering/src/reporter/mod.rs | 4 +- .../resource_metering/src/reporter/pubsub.rs | 4 +- .../src/reporter/single_target.rs | 2 +- .../resource_metering/tests/recorder_test.rs | 4 +- .../resource_metering/tests/summary_test.rs | 2 +- components/server/src/common.rs | 29 ++- components/server/src/raft_engine_switch.rs | 6 +- components/server/src/server.rs | 61 +++--- components/server/src/server2.rs | 47 ++--- components/service/src/service_manager.rs | 2 +- components/snap_recovery/Cargo.toml | 2 +- components/snap_recovery/src/data_resolver.rs | 6 +- components/snap_recovery/src/init_cluster.rs | 6 +- components/snap_recovery/src/leader_keeper.rs | 2 +- .../src/region_meta_collector.rs | 2 +- components/snap_recovery/src/services.rs | 8 +- .../sst_importer/src/caching/cache_map.rs | 4 +- components/sst_importer/src/config.rs | 2 +- components/sst_importer/src/import_file.rs | 6 +- components/sst_importer/src/import_mode.rs | 4 +- components/sst_importer/src/lib.rs | 4 +- components/sst_importer/src/metrics.rs | 1 + components/sst_importer/src/sst_importer.rs | 38 ++-- components/sst_importer/src/sst_writer.rs | 8 +- components/sst_importer/src/util.rs | 8 +- components/test_backup/src/disk_snap.rs | 4 +- components/test_backup/src/lib.rs | 10 +- components/test_coprocessor/src/column.rs | 2 +- components/test_coprocessor/src/dag.rs | 2 +- components/test_coprocessor/src/fixture.rs | 8 +- components/test_coprocessor/src/store.rs | 5 +- components/test_pd/src/lib.rs | 2 - components/test_pd/src/mocker/etcd.rs | 2 +- components/test_pd/src/mocker/meta_storage.rs | 12 +- components/test_pd/src/mocker/service.rs | 2 +- components/test_pd/src/server.rs | 6 +- components/test_pd_client/src/pd.rs | 18 +- components/test_raftstore-v2/src/cluster.rs | 32 +-- components/test_raftstore-v2/src/node.rs | 14 +- components/test_raftstore-v2/src/server.rs | 26 ++- .../src/transport_simulate.rs | 4 +- components/test_raftstore-v2/src/util.rs | 21 +- components/test_raftstore/src/cluster.rs | 17 +- components/test_raftstore/src/node.rs | 15 +- components/test_raftstore/src/router.rs | 6 +- components/test_raftstore/src/server.rs | 22 +- .../test_raftstore/src/transport_simulate.rs | 7 +- components/test_raftstore/src/util.rs | 24 ++- components/test_raftstore_macro/src/lib.rs | 4 +- components/test_sst_importer/src/lib.rs | 2 +- components/test_sst_importer/src/util.rs | 2 +- components/test_storage/src/assert_storage.rs | 3 +- components/test_storage/src/lib.rs | 1 - components/test_storage/src/sync_storage.rs | 10 +- components/test_storage/src/util.rs | 2 +- components/test_util/src/encryption.rs | 4 +- components/test_util/src/lib.rs | 2 +- components/test_util/src/logging.rs | 2 +- components/tidb_query_aggr/src/impl_avg.rs | 6 +- components/tidb_query_aggr/src/impl_bit_op.rs | 2 +- components/tidb_query_aggr/src/impl_count.rs | 2 +- components/tidb_query_aggr/src/impl_first.rs | 3 +- .../tidb_query_aggr/src/impl_max_min.rs | 37 ++-- components/tidb_query_aggr/src/impl_sum.rs | 4 +- .../tidb_query_aggr/src/impl_variance.rs | 6 +- components/tidb_query_aggr/src/lib.rs | 14 +- components/tidb_query_aggr/src/parser.rs | 2 +- components/tidb_query_aggr/src/summable.rs | 1 + components/tidb_query_aggr/src/util.rs | 4 +- components/tidb_query_codegen/src/lib.rs | 2 +- .../tidb_query_codegen/src/rpn_function.rs | 2 +- .../tidb_query_common/src/storage/scanner.rs | 6 +- .../src/storage/test_fixture.rs | 10 +- components/tidb_query_datatype/Cargo.toml | 2 +- .../benches/bench_vector_distance.rs | 2 +- .../src/codec/batch/lazy_column.rs | 11 +- .../src/codec/batch/lazy_column_vec.rs | 2 +- .../src/codec/chunk/chunk.rs | 8 +- .../src/codec/chunk/column.rs | 14 +- .../collation/collator/gb18030_collation.rs | 2 +- .../codec/collation/collator/latin1_bin.rs | 2 +- .../src/codec/collation/collator/mod.rs | 4 +- .../src/codec/collation/encoding/gb18030.rs | 2 +- .../src/codec/collation/encoding/mod.rs | 2 +- .../src/codec/collation/mod.rs | 12 +- .../tidb_query_datatype/src/codec/convert.rs | 64 ++++-- .../src/codec/data_type/chunked_vec_bytes.rs | 4 +- .../src/codec/data_type/chunked_vec_enum.rs | 6 +- .../src/codec/data_type/chunked_vec_json.rs | 4 +- .../src/codec/data_type/chunked_vec_set.rs | 4 +- .../src/codec/data_type/chunked_vec_sized.rs | 4 +- .../data_type/chunked_vec_vector_float32.rs | 4 +- .../src/codec/data_type/mod.rs | 17 +- .../src/codec/data_type/scalar.rs | 4 +- .../src/codec/data_type/vector.rs | 7 +- .../tidb_query_datatype/src/codec/datum.rs | 29 ++- .../src/codec/datum_codec.rs | 25 ++- .../src/codec/mysql/binary_literal.rs | 11 +- .../src/codec/mysql/decimal.rs | 111 +++++----- .../src/codec/mysql/duration.rs | 10 +- .../src/codec/mysql/enums.rs | 11 +- .../src/codec/mysql/json/binary.rs | 6 +- .../src/codec/mysql/json/comparison.rs | 7 +- .../src/codec/mysql/json/jcodec.rs | 4 +- .../src/codec/mysql/json/json_extract.rs | 6 +- .../src/codec/mysql/json/json_keys.rs | 2 +- .../src/codec/mysql/json/json_length.rs | 4 +- .../src/codec/mysql/json/json_merge.rs | 2 +- .../src/codec/mysql/json/json_modify.rs | 2 +- .../src/codec/mysql/json/json_remove.rs | 2 +- .../src/codec/mysql/json/json_unquote.rs | 7 +- .../src/codec/mysql/json/mod.rs | 20 +- .../src/codec/mysql/json/modifier.rs | 6 +- .../src/codec/mysql/json/path_expr.rs | 2 +- .../src/codec/mysql/json/serde.rs | 16 +- .../src/codec/mysql/mod.rs | 6 +- .../src/codec/mysql/time/extension.rs | 2 +- .../src/codec/mysql/time/interval.rs | 6 +- .../src/codec/mysql/time/mod.rs | 16 +- .../src/codec/mysql/vector.rs | 20 +- .../tidb_query_datatype/src/codec/overflow.rs | 2 - .../src/codec/row/v2/compat_v1.rs | 6 +- .../src/codec/row/v2/encoder_for_test.rs | 17 +- .../src/codec/row/v2/row_slice.rs | 15 +- .../tidb_query_datatype/src/codec/table.rs | 11 +- .../tidb_query_datatype/src/def/field_type.rs | 2 - .../tidb_query_datatype/src/expr/ctx.rs | 4 +- components/tidb_query_datatype/src/lib.rs | 1 - .../src/fast_hash_aggr_executor.rs | 11 +- .../src/index_scan_executor.rs | 27 +-- .../tidb_query_executors/src/interface.rs | 2 +- components/tidb_query_executors/src/lib.rs | 2 +- .../src/limit_executor.rs | 4 +- .../src/partition_top_n_executor.rs | 8 +- .../src/projection_executor.rs | 4 +- components/tidb_query_executors/src/runner.rs | 8 +- .../src/selection_executor.rs | 4 +- .../src/simple_aggr_executor.rs | 4 +- .../src/slow_hash_aggr_executor.rs | 8 +- .../src/stream_aggr_executor.rs | 9 +- .../src/table_scan_executor.rs | 8 +- .../src/top_n_executor.rs | 10 +- .../src/util/aggr_executor.rs | 6 +- .../src/util/hash_aggr_helper.rs | 2 +- .../src/util/scan_executor.rs | 6 +- .../src/util/top_n_heap.rs | 1 + components/tidb_query_expr/Cargo.toml | 3 +- .../tidb_query_expr/src/impl_arithmetic.rs | 4 +- components/tidb_query_expr/src/impl_cast.rs | 23 +- .../tidb_query_expr/src/impl_compare.rs | 13 +- .../tidb_query_expr/src/impl_compare_in.rs | 8 +- .../tidb_query_expr/src/impl_encryption.rs | 13 +- components/tidb_query_expr/src/impl_json.rs | 2 +- components/tidb_query_expr/src/impl_like.rs | 2 +- components/tidb_query_expr/src/impl_math.rs | 65 +++--- .../tidb_query_expr/src/impl_miscellaneous.rs | 4 +- components/tidb_query_expr/src/impl_op.rs | 6 +- components/tidb_query_expr/src/impl_other.rs | 2 - components/tidb_query_expr/src/impl_regexp.rs | 10 +- components/tidb_query_expr/src/impl_string.rs | 90 ++++---- components/tidb_query_expr/src/impl_time.rs | 31 ++- components/tidb_query_expr/src/lib.rs | 6 +- .../tidb_query_expr/src/types/expr_builder.rs | 5 +- .../tidb_query_expr/src/types/expr_eval.rs | 12 +- .../tidb_query_expr/src/types/function.rs | 4 +- components/tidb_query_expr/src/types/mod.rs | 2 +- .../tidb_query_expr/src/types/test_util.rs | 8 +- components/tikv_alloc/Cargo.toml | 3 + components/tikv_alloc/src/jemalloc.rs | 2 +- components/tikv_alloc/src/trace.rs | 2 +- components/tikv_kv/src/btree_engine.rs | 14 +- components/tikv_kv/src/cursor.rs | 32 +-- components/tikv_kv/src/lib.rs | 13 +- components/tikv_kv/src/raftstore_impls.rs | 2 +- components/tikv_kv/src/rocksdb_engine.rs | 17 +- components/tikv_kv/src/stats.rs | 2 +- .../benches/channel/bench_channel.rs | 2 +- components/tikv_util/src/callback.rs | 2 +- .../tikv_util/src/codec/stream_event.rs | 4 +- components/tikv_util/src/config.rs | 14 +- components/tikv_util/src/future.rs | 2 +- components/tikv_util/src/lib.rs | 12 +- components/tikv_util/src/log.rs | 2 +- components/tikv_util/src/logger/file_log.rs | 13 +- components/tikv_util/src/logger/formatter.rs | 4 +- components/tikv_util/src/logger/mod.rs | 8 +- components/tikv_util/src/math.rs | 2 +- components/tikv_util/src/memory.rs | 7 +- .../src/metrics/allocator_metrics.rs | 2 +- components/tikv_util/src/metrics/mod.rs | 4 +- .../tikv_util/src/metrics/process_linux.rs | 7 +- .../tikv_util/src/metrics/threads_linux.rs | 8 +- components/tikv_util/src/mpsc/future.rs | 5 +- components/tikv_util/src/mpsc/mod.rs | 2 +- .../tikv_util/src/mpsc/priority_queue.rs | 4 +- components/tikv_util/src/quota_limiter.rs | 2 +- components/tikv_util/src/range_latch.rs | 4 +- components/tikv_util/src/smoother.rs | 2 +- components/tikv_util/src/store/mod.rs | 2 +- components/tikv_util/src/stream.rs | 12 +- components/tikv_util/src/sys/cpu_time.rs | 7 +- components/tikv_util/src/sys/inspector.rs | 6 +- components/tikv_util/src/sys/mod.rs | 2 +- components/tikv_util/src/thread_group.rs | 2 +- components/tikv_util/src/time.rs | 4 +- components/tikv_util/src/timer.rs | 9 +- components/tikv_util/src/topn.rs | 2 +- components/tikv_util/src/worker/future.rs | 2 +- components/tikv_util/src/worker/mod.rs | 8 +- components/tikv_util/src/worker/pool.rs | 6 +- .../tikv_util/src/yatp_pool/future_pool.rs | 13 +- components/tikv_util/src/yatp_pool/mod.rs | 14 +- .../tipb_helper/src/expr_def_builder.rs | 2 +- components/tracker/src/lib.rs | 4 +- components/tracker/src/slab.rs | 2 +- components/tracker/src/tls.rs | 2 +- components/txn_types/src/lib.rs | 8 +- components/txn_types/src/lock.rs | 6 +- components/txn_types/src/write.rs | 19 +- deny.toml | 12 +- fuzz/cli.rs | 4 +- fuzz/fuzzer-afl/Cargo.toml | 1 + fuzz/fuzzer-honggfuzz/Cargo.toml | 1 + fuzz/fuzzer-libfuzzer/Cargo.toml | 1 + rust-toolchain.toml | 2 +- rustfmt.toml | 2 +- scripts/clippy | 2 + scripts/deny | 2 +- src/config/configurable.rs | 2 +- src/config/mod.rs | 119 +++++------ src/coprocessor/checksum.rs | 4 +- src/coprocessor/dag/mod.rs | 2 +- src/coprocessor/dag/storage_impl.rs | 2 +- src/coprocessor/endpoint.rs | 12 +- .../interceptors/concurrency_limiter.rs | 3 +- src/coprocessor/metrics.rs | 3 +- src/coprocessor/mod.rs | 10 +- src/coprocessor/readpool_impl.rs | 4 +- src/coprocessor/statistics/analyze.rs | 18 +- src/coprocessor/statistics/analyze_context.rs | 7 +- src/coprocessor/statistics/cmsketch.rs | 4 +- src/coprocessor/statistics/fmsketch.rs | 6 +- src/coprocessor/statistics/histogram.rs | 8 +- src/coprocessor/tracker.rs | 17 +- src/coprocessor_v2/endpoint.rs | 2 +- src/coprocessor_v2/plugin_registry.rs | 6 +- src/coprocessor_v2/raw_storage_impl.rs | 5 +- src/import/duplicate_detect.rs | 4 +- src/import/ingest.rs | 6 +- src/import/raft_writer.rs | 6 +- src/import/sst_service.rs | 32 +-- src/lib.rs | 2 - src/read_pool.rs | 15 +- src/server/config.rs | 9 +- src/server/debug.rs | 27 ++- src/server/debug2.rs | 30 ++- src/server/engine_factory.rs | 10 +- src/server/errors.rs | 2 +- src/server/gc_worker/compaction_filter.rs | 22 +- src/server/gc_worker/gc_manager.rs | 9 +- src/server/gc_worker/gc_worker.rs | 42 ++-- src/server/gc_worker/mod.rs | 12 +- .../gc_worker/rawkv_compaction_filter.rs | 20 +- src/server/load_statistics/linux.rs | 2 +- src/server/load_statistics/mod.rs | 2 +- src/server/lock_manager/config.rs | 2 +- src/server/lock_manager/deadlock.rs | 4 +- src/server/lock_manager/metrics.rs | 1 + src/server/lock_manager/mod.rs | 8 +- src/server/lock_manager/waiter_manager.rs | 6 +- src/server/metrics.rs | 3 +- src/server/mod.rs | 8 +- src/server/proxy.rs | 5 +- src/server/raft_client.rs | 14 +- src/server/raft_server.rs | 14 +- src/server/raftkv/mod.rs | 20 +- src/server/raftkv/raft_extension.rs | 2 +- src/server/raftkv2/mod.rs | 28 +-- src/server/raftkv2/node.rs | 10 +- src/server/raftkv2/raft_extension.rs | 2 +- src/server/reset_to_version.rs | 6 +- src/server/resolve.rs | 2 +- src/server/server.rs | 21 +- src/server/service/batch.rs | 8 +- src/server/service/diagnostics/log.rs | 10 +- src/server/service/diagnostics/mod.rs | 3 +- src/server/service/diagnostics/sys.rs | 2 +- src/server/service/kv.rs | 25 ++- src/server/service/mod.rs | 5 +- src/server/snap.rs | 15 +- src/server/status_server/lite.rs | 4 +- src/server/status_server/metrics.rs | 2 +- src/server/status_server/mod.rs | 28 +-- src/server/status_server/profile.rs | 9 +- src/server/tablet_snap.rs | 20 +- src/server/transport.rs | 2 +- src/server/ttl/mod.rs | 2 +- src/server/ttl/ttl_checker.rs | 2 +- src/server/ttl/ttl_compaction_filter.rs | 3 +- src/storage/config_manager.rs | 6 +- src/storage/errors.rs | 5 +- src/storage/kv/test_engine_builder.rs | 2 +- src/storage/lock_manager/lock_wait_context.rs | 40 ++-- .../lock_manager/lock_waiting_queue.rs | 16 +- src/storage/lock_manager/mod.rs | 4 +- src/storage/metrics.rs | 3 +- src/storage/mod.rs | 82 ++++---- src/storage/mvcc/consistency_check.rs | 8 +- src/storage/mvcc/metrics.rs | 1 + src/storage/mvcc/mod.rs | 6 +- src/storage/mvcc/reader/mod.rs | 4 +- src/storage/mvcc/reader/point_getter.rs | 4 +- src/storage/mvcc/reader/reader.rs | 27 ++- src/storage/mvcc/reader/scanner/backward.rs | 6 +- src/storage/mvcc/reader/scanner/forward.rs | 29 ++- src/storage/mvcc/reader/scanner/mod.rs | 12 +- src/storage/mvcc/txn.rs | 24 +-- src/storage/raw/encoded.rs | 9 +- src/storage/raw/raw_mvcc.rs | 15 +- src/storage/raw/store.rs | 7 +- src/storage/read_pool.rs | 4 +- .../txn/actions/acquire_pessimistic_lock.rs | 10 +- .../txn/actions/check_data_constraint.rs | 4 +- src/storage/txn/actions/check_txn_status.rs | 38 ++-- src/storage/txn/actions/cleanup.rs | 24 +-- src/storage/txn/actions/commit.rs | 14 +- .../txn/actions/flashback_to_version.rs | 8 +- src/storage/txn/actions/gc.rs | 6 +- src/storage/txn/actions/mvcc.rs | 6 +- src/storage/txn/actions/prewrite.rs | 19 +- src/storage/txn/actions/tests.rs | 6 +- .../txn/commands/acquire_pessimistic_lock.rs | 11 +- .../acquire_pessimistic_lock_resumed.rs | 23 +- src/storage/txn/commands/atomic_store.rs | 8 +- .../txn/commands/check_secondary_locks.rs | 6 +- src/storage/txn/commands/check_txn_status.rs | 12 +- src/storage/txn/commands/cleanup.rs | 5 +- src/storage/txn/commands/commit.rs | 5 +- src/storage/txn/commands/compare_and_swap.rs | 12 +- .../txn/commands/flashback_to_version.rs | 5 +- .../flashback_to_version_read_phase.rs | 24 +-- src/storage/txn/commands/flush.rs | 12 +- src/storage/txn/commands/macros.rs | 8 +- src/storage/txn/commands/mod.rs | 42 ++-- src/storage/txn/commands/mvcc_by_key.rs | 4 +- src/storage/txn/commands/mvcc_by_start_ts.rs | 4 +- src/storage/txn/commands/pause.rs | 4 +- .../txn/commands/pessimistic_rollback.rs | 6 +- .../pessimistic_rollback_read_phase.rs | 6 +- src/storage/txn/commands/prewrite.rs | 22 +- src/storage/txn/commands/resolve_lock.rs | 10 +- src/storage/txn/commands/resolve_lock_lite.rs | 6 +- .../txn/commands/resolve_lock_readphase.rs | 4 +- src/storage/txn/commands/rollback.rs | 7 +- src/storage/txn/commands/txn_heart_beat.rs | 6 +- .../singleton_flow_controller.rs | 7 +- .../flow_controller/tablet_flow_controller.rs | 6 +- src/storage/txn/latch.rs | 3 +- src/storage/txn/mod.rs | 8 +- src/storage/txn/sched_pool.rs | 6 +- src/storage/txn/scheduler.rs | 27 ++- src/storage/txn/store.rs | 2 +- src/storage/txn/task.rs | 4 +- src/storage/txn/tracker.rs | 4 +- src/storage/txn/txn_status_cache.rs | 34 +-- src/storage/types.rs | 2 +- .../coprocessor_executors/hash_aggr/util.rs | 4 +- .../coprocessor_executors/index_scan/mod.rs | 2 +- .../coprocessor_executors/index_scan/util.rs | 4 +- .../coprocessor_executors/integrated/mod.rs | 2 +- .../coprocessor_executors/selection/util.rs | 4 +- .../coprocessor_executors/simple_aggr/util.rs | 4 +- .../coprocessor_executors/stream_aggr/util.rs | 4 +- .../coprocessor_executors/table_scan/mod.rs | 2 +- .../coprocessor_executors/table_scan/util.rs | 4 +- .../coprocessor_executors/top_n/util.rs | 2 +- .../coprocessor_executors/util/fixture.rs | 4 +- tests/benches/deadlock_detector/mod.rs | 6 +- tests/benches/hierarchy/engine/mod.rs | 4 +- tests/benches/hierarchy/engine_factory.rs | 2 +- tests/benches/hierarchy/mvcc/mod.rs | 6 +- tests/benches/hierarchy/storage/mod.rs | 4 +- tests/benches/hierarchy/txn/mod.rs | 6 +- tests/benches/memory/mod.rs | 4 +- tests/benches/misc/coprocessor/codec/mod.rs | 2 +- .../misc/coprocessor/dag/expr/scalar.rs | 4 +- .../misc/keybuilder/bench_keybuilder.rs | 2 +- tests/benches/misc/raftkv/mod.rs | 10 +- .../misc/serialization/bench_serialization.rs | 2 +- tests/benches/misc/storage/incremental_get.rs | 2 +- tests/benches/misc/storage/key.rs | 2 +- tests/benches/misc/storage/mvcc_reader.rs | 2 +- tests/benches/misc/util/slice_compare.rs | 2 +- .../misc/writebatch/bench_writebatch.rs | 2 +- tests/failpoints/cases/test_async_fetch.rs | 4 +- tests/failpoints/cases/test_async_io.rs | 2 +- tests/failpoints/cases/test_backup.rs | 2 +- .../cases/test_cmd_epoch_checker.rs | 2 +- tests/failpoints/cases/test_conf_change.rs | 4 +- tests/failpoints/cases/test_coprocessor.rs | 2 +- tests/failpoints/cases/test_debugger.rs | 4 +- tests/failpoints/cases/test_disk_snap_br.rs | 2 +- tests/failpoints/cases/test_early_apply.rs | 2 +- tests/failpoints/cases/test_engine.rs | 4 +- tests/failpoints/cases/test_gc_metrics.rs | 21 +- tests/failpoints/cases/test_gc_worker.rs | 4 +- tests/failpoints/cases/test_hibernate.rs | 2 +- tests/failpoints/cases/test_import_service.rs | 6 +- .../failpoints/cases/test_in_memory_engine.rs | 22 +- tests/failpoints/cases/test_kv_service.rs | 4 +- .../cases/test_memory_usage_limit.rs | 2 +- tests/failpoints/cases/test_merge.rs | 13 +- tests/failpoints/cases/test_pd_client.rs | 4 +- .../failpoints/cases/test_pd_client_legacy.rs | 4 +- tests/failpoints/cases/test_pending_peers.rs | 2 +- tests/failpoints/cases/test_rawkv.rs | 2 +- .../cases/test_read_execution_tracker.rs | 2 +- tests/failpoints/cases/test_replica_read.rs | 4 +- tests/failpoints/cases/test_server.rs | 4 +- tests/failpoints/cases/test_snap.rs | 4 +- tests/failpoints/cases/test_split_region.rs | 18 +- tests/failpoints/cases/test_sst_ingest.rs | 2 +- tests/failpoints/cases/test_sst_recovery.rs | 2 +- tests/failpoints/cases/test_stale_peer.rs | 4 +- tests/failpoints/cases/test_stale_read.rs | 4 +- tests/failpoints/cases/test_storage.rs | 13 +- .../failpoints/cases/test_table_properties.rs | 10 +- tests/failpoints/cases/test_transaction.rs | 13 +- .../failpoints/cases/test_transfer_leader.rs | 4 +- tests/failpoints/cases/test_ttl.rs | 6 +- .../failpoints/cases/test_unsafe_recovery.rs | 4 +- tests/integrations/backup/disk_snap.rs | 4 +- tests/integrations/backup/mod.rs | 2 +- .../integrations/config/dynamic/gc_worker.rs | 4 +- .../config/dynamic/pessimistic_txn.rs | 2 +- .../integrations/config/dynamic/raftstore.rs | 10 +- tests/integrations/config/dynamic/snap.rs | 4 +- .../config/dynamic/split_check.rs | 2 +- tests/integrations/config/mod.rs | 4 +- .../integrations/coprocessor/test_checksum.rs | 6 +- tests/integrations/coprocessor/test_select.rs | 4 +- tests/integrations/import/test_sst_service.rs | 2 +- tests/integrations/pd/test_rpc_client.rs | 4 +- .../integrations/pd/test_rpc_client_legacy.rs | 5 +- .../integrations/raftstore/test_bootstrap.rs | 14 +- .../raftstore/test_clear_stale_data.rs | 4 +- .../raftstore/test_compact_after_delete.rs | 6 +- .../raftstore/test_compact_lock_cf.rs | 2 +- .../raftstore/test_conf_change.rs | 2 +- .../integrations/raftstore/test_hibernate.rs | 2 +- .../integrations/raftstore/test_lease_read.rs | 6 +- tests/integrations/raftstore/test_life.rs | 8 +- tests/integrations/raftstore/test_merge.rs | 10 +- tests/integrations/raftstore/test_multi.rs | 8 +- tests/integrations/raftstore/test_prevote.rs | 2 +- .../raftstore/test_region_change_observer.rs | 6 +- .../raftstore/test_region_heartbeat.rs | 4 +- .../raftstore/test_region_info_accessor.rs | 6 +- .../raftstore/test_replica_read.rs | 8 +- .../raftstore/test_replication_mode.rs | 2 +- .../integrations/raftstore/test_scale_pool.rs | 4 +- tests/integrations/raftstore/test_snap.rs | 6 +- .../raftstore/test_snap_recovery.rs | 6 +- .../raftstore/test_split_region.rs | 8 +- .../integrations/raftstore/test_stale_peer.rs | 6 +- tests/integrations/raftstore/test_stats.rs | 8 +- .../raftstore/test_status_command.rs | 2 +- .../integrations/raftstore/test_tombstone.rs | 2 +- .../raftstore/test_transfer_leader.rs | 2 +- .../raftstore/test_unsafe_recovery.rs | 2 +- .../raftstore/test_v1_v2_mixed.rs | 6 +- tests/integrations/raftstore/test_witness.rs | 2 +- .../resource_metering/test_cpu.rs | 6 +- .../resource_metering/test_dynamic_config.rs | 3 +- .../resource_metering/test_pubsub.rs | 2 +- .../resource_metering/test_read_keys.rs | 8 +- .../test_suite/mock_receiver_server.rs | 4 +- .../resource_metering/test_suite/mod.rs | 8 +- tests/integrations/server/debugger.rs | 4 +- tests/integrations/server/gc_worker.rs | 2 +- tests/integrations/server/kv_service.rs | 14 +- tests/integrations/server/lock_manager.rs | 5 +- tests/integrations/server/mod.rs | 2 +- tests/integrations/server/raft_client.rs | 7 +- tests/integrations/server/server.rs | 2 +- tests/integrations/server/status_server.rs | 2 +- .../integrations/storage/test_raft_storage.rs | 6 +- tests/integrations/storage/test_raftkv.rs | 8 +- .../storage/test_region_info_accessor.rs | 4 +- tests/integrations/storage/test_storage.rs | 7 +- tests/integrations/storage/test_titan.rs | 8 +- 874 files changed, 3803 insertions(+), 4023 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c2db0f7b04..5b3b2876bc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,9 +43,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ "getrandom 0.2.11", "once_cell", @@ -338,7 +338,7 @@ dependencies = [ "bytes", "fastrand 2.0.1", "http 0.2.12", - "time 0.3.20", + "time 0.3.41", "tokio", "tracing", "url", @@ -433,7 +433,7 @@ dependencies = [ "once_cell", "percent-encoding", "regex-lite", - "sha2 0.10.8", + "sha2", "tracing", "url", ] @@ -480,8 +480,8 @@ dependencies = [ "http 1.1.0", "once_cell", "percent-encoding", - "sha2 0.10.8", - "time 0.3.20", + "sha2", + "time 0.3.41", "tracing", ] @@ -513,7 +513,7 @@ dependencies = [ "md-5", "pin-project-lite", "sha1", - "sha2 0.10.8", + "sha2", "tracing", ] @@ -655,7 +655,7 @@ dependencies = [ "pin-utils", "ryu", "serde", - "time 0.3.20", + "time 0.3.41", "tokio", "tokio-util", ] @@ -705,7 +705,7 @@ dependencies = [ "slog", "slog-global", "tikv_util", - "time 0.3.20", + "time 0.3.41", "tokio", "url", "uuid 1.7.0", @@ -732,7 +732,7 @@ dependencies = [ "rustc_version 0.4.0", "serde", "serde_json", - "time 0.3.20", + "time 0.3.41", "url", "uuid 1.7.0", ] @@ -752,7 +752,7 @@ dependencies = [ "pin-project", "serde", "serde_json", - "time 0.3.20", + "time 0.3.41", "url", "uuid 1.7.0", ] @@ -768,7 +768,7 @@ dependencies = [ "futures 0.3.15", "serde", "serde_json", - "time 0.3.20", + "time 0.3.41", "url", ] @@ -788,7 +788,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "time 0.3.20", + "time 0.3.41", "url", "uuid 1.7.0", ] @@ -807,7 +807,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "time 0.3.20", + "time 0.3.41", "url", "uuid 1.7.0", ] @@ -881,7 +881,7 @@ dependencies = [ "concurrency_manager", "crc64fast", "dashmap", - "derive_more", + "derive_more 0.99.3", "encryption", "encryption_export", "engine_panic", @@ -964,7 +964,7 @@ dependencies = [ "criterion 0.3.5", "crossbeam", "dashmap", - "derive_more", + "derive_more 0.99.3", "fail", "file_system", "kvproto", @@ -1083,15 +1083,6 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -1457,7 +1448,7 @@ name = "cloud" version = "0.0.1" dependencies = [ "async-trait", - "derive_more", + "derive_more 0.99.3", "error_code", "futures 0.3.15", "futures-io", @@ -1527,7 +1518,7 @@ dependencies = [ "codec", "crc64fast", "dashmap", - "derive_more", + "derive_more 0.99.3", "encryption", "engine_rocks", "engine_traits", @@ -1664,12 +1655,6 @@ dependencies = [ "libc 0.2.151", ] -[[package]] -name = "cpuid-bool" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" - [[package]] name = "crc32c" version = "0.6.8" @@ -1974,14 +1959,13 @@ dependencies = [ ] [[package]] -name = "derivative" -version = "2.2.0" +name = "deranged" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.103", + "powerfmt", + "serde", ] [[package]] @@ -1996,27 +1980,39 @@ dependencies = [ ] [[package]] -name = "diff" -version = "0.1.13" +name = "derive_more" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +dependencies = [ + "derive_more-impl", +] [[package]] -name = "digest" -version = "0.9.0" +name = "derive_more-impl" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ - "generic-array", + "proc-macro2", + "quote", + "syn 2.0.79", + "unicode-xid", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", + "block-buffer", "crypto-common", "subtle", ] @@ -2163,7 +2159,7 @@ version = "0.0.1" dependencies = [ "api_version", "collections", - "derive_more", + "derive_more 0.99.3", "encryption", "engine_traits", "fail", @@ -3126,7 +3122,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.7", + "digest", ] [[package]] @@ -3584,7 +3580,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16d4bde3a7105e59c66a4104cfe9606453af1c7a0eac78cb7d5bc263eb762a70" dependencies = [ - "ahash 0.7.7", + "ahash 0.7.8", "atty", "indexmap 1.6.2", "itoa 1.0.1", @@ -3983,7 +3979,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if 1.0.0", - "digest 0.10.7", + "digest", ] [[package]] @@ -4362,15 +4358,10 @@ dependencies = [ ] [[package]] -name = "num-derive" -version = "0.3.0" +name = "num-conv" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8b15b261814f992e33760b1fca9fe8b693d8a65299f20c9901688636cfb746" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.103", -] +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" [[package]] name = "num-derive" @@ -4468,9 +4459,9 @@ dependencies = [ [[package]] name = "oauth2" -version = "4.1.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e47cfc4c0a1a519d9a025ebfbac3a2439d1b5cdf397d72dcb79b11d9920dab" +checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" dependencies = [ "base64 0.13.0", "chrono", @@ -4480,7 +4471,7 @@ dependencies = [ "serde", "serde_json", "serde_path_to_error", - "sha2 0.9.1", + "sha2", "thiserror", "url", ] @@ -4525,12 +4516,6 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "openssl" version = "0.10.57" @@ -4903,6 +4888,12 @@ dependencies = [ "ws2_32-sys", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "pprof" version = "0.14.0" @@ -5217,7 +5208,7 @@ dependencies = [ "lz4-sys", "memmap2", "nix 0.26.2", - "num-derive 0.4.2", + "num-derive", "num-traits", "parking_lot 0.12.1", "prometheus", @@ -5285,7 +5276,7 @@ dependencies = [ "concurrency_manager", "crc32fast", "crossbeam", - "derivative", + "derive_more 2.0.1", "encryption", "encryption_export", "engine_panic", @@ -5780,7 +5771,7 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f06953bb8b9e4307cb7ccc0d9d018e2ddd25a30d32831f631ce4fe8f17671f7" dependencies = [ - "ahash 0.7.7", + "ahash 0.7.8", "bitflags 1.3.2", "instant", "num-traits", @@ -5920,12 +5911,6 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" -[[package]] -name = "safemem" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b08423011dae9a5ca23f07cf57dac3857f5c885d352b76f6d95f4aea9434d0" - [[package]] name = "same-file" version = "1.0.5" @@ -6240,20 +6225,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest 0.10.7", -] - -[[package]] -name = "sha2" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if 0.1.10", - "cpuid-bool", - "digest 0.9.0", - "opaque-debug", + "digest", ] [[package]] @@ -6264,7 +6236,7 @@ checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest 0.10.7", + "digest", ] [[package]] @@ -7199,7 +7171,7 @@ dependencies = [ "anyhow", "api_version", "byteorder", - "derive_more", + "derive_more 0.99.3", "error_code", "futures 0.3.15", "kvproto", @@ -7240,7 +7212,7 @@ dependencies = [ "match-template", "nom 7.1.3", "num", - "num-derive 0.3.0", + "num-derive", "num-traits", "ordered-float", "protobuf", @@ -7302,6 +7274,7 @@ dependencies = [ "hex 0.4.3", "log_wrappers", "match-template", + "memchr", "num", "num-traits", "openssl", @@ -7309,7 +7282,6 @@ dependencies = [ "profiler", "protobuf", "regex", - "safemem", "serde", "serde_json", "static_assertions", @@ -7320,7 +7292,6 @@ dependencies = [ "time 0.1.43", "tipb", "tipb_helper", - "twoway", "uuid 0.8.2", ] @@ -7607,7 +7578,7 @@ dependencies = [ "cpu-time", "crossbeam", "crossbeam-skiplist 0.1.3", - "derive_more", + "derive_more 0.99.3", "error_code", "fail", "fs2", @@ -7672,13 +7643,16 @@ dependencies = [ [[package]] name = "time" -version = "0.3.20" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ + "deranged", "itoa 1.0.1", "libc 0.2.151", + "num-conv", "num_threads", + "powerfmt", "serde", "time-core", "time-macros", @@ -7686,16 +7660,17 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.8" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ + "num-conv", "time-core", ] @@ -7982,16 +7957,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" -[[package]] -name = "twoway" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766345ed3891b291d01af307cd3ad2992a4261cb6c0c7e665cd3e01cf379dd24" -dependencies = [ - "memchr", - "unchecked-index", -] - [[package]] name = "twox-hash" version = "1.6.3" @@ -8041,12 +8006,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" -[[package]] -name = "unchecked-index" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c" - [[package]] name = "unicode-ident" version = "1.0.5" diff --git a/cmd/tikv-ctl/src/cmd.rs b/cmd/tikv-ctl/src/cmd.rs index a42e1781383..f02c20ef38f 100644 --- a/cmd/tikv-ctl/src/cmd.rs +++ b/cmd/tikv-ctl/src/cmd.rs @@ -1,9 +1,9 @@ // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. -use std::{borrow::ToOwned, str, string::ToString, sync::LazyLock, u64}; +use std::{borrow::ToOwned, str, string::ToString, sync::LazyLock}; -use clap::{crate_authors, AppSettings}; -use engine_traits::{SstCompressionType, CF_DEFAULT}; +use clap::{AppSettings, crate_authors}; +use engine_traits::{CF_DEFAULT, SstCompressionType}; use raft_engine::ReadableSize; use structopt::StructOpt; diff --git a/cmd/tikv-ctl/src/executor.rs b/cmd/tikv-ctl/src/executor.rs index 30d4aa192ac..ce3c2c09134 100644 --- a/cmd/tikv-ctl/src/executor.rs +++ b/cmd/tikv-ctl/src/executor.rs @@ -9,15 +9,15 @@ use api_version::{ApiV1, KvFormat}; use encryption_export::data_key_manager_from_config; use engine_rocks::util::{db_exist, new_engine_opt}; use engine_traits::{ - Engines, Error as EngineError, RaftEngine, TabletRegistry, ALL_CFS, CF_DEFAULT, CF_LOCK, - CF_WRITE, DATA_CFS, + ALL_CFS, CF_DEFAULT, CF_LOCK, CF_WRITE, DATA_CFS, Engines, Error as EngineError, RaftEngine, + TabletRegistry, }; use file_system::read_dir; use futures::{ + StreamExt, TryStreamExt, executor::block_on, future, stream::{self, BoxStream}, - StreamExt, TryStreamExt, }; use grpcio::{ChannelBuilder, Environment}; use kvproto::{ @@ -31,7 +31,7 @@ use pd_client::{Config as PdConfig, PdClient, RpcClient}; use protobuf::Message; use raft::eraftpb::{ConfChange, ConfChangeV2, Entry, EntryType}; use raft_log_engine::RaftLogEngine; -use raftstore::store::{util::build_key_range, INIT_EPOCH_CONF_VER}; +use raftstore::store::{INIT_EPOCH_CONF_VER, util::build_key_range}; use security::SecurityManager; use serde_json::json; use server::fatal; @@ -39,15 +39,15 @@ use slog_global::crit; use tikv::{ config::{ConfigController, TikvConfig}, server::{ + KvEngineFactoryBuilder, debug::{BottommostLevelCompaction, Debugger, DebuggerImpl, RegionInfo}, debug2::DebuggerImplV2, - KvEngineFactoryBuilder, }, storage::{ + Engine, config::EngineType, kv::MockEngine, lock_manager::{LockManager, MockLockManager}, - Engine, }, }; use tikv_util::escape; @@ -223,7 +223,7 @@ pub trait DebugExecutor { let r = self.get_region_info(region_id); if skip_tombstone { let region_state = r.region_local_state.as_ref(); - if region_state.map_or(false, |s| s.get_state() == PeerState::Tombstone) { + if region_state.is_some_and(|s| s.get_state() == PeerState::Tombstone) { continue; } } @@ -386,21 +386,21 @@ pub trait DebugExecutor { println!("key: {}", escape(&key)); if cfs.contains(&CF_LOCK) && mvcc.has_lock() { let lock_info = mvcc.get_lock(); - if start_ts.map_or(true, |ts| lock_info.get_start_ts() == ts) { + if start_ts.is_none_or(|ts| lock_info.get_start_ts() == ts) { println!("\tlock cf value: {:?}", lock_info); } } if cfs.contains(&CF_DEFAULT) { for value_info in mvcc.get_values() { - if commit_ts.map_or(true, |ts| value_info.get_start_ts() == ts) { + if commit_ts.is_none_or(|ts| value_info.get_start_ts() == ts) { println!("\tdefault cf value: {:?}", value_info); } } } if cfs.contains(&CF_WRITE) { for write_info in mvcc.get_writes() { - if start_ts.map_or(true, |ts| write_info.get_start_ts() == ts) - && commit_ts.map_or(true, |ts| write_info.get_commit_ts() == ts) + if start_ts.is_none_or(|ts| write_info.get_start_ts() == ts) + && commit_ts.is_none_or(|ts| write_info.get_commit_ts() == ts) { println!("\t write cf value: {:?}", write_info); } diff --git a/cmd/tikv-ctl/src/fork_readonly_tikv.rs b/cmd/tikv-ctl/src/fork_readonly_tikv.rs index 934ef173a67..4d16f1e382f 100644 --- a/cmd/tikv-ctl/src/fork_readonly_tikv.rs +++ b/cmd/tikv-ctl/src/fork_readonly_tikv.rs @@ -8,7 +8,7 @@ use std::{ }; use encryption_export::data_key_manager_from_config; -use raft_engine::{env::DefaultFileSystem, Engine as RaftEngine}; +use raft_engine::{Engine as RaftEngine, env::DefaultFileSystem}; use regex::Regex; use tikv::config::TikvConfig; diff --git a/cmd/tikv-ctl/src/main.rs b/cmd/tikv-ctl/src/main.rs index bb73f55810a..67cc425d3d8 100644 --- a/cmd/tikv-ctl/src/main.rs +++ b/cmd/tikv-ctl/src/main.rs @@ -1,6 +1,5 @@ // Copyright 2016 TiKV Project Authors. Licensed under Apache-2.0. -#![feature(lazy_cell)] #![feature(let_chains)] #[macro_use] @@ -16,19 +15,19 @@ use std::{ sync::{Arc, Mutex, RwLock}, thread, time::Duration, - u64, }; use collections::HashMap; use compact_log_backup::{ + TraceResultExt, exec_hooks::{self as compact_log_hooks, skip_small_compaction::SkipSmallCompaction}, - execute as compact_log, TraceResultExt, + execute as compact_log, }; use crypto::fips; use encryption_export::{ - create_backend, data_key_manager_from_config, DataKeyManager, DecrypterReader, Iv, + DataKeyManager, DecrypterReader, Iv, create_backend, data_key_manager_from_config, }; -use engine_rocks::{get_env, util::new_engine_opt, RocksEngine}; +use engine_rocks::{RocksEngine, get_env, util::new_engine_opt}; use engine_traits::Peekable; use file_system::calc_crc32; use futures::{executor::block_on, future::try_join_all}; @@ -49,16 +48,16 @@ use raft_log_engine::ManagedFileSystem; use raftstore::store::util::build_key_range; use regex::Regex; use security::{SecurityConfig, SecurityManager}; -use structopt::{clap::ErrorKind, StructOpt}; +use structopt::{StructOpt, clap::ErrorKind}; use tempfile::TempDir; use tikv::{ config::TikvConfig, - server::{debug::BottommostLevelCompaction, KvEngineFactoryBuilder}, + server::{KvEngineFactoryBuilder, debug::BottommostLevelCompaction}, storage::config::EngineType, }; use tikv_util::{ escape, - logger::{get_log_level, Level}, + logger::{Level, get_log_level}, run_and_wait_child_process, sys::thread::StdThreadBuildWrapper, unescape, warn, @@ -1292,6 +1291,9 @@ fn print_bad_ssts(data_dir: &str, manifest: Option<&str>, pd_client: RpcClient, stderr_buf.read_to_end(&mut buffer).unwrap(); let corruptions = unsafe { String::from_utf8_unchecked(buffer) }; + let r = Regex::new(r"/\w*\.sst").unwrap(); + let column_r = Regex::new(r"--------------- (.*) --------------\n(.*)").unwrap(); + let sst_stat_r = Regex::new(r".*\n\d+:\d+\[\d+ .. \d+\]\['(\w*)' seq:\d+, type:\d+ .. '(\w*)' seq:\d+, type:\d+\] at level \d+").unwrap(); for line in corruptions.lines() { println!("--------------------------------------------------------"); // The corruption format may like this: @@ -1300,7 +1302,6 @@ fn print_bad_ssts(data_dir: &str, manifest: Option<&str>, pd_client: RpcClient, // ``` println!("corruption info:\n{}", line); - let r = Regex::new(r"/\w*\.sst").unwrap(); let sst_file_number = match r.captures(line) { None => { println!("skip bad line format"); @@ -1361,15 +1362,13 @@ fn print_bad_ssts(data_dir: &str, manifest: Option<&str>, pd_client: RpcClient, // --------------- Column family "write" (ID 2) -------------- // 63:132906243[3555338 .. 3555338]['7A311B40EFCC2CB4C5911ECF3937D728DED26AE53FA5E61BE04F23F2BE54EACC73' seq:3555338, type:1 .. '7A313030302E25CD5F57252E' seq:3555338, type:1] at level 0 // ``` - let column_r = Regex::new(r"--------------- (.*) --------------\n(.*)").unwrap(); if let Some(m) = column_r.captures(&output) { println!( "{} for {}", m.get(2).unwrap().as_str(), m.get(1).unwrap().as_str() ); - let r = Regex::new(r".*\n\d+:\d+\[\d+ .. \d+\]\['(\w*)' seq:\d+, type:\d+ .. '(\w*)' seq:\d+, type:\d+\] at level \d+").unwrap(); - let matches = match r.captures(&output) { + let matches = match sst_stat_r.captures(&output) { None => { println!("sst start key format is not correct: {}", output); continue; diff --git a/cmd/tikv-ctl/src/util.rs b/cmd/tikv-ctl/src/util.rs index 90322b6000d..913df27675b 100644 --- a/cmd/tikv-ctl/src/util.rs +++ b/cmd/tikv-ctl/src/util.rs @@ -1,6 +1,6 @@ // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. -use std::{borrow::ToOwned, error::Error, str, str::FromStr, u64}; +use std::{borrow::ToOwned, error::Error, str, str::FromStr}; use kvproto::kvrpcpb::KeyRange; use server::setup::initial_logger; diff --git a/cmd/tikv-server/src/main.rs b/cmd/tikv-server/src/main.rs index 9fe7f6986d0..e9f6fe8ab64 100644 --- a/cmd/tikv-server/src/main.rs +++ b/cmd/tikv-server/src/main.rs @@ -4,12 +4,12 @@ use std::{path::Path, process}; -use clap::{crate_authors, App, Arg}; +use clap::{App, Arg, crate_authors}; use crypto::fips; use serde_json::{Map, Value}; use server::setup::{ensure_no_unrecognized_config, validate_and_persist_config}; use tikv::{ - config::{to_flatten_config_info, TikvConfig}, + config::{TikvConfig, to_flatten_config_info}, storage::config::EngineType, }; diff --git a/components/api_version/src/api_v1ttl.rs b/components/api_version/src/api_v1ttl.rs index 2a2df6bfb33..bd66e0c4523 100644 --- a/components/api_version/src/api_v1ttl.rs +++ b/components/api_version/src/api_v1ttl.rs @@ -4,8 +4,8 @@ use engine_traits::Result; use tikv_util::{ box_err, codec::{ - number::{self, NumberEncoder}, Error, + number::{self, NumberEncoder}, }, }; diff --git a/components/api_version/src/api_v2.rs b/components/api_version/src/api_v2.rs index a56d5deac30..9096e8a18fa 100644 --- a/components/api_version/src/api_v2.rs +++ b/components/api_version/src/api_v2.rs @@ -3,9 +3,8 @@ use codec::byte::MemComparableByteCodec; use engine_traits::Result; use tikv_util::codec::{ - bytes, + Error, bytes, number::{self, NumberEncoder}, - Error, }; use txn_types::{Key, TimeStamp}; diff --git a/components/api_version/src/lib.rs b/components/api_version/src/lib.rs index 879751e7b62..18de8148228 100644 --- a/components/api_version/src/lib.rs +++ b/components/api_version/src/lib.rs @@ -259,7 +259,7 @@ impl> RawValue { #[inline] pub fn is_ttl_expired(&self, current_ts: u64) -> bool { self.expire_ts - .map_or(false, |expire_ts| expire_ts <= current_ts) + .is_some_and(|expire_ts| expire_ts <= current_ts) } } diff --git a/components/backup-stream/src/checkpoint_manager.rs b/components/backup-stream/src/checkpoint_manager.rs index 9896ea99b69..778ba988716 100644 --- a/components/backup-stream/src/checkpoint_manager.rs +++ b/components/backup-stream/src/checkpoint_manager.rs @@ -3,9 +3,9 @@ use std::{cell::RefCell, collections::HashMap, sync::Arc, time::Duration}; use futures::{ + FutureExt, SinkExt, StreamExt, channel::mpsc::{self as async_mpsc, Receiver, Sender}, future::BoxFuture, - FutureExt, SinkExt, StreamExt, }; use grpcio::{RpcStatus, RpcStatusCode, WriteFlags}; use kvproto::{ @@ -20,13 +20,13 @@ use txn_types::TimeStamp; use uuid::Uuid; use crate::{ - annotate, + RegionCheckpointOperation, Task, annotate, errors::{Error, Result}, future, - metadata::{store::MetaStore, Checkpoint, CheckpointProvider, MetadataClient}, + metadata::{Checkpoint, CheckpointProvider, MetadataClient, store::MetaStore}, metrics, subscription_track::ResolveResult, - try_send, RegionCheckpointOperation, Task, + try_send, }; /// A manager for maintaining the last flush ts. @@ -609,7 +609,7 @@ pub mod tests { time::Duration, }; - use futures::{future::ok, Sink}; + use futures::{Sink, future::ok}; use grpcio::{RpcStatus, RpcStatusCode}; use kvproto::{logbackuppb::SubscribeFlushEventResponse, metapb::*}; use pd_client::{PdClient, PdFuture}; @@ -617,8 +617,8 @@ pub mod tests { use super::{BasicFlushObserver, FlushObserver, RegionIdWithVersion}; use crate::{ - subscription_track::{CheckpointType, ResolveResult}, GetCheckpointResult, + subscription_track::{CheckpointType, ResolveResult}, }; fn region(id: u64, version: u64, conf_version: u64) -> Region { diff --git a/components/backup-stream/src/endpoint.rs b/components/backup-stream/src/endpoint.rs index b81279ed186..9201fcfee0c 100644 --- a/components/backup-stream/src/endpoint.rs +++ b/components/backup-stream/src/endpoint.rs @@ -15,7 +15,7 @@ use dashmap::DashMap; use encryption::BackupEncryptionManager; use engine_traits::KvEngine; use error_code::ErrorCodeExt; -use futures::{stream::AbortHandle, FutureExt, TryFutureExt}; +use futures::{FutureExt, TryFutureExt, stream::AbortHandle}; use kvproto::{ brpb::{StreamBackupError, StreamBackupTaskInfo}, metapb::{Region, RegionEpoch}, @@ -26,13 +26,13 @@ use raftstore::{ coprocessor::{CmdBatch, ObserveHandle, RegionInfoProvider}, router::CdcHandle, }; -use resolved_ts::{resolve_by_raft, LeadershipResolver}; +use resolved_ts::{LeadershipResolver, resolve_by_raft}; use tikv::{ config::{BackupStreamConfig, ResolvedTsConfig}, storage::txn::txn_status_cache::TxnStatusCache, }; use tikv_util::{ - box_err, + HandyRwLock, box_err, config::ReadableDuration, debug, defer, error, info, memory::MemoryQuota, @@ -40,12 +40,11 @@ use tikv_util::{ time::{Instant, Limiter}, warn, worker::{Runnable, Scheduler}, - HandyRwLock, }; use tokio::{ io::Result as TokioResult, runtime::{Handle, Runtime}, - sync::{mpsc::Sender, Semaphore}, + sync::{Semaphore, mpsc::Sender}, }; use tokio_stream::StreamExt; use tracing::instrument; @@ -62,7 +61,7 @@ use crate::{ errors::{Error, ReportableResult, Result}, event_loader::InitialDataLoader, future, - metadata::{store::MetaStore, MetadataClient, MetadataEvent, StreamTask}, + metadata::{MetadataClient, MetadataEvent, StreamTask, store::MetaStore}, metrics::{self, TaskStatus}, observer::BackupStreamObserver, router::{self, ApplyEvents, FlushContext, Router, TaskSelector, TaskSelectorRef}, @@ -186,7 +185,7 @@ where ), accessor.clone(), meta_client.clone(), - ((config.num_threads + 1) / 2).max(1), + (config.num_threads.div_ceil(2)).max(1), resolver, resolved_ts_config.advance_ts_interval.0, ); diff --git a/components/backup-stream/src/errors.rs b/components/backup-stream/src/errors.rs index 657fab840fa..f19016fcd38 100644 --- a/components/backup-stream/src/errors.rs +++ b/components/backup-stream/src/errors.rs @@ -79,7 +79,7 @@ impl ErrorCodeExt for Error { } } -impl<'a> ErrorCodeExt for &'a Error { +impl ErrorCodeExt for &Error { fn error_code(&self) -> error_code::ErrorCode { Error::error_code(*self) } @@ -198,7 +198,7 @@ impl Error { mod test { extern crate test; - use std::io::{self, ErrorKind}; + use std::io; use error_code::ErrorCodeExt; @@ -206,8 +206,7 @@ mod test { #[test] fn test_contextual_error() { - let err = Error::Io(io::Error::new( - ErrorKind::Other, + let err = Error::Io(io::Error::other( "the absence of error messages, is also a kind of error message", )); let result: Result<()> = Err(err); @@ -232,8 +231,7 @@ mod test { // 2,685 ns/iter (+/- 194) fn contextual_add_format_strings_directly(b: &mut test::Bencher) { b.iter(|| { - let err = Error::Io(io::Error::new( - ErrorKind::Other, + let err = Error::Io(io::Error::other( "basement, it is the fundamental basement.", )); let result: Result<()> = Err(err); @@ -253,8 +251,7 @@ mod test { // 1,922 ns/iter (+/- 273) fn contextual_add_format_strings(b: &mut test::Bencher) { b.iter(|| { - let err = Error::Io(io::Error::new( - ErrorKind::Other, + let err = Error::Io(io::Error::other( "basement, it is the fundamental basement.", )); let result: Result<()> = Err(err); @@ -274,8 +271,7 @@ mod test { // 1,988 ns/iter (+/- 89) fn contextual_add_closure(b: &mut test::Bencher) { b.iter(|| { - let err = Error::Io(io::Error::new( - ErrorKind::Other, + let err = Error::Io(io::Error::other( "basement, it is the fundamental basement.", )); let result: Result<()> = Err(err); @@ -296,8 +292,7 @@ mod test { // 773 ns/iter (+/- 8) fn baseline(b: &mut test::Bencher) { b.iter(|| { - let err = Error::Io(io::Error::new( - ErrorKind::Other, + let err = Error::Io(io::Error::other( "basement, it is the fundamental basement.", )); let result: Result<()> = Err(err); diff --git a/components/backup-stream/src/event_loader.rs b/components/backup-stream/src/event_loader.rs index a990ea2a82b..f7f18d18ea4 100644 --- a/components/backup-stream/src/event_loader.rs +++ b/components/backup-stream/src/event_loader.rs @@ -2,18 +2,18 @@ use std::{marker::PhantomData, sync::Arc, time::Duration}; -use engine_traits::{KvEngine, CF_DEFAULT, CF_WRITE}; +use engine_traits::{CF_DEFAULT, CF_WRITE, KvEngine}; use kvproto::{kvrpcpb::ExtraOp, metapb::Region, raft_cmdpb::CmdType}; use raftstore::{ coprocessor::ObserveHandle, router::CdcHandle, - store::{fsm::ChangeObserver, Callback}, + store::{Callback, fsm::ChangeObserver}, }; use tikv::storage::{ + Snapshot, Statistics, kv::StatisticsSummary, mvcc::{DeltaScanner, ScannerBuilder}, txn::{TxnEntry, TxnEntryScanner}, - Snapshot, Statistics, }; use tikv_util::{ box_err, @@ -27,12 +27,12 @@ use tracing_active_tree::frame; use txn_types::{Key, Lock, TimeStamp}; use crate::{ - annotate, debug, + Task, annotate, debug, errors::{ContextualResultExt, Error, Result}, metrics, router::{ApplyEvent, ApplyEvents, Router}, subscription_track::{Ref, RefMut, SubscriptionTracer, TwoPhaseResolver}, - utils, Task, + utils, }; const MAX_GET_SNAPSHOT_RETRY: usize = 5; @@ -484,8 +484,8 @@ mod tests { use futures::executor::block_on; use kvproto::metapb::*; use tikv::storage::{ - txn::{tests::*, txn_status_cache::TxnStatusCache}, TestEngineBuilder, + txn::{tests::*, txn_status_cache::TxnStatusCache}, }; use tikv_kv::SnapContext; use tikv_util::memory::{MemoryQuota, OwnedAllocated}; diff --git a/components/backup-stream/src/lib.rs b/components/backup-stream/src/lib.rs index 6d7afa334c9..97467fc169c 100644 --- a/components/backup-stream/src/lib.rs +++ b/components/backup-stream/src/lib.rs @@ -1,5 +1,4 @@ // Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. -#![feature(slice_group_by)] #![feature(trait_alias)] #![feature(result_flattening)] #![feature(assert_matches)] diff --git a/components/backup-stream/src/metadata/store/pd.rs b/components/backup-stream/src/metadata/store/pd.rs index 5b2e2b466e5..6ed5f693f2e 100644 --- a/components/backup-stream/src/metadata/store/pd.rs +++ b/components/backup-stream/src/metadata/store/pd.rs @@ -3,7 +3,7 @@ use std::{collections::VecDeque, fmt::Display, pin::Pin, task::ready}; use async_trait::async_trait; -use futures::{stream, Stream}; +use futures::{Stream, stream}; use kvproto::meta_storagepb::{self as mpb, WatchResponse}; use pd_client::meta_storage::{Get, MetaStorageClient, Put, Watch}; use pin_project::pin_project; @@ -200,10 +200,10 @@ mod tests { use futures::{Future, StreamExt}; use pd_client::{ - meta_storage::{Checked, Source, Sourced}, RpcClient, + meta_storage::{Checked, Source, Sourced}, }; - use test_pd::{mocker::MetaStorage, util::*, Server as PdServer}; + use test_pd::{Server as PdServer, mocker::MetaStorage, util::*}; use tikv_util::config::ReadableDuration; use super::PdStore; diff --git a/components/backup-stream/src/metadata/store/slash_etc.rs b/components/backup-stream/src/metadata/store/slash_etc.rs index a564d069d14..1afb387e596 100644 --- a/components/backup-stream/src/metadata/store/slash_etc.rs +++ b/components/backup-stream/src/metadata/store/slash_etc.rs @@ -9,8 +9,8 @@ use std::{ use async_trait::async_trait; use tokio::sync::{ - mpsc::{self, Sender}, Mutex, + mpsc::{self, Sender}, }; use tokio_stream::StreamExt; @@ -113,7 +113,7 @@ impl SlashEtc { .collect::>(); let kvs = mvccs .as_slice() - .group_by(|k1, k2| k1.0.0 == k2.0.0) + .chunk_by(|k1, k2| k1.0.0 == k2.0.0) .filter_map(|k| { let (k, v) = k.last()?; match v { diff --git a/components/backup-stream/src/metadata/test.rs b/components/backup-stream/src/metadata/test.rs index bb2b7fe1577..c7343e86754 100644 --- a/components/backup-stream/src/metadata/test.rs +++ b/components/backup-stream/src/metadata/test.rs @@ -3,17 +3,17 @@ #![cfg(test)] use std::{ - collections::{hash_map::RandomState, HashSet}, + collections::{HashSet, hash_map::RandomState}, iter::FromIterator, }; use kvproto::brpb::{Noop, StorageBackend}; use tokio_stream::StreamExt; -use super::{keys::MetaKey, MetadataClient, StreamTask}; +use super::{MetadataClient, StreamTask, keys::MetaKey}; use crate::{ errors::Result, - metadata::{store::SlashEtcStore, MetadataEvent}, + metadata::{MetadataEvent, store::SlashEtcStore}, }; pub fn test_meta_cli() -> MetadataClient { diff --git a/components/backup-stream/src/observer.rs b/components/backup-stream/src/observer.rs index f808e582e3c..ce6d8f130f8 100644 --- a/components/backup-stream/src/observer.rs +++ b/components/backup-stream/src/observer.rs @@ -6,7 +6,7 @@ use engine_traits::KvEngine; use kvproto::metapb::Region; use raft::StateRole; use raftstore::coprocessor::*; -use tikv_util::{worker::Scheduler, HandyRwLock}; +use tikv_util::{HandyRwLock, worker::Scheduler}; use crate::{ debug, @@ -185,7 +185,6 @@ impl RegionChangeObserver for BackupStreamObserver { } #[cfg(test)] - mod tests { use std::{assert_matches::assert_matches, sync::Arc, time::Duration}; @@ -198,7 +197,7 @@ mod tests { RegionChangeEvent, RegionChangeObserver, RegionChangeReason, RoleChange, RoleObserver, }; use tikv::storage::txn::txn_status_cache::TxnStatusCache; - use tikv_util::{worker::dummy_scheduler, HandyRwLock}; + use tikv_util::{HandyRwLock, worker::dummy_scheduler}; use super::BackupStreamObserver; use crate::{ diff --git a/components/backup-stream/src/router.rs b/components/backup-stream/src/router.rs index ed307871a46..93be78e2daf 100644 --- a/components/backup-stream/src/router.rs +++ b/components/backup-stream/src/router.rs @@ -7,8 +7,8 @@ use std::{ path::{Path, PathBuf}, result, sync::{ - atomic::{AtomicBool, AtomicPtr, AtomicU64, AtomicUsize, Ordering}, Arc, RwLock as SyncRwLock, + atomic::{AtomicBool, AtomicPtr, AtomicU64, AtomicUsize, Ordering}, }, time::Duration, }; @@ -16,8 +16,8 @@ use std::{ use dashmap::DashMap; use encryption::{BackupEncryptionManager, EncrypterReader, Iv, MultiMasterKeyBackend}; use encryption_export::create_async_backend; -use engine_traits::{CfName, CF_DEFAULT, CF_LOCK, CF_WRITE}; -use external_storage::{create_storage, BackendConfig, ExternalStorage, UnpinReader}; +use engine_traits::{CF_DEFAULT, CF_LOCK, CF_WRITE, CfName}; +use external_storage::{BackendConfig, ExternalStorage, UnpinReader, create_storage}; use file_system::Sha256Reader; use futures::io::Cursor; use kvproto::{ @@ -36,14 +36,13 @@ use slog_global::debug; use tidb_query_datatype::codec::table::decode_table_id; use tikv::config::BackupStreamConfig; use tikv_util::{ - box_err, + Either, HandyRwLock, box_err, codec::stream_event::EventEncoder, config::ReadableSize, error, info, time::{Instant, Limiter}, warn, worker::Scheduler, - Either, HandyRwLock, }; use tokio::{ io::AsyncWriteExt, @@ -1150,7 +1149,6 @@ impl StreamTaskHandler { ) .await .into_iter() - .map(|r| r.map_err(Error::from)) .fold(Ok(()), Result::and)?; let mut metadata = MetadataInfo::with_capacity(w.len() + wm.len()); @@ -1940,9 +1938,6 @@ impl std::fmt::Debug for DataFile { } } -#[derive(Clone, Ord, PartialOrd, PartialEq, Eq, Debug)] -struct KeyRange(Vec); - #[derive(Clone, Debug)] #[allow(dead_code)] struct TaskRange { @@ -1957,7 +1952,7 @@ mod tests { use async_compression::tokio::bufread::ZstdDecoder; use encryption::{DecrypterReader, FileConfig, MasterKeyConfig, MultiMasterKeyBackend}; use external_storage::{BlobObject, ExternalData, NoopStorage}; - use futures::{future::LocalBoxFuture, stream::LocalBoxStream, AsyncReadExt}; + use futures::{AsyncReadExt, future::LocalBoxFuture, stream::LocalBoxStream}; use kvproto::{ brpb::{CipherInfo, Noop, StorageBackend, StreamBackupTaskInfo}, encryptionpb::EncryptionMethod, @@ -1971,7 +1966,7 @@ mod tests { stream_event::{EventIterator, Iterator}, }, config::ReadableDuration, - worker::{dummy_scheduler, ReceiverWrapper}, + worker::{ReceiverWrapper, dummy_scheduler}, }; use tokio::{fs::File, io::BufReader}; use txn_types::{Write, WriteType}; @@ -2358,8 +2353,7 @@ mod tests { let mut fst = first_time.lock().unwrap(); if *fst { *fst = false; - return Err(io::Error::new( - io::ErrorKind::Other, + return Err(io::Error::other( "the absence of the result, is also a kind of result", )); } @@ -2787,8 +2781,7 @@ mod tests { if data_len == content_length { Ok(()) } else { - Err(io::Error::new( - io::ErrorKind::Other, + Err(io::Error::other( "the length of content in reader is not equal with content_length", )) } diff --git a/components/backup-stream/src/service.rs b/components/backup-stream/src/service.rs index 3e1db6f1d0d..1dc38416468 100644 --- a/components/backup-stream/src/service.rs +++ b/components/backup-stream/src/service.rs @@ -8,10 +8,11 @@ use kvproto::{logbackuppb::*, metapb::Region}; use tikv_util::{info, warn, worker::Scheduler}; use crate::{ + Task, checkpoint_manager::{GetCheckpointResult, RegionIdWithVersion}, endpoint::{RegionCheckpointOperation, RegionSet}, router::TaskSelector, - try_send, Task, + try_send, }; #[derive(Clone)] diff --git a/components/backup-stream/src/subscription_manager.rs b/components/backup-stream/src/subscription_manager.rs index 4eae7b7ee11..0ec1d13f285 100644 --- a/components/backup-stream/src/subscription_manager.rs +++ b/components/backup-stream/src/subscription_manager.rs @@ -17,24 +17,23 @@ use tikv_util::{ box_err, debug, info, memory::MemoryQuota, sys::thread::ThreadBuildWrapper, time::Instant, warn, worker::Scheduler, }; -use tokio::sync::mpsc::{channel, error::SendError, Receiver, Sender, WeakSender}; +use tokio::sync::mpsc::{Receiver, Sender, WeakSender, channel, error::SendError}; use tracing::instrument; use tracing_active_tree::root; use txn_types::TimeStamp; use crate::{ - annotate, + Task, annotate, endpoint::{BackupStreamResolver, ObserveOp}, errors::{Error, ReportableResult, Result}, event_loader::InitialDataLoader, future, - metadata::{store::MetaStore, CheckpointProvider, MetadataClient}, + metadata::{CheckpointProvider, MetadataClient, store::MetaStore}, metrics, router::{Router, TaskSelector}, subscription_track::{CheckpointType, Ref, RefMut, ResolveResult, SubscriptionTracer}, try_send, utils::{self, FutureWaitGroup, Work}, - Task, }; type ScanPool = tokio::runtime::Runtime; @@ -236,7 +235,7 @@ async fn scan_executor_loop(init: impl InitialScan, mut cmds: Receiver) /// spawn the executors to some runtime. #[cfg(test)] fn spawn_executors_to( - init: impl InitialScan + Send + Sync + 'static, + init: impl InitialScan + 'static, handle: &tokio::runtime::Handle, ) -> ScanPoolHandle { let (tx, rx) = tokio::sync::mpsc::channel(MESSAGE_BUFFER_SIZE); @@ -247,10 +246,7 @@ fn spawn_executors_to( } /// spawn the executors in the scan pool. -fn spawn_executors( - init: impl InitialScan + Send + Sync + 'static, - number: usize, -) -> ScanPoolHandle { +fn spawn_executors(init: impl InitialScan + 'static, number: usize) -> ScanPoolHandle { let (tx, rx) = tokio::sync::mpsc::channel(MESSAGE_BUFFER_SIZE); let pool = create_scan_pool(number); let handle = pool.handle().clone(); @@ -819,8 +815,8 @@ mod test { use std::{ collections::HashMap, sync::{ - atomic::{AtomicBool, Ordering}, Arc, Mutex, + atomic::{AtomicBool, Ordering}, }, time::Duration, }; @@ -833,27 +829,27 @@ mod test { metapb::{Region, RegionEpoch}, }; use raftstore::{ + RegionInfo, coprocessor::{ObserveHandle, RegionInfoCallback, RegionInfoProvider}, router::{CdcRaftRouter, ServerRaftStoreRouter}, - RegionInfo, }; use tikv::{ config::BackupStreamConfig, - storage::{txn::txn_status_cache::TxnStatusCache, Statistics}, + storage::{Statistics, txn::txn_status_cache::TxnStatusCache}, }; use tikv_util::{box_err, info, memory::MemoryQuota, worker::dummy_scheduler}; use tokio::{sync::mpsc::Sender, task::JoinHandle}; use txn_types::TimeStamp; - use super::{spawn_executors_to, InitialScan, RegionSubscriptionManager}; + use super::{InitialScan, RegionSubscriptionManager, spawn_executors_to}; use crate::{ + BackupStreamResolver, ObserveOp, Task, errors::Error, - metadata::{store::SlashEtcStore, MetadataClient, StreamTask}, + metadata::{MetadataClient, StreamTask, store::SlashEtcStore}, router::{Router, RouterInner}, subscription_manager::{OOM_BACKOFF_BASE, OOM_BACKOFF_JITTER_SECS}, subscription_track::{CheckpointType, SubscriptionTracer}, utils::FutureWaitGroup, - BackupStreamResolver, ObserveOp, Task, }; #[derive(Clone, Copy)] diff --git a/components/backup-stream/src/subscription_track.rs b/components/backup-stream/src/subscription_track.rs index e0b10418bd9..f3c87d3d00a 100644 --- a/components/backup-stream/src/subscription_track.rs +++ b/components/backup-stream/src/subscription_track.rs @@ -3,8 +3,8 @@ use std::{collections::HashSet, result::Result, sync::Arc}; use dashmap::{ - mapref::{entry::Entry, one::RefMut as DashRefMut}, DashMap, + mapref::{entry::Entry, one::RefMut as DashRefMut}, }; use kvproto::metapb::Region; use raftstore::coprocessor::*; @@ -405,6 +405,7 @@ pub trait Ref { type Key; type Value; + #[allow(dead_code)] fn key(&self) -> &Self::Key; fn value(&self) -> &Self::Value; } @@ -605,7 +606,7 @@ impl TwoPhaseResolver { txn_status_cache: Arc, ) -> Self { // TODO: limit the memory usage of the resolver. - let memory_quota = Arc::new(MemoryQuota::new(std::usize::MAX)); + let memory_quota = Arc::new(MemoryQuota::new(usize::MAX)); Self { resolver: Resolver::new(region_id, memory_quota, txn_status_cache), future_locks: Default::default(), diff --git a/components/backup-stream/src/tempfiles.rs b/components/backup-stream/src/tempfiles.rs index 612ba4f9e5f..c47406f5c32 100644 --- a/components/backup-stream/src/tempfiles.rs +++ b/components/backup-stream/src/tempfiles.rs @@ -22,10 +22,10 @@ use std::{ path::{Path, PathBuf}, pin::Pin, sync::{ - atomic::{AtomicU8, AtomicUsize, Ordering}, Arc, Mutex as BlockMutex, + atomic::{AtomicU8, AtomicUsize, Ordering}, }, - task::{ready, Context, Poll}, + task::{Context, Poll, ready}, time::Instant, }; @@ -234,8 +234,7 @@ impl TempFilePool { } }); if f.reader_count.load(Ordering::SeqCst) > 0 { - return Err(Error::new( - ErrorKind::Other, + return Err(Error::other( "open_for_write isn't allowed when there are concurrent reading.", )); } @@ -288,8 +287,7 @@ impl TempFilePool { // like cursors to allow the reader be able to access consistent // File snapshot even there are writers appending contents // to the file. But that isn't needed for now. - return Err(IoErr::new( - ErrorKind::Other, + return Err(IoErr::other( format!( "open_for_read isn't allowed when there are concurrent writing (there are still {} reads for file {}.).", refc, @@ -339,9 +337,8 @@ impl TempFilePool { fn create_relative(&self, p: &Path) -> std::io::Result { let abs_path = self.cfg.swap_files.join(p); #[cfg(test)] - match &self.override_swapout { - Some(f) => return Ok(SwappedOut::Dynamic(f(&abs_path))), - None => {} + if let Some(f) = &self.override_swapout { + return Ok(SwappedOut::Dynamic(f(&abs_path))); } let file = OsFile::from_std(SyncOsFile::create(&abs_path)?); @@ -788,15 +785,15 @@ mod test { path::Path, pin::Pin, sync::{ - atomic::{AtomicUsize, Ordering}, Arc, + atomic::{AtomicUsize, Ordering}, }, }; use async_compression::tokio::bufread::ZstdDecoder; use encryption::{BackupEncryptionManager, DataKeyManager, MultiMasterKeyBackend}; use kvproto::{brpb::CompressionType, encryptionpb::EncryptionMethod}; - use tempfile::{tempdir, TempDir}; + use tempfile::{TempDir, tempdir}; use test_util::new_test_key_manager; use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader}; use walkdir::WalkDir; diff --git a/components/backup-stream/src/utils.rs b/components/backup-stream/src/utils.rs index 051a3be033c..b019d7839f4 100644 --- a/components/backup-stream/src/utils.rs +++ b/components/backup-stream/src/utils.rs @@ -4,21 +4,21 @@ use core::pin::Pin; use std::{ borrow::Borrow, cell::RefCell, - collections::{hash_map::RandomState, BTreeMap, HashMap}, + collections::{BTreeMap, HashMap, hash_map::RandomState}, future::Future, ops::{Bound, RangeBounds}, path::Path, sync::{ - atomic::{AtomicUsize, Ordering}, Arc, + atomic::{AtomicUsize, Ordering}, }, task::{Context, Waker}, time::Duration, }; -use async_compression::{tokio::write::ZstdEncoder, Level}; +use async_compression::{Level, tokio::write::ZstdEncoder}; use engine_rocks::ReadPerfInstant; -use engine_traits::{CfName, CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE}; +use engine_traits::{CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE, CfName}; use futures::{ready, task::Poll}; use kvproto::{ brpb::CompressionType, @@ -27,13 +27,12 @@ use kvproto::{ }; use tikv::storage::CfStatistics; use tikv_util::{ - box_err, + Either, box_err, sys::inspector::{ - self_thread_inspector, IoStat, ThreadInspector, ThreadInspectorImpl as OsInspector, + IoStat, ThreadInspector, ThreadInspectorImpl as OsInspector, self_thread_inspector, }, time::Instant, worker::Scheduler, - Either, }; use tokio::{ fs::File, @@ -43,9 +42,9 @@ use tokio::{ use txn_types::{Key, Lock, LockType}; use crate::{ + Task, errors::{Error, Result}, router::TaskSelector, - Task, }; /// wrap a user key with encoded data key. @@ -812,8 +811,8 @@ impl> std::fmt::Debug for DebugIter { mod softlimit_test { use std::{ sync::{ - atomic::{AtomicU8, Ordering}, Arc, + atomic::{AtomicU8, Ordering}, }, time::Duration, }; diff --git a/components/backup/src/utils.rs b/components/backup/src/utils.rs index b13e51466f9..358a61e65b0 100644 --- a/components/backup/src/utils.rs +++ b/components/backup/src/utils.rs @@ -1,6 +1,6 @@ // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. -use api_version::{dispatch_api_version, ApiV2, KeyMode, KvFormat}; +use api_version::{ApiV2, KeyMode, KvFormat, dispatch_api_version}; use file_system::IoType; use kvproto::kvrpcpb::ApiVersion; use tikv_util::{error, sys::thread::ThreadBuildWrapper}; diff --git a/components/backup/src/writer.rs b/components/backup/src/writer.rs index 4e77f20350d..9e2e3301015 100644 --- a/components/backup/src/writer.rs +++ b/components/backup/src/writer.rs @@ -4,8 +4,8 @@ use std::{fmt::Display, io::Read}; use encryption::{EncrypterReader, Iv}; use engine_traits::{ - CfName, ExternalSstFileInfo, KvEngine, SstCompressionType, SstExt, SstWriter, SstWriterBuilder, - CF_DEFAULT, CF_WRITE, + CF_DEFAULT, CF_WRITE, CfName, ExternalSstFileInfo, KvEngine, SstCompressionType, SstExt, + SstWriter, SstWriterBuilder, }; use external_storage::{ExternalStorage, UnpinReader}; use file_system::Sha256Reader; @@ -21,7 +21,7 @@ use tikv_util::{ }; use txn_types::KvPair; -use crate::{backup_file_name, metrics::*, utils::KeyValueCodec, Error, Result}; +use crate::{Error, Result, backup_file_name, metrics::*, utils::KeyValueCodec}; #[derive(Debug, Clone, Copy)] /// CfNameWrap wraps the CfName type. @@ -467,7 +467,7 @@ mod tests { .unwrap(); assert_eq!(map.len(), kv.len(), "{} {:?} {:?}", cf, map, kv); for (k, v) in *kv { - assert_eq!(&v.to_vec(), map.get(&k.to_vec()).unwrap()); + assert_eq!(&v.to_vec(), map.get(*k).unwrap()); } } } @@ -542,10 +542,7 @@ mod tests { engine_traits::CF_WRITE, &temp.path().join(files[0].get_name()), )], - &[( - engine_traits::CF_WRITE, - &[(&keys::data_key(&[b'a']), &[b'a'])], - )], + &[(engine_traits::CF_WRITE, &[(&keys::data_key(b"a"), b"a")])], ); // Test write and default. @@ -595,16 +592,10 @@ mod tests { ), ], &[ - ( - engine_traits::CF_DEFAULT, - &[(&keys::data_key(&[b'a']), &[b'a'])], - ), + (engine_traits::CF_DEFAULT, &[(&keys::data_key(b"a"), b"a")]), ( engine_traits::CF_WRITE, - &[ - (&keys::data_key(&[b'a']), &[b'a']), - (&keys::data_key(&[b'b']), &[]), - ], + &[(&keys::data_key(b"a"), b"a"), (&keys::data_key(b"b"), &[])], ), ], ); diff --git a/components/batch-system/benches/batch-system.rs b/components/batch-system/benches/batch-system.rs index 9edf72f0ff9..d849d2460dc 100644 --- a/components/batch-system/benches/batch-system.rs +++ b/components/batch-system/benches/batch-system.rs @@ -2,7 +2,7 @@ #![feature(test)] -use std::sync::{atomic::*, Arc}; +use std::sync::{Arc, atomic::*}; use batch_system::{test_runner::*, *}; use criterion::*; diff --git a/components/batch-system/src/batch.rs b/components/batch-system/src/batch.rs index 0b7f9f45489..4791d2d02aa 100644 --- a/components/batch-system/src/batch.rs +++ b/components/batch-system/src/batch.rs @@ -10,16 +10,16 @@ use std::{ borrow::Cow, ops::{Deref, DerefMut}, - sync::{atomic::AtomicUsize, Arc, Mutex}, - thread::{self, current, JoinHandle, ThreadId}, + sync::{Arc, Mutex, atomic::AtomicUsize}, + thread::{self, JoinHandle, ThreadId, current}, time::Duration, }; use fail::fail_point; -use file_system::{set_io_type, IoType}; +use file_system::{IoType, set_io_type}; use resource_control::{ - channel::{unbounded, Receiver, Sender}, ResourceController, + channel::{Receiver, Sender, unbounded}, }; use tikv_util::{ debug, error, info, mpsc, safe_panic, sys::thread::StdThreadBuildWrapper, thd_name, diff --git a/components/batch-system/src/fsm.rs b/components/batch-system/src/fsm.rs index 9002e16e0a2..427e37e1015 100644 --- a/components/batch-system/src/fsm.rs +++ b/components/batch-system/src/fsm.rs @@ -4,10 +4,9 @@ use std::{ borrow::Cow, ptr, sync::{ - atomic::{AtomicPtr, AtomicUsize, Ordering}, Arc, + atomic::{AtomicPtr, AtomicUsize, Ordering}, }, - usize, }; use resource_control::ResourceMetered; diff --git a/components/batch-system/src/lib.rs b/components/batch-system/src/lib.rs index 1a45f2c5cdf..1f6f3ba4a60 100644 --- a/components/batch-system/src/lib.rs +++ b/components/batch-system/src/lib.rs @@ -13,8 +13,8 @@ pub mod test_runner; pub use self::{ batch::{ - create_system, BatchRouter, BatchSystem, FsmTypes, HandleResult, HandlerBuilder, - PollHandler, Poller, PoolState, + BatchRouter, BatchSystem, FsmTypes, HandleResult, HandlerBuilder, PollHandler, Poller, + PoolState, create_system, }, config::Config, fsm::{Fsm, FsmScheduler, Priority}, diff --git a/components/batch-system/src/mailbox.rs b/components/batch-system/src/mailbox.rs index 869031392af..024a275cd49 100644 --- a/components/batch-system/src/mailbox.rs +++ b/components/batch-system/src/mailbox.rs @@ -3,7 +3,7 @@ // #[PerformanceCriticalPath] use std::{ borrow::Cow, - sync::{atomic::AtomicUsize, Arc}, + sync::{Arc, atomic::AtomicUsize}, }; use crossbeam::channel::{SendError, TrySendError}; diff --git a/components/batch-system/src/router.rs b/components/batch-system/src/router.rs index 4f886fe3b3d..b37bed91476 100644 --- a/components/batch-system/src/router.rs +++ b/components/batch-system/src/router.rs @@ -2,13 +2,13 @@ // #[PerformanceCriticalPath] use std::sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering}, Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, }; use crossbeam::channel::{SendError, TrySendError}; use dashmap::DashMap; -use tikv_util::{debug, info, time::Instant, Either}; +use tikv_util::{Either, debug, info, time::Instant}; use crate::{ fsm::{Fsm, FsmScheduler}, diff --git a/components/batch-system/src/scheduler.rs b/components/batch-system/src/scheduler.rs index 12db92b8fa7..a9289cfbd34 100644 --- a/components/batch-system/src/scheduler.rs +++ b/components/batch-system/src/scheduler.rs @@ -5,8 +5,8 @@ use resource_control::channel::Sender; use tikv_util::{time::Instant, warn}; use crate::{ - fsm::{Fsm, FsmScheduler, Priority}, FsmTypes, + fsm::{Fsm, FsmScheduler, Priority}, }; pub struct NormalScheduler { pub(crate) sender: Sender>, diff --git a/components/batch-system/src/test_runner.rs b/components/batch-system/src/test_runner.rs index 7723fa88882..bcbdfbc2a27 100644 --- a/components/batch-system/src/test_runner.rs +++ b/components/batch-system/src/test_runner.rs @@ -6,8 +6,8 @@ use std::{ borrow::Cow, ops::DerefMut, sync::{ - atomic::{AtomicUsize, Ordering}, Arc, Mutex, + atomic::{AtomicUsize, Ordering}, }, }; diff --git a/components/batch-system/tests/cases/batch.rs b/components/batch-system/tests/cases/batch.rs index dc13affc363..9246edffe0f 100644 --- a/components/batch-system/tests/cases/batch.rs +++ b/components/batch-system/tests/cases/batch.rs @@ -1,7 +1,7 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. use std::{ - sync::{atomic::AtomicUsize, Arc}, + sync::{Arc, atomic::AtomicUsize}, thread::sleep, time::Duration, }; diff --git a/components/batch-system/tests/cases/router.rs b/components/batch-system/tests/cases/router.rs index 66d0770d544..b5dc0688d77 100644 --- a/components/batch-system/tests/cases/router.rs +++ b/components/batch-system/tests/cases/router.rs @@ -1,7 +1,7 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. use std::{ - sync::{atomic::*, Arc}, + sync::{Arc, atomic::*}, time::Duration, }; diff --git a/components/causal_ts/src/lib.rs b/components/causal_ts/src/lib.rs index ab57fbf734f..fa34b032f14 100644 --- a/components/causal_ts/src/lib.rs +++ b/components/causal_ts/src/lib.rs @@ -1,7 +1,5 @@ // Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. -#![feature(div_duration)] - #[macro_use] extern crate tikv_util; @@ -42,8 +40,8 @@ pub enum CausalTsProviderImpl { pub mod tests { use std::sync::{ - atomic::{AtomicU64, Ordering}, Arc, + atomic::{AtomicU64, Ordering}, }; use super::*; diff --git a/components/causal_ts/src/tso.rs b/components/causal_ts/src/tso.rs index 51f1824f7a6..b4527bd95b7 100644 --- a/components/causal_ts/src/tso.rs +++ b/components/causal_ts/src/tso.rs @@ -2,8 +2,8 @@ //! ## The algorithm to make the TSO cache tolerate failure of TSO service //! -//! 1. The expected total size (in duration) of TSO cache is specified by -//! config item `causal-ts.alloc-ahead-buffer`. +//! 1. The expected total size (in duration) of TSO cache is specified by config +//! item `causal-ts.alloc-ahead-buffer`. //! //! 2. Count usage of TSO on every renew interval. //! @@ -11,10 +11,10 @@ //! causal-ts.renew-interval`. //! //! 4. Then `tso_usage x cache_multiplier` is the expected number of TSO should -//! be cached. +//! be cached. //! //! 5. And `tso_usage x cache_multiplier - tso_remain` is the expected number of -//! TSO to be requested from TSO service (if it's not a flush). +//! TSO to be requested from TSO service (if it's not a flush). //! //! Others: //! * `cache_multiplier` is also used as capacity of TSO batch list, as we @@ -25,8 +25,8 @@ use std::{ collections::BTreeMap, error, result, sync::{ - atomic::{AtomicI32, AtomicU32, AtomicU64, Ordering}, Arc, + atomic::{AtomicI32, AtomicU32, AtomicU64, Ordering}, }, }; @@ -47,9 +47,9 @@ use tokio::sync::{ use txn_types::TimeStamp; use crate::{ + CausalTsProvider, errors::{Error, Result}, metrics::*, - CausalTsProvider, }; /// Renew on every 100ms, to adjust batch size rapidly enough. @@ -131,11 +131,11 @@ impl TsoBatch { /// `TsoBatchList` is a ordered list of `TsoBatch`. It aims to: /// /// 1. Cache more number of TSO to improve high availability. See issue #12794. -/// `TsoBatch` can only cache at most 262144 TSO as logical clock is 18 bits. +/// `TsoBatch` can only cache at most 262144 TSO as logical clock is 18 bits. /// /// 2. Fully utilize cached TSO when some regions require latest TSO (e.g. in -/// the scenario of leader transfer). Other regions without the requirement can -/// still use older TSO cache. +/// the scenario of leader transfer). Other regions without the requirement +/// can still use older TSO cache. #[derive(Default, Debug)] pub struct TsoBatchList { inner: RwLock, @@ -157,11 +157,11 @@ pub struct TsoBatchList { /// The reasons why `crossbeam_skiplist::SkipMap` is not chosen: /// /// 1. In `flush()` procedure, a reader of `SkipMap` can still acquire a batch -/// after the it is removed, which would violate the causality requirement. -/// The `RwLock` avoid this scenario by lock synchronization. +/// after the it is removed, which would violate the causality requirement. +/// The `RwLock` avoid this scenario by lock synchronization. /// /// 2. It is a scenario with much more reads than writes. The `RwLock` would not -/// be less efficient than lock free implementation. +/// be less efficient than lock free implementation. type TsoBatchListInner = BTreeMap; impl TsoBatchList { @@ -423,11 +423,10 @@ impl BatchTsoProvider { Ok(ts) => { tso_batch_list .push(new_batch_size, ts, need_flush) - .map_err(|e| { + .inspect_err(|_e| { if need_flush { tso_batch_list.flush(); } - e })?; debug!("BatchTsoProvider::renew_tso_batch"; "tso_batch_list.remain" => tso_batch_list.remain(), "ts" => ?ts); diff --git a/components/cdc/benches/cdc_event.rs b/components/cdc/benches/cdc_event.rs index 1ea57f5c703..c23852149b1 100644 --- a/components/cdc/benches/cdc_event.rs +++ b/components/cdc/benches/cdc_event.rs @@ -1,7 +1,7 @@ // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. use cdc::CdcEvent; -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; use kvproto::cdcpb::ResolvedTs; use protobuf::Message; diff --git a/components/cdc/src/channel.rs b/components/cdc/src/channel.rs index 98520dc7b02..0a3eb15d04c 100644 --- a/components/cdc/src/channel.rs +++ b/components/cdc/src/channel.rs @@ -3,19 +3,20 @@ use std::{ fmt, sync::{ - atomic::{AtomicBool, Ordering}, Arc, + atomic::{AtomicBool, Ordering}, }, time::Duration, }; use futures::{ + SinkExt, Stream, StreamExt, channel::mpsc::{ - channel as bounded, unbounded, Receiver, SendError as FuturesSendError, Sender, - TrySendError, UnboundedReceiver, UnboundedSender, + Receiver, SendError as FuturesSendError, Sender, TrySendError, UnboundedReceiver, + UnboundedSender, channel as bounded, unbounded, }, executor::block_on, - stream, SinkExt, Stream, StreamExt, + stream, }; use grpcio::WriteFlags; use kvproto::cdcpb::{ChangeDataEvent, Event, ResolvedTs}; @@ -459,13 +460,13 @@ where mod tests { use std::{ assert_matches::assert_matches, - sync::{mpsc, Arc}, + sync::{Arc, mpsc}, time::Duration, }; use futures::executor::block_on; use kvproto::cdcpb::{ - ChangeDataEvent, Event, EventEntries, EventRow, Event_oneof_event, ResolvedTs, + ChangeDataEvent, Event, Event_oneof_event, EventEntries, EventRow, ResolvedTs, }; use super::*; diff --git a/components/cdc/src/delegate.rs b/components/cdc/src/delegate.rs index db5d67f5887..2c28e2069a4 100644 --- a/components/cdc/src/delegate.rs +++ b/components/cdc/src/delegate.rs @@ -7,8 +7,8 @@ use std::{ result::Result as StdResult, string::String, sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering}, Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, }, time::Duration, }; @@ -18,19 +18,19 @@ use collections::HashMap; use crossbeam::atomic::AtomicCell; use kvproto::{ cdcpb::{ - ChangeDataRequestKvApi, Error as EventError, Event, EventEntries, EventLogType, EventRow, - EventRowOpType, Event_oneof_event, + ChangeDataRequestKvApi, Error as EventError, Event, Event_oneof_event, EventEntries, + EventLogType, EventRow, EventRowOpType, }, kvrpcpb::ExtraOp as TxnExtraOp, metapb::{Region, RegionEpoch}, raft_cmdpb::{AdminCmdType, AdminRequest, AdminResponse, CmdType, PutRequest, Request}, }; use raftstore::{ + Error as RaftStoreError, coprocessor::{Cmd, CmdBatch, ObserveHandle}, store::util::compare_region_epoch, - Error as RaftStoreError, }; -use tikv::storage::{txn::TxnEntry, Statistics}; +use tikv::storage::{Statistics, txn::TxnEntry}; use tikv_util::{ debug, info, memory::{HeapSize, MemoryQuota}, @@ -40,14 +40,14 @@ use tikv_util::{ use txn_types::{Key, Lock, LockType, TimeStamp, WriteBatchFlags, WriteRef, WriteType}; use crate::{ - channel::{CdcEvent, SendError, Sink, CDC_EVENT_MAX_BYTES}, + Error, Result, + channel::{CDC_EVENT_MAX_BYTES, CdcEvent, SendError, Sink}, endpoint::Advance, initializer::KvEntry, metrics::*, old_value::{OldValueCache, OldValueCallback}, service::{Conn, ConnId, FeatureGate, RequestId}, txn_source::TxnSource, - Error, Result, }; static DOWNSTREAM_ID_ALLOC: AtomicUsize = AtomicUsize::new(0); diff --git a/components/cdc/src/endpoint.rs b/components/cdc/src/endpoint.rs index 0100fd1e35c..7527d7081ac 100644 --- a/components/cdc/src/endpoint.rs +++ b/components/cdc/src/endpoint.rs @@ -5,8 +5,8 @@ use std::{ collections::{BTreeMap, BinaryHeap}, fmt, sync::{ - atomic::{AtomicBool, AtomicIsize, Ordering}, Arc, Mutex as StdMutex, + atomic::{AtomicBool, AtomicIsize, Ordering}, }, time::Duration, }; @@ -35,11 +35,11 @@ use raftstore::{ router::CdcHandle, store::fsm::store::StoreRegionMeta, }; -use resolved_ts::{resolve_by_raft, LeadershipResolver}; +use resolved_ts::{LeadershipResolver, resolve_by_raft}; use security::SecurityManager; use tikv::{ config::{CdcConfig, ResolvedTsConfig}, - storage::{kv::LocalTablets, Statistics}, + storage::{Statistics, kv::LocalTablets}, }; use tikv_util::{ debug, defer, error, impl_display_as_debug, info, @@ -59,13 +59,13 @@ use tokio::{ use txn_types::{Key, TimeStamp, TxnExtra, TxnExtraScheduler}; use crate::{ + CdcObserver, Error, channel::{CdcEvent, SendError}, - delegate::{on_init_downstream, Delegate, Downstream, DownstreamId, DownstreamState, MiniLock}, + delegate::{Delegate, Downstream, DownstreamId, DownstreamState, MiniLock, on_init_downstream}, initializer::Initializer, metrics::*, old_value::{OldValueCache, OldValueCallback}, - service::{validate_kv_api, Conn, ConnId, FeatureGate, RequestId}, - CdcObserver, Error, + service::{Conn, ConnId, FeatureGate, RequestId, validate_kv_api}, }; const FEATURE_RESOLVED_TS_STORE: Feature = Feature::require(5, 0, 0); @@ -640,25 +640,25 @@ impl, E: KvEngine, S: StoreRegionMeta> Endpoint 0 { self.config.incremental_scan_speed_limit.0 as f64 } else { @@ -667,7 +667,7 @@ impl, E: KvEngine, S: StoreRegionMeta> Endpoint 0 { self.config.incremental_fetch_speed_limit.0 as f64 } else { @@ -1383,23 +1383,23 @@ mod tests { use raftstore::{ errors::{DiscardReason, Error as RaftStoreError}, router::{CdcRaftRouter, RaftStoreRouter}, - store::{fsm::StoreMeta, msg::CasualMessage, PeerMsg, ReadDelegate}, + store::{PeerMsg, ReadDelegate, fsm::StoreMeta, msg::CasualMessage}, }; use test_pd_client::TestPdClient; use test_raftstore::MockRaftStoreRouter; use tikv::{ server::DEFAULT_CLUSTER_ID, - storage::{kv::Engine, TestEngineBuilder}, + storage::{TestEngineBuilder, kv::Engine}, }; use tikv_util::{ config::{ReadableDuration, ReadableSize}, - worker::{dummy_scheduler, ReceiverWrapper}, + worker::{ReceiverWrapper, dummy_scheduler}, }; use super::*; use crate::{ channel, - delegate::{post_init_downstream, ObservedRange}, + delegate::{ObservedRange, post_init_downstream}, recv_timeout, }; diff --git a/components/cdc/src/initializer.rs b/components/cdc/src/initializer.rs index 7017cf36ba3..a56f0831615 100644 --- a/components/cdc/src/initializer.rs +++ b/components/cdc/src/initializer.rs @@ -2,18 +2,19 @@ use std::{ collections::BTreeMap, sync::{ - atomic::{AtomicBool, Ordering}, Arc, + atomic::{AtomicBool, Ordering}, }, time::Duration, }; use api_version::ApiV2; use crossbeam::atomic::AtomicCell; -use engine_rocks::{ReadPerfContext, ReadPerfInstant, PROP_MAX_TS}; +use engine_rocks::{PROP_MAX_TS, ReadPerfContext, ReadPerfInstant}; use engine_traits::{ - IterOptions, KvEngine, Range, Snapshot as EngineSnapshot, TablePropertiesCollection, - TablePropertiesExt, UserCollectedProperties, CF_DEFAULT, CF_WRITE, DATA_KEY_PREFIX_LEN, + CF_DEFAULT, CF_WRITE, DATA_KEY_PREFIX_LEN, IterOptions, KvEngine, Range, + Snapshot as EngineSnapshot, TablePropertiesCollection, TablePropertiesExt, + UserCollectedProperties, }; use fail::fail_point; use keys::{data_end_key, data_key}; @@ -31,36 +32,35 @@ use raftstore::{ }, }; use tikv::storage::{ + Statistics, kv::Snapshot, mvcc::{DeltaScanner, MvccReader, ScannerBuilder}, raw::raw_mvcc::{RawMvccIterator, RawMvccSnapshot}, txn::{TxnEntry, TxnEntryScanner}, - Statistics, }; use tikv_kv::{Iterator, ScanMode}; use tikv_util::{ - box_err, + Either, box_err, codec::number, debug, defer, error, info, - sys::inspector::{self_thread_inspector, ThreadInspector}, - time::{duration_to_sec, Instant, Limiter}, + sys::inspector::{ThreadInspector, self_thread_inspector}, + time::{Instant, Limiter, duration_to_sec}, warn, worker::Scheduler, - Either, }; use tokio::sync::Semaphore; use txn_types::{Key, KvPair, LockType, OldValue, TimeStamp}; use crate::{ + Error, Result, Task, channel::CdcEvent, delegate::{ - post_init_downstream, Delegate, DownstreamId, DownstreamState, MiniLock, ObservedRange, + Delegate, DownstreamId, DownstreamState, MiniLock, ObservedRange, post_init_downstream, }, endpoint::Deregister, metrics::*, - old_value::{near_seek_old_value, OldValueCursors}, + old_value::{OldValueCursors, near_seek_old_value}, service::{ConnId, RequestId}, - Error, Result, Task, }; #[derive(Copy, Clone, Debug, Default)] @@ -611,7 +611,7 @@ impl Initializer { if let Some((_, keys)) = prop.approximate_size_and_keys(&start_key, &end_key) { total_count += keys; if Self::parse_u64_prop(prop, PROP_MAX_TS) - .map_or(false, |max_ts| max_ts < hint_min_ts) + .is_some_and(|max_ts| max_ts < hint_min_ts) { filtered_count += keys; } @@ -645,18 +645,18 @@ mod tests { collections::BTreeMap, fmt::Display, sync::{ - atomic::AtomicBool, - mpsc::{channel, sync_channel, Receiver, RecvTimeoutError, Sender}, Arc, + atomic::AtomicBool, + mpsc::{Receiver, RecvTimeoutError, Sender, channel, sync_channel}, }, time::Duration, }; use engine_rocks::{BlobRunMode, RocksEngine}; - use engine_traits::{MiscExt, CF_WRITE}; - use futures::{executor::block_on, StreamExt}; + use engine_traits::{CF_WRITE, MiscExt}; + use futures::{StreamExt, executor::block_on}; use kvproto::{ - cdcpb::{EventLogType, Event_oneof_event}, + cdcpb::{Event_oneof_event, EventLogType}, errorpb::Error as ErrorHeader, }; use raftstore::{coprocessor::ObserveHandle, router::CdcRaftRouter}; @@ -664,12 +664,12 @@ mod tests { use tikv::{ config::DbConfig, storage::{ + TestEngineBuilder, kv::Engine, txn::tests::{ must_acquire_pessimistic_lock, must_commit, must_prewrite_delete, must_prewrite_put, must_prewrite_put_with_txn_soucre, }, - TestEngineBuilder, }, }; use tikv_util::{ diff --git a/components/cdc/src/lib.rs b/components/cdc/src/lib.rs index 64f110f5c45..3498fbdba00 100644 --- a/components/cdc/src/lib.rs +++ b/components/cdc/src/lib.rs @@ -15,7 +15,7 @@ mod old_value; mod service; mod txn_source; -pub use channel::{recv_timeout, CdcEvent}; +pub use channel::{CdcEvent, recv_timeout}; pub use config::CdcConfigManager; pub use delegate::Delegate; pub use endpoint::{CdcTxnExtraScheduler, Endpoint, Task, Validate}; diff --git a/components/cdc/src/observer.rs b/components/cdc/src/observer.rs index de898a8ae72..7a3e0d63685 100644 --- a/components/cdc/src/observer.rs +++ b/components/cdc/src/observer.rs @@ -7,14 +7,14 @@ use engine_traits::KvEngine; use fail::fail_point; use kvproto::metapb::{Peer, Region}; use raft::StateRole; -use raftstore::{coprocessor::*, store::RegionSnapshot, Error as RaftStoreError}; +use raftstore::{Error as RaftStoreError, coprocessor::*, store::RegionSnapshot}; use tikv::storage::Statistics; use tikv_util::{error, memory::MemoryQuota, warn, worker::Scheduler}; use crate::{ + Error as CdcError, endpoint::{Deregister, Task}, old_value::{self, OldValueCache}, - Error as CdcError, }; /// An Observer for CDC. diff --git a/components/cdc/src/old_value.rs b/components/cdc/src/old_value.rs index 1ab5a3a7510..bbb5de229a6 100644 --- a/components/cdc/src/old_value.rs +++ b/components/cdc/src/old_value.rs @@ -2,22 +2,22 @@ use std::ops::{Bound, Deref}; -use engine_traits::{ReadOptions, CF_DEFAULT, CF_WRITE}; +use engine_traits::{CF_DEFAULT, CF_WRITE, ReadOptions}; use getset::CopyGetters; use tikv::storage::{ - mvcc::near_load_data_by_write, Cursor, CursorBuilder, ScanMode, Snapshot as EngineSnapshot, - Statistics, + Cursor, CursorBuilder, ScanMode, Snapshot as EngineSnapshot, Statistics, + mvcc::near_load_data_by_write, }; use tikv_kv::Snapshot; use tikv_util::{ + Either, config::ReadableSize, lru::{LruCache, SizePolicy}, time::Instant, - Either, }; use txn_types::{Key, MutationType, OldValue, TimeStamp, Value, WriteRef, WriteType}; -use crate::{metrics::*, Result}; +use crate::{Result, metrics::*}; pub(crate) type OldValueCallback = Box< dyn Fn(Key, TimeStamp, &mut OldValueCache, &mut Statistics) -> Result>> + Send, @@ -534,7 +534,10 @@ mod tests { let mut default_cursor = new_old_value_cursor(&snapshot, CF_DEFAULT); let mut load_default = |use_default_cursor: bool| { if use_default_cursor { - let x = unsafe { std::mem::transmute::<_, &'static mut _>(&mut default_cursor) }; + let x = unsafe { + #[allow(clippy::missing_transmute_annotations)] + std::mem::transmute::<_, &'static mut _>(&mut default_cursor) + }; Either::Right(x) } else { Either::Left(&snapshot) @@ -548,7 +551,7 @@ mod tests { let key = Key::from_raw(&raw_key).append_ts(150.into()); let ld = load_default(use_default_cursor); let v = near_seek_old_value(&key, &mut cursor, ld, &mut stats).unwrap(); - assert!(v.map_or(false, |x| x == value())); + assert!(v.is_some_and(|x| x == value())); } assert_eq!(stats.write.seek, 1); assert_eq!(stats.write.next, 58); @@ -567,7 +570,7 @@ mod tests { let key = Key::from_raw(&raw_key).append_ts(150.into()); let ld = load_default(use_default_cursor); let v = near_seek_old_value(&key, &mut cursor, ld, &mut stats).unwrap(); - assert!(v.map_or(false, |x| x == value())); + assert!(v.is_some_and(|x| x == value())); } assert_eq!(stats.write.seek, 2); assert_eq!(stats.write.next, 144); diff --git a/components/cdc/src/service.rs b/components/cdc/src/service.rs index 6f1a7c08d08..999f4df5ea9 100644 --- a/components/cdc/src/service.rs +++ b/components/cdc/src/service.rs @@ -1,8 +1,8 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. use std::sync::{ - atomic::{AtomicUsize, Ordering}, Arc, + atomic::{AtomicUsize, Ordering}, }; use collections::{HashMap, HashMapEntry}; @@ -11,15 +11,15 @@ use futures::stream::TryStreamExt; use grpcio::{DuplexSink, RequestStream, RpcContext, RpcStatus, RpcStatusCode}; use kvproto::{ cdcpb::{ - ChangeData, ChangeDataEvent, ChangeDataRequest, ChangeDataRequestKvApi, - ChangeDataRequest_oneof_request, + ChangeData, ChangeDataEvent, ChangeDataRequest, ChangeDataRequest_oneof_request, + ChangeDataRequestKvApi, }, kvrpcpb::ApiVersion, }; use tikv_util::{error, info, memory::MemoryQuota, warn, worker::*}; use crate::{ - channel::{channel, Sink, CDC_CHANNLE_CAPACITY}, + channel::{CDC_CHANNLE_CAPACITY, Sink, channel}, delegate::{Downstream, DownstreamId, DownstreamState, ObservedRange}, endpoint::{Deregister, Task}, }; @@ -526,13 +526,13 @@ async fn sleep_before_drain_change_event() { mod tests { use std::{sync::Arc, time::Duration}; - use futures::{executor::block_on, SinkExt}; + use futures::{SinkExt, executor::block_on}; use grpcio::{self, ChannelBuilder, EnvBuilder, Server, ServerBuilder, WriteFlags}; - use kvproto::cdcpb::{create_change_data, ChangeDataClient, ResolvedTs}; + use kvproto::cdcpb::{ChangeDataClient, ResolvedTs, create_change_data}; use tikv_util::future::block_on_timeout; use super::*; - use crate::channel::{recv_timeout, CdcEvent}; + use crate::channel::{CdcEvent, recv_timeout}; fn new_rpc_suite(capacity: usize) -> (Server, ChangeDataClient, ReceiverWrapper) { let memory_quota = Arc::new(MemoryQuota::new(capacity)); diff --git a/components/cdc/tests/failpoints/test_endpoint.rs b/components/cdc/tests/failpoints/test_endpoint.rs index 018946fd89b..59c3e06e542 100644 --- a/components/cdc/tests/failpoints/test_endpoint.rs +++ b/components/cdc/tests/failpoints/test_endpoint.rs @@ -1,23 +1,23 @@ // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. use std::{ - sync::{mpsc, Arc}, + sync::{Arc, mpsc}, thread, time::{Duration, Instant}, }; -use api_version::{test_kv_format_impl, KvFormat}; +use api_version::{KvFormat, test_kv_format_impl}; use causal_ts::CausalTsProvider; -use cdc::{recv_timeout, Delegate, OldValueCache, Task, Validate}; +use cdc::{Delegate, OldValueCache, Task, Validate, recv_timeout}; use futures::{executor::block_on, sink::SinkExt}; use grpcio::{ChannelBuilder, Environment, WriteFlags}; use kvproto::{cdcpb::*, kvrpcpb::*, tikvpb_grpc::TikvClient}; use pd_client::PdClient; use test_raftstore::*; -use tikv_util::{debug, worker::Scheduler, HandyRwLock}; +use tikv_util::{HandyRwLock, debug, worker::Scheduler}; use txn_types::{Key, TimeStamp}; -use crate::{new_event_feed, new_event_feed_v2, ClientReceiver, TestSuite, TestSuiteBuilder}; +use crate::{ClientReceiver, TestSuite, TestSuiteBuilder, new_event_feed, new_event_feed_v2}; #[test] fn test_cdc_double_scan_deregister() { diff --git a/components/cdc/tests/failpoints/test_memory_quota.rs b/components/cdc/tests/failpoints/test_memory_quota.rs index 3d331deda3d..07184993336 100644 --- a/components/cdc/tests/failpoints/test_memory_quota.rs +++ b/components/cdc/tests/failpoints/test_memory_quota.rs @@ -3,13 +3,13 @@ use std::{sync::*, time::Duration}; use cdc::{Task, Validate}; -use futures::{executor::block_on, SinkExt}; +use futures::{SinkExt, executor::block_on}; use grpcio::WriteFlags; use kvproto::{cdcpb::*, kvrpcpb::*}; use pd_client::PdClient; use test_raftstore::*; -use crate::{new_event_feed, TestSuiteBuilder}; +use crate::{TestSuiteBuilder, new_event_feed}; #[test] fn test_resolver_track_lock_memory_quota_exceeded() { diff --git a/components/cdc/tests/failpoints/test_observe.rs b/components/cdc/tests/failpoints/test_observe.rs index 4a34185de76..1b350fe3493 100644 --- a/components/cdc/tests/failpoints/test_observe.rs +++ b/components/cdc/tests/failpoints/test_observe.rs @@ -2,22 +2,23 @@ use std::{ sync::{ + Arc, atomic::{AtomicBool, Ordering}, - mpsc, Arc, + mpsc, }, time::Duration, }; -use api_version::{test_kv_format_impl, KvFormat}; +use api_version::{KvFormat, test_kv_format_impl}; use futures::{executor::block_on, sink::SinkExt}; use grpcio::WriteFlags; use kvproto::{cdcpb::*, kvrpcpb::*, raft_serverpb::RaftMessage}; use pd_client::PdClient; use raft::eraftpb::MessageType; use test_raftstore::*; -use tikv_util::{config::ReadableDuration, HandyRwLock}; +use tikv_util::{HandyRwLock, config::ReadableDuration}; -use crate::{new_event_feed, TestSuite, TestSuiteBuilder}; +use crate::{TestSuite, TestSuiteBuilder, new_event_feed}; #[test] fn test_observe_duplicate_cmd() { diff --git a/components/cdc/tests/failpoints/test_register.rs b/components/cdc/tests/failpoints/test_register.rs index 2128dff08e1..0ad85de1a6b 100644 --- a/components/cdc/tests/failpoints/test_register.rs +++ b/components/cdc/tests/failpoints/test_register.rs @@ -1,7 +1,7 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. use std::{thread, time::Duration}; -use api_version::{test_kv_format_impl, KvFormat}; +use api_version::{KvFormat, test_kv_format_impl}; use futures::{executor::block_on, sink::SinkExt}; use grpcio::WriteFlags; use kvproto::{cdcpb::*, kvrpcpb::*, metapb::RegionEpoch}; @@ -10,7 +10,7 @@ use raft::StateRole; use raftstore::coprocessor::{ObserverContext, RoleChange, RoleObserver}; use test_raftstore::sleep_ms; -use crate::{new_event_feed, TestSuite}; +use crate::{TestSuite, new_event_feed}; #[test] fn test_failed_pending_batch() { diff --git a/components/cdc/tests/failpoints/test_resolve.rs b/components/cdc/tests/failpoints/test_resolve.rs index 560eb68ba44..6f1e63f2d6c 100644 --- a/components/cdc/tests/failpoints/test_resolve.rs +++ b/components/cdc/tests/failpoints/test_resolve.rs @@ -1,7 +1,7 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. use std::time::Duration; -use api_version::{test_kv_format_impl, KvFormat}; +use api_version::{KvFormat, test_kv_format_impl}; use futures::{executor::block_on, sink::SinkExt}; use grpcio::WriteFlags; use kvproto::{cdcpb::*, kvrpcpb::*}; @@ -10,7 +10,7 @@ use raft::eraftpb::ConfChangeType; use test_raftstore::*; use tikv_util::config::*; -use crate::{new_event_feed, TestSuite, TestSuiteBuilder}; +use crate::{TestSuite, TestSuiteBuilder, new_event_feed}; #[test] fn test_stale_resolver() { diff --git a/components/cdc/tests/integrations/test_cdc.rs b/components/cdc/tests/integrations/test_cdc.rs index c89bdcbb55a..2635f1efda3 100644 --- a/components/cdc/tests/integrations/test_cdc.rs +++ b/components/cdc/tests/integrations/test_cdc.rs @@ -2,20 +2,20 @@ use std::{sync::*, time::Duration}; -use api_version::{test_kv_format_impl, KvFormat}; -use cdc::{metrics::CDC_RESOLVED_TS_ADVANCE_METHOD, Task, Validate}; +use api_version::{KvFormat, test_kv_format_impl}; +use cdc::{Task, Validate, metrics::CDC_RESOLVED_TS_ADVANCE_METHOD}; use concurrency_manager::ConcurrencyManager; -use futures::{executor::block_on, SinkExt}; +use futures::{SinkExt, executor::block_on}; use grpcio::WriteFlags; use kvproto::{cdcpb::*, kvrpcpb::*}; use pd_client::PdClient; use raft::eraftpb::MessageType; use test_raftstore::*; use tikv::server::DEFAULT_CLUSTER_ID; -use tikv_util::{config::ReadableDuration, HandyRwLock}; +use tikv_util::{HandyRwLock, config::ReadableDuration}; use txn_types::{Key, Lock, LockType, TimeStamp}; -use crate::{new_event_feed, new_event_feed_v2, TestSuite, TestSuiteBuilder}; +use crate::{TestSuite, TestSuiteBuilder, new_event_feed, new_event_feed_v2}; #[test] fn test_cdc_basic() { diff --git a/components/cdc/tests/integrations/test_flow_control.rs b/components/cdc/tests/integrations/test_flow_control.rs index 77edbcc7e2d..854f4e82ea8 100644 --- a/components/cdc/tests/integrations/test_flow_control.rs +++ b/components/cdc/tests/integrations/test_flow_control.rs @@ -3,13 +3,13 @@ use std::{sync::*, time::Duration}; use cdc::{Task, Validate}; -use futures::{executor::block_on, SinkExt}; +use futures::{SinkExt, executor::block_on}; use grpcio::WriteFlags; use kvproto::{cdcpb::*, kvrpcpb::*}; use pd_client::PdClient; use test_raftstore::*; -use crate::{new_event_feed, TestSuiteBuilder}; +use crate::{TestSuiteBuilder, new_event_feed}; #[test] fn test_cdc_congest() { diff --git a/components/cdc/tests/mod.rs b/components/cdc/tests/mod.rs index 56fa39720ab..c0a73592108 100644 --- a/components/cdc/tests/mod.rs +++ b/components/cdc/tests/mod.rs @@ -6,7 +6,7 @@ use std::{ }; use causal_ts::CausalTsProvider; -use cdc::{recv_timeout, CdcObserver, Delegate, FeatureGate, Task, Validate}; +use cdc::{CdcObserver, Delegate, FeatureGate, Task, Validate, recv_timeout}; use collections::HashMap; use concurrency_manager::ConcurrencyManager; use engine_rocks::RocksEngine; @@ -16,7 +16,7 @@ use grpcio::{ Environment, MetadataBuilder, }; use kvproto::{ - cdcpb::{create_change_data, ChangeDataClient, ChangeDataEvent, ChangeDataRequest}, + cdcpb::{ChangeDataClient, ChangeDataEvent, ChangeDataRequest, create_change_data}, kvrpcpb::{PrewriteRequestPessimisticAction::*, *}, tikvpb::TikvClient, }; @@ -29,10 +29,10 @@ use tikv::{ storage::kv::LocalTablets, }; use tikv_util::{ + HandyRwLock, config::ReadableDuration, memory::MemoryQuota, worker::{LazyWorker, Runnable}, - HandyRwLock, }; use txn_types::TimeStamp; static INIT: Once = Once::new(); diff --git a/components/cloud/aws/src/kms.rs b/components/cloud/aws/src/kms.rs index f877d159778..08bfe67aa81 100644 --- a/components/cloud/aws/src/kms.rs +++ b/components/cloud/aws/src/kms.rs @@ -4,12 +4,12 @@ use std::ops::Deref; use async_trait::async_trait; use aws_config::BehaviorVersion; -use aws_credential_types::provider::{error::CredentialsError, ProvideCredentials}; +use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError}; use aws_sdk_kms::{ + Client, operation::{decrypt::DecryptError, generate_data_key::GenerateDataKeyError}, primitives::Blob, types::DataKeySpec, - Client, }; use aws_sdk_s3::config::HttpClient; use cloud::{ @@ -18,7 +18,7 @@ use cloud::{ }; use futures::executor::block_on; -use crate::util::{self, is_retryable, SdkError}; +use crate::util::{self, SdkError, is_retryable}; const AWS_KMS_DATA_KEY_SPEC: DataKeySpec = DataKeySpec::Aes256; @@ -352,8 +352,8 @@ mod tests { client.assert_requests_match(&[]); } + #[ignore = "no available aws ksm backend for the test env."] #[tokio::test] - #[cfg(FALSE)] // FIXME: enable this (or move this to an integration test) async fn test_aws_kms_localstack() { let config = Config { @@ -365,6 +365,7 @@ mod tests { }, azure: None, gcp: None, + aws: None, }; let creds = diff --git a/components/cloud/aws/src/s3.rs b/components/cloud/aws/src/s3.rs index 0c2092bc2bc..ec52a4bfe61 100644 --- a/components/cloud/aws/src/s3.rs +++ b/components/cloud/aws/src/s3.rs @@ -7,41 +7,41 @@ use std::{ }; use async_trait::async_trait; -use aws_config::{sts::AssumeRoleProvider, BehaviorVersion, Region, SdkConfig}; -use aws_credential_types::{provider::ProvideCredentials, Credentials}; +use aws_config::{BehaviorVersion, Region, SdkConfig, sts::AssumeRoleProvider}; +use aws_credential_types::{Credentials, provider::ProvideCredentials}; use aws_sdk_s3::{ + Client, config::HttpClient, operation::get_object::GetObjectError, types::{CompletedMultipartUpload, CompletedPart}, - Client, }; use bytes::Bytes; use cloud::{ blob::{ - none_to_empty, BlobConfig, BlobObject, BlobStorage, BucketConf, DeletableStorage, - IterableStorage, PutResource, StringNonEmpty, + BlobConfig, BlobObject, BlobStorage, BucketConf, DeletableStorage, IterableStorage, + PutResource, StringNonEmpty, none_to_empty, }, metrics::CLOUD_REQUEST_HISTOGRAM_VEC, }; use fail::fail_point; use futures::{executor::block_on, stream::Stream}; use futures_util::{ + StreamExt, future::{FutureExt, LocalBoxFuture}, io::{AsyncRead, AsyncReadExt}, stream::TryStreamExt, - StreamExt, }; pub use kvproto::brpb::S3 as InputConfig; use thiserror::Error; use tikv_util::{ debug, - stream::{error_stream, RetryError}, + stream::{RetryError, error_stream}, time::Instant, }; use tokio::time::{sleep, timeout}; use tokio_util::io::ReaderStream; -use crate::util::{self, retry_and_count, SdkError}; +use crate::util::{self, SdkError, retry_and_count}; const CONNECTION_TIMEOUT: Duration = Duration::from_secs(900); pub const STORAGE_VENDOR_NAME_AWS: &str = "aws"; @@ -143,12 +143,9 @@ impl BlobConfig for Config { } fn url(&self) -> io::Result { - self.bucket.url("s3").map_err(|s| { - io::Error::new( - io::ErrorKind::Other, - format!("error creating bucket url: {}", s), - ) - }) + self.bucket + .url("s3") + .map_err(|s| io::Error::other(format!("error creating bucket url: {}", s))) } } @@ -775,10 +772,7 @@ impl DeletableStorage for S3Storage { .observe(now.saturating_elapsed().as_secs_f64()); match res { Ok(_) => Ok(()), - Err(e) => Err(io::Error::new( - io::ErrorKind::Other, - format!("failed to delete object {}", e), - )), + Err(e) => Err(io::Error::other(format!("failed to delete object {}", e))), } } .boxed_local() @@ -819,10 +813,7 @@ impl IterableStorage for S3Storage { .unwrap_or_else(|| futures::stream::empty().right_stream()) }) .map_err(|err| { - io::Error::new( - io::ErrorKind::Other, - format!("sdk encounters an unexpected error: {:?}", err), - ) + io::Error::other(format!("sdk encounters an unexpected error: {:?}", err)) }) .try_flatten() .boxed_local() @@ -1145,8 +1136,8 @@ mod tests { client.assert_requests_match(&[]); } + #[ignore = "s3 test env is unavailable"] #[tokio::test] - #[cfg(FALSE)] // FIXME: enable this (or move this to an integration test) if we've got a // reliable way to test s3 (aws test_util requires custom logic to verify the // body stream which itself can have bug) diff --git a/components/cloud/aws/src/util.rs b/components/cloud/aws/src/util.rs index 8938cdf607b..780163846ad 100644 --- a/components/cloud/aws/src/util.rs +++ b/components/cloud/aws/src/util.rs @@ -3,14 +3,14 @@ use std::{error::Error as StdError, io}; use ::aws_smithy_runtime_api::client::orchestrator::HttpResponse; use aws_config::{ + ConfigLoader, Region, default_provider::credentials::DefaultCredentialsChain, environment::EnvironmentVariableRegionProvider, meta::region::{self, ProvideRegion, RegionProviderChain}, profile::ProfileFileRegionProvider, provider_config::ProviderConfig, - ConfigLoader, Region, }; -use aws_credential_types::provider::{error::CredentialsError, ProvideCredentials}; +use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError}; use aws_sdk_kms::config::SharedHttpClient; use aws_sdk_s3::config::HttpClient; use aws_smithy_runtime::client::http::hyper_014::HyperClientBuilder; @@ -19,7 +19,7 @@ use futures::{Future, TryFutureExt}; use hyper::Client; use hyper_tls::HttpsConnector; use tikv_util::{ - stream::{block_on_external_io, retry_ext, RetryError, RetryExt}, + stream::{RetryError, RetryExt, block_on_external_io, retry_ext}, warn, }; diff --git a/components/cloud/azure/Cargo.toml b/components/cloud/azure/Cargo.toml index a9aaf2244c6..0ab0f82165b 100644 --- a/components/cloud/azure/Cargo.toml +++ b/components/cloud/azure/Cargo.toml @@ -19,7 +19,7 @@ cloud = { workspace = true } futures = "0.3" futures-util = { version = "0.3", default-features = false, features = ["io"] } kvproto = { workspace = true } -oauth2 = { version = "4.0.0", default-features = false } +oauth2 = { version = "4.4", default-features = false } openssl = { workspace = true } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/components/cloud/azure/src/azblob.rs b/components/cloud/azure/src/azblob.rs index 04398c935e1..210ac0e3d92 100644 --- a/components/cloud/azure/src/azblob.rs +++ b/components/cloud/azure/src/azblob.rs @@ -15,28 +15,28 @@ use azure_identity::{ AutoRefreshingTokenCredential, ClientSecretCredential, DefaultAzureCredential, TokenCredentialOptions, }; -use azure_storage::{prelude::*, ConnectionString, ConnectionStringBuilder}; +use azure_storage::{ConnectionString, ConnectionStringBuilder, prelude::*}; use azure_storage_blobs::{blob::operations::PutBlockBlobBuilder, prelude::*}; use cloud::{ blob::{ - none_to_empty, read_to_end, unimplemented, BlobConfig, BlobObject, BlobStorage, BucketConf, - DeletableStorage, IterableStorage, PutResource, StringNonEmpty, + BlobConfig, BlobObject, BlobStorage, BucketConf, DeletableStorage, IterableStorage, + PutResource, StringNonEmpty, none_to_empty, read_to_end, unimplemented, }, metrics::AZBLOB_UPLOAD_DURATION, }; use futures::TryFutureExt; -use futures_util::{future::FutureExt, io::AsyncRead, stream, stream::StreamExt, TryStreamExt}; +use futures_util::{TryStreamExt, future::FutureExt, io::AsyncRead, stream, stream::StreamExt}; pub use kvproto::brpb::{AzureBlobStorage as InputConfig, AzureCustomerKey}; use oauth2::{ClientId, ClientSecret}; use tikv_util::{ debug, defer, - stream::{retry, RetryError}, + stream::{RetryError, retry}, time::Instant, }; use time::OffsetDateTime; use tokio::{ sync::Mutex, - time::{timeout, Duration}, + time::{Duration, timeout}, }; const ENV_CLIENT_ID: &str = "AZURE_CLIENT_ID"; @@ -239,12 +239,9 @@ impl BlobConfig for Config { } fn url(&self) -> io::Result { - self.bucket.url("azure").map_err(|s| { - io::Error::new( - io::ErrorKind::Other, - format!("error creating bucket url: {}", s), - ) - }) + self.bucket + .url("azure") + .map_err(|s| io::Error::other(format!("error creating bucket url: {}", s))) } } @@ -818,6 +815,8 @@ impl DeletableStorage for AzureStorage { #[cfg(test)] mod tests { + use futures::AsyncReadExt; + use super::*; #[test] @@ -895,8 +894,8 @@ mod tests { env::remove_var(ENV_CLIENT_SECRET); } + #[ignore = "no available azure cloud service for the test env."] #[tokio::test] - #[cfg(feature = "azurite")] // test in Azurite emulator async fn test_azblob_storage() { use futures_util::stream; diff --git a/components/cloud/azure/src/kms.rs b/components/cloud/azure/src/kms.rs index a7d72c2afa4..3e7e58c278b 100644 --- a/components/cloud/azure/src/kms.rs +++ b/components/cloud/azure/src/kms.rs @@ -3,11 +3,11 @@ use std::{ops::Deref, sync::Arc}; use async_trait::async_trait; -use azure_core::{auth::TokenCredential, new_http_client, Error as AzureError}; +use azure_core::{Error as AzureError, auth::TokenCredential, new_http_client}; use azure_identity::{ AutoRefreshingTokenCredential, ClientSecretCredential, TokenCredentialOptions, }; -use azure_security_keyvault::{prelude::*, KeyClient}; +use azure_security_keyvault::{KeyClient, prelude::*}; use cloud::{ error::{Error as CloudError, KmsError, OtherError, Result}, kms::{Config, CryptographyType, DataKeyPair, EncryptedKey, KeyId, KmsProvider, PlainKey}, @@ -190,7 +190,7 @@ impl KmsProvider for AzureKms { ), }; self.client - .encrypt(&self.current_key_id.clone().into_inner(), encrypt_params) + .encrypt(self.current_key_id.clone().into_inner(), encrypt_params) .await .map_err(convert_azure_error) .and_then(|response| { @@ -237,10 +237,7 @@ fn convert_azure_error(err: AzureError) -> CloudError { let err_msg = if let Ok(e) = err.into_inner() { e } else { - Box::new(std::io::Error::new( - std::io::ErrorKind::Other, - "unknown error", - )) + Box::new(std::io::Error::other("unknown error")) }; CloudError::KmsError(KmsError::Other(OtherError::from_box(err_msg))) } diff --git a/components/cloud/azure/src/token_credentials/certificate_credentials.rs b/components/cloud/azure/src/token_credentials/certificate_credentials.rs index af39e7be5fa..ce32e3231ac 100644 --- a/components/cloud/azure/src/token_credentials/certificate_credentials.rs +++ b/components/cloud/azure/src/token_credentials/certificate_credentials.rs @@ -3,15 +3,16 @@ use std::{str, sync::Arc, time::Duration}; use azure_core::{ + HttpClient, Method, Request, auth::{AccessToken, TokenCredential, TokenResponse}, base64, content_type, error::{Error, ErrorKind}, - headers, new_http_client, HttpClient, Method, Request, + headers, new_http_client, }; use azure_identity::authority_hosts::AZURE_PUBLIC_CLOUD; use openssl::{ error::ErrorStack, - hash::{hash, DigestBytes, MessageDigest}, + hash::{DigestBytes, MessageDigest, hash}, pkcs12::Pkcs12, pkey::{PKey, Private}, sign::Signer, @@ -19,7 +20,7 @@ use openssl::{ }; use serde::Deserialize; use time::OffsetDateTime; -use url::{form_urlencoded, Url}; +use url::{Url, form_urlencoded}; /// Refresh time to use in seconds const DEFAULT_REFRESH_TIME: i64 = 300; diff --git a/components/cloud/gcp/src/client.rs b/components/cloud/gcp/src/client.rs index 7dc99c0e1f2..b6bca3529a0 100644 --- a/components/cloud/gcp/src/client.rs +++ b/components/cloud/gcp/src/client.rs @@ -8,13 +8,13 @@ use std::{ sync::Arc, }; -use hyper::{client::HttpConnector, Body, Client, Request, Response, StatusCode}; +use hyper::{Body, Client, Request, Response, StatusCode, client::HttpConnector}; use hyper_tls::HttpsConnector; use serde::Deserialize; use tame_oauth::gcp::{ - end_user::EndUserCredentialsInner, service_account::ServiceAccountProviderInner, EndUserCredentialsInfo, ServiceAccountInfo, TokenOrRequest, TokenProvider, - TokenProviderWrapper, TokenProviderWrapperInner, + TokenProviderWrapper, TokenProviderWrapperInner, end_user::EndUserCredentialsInner, + service_account::ServiceAccountProviderInner, }; use tikv_util::stream::RetryError; @@ -166,6 +166,7 @@ trait ResultExt { // Maps the error of this result as an `std::io::Error` with `Other` error // kind. + #[allow(dead_code)] fn or_io_error(self, msg: D) -> io::Result; // Maps the error of this result as an `std::io::Error` with `InvalidInput` @@ -176,7 +177,7 @@ trait ResultExt { impl ResultExt for StdResult { type Ok = T; fn or_io_error(self, msg: D) -> io::Result { - self.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("{}: {}", msg, e))) + self.map_err(|e| io::Error::other(format!("{}: {}", msg, e))) } fn or_invalid_input(self, msg: D) -> io::Result { self.map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, format!("{}: {}", msg, e))) diff --git a/components/cloud/gcp/src/gcs.rs b/components/cloud/gcp/src/gcs.rs index 794367e43e7..3645d419874 100644 --- a/components/cloud/gcp/src/gcs.rs +++ b/components/cloud/gcp/src/gcs.rs @@ -4,8 +4,8 @@ use std::{fmt::Display, io}; use async_trait::async_trait; use cloud::{ blob::{ - none_to_empty, read_to_end, BlobConfig, BlobObject, BlobStorage, BucketConf, - DeletableStorage, IterableStorage, PutResource, StringNonEmpty, + BlobConfig, BlobObject, BlobStorage, BucketConf, DeletableStorage, IterableStorage, + PutResource, StringNonEmpty, none_to_empty, read_to_end, }, metrics, }; @@ -24,12 +24,12 @@ use tame_gcs::{ }; use tame_oauth::gcp::ServiceAccountInfo; use tikv_util::{ - stream::{error_stream, AsyncReadAsSyncStreamOfBytes}, + stream::{AsyncReadAsSyncStreamOfBytes, error_stream}, time::Instant, }; use crate::{ - client::{status_code_error, GcpClient, RequestError}, + client::{GcpClient, RequestError, status_code_error}, utils::{self, retry}, }; @@ -100,12 +100,9 @@ impl BlobConfig for Config { } fn url(&self) -> io::Result { - self.bucket.url("gcs").map_err(|s| { - io::Error::new( - io::ErrorKind::Other, - format!("error creating bucket url: {}", s), - ) - }) + self.bucket + .url("gcs") + .map_err(|s| io::Error::other(format!("error creating bucket url: {}", s))) } } @@ -131,7 +128,7 @@ pub trait ResultExt { impl ResultExt for Result { type Ok = T; fn or_io_error(self, msg: D) -> io::Result { - self.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("{}: {}", msg, e))) + self.map_err(|e| io::Error::other(format!("{}: {}", msg, e))) } fn or_invalid_input(self, msg: D) -> io::Result { self.map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, format!("{}: {}", msg, e))) @@ -438,7 +435,7 @@ impl<'cli> GcsPrefixIter<'cli> { .cli .make_request(req.map(|_e| Body::empty()), tame_gcs::Scopes::ReadOnly) .await - .map_err(|err| io::Error::new(io::ErrorKind::Other, err))?; + .map_err(|err| io::Error::other(err))?; let resp = utils::read_from_http_body::(res).await?; metrics::CLOUD_REQUEST_HISTOGRAM_VEC .with_label_values(&["gcp", "list"]) diff --git a/components/cloud/gcp/src/kms.rs b/components/cloud/gcp/src/kms.rs index 3d4c84d06a1..9a83851e411 100644 --- a/components/cloud/gcp/src/kms.rs +++ b/components/cloud/gcp/src/kms.rs @@ -4,9 +4,10 @@ use std::{fmt, result::Result as StdResult}; use async_trait::async_trait; use cloud::{ + KeyId, error::{Error as CloudError, KmsError, Result}, kms::{Config, CryptographyType, DataKeyPair, EncryptedKey, KmsProvider, PlainKey}, - metrics, KeyId, + metrics, }; use futures_util::stream::StreamExt; use http::Method; @@ -18,8 +19,8 @@ use tame_gcs::error::HttpStatusError; use tikv_util::{box_err, stream::RetryError, time::Instant}; use crate::{ - client::{GcpClient, RequestError}, STORAGE_VENDOR_NAME_GCP, + client::{GcpClient, RequestError}, }; // generated random encryption data key length. diff --git a/components/cloud/gcp/src/lib.rs b/components/cloud/gcp/src/lib.rs index 81ab327f724..69ebc4bd32c 100644 --- a/components/cloud/gcp/src/lib.rs +++ b/components/cloud/gcp/src/lib.rs @@ -16,9 +16,9 @@ pub mod utils { use std::{future::Future, io}; use cloud::metrics; - use hyper::{body::Bytes, Body}; + use hyper::{Body, body::Bytes}; use tame_gcs::ApiResponse; - use tikv_util::stream::{retry_ext, RetryError, RetryExt}; + use tikv_util::stream::{RetryError, RetryExt, retry_ext}; pub async fn retry(action: G, name: &'static str) -> Result where G: FnMut() -> F, diff --git a/components/cloud/src/error.rs b/components/cloud/src/error.rs index 8fd1dda3e8e..ea400d95466 100644 --- a/components/cloud/src/error.rs +++ b/components/cloud/src/error.rs @@ -3,7 +3,7 @@ use std::{ error, fmt::{self, Debug, Display}, - io::{Error as IoError, ErrorKind}, + io::Error as IoError, result, }; @@ -59,7 +59,7 @@ impl From for IoError { fn from(err: Error) -> IoError { match err { Error::Io(e) => e, - other => IoError::new(ErrorKind::Other, format!("{}", other)), + other => IoError::other(format!("{}", other)), } } } diff --git a/components/cloud/src/lib.rs b/components/cloud/src/lib.rs index 28debed51b9..978a1558a3e 100644 --- a/components/cloud/src/lib.rs +++ b/components/cloud/src/lib.rs @@ -13,6 +13,6 @@ pub mod kms; pub use kms::{Config, DataKeyPair, EncryptedKey, KeyId, KmsProvider, PlainKey, SubConfigAzure}; pub mod blob; -pub use blob::{none_to_empty, BucketConf, StringNonEmpty}; +pub use blob::{BucketConf, StringNonEmpty, none_to_empty}; pub mod metrics; diff --git a/components/codec/src/buffer.rs b/components/codec/src/buffer.rs index f40ee1fae4f..e47249e05ae 100644 --- a/components/codec/src/buffer.rs +++ b/components/codec/src/buffer.rs @@ -65,7 +65,7 @@ impl> BufferReader for std::io::Cursor { } } -impl<'a> BufferReader for &'a [u8] { +impl BufferReader for &[u8] { #[inline] fn bytes(&self) -> &[u8] { self @@ -86,7 +86,7 @@ impl<'a> BufferReader for &'a [u8] { } } -impl<'a, T: BufferReader + ?Sized> BufferReader for &'a mut T { +impl BufferReader for &mut T { #[inline] fn bytes(&self) -> &[u8] { (**self).bytes() @@ -204,7 +204,7 @@ impl> BufferWriter for std::io::Cursor { } } -impl<'a> BufferWriter for &'a mut [u8] { +impl BufferWriter for &mut [u8] { #[inline] unsafe fn bytes_mut(&mut self, _size: usize) -> &mut [u8] { self @@ -252,7 +252,7 @@ impl BufferWriter for Vec { } } -impl<'a, T: BufferWriter + ?Sized> BufferWriter for &'a mut T { +impl BufferWriter for &mut T { #[inline] unsafe fn bytes_mut(&mut self, size: usize) -> &mut [u8] { (**self).bytes_mut(size) @@ -635,7 +635,7 @@ mod tests { // Re-allocate the vector space and ensure that the address is changed. vec.reserve(::std::cmp::max(payload_len * 3, 32)); - if vec_ptr == vec.as_ptr() { + if std::ptr::eq(vec_ptr, vec.as_ptr()) { in_place_reallocs += 1; } diff --git a/components/codec/src/byte.rs b/components/codec/src/byte.rs index 8b5fd928edf..f76985fc424 100644 --- a/components/codec/src/byte.rs +++ b/components/codec/src/byte.rs @@ -3,9 +3,9 @@ use std::{intrinsics::unlikely, io::Read}; use crate::{ + ErrorInner, Result, buffer::BufferReader, number::{self, NumberCodec, NumberDecoder, NumberEncoder}, - ErrorInner, Result, }; const MEMCMP_GROUP_SIZE: usize = 8; diff --git a/components/codec/src/number.rs b/components/codec/src/number.rs index af47905334d..7422573588f 100644 --- a/components/codec/src/number.rs +++ b/components/codec/src/number.rs @@ -1,12 +1,15 @@ // Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0. -use std::intrinsics::{likely, unlikely}; +use std::{ + intrinsics::{likely, unlikely}, + ptr, +}; use byteorder::{BigEndian, ByteOrder, LittleEndian}; use crate::{ - buffer::{BufferReader, BufferWriter}, ErrorInner, Result, + buffer::{BufferReader, BufferWriter}, }; pub const MAX_VARINT64_LENGTH: usize = 10; @@ -466,12 +469,12 @@ impl NumberCodec { let ptr_end = buf.as_ptr().add(len); // Slow path let mut shift = 0; - while ptr != ptr_end && *ptr >= 0x80 { + while !ptr::eq(ptr, ptr_end) && *ptr >= 0x80 { val |= ((*ptr & 0x7f) as u64) << shift; shift += 7; ptr = ptr.add(1); } - if unlikely(ptr == ptr_end) { + if unlikely(ptr::eq(ptr, ptr_end)) { return Err(ErrorInner::eof().into()); } val |= (*ptr as u64) << shift; @@ -541,7 +544,7 @@ impl NumberCodec { } else { let ptr_end = buf.as_ptr().add(len); // Slow path - while ptr != ptr_end && *ptr >= 0x80 { + while !ptr::eq(ptr, ptr_end) && *ptr >= 0x80 { ptr = ptr.add(1); } // When we got here, we are either `ptr == ptr_end`, or `*ptr < 0x80`. @@ -549,7 +552,7 @@ impl NumberCodec { // but meet EOF, so only `len` is returned. // For `*ptr < 0x80` case, it means currently it is pointing to the last byte // of the VarInt, so we return `delta + 1` as length. - if unlikely(ptr == ptr_end) { + if unlikely(ptr::eq(ptr, ptr_end)) { return len; } ptr.offset_from(buf.as_ptr()) as usize + 1 diff --git a/components/compact-log-backup/src/compaction/collector.rs b/components/compact-log-backup/src/compaction/collector.rs index 6113a82bccd..d7da5a33711 100644 --- a/components/compact-log-backup/src/compaction/collector.rs +++ b/components/compact-log-backup/src/compaction/collector.rs @@ -12,7 +12,7 @@ use crate::{ }; /// A collecting subcompaction. - +/// /// Collecting a stream of [`LogFile`], and generate a stream of compactions. #[pin_project::pin_project] pub struct CollectSubcompaction>> { diff --git a/components/compact-log-backup/src/compaction/exec.rs b/components/compact-log-backup/src/compaction/exec.rs index 481c2340048..a4ff62a1812 100644 --- a/components/compact-log-backup/src/compaction/exec.rs +++ b/components/compact-log-backup/src/compaction/exec.rs @@ -6,8 +6,8 @@ use std::{ use engine_rocks::RocksEngine; use engine_traits::{ - ExternalSstFileInfo, SstCompressionType, SstExt, SstWriter, SstWriterBuilder, - DATA_KEY_PREFIX_LEN, + DATA_KEY_PREFIX_LEN, ExternalSstFileInfo, SstCompressionType, SstExt, SstWriter, + SstWriterBuilder, }; use external_storage::{ExternalStorage, UnpinReader}; use file_system::Sha256Reader; @@ -22,7 +22,7 @@ use crate::{ compaction::SST_OUT_REL, errors::{OtherErrExt, Result, TraceResultExt}, source::{Record, Source}, - statistic::{prom::*, LoadStatistic, SubcompactStatistic}, + statistic::{LoadStatistic, SubcompactStatistic, prom::*}, storage::DEFAULT_COMPACTION_OUT_PREFIX, util::{self, Cooperate, ExecuteAllExt}, }; @@ -410,7 +410,7 @@ mod test { compaction::Subcompaction, storage::{Epoch, MetaFile}, test_util::{ - gen_step, save_many_log_files, CompactInMem, KvGen, LogFileBuilder, TmpStorage, + CompactInMem, KvGen, LogFileBuilder, TmpStorage, gen_step, save_many_log_files, }, }; diff --git a/components/compact-log-backup/src/compaction/meta.rs b/components/compact-log-backup/src/compaction/meta.rs index a54c35e70d1..f9998e27a73 100644 --- a/components/compact-log-backup/src/compaction/meta.rs +++ b/components/compact-log-backup/src/compaction/meta.rs @@ -9,8 +9,8 @@ use futures::stream::TryStreamExt; use kvproto::brpb::{self, DeleteSpansOfFile}; use super::{ - collector::CollectSubcompactionConfig, EpochHint, Subcompaction, SubcompactionCollectKey, - SubcompactionResult, UnformedSubcompaction, + EpochHint, Subcompaction, SubcompactionCollectKey, SubcompactionResult, UnformedSubcompaction, + collector::CollectSubcompactionConfig, }; use crate::{ errors::Result, @@ -358,8 +358,8 @@ mod test { use super::CompactionRunInfoBuilder; use crate::{ - compaction::{exec::SubcompactionExec, Subcompaction, SubcompactionResult}, - test_util::{gen_min_max, KvGen, LogFileBuilder, TmpStorage}, + compaction::{Subcompaction, SubcompactionResult, exec::SubcompactionExec}, + test_util::{KvGen, LogFileBuilder, TmpStorage, gen_min_max}, }; impl CompactionRunInfoBuilder { diff --git a/components/compact-log-backup/src/exec_hooks/checkpoint.rs b/components/compact-log-backup/src/exec_hooks/checkpoint.rs index 6bbfe42422b..ff1e85c3b61 100644 --- a/components/compact-log-backup/src/exec_hooks/checkpoint.rs +++ b/components/compact-log-backup/src/exec_hooks/checkpoint.rs @@ -7,9 +7,9 @@ use futures::stream::TryStreamExt; use tikv_util::{info, time::Instant, warn}; use crate::{ + ErrorKind, OtherErrExt, Result, TraceResultExt, compaction::META_OUT_REL, execute::hooking::{BeforeStartCtx, CId, ExecHooks, SubcompactionStartCtx}, - ErrorKind, OtherErrExt, Result, TraceResultExt, }; #[derive(Default)] diff --git a/components/compact-log-backup/src/exec_hooks/consistency.rs b/components/compact-log-backup/src/exec_hooks/consistency.rs index c22a49c9719..0b657dc5ca0 100644 --- a/components/compact-log-backup/src/exec_hooks/consistency.rs +++ b/components/compact-log-backup/src/exec_hooks/consistency.rs @@ -1,16 +1,16 @@ // Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0. pub use engine_traits::SstCompressionType; -use external_storage::{locking::RemoteLock, ExternalStorage}; +use external_storage::{ExternalStorage, locking::RemoteLock}; use futures::{io::AsyncReadExt, stream::TryStreamExt}; use tikv_util::warn; use crate::{ + ErrorKind, TraceResultExt, errors::Result, execute::hooking::{AbortedCtx, AfterFinishCtx, BeforeStartCtx, ExecHooks}, storage::LOCK_PREFIX, util::storage_url, - ErrorKind, TraceResultExt, }; #[derive(Default)] diff --git a/components/compact-log-backup/src/exec_hooks/observability.rs b/components/compact-log-backup/src/exec_hooks/observability.rs index 1a24e869529..1b628d4637b 100644 --- a/components/compact-log-backup/src/exec_hooks/observability.rs +++ b/components/compact-log-backup/src/exec_hooks/observability.rs @@ -6,6 +6,7 @@ use tokio::{io::AsyncWriteExt, signal::unix::SignalKind}; use super::CollectStatistic; use crate::{ + ErrorKind, errors::Result, execute::hooking::{ AbortedCtx, AfterFinishCtx, BeforeStartCtx, CId, ExecHooks, SubcompactionFinishCtx, @@ -14,7 +15,6 @@ use crate::{ statistic::prom, storage::StreamMetaStorage, util::storage_url, - ErrorKind, }; /// The hooks that used for an execution from a TTY. Providing the basic diff --git a/components/compact-log-backup/src/exec_hooks/save_meta.rs b/components/compact-log-backup/src/exec_hooks/save_meta.rs index adfc7225ec4..b611d706133 100644 --- a/components/compact-log-backup/src/exec_hooks/save_meta.rs +++ b/components/compact-log-backup/src/exec_hooks/save_meta.rs @@ -5,13 +5,13 @@ use external_storage::UnpinReader; use futures::{future::TryFutureExt, io::Cursor}; use kvproto::brpb; use tikv_util::{ - stream::{retry, JustRetry}, + stream::{JustRetry, retry}, warn, }; use super::CollectStatistic; use crate::{ - compaction::{meta::CompactionRunInfoBuilder, META_OUT_REL, SST_OUT_REL}, + compaction::{META_OUT_REL, SST_OUT_REL, meta::CompactionRunInfoBuilder}, errors::Result, execute::hooking::{ AfterFinishCtx, BeforeStartCtx, CId, ExecHooks, SubcompactionFinishCtx, diff --git a/components/compact-log-backup/src/execute/hooking.rs b/components/compact-log-backup/src/execute/hooking.rs index c0443dc968f..8199fe82623 100644 --- a/components/compact-log-backup/src/execute/hooking.rs +++ b/components/compact-log-backup/src/execute/hooking.rs @@ -7,11 +7,11 @@ use external_storage::ExternalStorage; use tokio::runtime::Handle; use crate::{ + Error, compaction::{Subcompaction, SubcompactionResult}, errors::Result, execute::Execution, statistic::{CollectSubcompactionStatistic, LoadMetaStatistic}, - Error, }; pub struct NoHooks; diff --git a/components/compact-log-backup/src/execute/mod.rs b/components/compact-log-backup/src/execute/mod.rs index b9f2cbc71c5..58e976b4720 100644 --- a/components/compact-log-backup/src/execute/mod.rs +++ b/components/compact-log-backup/src/execute/mod.rs @@ -18,7 +18,7 @@ use hooking::{ use kvproto::brpb::StorageBackend; use tikv_util::config::ReadableSize; use tokio::runtime::Handle; -use tracing::{trace_span, Instrument}; +use tracing::{Instrument, trace_span}; use tracing_active_tree::{frame, root}; use txn_types::TimeStamp; @@ -31,9 +31,10 @@ use super::{ storage::{LoadFromExt, StreamMetaStorage}, }; use crate::{ - compaction::{exec::SubcompactionExecArg, SubcompactionResult}, + ErrorKind, + compaction::{SubcompactionResult, exec::SubcompactionExecArg}, errors::{Result, TraceResultExt}, - util, ErrorKind, + util, }; const COMPACTION_V1_PREFIX: &str = "v1/compactions"; diff --git a/components/compact-log-backup/src/execute/test.rs b/components/compact-log-backup/src/execute/test.rs index 743316cdf69..b2b9b086496 100644 --- a/components/compact-log-backup/src/execute/test.rs +++ b/components/compact-log-backup/src/execute/test.rs @@ -4,8 +4,8 @@ use std::{ collections::HashMap, future::Future, sync::{ - atomic::{AtomicU64, Ordering}, Arc, + atomic::{AtomicU64, Ordering}, }, }; @@ -17,6 +17,7 @@ use tokio::sync::mpsc::Sender; use super::{Execution, ExecutionConfig}; use crate::{ + ErrorKind, compaction::SubcompactionResult, errors::OtherErrExt, exec_hooks::{ @@ -25,8 +26,7 @@ use crate::{ }, execute::hooking::{CId, ExecHooks, SubcompactionFinishCtx}, storage::LOCK_PREFIX, - test_util::{gen_step, CompactInMem, KvGen, LogFileBuilder, TmpStorage}, - ErrorKind, + test_util::{CompactInMem, KvGen, LogFileBuilder, TmpStorage, gen_step}, }; #[derive(Clone)] diff --git a/components/compact-log-backup/src/source.rs b/components/compact-log-backup/src/source.rs index 44a6c75745b..a82fd1dc8a6 100644 --- a/components/compact-log-backup/src/source.rs +++ b/components/compact-log-backup/src/source.rs @@ -1,6 +1,6 @@ // Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0. use std::{ - pin::{pin, Pin}, + pin::{Pin, pin}, sync::Arc, }; @@ -12,7 +12,7 @@ use kvproto::brpb; use prometheus::core::{Atomic, AtomicU64}; use tikv_util::{ codec::stream_event::{self, Iterator}, - stream::{retry_all_ext, JustRetry, RetryExt}, + stream::{JustRetry, RetryExt, retry_all_ext}, }; use txn_types::Key; @@ -133,7 +133,7 @@ mod tests { compaction::{Input, Subcompaction}, statistic::LoadStatistic, storage::{LogFile, MetaFile}, - test_util::{gen_adjacent_with_ts, KvGen, LogFileBuilder, TmpStorage}, + test_util::{KvGen, LogFileBuilder, TmpStorage, gen_adjacent_with_ts}, }; const NUM_FLUSH: usize = 2; diff --git a/components/compact-log-backup/src/statistic.rs b/components/compact-log-backup/src/statistic.rs index 347b1f25342..1af2fc31f42 100644 --- a/components/compact-log-backup/src/statistic.rs +++ b/components/compact-log-backup/src/statistic.rs @@ -58,7 +58,6 @@ pub struct LoadMetaStatistic { /// The statistic of loading data files for a subcompaction. #[derive(Default, Debug, Add, AddAssign, Clone, Serialize)] #[serde(rename_all = "kebab-case")] - pub struct LoadStatistic { /// How many logical "files" we have loaded? pub files_in: u64, @@ -119,7 +118,7 @@ pub struct CollectSubcompactionStatistic { pub mod prom { use prometheus::*; - use serde::{ser::SerializeMap, Serialize}; + use serde::{Serialize, ser::SerializeMap}; struct ShowPromHist<'a>(&'a Histogram); diff --git a/components/compact-log-backup/src/storage.rs b/components/compact-log-backup/src/storage.rs index a4692f1f986..69cfb06b481 100644 --- a/components/compact-log-backup/src/storage.rs +++ b/components/compact-log-backup/src/storage.rs @@ -6,7 +6,7 @@ use std::{ path::Path, pin::Pin, sync::Arc, - task::{ready, Context, Poll}, + task::{Context, Poll, ready}, }; use derive_more::Display; @@ -28,14 +28,14 @@ use tikv_util::{ time::Instant, }; use tokio_stream::Stream; -use tracing::{span::Entered, Span}; +use tracing::{Span, span::Entered}; use tracing_active_tree::frame; use super::{ errors::{Error, Result}, statistic::LoadMetaStatistic, }; -use crate::{compaction::EpochHint, errors::ErrorKind, util, OtherErrExt}; +use crate::{OtherErrExt, compaction::EpochHint, errors::ErrorKind, util}; pub const METADATA_PREFIX: &str = "v1/backupmeta"; pub const DEFAULT_COMPACTION_OUT_PREFIX: &str = "v1/compaction_out"; @@ -788,7 +788,7 @@ mod test { use super::{LoadFromExt, MetaFile, StreamMetaStorage}; use crate::{ storage::MigrationStorageWrapper, - test_util::{gen_step, KvGen, LogFileBuilder, TmpStorage}, + test_util::{KvGen, LogFileBuilder, TmpStorage, gen_step}, }; async fn construct_storage( diff --git a/components/compact-log-backup/src/test_util.rs b/components/compact-log-backup/src/test_util.rs index c5c0a01b493..2bfbdfacb6f 100644 --- a/components/compact-log-backup/src/test_util.rs +++ b/components/compact-log-backup/src/test_util.rs @@ -6,7 +6,6 @@ use std::{ ops::Not, path::{Path, PathBuf}, sync::{Arc, Mutex}, - u64, }; use engine_rocks::RocksEngine; @@ -19,7 +18,7 @@ use futures::{ }; use keys::origin_key; use kvproto::brpb::{self, Metadata}; -use protobuf::{parse_from_bytes, Message}; +use protobuf::{Message, parse_from_bytes}; use tempdir::TempDir; use tidb_query_datatype::codec::table::encode_row_key; use tikv_util::codec::stream_event::EventEncoder; @@ -27,11 +26,11 @@ use txn_types::Key; use crate::{ compaction::{ - exec::{SubcompactExt, SubcompactionExec}, Subcompaction, SubcompactionResult, + exec::{SubcompactExt, SubcompactionExec}, }, errors::{OtherErrExt, Result}, - storage::{id_of_migration, Epoch, LogFile, LogFileId, MetaFile}, + storage::{Epoch, LogFile, LogFileId, MetaFile, id_of_migration}, }; #[derive(Debug, PartialEq, Eq)] diff --git a/components/compact-log-backup/src/util.rs b/components/compact-log-backup/src/util.rs index 4bfd31055ff..b5250d46ed9 100644 --- a/components/compact-log-backup/src/util.rs +++ b/components/compact-log-backup/src/util.rs @@ -2,7 +2,7 @@ use std::{fmt::Display, future::Future, task::Poll}; -use engine_traits::{CfName, SstCompressionType, CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE}; +use engine_traits::{CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE, CfName, SstCompressionType}; use external_storage::ExternalStorage; /// A helper for cooperative concurrency. diff --git a/components/concurrency_manager/benches/update_max_ts.rs b/components/concurrency_manager/benches/update_max_ts.rs index df2e61aeb42..b93c5b42afc 100644 --- a/components/concurrency_manager/benches/update_max_ts.rs +++ b/components/concurrency_manager/benches/update_max_ts.rs @@ -3,7 +3,7 @@ use std::time::Duration; use concurrency_manager::{ActionOnInvalidMaxTs, ConcurrencyManager}; -use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use criterion::{Criterion, black_box, criterion_group, criterion_main}; use txn_types::TimeStamp; fn benchmark_update_max_ts(c: &mut Criterion) { diff --git a/components/concurrency_manager/src/key_handle.rs b/components/concurrency_manager/src/key_handle.rs index c7aebbc49e0..e8183e99b88 100644 --- a/components/concurrency_manager/src/key_handle.rs +++ b/components/concurrency_manager/src/key_handle.rs @@ -31,7 +31,10 @@ impl KeyHandle { // Safety: `_mutex_guard` is declared before `handle_ref` in `KeyHandleGuard`. // So the mutex guard will be released earlier than the `Arc`. // Then we can make sure the mutex guard doesn't point to released memory. - let mutex_guard = unsafe { mem::transmute(self.mutex.lock().await) }; + let mutex_guard = unsafe { + #[allow(clippy::missing_transmute_annotations)] + mem::transmute(self.mutex.lock().await) + }; KeyHandleGuard { _mutex_guard: mutex_guard, handle: self, diff --git a/components/concurrency_manager/src/lib.rs b/components/concurrency_manager/src/lib.rs index 8c8c17f685e..719528ca703 100644 --- a/components/concurrency_manager/src/lib.rs +++ b/components/concurrency_manager/src/lib.rs @@ -21,8 +21,8 @@ use std::{ fmt::Display, mem::MaybeUninit, sync::{ - atomic::{AtomicU64, AtomicUsize, Ordering}, Arc, + atomic::{AtomicU64, AtomicUsize, Ordering}, }, time::Duration, }; @@ -31,7 +31,7 @@ use crossbeam::atomic::AtomicCell; use lazy_static::lazy_static; use mockall::automock; use pd_client::{PdClient, PdFuture}; -use prometheus::{register_int_gauge, IntGauge}; +use prometheus::{IntGauge, register_int_gauge}; use thiserror::Error; use tikv_util::{error, future::block_on_timeout, time::Instant, warn}; use txn_types::{Key, Lock, TimeStamp}; @@ -209,7 +209,6 @@ impl ConcurrencyManager { /// - Ok(()): If the update is successful or has no effect /// - Err(limit): If new_ts is greater than the max_ts_limit, returns the /// current limit value - pub fn update_max_ts( &self, new_ts: TimeStamp, @@ -575,7 +574,7 @@ pub trait IntoErrorSource: sealed::Sealed { } // &str impl -impl<'a> sealed::Sealed for &'a str {} +impl sealed::Sealed for &str {} impl<'a> IntoErrorSource for &'a str { type Output = &'a str; fn into_error_source(self) -> Self::Output { diff --git a/components/concurrency_manager/tests/memory_usage.rs b/components/concurrency_manager/tests/memory_usage.rs index 6822c2fdc52..e1d9e0c3f20 100644 --- a/components/concurrency_manager/tests/memory_usage.rs +++ b/components/concurrency_manager/tests/memory_usage.rs @@ -1,7 +1,7 @@ // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. use std::{ - mem::{forget, ManuallyDrop}, + mem::{ManuallyDrop, forget}, thread, }; diff --git a/components/coprocessor_plugin_api/src/allocator.rs b/components/coprocessor_plugin_api/src/allocator.rs index d8c2ab5062f..555230a9141 100644 --- a/components/coprocessor_plugin_api/src/allocator.rs +++ b/components/coprocessor_plugin_api/src/allocator.rs @@ -18,6 +18,7 @@ pub struct HostAllocatorPtr { } /// An allocator that forwards invocations to the host (TiKV) of the plugin. +#[derive(Default)] pub struct HostAllocator { alloc_fn: Atomic>, dealloc_fn: Atomic>, diff --git a/components/crossbeam-skiplist/src/lib.rs b/components/crossbeam-skiplist/src/lib.rs index 222b735024a..904f833ea0e 100644 --- a/components/crossbeam-skiplist/src/lib.rs +++ b/components/crossbeam-skiplist/src/lib.rs @@ -91,9 +91,9 @@ //! A solution to the above is to have the implementation wrap //! each value in a lock. However, this has some repercussions: //! * The map would no longer be lock-free, inhibiting scalability -//! and allowing for deadlocks. +//! and allowing for deadlocks. //! * If a user of the map doesn't need mutable access, then they pay -//! the price of locks without actually needing them. +//! the price of locks without actually needing them. //! //! Instead, the approach taken by this crate gives more control to the user. //! If mutable access is needed, then you can use interior mutability, @@ -150,7 +150,7 @@ //! Crossbeam [does not currently provide a concurrent unordered map](https://github.com/crossbeam-rs/rfcs/issues/32). //! That said, here are some other crates which may suit you: //! * [`DashMap`](https://docs.rs/dashmap) implements a novel concurrent hash map -//! with good performance characteristics. +//! with good performance characteristics. //! * [`flurry`](https://docs.rs/flurry) is a Rust port of Java's `ConcurrentHashMap`. //! //! [`insert`]: SkipMap::insert @@ -260,6 +260,7 @@ pub use crate::{map::SkipMap, set::SkipSet}; // Prevent cargo machete warnings. mod seal { + #[allow(dead_code)] pub trait Sealed {} impl Sealed for crossbeam_skiplist_offical::SkipList<(), ()> {} } diff --git a/components/crypto/Cargo.toml b/components/crypto/Cargo.toml index b80628c2912..d1156484e37 100644 --- a/components/crypto/Cargo.toml +++ b/components/crypto/Cargo.toml @@ -13,3 +13,6 @@ openssl-sys = { workspace = true } slog = { workspace = true } # better to not use slog-global, but pass in the logger slog-global = { workspace = true } + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(ossl1)', 'cfg(ossl3)', 'cfg(disable_fips)'] } diff --git a/components/crypto/build.rs b/components/crypto/build.rs index 4b5b6fda112..cf974133ee6 100644 --- a/components/crypto/build.rs +++ b/components/crypto/build.rs @@ -3,7 +3,7 @@ use std::env; fn main() { - if !option_env!("ENABLE_FIPS").map_or(false, |v| v == "1") { + if option_env!("ENABLE_FIPS").is_none_or(|v| v != "1") { println!("cargo:rustc-cfg=disable_fips"); return; } diff --git a/components/encryption/export/examples/ecli.rs b/components/encryption/export/examples/ecli.rs index 106b22ded75..5eb1f92717e 100644 --- a/components/encryption/export/examples/ecli.rs +++ b/components/encryption/export/examples/ecli.rs @@ -5,13 +5,13 @@ use std::io::{Read, Write}; use azure::STORAGE_VENDOR_NAME_AZURE; pub use cloud::kms::Config as CloudConfig; use encryption::{GcpConfig, KmsBackend}; -use encryption_export::{create_cloud_backend, AzureConfig, Backend, Error, KmsConfig, Result}; +use encryption_export::{AzureConfig, Backend, Error, KmsConfig, Result, create_cloud_backend}; use file_system::{File, OpenOptions}; use gcp::STORAGE_VENDOR_NAME_GCP; use ini::ini::Ini; use kvproto::encryptionpb::EncryptedContent; use protobuf::Message; -use structopt::{clap::arg_enum, StructOpt}; +use structopt::{StructOpt, clap::arg_enum}; use tikv_util::box_err; arg_enum! { diff --git a/components/encryption/export/src/lib.rs b/components/encryption/export/src/lib.rs index 1b7d23d5042..393e5bfddbd 100644 --- a/components/encryption/export/src/lib.rs +++ b/components/encryption/export/src/lib.rs @@ -4,11 +4,11 @@ use std::path::Path; use aws::{AwsKms, STORAGE_VENDOR_NAME_AWS}; use azure::{AzureKms, STORAGE_VENDOR_NAME_AZURE}; pub use encryption::{ - clean_up_dir, clean_up_trash, trash_dir_all, AsyncBackend, AzureConfig, Backend, - DataKeyImporter, DataKeyManager, DataKeyManagerArgs, DecrypterReader, EncryptionConfig, Error, - FileConfig, Iv, KmsBackend, KmsConfig, MasterKeyConfig, Result, + AsyncBackend, AzureConfig, Backend, DataKeyImporter, DataKeyManager, DataKeyManagerArgs, + DecrypterReader, EncryptionConfig, Error, FileConfig, Iv, KmsBackend, KmsConfig, + MasterKeyConfig, Result, clean_up_dir, clean_up_trash, trash_dir_all, }; -use encryption::{cloud_convert_error, FileBackend, PlaintextBackend}; +use encryption::{FileBackend, PlaintextBackend, cloud_convert_error}; use gcp::{GcpKms, STORAGE_VENDOR_NAME_GCP}; use tikv_util::{box_err, error, info}; diff --git a/components/encryption/src/config.rs b/components/encryption/src/config.rs index 2b8ed30a73d..50620d5c183 100644 --- a/components/encryption/src/config.rs +++ b/components/encryption/src/config.rs @@ -3,10 +3,10 @@ use std::result; use cloud::{ - kms::{SubConfigAws, SubConfigAzure, SubConfigGcp}, Config as CloudConfig, + kms::{SubConfigAws, SubConfigAzure, SubConfigGcp}, }; -use kvproto::encryptionpb::{EncryptionMethod, MasterKey, MasterKeyKms, MasterKey_oneof_backend}; +use kvproto::encryptionpb::{EncryptionMethod, MasterKey, MasterKey_oneof_backend, MasterKeyKms}; use online_config::OnlineConfig; use serde_derive::{Deserialize, Serialize}; use tikv_util::{box_err, config::ReadableDuration}; @@ -266,8 +266,8 @@ mod encryption_method_serde { use std::fmt; use serde::{ - de::{self, Unexpected, Visitor}, Deserializer, Serializer, + de::{self, Unexpected, Visitor}, }; use super::EncryptionMethod; diff --git a/components/encryption/src/encrypted_file/mod.rs b/components/encryption/src/encrypted_file/mod.rs index 8cac47077f4..4b1f9b42056 100644 --- a/components/encryption/src/encrypted_file/mod.rs +++ b/components/encryption/src/encrypted_file/mod.rs @@ -6,13 +6,13 @@ use std::{ }; use crypto::rand; -use file_system::{rename, File, OpenOptions}; +use file_system::{File, OpenOptions, rename}; use kvproto::encryptionpb::EncryptedContent; use protobuf::Message; use slog_global::error; use tikv_util::time::Instant; -use crate::{master_key::*, metrics::*, Result}; +use crate::{Result, master_key::*, metrics::*}; mod header; pub use header::*; diff --git a/components/encryption/src/errors.rs b/components/encryption/src/errors.rs index da23d923be7..3198fd5d2a6 100644 --- a/components/encryption/src/errors.rs +++ b/components/encryption/src/errors.rs @@ -3,7 +3,7 @@ use std::{ error, fmt::{Debug, Display}, - io::{Error as IoError, ErrorKind}, + io::Error as IoError, result, }; @@ -67,7 +67,7 @@ impl From for IoError { fn from(err: Error) -> IoError { match err { Error::Io(e) => e, - other => IoError::new(ErrorKind::Other, format!("{}", other)), + other => IoError::other(format!("{}", other)), } } } diff --git a/components/encryption/src/file_dict_file.rs b/components/encryption/src/file_dict_file.rs index a40fb912b3b..0b172a58194 100644 --- a/components/encryption/src/file_dict_file.rs +++ b/components/encryption/src/file_dict_file.rs @@ -7,16 +7,16 @@ use std::{ use byteorder::{BigEndian, ByteOrder}; use crypto::rand; -use file_system::{rename, File, OpenOptions}; +use file_system::{File, OpenOptions, rename}; use kvproto::encryptionpb::{EncryptedContent, FileDictionary, FileInfo}; use protobuf::Message; use tikv_util::{box_err, info, set_panic_mark, warn}; use crate::{ - encrypted_file::{EncryptedFile, Header, Version, TMP_FILE_SUFFIX}, + Error, Result, + encrypted_file::{EncryptedFile, Header, TMP_FILE_SUFFIX, Version}, master_key::{Backend, PlaintextBackend}, metrics::*, - Error, Result, }; #[derive(Debug)] @@ -412,7 +412,7 @@ mod tests { use kvproto::encryptionpb::EncryptionMethod; use super::*; - use crate::{encrypted_file::EncryptedFile, Error}; + use crate::{Error, encrypted_file::EncryptedFile}; fn test_file_dict_file_normal(enable_log: bool) { let tempdir = tempfile::tempdir().unwrap(); diff --git a/components/encryption/src/io.rs b/components/encryption/src/io.rs index 018b7f6f89f..1f12883989b 100644 --- a/components/encryption/src/io.rs +++ b/components/encryption/src/io.rs @@ -1,16 +1,16 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. use std::{ - io::{Error as IoError, ErrorKind, Read, Result as IoResult, Seek, SeekFrom, Write}, + io::{Error as IoError, Read, Result as IoResult, Seek, SeekFrom, Write}, pin::Pin, task::ready, }; use file_system::File; use futures_util::{ + AsyncWrite, io::AsyncRead, task::{Context, Poll}, - AsyncWrite, }; use kvproto::encryptionpb::EncryptionMethod; use openssl::symm::{Cipher as OCipher, Crypter as OCrypter, Mode}; @@ -564,13 +564,10 @@ impl CrypterCore { let crypter_count = crypter.update(&partial_block, &mut self.buffer)?; if crypter_count != partial_offset { self.lazy_reset_crypter(offset); - return Err(IoError::new( - ErrorKind::Other, - format!( - "crypter output size mismatch, expect {} vs actual {}", - partial_offset, crypter_count, - ), - )); + return Err(IoError::other(format!( + "crypter output size mismatch, expect {} vs actual {}", + partial_offset, crypter_count, + ))); } self.offset = offset; self.crypter = Some(crypter); @@ -596,14 +593,11 @@ impl CrypterCore { let crypter_count = crypter.update(&buf[encrypted..target], &mut self.buffer)?; if crypter_count != target - encrypted { self.crypter.take(); - return Err(IoError::new( - ErrorKind::Other, - format!( - "crypter output size mismatch, expect {} vs actual {}", - target - encrypted, - crypter_count, - ), - )); + return Err(IoError::other(format!( + "crypter output size mismatch, expect {} vs actual {}", + target - encrypted, + crypter_count, + ))); } buf[encrypted..target].copy_from_slice(&self.buffer[..crypter_count]); encrypted += crypter_count; @@ -630,13 +624,10 @@ impl CrypterCore { let crypter_count = crypter.update(buf, &mut self.buffer)?; if crypter_count != count { self.crypter.take(); - return Err(IoError::new( - ErrorKind::Other, - format!( - "crypter output size mismatch, expect {} vs actual {}", - count, crypter_count, - ), - )); + return Err(IoError::other(format!( + "crypter output size mismatch, expect {} vs actual {}", + count, crypter_count, + ))); } self.offset += count as u64; Ok(&self.buffer[..count]) @@ -1077,7 +1068,7 @@ mod tests { let key = generate_data_key(method).unwrap().1; let mut wt = EncrypterWriter::new(YieldOnce::new(buf), method, &key, iv).unwrap(); let waker = Waker::noop(); - let mut cx = Context::from_waker(&waker); + let mut cx = Context::from_waker(waker); assert_matches!( Pin::new(&mut wt).poll_write(&mut cx, &plain_text[..size / 2]), Poll::Pending diff --git a/components/encryption/src/lib.rs b/components/encryption/src/lib.rs index 086c2c0ae8f..9bc7881d87b 100644 --- a/components/encryption/src/lib.rs +++ b/components/encryption/src/lib.rs @@ -1,7 +1,6 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. #![feature(let_chains)] -#![feature(noop_waker)] mod backup; mod config; @@ -23,12 +22,12 @@ use std::{io::ErrorKind, path::Path}; pub use self::{ backup::backup_encryption::*, config::*, - crypter::{verify_encryption_config, AesGcmCrypter, FileEncryptionInfo, Iv}, + crypter::{AesGcmCrypter, FileEncryptionInfo, Iv, verify_encryption_config}, encrypted_file::EncryptedFile, - errors::{cloud_convert_error, Error, Result, RetryCodedError}, + errors::{Error, Result, RetryCodedError, cloud_convert_error}, file_dict_file::FileDictionaryFile, io::{ - create_aes_ctr_crypter, DecrypterReader, DecrypterWriter, EncrypterReader, EncrypterWriter, + DecrypterReader, DecrypterWriter, EncrypterReader, EncrypterWriter, create_aes_ctr_crypter, }, manager::{DataKeyImporter, DataKeyManager, DataKeyManagerArgs}, master_key::{ diff --git a/components/encryption/src/manager/mod.rs b/components/encryption/src/manager/mod.rs index 9250ed0714c..a8797fed2bc 100644 --- a/components/encryption/src/manager/mod.rs +++ b/components/encryption/src/manager/mod.rs @@ -5,8 +5,8 @@ use std::{ io::{self, Error as IoError, ErrorKind, Result as IoResult}, path::{Path, PathBuf}, sync::{ - atomic::{AtomicU64, Ordering}, Arc, Mutex, + atomic::{AtomicU64, Ordering}, }, thread::JoinHandle, time::{Duration, SystemTime, UNIX_EPOCH}, @@ -21,6 +21,7 @@ use protobuf::Message; use tikv_util::{box_err, debug, error, info, sys::thread::StdThreadBuildWrapper, thd_name, warn}; use crate::{ + Error, Result, config::EncryptionConfig, crypter::{self, FileEncryptionInfo, Iv}, encrypted_file::EncryptedFile, @@ -28,7 +29,6 @@ use crate::{ io::{DecrypterReader, EncrypterWriter}, master_key::Backend, metrics::*, - Error, Result, }; const KEY_DICT_NAME: &str = "key.dict"; @@ -383,7 +383,7 @@ fn check_stale_file_exist( file_dict: &mut FileDictionary, file_dict_file: &mut FileDictionaryFile, ) -> Result<()> { - if file_dict.files.get(fname).is_some() { + if file_dict.files.contains_key(fname) { if Path::new(fname).exists() { return Err(Error::Io(IoError::new( ErrorKind::AlreadyExists, @@ -816,10 +816,10 @@ impl DataKeyManager { while let Some(e) = iter.next() { let e = e?; if e.path().is_symlink() { - return Err(io::Error::new( - io::ErrorKind::Other, - format!("unexpected symbolic link: {}", e.path().display()), - )); + return Err(io::Error::other(format!( + "unexpected symbolic link: {}", + e.path().display() + ))); } let fname = e.path().to_str().unwrap(); let sync = iter.peek().is_none(); @@ -933,10 +933,10 @@ impl DataKeyManager { while let Some(e) = iter.next() { let e = e?; if e.path().is_symlink() { - return Err(io::Error::new( - io::ErrorKind::Other, - format!("unexpected symbolic link: {}", e.path().display()), - )); + return Err(io::Error::other(format!( + "unexpected symbolic link: {}", + e.path().display() + ))); } let sub_path = e.path().strip_prefix(src_path).unwrap(); let src = e.path().to_str().unwrap(); @@ -1130,7 +1130,7 @@ impl<'a> Drop for DataKeyImporter<'a> { #[cfg(test)] mod tests { - use file_system::{remove_file, File}; + use file_system::{File, remove_file}; use kvproto::encryptionpb::EncryptionMethod; use matches::assert_matches; use tempfile::TempDir; @@ -1138,8 +1138,8 @@ mod tests { use super::*; use crate::master_key::{ - tests::{decrypt_called, encrypt_called, MockBackend}, FileBackend, PlaintextBackend, + tests::{MockBackend, decrypt_called, encrypt_called}, }; lazy_static::lazy_static! { diff --git a/components/encryption/src/master_key/kms.rs b/components/encryption/src/master_key/kms.rs index ad89d217f92..fe088595292 100644 --- a/components/encryption/src/master_key/kms.rs +++ b/components/encryption/src/master_key/kms.rs @@ -12,8 +12,8 @@ use tikv_util::{ }; use tokio::runtime::{Builder, Runtime}; -use super::{metadata::MetadataKey, AsyncBackend, Backend, MemAesGcmBackend}; -use crate::{crypter::Iv, errors::cloud_convert_error, Error, Result}; +use super::{AsyncBackend, Backend, MemAesGcmBackend, metadata::MetadataKey}; +use crate::{Error, Result, crypter::Iv, errors::cloud_convert_error}; #[derive(Debug)] struct State { diff --git a/components/encryption/src/master_key/mem.rs b/components/encryption/src/master_key/mem.rs index c19351f5dc7..77c2fdbb2e3 100644 --- a/components/encryption/src/master_key/mem.rs +++ b/components/encryption/src/master_key/mem.rs @@ -5,7 +5,7 @@ use kvproto::encryptionpb::EncryptedContent; use tikv_util::box_err; use super::metadata::*; -use crate::{crypter::*, errors::cloud_convert_error, AesGcmCrypter, Error, Iv, Result}; +use crate::{AesGcmCrypter, Error, Iv, Result, crypter::*, errors::cloud_convert_error}; /// An in-memory backend, it saves master key in memory. #[derive(Debug)] diff --git a/components/encryption/src/master_key/mod.rs b/components/encryption/src/master_key/mod.rs index ab17ecdd0f0..c2c81950ba3 100644 --- a/components/encryption/src/master_key/mod.rs +++ b/components/encryption/src/master_key/mod.rs @@ -7,7 +7,7 @@ use kvproto::encryptionpb::{EncryptedContent, EncryptionMethod, MasterKey}; use tikv_util::{box_err, error}; use tokio::sync::RwLock; -use crate::{manager::generate_data_key, Error, MasterKeyConfig, Result}; +use crate::{Error, MasterKeyConfig, Result, manager::generate_data_key}; /// Provide API to encrypt/decrypt key dictionary content. /// @@ -38,9 +38,9 @@ mod metadata; use self::metadata::*; mod kms; +pub use self::kms::KmsBackend; #[cfg(any(test, feature = "testexport"))] pub use self::kms::fake; -pub use self::kms::KmsBackend; #[derive(Default, Debug, Clone)] pub struct PlaintextBackend {} @@ -371,14 +371,12 @@ pub mod tests { .update_from_config_if_needed(configs, |_| { Ok(create_mock_backend( || { - Err(Error::Other(Box::new(std::io::Error::new( - ErrorKind::Other, + Err(Error::Other(Box::new(std::io::Error::other( "Encrypt error", )))) }, || { - Err(Error::Other(Box::new(std::io::Error::new( - ErrorKind::Other, + Err(Error::Other(Box::new(std::io::Error::other( "Decrypt error", )))) }, @@ -432,8 +430,7 @@ pub mod tests { let backends: Vec> = vec![ create_mock_backend( || { - Err(Error::Other(Box::new(std::io::Error::new( - ErrorKind::Other, + Err(Error::Other(Box::new(std::io::Error::other( "Encrypt error 1", )))) }, @@ -442,8 +439,7 @@ pub mod tests { create_mock_backend( || Ok(EncryptedContent::default()), || { - Err(Error::Other(Box::new(std::io::Error::new( - ErrorKind::Other, + Err(Error::Other(Box::new(std::io::Error::other( "Decrypt error 2", )))) }, diff --git a/components/engine_panic/src/db_vector.rs b/components/engine_panic/src/db_vector.rs index 3daf6dc9500..5b7c6d73bad 100644 --- a/components/engine_panic/src/db_vector.rs +++ b/components/engine_panic/src/db_vector.rs @@ -17,7 +17,7 @@ impl Deref for PanicDbVector { } } -impl<'a> PartialEq<&'a [u8]> for PanicDbVector { +impl PartialEq<&[u8]> for PanicDbVector { fn eq(&self, rhs: &&[u8]) -> bool { **rhs == **self } diff --git a/components/engine_rocks/src/checkpoint.rs b/components/engine_rocks/src/checkpoint.rs index 3e085621f34..65f44a0ddc9 100644 --- a/components/engine_rocks/src/checkpoint.rs +++ b/components/engine_rocks/src/checkpoint.rs @@ -4,7 +4,7 @@ use std::path::Path; use engine_traits::{Checkpointable, Checkpointer, Result}; -use crate::{r2e, RocksEngine}; +use crate::{RocksEngine, r2e}; impl Checkpointable for RocksEngine { type Checkpointer = RocksEngineCheckpointer; @@ -44,7 +44,7 @@ impl Checkpointer for RocksEngineCheckpointer { #[cfg(test)] mod tests { - use engine_traits::{Checkpointable, Checkpointer, MiscExt, Peekable, SyncMutable, ALL_CFS}; + use engine_traits::{ALL_CFS, Checkpointable, Checkpointer, MiscExt, Peekable, SyncMutable}; use tempfile::tempdir; use crate::util::new_engine; diff --git a/components/engine_rocks/src/compact.rs b/components/engine_rocks/src/compact.rs index 53447e9a582..91fbe1f3313 100644 --- a/components/engine_rocks/src/compact.rs +++ b/components/engine_rocks/src/compact.rs @@ -134,7 +134,7 @@ mod tests { use engine_traits::{CfNamesExt, CfOptionsExt, CompactExt, MiscExt, SyncMutable}; use tempfile::Builder; - use crate::{util, RocksCfOptions, RocksDbOptions}; + use crate::{RocksCfOptions, RocksDbOptions, util}; #[test] fn test_compact_files_in_range() { diff --git a/components/engine_rocks/src/compact_listener.rs b/components/engine_rocks/src/compact_listener.rs index e679410c8b9..782d82a182d 100644 --- a/components/engine_rocks/src/compact_listener.rs +++ b/components/engine_rocks/src/compact_listener.rs @@ -138,11 +138,8 @@ impl RocksCompactedEvent { impl CompactedEvent for RocksCompactedEvent { fn total_bytes_declined(&self) -> u64 { - if self.total_input_bytes > self.total_output_bytes { - self.total_input_bytes - self.total_output_bytes - } else { - 0 - } + self.total_input_bytes + .saturating_sub(self.total_output_bytes) } fn is_size_declining_trivial(&self, split_check_diff: u64) -> bool { diff --git a/components/engine_rocks/src/config.rs b/components/engine_rocks/src/config.rs index 2a444a22d3e..11af54d8c5c 100644 --- a/components/engine_rocks/src/config.rs +++ b/components/engine_rocks/src/config.rs @@ -65,9 +65,9 @@ pub mod compression_type_level_serde { use rocksdb::DBCompressionType; use serde::{ + Deserializer, Serializer, de::{Error, SeqAccess, Unexpected, Visitor}, ser::SerializeSeq, - Deserializer, Serializer, }; pub fn serialize(ts: &[DBCompressionType; 7], serializer: S) -> Result @@ -152,8 +152,8 @@ pub mod compression_type_serde { use rocksdb::DBCompressionType; use serde::{ - de::{Error, Unexpected, Visitor}, Deserializer, Serializer, + de::{Error, Unexpected, Visitor}, }; pub fn serialize(t: &DBCompressionType, serializer: S) -> Result @@ -257,8 +257,8 @@ pub mod checksum_serde { use rocksdb::ChecksumType; use serde::{ - de::{Error, Unexpected, Visitor}, Deserializer, Serializer, + de::{Error, Unexpected, Visitor}, }; pub fn serialize(t: &ChecksumType, serializer: S) -> Result @@ -317,8 +317,8 @@ pub mod prepopulate_block_cache_serde { use rocksdb::PrepopulateBlockCache; use serde::{ - de::{Error, Unexpected, Visitor}, Deserializer, Serializer, + de::{Error, Unexpected, Visitor}, }; pub fn serialize(t: &PrepopulateBlockCache, serializer: S) -> Result diff --git a/components/engine_rocks/src/db_vector.rs b/components/engine_rocks/src/db_vector.rs index 97fa65b7072..3b88e69717b 100644 --- a/components/engine_rocks/src/db_vector.rs +++ b/components/engine_rocks/src/db_vector.rs @@ -32,7 +32,7 @@ impl Debug for RocksDbVector { } } -impl<'a> PartialEq<&'a [u8]> for RocksDbVector { +impl PartialEq<&[u8]> for RocksDbVector { fn eq(&self, rhs: &&[u8]) -> bool { **rhs == **self } diff --git a/components/engine_rocks/src/decode_properties.rs b/components/engine_rocks/src/decode_properties.rs index 70b1d52ba8d..fc3258cc4d5 100644 --- a/components/engine_rocks/src/decode_properties.rs +++ b/components/engine_rocks/src/decode_properties.rs @@ -9,8 +9,8 @@ use std::{ }; use tikv_util::codec::{ - number::{self, NumberEncoder}, Result, + number::{self, NumberEncoder}, }; #[derive(Clone, Debug, Default)] diff --git a/components/engine_rocks/src/engine.rs b/components/engine_rocks/src/engine.rs index 4cd51a7fcad..4273d4fbd91 100644 --- a/components/engine_rocks/src/engine.rs +++ b/components/engine_rocks/src/engine.rs @@ -3,12 +3,12 @@ use std::{any::Any, sync::Arc}; use engine_traits::{IterOptions, Iterable, KvEngine, Peekable, ReadOptions, Result, SyncMutable}; -use rocksdb::{DBIterator, Writable, DB}; +use rocksdb::{DB, DBIterator, Writable}; use tikv_util::range_latch::RangeLatch; use crate::{ - db_vector::RocksDbVector, options::RocksReadOptions, r2e, util::get_cf_handle, - RocksEngineIterator, RocksSnapshot, + RocksEngineIterator, RocksSnapshot, db_vector::RocksDbVector, options::RocksReadOptions, r2e, + util::get_cf_handle, }; #[cfg(feature = "trace-lifetime")] @@ -30,8 +30,8 @@ mod trace { collections::BTreeMap, ops::Bound::Included, sync::{ - atomic::{AtomicU64, Ordering}, Mutex, + atomic::{AtomicU64, Ordering}, }, }; @@ -278,14 +278,14 @@ impl SyncMutable for RocksEngine { #[cfg(test)] mod tests { - use engine_traits::{Iterable, KvEngine, Peekable, SyncMutable, CF_DEFAULT}; + use engine_traits::{CF_DEFAULT, Iterable, KvEngine, Peekable, SyncMutable}; use kvproto::metapb::Region; use proptest::prelude::*; - use rocksdb::{DBOptions, FlushOptions, SeekKey, TitanDBOptions, Writable, DB}; + use rocksdb::{DB, DBOptions, FlushOptions, SeekKey, TitanDBOptions, Writable}; use tempfile::Builder; use tikv_util::config::ReadableSize; - use crate::{util, RocksSnapshot}; + use crate::{RocksSnapshot, util}; #[test] fn test_base() { diff --git a/components/engine_rocks/src/engine_iterator.rs b/components/engine_rocks/src/engine_iterator.rs index 5db3e21a03d..6f69da2c7eb 100644 --- a/components/engine_rocks/src/engine_iterator.rs +++ b/components/engine_rocks/src/engine_iterator.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use engine_traits::{self, IterMetricsCollector, MetricsExt, Result}; -use rocksdb::{DBIterator, PerfContext, DB}; +use rocksdb::{DB, DBIterator, PerfContext}; use crate::r2e; diff --git a/components/engine_rocks/src/event_listener.rs b/components/engine_rocks/src/event_listener.rs index 4ba4061a60f..12ea6504191 100644 --- a/components/engine_rocks/src/event_listener.rs +++ b/components/engine_rocks/src/event_listener.rs @@ -1,7 +1,7 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. use engine_traits::PersistenceListener; -use file_system::{get_io_type, set_io_type, IoType}; +use file_system::{IoType, get_io_type, set_io_type}; use regex::Regex; use rocksdb::{ CompactionJobInfo, DBBackgroundErrorReason, FlushJobInfo, IngestionInfo, MemTableInfo, @@ -175,10 +175,7 @@ impl rocksdb::EventListener for RocksEventListener { // We assume that only the corruption sst file path is printed inside error. fn resolve_sst_filename_from_err(err: &str) -> Option { let r = Regex::new(r"/\w*\.sst").unwrap(); - let matches = match r.captures(err) { - None => return None, - Some(v) => v, - }; + let matches = r.captures(err)?; let filename = matches.get(0).unwrap().as_str().to_owned(); Some(filename) } @@ -227,17 +224,17 @@ impl rocksdb::EventListener for RocksPersistenceListener { #[cfg(test)] mod tests { use std::sync::{ - mpsc::{self, Sender}, Arc, Mutex, + mpsc::{self, Sender}, }; use engine_traits::{ - ApplyProgress, FlushState, MiscExt, StateStorage, SyncMutable, CF_DEFAULT, DATA_CFS, + ApplyProgress, CF_DEFAULT, DATA_CFS, FlushState, MiscExt, StateStorage, SyncMutable, }; use tempfile::Builder; use super::*; - use crate::{util, RocksCfOptions, RocksDbOptions}; + use crate::{RocksCfOptions, RocksDbOptions, util}; #[test] fn test_resolve_sst_filename() { @@ -316,7 +313,7 @@ mod tests { Ok(p) => p, Err(_) => return false, }; - p.path().extension().map_or(false, |ext| ext == "sst") + p.path().extension().is_some_and(|ext| ext == "sst") }) .count() }; diff --git a/components/engine_rocks/src/file_system.rs b/components/engine_rocks/src/file_system.rs index 4f584347747..ee5762d1942 100644 --- a/components/engine_rocks/src/file_system.rs +++ b/components/engine_rocks/src/file_system.rs @@ -42,15 +42,15 @@ impl DBFileSystemInspector for WrappedFileSystemInspecto mod tests { use std::sync::Arc; - use engine_traits::{CompactExt, ManualCompactionOptions, MiscExt, SyncMutable, CF_DEFAULT}; + use engine_traits::{CF_DEFAULT, CompactExt, ManualCompactionOptions, MiscExt, SyncMutable}; use file_system::{IoOp, IoRateLimiter, IoRateLimiterStatistics, IoType}; use keys::data_key; use tempfile::Builder; use super::*; use crate::{ - event_listener::RocksEventListener, raw::DBCompressionType, util::new_engine_opt, - RocksCfOptions, RocksDbOptions, RocksEngine, + RocksCfOptions, RocksDbOptions, RocksEngine, event_listener::RocksEventListener, + raw::DBCompressionType, util::new_engine_opt, }; fn new_test_db(dir: &str) -> (RocksEngine, Arc) { diff --git a/components/engine_rocks/src/flow_listener.rs b/components/engine_rocks/src/flow_listener.rs index 4a4f80cc46f..3d9f048f2df 100644 --- a/components/engine_rocks/src/flow_listener.rs +++ b/components/engine_rocks/src/flow_listener.rs @@ -1,6 +1,6 @@ // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. -use std::sync::{mpsc::Sender, Arc, Mutex}; +use std::sync::{Arc, Mutex, mpsc::Sender}; use collections::hash_set_with_capacity; use rocksdb::{CompactionJobInfo, EventListener, FlushJobInfo, IngestionInfo}; @@ -122,8 +122,7 @@ impl EventListener for FlowListener { } } - let diff = if output < input { input - output } else { 0 }; - + let diff = input.saturating_sub(output); let _ = self .flow_info_sender .lock() diff --git a/components/engine_rocks/src/import.rs b/components/engine_rocks/src/import.rs index 14947149c1b..08ad9f245cd 100644 --- a/components/engine_rocks/src/import.rs +++ b/components/engine_rocks/src/import.rs @@ -99,13 +99,13 @@ impl IngestExternalFileOptions for RocksIngestExternalFileOptions { #[cfg(test)] mod tests { use engine_traits::{ - FlowControlFactorsExt, MiscExt, Mutable, SstWriter, SstWriterBuilder, WriteBatch, - WriteBatchExt, ALL_CFS, CF_DEFAULT, + ALL_CFS, CF_DEFAULT, FlowControlFactorsExt, MiscExt, Mutable, SstWriter, SstWriterBuilder, + WriteBatch, WriteBatchExt, }; use tempfile::Builder; use super::*; - use crate::{util::new_engine_opt, RocksCfOptions, RocksDbOptions, RocksSstWriterBuilder}; + use crate::{RocksCfOptions, RocksDbOptions, RocksSstWriterBuilder, util::new_engine_opt}; #[test] fn test_ingest_multiple_file() { diff --git a/components/engine_rocks/src/lib.rs b/components/engine_rocks/src/lib.rs index 28c7c97d0a8..6d45211e26a 100644 --- a/components/engine_rocks/src/lib.rs +++ b/components/engine_rocks/src/lib.rs @@ -17,7 +17,6 @@ #![cfg_attr(test, feature(test))] #![feature(let_chains)] -#![feature(option_get_or_insert_default)] #![feature(path_file_prefix)] #[allow(unused_extern_crates)] @@ -108,8 +107,8 @@ pub mod file_system; mod raft_engine; pub use rocksdb::{ - set_perf_flags, set_perf_level, PerfContext, PerfFlag, PerfFlags, PerfLevel, - Statistics as RocksStatistics, + PerfContext, PerfFlag, PerfFlags, PerfLevel, Statistics as RocksStatistics, set_perf_flags, + set_perf_level, }; pub mod flow_control_factors; diff --git a/components/engine_rocks/src/misc.rs b/components/engine_rocks/src/misc.rs index 464106f302e..171443f8678 100644 --- a/components/engine_rocks/src/misc.rs +++ b/components/engine_rocks/src/misc.rs @@ -10,12 +10,13 @@ use rocksdb::{FlushOptions, Range as RocksRange}; use tikv_util::{box_try, keybuilder::KeyBuilder}; use crate::{ + RocksSstWriter, engine::RocksEngine, r2e, rocks_metrics::{RocksStatisticsReporter, STORE_ENGINE_EVENT_COUNTER_VEC}, rocks_metrics_defs::*, sst::RocksSstWriterBuilder, - util, RocksSstWriter, + util, }; pub const MAX_DELETE_COUNT_BY_KEY: usize = 2048; @@ -46,7 +47,7 @@ impl RocksEngine { // There may be a range overlap with next range if last_end_key .as_ref() - .map_or(false, |key| key.as_slice() > r.start_key) + .is_some_and(|key| key.as_slice() > r.start_key) { written |= self.delete_all_in_range_cf_by_key(wopts, cf, r)?; continue; @@ -211,7 +212,7 @@ impl MiscExt for RocksEngine { .map(|(_, time)| (handle, time)) }) .min_by(|(_, a), (_, b)| a.cmp(b)) - && age_threshold.map_or(true, |threshold| time <= threshold) + && age_threshold.is_none_or(|threshold| time <= threshold) { let mut fopts = FlushOptions::default(); fopts.set_wait(wait); @@ -509,16 +510,16 @@ impl MiscExt for RocksEngine { #[cfg(test)] mod tests { use engine_traits::{ - CompactExt, DeleteStrategy, Iterable, Iterator, ManualCompactionOptions, Mutable, - SyncMutable, WriteBatchExt, ALL_CFS, + ALL_CFS, CompactExt, DeleteStrategy, Iterable, Iterator, ManualCompactionOptions, Mutable, + SyncMutable, WriteBatchExt, }; use tempfile::Builder; use super::*; use crate::{ + RocksCfOptions, RocksDbOptions, engine::RocksEngine, util::{new_engine, new_engine_opt}, - RocksCfOptions, RocksDbOptions, }; fn check_data(db: &RocksEngine, cfs: &[&str], expected: &[(&[u8], &[u8])]) { diff --git a/components/engine_rocks/src/mvcc_properties.rs b/components/engine_rocks/src/mvcc_properties.rs index 99ac5fbe6b2..3384904de73 100644 --- a/components/engine_rocks/src/mvcc_properties.rs +++ b/components/engine_rocks/src/mvcc_properties.rs @@ -3,7 +3,7 @@ use engine_traits::{MvccProperties, MvccPropertiesExt, Result}; use txn_types::TimeStamp; -use crate::{decode_properties::DecodeProperties, RocksEngine, RocksTtlProperties, UserProperties}; +use crate::{RocksEngine, RocksTtlProperties, UserProperties, decode_properties::DecodeProperties}; pub const PROP_NUM_ERRORS: &str = "tikv.num_errors"; pub const PROP_MIN_TS: &str = "tikv.min_ts"; diff --git a/components/engine_rocks/src/perf_context_impl.rs b/components/engine_rocks/src/perf_context_impl.rs index 59086127154..c33fed20d3f 100644 --- a/components/engine_rocks/src/perf_context_impl.rs +++ b/components/engine_rocks/src/perf_context_impl.rs @@ -7,11 +7,11 @@ use engine_traits::{PerfContextKind, PerfLevel}; use lazy_static::lazy_static; use slog_derive::KV; use tikv_util::time::Instant; -use tracker::{Tracker, TrackerToken, GLOBAL_TRACKERS}; +use tracker::{GLOBAL_TRACKERS, Tracker, TrackerToken}; use crate::{ - perf_context_metrics::*, set_perf_flags, set_perf_level, util, PerfContext as RawPerfContext, - PerfFlag, PerfFlags, + PerfContext as RawPerfContext, PerfFlag, PerfFlags, perf_context_metrics::*, set_perf_flags, + set_perf_level, util, }; macro_rules! report_write_perf_context { diff --git a/components/engine_rocks/src/properties.rs b/components/engine_rocks/src/properties.rs index 01ca0447a06..7a37307c7cc 100644 --- a/components/engine_rocks/src/properties.rs +++ b/components/engine_rocks/src/properties.rs @@ -5,19 +5,18 @@ use std::{ collections::HashMap, io::Read, ops::{Deref, DerefMut}, - u64, }; use api_version::{ApiV2, KeyMode, KvFormat}; -use engine_traits::{raw_ttl::ttl_current_ts, MvccProperties, Range, RangeStats}; +use engine_traits::{MvccProperties, Range, RangeStats, raw_ttl::ttl_current_ts}; use rocksdb::{ DBEntryType, TablePropertiesCollector, TablePropertiesCollectorFactory, TitanBlobIndex, UserCollectedProperties, }; use tikv_util::{ codec::{ - number::{self, NumberEncoder}, Error, Result, + number::{self, NumberEncoder}, }, info, }; @@ -574,7 +573,7 @@ pub fn get_range_stats( #[cfg(test)] mod tests { use api_version::RawValue; - use engine_traits::{MiscExt, SyncMutable, CF_WRITE, LARGE_CFS}; + use engine_traits::{CF_WRITE, LARGE_CFS, MiscExt, SyncMutable}; use rand::Rng; use tempfile::Builder; use test::Bencher; @@ -582,8 +581,8 @@ mod tests { use super::*; use crate::{ - raw::{DBEntryType, TablePropertiesCollector}, RocksCfOptions, RocksDbOptions, + raw::{DBEntryType, TablePropertiesCollector}, }; #[allow(clippy::many_single_char_names)] diff --git a/components/engine_rocks/src/raft_engine.rs b/components/engine_rocks/src/raft_engine.rs index df0988f4cdb..2f3e8ad5df4 100644 --- a/components/engine_rocks/src/raft_engine.rs +++ b/components/engine_rocks/src/raft_engine.rs @@ -2,9 +2,9 @@ // #[PerformanceCriticalPath] use engine_traits::{ - Error, Iterable, KvEngine, MiscExt, Mutable, Peekable, RaftEngine, RaftEngineDebug, - RaftEngineReadOnly, RaftLogBatch, Result, WriteBatch, WriteBatchExt, WriteOptions, CF_DEFAULT, - RAFT_LOG_MULTI_GET_CNT, + CF_DEFAULT, Error, Iterable, KvEngine, MiscExt, Mutable, Peekable, RAFT_LOG_MULTI_GET_CNT, + RaftEngine, RaftEngineDebug, RaftEngineReadOnly, RaftLogBatch, Result, WriteBatch, + WriteBatchExt, WriteOptions, }; use kvproto::{ metapb::Region, @@ -16,7 +16,7 @@ use protobuf::Message; use raft::eraftpb::Entry; use tikv_util::{box_err, box_try}; -use crate::{util, RocksEngine, RocksWriteBatchVec}; +use crate::{RocksEngine, RocksWriteBatchVec, util}; impl RaftEngineReadOnly for RocksEngine { fn get_raft_state(&self, raft_group_id: u64) -> Result> { diff --git a/components/engine_rocks/src/range_properties.rs b/components/engine_rocks/src/range_properties.rs index dfc41db5f6e..b0b7c0e2b8d 100644 --- a/components/engine_rocks/src/range_properties.rs +++ b/components/engine_rocks/src/range_properties.rs @@ -3,13 +3,13 @@ use std::path::Path; use engine_traits::{ - MiscExt, Range, RangePropertiesExt, Result, CF_DEFAULT, CF_LOCK, CF_WRITE, LARGE_CFS, + CF_DEFAULT, CF_LOCK, CF_WRITE, LARGE_CFS, MiscExt, Range, RangePropertiesExt, Result, }; use tikv_util::{box_err, box_try, debug, info}; use crate::{ engine::RocksEngine, - properties::{get_range_stats, RangeProperties}, + properties::{RangeProperties, get_range_stats}, }; impl RangePropertiesExt for RocksEngine { diff --git a/components/engine_rocks/src/raw.rs b/components/engine_rocks/src/raw.rs index f2c6d862280..b273ac2d7c0 100644 --- a/components/engine_rocks/src/raw.rs +++ b/components/engine_rocks/src/raw.rs @@ -7,13 +7,14 @@ //! crate, but only until the engine interface is completely abstracted. pub use rocksdb::{ - run_ldb_tool, run_sst_dump_tool, BlockBasedOptions, Cache, ChecksumType, CompactOptions, - CompactionFilter, CompactionFilterContext, CompactionFilterDecision, CompactionFilterFactory, + BlockBasedOptions, Cache, ChecksumType, CompactOptions, CompactionFilter, + CompactionFilterContext, CompactionFilterDecision, CompactionFilterFactory, CompactionFilterValueType, CompactionJobInfo, CompactionOptions, CompactionPriority, ConcurrentTaskLimiter, DBBottommostLevelCompaction, DBCompactionFilter, DBCompactionStyle, DBCompressionType, DBEntryType, DBRateLimiterMode, DBRecoveryMode, DBStatisticsTickerType, DBTableFileCreationReason, DBTitanDBBlobRunMode, Env, EventListener, FlushOptions, IngestExternalFileOptions, LRUCacheOptions, MemoryAllocator, PerfContext, PrepopulateBlockCache, Range, RateLimiter, SliceTransform, Statistics, - TablePropertiesCollector, TablePropertiesCollectorFactory, WriteBufferManager, + TablePropertiesCollector, TablePropertiesCollectorFactory, WriteBufferManager, run_ldb_tool, + run_sst_dump_tool, }; diff --git a/components/engine_rocks/src/rocks_metrics.rs b/components/engine_rocks/src/rocks_metrics.rs index 7f7294f0188..c5c71b4b5cc 100644 --- a/components/engine_rocks/src/rocks_metrics.rs +++ b/components/engine_rocks/src/rocks_metrics.rs @@ -1,7 +1,7 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. use collections::HashMap; -use engine_traits::{StatisticsReporter, CF_DEFAULT}; +use engine_traits::{CF_DEFAULT, StatisticsReporter}; use lazy_static::lazy_static; use prometheus::*; use prometheus_static_metric::*; @@ -9,7 +9,7 @@ use rocksdb::{ DBStatisticsHistogramType as HistType, DBStatisticsTickerType as TickerType, HistogramData, }; -use crate::{engine::RocksEngine, rocks_metrics_defs::*, RocksStatistics}; +use crate::{RocksStatistics, engine::RocksEngine, rocks_metrics_defs::*}; make_auto_flush_static_metric! { pub label_enum TickerName { diff --git a/components/engine_rocks/src/snapshot.rs b/components/engine_rocks/src/snapshot.rs index 22cdea503ab..1f3ed47f0b5 100644 --- a/components/engine_rocks/src/snapshot.rs +++ b/components/engine_rocks/src/snapshot.rs @@ -9,11 +9,11 @@ use engine_traits::{ self, CfNamesExt, IterOptions, Iterable, Peekable, ReadOptions, Result, Snapshot, SnapshotMiscExt, }; -use rocksdb::{rocksdb_options::UnsafeSnap, DBIterator, DB}; +use rocksdb::{DB, DBIterator, rocksdb_options::UnsafeSnap}; use crate::{ - db_vector::RocksDbVector, options::RocksReadOptions, r2e, util::get_cf_handle, - RocksEngineIterator, + RocksEngineIterator, db_vector::RocksDbVector, options::RocksReadOptions, r2e, + util::get_cf_handle, }; pub struct RocksSnapshot { diff --git a/components/engine_rocks/src/sst.rs b/components/engine_rocks/src/sst.rs index d72456cc572..dcf3dfb90d4 100644 --- a/components/engine_rocks/src/sst.rs +++ b/components/engine_rocks/src/sst.rs @@ -4,15 +4,15 @@ use std::{path::PathBuf, sync::Arc}; use ::encryption::DataKeyManager; use engine_traits::{ - Error, ExternalSstFileInfo, ExternalSstFileReader, IterOptions, Iterator, RefIterable, Result, - SstCompressionType, SstExt, SstReader, SstWriter, SstWriterBuilder, CF_DEFAULT, + CF_DEFAULT, Error, ExternalSstFileInfo, ExternalSstFileReader, IterOptions, Iterator, + RefIterable, Result, SstCompressionType, SstExt, SstReader, SstWriter, SstWriterBuilder, }; use fail::fail_point; use file_system::get_io_rate_limiter; use rocksdb::{ - rocksdb::supported_compression, ColumnFamilyOptions, DBCompressionType, DBIterator, Env, - EnvOptions, ExternalSstFileInfo as RawExternalSstFileInfo, SequentialFile, SstFileReader, - SstFileWriter, DB, + ColumnFamilyOptions, DB, DBCompressionType, DBIterator, Env, EnvOptions, + ExternalSstFileInfo as RawExternalSstFileInfo, SequentialFile, SstFileReader, SstFileWriter, + rocksdb::supported_compression, }; use crate::{engine::RocksEngine, get_env, options::RocksReadOptions, r2e}; diff --git a/components/engine_rocks/src/table_properties.rs b/components/engine_rocks/src/table_properties.rs index 19b2141483d..2402fe6000d 100644 --- a/components/engine_rocks/src/table_properties.rs +++ b/components/engine_rocks/src/table_properties.rs @@ -2,7 +2,7 @@ use engine_traits::{Range, Result}; -use crate::{r2e, util, RangeProperties, RocksEngine}; +use crate::{RangeProperties, RocksEngine, r2e, util}; #[repr(transparent)] pub struct UserCollectedProperties(rocksdb::UserCollectedProperties); @@ -27,7 +27,10 @@ impl engine_traits::TablePropertiesCollection for TablePropertiesCollection { F: FnMut(&Self::UserCollectedProperties) -> bool, { for (_, props) in self.0.into_iter() { - let props = unsafe { std::mem::transmute(props.user_collected_properties()) }; + let props = unsafe { + #[allow(clippy::missing_transmute_annotations)] + std::mem::transmute(props.user_collected_properties()) + }; if !f(props) { break; } diff --git a/components/engine_rocks/src/ttl_properties.rs b/components/engine_rocks/src/ttl_properties.rs index 62731ac1aa4..43457e68b27 100644 --- a/components/engine_rocks/src/ttl_properties.rs +++ b/components/engine_rocks/src/ttl_properties.rs @@ -7,7 +7,7 @@ use engine_traits::{Range, Result, TtlProperties, TtlPropertiesExt}; use rocksdb::{DBEntryType, TablePropertiesCollector, TablePropertiesCollectorFactory}; use tikv_util::error; -use crate::{decode_properties::DecodeProperties, RocksEngine, UserProperties}; +use crate::{RocksEngine, UserProperties, decode_properties::DecodeProperties}; const PROP_MAX_EXPIRE_TS: &str = "tikv.max_expire_ts"; const PROP_MIN_EXPIRE_TS: &str = "tikv.min_expire_ts"; diff --git a/components/engine_rocks/src/util.rs b/components/engine_rocks/src/util.rs index e51e1178854..e08f807ed3c 100644 --- a/components/engine_rocks/src/util.rs +++ b/components/engine_rocks/src/util.rs @@ -2,19 +2,19 @@ use std::{ffi::CString, fs, path::Path, str::FromStr, sync::Arc}; -use engine_traits::{Engines, Range, Result, CF_DEFAULT}; +use engine_traits::{CF_DEFAULT, Engines, Range, Result}; use fail::fail_point; use rocksdb::{ - load_latest_options, CColumnFamilyDescriptor, CFHandle, ColumnFamilyOptions, CompactionFilter, + CColumnFamilyDescriptor, CFHandle, ColumnFamilyOptions, CompactionFilter, CompactionFilterContext, CompactionFilterDecision, CompactionFilterFactory, - CompactionFilterValueType, DBTableFileCreationReason, Env, Range as RocksRange, SliceTransform, - DB, + CompactionFilterValueType, DB, DBTableFileCreationReason, Env, Range as RocksRange, + SliceTransform, load_latest_options, }; use slog_global::warn; use crate::{ - cf_options::RocksCfOptions, db_options::RocksDbOptions, engine::RocksEngine, r2e, - rocks_metrics_defs::*, RocksStatistics, + RocksStatistics, cf_options::RocksCfOptions, db_options::RocksDbOptions, engine::RocksEngine, + r2e, rocks_metrics_defs::*, }; pub fn new_temp_engine(path: &tempfile::TempDir) -> Engines { @@ -517,7 +517,7 @@ impl CompactionFilter for RangeCompactionFilter { #[cfg(test)] mod tests { use engine_traits::{ - CfOptionsExt, FlowControlFactorsExt, Iterable, MiscExt, Peekable, SyncMutable, CF_DEFAULT, + CF_DEFAULT, CfOptionsExt, FlowControlFactorsExt, Iterable, MiscExt, Peekable, SyncMutable, }; use rocksdb::DB; use tempfile::Builder; diff --git a/components/engine_rocks/src/write_batch.rs b/components/engine_rocks/src/write_batch.rs index 8c5aa1dd9b9..bbf5eb4248c 100644 --- a/components/engine_rocks/src/write_batch.rs +++ b/components/engine_rocks/src/write_batch.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use engine_traits::{self, Mutable, Result, WriteBatchExt, WriteOptions}; -use rocksdb::{Writable, WriteBatch as RawWriteBatch, DB}; +use rocksdb::{DB, Writable, WriteBatch as RawWriteBatch}; use crate::{engine::RocksEngine, options::RocksWriteOptions, r2e, util::get_cf_handle}; @@ -240,12 +240,12 @@ impl Mutable for RocksWriteBatchVec { #[cfg(test)] mod tests { - use engine_traits::{Peekable, WriteBatch, CF_DEFAULT}; + use engine_traits::{CF_DEFAULT, Peekable, WriteBatch}; use rocksdb::DBOptions as RawDBOptions; use tempfile::Builder; use super::{ - super::{util::new_engine_opt, RocksDbOptions}, + super::{RocksDbOptions, util::new_engine_opt}, *, }; use crate::RocksCfOptions; diff --git a/components/engine_rocks_helper/src/sst_recovery.rs b/components/engine_rocks_helper/src/sst_recovery.rs index 8a2d31b22fb..00bbce9e29b 100644 --- a/components/engine_rocks_helper/src/sst_recovery.rs +++ b/components/engine_rocks_helper/src/sst_recovery.rs @@ -210,7 +210,7 @@ mod tests { use std::{collections::BTreeMap, sync::Arc}; use engine_rocks::util; - use engine_traits::{CompactExt, ManualCompactionOptions, SyncMutable, CF_DEFAULT}; + use engine_traits::{CF_DEFAULT, CompactExt, ManualCompactionOptions, SyncMutable}; use kvproto::metapb::{Peer, Region}; use tempfile::Builder; diff --git a/components/engine_test/src/lib.rs b/components/engine_test/src/lib.rs index b8eb340c55a..3760dd4c63a 100644 --- a/components/engine_test/src/lib.rs +++ b/components/engine_test/src/lib.rs @@ -371,13 +371,12 @@ pub mod ctor { mod rocks { use engine_rocks::{ - get_env, + RocksCfOptions, RocksDbOptions, RocksPersistenceListener, get_env, properties::{MvccPropertiesCollectorFactory, RangePropertiesCollectorFactory}, - util::{new_engine_opt as rocks_new_engine_opt, RangeCompactionFilterFactory}, - RocksCfOptions, RocksDbOptions, RocksPersistenceListener, + util::{RangeCompactionFilterFactory, new_engine_opt as rocks_new_engine_opt}, }; use engine_traits::{ - CfOptions as _, PersistenceListener, Result, TabletContext, CF_DEFAULT, + CF_DEFAULT, CfOptions as _, PersistenceListener, Result, TabletContext, }; use super::{ diff --git a/components/engine_tirocks/src/db_vector.rs b/components/engine_tirocks/src/db_vector.rs index 67a7609ac15..7e71a8905ee 100644 --- a/components/engine_tirocks/src/db_vector.rs +++ b/components/engine_tirocks/src/db_vector.rs @@ -27,7 +27,7 @@ impl Debug for RocksPinSlice { } } -impl<'a> PartialEq<&'a [u8]> for RocksPinSlice { +impl PartialEq<&[u8]> for RocksPinSlice { #[inline] fn eq(&self, rhs: &&[u8]) -> bool { **rhs == **self diff --git a/components/engine_tirocks/src/write_batch.rs b/components/engine_tirocks/src/write_batch.rs index 1671e686917..684f7ba0863 100644 --- a/components/engine_tirocks/src/write_batch.rs +++ b/components/engine_tirocks/src/write_batch.rs @@ -97,7 +97,7 @@ impl engine_traits::WriteBatch for RocksWriteBatchVec { fn data_size(&self) -> usize { let mut size = 0; for w in &self.wbs[..=self.index] { - size += w.as_bytes().len(); + size += w.len(); } size } diff --git a/components/engine_traits/src/cf_defs.rs b/components/engine_traits/src/cf_defs.rs index 46e56351387..68133424b9e 100644 --- a/components/engine_traits/src/cf_defs.rs +++ b/components/engine_traits/src/cf_defs.rs @@ -28,5 +28,5 @@ pub fn name_to_cf(name: &str) -> Option { } pub fn is_data_cf(cf: &str) -> bool { - DATA_CFS.iter().any(|c| *c == cf) + DATA_CFS.contains(&cf) } diff --git a/components/engine_traits/src/cf_options.rs b/components/engine_traits/src/cf_options.rs index 1ed44825d37..35143bf0b18 100644 --- a/components/engine_traits/src/cf_options.rs +++ b/components/engine_traits/src/cf_options.rs @@ -1,6 +1,6 @@ // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. -use crate::{db_options::TitanCfOptions, sst_partitioner::SstPartitionerFactory, Result}; +use crate::{Result, db_options::TitanCfOptions, sst_partitioner::SstPartitionerFactory}; /// Trait for engines with column family options pub trait CfOptionsExt { diff --git a/components/engine_traits/src/compact.rs b/components/engine_traits/src/compact.rs index 9be2543baa6..275c6b2b8c0 100644 --- a/components/engine_traits/src/compact.rs +++ b/components/engine_traits/src/compact.rs @@ -4,7 +4,7 @@ use std::collections::BTreeMap; -use crate::{errors::Result, CfNamesExt}; +use crate::{CfNamesExt, errors::Result}; #[derive(Clone, Debug)] pub struct ManualCompactionOptions { diff --git a/components/engine_traits/src/file_system.rs b/components/engine_traits/src/file_system.rs index 51911b1f58e..77b7a40511f 100644 --- a/components/engine_traits/src/file_system.rs +++ b/components/engine_traits/src/file_system.rs @@ -2,7 +2,7 @@ use std::sync::Arc; -use file_system::{get_io_rate_limiter, get_io_type, IoOp, IoRateLimiter}; +use file_system::{IoOp, IoRateLimiter, get_io_rate_limiter, get_io_type}; use crate::Result; diff --git a/components/engine_traits/src/flush.rs b/components/engine_traits/src/flush.rs index 2a98c69bc8b..2e1c1602936 100644 --- a/components/engine_traits/src/flush.rs +++ b/components/engine_traits/src/flush.rs @@ -15,8 +15,8 @@ use std::{ collections::LinkedList, sync::{ - atomic::{AtomicU64, Ordering}, Arc, Mutex, RwLock, + atomic::{AtomicU64, Ordering}, }, time::Duration, }; @@ -25,7 +25,7 @@ use kvproto::import_sstpb::SstMeta; use slog_global::{info, warn}; use tikv_util::{set_panic_mark, time::Instant}; -use crate::{data_cf_offset, RaftEngine, RaftLogBatch, DATA_CFS_LEN}; +use crate::{DATA_CFS_LEN, RaftEngine, RaftLogBatch, data_cf_offset}; const HEAVY_WORKER_THRESHOLD: Duration = Duration::from_millis(25); diff --git a/components/engine_traits/src/import.rs b/components/engine_traits/src/import.rs index 1a6ef4f9334..110cf00f1c7 100644 --- a/components/engine_traits/src/import.rs +++ b/components/engine_traits/src/import.rs @@ -2,7 +2,7 @@ use tikv_util::range_latch::RangeLatchGuard; -use crate::{errors::Result, Range}; +use crate::{Range, errors::Result}; pub trait ImportExt { type IngestExternalFileOptions: IngestExternalFileOptions; diff --git a/components/engine_traits/src/lib.rs b/components/engine_traits/src/lib.rs index 0c3131f92f3..657080ee820 100644 --- a/components/engine_traits/src/lib.rs +++ b/components/engine_traits/src/lib.rs @@ -342,8 +342,8 @@ pub use crate::range::*; mod raft_engine; pub use raft_engine::{ - CacheStats, RaftEngine, RaftEngineDebug, RaftEngineReadOnly, RaftLogBatch, - RAFT_LOG_MULTI_GET_CNT, + CacheStats, RAFT_LOG_MULTI_GET_CNT, RaftEngine, RaftEngineDebug, RaftEngineReadOnly, + RaftLogBatch, }; // These modules need further scrutiny diff --git a/components/engine_traits/src/misc.rs b/components/engine_traits/src/misc.rs index f8318948fc1..45746c9d3f6 100644 --- a/components/engine_traits/src/misc.rs +++ b/components/engine_traits/src/misc.rs @@ -6,8 +6,8 @@ //! FIXME: Things here need to be moved elsewhere. use crate::{ - cf_names::CfNamesExt, errors::Result, flow_control_factors::FlowControlFactorsExt, - range::Range, KvEngine, WriteBatchExt, WriteOptions, + KvEngine, WriteBatchExt, WriteOptions, cf_names::CfNamesExt, errors::Result, + flow_control_factors::FlowControlFactorsExt, range::Range, }; #[derive(Clone, Debug)] diff --git a/components/engine_traits/src/raft_engine.rs b/components/engine_traits/src/raft_engine.rs index 01b8fca875b..2e2444cb46a 100644 --- a/components/engine_traits/src/raft_engine.rs +++ b/components/engine_traits/src/raft_engine.rs @@ -193,9 +193,9 @@ pub trait RaftLogBatch: Send { /// /// There are two types of apply index: /// 1. Normal apply index that only related to single tablet. These apply - /// indexes are recorded using its own CF. + /// indexes are recorded using its own CF. /// 2. Apply index that can affect other tablets, like split, merge. These - /// apply indexes are recorded using special Raft CF. + /// apply indexes are recorded using special Raft CF. /// /// Because a peer may have multiple tablets (only one is latest), we use /// `tablet_index` to avoid conflicts. diff --git a/components/engine_traits/src/range_properties.rs b/components/engine_traits/src/range_properties.rs index f97008dd929..5646443cd99 100644 --- a/components/engine_traits/src/range_properties.rs +++ b/components/engine_traits/src/range_properties.rs @@ -5,7 +5,7 @@ //! In RocksDB these are typically implemented with user collected properties, //! which might require the database to be constructed with specific options. -use crate::{errors::Result, Range}; +use crate::{Range, errors::Result}; pub trait RangePropertiesExt { /// Gets the number of keys in a range. diff --git a/components/engine_traits/src/snapshot.rs b/components/engine_traits/src/snapshot.rs index e8f61fc9d68..95709d73f5a 100644 --- a/components/engine_traits/src/snapshot.rs +++ b/components/engine_traits/src/snapshot.rs @@ -2,7 +2,7 @@ use std::fmt::Debug; -use crate::{iterable::Iterable, peekable::Peekable, CfNamesExt, SnapshotMiscExt}; +use crate::{CfNamesExt, SnapshotMiscExt, iterable::Iterable, peekable::Peekable}; /// A consistent read-only view of the database. /// diff --git a/components/engine_traits/src/sst.rs b/components/engine_traits/src/sst.rs index c0cdf998d16..d0474b5186f 100644 --- a/components/engine_traits/src/sst.rs +++ b/components/engine_traits/src/sst.rs @@ -5,7 +5,7 @@ use std::{path::PathBuf, str::FromStr, sync::Arc}; use encryption::DataKeyManager; use kvproto::import_sstpb::SstMeta; -use crate::{errors::Result, RefIterable}; +use crate::{RefIterable, errors::Result}; #[derive(Clone, Debug)] pub struct SstMetaInfo { diff --git a/components/engine_traits/src/tablet.rs b/components/engine_traits/src/tablet.rs index 64e6dcbd4b4..68144a826c3 100644 --- a/components/engine_traits/src/tablet.rs +++ b/components/engine_traits/src/tablet.rs @@ -4,8 +4,8 @@ use std::{ fmt::{self, Debug, Formatter}, path::{Path, PathBuf}, sync::{ - atomic::{AtomicU64, Ordering}, Arc, Mutex, + atomic::{AtomicU64, Ordering}, }, }; diff --git a/components/engine_traits_tests/src/basic_read_write.rs b/components/engine_traits_tests/src/basic_read_write.rs index 38a1921dd85..9847751a22e 100644 --- a/components/engine_traits_tests/src/basic_read_write.rs +++ b/components/engine_traits_tests/src/basic_read_write.rs @@ -2,7 +2,7 @@ //! Reading and writing -use engine_traits::{Peekable, SyncMutable, ALL_CFS, CF_DEFAULT}; +use engine_traits::{ALL_CFS, CF_DEFAULT, Peekable, SyncMutable}; use super::engine_cfs; diff --git a/components/engine_traits_tests/src/cf_names.rs b/components/engine_traits_tests/src/cf_names.rs index f85c2f5df97..cd3e65c2691 100644 --- a/components/engine_traits_tests/src/cf_names.rs +++ b/components/engine_traits_tests/src/cf_names.rs @@ -1,6 +1,6 @@ // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. -use engine_traits::{CfNamesExt, ALL_CFS, CF_DEFAULT}; +use engine_traits::{ALL_CFS, CF_DEFAULT, CfNamesExt}; use super::{default_engine, engine_cfs}; diff --git a/components/engine_traits_tests/src/checkpoint.rs b/components/engine_traits_tests/src/checkpoint.rs index e531e55a55e..523a2260c73 100644 --- a/components/engine_traits_tests/src/checkpoint.rs +++ b/components/engine_traits_tests/src/checkpoint.rs @@ -10,7 +10,7 @@ use engine_test::{ kv::KvTestEngine, }; use engine_traits::{ - Checkpointable, Checkpointer, KvEngine, Peekable, SyncMutable, ALL_CFS, CF_DEFAULT, + ALL_CFS, CF_DEFAULT, Checkpointable, Checkpointer, KvEngine, Peekable, SyncMutable, }; use super::tempdir; diff --git a/components/engine_traits_tests/src/ctor.rs b/components/engine_traits_tests/src/ctor.rs index ba3154d9267..3e0b6bd9b27 100644 --- a/components/engine_traits_tests/src/ctor.rs +++ b/components/engine_traits_tests/src/ctor.rs @@ -9,7 +9,7 @@ use engine_test::{ ctor::{CfOptions, DbOptions, KvEngineConstructorExt}, kv::KvTestEngine, }; -use engine_traits::{KvEngine, Peekable, SyncMutable, ALL_CFS, CF_DEFAULT}; +use engine_traits::{ALL_CFS, CF_DEFAULT, KvEngine, Peekable, SyncMutable}; use super::tempdir; diff --git a/components/engine_traits_tests/src/iterator.rs b/components/engine_traits_tests/src/iterator.rs index 714ca4cb0b4..8c8315024d0 100644 --- a/components/engine_traits_tests/src/iterator.rs +++ b/components/engine_traits_tests/src/iterator.rs @@ -1,6 +1,6 @@ // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. -use engine_traits::{Iterable, Iterator, KvEngine, CF_DEFAULT}; +use engine_traits::{CF_DEFAULT, Iterable, Iterator, KvEngine}; use panic_hook::recover_safe; use super::default_engine; diff --git a/components/engine_traits_tests/src/read_consistency.rs b/components/engine_traits_tests/src/read_consistency.rs index 8c7ab50657f..d0e639af96c 100644 --- a/components/engine_traits_tests/src/read_consistency.rs +++ b/components/engine_traits_tests/src/read_consistency.rs @@ -2,7 +2,7 @@ //! Testing iterator and snapshot behavior in the presence of intermixed writes -use engine_traits::{Iterable, Iterator, KvEngine, Peekable, SyncMutable, CF_DEFAULT}; +use engine_traits::{CF_DEFAULT, Iterable, Iterator, KvEngine, Peekable, SyncMutable}; use super::default_engine; diff --git a/components/engine_traits_tests/src/scenario_writes.rs b/components/engine_traits_tests/src/scenario_writes.rs index 169be158006..204db7064d2 100644 --- a/components/engine_traits_tests/src/scenario_writes.rs +++ b/components/engine_traits_tests/src/scenario_writes.rs @@ -2,8 +2,8 @@ use engine_test::kv::KvTestEngine; use engine_traits::{ - Mutable, Peekable, Result, SyncMutable, WriteBatch, WriteBatchExt, ALL_CFS, CF_DEFAULT, - CF_WRITE, + ALL_CFS, CF_DEFAULT, CF_WRITE, Mutable, Peekable, Result, SyncMutable, WriteBatch, + WriteBatchExt, }; use panic_hook::recover_safe; diff --git a/components/engine_traits_tests/src/snapshot_basic.rs b/components/engine_traits_tests/src/snapshot_basic.rs index c0f93480830..c35a897a557 100644 --- a/components/engine_traits_tests/src/snapshot_basic.rs +++ b/components/engine_traits_tests/src/snapshot_basic.rs @@ -1,6 +1,6 @@ // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. -use engine_traits::{KvEngine, Peekable, SyncMutable, ALL_CFS, CF_WRITE}; +use engine_traits::{ALL_CFS, CF_WRITE, KvEngine, Peekable, SyncMutable}; use super::{default_engine, engine_cfs}; diff --git a/components/external_storage/examples/scli.rs b/components/external_storage/examples/scli.rs index 9621f840e6c..c7b18e73096 100644 --- a/components/external_storage/examples/scli.rs +++ b/components/external_storage/examples/scli.rs @@ -2,18 +2,18 @@ use std::{ fs::{self, File}, - io::{Error, ErrorKind, Result}, + io::{Error, Result}, path::Path, }; use external_storage::{ - create_storage, make_azblob_backend, make_gcs_backend, make_hdfs_backend, make_local_backend, - make_noop_backend, make_s3_backend, ExternalStorage, UnpinReader, + ExternalStorage, UnpinReader, create_storage, make_azblob_backend, make_gcs_backend, + make_hdfs_backend, make_local_backend, make_noop_backend, make_s3_backend, }; -use futures_util::io::{copy, AllowStdIo}; +use futures_util::io::{AllowStdIo, copy}; use ini::ini::Ini; -use kvproto::brpb::{AzureBlobStorage, Gcs, StorageBackend, S3}; -use structopt::{clap::arg_enum, StructOpt}; +use kvproto::brpb::{AzureBlobStorage, Gcs, S3, StorageBackend}; +use structopt::{StructOpt, clap::arg_enum}; use tikv_util::stream::block_on_external_io; use tokio::runtime::Runtime; @@ -77,22 +77,18 @@ fn create_s3_storage(opt: &Opt) -> Result { let mut config = S3::default(); if let Some(credential_file) = &opt.credential_file { - let ini = Ini::load_from_file(credential_file).map_err(|e| { - Error::new( - ErrorKind::Other, - format!("Failed to parse credential file as ini: {}", e), - ) - })?; + let ini = Ini::load_from_file(credential_file) + .map_err(|e| Error::other(format!("Failed to parse credential file as ini: {}", e)))?; let props = ini .section(Some("default")) - .ok_or_else(|| Error::new(ErrorKind::Other, "fail to parse section"))?; + .ok_or_else(|| Error::other("fail to parse section"))?; config.access_key = props .get("aws_access_key_id") - .ok_or_else(|| Error::new(ErrorKind::Other, "fail to parse credential"))? + .ok_or_else(|| Error::other("fail to parse credential"))? .clone(); config.secret_access_key = props .get("aws_secret_access_key") - .ok_or_else(|| Error::new(ErrorKind::Other, "fail to parse credential"))? + .ok_or_else(|| Error::other("fail to parse credential"))? .clone(); } @@ -102,12 +98,12 @@ fn create_s3_storage(opt: &Opt) -> Result { if let Some(region) = &opt.region { config.region = region.to_string(); } else { - return Err(Error::new(ErrorKind::Other, "missing region")); + return Err(Error::other("missing region")); } if let Some(bucket) = &opt.bucket { config.bucket = bucket.to_string(); } else { - return Err(Error::new(ErrorKind::Other, "missing bucket")); + return Err(Error::other("missing bucket")); } if let Some(prefix) = &opt.prefix { config.prefix = prefix.to_string(); @@ -127,7 +123,7 @@ fn create_gcs_storage(opt: &Opt) -> Result { if let Some(bucket) = &opt.bucket { config.bucket = bucket.to_string(); } else { - return Err(Error::new(ErrorKind::Other, "missing bucket")); + return Err(Error::other("missing bucket")); } if let Some(prefix) = &opt.prefix { config.prefix = prefix.to_string(); @@ -139,22 +135,18 @@ fn create_azure_storage(opt: &Opt) -> Result { let mut config = AzureBlobStorage::default(); if let Some(credential_file) = &opt.credential_file { - let ini = Ini::load_from_file(credential_file).map_err(|e| { - Error::new( - ErrorKind::Other, - format!("Failed to parse credential file as ini: {}", e), - ) - })?; + let ini = Ini::load_from_file(credential_file) + .map_err(|e| Error::other(format!("Failed to parse credential file as ini: {}", e)))?; let props = ini .section(Some("default")) - .ok_or_else(|| Error::new(ErrorKind::Other, "fail to parse section"))?; + .ok_or_else(|| Error::other("fail to parse section"))?; config.account_name = props .get("azure_storage_name") - .ok_or_else(|| Error::new(ErrorKind::Other, "fail to parse credential"))? + .ok_or_else(|| Error::other("fail to parse credential"))? .clone(); config.shared_key = props .get("azure_storage_key") - .ok_or_else(|| Error::new(ErrorKind::Other, "fail to parse credential"))? + .ok_or_else(|| Error::other("fail to parse credential"))? .clone(); } if let Some(endpoint) = &opt.endpoint { @@ -163,7 +155,7 @@ fn create_azure_storage(opt: &Opt) -> Result { if let Some(bucket) = &opt.bucket { config.bucket = bucket.to_string(); } else { - return Err(Error::new(ErrorKind::Other, "missing bucket")); + return Err(Error::other("missing bucket")); } if let Some(prefix) = &opt.prefix { config.prefix = prefix.to_string(); diff --git a/components/external_storage/src/export.rs b/components/external_storage/src/export.rs index 127bace7f52..ea0754124aa 100644 --- a/components/external_storage/src/export.rs +++ b/components/external_storage/src/export.rs @@ -11,14 +11,14 @@ use encryption::DataKeyManager; use futures_util::{future::LocalBoxFuture, stream::LocalBoxStream}; use gcp::GcsStorage; use kvproto::brpb::{ - AzureBlobStorage, Gcs, Noop, StorageBackend, StorageBackend_oneof_backend as Backend, S3, + AzureBlobStorage, Gcs, Noop, S3, StorageBackend, StorageBackend_oneof_backend as Backend, }; use tikv_util::time::{Instant, Limiter}; use crate::{ - compression_reader_dispatcher, encrypt_wrap_reader, read_external_storage_into_file, - record_storage_create, wrap_with_checksum_reader_if_needed, BackendConfig, ExternalData, - ExternalStorage, HdfsStorage, LocalStorage, NoopStorage, RestoreConfig, UnpinReader, + BackendConfig, ExternalData, ExternalStorage, HdfsStorage, LocalStorage, NoopStorage, + RestoreConfig, UnpinReader, compression_reader_dispatcher, encrypt_wrap_reader, + read_external_storage_into_file, record_storage_create, wrap_with_checksum_reader_if_needed, }; pub fn create_storage( storage_backend: &StorageBackend, diff --git a/components/external_storage/src/hdfs.rs b/components/external_storage/src/hdfs.rs index 05934463091..b5b4840edda 100644 --- a/components/external_storage/src/hdfs.rs +++ b/components/external_storage/src/hdfs.rs @@ -37,7 +37,7 @@ pub struct HdfsStorage { impl HdfsStorage { pub fn new(remote: &str, config: HdfsConfig) -> io::Result { - let mut remote = Url::parse(remote).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + let mut remote = Url::parse(remote).map_err(|e| io::Error::other(e))?; if !remote.path().ends_with('/') { let mut new_path = remote.path().to_owned(); new_path.push('/'); @@ -88,17 +88,14 @@ impl ExternalStorage for HdfsStorage { _content_length: u64, ) -> io::Result<()> { if name.contains(path::MAIN_SEPARATOR) { - return Err(io::Error::new( - io::ErrorKind::Other, - format!("[{}] parent is not allowed in storage", name), - )); + return Err(io::Error::other(format!( + "[{}] parent is not allowed in storage", + name + ))); } let cmd_path = self.get_hdfs_bin().ok_or_else(|| { - io::Error::new( - io::ErrorKind::Other, - "Cannot found hdfs command, please specify HADOOP_HOME", - ) + io::Error::other("Cannot found hdfs command, please specify HADOOP_HOME") })?; let remote_url = self.remote.clone().join(name).unwrap(); let path = try_convert_to_path(&remote_url); @@ -134,10 +131,10 @@ impl ExternalStorage for HdfsStorage { "stdout" => stdout.as_ref(), "stderr" => stderr.as_ref(), ); - Err(io::Error::new( - io::ErrorKind::Other, - format!("hdfs returned non-zero status: {:?}", output.status.code()), - )) + Err(io::Error::other(format!( + "hdfs returned non-zero status: {:?}", + output.status.code() + ))) } } diff --git a/components/external_storage/src/lib.rs b/components/external_storage/src/lib.rs index c9b3aafe586..eb19625eaaa 100644 --- a/components/external_storage/src/lib.rs +++ b/components/external_storage/src/lib.rs @@ -23,7 +23,7 @@ use encryption::{DecrypterReader, FileEncryptionInfo, Iv}; use file_system::{File, Sha256Reader}; use futures::io::BufReader; use futures_io::AsyncRead; -use futures_util::{future::LocalBoxFuture, stream::LocalBoxStream, AsyncReadExt}; +use futures_util::{AsyncReadExt, future::LocalBoxFuture, stream::LocalBoxStream}; use kvproto::brpb::CompressionType; use openssl::hash::{Hasher, MessageDigest}; use tikv_util::{ @@ -94,13 +94,10 @@ pub fn compression_reader_dispatcher( // So here regard Unkown(0) as uncompressed type. CompressionType::Unknown => Ok(inner), CompressionType::Zstd => Ok(Box::new(ZstdDecoder::new(BufReader::new(inner)))), - _ => Err(io::Error::new( - io::ErrorKind::Other, - format!( - "the compression type is unimplemented, compression type id {:?}", - c - ), - )), + _ => Err(io::Error::other(format!( + "the compression type is unimplemented, compression type id {:?}", + c + ))), }, None => Ok(inner), } @@ -492,30 +489,23 @@ pub async fn read_external_storage_info_buff( } fn build_hasher() -> Result { - Hasher::new(MessageDigest::sha256()).map_err(|err| { - io::Error::new( - io::ErrorKind::Other, - format!("openssl hasher failed to init: {}", err), - ) - }) + Hasher::new(MessageDigest::sha256()) + .map_err(|err| io::Error::other(format!("openssl hasher failed to init: {}", err))) } fn update_hasher(hasher: &mut Hasher, data: &[u8]) -> Result<(), io::Error> { - hasher.update(data).map_err(|err| { - io::Error::new( - io::ErrorKind::Other, - format!("openssl hasher update failed: {}", err), - ) - }) + hasher + .update(data) + .map_err(|err| io::Error::other(format!("openssl hasher update failed: {}", err))) } fn finish_hasher(mut hasher: Hasher) -> Result, io::Error> { hasher.finish().map_or_else( |err| { - Err(io::Error::new( - io::ErrorKind::Other, - format!("openssl hasher finish failed: {}", err), - )) + Err(io::Error::other(format!( + "openssl hasher finish failed: {}", + err + ))) }, |bytes| Ok(bytes.to_vec()), ) @@ -529,10 +519,10 @@ fn calc_and_compare_checksums( if let Some(hasher) = opt_encrypted_file_hasher { let calc_checksum = hasher.lock().unwrap().finish().map_or_else( |err| { - Err(io::Error::new( - io::ErrorKind::Other, - format!("openssl hasher finish failed: {}", err), - )) + Err(io::Error::other(format!( + "openssl hasher finish failed: {}", + err + ))) }, |bytes| Ok(bytes.to_vec()), )?; diff --git a/components/external_storage/src/local.rs b/components/external_storage/src/local.rs index cf974df5dd2..d6af0fa8a53 100644 --- a/components/external_storage/src/local.rs +++ b/components/external_storage/src/local.rs @@ -219,8 +219,7 @@ impl ExternalStorage for LocalStorage { .path() .strip_prefix(&self.base); match rel { - Err(_) => futures::future::err(io::Error::new( - io::ErrorKind::Other, + Err(_) => futures::future::err(io::Error::other( format!("unknown: we found something not match the prefix... it is {}, our prefix is {}", v.path().display(), self.base.display()), )), Ok(item) => futures::future::ok(BlobObject{ diff --git a/components/external_storage/src/locking.rs b/components/external_storage/src/locking.rs index b44c974c3ec..ed85271e995 100644 --- a/components/external_storage/src/locking.rs +++ b/components/external_storage/src/locking.rs @@ -22,13 +22,13 @@ use std::io; use chrono::Utc; use futures_util::{ - future::{ok, FutureExt, LocalBoxFuture}, + future::{FutureExt, LocalBoxFuture, ok}, io::AsyncReadExt, stream::TryStreamExt, }; use tikv_util::sys::{ hostname, - thread::{process_id, Pid}, + thread::{Pid, process_id}, }; use uuid::Uuid; diff --git a/components/file_system/src/file.rs b/components/file_system/src/file.rs index 76d662464f8..a49efc58d00 100644 --- a/components/file_system/src/file.rs +++ b/components/file_system/src/file.rs @@ -13,7 +13,7 @@ use std::{ // Extention Traits use fs2::FileExt; -use super::{get_io_rate_limiter, get_io_type, IoOp, IoRateLimiter}; +use super::{IoOp, IoRateLimiter, get_io_rate_limiter, get_io_type}; /// A wrapper around `std::fs::File` with capability to track and regulate IO /// flow. @@ -169,7 +169,7 @@ impl File { } pub fn lock_shared(&self) -> io::Result<()> { - self.inner.lock_shared() + fs2::FileExt::lock_shared(&self.inner) } pub fn lock_exclusive(&self) -> io::Result<()> { @@ -177,7 +177,7 @@ impl File { } pub fn try_lock_shared(&self) -> io::Result<()> { - self.inner.try_lock_shared() + fs2::FileExt::try_lock_shared(&self.inner) } pub fn try_lock_exclusive(&self) -> io::Result<()> { @@ -185,7 +185,7 @@ impl File { } pub fn unlock(&self) -> io::Result<()> { - self.inner.unlock() + fs2::FileExt::unlock(&self.inner) } } diff --git a/components/file_system/src/io_stats/biosnoop.rs b/components/file_system/src/io_stats/biosnoop.rs index 2267193a3ec..ae3ceec8d74 100644 --- a/components/file_system/src/io_stats/biosnoop.rs +++ b/components/file_system/src/io_stats/biosnoop.rs @@ -4,17 +4,17 @@ use std::{ collections::VecDeque, ptr, sync::{ - atomic::{AtomicUsize, Ordering}, Mutex, + atomic::{AtomicUsize, Ordering}, }, }; -use bcc::{table::Table, Kprobe, BPF}; +use bcc::{BPF, Kprobe, table::Table}; use crossbeam_utils::CachePadded; use strum::{EnumCount, IntoEnumIterator}; use tikv_util::sys::thread; -use crate::{metrics::*, IoBytes, IoType}; +use crate::{IoBytes, IoType, metrics::*}; /// Biosnoop leverages BCC to make use of eBPF to get disk IO of TiKV requests. /// The BCC code is in `biosnoop.c` which is compiled and attached kernel on @@ -288,10 +288,10 @@ mod tests { use test::Bencher; use super::{ - fetch_io_bytes, flush_io_latency_metrics, get_io_type, init, set_io_type, BPF_CONTEXT, - MAX_THREAD_IDX, + BPF_CONTEXT, MAX_THREAD_IDX, fetch_io_bytes, flush_io_latency_metrics, get_io_type, init, + set_io_type, }; - use crate::{io_stats::A512, metrics::*, IoType, OpenOptions}; + use crate::{IoType, OpenOptions, io_stats::A512, metrics::*}; #[test] fn test_biosnoop() { diff --git a/components/file_system/src/io_stats/proc.rs b/components/file_system/src/io_stats/proc.rs index 087672d4fc2..9101e394db9 100644 --- a/components/file_system/src/io_stats/proc.rs +++ b/components/file_system/src/io_stats/proc.rs @@ -200,7 +200,7 @@ mod tests { use tikv_util::sys::thread::StdThreadBuildWrapper; use super::*; - use crate::{io_stats::A512, OpenOptions, WithIoType}; + use crate::{OpenOptions, WithIoType, io_stats::A512}; #[test] fn test_read_bytes() { diff --git a/components/file_system/src/lib.rs b/components/file_system/src/lib.rs index 01bc0105ddf..30d08ef0e70 100644 --- a/components/file_system/src/lib.rs +++ b/components/file_system/src/lib.rs @@ -1,7 +1,6 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. #![feature(test)] -#![feature(duration_consts_float)] #[macro_use] extern crate lazy_static; @@ -15,9 +14,9 @@ use std::cell::Cell; pub use std::{ convert::TryFrom, fs::{ - canonicalize, create_dir, create_dir_all, hard_link, metadata, read_dir, read_link, - remove_dir, remove_dir_all, remove_file, rename, set_permissions, symlink_metadata, - DirBuilder, DirEntry, FileType, Metadata, Permissions, ReadDir, + DirBuilder, DirEntry, FileType, Metadata, Permissions, ReadDir, canonicalize, create_dir, + create_dir_all, hard_link, metadata, read_dir, read_link, remove_dir, remove_dir_all, + remove_file, rename, set_permissions, symlink_metadata, }, }; use std::{ @@ -26,7 +25,7 @@ use std::{ pin::Pin, str::FromStr, sync::{Arc, Mutex}, - task::{ready, Context, Poll}, + task::{Context, Poll, ready}, }; pub use file::{File, OpenOptions}; @@ -41,8 +40,8 @@ use openssl::{ hash::{self, Hasher, MessageDigest}, }; pub use rate_limiter::{ - get_io_rate_limiter, set_io_rate_limiter, IoBudgetAdjustor, IoRateLimitMode, IoRateLimiter, - IoRateLimiterStatistics, + IoBudgetAdjustor, IoRateLimitMode, IoRateLimiter, IoRateLimiterStatistics, get_io_rate_limiter, + set_io_rate_limiter, }; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use strum::{EnumCount, EnumIter}; @@ -583,7 +582,7 @@ impl AsyncRead for Sha256Reader { .lock() .expect("failed to lock hasher in Sha256Reader async read"); if let Err(e) = hasher.update(new_data) { - return Poll::Ready(Err(io::Error::new(ErrorKind::Other, e))); + return Poll::Ready(Err(io::Error::other(e))); } Poll::Ready(Ok(())) @@ -622,7 +621,7 @@ pub fn reserve_space_for_recover>(data_dir: P, file_size: u64) -> mod tests { use std::{io::Write, iter}; - use rand::{distributions::Alphanumeric, thread_rng, Rng}; + use rand::{Rng, distributions::Alphanumeric, thread_rng}; use tempfile::{Builder, TempDir}; use super::*; diff --git a/components/file_system/src/metrics_manager.rs b/components/file_system/src/metrics_manager.rs index 89e822b24e7..d67d3b665e8 100644 --- a/components/file_system/src/metrics_manager.rs +++ b/components/file_system/src/metrics_manager.rs @@ -6,9 +6,9 @@ use strum::{EnumCount, IntoEnumIterator}; use tikv_util::time::Instant; use crate::{ - io_stats::fetch_io_bytes, - metrics::{tls_flush, IO_BYTES_VEC}, IoBytes, IoOp, IoRateLimiterStatistics, IoType, + io_stats::fetch_io_bytes, + metrics::{IO_BYTES_VEC, tls_flush}, }; pub enum BytesFetcher { diff --git a/components/file_system/src/rate_limiter.rs b/components/file_system/src/rate_limiter.rs index 79c7094b186..5b7fbfdb9de 100644 --- a/components/file_system/src/rate_limiter.rs +++ b/components/file_system/src/rate_limiter.rs @@ -3,8 +3,8 @@ use std::{ str::FromStr, sync::{ - atomic::{AtomicU32, AtomicUsize, Ordering}, Arc, + atomic::{AtomicU32, AtomicUsize, Ordering}, }, time::Duration, }; @@ -16,8 +16,8 @@ use strum::EnumCount; use tikv_util::time::Instant; use super::{ - metrics::{tls_collect_rate_limiter_request_wait, RATE_LIMITER_MAX_BYTES_PER_SEC}, IoOp, IoPriority, IoType, + metrics::{RATE_LIMITER_MAX_BYTES_PER_SEC, tls_collect_rate_limiter_request_wait}, }; const DEFAULT_REFILL_PERIOD: Duration = Duration::from_millis(50); @@ -263,8 +263,7 @@ macro_rules! request_imp { // Bytes served by next epoch (and skipped epochs) during refill are subtracted // from pending_bytes, round up the rest. DEFAULT_REFILL_PERIOD - * ((locked.pending_bytes[priority_idx] + cached_bytes_per_epoch - 1) - / cached_bytes_per_epoch) as u32 + * locked.pending_bytes[priority_idx].div_ceil(cached_bytes_per_epoch) as u32 } else { // `(a-1)/b` is equivalent to `roundup(a.saturating_sub(b)/b)`. locked.next_refill_time - now diff --git a/components/health_controller/src/lib.rs b/components/health_controller/src/lib.rs index 75427cd8e7c..ed940938da0 100644 --- a/components/health_controller/src/lib.rs +++ b/components/health_controller/src/lib.rs @@ -30,8 +30,6 @@ //! that are specific to different modules, increasing the complexity and //! possibility to misuse of `HealthController`. -#![feature(div_duration)] - pub mod reporters; pub mod slow_score; pub mod trend; @@ -41,8 +39,8 @@ use std::{ collections::HashSet, ops::Deref, sync::{ - atomic::{AtomicU64, AtomicUsize, Ordering}, Arc, + atomic::{AtomicU64, AtomicUsize, Ordering}, }, }; @@ -322,7 +320,7 @@ impl RollingRetriever { #[cfg(test)] mod tests { use std::{ - sync::mpsc::{sync_channel, RecvTimeoutError}, + sync::mpsc::{RecvTimeoutError, sync_channel}, time::Duration, }; diff --git a/components/health_controller/src/reporters.rs b/components/health_controller/src/reporters.rs index 56624c37d64..f14810e4644 100644 --- a/components/health_controller/src/reporters.rs +++ b/components/health_controller/src/reporters.rs @@ -10,10 +10,10 @@ use pdpb::SlowTrend as SlowTrendPb; use prometheus::IntGauge; use crate::{ + HealthController, HealthControllerInner, RaftstoreDuration, slow_score::{SlowScore, SlowScoreTickResult}, trend::{RequestPerSecRecorder, Trend}, types::InspectFactor, - HealthController, HealthControllerInner, RaftstoreDuration, }; /// The parameters for building a [`RaftstoreReporter`]. diff --git a/components/health_controller/src/types.rs b/components/health_controller/src/types.rs index 7342273e972..6fa775b7444 100644 --- a/components/health_controller/src/types.rs +++ b/components/health_controller/src/types.rs @@ -1,6 +1,6 @@ // Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0. -use std::{fmt::Debug, u64}; +use std::fmt::Debug; /// Represent the duration of all stages of raftstore recorded by one /// inspecting. diff --git a/components/hybrid_engine/src/db_vector.rs b/components/hybrid_engine/src/db_vector.rs index 1b286e182e7..5251c204913 100644 --- a/components/hybrid_engine/src/db_vector.rs +++ b/components/hybrid_engine/src/db_vector.rs @@ -80,7 +80,7 @@ where } } -impl<'a, EK, EC> PartialEq<&'a [u8]> for HybridDbVector +impl PartialEq<&[u8]> for HybridDbVector where EK: KvEngine, EC: RegionCacheEngine, diff --git a/components/hybrid_engine/src/engine.rs b/components/hybrid_engine/src/engine.rs index 0fb70eed7ac..13a94092ac8 100644 --- a/components/hybrid_engine/src/engine.rs +++ b/components/hybrid_engine/src/engine.rs @@ -76,10 +76,10 @@ mod tests { use std::sync::Arc; use engine_rocks::util::new_engine; - use engine_traits::{CacheRegion, CF_DEFAULT, CF_LOCK, CF_WRITE}; + use engine_traits::{CF_DEFAULT, CF_LOCK, CF_WRITE, CacheRegion}; use in_memory_engine::{ - config::InMemoryEngineConfigManager, test_util::new_region, InMemoryEngineConfig, - InMemoryEngineContext, RegionCacheMemoryEngine, + InMemoryEngineConfig, InMemoryEngineContext, RegionCacheMemoryEngine, + config::InMemoryEngineConfigManager, test_util::new_region, }; use online_config::{ConfigChange, ConfigManager, ConfigValue}; use tempfile::Builder; diff --git a/components/hybrid_engine/src/metrics.rs b/components/hybrid_engine/src/metrics.rs index 76329179375..11ef247c293 100644 --- a/components/hybrid_engine/src/metrics.rs +++ b/components/hybrid_engine/src/metrics.rs @@ -1,7 +1,7 @@ // Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0. use lazy_static::lazy_static; -use prometheus::{register_int_counter_vec, IntCounterVec}; +use prometheus::{IntCounterVec, register_int_counter_vec}; use prometheus_static_metric::{auto_flush_from, make_auto_flush_static_metric}; make_auto_flush_static_metric! { diff --git a/components/hybrid_engine/src/observer/load_eviction.rs b/components/hybrid_engine/src/observer/load_eviction.rs index cba8029e31a..16b4d5e6e8a 100644 --- a/components/hybrid_engine/src/observer/load_eviction.rs +++ b/components/hybrid_engine/src/observer/load_eviction.rs @@ -13,11 +13,11 @@ use protobuf::ProtobufEnum as _; use raft::StateRole; use raftstore::{ coprocessor::{ - dispatcher::{BoxDestroyPeerObserver, BoxTransferLeaderObserver}, AdminObserver, ApplyCtxInfo, ApplySnapshotObserver, BoxAdminObserver, BoxApplySnapshotObserver, BoxQueryObserver, BoxRoleObserver, Cmd, Coprocessor, CoprocessorHost, DestroyPeerObserver, ObserverContext, QueryObserver, RegionState, RoleObserver, TransferLeaderCustomContext, TransferLeaderObserver, + dispatcher::{BoxDestroyPeerObserver, BoxTransferLeaderObserver}, }, store::TransferLeaderContext, }; diff --git a/components/hybrid_engine/src/observer/snapshot.rs b/components/hybrid_engine/src/observer/snapshot.rs index edf8d9729f9..6c344fd6433 100644 --- a/components/hybrid_engine/src/observer/snapshot.rs +++ b/components/hybrid_engine/src/observer/snapshot.rs @@ -6,7 +6,7 @@ use engine_traits::{CacheRegion, FailedReason, KvEngine, RegionCacheEngine}; use in_memory_engine::{RegionCacheMemoryEngine, RegionCacheSnapshot}; use kvproto::metapb::Region; use raftstore::coprocessor::{ - dispatcher::BoxSnapshotObserver, CoprocessorHost, ObservedSnapshot, SnapshotObserver, + CoprocessorHost, ObservedSnapshot, SnapshotObserver, dispatcher::BoxSnapshotObserver, }; use crate::metrics::{ diff --git a/components/hybrid_engine/src/observer/test_write_batch.rs b/components/hybrid_engine/src/observer/test_write_batch.rs index faa1cbf30d1..9c3dc233400 100644 --- a/components/hybrid_engine/src/observer/test_write_batch.rs +++ b/components/hybrid_engine/src/observer/test_write_batch.rs @@ -5,8 +5,8 @@ use std::{sync::mpsc::sync_channel, time::Duration}; use crossbeam::epoch; use engine_traits::{CacheRegion, Mutable, Peekable, RegionCacheEngine, WriteBatch, WriteBatchExt}; use in_memory_engine::{ - decode_key, test_util::new_region, InMemoryEngineConfig, InternalKey, RegionCacheStatus, - ValueType, + InMemoryEngineConfig, InternalKey, RegionCacheStatus, ValueType, decode_key, + test_util::new_region, }; use raftstore::coprocessor::{WriteBatchObserver, WriteBatchWrapper}; diff --git a/components/hybrid_engine/src/observer/write_batch.rs b/components/hybrid_engine/src/observer/write_batch.rs index 854dc6a2eb7..7d9271f365d 100644 --- a/components/hybrid_engine/src/observer/write_batch.rs +++ b/components/hybrid_engine/src/observer/write_batch.rs @@ -1,11 +1,11 @@ // Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0. -use engine_traits::{is_data_cf, KvEngine, Mutable, Result, WriteBatch, WriteOptions}; +use engine_traits::{KvEngine, Mutable, Result, WriteBatch, WriteOptions, is_data_cf}; use in_memory_engine::{RegionCacheMemoryEngine, RegionCacheWriteBatch}; use kvproto::metapb; use raftstore::coprocessor::{ - dispatcher::BoxWriteBatchObserver, Coprocessor, CoprocessorHost, ObservableWriteBatch, - WriteBatchObserver, + Coprocessor, CoprocessorHost, ObservableWriteBatch, WriteBatchObserver, + dispatcher::BoxWriteBatchObserver, }; #[derive(Clone)] diff --git a/components/hybrid_engine/src/snapshot.rs b/components/hybrid_engine/src/snapshot.rs index 01587de3fa9..7b72489be7d 100644 --- a/components/hybrid_engine/src/snapshot.rs +++ b/components/hybrid_engine/src/snapshot.rs @@ -6,8 +6,8 @@ use std::{ }; use engine_traits::{ - is_data_cf, CfNamesExt, IterOptions, Iterable, KvEngine, Peekable, ReadOptions, - RegionCacheEngine, Result, Snapshot, SnapshotMiscExt, CF_DEFAULT, + CF_DEFAULT, CfNamesExt, IterOptions, Iterable, KvEngine, Peekable, ReadOptions, + RegionCacheEngine, Result, Snapshot, SnapshotMiscExt, is_data_cf, }; use in_memory_engine::RegionCacheMemoryEngine; use raftstore::coprocessor::ObservedSnapshot; @@ -161,10 +161,10 @@ where #[cfg(test)] mod tests { use engine_traits::{ - CacheRegion, IterOptions, Iterable, Iterator, Mutable, WriteBatch, WriteBatchExt, - CF_DEFAULT, + CF_DEFAULT, CacheRegion, IterOptions, Iterable, Iterator, Mutable, WriteBatch, + WriteBatchExt, }; - use in_memory_engine::{test_util::new_region, InMemoryEngineConfig, RegionCacheStatus}; + use in_memory_engine::{InMemoryEngineConfig, RegionCacheStatus, test_util::new_region}; use raftstore::coprocessor::WriteBatchWrapper; use crate::{ diff --git a/components/hybrid_engine/src/util.rs b/components/hybrid_engine/src/util.rs index 2a5828a7658..35afac31244 100644 --- a/components/hybrid_engine/src/util.rs +++ b/components/hybrid_engine/src/util.rs @@ -2,8 +2,8 @@ use std::sync::Arc; -use engine_rocks::{util::new_engine, RocksEngine}; -use engine_traits::{RegionCacheEngine, Result, CF_DEFAULT, CF_LOCK, CF_WRITE}; +use engine_rocks::{RocksEngine, util::new_engine}; +use engine_traits::{CF_DEFAULT, CF_LOCK, CF_WRITE, RegionCacheEngine, Result}; use in_memory_engine::{InMemoryEngineConfig, InMemoryEngineContext, RegionCacheMemoryEngine}; use tempfile::{Builder, TempDir}; use tikv_util::config::VersionTrack; @@ -17,7 +17,7 @@ use crate::HybridEngine; /// /// ``` /// use hybrid_engine::util::hybrid_engine_for_tests; -/// use in_memory_engine::{test_util::new_region, InMemoryEngineConfig}; +/// use in_memory_engine::{InMemoryEngineConfig, test_util::new_region}; /// let mut config = InMemoryEngineConfig::default(); /// config.enable = true; /// let (_path, _hybrid_engine) = hybrid_engine_for_tests("temp", config, |memory_engine| { diff --git a/components/in_memory_engine/benches/load_region.rs b/components/in_memory_engine/benches/load_region.rs index 77315ffc3cb..f5a307c3d8e 100644 --- a/components/in_memory_engine/benches/load_region.rs +++ b/components/in_memory_engine/benches/load_region.rs @@ -5,17 +5,17 @@ use std::sync::Arc; use criterion::*; -use engine_rocks::{util::new_engine, RocksEngine}; +use engine_rocks::{RocksEngine, util::new_engine}; use engine_traits::{ - CacheRegion, KvEngine, Mutable, RegionCacheEngine, WriteBatch, WriteBatchExt, CF_DEFAULT, - CF_WRITE, DATA_CFS, + CF_DEFAULT, CF_WRITE, CacheRegion, DATA_CFS, KvEngine, Mutable, RegionCacheEngine, WriteBatch, + WriteBatchExt, }; use futures::future::ready; use in_memory_engine::{BackgroundRunner, *}; use keys::{DATA_MAX_KEY, DATA_MIN_KEY}; use pd_client::PdClient; use raftstore::coprocessor::config::SPLIT_SIZE; -use rand::{thread_rng, RngCore}; +use rand::{RngCore, thread_rng}; use tikv_util::config::VersionTrack; use txn_types::{Key, TimeStamp, Write, WriteType}; diff --git a/components/in_memory_engine/src/background.rs b/components/in_memory_engine/src/background.rs index 4a30c6ca4e2..74a1dd3d78f 100644 --- a/components/in_memory_engine/src/background.rs +++ b/components/in_memory_engine/src/background.rs @@ -4,13 +4,13 @@ use std::{borrow::Cow, fmt, sync::Arc, time::Duration}; use bytes::Bytes; use crossbeam::{ - channel::{bounded, tick, Sender}, + channel::{Sender, bounded, tick}, epoch, select, }; use engine_rocks::{RocksEngine, RocksSnapshot}; use engine_traits::{ - CacheRegion, EvictReason, IterOptions, Iterable, Iterator, MiscExt, OnEvictFinishedCallback, - RangeHintService, SnapshotMiscExt, CF_DEFAULT, CF_WRITE, DATA_CFS, + CF_DEFAULT, CF_WRITE, CacheRegion, DATA_CFS, EvictReason, IterOptions, Iterable, Iterator, + MiscExt, OnEvictFinishedCallback, RangeHintService, SnapshotMiscExt, }; use fail::fail_point; use keys::{origin_end_key, origin_key}; @@ -33,11 +33,12 @@ use txn_types::{Key, TimeStamp, WriteRef, WriteType}; use yatp::Remote; use crate::{ + InMemoryEngineConfig, RegionCacheMemoryEngine, cross_check::CrossChecker, engine::{RegionCacheMemoryEngineCore, SkiplistHandle}, keys::{ - decode_key, encode_key, encode_key_for_boundary_with_mvcc, encoding_for_filter, - InternalBytes, InternalKey, ValueType, + InternalBytes, InternalKey, ValueType, decode_key, encode_key, + encode_key_for_boundary_with_mvcc, encoding_for_filter, }, memory_controller::{MemoryController, MemoryUsage}, metrics::{ @@ -50,9 +51,8 @@ use crate::{ LabelRule, RegionLabelChangedCallback, RegionLabelRulesManager, RegionLabelServiceBuilder, }, region_manager::{CacheRegionMeta, RegionState}, - region_stats::{RegionStatsManager, DEFAULT_EVICT_MIN_DURATION}, + region_stats::{DEFAULT_EVICT_MIN_DURATION, RegionStatsManager}, write_batch::RegionCacheWriteBatchEntry, - InMemoryEngineConfig, RegionCacheMemoryEngine, }; // 5 seconds should be long enough for getting a TSO from PD. @@ -1077,9 +1077,7 @@ impl Runnable for BackgroundRunner { Some(t.unwrap().parse::().unwrap()) }); - let Some(ref rocks_engine) = self.rocks_engine else { - return None; - }; + let rocks_engine = self.rocks_engine.as_ref()?; let latest_seqno = rocks_engine.get_latest_sequence_number(); Some( rocks_engine @@ -1727,8 +1725,8 @@ impl Filter { pub mod tests { use std::{ sync::{ - mpsc::{channel, Sender}, Arc, Mutex, + mpsc::{Sender, channel}, }, time::Duration, }; @@ -1736,11 +1734,11 @@ pub mod tests { use crossbeam::epoch; use engine_rocks::util::new_engine; use engine_traits::{ - CacheRegion, IterOptions, Iterable, Iterator, RegionCacheEngine, RegionCacheEngineExt, - RegionEvent, SyncMutable, CF_DEFAULT, CF_LOCK, CF_WRITE, DATA_CFS, + CF_DEFAULT, CF_LOCK, CF_WRITE, CacheRegion, DATA_CFS, IterOptions, Iterable, Iterator, + RegionCacheEngine, RegionCacheEngineExt, RegionEvent, SyncMutable, }; use futures::future::ready; - use keys::{data_key, DATA_MAX_KEY, DATA_MIN_KEY}; + use keys::{DATA_MAX_KEY, DATA_MIN_KEY, data_key}; use kvproto::metapb::Region; use online_config::{ConfigChange, ConfigManager, ConfigValue}; use pd_client::PdClient; @@ -1753,12 +1751,13 @@ pub mod tests { use super::*; use crate::{ + InMemoryEngineConfig, InMemoryEngineContext, RegionCacheMemoryEngine, background::BackgroundRunner, config::InMemoryEngineConfigManager, engine::{SkiplistEngine, SkiplistHandle}, keys::{ - construct_key, construct_region_key, construct_value, encode_key, encode_seek_key, - encoding_for_filter, InternalBytes, ValueType, + InternalBytes, ValueType, construct_key, construct_region_key, construct_value, + encode_key, encode_seek_key, encoding_for_filter, }, memory_controller::MemoryController, region_label::{ @@ -1768,7 +1767,6 @@ pub mod tests { region_manager::RegionState::*, test_util::{new_region, put_data, put_data_with_overwrite}, write_batch::RegionCacheWriteBatchEntry, - InMemoryEngineConfig, InMemoryEngineContext, RegionCacheMemoryEngine, }; fn delete_data( diff --git a/components/in_memory_engine/src/config.rs b/components/in_memory_engine/src/config.rs index 904143755b6..d9ecd53aeda 100644 --- a/components/in_memory_engine/src/config.rs +++ b/components/in_memory_engine/src/config.rs @@ -248,7 +248,7 @@ mod tests { use super::*; const DEFAULT_REGION_SPLIT_SIZE: ReadableSize = ReadableSize::mb(256); - const SMALL_ENOUGH_BLOCK_CACHE_CAPACITY: u64 = (MIN_CAPACITY + 1) / 2; + const SMALL_ENOUGH_BLOCK_CACHE_CAPACITY: u64 = MIN_CAPACITY.div_ceil(2); const LARGE_ENOUGH_BLOCK_CACHE_CAPACITY: u64 = (MIN_CAPACITY + 1) * (1.0 / DEFAULT_CAPACITY_FROM_BLOCK_CACHE_RATIO) as u64; diff --git a/components/in_memory_engine/src/cross_check.rs b/components/in_memory_engine/src/cross_check.rs index 36165af1a77..f503de2a4f6 100644 --- a/components/in_memory_engine/src/cross_check.rs +++ b/components/in_memory_engine/src/cross_check.rs @@ -4,8 +4,8 @@ use std::{fmt::Display, sync::Arc, time::Duration}; use engine_rocks::{RocksEngine, RocksEngineIterator, RocksSnapshot}; use engine_traits::{ - iter_option, CacheRegion, Iterable, Iterator, KvEngine, Peekable, RegionCacheEngine, - SnapshotMiscExt, CF_LOCK, CF_WRITE, + CF_LOCK, CF_WRITE, CacheRegion, Iterable, Iterator, KvEngine, Peekable, RegionCacheEngine, + SnapshotMiscExt, iter_option, }; use pd_client::PdClient; use slog_global::{error, info, warn}; @@ -17,9 +17,9 @@ use tikv_util::{ use txn_types::{Key, TimeStamp, WriteRef, WriteType}; use crate::{ + RegionCacheMemoryEngine, RegionState, background::{parse_write, split_ts}, read::{RegionCacheIterator, RegionCacheSnapshot}, - RegionCacheMemoryEngine, RegionState, }; // Cross check stops for some reason. @@ -945,18 +945,18 @@ impl std::fmt::Debug for KeyCheckingInfo { mod tests { use std::{sync::Arc, time::Duration}; - use engine_rocks::{util::new_engine_opt, RocksDbOptions, RocksWriteBatchVec}; + use engine_rocks::{RocksDbOptions, RocksWriteBatchVec, util::new_engine_opt}; use engine_traits::{ - CacheRegion, KvEngine, Mutable, RegionCacheEngine, WriteBatch, WriteBatchExt, CF_DEFAULT, - CF_LOCK, CF_WRITE, + CF_DEFAULT, CF_LOCK, CF_WRITE, CacheRegion, KvEngine, Mutable, RegionCacheEngine, + WriteBatch, WriteBatchExt, }; use futures::future::ready; use keys::data_key; use kvproto::metapb::{Region, RegionEpoch}; use pd_client::PdClient; use raftstore::{ - coprocessor::{RegionInfoCallback, RegionInfoProvider}, RegionInfo, SeekRegionCallback, + coprocessor::{RegionInfoCallback, RegionInfoProvider}, }; use tempfile::Builder; use tikv_util::{config::VersionTrack, store::new_peer}; @@ -964,8 +964,8 @@ mod tests { use super::Result; use crate::{ - cross_check::CrossChecker, InMemoryEngineConfig, InMemoryEngineContext, - RegionCacheMemoryEngine, RegionCacheWriteBatch, + InMemoryEngineConfig, InMemoryEngineContext, RegionCacheMemoryEngine, + RegionCacheWriteBatch, cross_check::CrossChecker, }; #[derive(Clone)] diff --git a/components/in_memory_engine/src/engine.rs b/components/in_memory_engine/src/engine.rs index 81925dc6ada..70ad4b37e4b 100644 --- a/components/in_memory_engine/src/engine.rs +++ b/components/in_memory_engine/src/engine.rs @@ -4,18 +4,18 @@ use std::{ fmt::{self, Debug}, ops::Bound, result, - sync::{atomic::AtomicU64, Arc}, + sync::{Arc, atomic::AtomicU64}, }; -use crossbeam::epoch::{self, default_collector, Guard}; +use crossbeam::epoch::{self, Guard, default_collector}; use crossbeam_skiplist::{ - base::{Entry, OwnedIter}, SkipList, + base::{Entry, OwnedIter}, }; use engine_rocks::RocksEngine; use engine_traits::{ - CacheRegion, EvictReason, FailedReason, KvEngine, OnEvictFinishedCallback, RegionCacheEngine, - RegionCacheEngineExt, RegionEvent, CF_DEFAULT, CF_LOCK, CF_WRITE, DATA_CFS, + CF_DEFAULT, CF_LOCK, CF_WRITE, CacheRegion, DATA_CFS, EvictReason, FailedReason, KvEngine, + OnEvictFinishedCallback, RegionCacheEngine, RegionCacheEngineExt, RegionEvent, }; use fail::fail_point; use kvproto::metapb::Region; @@ -25,15 +25,15 @@ use slog_global::error; use tikv_util::{config::VersionTrack, info, warn, worker::Scheduler}; use crate::{ + InMemoryEngineConfig, InMemoryEngineContext, background::{BackgroundTask, BgWorkManager, PdRangeHintService}, keys::{ - encode_key_for_boundary_with_mvcc, encode_key_for_boundary_without_mvcc, InternalBytes, + InternalBytes, encode_key_for_boundary_with_mvcc, encode_key_for_boundary_without_mvcc, }, memory_controller::MemoryController, read::RegionCacheSnapshot, region_manager::{LoadFailedReason, RegionCacheStatus, RegionManager, RegionState}, statistics::Statistics, - InMemoryEngineConfig, InMemoryEngineContext, }; pub(crate) const CF_DEFAULT_USIZE: usize = 0; @@ -638,24 +638,24 @@ pub mod tests { use crossbeam::epoch; use engine_rocks::util::new_engine; use engine_traits::{ - CacheRegion, EvictReason, Mutable, RegionCacheEngine, RegionCacheEngineExt, RegionEvent, - WriteBatch, WriteBatchExt, CF_DEFAULT, CF_LOCK, CF_WRITE, DATA_CFS, + CF_DEFAULT, CF_LOCK, CF_WRITE, CacheRegion, DATA_CFS, EvictReason, Mutable, + RegionCacheEngine, RegionCacheEngineExt, RegionEvent, WriteBatch, WriteBatchExt, }; use tikv_util::config::{ReadableDuration, VersionTrack}; use tokio::{ runtime::Builder, - sync::{mpsc, Mutex}, + sync::{Mutex, mpsc}, time::timeout, }; use super::SkiplistEngine; use crate::{ + InMemoryEngineConfig, InMemoryEngineContext, InternalBytes, RegionCacheMemoryEngine, + ValueType, keys::{construct_key, construct_user_key, encode_key}, memory_controller::MemoryController, region_manager::{CacheRegionMeta, RegionManager, RegionState::*}, test_util::new_region, - InMemoryEngineConfig, InMemoryEngineContext, InternalBytes, RegionCacheMemoryEngine, - ValueType, }; fn count_region(mgr: &RegionManager, mut f: impl FnMut(&CacheRegionMeta) -> bool) -> usize { diff --git a/components/in_memory_engine/src/keys.rs b/components/in_memory_engine/src/keys.rs index 90845af7f54..afd528be84d 100644 --- a/components/in_memory_engine/src/keys.rs +++ b/components/in_memory_engine/src/keys.rs @@ -305,7 +305,7 @@ mod tests { use bytes::BufMut; use super::*; - use crate::keys::{encode_key, ValueType}; + use crate::keys::{ValueType, encode_key}; fn construct_key(i: u64, mvcc: u64) -> Vec { let k = format!("zk{:08}", i); diff --git a/components/in_memory_engine/src/lib.rs b/components/in_memory_engine/src/lib.rs index e4011f20235..ccdd858d0c7 100644 --- a/components/in_memory_engine/src/lib.rs +++ b/components/in_memory_engine/src/lib.rs @@ -36,8 +36,8 @@ pub use background::{BackgroundRunner, BackgroundTask, GcTask}; pub use config::InMemoryEngineConfig; pub use engine::{RegionCacheMemoryEngine, SkiplistHandle}; pub use keys::{ - decode_key, encode_key_for_boundary_without_mvcc, encoding_for_filter, InternalBytes, - InternalKey, ValueType, + InternalBytes, InternalKey, ValueType, decode_key, encode_key_for_boundary_without_mvcc, + encoding_for_filter, }; pub use metrics::flush_in_memory_engine_statistics; pub use read::RegionCacheSnapshot; diff --git a/components/in_memory_engine/src/memory_controller.rs b/components/in_memory_engine/src/memory_controller.rs index da6ffe64071..5e0320d1523 100644 --- a/components/in_memory_engine/src/memory_controller.rs +++ b/components/in_memory_engine/src/memory_controller.rs @@ -3,15 +3,15 @@ use std::{ fmt, sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering}, Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, }, }; use tikv_util::config::VersionTrack; use crate::{ - engine::SkiplistEngine, write_batch::NODE_OVERHEAD_SIZE_EXPECTATION, InMemoryEngineConfig, + InMemoryEngineConfig, engine::SkiplistEngine, write_batch::NODE_OVERHEAD_SIZE_EXPECTATION, }; #[derive(Debug, PartialEq)] @@ -124,7 +124,7 @@ mod tests { use tikv_util::config::ReadableSize; use super::*; - use crate::keys::{encode_key, InternalBytes, ValueType}; + use crate::keys::{InternalBytes, ValueType, encode_key}; #[test] fn test_memory_controller() { diff --git a/components/in_memory_engine/src/metrics.rs b/components/in_memory_engine/src/metrics.rs index 470507a749d..2016777a434 100644 --- a/components/in_memory_engine/src/metrics.rs +++ b/components/in_memory_engine/src/metrics.rs @@ -8,8 +8,8 @@ use prometheus::*; use prometheus_static_metric::*; use crate::{ - statistics::{Tickers, ENGINE_TICKER_TYPES}, InMemoryEngineStatistics, + statistics::{ENGINE_TICKER_TYPES, Tickers}, }; make_auto_flush_static_metric! { diff --git a/components/in_memory_engine/src/prop_test.rs b/components/in_memory_engine/src/prop_test.rs index be9912bec01..ea00dc2f888 100644 --- a/components/in_memory_engine/src/prop_test.rs +++ b/components/in_memory_engine/src/prop_test.rs @@ -3,9 +3,9 @@ use std::sync::Arc; use crossbeam::epoch; -use engine_rocks::{util::new_engine, RocksEngine}; +use engine_rocks::{RocksEngine, util::new_engine}; use engine_traits::{ - CacheRegion, CfName, Iterable, Iterator, Peekable, SyncMutable, CF_DEFAULT, CF_LOCK, CF_WRITE, + CF_DEFAULT, CF_LOCK, CF_WRITE, CacheRegion, CfName, Iterable, Iterator, Peekable, SyncMutable, }; use proptest::prelude::*; use tikv_util::config::{ReadableSize, VersionTrack}; @@ -13,8 +13,8 @@ use txn_types::{Key, TimeStamp}; use super::engine::SkiplistHandle; use crate::{ - decode_key, engine::SkiplistEngine, keys::encode_key, memory_controller::MemoryController, - InMemoryEngineConfig, InternalBytes, + InMemoryEngineConfig, InternalBytes, decode_key, engine::SkiplistEngine, keys::encode_key, + memory_controller::MemoryController, }; // This fixed mvcc suffix is used for CF_WRITE and CF_DEFAULT in prop test. diff --git a/components/in_memory_engine/src/read.rs b/components/in_memory_engine/src/read.rs index 3619c4c4555..e587b93921c 100644 --- a/components/in_memory_engine/src/read.rs +++ b/components/in_memory_engine/src/read.rs @@ -5,29 +5,29 @@ use std::{fmt::Debug, ops::Deref, result, sync::Arc}; use bytes::Bytes; use crossbeam::epoch; -use crossbeam_skiplist::{base::OwnedIter, SkipList}; +use crossbeam_skiplist::{SkipList, base::OwnedIter}; use engine_rocks::{raw::SliceTransform, util::FixedSuffixSliceTransform}; use engine_traits::{ - CacheRegion, CfNamesExt, DbVector, Error, FailedReason, IterMetricsCollector, IterOptions, - Iterable, Iterator, MetricsExt, Peekable, ReadOptions, Result, Snapshot, SnapshotMiscExt, - CF_DEFAULT, + CF_DEFAULT, CacheRegion, CfNamesExt, DbVector, Error, FailedReason, IterMetricsCollector, + IterOptions, Iterable, Iterator, MetricsExt, Peekable, ReadOptions, Result, Snapshot, + SnapshotMiscExt, }; use prometheus::local::LocalHistogram; use slog_global::error; use tikv_util::{box_err, time::Instant}; use crate::{ + RegionCacheMemoryEngine, background::BackgroundTask, - engine::{cf_to_id, SkiplistEngine}, + engine::{SkiplistEngine, cf_to_id}, keys::{ - decode_key, encode_seek_for_prev_key, encode_seek_key, InternalBytes, InternalKey, - ValueType, + InternalBytes, InternalKey, ValueType, decode_key, encode_seek_for_prev_key, + encode_seek_key, }, metrics::IN_MEMORY_ENGINE_SEEK_DURATION, perf_context::PERF_CONTEXT, perf_counter_add, statistics::{LocalStatistics, Statistics, Tickers}, - RegionCacheMemoryEngine, }; // The max snapshot number that can exist in the RocksDB. This is typically used @@ -653,7 +653,7 @@ impl Deref for RegionCacheDbVector { impl DbVector for RegionCacheDbVector {} -impl<'a> PartialEq<&'a [u8]> for RegionCacheDbVector { +impl PartialEq<&[u8]> for RegionCacheDbVector { fn eq(&self, rhs: &&[u8]) -> bool { self.0.as_slice() == *rhs } @@ -673,13 +673,13 @@ mod tests { use crossbeam::epoch; use crossbeam_skiplist::SkipList; use engine_rocks::{ - raw::DBStatisticsTickerType, util::new_engine_opt, RocksDbOptions, RocksStatistics, + RocksDbOptions, RocksStatistics, raw::DBStatisticsTickerType, util::new_engine_opt, }; use engine_traits::{ - CacheRegion, EvictReason, FailedReason, IterMetricsCollector, IterOptions, Iterable, - Iterator, MetricsExt, Mutable, Peekable, ReadOptions, RegionCacheEngine, - RegionCacheEngineExt, RegionEvent, WriteBatch, WriteBatchExt, CF_DEFAULT, CF_LOCK, - CF_WRITE, + CF_DEFAULT, CF_LOCK, CF_WRITE, CacheRegion, EvictReason, FailedReason, + IterMetricsCollector, IterOptions, Iterable, Iterator, MetricsExt, Mutable, Peekable, + ReadOptions, RegionCacheEngine, RegionCacheEngineExt, RegionEvent, WriteBatch, + WriteBatchExt, }; use keys::DATA_PREFIX_KEY; use tempfile::Builder; @@ -687,17 +687,17 @@ mod tests { use super::{RegionCacheIterator, RegionCacheSnapshot}; use crate::{ - engine::{cf_to_id, SkiplistEngine}, + InMemoryEngineConfig, InMemoryEngineContext, RegionCacheMemoryEngine, + RegionCacheWriteBatch, + engine::{SkiplistEngine, cf_to_id}, keys::{ - construct_key, construct_region_key, construct_user_key, construct_value, decode_key, - encode_key, encode_seek_key, InternalBytes, ValueType, + InternalBytes, ValueType, construct_key, construct_region_key, construct_user_key, + construct_value, decode_key, encode_key, encode_seek_key, }, perf_context::PERF_CONTEXT, region_manager::RegionState, statistics::Tickers, test_util::new_region, - InMemoryEngineConfig, InMemoryEngineContext, RegionCacheMemoryEngine, - RegionCacheWriteBatch, }; #[test] @@ -723,13 +723,12 @@ mod tests { ); } else { assert!( - regions_map.regions()[®ion.id] + !regions_map.regions()[®ion.id] .region_snapshot_list() .lock() .unwrap() .0 - .get(&snapshot_ts) - .is_none() + .contains_key(&snapshot_ts) ) } }; diff --git a/components/in_memory_engine/src/region_label.rs b/components/in_memory_engine/src/region_label.rs index c6e643beb53..739496e42f4 100644 --- a/components/in_memory_engine/src/region_label.rs +++ b/components/in_memory_engine/src/region_label.rs @@ -11,8 +11,8 @@ use futures::{ use keys::{data_end_key, data_key}; use kvproto::meta_storagepb::EventEventType; use pd_client::{ + Error as PdError, PdClient, REGION_LABEL_PATH_PREFIX, RpcClient, meta_storage::{Checked, Get, MetaStorageClient, Sourced, Watch}, - Error as PdError, PdClient, RpcClient, REGION_LABEL_PATH_PREFIX, }; use serde::{Deserialize, Serialize}; use tikv_util::{error, info, timer::GLOBAL_TIMER_HANDLE}; @@ -296,7 +296,7 @@ pub mod tests { use futures::executor::block_on; use pd_client::meta_storage::{Delete, Put}; use security::{SecurityConfig, SecurityManager}; - use test_pd::{mocker::MetaStorage, util::*, Server as MockServer}; + use test_pd::{Server as MockServer, mocker::MetaStorage, util::*}; use tikv_util::{config::ReadableDuration, worker::Builder}; use super::*; diff --git a/components/in_memory_engine/src/region_manager.rs b/components/in_memory_engine/src/region_manager.rs index 338f6f56aed..0b2236f59ef 100644 --- a/components/in_memory_engine/src/region_manager.rs +++ b/components/in_memory_engine/src/region_manager.rs @@ -9,8 +9,8 @@ use std::{ fmt::Debug, result, sync::{ - atomic::{AtomicBool, Ordering}, Arc, Mutex, + atomic::{AtomicBool, Ordering}, }, }; diff --git a/components/in_memory_engine/src/region_stats.rs b/components/in_memory_engine/src/region_stats.rs index 634d5d2bb73..9abbc78ebf6 100644 --- a/components/in_memory_engine/src/region_stats.rs +++ b/components/in_memory_engine/src/region_stats.rs @@ -3,8 +3,8 @@ use std::{ collections::BTreeMap, num::NonZeroUsize, sync::{ - atomic::{AtomicBool, Ordering}, Arc, Mutex as StdMutex, + atomic::{AtomicBool, Ordering}, }, time::{Duration, Instant}, }; @@ -21,6 +21,7 @@ use tikv_util::{config::VersionTrack, info, worker::Scheduler}; use tokio::sync::mpsc; use crate::{ + BackgroundTask, InMemoryEngineConfig, memory_controller::MemoryController, metrics::{ IN_MEMORY_ENGINE_AUTO_LOAD_EVICT_CACHED_REGION_COP_REQ, @@ -29,7 +30,6 @@ use crate::{ IN_MEMORY_ENGINE_AUTO_LOAD_EVICT_TOP_REGION_MVCC_AMP, }, region_manager::CopRequestsSma, - BackgroundTask, InMemoryEngineConfig, }; /// Do not evict a region if has been cached for less than this duration. @@ -282,7 +282,7 @@ impl RegionStatsManager { let load_time = regions_loaded.entry(crs.region.id).or_insert(now); let can_evict = now .checked_duration_since(*load_time) - .map_or(false, |d| d > self.evict_min_duration); + .is_some_and(|d| d > self.evict_min_duration); if !can_evict { continue; } @@ -473,7 +473,7 @@ fn sort_cached_region_stats( pub mod tests { use futures::executor::block_on; use pd_client::{RegionStat, RegionWriteCfCopDetail}; - use raftstore::coprocessor::{self, region_info_accessor::TopRegions, RegionInfoProvider}; + use raftstore::coprocessor::{self, RegionInfoProvider, region_info_accessor::TopRegions}; use tikv_util::{ box_err, config::{ReadableDuration, ReadableSize, VersionTrack}, @@ -482,8 +482,8 @@ pub mod tests { use super::*; use crate::{ - engine::SkiplistEngine, region_manager::COP_REQUEST_SMA_RECORD_COUNT, - test_util::new_region, InMemoryEngineConfig, + InMemoryEngineConfig, engine::SkiplistEngine, region_manager::COP_REQUEST_SMA_RECORD_COUNT, + test_util::new_region, }; struct RegionInfoSimulator { diff --git a/components/in_memory_engine/src/statistics.rs b/components/in_memory_engine/src/statistics.rs index 47e5a912bb7..64608dd6880 100644 --- a/components/in_memory_engine/src/statistics.rs +++ b/components/in_memory_engine/src/statistics.rs @@ -1,8 +1,8 @@ // Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0. use std::sync::{ - atomic::{AtomicU64, Ordering}, Mutex, + atomic::{AtomicU64, Ordering}, }; use crossbeam::utils::CachePadded; diff --git a/components/in_memory_engine/src/test_util.rs b/components/in_memory_engine/src/test_util.rs index c11ba667ae4..217716e2179 100644 --- a/components/in_memory_engine/src/test_util.rs +++ b/components/in_memory_engine/src/test_util.rs @@ -4,14 +4,14 @@ use std::sync::Arc; use crossbeam::epoch; use engine_rocks::RocksEngine; -use engine_traits::{SyncMutable, CF_WRITE}; +use engine_traits::{CF_WRITE, SyncMutable}; use keys::data_key; use kvproto::metapb::{Peer, Region}; use txn_types::{Key, TimeStamp, Write, WriteType}; use crate::{ engine::SkiplistHandle, - keys::{encode_key, InternalBytes, ValueType}, + keys::{InternalBytes, ValueType, encode_key}, memory_controller::MemoryController, write_batch::RegionCacheWriteBatchEntry, }; diff --git a/components/in_memory_engine/src/write_batch.rs b/components/in_memory_engine/src/write_batch.rs index f2e4b6aeed9..1ef5f5e66a6 100644 --- a/components/in_memory_engine/src/write_batch.rs +++ b/components/in_memory_engine/src/write_batch.rs @@ -1,31 +1,31 @@ // Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0. use std::{ - sync::{atomic::Ordering, Arc}, + sync::{Arc, atomic::Ordering}, time::Duration, }; use bytes::Bytes; use crossbeam::epoch; use engine_traits::{ - CacheRegion, EvictReason, MiscExt, Mutable, RegionCacheEngine, Result, WriteBatch, - WriteBatchExt, WriteOptions, CF_DEFAULT, + CF_DEFAULT, CacheRegion, EvictReason, MiscExt, Mutable, RegionCacheEngine, Result, WriteBatch, + WriteBatchExt, WriteOptions, }; use kvproto::metapb; use smallvec::SmallVec; use tikv_util::{box_err, config::ReadableSize, error, info, time::Instant, warn}; use crate::{ + RegionCacheMemoryEngine, background::BackgroundTask, - engine::{cf_to_id, id_to_cf, is_lock_cf, SkiplistEngine}, - keys::{encode_key, InternalBytes, ValueType, ENC_KEY_SEQ_LENGTH}, + engine::{SkiplistEngine, cf_to_id, id_to_cf, is_lock_cf}, + keys::{ENC_KEY_SEQ_LENGTH, InternalBytes, ValueType, encode_key}, memory_controller::{MemoryController, MemoryUsage}, metrics::{ - count_operations_for_cfs, IN_MEMORY_ENGINE_PREPARE_FOR_WRITE_DURATION_HISTOGRAM, - IN_MEMORY_ENGINE_WRITE_DURATION_HISTOGRAM, + IN_MEMORY_ENGINE_PREPARE_FOR_WRITE_DURATION_HISTOGRAM, + IN_MEMORY_ENGINE_WRITE_DURATION_HISTOGRAM, count_operations_for_cfs, }, region_manager::RegionCacheStatus, - RegionCacheMemoryEngine, }; // This is a bit of a hack. It's the overhead of a node in the skiplist with @@ -592,7 +592,7 @@ mod tests { use crossbeam_skiplist::SkipList; use engine_rocks::util::new_engine; use engine_traits::{ - CacheRegion, FailedReason, Peekable, RegionCacheEngine, WriteBatch, DATA_CFS, + CacheRegion, DATA_CFS, FailedReason, Peekable, RegionCacheEngine, WriteBatch, }; use kvproto::metapb::{Region, RegionEpoch}; use online_config::{ConfigChange, ConfigManager, ConfigValue}; @@ -601,8 +601,8 @@ mod tests { use super::*; use crate::{ - background::flush_epoch, config::InMemoryEngineConfigManager, region_manager::RegionState, - test_util::new_region, InMemoryEngineConfig, InMemoryEngineContext, + InMemoryEngineConfig, InMemoryEngineContext, background::flush_epoch, + config::InMemoryEngineConfigManager, region_manager::RegionState, test_util::new_region, }; // We should not use skiplist.get directly as we only cares keys without diff --git a/components/in_memory_engine/tests/failpoints/test_memory_engine.rs b/components/in_memory_engine/tests/failpoints/test_memory_engine.rs index 84bce6bdd64..8431e33f701 100644 --- a/components/in_memory_engine/tests/failpoints/test_memory_engine.rs +++ b/components/in_memory_engine/tests/failpoints/test_memory_engine.rs @@ -1,28 +1,28 @@ // Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0. use std::{ - sync::{mpsc::sync_channel, Arc}, + sync::{Arc, mpsc::sync_channel}, time::Duration, }; use crossbeam::epoch; use engine_rocks::util::new_engine; use engine_traits::{ - CacheRegion, EvictReason, Mutable, RegionCacheEngine, RegionCacheEngineExt, RegionEvent, - WriteBatch, WriteBatchExt, CF_DEFAULT, CF_LOCK, CF_WRITE, DATA_CFS, + CF_DEFAULT, CF_LOCK, CF_WRITE, CacheRegion, DATA_CFS, EvictReason, Mutable, RegionCacheEngine, + RegionCacheEngineExt, RegionEvent, WriteBatch, WriteBatchExt, }; use in_memory_engine::{ - decode_key, encode_key_for_boundary_without_mvcc, encoding_for_filter, - test_util::{new_region, put_data, put_data_in_rocks}, BackgroundTask, InMemoryEngineConfig, InMemoryEngineContext, InternalBytes, InternalKey, - RegionCacheMemoryEngine, RegionState, SkiplistHandle, ValueType, + RegionCacheMemoryEngine, RegionState, SkiplistHandle, ValueType, decode_key, + encode_key_for_boundary_without_mvcc, encoding_for_filter, + test_util::{new_region, put_data, put_data_in_rocks}, }; -use keys::{data_key, DATA_MAX_KEY, DATA_MIN_KEY}; +use keys::{DATA_MAX_KEY, DATA_MIN_KEY, data_key}; use kvproto::metapb::Region; use tempfile::Builder; use tikv_util::config::{ReadableDuration, ReadableSize, VersionTrack}; use tokio::{ - sync::{mpsc, Mutex}, + sync::{Mutex, mpsc}, time::timeout, }; use txn_types::{Key, TimeStamp, WriteType}; diff --git a/components/keys/src/lib.rs b/components/keys/src/lib.rs index 304e13f1e66..caa1f0b7cc1 100644 --- a/components/keys/src/lib.rs +++ b/components/keys/src/lib.rs @@ -361,7 +361,7 @@ mod tests { #[test] fn test_region_key() { - let ids = vec![1, 1024, u64::max_value()]; + let ids = vec![1, 1024, u64::MAX]; for id in ids { // region meta key let meta_prefix = region_meta_prefix(id); diff --git a/components/log_wrappers/src/lib.rs b/components/log_wrappers/src/lib.rs index d4097ce5a13..71aa924c50c 100644 --- a/components/log_wrappers/src/lib.rs +++ b/components/log_wrappers/src/lib.rs @@ -14,10 +14,10 @@ use std::{fmt, str::FromStr, sync::atomic::Ordering}; use atomic::Atomic; use online_config::ConfigValue; use protobuf::atomic_flags::{ - set_redact_level as proto_set_redact_level, RedactLevel, DEFAULT_REDACT_MARKER_HEAD, - DEFAULT_REDACT_MARKER_TAIL, + DEFAULT_REDACT_MARKER_HEAD, DEFAULT_REDACT_MARKER_TAIL, RedactLevel, + set_redact_level as proto_set_redact_level, }; -use serde::{de, Deserialize, Serialize, Serializer}; +use serde::{Deserialize, Serialize, Serializer, de}; pub use crate::hex::*; diff --git a/components/memory_trace_macros/src/lib.rs b/components/memory_trace_macros/src/lib.rs index be5e516b437..ab073dcef32 100644 --- a/components/memory_trace_macros/src/lib.rs +++ b/components/memory_trace_macros/src/lib.rs @@ -4,7 +4,7 @@ use proc_macro::TokenStream; use quote::quote; -use syn::{parse_macro_input, Data, DeriveInput, Fields}; +use syn::{Data, DeriveInput, Fields, parse_macro_input}; /// MemoryTraceHelper adds two methods `reset` and `sum` to derived struct. /// All fields of derived struct should be `usize`. diff --git a/components/online_config/src/lib.rs b/components/online_config/src/lib.rs index b02a4e94ed9..7d4571ba61c 100644 --- a/components/online_config/src/lib.rs +++ b/components/online_config/src/lib.rs @@ -109,15 +109,15 @@ impl_into!(ConfigChange, Module); /// The OnlineConfig trait /// /// There are four type of fields inside derived OnlineConfig struct: -/// 1. `#[online_config(skip)]` field, these fields will not return -/// by `diff` method and have not effect of `update` method +/// 1. `#[online_config(skip)]` field, these fields will not return by `diff` +/// method and have not effect of `update` method /// 2. `#[online_config(hidden)]` field, these fields have the same effect of -/// `#[online_config(skip)]` field, in addition, these fields will not appear -/// at the output of serializing `Self::Encoder` +/// `#[online_config(skip)]` field, in addition, these fields will not appear +/// at the output of serializing `Self::Encoder` /// 3. `#[online_config(submodule)]` field, these fields represent the -/// submodule, and should also derive `OnlineConfig` -/// 4. normal fields, the type of these fields should be implment -/// `Into` and `From`/`TryFrom` for `ConfigValue` +/// submodule, and should also derive `OnlineConfig` +/// 4. normal fields, the type of these fields should be implment `Into` and +/// `From`/`TryFrom` for `ConfigValue` pub trait OnlineConfig<'a> { type Encoder: serde::Serialize; /// Compare to other config, return the difference diff --git a/components/panic_hook/src/lib.rs b/components/panic_hook/src/lib.rs index 7e95ea4071a..7c26b866f08 100644 --- a/components/panic_hook/src/lib.rs +++ b/components/panic_hook/src/lib.rs @@ -8,16 +8,16 @@ use std::{ cell::RefCell, - panic::{self, AssertUnwindSafe, PanicInfo}, + panic::{self, AssertUnwindSafe, PanicHookInfo}, sync::Once, }; static INIT: Once = Once::new(); // store the default panic hook defined in std. -static mut DEFAULT_HOOK: Option<*mut (dyn Fn(&PanicInfo<'_>) + 'static + Sync + Send)> = None; +static mut DEFAULT_HOOK: Option<*mut (dyn Fn(&PanicHookInfo<'_>) + 'static + Sync + Send)> = None; thread_local! { - static MUTED: RefCell = RefCell::new(false) + static MUTED: RefCell = const {RefCell::new(false)} } /// Replace the default hook if we haven't. @@ -40,7 +40,7 @@ pub fn unmute() { } /// Print the stacktrace according to the static MUTED. -fn track_hook(p: &PanicInfo<'_>) { +fn track_hook(p: &PanicHookInfo<'_>) { MUTED.with(|m| { if *m.borrow() { return; diff --git a/components/pd_client/src/client.rs b/components/pd_client/src/client.rs index 4cdab7c9364..af06d6d94b0 100644 --- a/components/pd_client/src/client.rs +++ b/components/pd_client/src/client.rs @@ -3,21 +3,20 @@ use std::{ fmt, sync::{ - atomic::{AtomicU64, Ordering}, Arc, + atomic::{AtomicU64, Ordering}, }, time::Duration, - u64, }; use futures::{ + TryStreamExt, channel::mpsc, compat::{Compat, Future01CompatExt}, executor::block_on, future::{self, BoxFuture, FutureExt, TryFlattenStream, TryFutureExt}, sink::SinkExt, stream::{ErrInto, StreamExt}, - TryStreamExt, }; use grpcio::{EnvBuilder, Environment, WriteFlags}; use kvproto::{ @@ -31,18 +30,18 @@ use kvproto::{ }; use security::SecurityManager; use tikv_util::{ - box_err, debug, error, info, thd_name, time::Instant, timer::GLOBAL_TIMER_HANDLE, warn, Either, - HandyRwLock, + Either, HandyRwLock, box_err, debug, error, info, thd_name, time::Instant, + timer::GLOBAL_TIMER_HANDLE, warn, }; use txn_types::TimeStamp; -use yatp::{task::future::TaskCell, ThreadPool}; +use yatp::{ThreadPool, task::future::TaskCell}; use super::{ + BucketStat, Config, Error, FeatureGate, PdClient, PdFuture, REQUEST_TIMEOUT, RegionInfo, + RegionStat, Result, UnixSecs, meta_storage::{Delete, Get, MetaStorageClient, Put, Watch}, metrics::*, - util::{call_option_inner, check_resp_header, sync_request, Client, PdConnector}, - BucketStat, Config, Error, FeatureGate, PdClient, PdFuture, RegionInfo, RegionStat, Result, - UnixSecs, REQUEST_TIMEOUT, + util::{Client, PdConnector, call_option_inner, check_resp_header, sync_request}, }; pub const CQ_COUNT: usize = 1; @@ -81,7 +80,7 @@ impl RpcClient { // -1 means the max. let retries = match cfg.retry_max_count { - -1 => std::isize::MAX, + -1 => isize::MAX, v => v.saturating_add(1), }; let monitor = Arc::new( diff --git a/components/pd_client/src/client_v2.rs b/components/pd_client/src/client_v2.rs index 7ffb0ca08bf..1030fc90281 100644 --- a/components/pd_client/src/client_v2.rs +++ b/components/pd_client/src/client_v2.rs @@ -15,11 +15,10 @@ use std::{ fmt::Debug, pin::Pin, sync::{ - atomic::{AtomicU64, Ordering}, Arc, Mutex, + atomic::{AtomicU64, Ordering}, }, time::{Duration, Instant as StdInstant}, - u64, }; use fail::fail_point; @@ -53,11 +52,11 @@ use tokio::sync::{broadcast, mpsc as tokio_mpsc}; use txn_types::TimeStamp; use super::{ + Config, Error, FeatureGate, REQUEST_TIMEOUT as REQUEST_TIMEOUT_SEC, RegionInfo, Result, + UnixSecs, client::{CLIENT_PREFIX, CQ_COUNT}, metrics::*, - util::{check_resp_header, PdConnector, TargetInfo}, - Config, Error, FeatureGate, RegionInfo, Result, UnixSecs, - REQUEST_TIMEOUT as REQUEST_TIMEOUT_SEC, + util::{PdConnector, TargetInfo, check_resp_header}, }; use crate::PdFuture; @@ -88,7 +87,7 @@ impl RawClient { async fn connect(ctx: &ConnectContext) -> Result { // -1 means the max. let retries = match ctx.cfg.retry_max_count { - -1 => std::isize::MAX, + -1 => isize::MAX, v => v.saturating_add(1), }; for i in 0..retries { diff --git a/components/pd_client/src/config.rs b/components/pd_client/src/config.rs index f42cc3528d7..e80ae19dcb8 100644 --- a/components/pd_client/src/config.rs +++ b/components/pd_client/src/config.rs @@ -45,7 +45,7 @@ impl Default for Config { Config { endpoints: vec!["127.0.0.1:2379".to_string()], retry_interval: ReadableDuration::millis(300), - retry_max_count: std::isize::MAX, + retry_max_count: isize::MAX, retry_log_every: 10, update_interval: ReadableDuration::minutes(10), enable_forwarding: false, diff --git a/components/pd_client/src/feature_gate.rs b/components/pd_client/src/feature_gate.rs index dc8bef853de..c55e17aa192 100644 --- a/components/pd_client/src/feature_gate.rs +++ b/components/pd_client/src/feature_gate.rs @@ -1,8 +1,8 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. use std::sync::{ - atomic::{AtomicU64, Ordering}, Arc, + atomic::{AtomicU64, Ordering}, }; use semver::{SemVerError, Version}; diff --git a/components/pd_client/src/lib.rs b/components/pd_client/src/lib.rs index 43646452c04..75fffa44b79 100644 --- a/components/pd_client/src/lib.rs +++ b/components/pd_client/src/lib.rs @@ -37,7 +37,7 @@ pub use self::{ config::Config, errors::{Error, Result}, feature_gate::{Feature, FeatureGate}, - util::{merge_bucket_stats, new_bucket_stats, PdConnector, REQUEST_RECONNECT_INTERVAL}, + util::{PdConnector, REQUEST_RECONNECT_INTERVAL, merge_bucket_stats, new_bucket_stats}, }; pub type Key = Vec; @@ -322,7 +322,7 @@ pub trait PdClient: Send + Sync { /// recovery mode recovery mode will do /// 1. update tikv cluster id from pd /// 2. all peer apply the log to last of the leader peer which has the most - /// log appended. 3. delete data to some point of time (resolved_ts) + /// log appended. 3. delete data to some point of time (resolved_ts) fn is_recovering_marked(&self) -> Result { unimplemented!(); } @@ -343,7 +343,7 @@ pub trait PdClient: Send + Sync { /// which Region and Peers will be removed. /// - For auto-balance, PD determines how to move the Region from one store /// to another. - + /// /// Gets store information if it is not a tombstone store. fn get_store(&self, _store_id: u64) -> Result { unimplemented!(); diff --git a/components/pd_client/src/tso.rs b/components/pd_client/src/tso.rs index feec5061a8c..6b8b2a0bf0b 100644 --- a/components/pd_client/src/tso.rs +++ b/components/pd_client/src/tso.rs @@ -27,7 +27,7 @@ use tikv_util::{box_err, info, sys::thread::StdThreadBuildWrapper}; use tokio::sync::{mpsc, oneshot, watch}; use txn_types::TimeStamp; -use crate::{metrics::PD_PENDING_TSO_REQUEST_GAUGE, Error, Result}; +use crate::{Error, Result, metrics::PD_PENDING_TSO_REQUEST_GAUGE}; /// It is an empirical value. const MAX_BATCH_SIZE: usize = 64; diff --git a/components/pd_client/src/util.rs b/components/pd_client/src/util.rs index 1ac1d80321b..2c896104322 100644 --- a/components/pd_client/src/util.rs +++ b/components/pd_client/src/util.rs @@ -3,7 +3,7 @@ use core::panic; use std::{ pin::Pin, - sync::{atomic::AtomicU64, Arc, RwLock}, + sync::{Arc, RwLock, atomic::AtomicU64}, thread, time::Duration, }; @@ -36,14 +36,14 @@ use kvproto::{ }; use security::SecurityManager; use tikv_util::{ - box_err, debug, error, info, slow_log, time::Instant, timer::GLOBAL_TIMER_HANDLE, warn, Either, - HandyRwLock, + Either, HandyRwLock, box_err, debug, error, info, slow_log, time::Instant, + timer::GLOBAL_TIMER_HANDLE, warn, }; use tokio_timer::timer::Handle; use super::{ - metrics::*, tso::TimestampOracle, BucketMeta, Config, Error, FeatureGate, PdFuture, Result, - REQUEST_TIMEOUT, + BucketMeta, Config, Error, FeatureGate, PdFuture, REQUEST_TIMEOUT, Result, metrics::*, + tso::TimestampOracle, }; const RETRY_INTERVAL: Duration = Duration::from_secs(1); // 1s diff --git a/components/profiler/src/lib.rs b/components/profiler/src/lib.rs index 2734d8f7877..2ebf851df41 100644 --- a/components/profiler/src/lib.rs +++ b/components/profiler/src/lib.rs @@ -43,6 +43,7 @@ //! //! Also see `examples/prime.rs`. +#![allow(clippy::doc_overindented_list_items)] #[allow(unused_extern_crates)] extern crate tikv_alloc; diff --git a/components/raft_log_engine/src/engine.rs b/components/raft_log_engine/src/engine.rs index c71b9fd65d9..44c335b838b 100644 --- a/components/raft_log_engine/src/engine.rs +++ b/components/raft_log_engine/src/engine.rs @@ -10,9 +10,8 @@ use std::{ use codec::number::NumberCodec; use encryption::{DataKeyManager, DecrypterReader, EncrypterWriter}; use engine_traits::{ - CacheStats, PerfContextExt, PerfContextKind, PerfLevel, RaftEngine, RaftEngineDebug, - RaftEngineReadOnly, RaftLogBatch as RaftLogBatchTrait, Result, CF_DEFAULT, CF_LOCK, CF_RAFT, - CF_WRITE, + CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE, CacheStats, PerfContextExt, PerfContextKind, PerfLevel, + RaftEngine, RaftEngineDebug, RaftEngineReadOnly, RaftLogBatch as RaftLogBatchTrait, Result, }; use file_system::{IoOp, IoRateLimiter, IoType, WithIoType}; use kvproto::{ @@ -24,8 +23,8 @@ use kvproto::{ }; use raft::eraftpb::Entry; use raft_engine::{ - env::{DefaultFileSystem, FileSystem, Handle, Permission, WriteExt}, Command, Engine as RawRaftEngine, Error as RaftEngineError, LogBatch, MessageExt, + env::{DefaultFileSystem, FileSystem, Handle, Permission, WriteExt}, }; pub use raft_engine::{Config as RaftEngineConfig, ReadableSize, RecoveryMode}; use tikv_util::Either; diff --git a/components/raft_log_engine/src/perf_context.rs b/components/raft_log_engine/src/perf_context.rs index 87946e2f48e..257f2d4a6dc 100644 --- a/components/raft_log_engine/src/perf_context.rs +++ b/components/raft_log_engine/src/perf_context.rs @@ -1,7 +1,7 @@ // Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. use raft_engine::get_perf_context; -use tracker::{TrackerToken, GLOBAL_TRACKERS}; +use tracker::{GLOBAL_TRACKERS, TrackerToken}; #[derive(Debug)] pub struct RaftEnginePerfContext; diff --git a/components/raftstore-v2/src/batch/mod.rs b/components/raftstore-v2/src/batch/mod.rs index 7daeebaa8f0..d6003224c45 100644 --- a/components/raftstore-v2/src/batch/mod.rs +++ b/components/raftstore-v2/src/batch/mod.rs @@ -7,4 +7,4 @@ mod store; -pub use store::{create_store_batch_system, StoreContext, StoreRouter, StoreSystem}; +pub use store::{StoreContext, StoreRouter, StoreSystem, create_store_batch_system}; diff --git a/components/raftstore-v2/src/batch/store.rs b/components/raftstore-v2/src/batch/store.rs index 056cd122e67..676ff274e2f 100644 --- a/components/raftstore-v2/src/batch/store.rs +++ b/components/raftstore-v2/src/batch/store.rs @@ -4,8 +4,8 @@ use std::{ ops::{Deref, DerefMut}, path::Path, sync::{ - atomic::{AtomicBool, Ordering}, Arc, Mutex, + atomic::{AtomicBool, Ordering}, }, time::Duration, }; @@ -19,52 +19,51 @@ use concurrency_manager::ConcurrencyManager; use crossbeam::channel::TrySendError; use encryption_export::DataKeyManager; use engine_traits::{KvEngine, RaftEngine, TabletRegistry}; -use file_system::{set_io_type, IoType, WithIoType}; +use file_system::{IoType, WithIoType, set_io_type}; use futures::compat::Future01CompatExt; use health_controller::types::LatencyInspector; use kvproto::{disk_usage::DiskUsage, raft_serverpb::RaftMessage}; use pd_client::PdClient; -use raft::{StateRole, INVALID_ID}; +use raft::{INVALID_ID, StateRole}; use raftstore::{ coprocessor::{CoprocessorHost, RegionChangeEvent}, store::{ + AutoSplitController, Config, ReadRunner, ReadTask, RefreshConfigTask, SplitCheckRunner, + SplitCheckTask, StoreWriters, StoreWritersContext, TabletSnapManager, Transport, + WriteRouterContext, WriteSenders, WriterContoller, fsm::{ - store::{PeerTickBatch, ENTRY_CACHE_EVICT_TICK_DURATION}, GlobalStoreStat, LocalStoreStat, + store::{ENTRY_CACHE_EVICT_TICK_DURATION, PeerTickBatch}, }, local_metrics::RaftMetrics, - AutoSplitController, Config, ReadRunner, ReadTask, RefreshConfigTask, SplitCheckRunner, - SplitCheckTask, StoreWriters, StoreWritersContext, TabletSnapManager, Transport, - WriteRouterContext, WriteSenders, WriterContoller, }, }; use resource_control::ResourceController; use resource_metering::CollectorRegHandle; use service::service_manager::GrpcServiceManager; -use slog::{warn, Logger}; +use slog::{Logger, warn}; use sst_importer::SstImporter; use tikv_util::{ - box_err, + Either, box_err, config::{Tracker, VersionTrack}, log::SlogFormat, - sys::{disk::get_disk_status, SysQuota}, - time::{duration_to_sec, monotonic_raw_now, Instant as TiInstant, Limiter}, - timer::{SteadyTimer, GLOBAL_TIMER_HANDLE}, + sys::{SysQuota, disk::get_disk_status}, + time::{Instant as TiInstant, Limiter, duration_to_sec, monotonic_raw_now}, + timer::{GLOBAL_TIMER_HANDLE, SteadyTimer}, worker::{Builder, LazyWorker, Scheduler, Worker}, yatp_pool::{DefaultTicker, FuturePool, YatpPoolBuilder}, - Either, }; use time::Timespec; use crate::{ + Error, Result, fsm::{PeerFsm, PeerFsmDelegate, SenderFsmPair, StoreFsm, StoreFsmDelegate, StoreMeta}, operation::{ - ReplayWatch, SharedReadTablet, MERGE_IN_PROGRESS_PREFIX, MERGE_SOURCE_PREFIX, SPLIT_PREFIX, + MERGE_IN_PROGRESS_PREFIX, MERGE_SOURCE_PREFIX, ReplayWatch, SPLIT_PREFIX, SharedReadTablet, }, raft::Storage, router::{PeerMsg, PeerTick, StoreMsg}, worker::{cleanup, pd, refresh_config, tablet}, - Error, Result, }; const MIN_MANUAL_FLUSH_RATE: f64 = 0.2; @@ -488,7 +487,7 @@ impl StorePollerBuilder { for entry in file_system::read_dir(self.tablet_registry.tablet_root())? { let entry = entry?; let path = entry.path(); - if path.extension().map_or(false, |s| s == "tmp") { + if path.extension().is_some_and(|s| s == "tmp") { // The directory may be generated by an aborted checkpoint. self.remove_dir(&path)?; continue; diff --git a/components/raftstore-v2/src/bootstrap.rs b/components/raftstore-v2/src/bootstrap.rs index 62bc9e4b8c5..54694184d19 100644 --- a/components/raftstore-v2/src/bootstrap.rs +++ b/components/raftstore-v2/src/bootstrap.rs @@ -12,10 +12,10 @@ use kvproto::{ use pd_client::PdClient; use raft::INVALID_ID; use raftstore::store::initial_region; -use slog::{debug, error, info, warn, Logger}; +use slog::{Logger, debug, error, info, warn}; use tikv_util::{box_err, box_try}; -use crate::{operation::write_initial_states, Result}; +use crate::{Result, operation::write_initial_states}; const MAX_CHECK_CLUSTER_BOOTSTRAPPED_RETRY_COUNT: u64 = 60; const CHECK_CLUSTER_BOOTSTRAPPED_RETRY_INTERVAL: Duration = Duration::from_secs(3); diff --git a/components/raftstore-v2/src/fsm/apply.rs b/components/raftstore-v2/src/fsm/apply.rs index 49530fcd6df..fb2c61e39db 100644 --- a/components/raftstore-v2/src/fsm/apply.rs +++ b/components/raftstore-v2/src/fsm/apply.rs @@ -9,7 +9,7 @@ use batch_system::{Fsm, FsmScheduler, Mailbox}; use crossbeam::channel::TryRecvError; use engine_traits::{FlushState, KvEngine, SstApplyState, TabletRegistry}; use fail::fail_point; -use futures::{compat::Future01CompatExt, FutureExt, StreamExt}; +use futures::{FutureExt, StreamExt, compat::Future01CompatExt}; use kvproto::{metapb, raft_serverpb::RegionLocalState}; use pd_client::BucketStat; use raftstore::{ @@ -26,10 +26,10 @@ use tikv_util::{ }; use crate::{ + TabletTask, operation::{CatchUpLogs, DataTrace}, raft::Apply, router::{ApplyRes, ApplyTask, PeerMsg}, - TabletTask, }; /// A trait for reporting apply result. diff --git a/components/raftstore-v2/src/fsm/peer.rs b/components/raftstore-v2/src/fsm/peer.rs index 7dc099c2944..4d1c5a06814 100644 --- a/components/raftstore-v2/src/fsm/peer.rs +++ b/components/raftstore-v2/src/fsm/peer.rs @@ -10,21 +10,21 @@ use encryption_export::DataKeyManager; use engine_traits::{KvEngine, RaftEngine, TabletRegistry}; use kvproto::{errorpb, raft_cmdpb::RaftCmdResponse}; use raftstore::store::{Config, ReadCallback, TabletSnapManager, Transport}; -use slog::{debug, info, trace, Logger}; +use slog::{Logger, debug, info, trace}; use tikv_util::{ is_zero_duration, mpsc::{self, LooseBoundedSender, Receiver}, slog_panic, - time::{duration_to_sec, Instant}, + time::{Instant, duration_to_sec}, }; -use tracker::{TrackerToken, GLOBAL_TRACKERS}; +use tracker::{GLOBAL_TRACKERS, TrackerToken}; use crate::{ + Result, batch::StoreContext, operation::ReplayWatch, raft::{Peer, Storage}, router::{PeerMsg, PeerTick, QueryResult}, - Result, }; pub type SenderFsmPair = (LooseBoundedSender, Box>); diff --git a/components/raftstore-v2/src/fsm/store.rs b/components/raftstore-v2/src/fsm/store.rs index 107c35f5559..30e253c1501 100644 --- a/components/raftstore-v2/src/fsm/store.rs +++ b/components/raftstore-v2/src/fsm/store.rs @@ -9,13 +9,13 @@ use std::{ use batch_system::Fsm; use collections::HashMap; use engine_traits::{KvEngine, RaftEngine}; -use futures::{compat::Future01CompatExt, FutureExt}; +use futures::{FutureExt, compat::Future01CompatExt}; use keys::{data_end_key, data_key}; use kvproto::metapb::Region; use raftstore::store::{ - fsm::store::StoreRegionMeta, Config, ReadDelegate, RegionReadProgressRegistry, Transport, + Config, ReadDelegate, RegionReadProgressRegistry, Transport, fsm::store::StoreRegionMeta, }; -use slog::{info, o, Logger}; +use slog::{Logger, info, o}; use tikv_util::{ future::poll_future_notify, is_zero_duration, @@ -81,8 +81,7 @@ impl StoreMeta { } else { assert!( self.region_ranges - .get(&(data_end_key(prev.get_end_key()), version)) - .is_some(), + .contains_key(&(data_end_key(prev.get_end_key()), version)), "{} region corrupted", SlogFormat(logger) ); diff --git a/components/raftstore-v2/src/lib.rs b/components/raftstore-v2/src/lib.rs index 5b5e132b9ce..5ef142b32f6 100644 --- a/components/raftstore-v2/src/lib.rs +++ b/components/raftstore-v2/src/lib.rs @@ -23,10 +23,8 @@ #![feature(let_chains)] #![feature(array_windows)] -#![feature(div_duration)] #![feature(box_into_inner)] #![feature(assert_matches)] -#![feature(option_get_or_insert_default)] mod batch; mod bootstrap; @@ -37,11 +35,11 @@ pub mod router; mod worker; pub(crate) use batch::StoreContext; -pub use batch::{create_store_batch_system, StoreRouter, StoreSystem}; +pub use batch::{StoreRouter, StoreSystem, create_store_batch_system}; pub use bootstrap::Bootstrap; pub use fsm::StoreMeta; -pub use operation::{write_initial_states, SimpleWriteBinary, SimpleWriteEncoder, StateStorage}; -pub use raftstore::{store::Config, Error, Result}; +pub use operation::{SimpleWriteBinary, SimpleWriteEncoder, StateStorage, write_initial_states}; +pub use raftstore::{Error, Result, store::Config}; pub use worker::{ cleanup::CompactTask, pd::{PdReporter, Task as PdTask}, diff --git a/components/raftstore-v2/src/operation/bucket.rs b/components/raftstore-v2/src/operation/bucket.rs index db57b815576..f28e7604c07 100644 --- a/components/raftstore-v2/src/operation/bucket.rs +++ b/components/raftstore-v2/src/operation/bucket.rs @@ -12,7 +12,7 @@ use kvproto::{ use pd_client::BucketMeta; use raftstore::{ coprocessor::RegionChangeEvent, - store::{util, Bucket, BucketRange, ReadProgress, SplitCheckTask, Transport}, + store::{Bucket, BucketRange, ReadProgress, SplitCheckTask, Transport, util}, }; use slog::{error, info}; diff --git a/components/raftstore-v2/src/operation/command/admin/compact_log.rs b/components/raftstore-v2/src/operation/command/admin/compact_log.rs index 4eff2e132a9..e56f23cfccd 100644 --- a/components/raftstore-v2/src/operation/command/admin/compact_log.rs +++ b/components/raftstore-v2/src/operation/command/admin/compact_log.rs @@ -16,8 +16,8 @@ use std::{ path::PathBuf, sync::{ - atomic::{AtomicU64, Ordering}, Arc, + atomic::{AtomicU64, Ordering}, }, }; @@ -25,11 +25,11 @@ use engine_traits::{KvEngine, RaftEngine, RaftLogBatch}; use kvproto::raft_cmdpb::{AdminCmdType, AdminRequest, AdminResponse, RaftCmdRequest}; use protobuf::Message; use raftstore::{ + Result, store::{ - fsm::new_admin_request, metrics::REGION_MAX_LOG_LAG, needs_evict_entry_cache, Transport, - WriteTask, RAFT_INIT_LOG_INDEX, + RAFT_INIT_LOG_INDEX, Transport, WriteTask, fsm::new_admin_request, + metrics::REGION_MAX_LOG_LAG, needs_evict_entry_cache, }, - Result, }; use slog::{debug, error, info}; use tikv_util::{box_err, log::SlogFormat}; diff --git a/components/raftstore-v2/src/operation/command/admin/conf_change.rs b/components/raftstore-v2/src/operation/command/admin/conf_change.rs index 5c7ff96a955..f80bb900c7b 100644 --- a/components/raftstore-v2/src/operation/command/admin/conf_change.rs +++ b/components/raftstore-v2/src/operation/command/admin/conf_change.rs @@ -19,13 +19,13 @@ use kvproto::{ use protobuf::Message; use raft::prelude::*; use raftstore::{ + Error, Result, coprocessor::{RegionChangeEvent, RegionChangeReason}, store::{ + ProposalContext, metrics::{PEER_ADMIN_CMD_COUNTER_VEC, PEER_PROPOSE_LOG_SIZE_HISTOGRAM}, util::{self, ChangePeerI, ConfChangeKind}, - ProposalContext, }, - Error, Result, }; use slog::{error, info, warn}; use tikv_util::{box_err, slog_panic}; diff --git a/components/raftstore-v2/src/operation/command/admin/flashback.rs b/components/raftstore-v2/src/operation/command/admin/flashback.rs index 7301736e380..a9fc47030e4 100644 --- a/components/raftstore-v2/src/operation/command/admin/flashback.rs +++ b/components/raftstore-v2/src/operation/command/admin/flashback.rs @@ -8,12 +8,12 @@ use kvproto::{ }; use protobuf::Message; use raftstore::{ + Result, coprocessor::RegionChangeReason, store::{ - metrics::{PEER_ADMIN_CMD_COUNTER, PEER_IN_FLASHBACK_STATE}, LocksStatus, + metrics::{PEER_ADMIN_CMD_COUNTER, PEER_IN_FLASHBACK_STATE}, }, - Result, }; use super::AdminCmdResult; diff --git a/components/raftstore-v2/src/operation/command/admin/merge/commit.rs b/components/raftstore-v2/src/operation/command/admin/merge/commit.rs index 166d3a98d86..a0b7d9fbd5b 100644 --- a/components/raftstore-v2/src/operation/command/admin/merge/commit.rs +++ b/components/raftstore-v2/src/operation/command/admin/merge/commit.rs @@ -64,15 +64,15 @@ use kvproto::{ raft_serverpb::{MergedRecord, PeerState, RegionLocalState}, }; use protobuf::Message; -use raft::{GetEntriesContext, Storage, INVALID_ID, NO_LIMIT}; +use raft::{GetEntriesContext, INVALID_ID, NO_LIMIT, Storage}; use raftstore::{ + Result, coprocessor::RegionChangeReason, store::{ - fsm::new_admin_request, metrics::PEER_ADMIN_CMD_COUNTER, util, ProposalContext, Transport, + ProposalContext, Transport, fsm::new_admin_request, metrics::PEER_ADMIN_CMD_COUNTER, util, }, - Result, }; -use slog::{debug, error, info, Logger}; +use slog::{Logger, debug, error, info}; use tikv_util::{ config::ReadableDuration, log::SlogFormat, @@ -81,7 +81,7 @@ use tikv_util::{ time::Instant, }; -use super::{merge_source_path, PrepareStatus}; +use super::{PrepareStatus, merge_source_path}; use crate::{ batch::StoreContext, fsm::ApplyResReporter, diff --git a/components/raftstore-v2/src/operation/command/admin/merge/mod.rs b/components/raftstore-v2/src/operation/command/admin/merge/mod.rs index 94adc1e1c3c..bc4d0d6dc81 100644 --- a/components/raftstore-v2/src/operation/command/admin/merge/mod.rs +++ b/components/raftstore-v2/src/operation/command/admin/merge/mod.rs @@ -13,9 +13,9 @@ use kvproto::{ raft_serverpb::{MergeState, PeerState, RegionLocalState}, }; use prepare::PrepareStatus; -use raft::{ProgressState, INVALID_INDEX}; +use raft::{INVALID_INDEX, ProgressState}; use raftstore::Result; -use slog::{info, warn, Logger}; +use slog::{Logger, info, warn}; use tikv_util::box_err; use crate::raft::Peer; diff --git a/components/raftstore-v2/src/operation/command/admin/merge/prepare.rs b/components/raftstore-v2/src/operation/command/admin/merge/prepare.rs index 0aa44ea45db..b4dbdaf9ab7 100644 --- a/components/raftstore-v2/src/operation/command/admin/merge/prepare.rs +++ b/components/raftstore-v2/src/operation/command/admin/merge/prepare.rs @@ -30,7 +30,7 @@ use std::{mem, time::Duration}; use collections::HashMap; -use engine_traits::{Checkpointer, KvEngine, RaftEngine, RaftLogBatch, CF_LOCK}; +use engine_traits::{CF_LOCK, Checkpointer, KvEngine, RaftEngine, RaftLogBatch}; use futures::channel::oneshot; use kvproto::{ metapb::RegionEpoch, @@ -44,11 +44,11 @@ use kvproto::{ }; use parking_lot::RwLockUpgradableReadGuard; use protobuf::Message; -use raft::{eraftpb::EntryType, GetEntriesContext, NO_LIMIT}; +use raft::{GetEntriesContext, NO_LIMIT, eraftpb::EntryType}; use raftstore::{ - coprocessor::RegionChangeReason, - store::{metrics::PEER_ADMIN_CMD_COUNTER, util, LocksStatus, ProposalContext, Transport}, Error, Result, + coprocessor::RegionChangeReason, + store::{LocksStatus, ProposalContext, Transport, metrics::PEER_ADMIN_CMD_COUNTER, util}, }; use slog::{debug, error, info}; use tikv_util::{ @@ -56,11 +56,11 @@ use tikv_util::{ }; use txn_types::WriteBatchFlags; -use super::{merge_source_path, CatchUpLogs}; +use super::{CatchUpLogs, merge_source_path}; use crate::{ batch::StoreContext, fsm::ApplyResReporter, - operation::{command::parse_at, AdminCmdResult, SimpleWriteReqDecoder}, + operation::{AdminCmdResult, SimpleWriteReqDecoder, command::parse_at}, raft::{Apply, Peer}, router::{CmdResChannel, PeerMsg, RaftRequest}, }; diff --git a/components/raftstore-v2/src/operation/command/admin/merge/rollback.rs b/components/raftstore-v2/src/operation/command/admin/merge/rollback.rs index adc49a928b3..e15a1511eb1 100644 --- a/components/raftstore-v2/src/operation/command/admin/merge/rollback.rs +++ b/components/raftstore-v2/src/operation/command/admin/merge/rollback.rs @@ -8,9 +8,9 @@ use kvproto::{ raft_serverpb::{PeerState, RegionLocalState}, }; use raftstore::{ - coprocessor::RegionChangeReason, - store::{fsm::new_admin_request, metrics::PEER_ADMIN_CMD_COUNTER, LocksStatus, Transport}, Result, + coprocessor::RegionChangeReason, + store::{LocksStatus, Transport, fsm::new_admin_request, metrics::PEER_ADMIN_CMD_COUNTER}, }; use slog::{error, info}; use tikv_util::slog_panic; diff --git a/components/raftstore-v2/src/operation/command/admin/mod.rs b/components/raftstore-v2/src/operation/command/admin/mod.rs index db836086172..1c54d428e8a 100644 --- a/components/raftstore-v2/src/operation/command/admin/mod.rs +++ b/components/raftstore-v2/src/operation/command/admin/mod.rs @@ -17,28 +17,28 @@ use kvproto::{ raft_cmdpb::{AdminCmdType, RaftCmdRequest}, raft_serverpb::{ExtraMessageType, FlushMemtable, RaftMessage}, }; -use merge::{ - commit::CommitMergeResult, prepare::PrepareMergeResult, rollback::RollbackMergeResult, -}; pub use merge::{ + MERGE_SOURCE_PREFIX, MergeContext, commit::{CatchUpLogs, MERGE_IN_PROGRESS_PREFIX}, - merge_source_path, MergeContext, MERGE_SOURCE_PREFIX, + merge_source_path, +}; +use merge::{ + commit::CommitMergeResult, prepare::PrepareMergeResult, rollback::RollbackMergeResult, }; use protobuf::Message; use raftstore::{ + Error, store::{ - cmd_resp, + ProposalContext, Transport, cmd_resp, fsm::{apply, apply::validate_batch_split}, msg::ErrorCallback, - ProposalContext, Transport, }, - Error, }; use slog::{debug, error, info}; use split::SplitResult; pub use split::{ - report_split_init_finish, temp_split_path, RequestHalfSplit, RequestSplit, SplitFlowControl, - SplitInit, SplitPendingAppend, SPLIT_PREFIX, + RequestHalfSplit, RequestSplit, SPLIT_PREFIX, SplitFlowControl, SplitInit, SplitPendingAppend, + report_split_init_finish, temp_split_path, }; use tikv_util::{box_err, log::SlogFormat, slog_panic, sys::disk::DiskUsage}; use txn_types::WriteBatchFlags; diff --git a/components/raftstore-v2/src/operation/command/admin/split.rs b/components/raftstore-v2/src/operation/command/admin/split.rs index 1f7ba9b9075..9969c1c0cd9 100644 --- a/components/raftstore-v2/src/operation/command/admin/split.rs +++ b/components/raftstore-v2/src/operation/command/admin/split.rs @@ -42,30 +42,30 @@ use kvproto::{ raft_serverpb::{RaftMessage, RaftSnapshotData}, }; use protobuf::Message; -use raft::{prelude::Snapshot, INVALID_ID}; +use raft::{INVALID_ID, prelude::Snapshot}; use raftstore::{ + Result, coprocessor::RegionChangeReason, store::{ + PeerPessimisticLocks, RAFT_INIT_LOG_INDEX, RAFT_INIT_LOG_TERM, SplitCheckTask, Transport, cmd_resp, - fsm::{apply::validate_batch_split, ApplyMetrics}, + fsm::{ApplyMetrics, apply::validate_batch_split}, metrics::PEER_ADMIN_CMD_COUNTER, snap::TABLET_SNAPSHOT_VERSION, util::{self, KeysInfoFormatter}, - PeerPessimisticLocks, SplitCheckTask, Transport, RAFT_INIT_LOG_INDEX, RAFT_INIT_LOG_TERM, }, - Result, }; use slog::{error, info, warn}; use tikv_util::{box_err, log::SlogFormat, slog_panic, time::Instant}; use crate::{ + Error, batch::StoreContext, fsm::{ApplyResReporter, PeerFsmDelegate}, operation::{AdminCmdResult, SharedReadTablet}, raft::{Apply, Peer}, router::{CmdResChannel, PeerMsg, PeerTick, StoreMsg}, worker::tablet, - Error, }; pub const SPLIT_PREFIX: &str = "split"; @@ -1005,8 +1005,8 @@ mod test { kv::{KvTestEngine, TestTabletFactory}, }; use engine_traits::{ - FlushState, Peekable, SstApplyState, TabletContext, TabletRegistry, WriteBatch, CF_DEFAULT, - DATA_CFS, + CF_DEFAULT, DATA_CFS, FlushState, Peekable, SstApplyState, TabletContext, TabletRegistry, + WriteBatch, }; use futures::executor::block_on; use kvproto::{ @@ -1016,7 +1016,7 @@ mod test { }; use raftstore::{ coprocessor::CoprocessorHost, - store::{cmd_resp::new_error, Config}, + store::{Config, cmd_resp::new_error}, }; use slog::o; use tempfile::TempDir; @@ -1028,7 +1028,7 @@ mod test { use super::*; use crate::{ - operation::test_util::{create_tmp_importer, MockReporter}, + operation::test_util::{MockReporter, create_tmp_importer}, raft::Apply, }; diff --git a/components/raftstore-v2/src/operation/command/admin/transfer_leader.rs b/components/raftstore-v2/src/operation/command/admin/transfer_leader.rs index c56128150ec..fa603f1761a 100644 --- a/components/raftstore-v2/src/operation/command/admin/transfer_leader.rs +++ b/components/raftstore-v2/src/operation/command/admin/transfer_leader.rs @@ -10,13 +10,13 @@ use kvproto::{ AdminCmdType, AdminRequest, AdminResponse, RaftCmdRequest, TransferLeaderRequest, }, }; -use raft::{eraftpb, ProgressState, Storage}; +use raft::{ProgressState, Storage, eraftpb}; use raftstore::{ + Result, store::{ - entry_storage::CacheWarmupState, fsm::new_admin_request, make_transfer_leader_response, - metrics::PEER_ADMIN_CMD_COUNTER, Config, TransferLeaderContext, Transport, + Config, TransferLeaderContext, Transport, entry_storage::CacheWarmupState, + fsm::new_admin_request, make_transfer_leader_response, metrics::PEER_ADMIN_CMD_COUNTER, }, - Result, }; use rand::prelude::SliceRandom; use slog::info; diff --git a/components/raftstore-v2/src/operation/command/control.rs b/components/raftstore-v2/src/operation/command/control.rs index f05c9ca5297..f6c61ae6df3 100644 --- a/components/raftstore-v2/src/operation/command/control.rs +++ b/components/raftstore-v2/src/operation/command/control.rs @@ -4,16 +4,16 @@ use std::{collections::LinkedList, mem}; use kvproto::{metapb, raft_cmdpb::AdminCmdType}; use raftstore::{ + Error, store::{ cmd_resp, fsm::apply, msg::ErrorCallback, util::{ - admin_cmd_epoch_lookup, AdminCmdEpochState, NORMAL_REQ_CHECK_CONF_VER, - NORMAL_REQ_CHECK_VER, + AdminCmdEpochState, NORMAL_REQ_CHECK_CONF_VER, NORMAL_REQ_CHECK_VER, + admin_cmd_epoch_lookup, }, }, - Error, }; use crate::router::CmdResChannel; diff --git a/components/raftstore-v2/src/operation/command/mod.rs b/components/raftstore-v2/src/operation/command/mod.rs index 8c372fdb278..2d8b55ae952 100644 --- a/components/raftstore-v2/src/operation/command/mod.rs +++ b/components/raftstore-v2/src/operation/command/mod.rs @@ -18,7 +18,7 @@ use std::{ mem, - sync::{atomic::Ordering, Arc}, + sync::{Arc, atomic::Ordering}, time::Duration, }; @@ -30,12 +30,13 @@ use kvproto::raft_cmdpb::{ use raft::eraftpb::{ConfChange, ConfChangeV2, Entry, EntryType}; use raft_proto::ConfChangeI; use raftstore::{ + Error, Result, coprocessor::ObserveLevel, store::{ - cmd_resp, + Config, ProposalContext, Transport, WriteCallback, cmd_resp, fsm::{ - apply::{self, APPLY_WB_SHRINK_SIZE, SHRINK_PENDING_CMD_QUEUE_CAP}, Proposal, + apply::{self, APPLY_WB_SHRINK_SIZE, SHRINK_PENDING_CMD_QUEUE_CAP}, }, local_metrics::RaftMetrics, metrics::{ @@ -43,16 +44,14 @@ use raftstore::{ }, msg::ErrorCallback, util::{self, check_flashback_state}, - Config, ProposalContext, Transport, WriteCallback, }, - Error, Result, }; use slog::{debug, error, warn}; use tikv_util::{ box_err, log::SlogFormat, slog_panic, - time::{duration_to_sec, monotonic_raw_now, Instant}, + time::{Instant, duration_to_sec, monotonic_raw_now}, }; use crate::{ @@ -67,9 +66,9 @@ mod control; mod write; pub use admin::{ - merge_source_path, report_split_init_finish, temp_split_path, AdminCmdResult, CatchUpLogs, - CompactLogContext, MergeContext, RequestHalfSplit, RequestSplit, SplitFlowControl, SplitInit, - SplitPendingAppend, MERGE_IN_PROGRESS_PREFIX, MERGE_SOURCE_PREFIX, SPLIT_PREFIX, + AdminCmdResult, CatchUpLogs, CompactLogContext, MERGE_IN_PROGRESS_PREFIX, MERGE_SOURCE_PREFIX, + MergeContext, RequestHalfSplit, RequestSplit, SPLIT_PREFIX, SplitFlowControl, SplitInit, + SplitPendingAppend, merge_source_path, report_split_init_finish, temp_split_path, }; pub use control::ProposalControl; use pd_client::{BucketMeta, BucketStat}; @@ -659,7 +658,7 @@ impl Apply { Ok(decoder) => { fail::fail_point!( "on_apply_write_cmd", - cfg!(release) || self.peer_id() == 3, + !cfg!(debug_assertions) || self.peer_id() == 3, |_| { unimplemented!(); } diff --git a/components/raftstore-v2/src/operation/command/write/ingest.rs b/components/raftstore-v2/src/operation/command/write/ingest.rs index 147bd83312f..5249c78bae6 100644 --- a/components/raftstore-v2/src/operation/command/write/ingest.rs +++ b/components/raftstore-v2/src/operation/command/write/ingest.rs @@ -2,12 +2,12 @@ use collections::HashMap; use crossbeam::channel::TrySendError; -use engine_traits::{data_cf_offset, KvEngine, RaftEngine, DATA_CFS_LEN}; +use engine_traits::{DATA_CFS_LEN, KvEngine, RaftEngine, data_cf_offset}; use kvproto::import_sstpb::SstMeta; use pd_client::metrics::STORE_SIZE_EVENT_INT_VEC; use raftstore::{ - store::{check_sst_for_ingestion, metrics::PEER_WRITE_CMD_COUNTER, util}, Result, + store::{check_sst_for_ingestion, metrics::PEER_WRITE_CMD_COUNTER, util}, }; use slog::{error, info}; use sst_importer::range_overlaps; diff --git a/components/raftstore-v2/src/operation/command/write/mod.rs b/components/raftstore-v2/src/operation/command/write/mod.rs index 5806614e192..88e37f0f921 100644 --- a/components/raftstore-v2/src/operation/command/write/mod.rs +++ b/components/raftstore-v2/src/operation/command/write/mod.rs @@ -1,32 +1,31 @@ // Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. use engine_traits::{ - data_cf_offset, name_to_cf, KvEngine, Mutable, RaftEngine, ALL_CFS, CF_DEFAULT, + ALL_CFS, CF_DEFAULT, KvEngine, Mutable, RaftEngine, data_cf_offset, name_to_cf, }; use fail::fail_point; use futures::channel::oneshot; use kvproto::raft_cmdpb::RaftRequestHeader; use raftstore::{ + Error, Result, store::{ - cmd_resp, - fsm::{apply, MAX_PROPOSAL_SIZE_RATIO}, + RaftCmdExtraOpts, cmd_resp, + fsm::{MAX_PROPOSAL_SIZE_RATIO, apply}, metrics::PEER_WRITE_CMD_COUNTER, msg::ErrorCallback, util::{self}, - RaftCmdExtraOpts, }, - Error, Result, }; use slog::{error, info}; use tikv_util::{box_err, slog_panic, time::Instant}; use crate::{ + TabletTask, batch::StoreContext, fsm::ApplyResReporter, operation::SimpleWriteReqEncoder, raft::{Apply, Peer}, router::{ApplyTask, CmdResChannel}, - TabletTask, }; mod ingest; @@ -254,7 +253,7 @@ impl Apply { cf = CF_DEFAULT; } - if !ALL_CFS.iter().any(|x| *x == cf) { + if !ALL_CFS.contains(&cf) { return Err(box_err!("invalid delete range command, cf: {:?}", cf)); } @@ -316,7 +315,7 @@ mod test { kv::{KvTestEngine, TestTabletFactory}, }; use engine_traits::{ - FlushState, Peekable, SstApplyState, TabletContext, TabletRegistry, CF_DEFAULT, DATA_CFS, + CF_DEFAULT, DATA_CFS, FlushState, Peekable, SstApplyState, TabletContext, TabletRegistry, }; use futures::executor::block_on; use kvproto::{ @@ -331,14 +330,14 @@ mod test { use tempfile::TempDir; use tikv_util::{ store::new_peer, - worker::{dummy_scheduler, Worker}, + worker::{Worker, dummy_scheduler}, yatp_pool::{DefaultTicker, YatpPoolBuilder}, }; use crate::{ operation::{ - test_util::{create_tmp_importer, new_delete_range_entry, new_put_entry, MockReporter}, CommittedEntries, + test_util::{MockReporter, create_tmp_importer, new_delete_range_entry, new_put_entry}, }, raft::Apply, worker::tablet, diff --git a/components/raftstore-v2/src/operation/life.rs b/components/raftstore-v2/src/operation/life.rs index 6dd0589b27c..38347375311 100644 --- a/components/raftstore-v2/src/operation/life.rs +++ b/components/raftstore-v2/src/operation/life.rs @@ -40,23 +40,23 @@ use kvproto::{ }; use raft::eraftpb::MessageType; use raftstore::{ + Error, Result, store::{ + DiskFullPeers, Transport, WriteTask, fsm::{ - apply, + Proposal, apply, life::{build_peer_destroyed_report, forward_destroy_to_source_peer}, - Proposal, }, local_metrics::IoType as InspectIoType, metrics::RAFT_PEER_PENDING_DURATION, - util, DiskFullPeers, Transport, WriteTask, + util, }, - Error, Result, }; use slog::{debug, error, info, warn}; use tikv_util::{ store::find_peer, sys::disk::DiskUsage, - time::{duration_to_sec, Instant}, + time::{Instant, duration_to_sec}, }; use super::command::SplitInit; @@ -251,7 +251,7 @@ fn check_if_to_peer_destroyed( .get_removed_records() .iter() .find(|p| p.get_store_id() == store_id) - .map_or(false, |p| to_peer.id <= p.get_id()) + .is_some_and(|p| to_peer.id <= p.get_id()) { return Ok(true); } diff --git a/components/raftstore-v2/src/operation/misc.rs b/components/raftstore-v2/src/operation/misc.rs index 0509722ebb3..8be6ea51b42 100644 --- a/components/raftstore-v2/src/operation/misc.rs +++ b/components/raftstore-v2/src/operation/misc.rs @@ -7,14 +7,15 @@ use std::collections::{ use collections::HashMap; use crossbeam::channel::TrySendError; -use engine_traits::{KvEngine, RaftEngine, CF_DEFAULT, CF_WRITE}; +use engine_traits::{CF_DEFAULT, CF_WRITE, KvEngine, RaftEngine}; use raftstore::{ - store::{CompactThreshold, TabletSnapKey}, Result, + store::{CompactThreshold, TabletSnapKey}, }; use slog::{debug, error, info}; use crate::{ + CompactTask::CheckAndCompact, batch::StoreContext, fsm::{Store, StoreFsmDelegate}, router::{PeerMsg, StoreTick}, @@ -22,7 +23,6 @@ use crate::{ cleanup::{self}, tablet, }, - CompactTask::CheckAndCompact, }; impl<'a, EK: KvEngine, ER: RaftEngine, T> StoreFsmDelegate<'a, EK, ER, T> { diff --git a/components/raftstore-v2/src/operation/mod.rs b/components/raftstore-v2/src/operation/mod.rs index df82f1abfe9..77ff21453b7 100644 --- a/components/raftstore-v2/src/operation/mod.rs +++ b/components/raftstore-v2/src/operation/mod.rs @@ -12,17 +12,16 @@ mod txn_ext; mod unsafe_recovery; pub use command::{ - merge_source_path, AdminCmdResult, ApplyFlowControl, CatchUpLogs, CommittedEntries, - CompactLogContext, MergeContext, ProposalControl, RequestHalfSplit, RequestSplit, - SimpleWriteBinary, SimpleWriteEncoder, SimpleWriteReqDecoder, SimpleWriteReqEncoder, - SplitFlowControl, SplitPendingAppend, MERGE_IN_PROGRESS_PREFIX, MERGE_SOURCE_PREFIX, - SPLIT_PREFIX, + AdminCmdResult, ApplyFlowControl, CatchUpLogs, CommittedEntries, CompactLogContext, + MERGE_IN_PROGRESS_PREFIX, MERGE_SOURCE_PREFIX, MergeContext, ProposalControl, RequestHalfSplit, + RequestSplit, SPLIT_PREFIX, SimpleWriteBinary, SimpleWriteEncoder, SimpleWriteReqDecoder, + SimpleWriteReqEncoder, SplitFlowControl, SplitPendingAppend, merge_source_path, }; pub use disk_snapshot_backup::UnimplementedHandle as DiskSnapBackupHandle; pub use life::{AbnormalPeerContext, DestroyProgress, GcPeerContext}; pub use ready::{ - write_initial_states, ApplyTrace, AsyncWriter, DataTrace, GenSnapTask, ReplayWatch, SnapState, - StateStorage, + ApplyTrace, AsyncWriter, DataTrace, GenSnapTask, ReplayWatch, SnapState, StateStorage, + write_initial_states, }; pub(crate) use self::{ @@ -34,11 +33,11 @@ pub(crate) use self::{ #[cfg(test)] pub mod test_util { use std::sync::{ - mpsc::{channel, Receiver, Sender}, Arc, + mpsc::{Receiver, Sender, channel}, }; - use engine_traits::{CfName, KvEngine, CF_DEFAULT}; + use engine_traits::{CF_DEFAULT, CfName, KvEngine}; use kvproto::{kvrpcpb::ApiVersion, metapb::RegionEpoch, raft_cmdpb::RaftRequestHeader}; use raft::prelude::{Entry, EntryType}; use raftstore::store::simple_write::SimpleWriteEncoder; diff --git a/components/raftstore-v2/src/operation/pd.rs b/components/raftstore-v2/src/operation/pd.rs index 8e392755c5e..864786df641 100644 --- a/components/raftstore-v2/src/operation/pd.rs +++ b/components/raftstore-v2/src/operation/pd.rs @@ -7,7 +7,7 @@ use std::sync::atomic::Ordering; use engine_traits::{KvEngine, RaftEngine}; use fail::fail_point; use kvproto::{metapb, pdpb}; -use raftstore::store::{metrics::STORE_SNAPSHOT_TRAFFIC_GAUGE_VEC, Transport}; +use raftstore::store::{Transport, metrics::STORE_SNAPSHOT_TRAFFIC_GAUGE_VEC}; use slog::{debug, error}; use tikv_util::{slog_panic, time::Instant}; diff --git a/components/raftstore-v2/src/operation/query/capture.rs b/components/raftstore-v2/src/operation/query/capture.rs index 4b6a8c24fc3..f3967bc7980 100644 --- a/components/raftstore-v2/src/operation/query/capture.rs +++ b/components/raftstore-v2/src/operation/query/capture.rs @@ -8,14 +8,14 @@ use kvproto::raft_cmdpb::{RaftCmdRequest, RaftCmdResponse}; use raftstore::{ coprocessor::{Cmd, CmdBatch, ObserveHandle, ObserveLevel}, store::{ - cmd_resp, + RegionSnapshot, cmd_resp, fsm::{ - apply::{notify_stale_req_with_msg, ObserverType, SHRINK_PENDING_CMD_QUEUE_CAP}, - new_read_index_request, ChangeObserver, + ChangeObserver, + apply::{ObserverType, SHRINK_PENDING_CMD_QUEUE_CAP, notify_stale_req_with_msg}, + new_read_index_request, }, msg::ErrorCallback, util::compare_region_epoch, - RegionSnapshot, }, }; use slog::info; @@ -24,7 +24,7 @@ use txn_types::WriteBatchFlags; use crate::{ fsm::{ApplyResReporter, PeerFsmDelegate}, raft::Apply, - router::{message::CaptureChange, ApplyTask, QueryResChannel, QueryResult}, + router::{ApplyTask, QueryResChannel, QueryResult, message::CaptureChange}, }; impl<'a, EK: KvEngine, ER: RaftEngine, T: raftstore::store::Transport> @@ -185,8 +185,8 @@ impl Apply { #[cfg(test)] mod test { use std::sync::{ - mpsc::{channel, Receiver, Sender}, Arc, Mutex, + mpsc::{Receiver, Sender, channel}, }; use engine_test::{ @@ -194,7 +194,7 @@ mod test { kv::{KvTestEngine, TestTabletFactory}, }; use engine_traits::{ - FlushState, Peekable, SstApplyState, TabletContext, TabletRegistry, CF_DEFAULT, DATA_CFS, + CF_DEFAULT, DATA_CFS, FlushState, Peekable, SstApplyState, TabletContext, TabletRegistry, }; use futures::executor::block_on; use kvproto::{ @@ -217,8 +217,8 @@ mod test { use super::*; use crate::{ operation::{ - test_util::{create_tmp_importer, new_put_entry, MockReporter}, CommittedEntries, + test_util::{MockReporter, create_tmp_importer, new_put_entry}, }, raft::Apply, router::build_any_channel, diff --git a/components/raftstore-v2/src/operation/query/lease.rs b/components/raftstore-v2/src/operation/query/lease.rs index 27b69d2cabe..9e530804467 100644 --- a/components/raftstore-v2/src/operation/query/lease.rs +++ b/components/raftstore-v2/src/operation/query/lease.rs @@ -5,21 +5,20 @@ use std::sync::Mutex; use engine_traits::{KvEngine, RaftEngine}; use kvproto::raft_cmdpb::{RaftCmdRequest, RaftRequestHeader}; use raft::{ - eraftpb::{self, MessageType}, Storage, + eraftpb::{self, MessageType}, }; use raftstore::{ + Error, Result, store::{ - can_amend_read, cmd_resp, + ReadDelegate, ReadIndexRequest, ReadProgress, Transport, can_amend_read, cmd_resp, fsm::{apply::notify_stale_req, new_read_index_request}, metrics::RAFT_READ_INDEX_PENDING_COUNT, msg::{ErrorCallback, ReadCallback}, propose_read_index, should_renew_lease, simple_write::SimpleWriteEncoder, - util::{check_req_region_epoch, LeaseState}, - ReadDelegate, ReadIndexRequest, ReadProgress, Transport, + util::{LeaseState, check_req_region_epoch}, }, - Error, Result, }; use slog::debug; use tikv_util::time::monotonic_raw_now; @@ -76,7 +75,7 @@ impl Peer { pub fn pre_read_index(&self) -> Result<()> { fail::fail_point!("before_propose_readindex", |s| if s - .map_or(true, |s| s.parse().unwrap_or(true)) + .is_none_or(|s| s.parse().unwrap_or(true)) { Ok(()) } else { @@ -344,17 +343,17 @@ impl Peer { None => return false, }; let max_lease = ctx.cfg.raft_store_max_leader_lease(); - let has_overlapped_reads = self.pending_reads().back().map_or(false, |read| { + let has_overlapped_reads = self.pending_reads().back().is_some_and(|read| { // If there is any read index whose lease can cover till next heartbeat // then we don't need to propose a new one read.propose_time + max_lease > renew_bound }); - let has_overlapped_writes = self.proposals().back().map_or(false, |proposal| { + let has_overlapped_writes = self.proposals().back().is_some_and(|proposal| { // If there is any write whose lease can cover till next heartbeat // then we don't need to propose a new one proposal .propose_time - .map_or(false, |propose_time| propose_time + max_lease > renew_bound) + .is_some_and(|propose_time| propose_time + max_lease > renew_bound) }); !has_overlapped_reads && !has_overlapped_writes } diff --git a/components/raftstore-v2/src/operation/query/local.rs b/components/raftstore-v2/src/operation/query/local.rs index 860d43ce514..7d7933d1765 100644 --- a/components/raftstore-v2/src/operation/query/local.rs +++ b/components/raftstore-v2/src/operation/query/local.rs @@ -4,7 +4,7 @@ use std::{ num::NonZeroU64, ops::Deref, - sync::{atomic, Arc, Mutex}, + sync::{Arc, Mutex, atomic}, }; use batch_system::Router; @@ -16,25 +16,24 @@ use kvproto::{ raft_cmdpb::{CmdType, RaftCmdRequest, RaftCmdResponse}, }; use raftstore::{ + Result, errors::RAFTSTORE_IS_BUSY, store::{ - cmd_resp, + LocalReaderCore, ReadDelegate, ReadExecutorProvider, RegionSnapshot, cmd_resp, util::LeaseState, worker_metrics::{self, TLS_LOCAL_READ_METRICS}, - LocalReaderCore, ReadDelegate, ReadExecutorProvider, RegionSnapshot, }, - Result, }; -use slog::{debug, Logger}; -use tikv_util::{box_err, codec::number::decode_u64, time::monotonic_raw_now, Either}; +use slog::{Logger, debug}; +use tikv_util::{Either, box_err, codec::number::decode_u64, time::monotonic_raw_now}; use time::Timespec; -use tracker::{get_tls_tracker_token, GLOBAL_TRACKERS}; +use tracker::{GLOBAL_TRACKERS, get_tls_tracker_token}; use txn_types::WriteBatchFlags; use crate::{ + StoreRouter, fsm::StoreMeta, router::{PeerMsg, QueryResult}, - StoreRouter, }; pub trait MsgRouter: Clone + Send + 'static { @@ -57,15 +56,15 @@ pub type ReadDelegatePair = (ReadDelegate, SharedReadTablet); /// /// Though it looks like `CachedTablet`, but there are subtle differences. /// 1. `CachedTablet` always hold the latest version of the tablet. But -/// `SharedReadTablet` should only hold the tablet that matches epoch. So it -/// will be updated only when the epoch is updated. +/// `SharedReadTablet` should only hold the tablet that matches epoch. So it +/// will be updated only when the epoch is updated. /// 2. `SharedReadTablet` should always hold a tablet and the same tablet. If -/// tablet is taken, then it should be considered as stale and should check -/// again epoch to load the new `SharedReadTablet`. +/// tablet is taken, then it should be considered as stale and should check +/// again epoch to load the new `SharedReadTablet`. /// 3. `SharedReadTablet` may be cloned into thread local. So its cache should -/// be released as soon as possible, so there should be no strong reference -/// that prevents tablet from being dropped after it's marked as stale by other -/// threads. +/// be released as soon as possible, so there should be no strong reference +/// that prevents tablet from being dropped after it's marked as stale by +/// other threads. pub struct SharedReadTablet { tablet: Arc>>, cache: Option, @@ -366,7 +365,7 @@ where || res .get_responses() .first() - .map_or(false, |r| r.get_read_index().has_locked()), + .is_some_and(|r| r.get_read_index().has_locked()), "{:?}", res ); @@ -667,13 +666,13 @@ mod tests { ctor::{CfOptions, DbOptions}, kv::{KvTestEngine, TestTabletFactory}, }; - use engine_traits::{MiscExt, SyncMutable, TabletContext, TabletRegistry, DATA_CFS}; + use engine_traits::{DATA_CFS, MiscExt, SyncMutable, TabletContext, TabletRegistry}; use futures::executor::block_on; use kvproto::{kvrpcpb::ExtraOp as TxnExtraOp, metapb, raft_cmdpb::*}; use pd_client::BucketMeta; use raftstore::store::{ - util::Lease, worker_metrics::TLS_LOCAL_READ_METRICS, ReadCallback, ReadProgress, - RegionReadProgress, TrackVer, TxnExt, + ReadCallback, ReadProgress, RegionReadProgress, TrackVer, TxnExt, util::Lease, + worker_metrics::TLS_LOCAL_READ_METRICS, }; use slog::o; use tempfile::Builder; @@ -851,7 +850,7 @@ mod tests { TLS_LOCAL_READ_METRICS.with(|m| m.borrow().reject_reason.cache_miss.get()), 1 ); - assert!(reader.local_reader.delegates.get(&1).is_none()); + assert!(!reader.local_reader.delegates.contains_key(&1)); // Register region 1 lease.renew(monotonic_raw_now()); diff --git a/components/raftstore-v2/src/operation/query/mod.rs b/components/raftstore-v2/src/operation/query/mod.rs index 10f6e3279c3..87e7611b372 100644 --- a/components/raftstore-v2/src/operation/query/mod.rs +++ b/components/raftstore-v2/src/operation/query/mod.rs @@ -21,13 +21,13 @@ use kvproto::{ }; use raft::{Ready, StateRole}; use raftstore::{ + Error, Result, errors::RAFTSTORE_IS_BUSY, store::{ - cmd_resp, local_metrics::RaftMetrics, metrics::RAFT_READ_INDEX_PENDING_COUNT, - msg::ErrorCallback, region_meta::RegionMeta, util, util::LeaseState, GroupState, - ReadIndexContext, ReadProgress, RequestPolicy, + GroupState, ReadIndexContext, ReadProgress, RequestPolicy, cmd_resp, + local_metrics::RaftMetrics, metrics::RAFT_READ_INDEX_PENDING_COUNT, msg::ErrorCallback, + region_meta::RegionMeta, util, util::LeaseState, }, - Error, Result, }; use slog::{debug, info}; use tikv_util::{box_err, log::SlogFormat}; @@ -38,7 +38,7 @@ use crate::{ fsm::PeerFsmDelegate, raft::Peer, router::{ - message::RaftRequest, DebugInfoChannel, PeerMsg, QueryResChannel, QueryResult, ReadResponse, + DebugInfoChannel, PeerMsg, QueryResChannel, QueryResult, ReadResponse, message::RaftRequest, }, }; diff --git a/components/raftstore-v2/src/operation/query/replica.rs b/components/raftstore-v2/src/operation/query/replica.rs index 8dab933f5ed..48451a90e93 100644 --- a/components/raftstore-v2/src/operation/query/replica.rs +++ b/components/raftstore-v2/src/operation/query/replica.rs @@ -4,14 +4,14 @@ use engine_traits::{KvEngine, RaftEngine}; use kvproto::raft_cmdpb::{self, RaftCmdRequest, RaftCmdResponse}; use pd_client::INVALID_ID; use raftstore::{ + Error, store::{ - cmd_resp, + Config, ReadIndexContext, ReadIndexRequest, cmd_resp, fsm::apply::notify_stale_req, metrics::RAFT_READ_INDEX_PENDING_COUNT, msg::{ErrorCallback, ReadCallback}, - propose_read_index, Config, ReadIndexContext, ReadIndexRequest, + propose_read_index, }, - Error, }; use slog::debug; use tikv_util::time::monotonic_raw_now; diff --git a/components/raftstore-v2/src/operation/ready/apply_trace.rs b/components/raftstore-v2/src/operation/ready/apply_trace.rs index 53756465cc4..1ab961938d0 100644 --- a/components/raftstore-v2/src/operation/ready/apply_trace.rs +++ b/components/raftstore-v2/src/operation/ready/apply_trace.rs @@ -17,11 +17,13 @@ //! `ApplyTrace::maybe_advance_admin_flushed`. //! //! The correctness of raft cf index relies on the fact that: +//! //! - apply is sequential, so if any apply index is updated to apply trace, all //! modification events before that must be processed. //! - admin commands that marked by raft cf index must flush all data before //! being executed. Note this contraint is not just for recovery, but also //! necessary to guarantee safety of operations like split init or log gc. +//! //! So data of logs before raft cf index must be applied and flushed to disk. //! //! All apply related states are associated with an apply index. During @@ -31,13 +33,13 @@ use std::{ cmp, collections::VecDeque, path::Path, - sync::{atomic::Ordering, mpsc::SyncSender, Mutex}, + sync::{Mutex, atomic::Ordering, mpsc::SyncSender}, }; use encryption_export::DataKeyManager; use engine_traits::{ - data_cf_offset, offset_to_cf, ApplyProgress, KvEngine, RaftEngine, RaftLogBatch, - TabletRegistry, ALL_CFS, CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE, DATA_CFS, DATA_CFS_LEN, + ALL_CFS, ApplyProgress, CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE, DATA_CFS, DATA_CFS_LEN, + KvEngine, RaftEngine, RaftLogBatch, TabletRegistry, data_cf_offset, offset_to_cf, }; use fail::fail_point; use kvproto::{ @@ -45,12 +47,13 @@ use kvproto::{ raft_serverpb::{PeerState, RaftApplyState, RaftLocalState, RegionLocalState}, }; use raftstore::store::{ - util, ReadTask, TabletSnapManager, WriteTask, RAFT_INIT_LOG_INDEX, RAFT_INIT_LOG_TERM, + RAFT_INIT_LOG_INDEX, RAFT_INIT_LOG_TERM, ReadTask, TabletSnapManager, WriteTask, util, }; -use slog::{info, trace, warn, Logger}; +use slog::{Logger, info, trace, warn}; use tikv_util::{box_err, slog_panic, worker::Scheduler}; use crate::{ + Result, StoreRouter, batch::StoreContext, operation::{ command::temp_split_path, @@ -59,7 +62,6 @@ use crate::{ raft::{Peer, Storage}, router::{PeerMsg, SstApplyIndex}, worker::tablet, - Result, StoreRouter, }; /// Write states for the given region. The region is supposed to have all its diff --git a/components/raftstore-v2/src/operation/ready/async_writer.rs b/components/raftstore-v2/src/operation/ready/async_writer.rs index c2a9427580a..0c55371546e 100644 --- a/components/raftstore-v2/src/operation/ready/async_writer.rs +++ b/components/raftstore-v2/src/operation/ready/async_writer.rs @@ -5,10 +5,10 @@ use std::collections::VecDeque; use engine_traits::{KvEngine, RaftEngine}; use kvproto::{metapb::RegionEpoch, raft_serverpb::RaftMessage}; use raftstore::store::{ - local_metrics::RaftMetrics, Config, PersistedNotifier, WriteRouter, WriteRouterContext, - WriteSenders, WriteTask, + Config, PersistedNotifier, WriteRouter, WriteRouterContext, WriteSenders, WriteTask, + local_metrics::RaftMetrics, }; -use slog::{warn, Logger}; +use slog::{Logger, warn}; use tikv_util::slog_panic; use crate::{ @@ -120,7 +120,7 @@ impl AsyncWriter { } let last_unpersisted = self.unpersisted_readies.back(); - if last_unpersisted.map_or(true, |u| u.number < ready_number) { + if last_unpersisted.is_none_or(|u| u.number < ready_number) { slog_panic!( logger, "ready number is too large"; diff --git a/components/raftstore-v2/src/operation/ready/mod.rs b/components/raftstore-v2/src/operation/ready/mod.rs index 3603f03078c..3a1a58d9cf9 100644 --- a/components/raftstore-v2/src/operation/ready/mod.rs +++ b/components/raftstore-v2/src/operation/ready/mod.rs @@ -25,42 +25,42 @@ use std::{ cmp, fmt::{self, Debug, Formatter}, sync::{ - atomic::{AtomicUsize, Ordering}, Arc, + atomic::{AtomicUsize, Ordering}, }, time::Instant, }; -use engine_traits::{KvEngine, RaftEngine, DATA_CFS}; +use engine_traits::{DATA_CFS, KvEngine, RaftEngine}; use error_code::ErrorCodeExt; use kvproto::{ raft_cmdpb::AdminCmdType, raft_serverpb::{ExtraMessageType, RaftMessage}, }; use protobuf::Message as _; -use raft::{eraftpb, prelude::MessageType, Ready, SnapshotStatus, StateRole, INVALID_ID}; +use raft::{INVALID_ID, Ready, SnapshotStatus, StateRole, eraftpb, prelude::MessageType}; use raftstore::{ coprocessor::{RegionChangeEvent, RoleChange}, store::{ + FetchedLogs, ReadProgress, Transport, WriteCallback, WriteTask, fsm::store::StoreRegionMeta, local_metrics::IoType, needs_evict_entry_cache, util::{self, is_first_append_entry, is_initial_msg}, worker_metrics::SNAP_COUNTER, - FetchedLogs, ReadProgress, Transport, WriteCallback, WriteTask, }, }; -use slog::{debug, error, info, warn, Logger}; +use slog::{Logger, debug, error, info, warn}; use tikv_util::{ log::SlogFormat, slog_panic, store::find_peer, sys::disk::DiskUsage, - time::{duration_to_sec, monotonic_raw_now, Duration, Instant as TiInstant}, + time::{Duration, Instant as TiInstant, duration_to_sec, monotonic_raw_now}, }; pub use self::{ - apply_trace::{write_initial_states, ApplyTrace, DataTrace, StateStorage}, + apply_trace::{ApplyTrace, DataTrace, StateStorage, write_initial_states}, async_writer::AsyncWriter, snapshot::{GenSnapTask, SnapState}, }; @@ -966,7 +966,7 @@ impl Peer { let now = Instant::now(); for i in old_index + 1..=new_index { if let Some((term, trackers)) = self.proposals().find_trackers(i) { - if self.entry_storage().term(i).map_or(false, |t| t == term) { + if self.entry_storage().term(i) == Ok(term) { for tracker in trackers { tracker.observe(now, &ctx.raft_metrics.wf_persist_log, |t| { &mut t.metrics.wf_persist_log_nanos @@ -991,7 +991,7 @@ impl Peer { let health_stats = &mut ctx.raft_metrics.health_stats; for i in old_index + 1..=new_index { if let Some((term, trackers)) = self.proposals().find_trackers(i) { - if self.entry_storage().term(i).map_or(false, |t| t == term) { + if self.entry_storage().term(i) == Ok(term) { let commit_persisted = i <= self.persisted_index(); let hist = if commit_persisted { &ctx.raft_metrics.wf_commit_log @@ -1260,7 +1260,7 @@ impl Peer { disk_full_peers.is_empty() || disk_full_peers .get(peer_id) - .map_or(true, |x| x != msg.disk_usage) + .is_none_or(|x| x != msg.disk_usage) }; if refill_disk_usages || self.has_region_merge_proposal { diff --git a/components/raftstore-v2/src/operation/ready/snapshot.rs b/components/raftstore-v2/src/operation/ready/snapshot.rs index b6a02d70eac..7393ecd158e 100644 --- a/components/raftstore-v2/src/operation/ready/snapshot.rs +++ b/components/raftstore-v2/src/operation/ready/snapshot.rs @@ -24,38 +24,38 @@ use std::{ fs, path::{Path, PathBuf}, sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, }, }; use encryption_export::DataKeyManager; -use engine_traits::{KvEngine, RaftEngine, RaftLogBatch, TabletContext, TabletRegistry, ALL_CFS}; +use engine_traits::{ALL_CFS, KvEngine, RaftEngine, RaftLogBatch, TabletContext, TabletRegistry}; use fail::fail_point; use kvproto::{ metapb::PeerRole, raft_serverpb::{PeerState, RaftSnapshotData}, }; use protobuf::Message; -use raft::{eraftpb::Snapshot, StateRole}; +use raft::{StateRole, eraftpb::Snapshot}; use raftstore::{ coprocessor::RegionChangeEvent, store::{ + GenSnapRes, RAFT_INIT_LOG_INDEX, RAFT_INIT_LOG_TERM, ReadTask, TabletSnapKey, + TabletSnapManager, Transport, WriteTask, metrics::STORE_SNAPSHOT_VALIDATION_FAILURE_COUNTER, worker_metrics::SNAP_COUNTER, - GenSnapRes, ReadTask, TabletSnapKey, TabletSnapManager, Transport, WriteTask, - RAFT_INIT_LOG_INDEX, RAFT_INIT_LOG_TERM, }, }; use slog::{debug, error, info, warn}; use tikv_util::{box_err, log::SlogFormat, slog_panic, store::find_peer_by_id}; use crate::{ + Result, StoreContext, fsm::ApplyResReporter, - operation::{command::temp_split_path, SharedReadTablet}, + operation::{SharedReadTablet, command::temp_split_path}, raft::{Apply, Peer, Storage}, router::ApplyTask, worker::tablet, - Result, StoreContext, }; /// Snapshot generating task state. @@ -322,9 +322,10 @@ impl Peer { } self.read_progress_mut().update_applied_core(snapshot_index); let split = self.storage_mut().split_init_mut().take(); - if split.as_ref().map_or(true, |s| { - !s.scheduled || snapshot_index != RAFT_INIT_LOG_INDEX - }) { + if split + .as_ref() + .is_none_or(|s| !s.scheduled || snapshot_index != RAFT_INIT_LOG_INDEX) + { info!(self.logger, "apply tablet snapshot completely"); // Tablet sent from region leader should have already be trimmed. self.storage_mut().set_has_dirty_data(false); diff --git a/components/raftstore-v2/src/operation/txn_ext.rs b/components/raftstore-v2/src/operation/txn_ext.rs index ebb8a3bb2a0..1ae265e3b98 100644 --- a/components/raftstore-v2/src/operation/txn_ext.rs +++ b/components/raftstore-v2/src/operation/txn_ext.rs @@ -5,10 +5,10 @@ //! This is the temporary (efficient) solution, it should be implemented as one //! type of coprocessor. -use std::sync::{atomic::Ordering, Arc}; +use std::sync::{Arc, atomic::Ordering}; use crossbeam::atomic::AtomicCell; -use engine_traits::{KvEngine, RaftEngine, CF_LOCK}; +use engine_traits::{CF_LOCK, KvEngine, RaftEngine}; use kvproto::{ kvrpcpb::{DiskFullOpt, ExtraOp}, metapb::Region, @@ -19,14 +19,14 @@ use raft::eraftpb; use raftstore::store::{ LocksStatus, PeerPessimisticLocks, RaftCmdExtraOpts, TransferLeaderContext, TxnExt, }; -use slog::{error, info, warn, Logger}; +use slog::{Logger, error, info, warn}; use crate::{ + SimpleWriteEncoder, batch::StoreContext, raft::Peer, router::{PeerMsg, PeerTick}, worker::pd, - SimpleWriteEncoder, }; pub struct TxnContext { diff --git a/components/raftstore-v2/src/operation/unsafe_recovery/create.rs b/components/raftstore-v2/src/operation/unsafe_recovery/create.rs index c96f3dc55c5..2547f857966 100644 --- a/components/raftstore-v2/src/operation/unsafe_recovery/create.rs +++ b/components/raftstore-v2/src/operation/unsafe_recovery/create.rs @@ -7,15 +7,15 @@ use engine_traits::{KvEngine, RaftEngine, TabletContext}; use keys::{data_end_key, data_key, enc_start_key}; use kvproto::metapb::Region; use raftstore::store::{ - PeerPessimisticLocks, Transport, UnsafeRecoveryExecutePlanSyncer, UnsafeRecoveryState, - RAFT_INIT_LOG_INDEX, + PeerPessimisticLocks, RAFT_INIT_LOG_INDEX, Transport, UnsafeRecoveryExecutePlanSyncer, + UnsafeRecoveryState, }; use slog::{error, info, warn}; use crate::{ batch::StoreContext, fsm::Store, - operation::{command::temp_split_path, SplitInit}, + operation::{SplitInit, command::temp_split_path}, raft::Peer, router::PeerMsg, }; diff --git a/components/raftstore-v2/src/operation/unsafe_recovery/demote.rs b/components/raftstore-v2/src/operation/unsafe_recovery/demote.rs index 0eb722a94c7..1a39cc3cb79 100644 --- a/components/raftstore-v2/src/operation/unsafe_recovery/demote.rs +++ b/components/raftstore-v2/src/operation/unsafe_recovery/demote.rs @@ -3,8 +3,8 @@ use engine_traits::{KvEngine, RaftEngine}; use kvproto::metapb; use raftstore::store::{ - demote_failed_voters_request, exit_joint_request, Transport, UnsafeRecoveryExecutePlanSyncer, - UnsafeRecoveryState, + Transport, UnsafeRecoveryExecutePlanSyncer, UnsafeRecoveryState, demote_failed_voters_request, + exit_joint_request, }; use slog::{error, info, warn}; diff --git a/components/raftstore-v2/src/operation/unsafe_recovery/force_leader.rs b/components/raftstore-v2/src/operation/unsafe_recovery/force_leader.rs index b80cc645b84..2875d687386 100644 --- a/components/raftstore-v2/src/operation/unsafe_recovery/force_leader.rs +++ b/components/raftstore-v2/src/operation/unsafe_recovery/force_leader.rs @@ -4,9 +4,9 @@ use std::mem; use collections::HashSet; use engine_traits::{KvEngine, RaftEngine}; -use raft::{eraftpb::MessageType, StateRole, Storage}; +use raft::{StateRole, Storage, eraftpb::MessageType}; use raftstore::store::{ - util::LeaseState, ForceLeaderState, UnsafeRecoveryForceLeaderSyncer, UnsafeRecoveryState, + ForceLeaderState, UnsafeRecoveryForceLeaderSyncer, UnsafeRecoveryState, util::LeaseState, }; use slog::{info, warn}; use tikv_util::time::Instant as TiInstant; diff --git a/components/raftstore-v2/src/operation/unsafe_recovery/report.rs b/components/raftstore-v2/src/operation/unsafe_recovery/report.rs index db78c61a0e7..e89d8b41227 100644 --- a/components/raftstore-v2/src/operation/unsafe_recovery/report.rs +++ b/components/raftstore-v2/src/operation/unsafe_recovery/report.rs @@ -2,7 +2,7 @@ use engine_traits::{KvEngine, RaftEngine}; use kvproto::{pdpb, raft_serverpb::RegionLocalState}; -use raft::{GetEntriesContext, Storage, NO_LIMIT}; +use raft::{GetEntriesContext, NO_LIMIT, Storage}; use raftstore::store::{ ProposalContext, Transport, UnsafeRecoveryFillOutReportSyncer, UnsafeRecoveryState, UnsafeRecoveryWaitApplySyncer, diff --git a/components/raftstore-v2/src/raft/apply.rs b/components/raftstore-v2/src/raft/apply.rs index 35959dd8aea..af0b1a776e6 100644 --- a/components/raftstore-v2/src/raft/apply.rs +++ b/components/raftstore-v2/src/raft/apply.rs @@ -3,15 +3,15 @@ use std::{mem, sync::Arc}; use engine_traits::{ - FlushState, KvEngine, PerfContextKind, SstApplyState, TabletRegistry, WriteBatch, DATA_CFS_LEN, + DATA_CFS_LEN, FlushState, KvEngine, PerfContextKind, SstApplyState, TabletRegistry, WriteBatch, }; use kvproto::{metapb, raft_cmdpb::RaftCmdResponse, raft_serverpb::RegionLocalState}; use pd_client::BucketStat; use raftstore::{ coprocessor::{Cmd, CmdObserveInfo, CoprocessorHost, ObserveLevel}, store::{ - fsm::{apply::DEFAULT_APPLY_WB_SIZE, ApplyMetrics}, Config, ReadTask, + fsm::{ApplyMetrics, apply::DEFAULT_APPLY_WB_SIZE}, }, }; use slog::Logger; @@ -19,9 +19,9 @@ use sst_importer::SstImporter; use tikv_util::{log::SlogFormat, worker::Scheduler, yatp_pool::FuturePool}; use crate::{ + TabletTask, operation::{AdminCmdResult, ApplyFlowControl, DataTrace}, router::{CmdResChannel, SstApplyIndex}, - TabletTask, }; pub(crate) struct Observe { @@ -239,7 +239,7 @@ impl Apply { #[inline] pub fn set_tablet(&mut self, tablet: EK) { assert!( - self.write_batch.as_ref().map_or(true, |wb| wb.is_empty()), + self.write_batch.as_ref().is_none_or(|wb| wb.is_empty()), "{} setting tablet while still have dirty write batch", SlogFormat(&self.logger) ); @@ -290,7 +290,7 @@ impl Apply { #[inline] pub fn release_memory(&mut self) { mem::take(&mut self.key_buffer); - if self.write_batch.as_ref().map_or(false, |wb| wb.is_empty()) { + if self.write_batch.as_ref().is_some_and(|wb| wb.is_empty()) { self.write_batch = None; } } diff --git a/components/raftstore-v2/src/raft/peer.rs b/components/raftstore-v2/src/raft/peer.rs index 48d51ece654..bbae8035f4a 100644 --- a/components/raftstore-v2/src/raft/peer.rs +++ b/components/raftstore-v2/src/raft/peer.rs @@ -16,23 +16,24 @@ use kvproto::{ pdpb, raft_serverpb::RaftMessage, }; -use raft::{eraftpb, RawNode, StateRole}; +use raft::{RawNode, StateRole, eraftpb}; use raftstore::{ coprocessor::{CoprocessorHost, RegionChangeEvent, RegionChangeReason}, store::{ - fsm::ApplyMetrics, - metrics::RAFT_PEER_PENDING_DURATION, - util::{Lease, RegionReadProgress}, BucketStatsInfo, Config, EntryStorage, ForceLeaderState, PeerStat, ProposalQueue, ReadDelegate, ReadIndexQueue, ReadProgress, TabletSnapManager, TransferLeaderState, UnsafeRecoveryState, WriteTask, + fsm::ApplyMetrics, + metrics::RAFT_PEER_PENDING_DURATION, + util::{Lease, RegionReadProgress}, }, }; -use slog::{debug, info, Logger}; +use slog::{Logger, debug, info}; use tikv_util::{slog_panic, time::duration_to_sec}; use super::storage::Storage; use crate::{ + Result, batch::StoreContext, fsm::ApplyScheduler, operation::{ @@ -41,7 +42,6 @@ use crate::{ SplitPendingAppend, TxnContext, }, router::{ApplyTask, CmdResChannel, PeerTick, QueryResChannel}, - Result, }; const REGION_READ_PROGRESS_CAP: usize = 128; diff --git a/components/raftstore-v2/src/raft/storage.rs b/components/raftstore-v2/src/raft/storage.rs index 4b78bee9b46..8ece3e41e26 100644 --- a/components/raftstore-v2/src/raft/storage.rs +++ b/components/raftstore-v2/src/raft/storage.rs @@ -13,16 +13,16 @@ use kvproto::{ raft_serverpb::{RaftApplyState, RaftLocalState, RegionLocalState}, }; use raft::{ + GetEntriesContext, INVALID_ID, RaftState, eraftpb::{ConfState, Entry, Snapshot}, - GetEntriesContext, RaftState, INVALID_ID, }; -use raftstore::store::{local_metrics::RaftMetrics, util, EntryStorage, ReadTask}; -use slog::{o, Logger}; +use raftstore::store::{EntryStorage, ReadTask, local_metrics::RaftMetrics, util}; +use slog::{Logger, o}; use tikv_util::{box_err, store::find_peer, worker::Scheduler}; use crate::{ - operation::{ApplyTrace, GenSnapTask, SnapState, SplitInit}, Result, + operation::{ApplyTrace, GenSnapTask, SnapState, SplitInit}, }; /// A storage for raft. @@ -337,8 +337,8 @@ impl raft::Storage for Storage { mod tests { use std::{ sync::{ - mpsc::{sync_channel, Receiver, SyncSender}, Arc, + mpsc::{Receiver, SyncSender, sync_channel}, }, time::Duration, }; @@ -348,8 +348,8 @@ mod tests { kv::{KvTestEngine, TestTabletFactory}, }; use engine_traits::{ - FlushState, RaftEngine, RaftLogBatch, SstApplyState, TabletContext, TabletRegistry, - DATA_CFS, + DATA_CFS, FlushState, RaftEngine, RaftLogBatch, SstApplyState, TabletContext, + TabletRegistry, }; use kvproto::{ metapb::Region, @@ -360,23 +360,23 @@ mod tests { use raftstore::{ coprocessor::CoprocessorHost, store::{ - util::new_empty_snapshot, write_to_db_for_test, AsyncReadNotifier, Config, FetchedLogs, - GenSnapRes, ReadRunner, TabletSnapKey, TabletSnapManager, WriteTask, - RAFT_INIT_LOG_INDEX, RAFT_INIT_LOG_TERM, + AsyncReadNotifier, Config, FetchedLogs, GenSnapRes, RAFT_INIT_LOG_INDEX, + RAFT_INIT_LOG_TERM, ReadRunner, TabletSnapKey, TabletSnapManager, WriteTask, + util::new_empty_snapshot, write_to_db_for_test, }, }; use slog::o; use tempfile::TempDir; use tikv_util::{ store::new_peer, - worker::{dummy_scheduler, Worker}, + worker::{Worker, dummy_scheduler}, yatp_pool::{DefaultTicker, YatpPoolBuilder}, }; use super::*; use crate::{ fsm::ApplyResReporter, - operation::{test_util::create_tmp_importer, write_initial_states, CatchUpLogs}, + operation::{CatchUpLogs, test_util::create_tmp_importer, write_initial_states}, raft::Apply, router::ApplyRes, }; diff --git a/components/raftstore-v2/src/router/imp.rs b/components/raftstore-v2/src/router/imp.rs index e7a63f6d48f..b403da3b179 100644 --- a/components/raftstore-v2/src/router/imp.rs +++ b/components/raftstore-v2/src/router/imp.rs @@ -19,18 +19,19 @@ use kvproto::{ use raftstore::{ router::CdcHandle, store::{ - fsm::ChangeObserver, AsyncReadNotifier, Callback, FetchedLogs, GenSnapRes, RegionSnapshot, + AsyncReadNotifier, Callback, FetchedLogs, GenSnapRes, RegionSnapshot, UnsafeRecoveryExecutePlanSyncer, UnsafeRecoveryFillOutReportSyncer, UnsafeRecoveryForceLeaderSyncer, UnsafeRecoveryHandle, UnsafeRecoveryWaitApplySyncer, + fsm::ChangeObserver, }, }; use slog::warn; use tikv_util::box_err; use super::{ - build_any_channel, message::CaptureChange, PeerMsg, QueryResChannel, QueryResult, StoreMsg, + PeerMsg, QueryResChannel, QueryResult, StoreMsg, build_any_channel, message::CaptureChange, }; -use crate::{batch::StoreRouter, operation::LocalReader, StoreMeta}; +use crate::{StoreMeta, batch::StoreRouter, operation::LocalReader}; impl AsyncReadNotifier for StoreRouter { fn notify_logs_fetched(&self, region_id: u64, fetched_logs: FetchedLogs) { diff --git a/components/raftstore-v2/src/router/message.rs b/components/raftstore-v2/src/router/message.rs index b66c84d9740..203306f26e6 100644 --- a/components/raftstore-v2/src/router/message.rs +++ b/components/raftstore-v2/src/router/message.rs @@ -1,7 +1,7 @@ // Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. // #[PerformanceCriticalPath] -use std::sync::{mpsc::SyncSender, Arc}; +use std::sync::{Arc, mpsc::SyncSender}; use collections::HashSet; use health_controller::types::LatencyInspector; @@ -14,10 +14,10 @@ use kvproto::{ raft_serverpb::RaftMessage, }; use raftstore::store::{ - fsm::ChangeObserver, metrics::RaftEventDurationType, simple_write::SimpleWriteBinary, FetchedLogs, GenSnapRes, RaftCmdExtraOpts, TabletSnapKey, UnsafeRecoveryExecutePlanSyncer, UnsafeRecoveryFillOutReportSyncer, UnsafeRecoveryForceLeaderSyncer, - UnsafeRecoveryWaitApplySyncer, + UnsafeRecoveryWaitApplySyncer, fsm::ChangeObserver, metrics::RaftEventDurationType, + simple_write::SimpleWriteBinary, }; use resource_control::ResourceMetered; use tikv_util::time::Instant; diff --git a/components/raftstore-v2/src/router/mod.rs b/components/raftstore-v2/src/router/mod.rs index d63e1abc733..2f80c6cd155 100644 --- a/components/raftstore-v2/src/router/mod.rs +++ b/components/raftstore-v2/src/router/mod.rs @@ -15,9 +15,9 @@ pub use self::{ internal_message::{ApplyRes, SstApplyIndex}, message::{PeerMsg, PeerTick, RaftRequest, StoreMsg, StoreTick}, response_channel::{ - build_any_channel, AnyResChannel, AnyResSubscriber, BaseSubscriber, CmdResChannel, - CmdResChannelBuilder, CmdResEvent, CmdResStream, CmdResSubscriber, DebugInfoChannel, - DebugInfoSubscriber, QueryResChannel, QueryResult, ReadResponse, + AnyResChannel, AnyResSubscriber, BaseSubscriber, CmdResChannel, CmdResChannelBuilder, + CmdResEvent, CmdResStream, CmdResSubscriber, DebugInfoChannel, DebugInfoSubscriber, + QueryResChannel, QueryResult, ReadResponse, build_any_channel, }, }; pub use super::operation::DiskSnapBackupHandle; diff --git a/components/raftstore-v2/src/router/response_channel.rs b/components/raftstore-v2/src/router/response_channel.rs index 4f47f971670..459381536ca 100644 --- a/components/raftstore-v2/src/router/response_channel.rs +++ b/components/raftstore-v2/src/router/response_channel.rs @@ -19,19 +19,19 @@ use std::{ future::Future, pin::Pin, sync::{ - atomic::{AtomicU64, Ordering}, Arc, + atomic::{AtomicU64, Ordering}, }, task::{Context, Poll}, }; -use futures::{task::AtomicWaker, FutureExt, Stream}; +use futures::{FutureExt, Stream, task::AtomicWaker}; use kvproto::{kvrpcpb::ExtraOp as TxnExtraOp, raft_cmdpb::RaftCmdResponse}; use raftstore::store::{ - local_metrics::TimeTracker, msg::ErrorCallback, region_meta::RegionMeta, ReadCallback, - WriteCallback, + ReadCallback, WriteCallback, local_metrics::TimeTracker, msg::ErrorCallback, + region_meta::RegionMeta, }; -use tracker::{get_tls_tracker_token, TrackerToken}; +use tracker::{TrackerToken, get_tls_tracker_token}; union Tracker { read: TrackerToken, @@ -49,6 +49,7 @@ struct EventCore { /// - 0b01 means the event is fired and not subscribed. /// - 0b10 means the event is not fired and subscribed. /// - 0b11 means the event is fired and subscribed. + /// /// Event 0 and Event 31 is reserved as payload and cancel respectively. /// Other events should be defined within [1, 30]. event: AtomicU64, @@ -729,7 +730,7 @@ pub use flush_channel::{FlushChannel, FlushSubscriber}; mod tests { use std::assert_matches::assert_matches; - use futures::{executor::block_on, StreamExt}; + use futures::{StreamExt, executor::block_on}; use super::*; diff --git a/components/raftstore-v2/src/worker/cleanup/compact.rs b/components/raftstore-v2/src/worker/cleanup/compact.rs index 3fe94daa684..ff78c8ffb09 100644 --- a/components/raftstore-v2/src/worker/cleanup/compact.rs +++ b/components/raftstore-v2/src/worker/cleanup/compact.rs @@ -5,11 +5,11 @@ use std::{ fmt::{self, Display, Formatter}, }; -use engine_traits::{KvEngine, ManualCompactionOptions, TabletRegistry, CF_WRITE}; +use engine_traits::{CF_WRITE, KvEngine, ManualCompactionOptions, TabletRegistry}; use fail::fail_point; use keys::{DATA_MAX_KEY, DATA_MIN_KEY}; -use raftstore::store::{need_compact, CompactThreshold}; -use slog::{debug, error, info, warn, Logger}; +use raftstore::store::{CompactThreshold, need_compact}; +use slog::{Logger, debug, error, info, warn}; use thiserror::Error; use tikv_util::{box_try, worker::Runnable}; @@ -190,7 +190,7 @@ mod tests { ctor::{CfOptions, DbOptions}, kv::{KvTestEngine, TestTabletFactory}, }; - use engine_traits::{MiscExt, SyncMutable, TabletContext, TabletRegistry, CF_DEFAULT, CF_LOCK}; + use engine_traits::{CF_DEFAULT, CF_LOCK, MiscExt, SyncMutable, TabletContext, TabletRegistry}; use keys::data_key; use kvproto::metapb::Region; use tempfile::Builder; diff --git a/components/raftstore-v2/src/worker/pd/misc.rs b/components/raftstore-v2/src/worker/pd/misc.rs index fd2848b95a4..eb6bdd203b0 100644 --- a/components/raftstore-v2/src/worker/pd/misc.rs +++ b/components/raftstore-v2/src/worker/pd/misc.rs @@ -1,15 +1,15 @@ // Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. use std::{ - sync::{atomic::Ordering, Arc}, + sync::{Arc, atomic::Ordering}, time::{Duration, Instant}, }; use causal_ts::CausalTsProvider; use engine_traits::{KvEngine, RaftEngine}; -use futures::{compat::Future01CompatExt, FutureExt}; +use futures::{FutureExt, compat::Future01CompatExt}; use pd_client::PdClient; -use raftstore::{store::TxnExt, Result}; +use raftstore::{Result, store::TxnExt}; use slog::{info, warn}; use tikv_util::{box_err, timer::GLOBAL_TIMER_HANDLE}; diff --git a/components/raftstore-v2/src/worker/pd/mod.rs b/components/raftstore-v2/src/worker/pd/mod.rs index 133598a10f6..01c6774f914 100644 --- a/components/raftstore-v2/src/worker/pd/mod.rs +++ b/components/raftstore-v2/src/worker/pd/mod.rs @@ -2,7 +2,7 @@ use std::{ fmt::{self, Display, Formatter}, - sync::{atomic::AtomicBool, Arc}, + sync::{Arc, atomic::AtomicBool}, }; use causal_ts::CausalTsProviderImpl; @@ -13,19 +13,19 @@ use health_controller::types::{InspectFactor, LatencyInspector, RaftstoreDuratio use kvproto::{metapb, pdpb}; use pd_client::{BucketStat, PdClient}; use raftstore::store::{ - metrics::STORE_INSPECT_DURATION_HISTOGRAM, util::KeysInfoFormatter, AutoSplitController, - Config, FlowStatsReporter, PdStatsMonitor, ReadStats, SplitInfo, StoreStatsReporter, - TabletSnapManager, TxnExt, WriteStats, NUM_COLLECT_STORE_INFOS_PER_HEARTBEAT, + AutoSplitController, Config, FlowStatsReporter, NUM_COLLECT_STORE_INFOS_PER_HEARTBEAT, + PdStatsMonitor, ReadStats, SplitInfo, StoreStatsReporter, TabletSnapManager, TxnExt, + WriteStats, metrics::STORE_INSPECT_DURATION_HISTOGRAM, util::KeysInfoFormatter, }; use resource_metering::{Collector, CollectorRegHandle, RawRecords}; use service::service_manager::GrpcServiceManager; -use slog::{error, warn, Logger}; +use slog::{Logger, error, warn}; use tikv_util::{ config::VersionTrack, time::{Instant as TiInstant, UnixSecs}, worker::{Runnable, Scheduler}, }; -use yatp::{task::future::TaskCell, Remote}; +use yatp::{Remote, task::future::TaskCell}; use crate::{ batch::StoreRouter, diff --git a/components/raftstore-v2/src/worker/pd/region.rs b/components/raftstore-v2/src/worker/pd/region.rs index be207e1ab16..7da269e20b6 100644 --- a/components/raftstore-v2/src/worker/pd/region.rs +++ b/components/raftstore-v2/src/worker/pd/region.rs @@ -6,14 +6,14 @@ use collections::HashMap; use engine_traits::{KvEngine, RaftEngine}; use kvproto::{metapb, pdpb}; use pd_client::{ - metrics::PD_HEARTBEAT_COUNTER_VEC, BucketStat, PdClient, RegionStat, RegionWriteCfCopDetail, + BucketStat, PdClient, RegionStat, RegionWriteCfCopDetail, metrics::PD_HEARTBEAT_COUNTER_VEC, }; use raftstore::store::{ReadStats, WriteStats}; use resource_metering::RawRecords; use slog::{debug, error, info}; use tikv_util::{store::QueryStats, time::UnixSecs}; -use super::{requests::*, Runner}; +use super::{Runner, requests::*}; use crate::{ operation::{RequestHalfSplit, RequestSplit}, router::{CmdResChannel, PeerMsg}, diff --git a/components/raftstore-v2/src/worker/pd/slowness.rs b/components/raftstore-v2/src/worker/pd/slowness.rs index 4f2aee6102e..b91726d2d1c 100644 --- a/components/raftstore-v2/src/worker/pd/slowness.rs +++ b/components/raftstore-v2/src/worker/pd/slowness.rs @@ -10,7 +10,7 @@ use health_controller::{ }; use kvproto::pdpb; use pd_client::PdClient; -use raftstore::store::{metrics::*, Config}; +use raftstore::store::{Config, metrics::*}; use super::Runner; pub struct SlownessStatistics { diff --git a/components/raftstore-v2/src/worker/pd/split.rs b/components/raftstore-v2/src/worker/pd/split.rs index 7bafb6c442a..07b3f99577f 100644 --- a/components/raftstore-v2/src/worker/pd/split.rs +++ b/components/raftstore-v2/src/worker/pd/split.rs @@ -7,10 +7,10 @@ use kvproto::{ }; use pd_client::PdClient; use raftstore::store::SplitInfo; -use slog::{info, warn, Logger}; -use yatp::{task::future::TaskCell, Remote}; +use slog::{Logger, info, warn}; +use yatp::{Remote, task::future::TaskCell}; -use super::{requests::*, Runner}; +use super::{Runner, requests::*}; use crate::{batch::StoreRouter, router::CmdResChannel}; fn new_batch_split_region_request( diff --git a/components/raftstore-v2/src/worker/pd/store.rs b/components/raftstore-v2/src/worker/pd/store.rs index 3502e0ffb85..59f6f7ca76f 100644 --- a/components/raftstore-v2/src/worker/pd/store.rs +++ b/components/raftstore-v2/src/worker/pd/store.rs @@ -8,16 +8,16 @@ use fail::fail_point; use health_controller::types::LatencyInspector; use kvproto::pdpb; use pd_client::{ + PdClient, metrics::{ REGION_READ_BYTES_HISTOGRAM, REGION_READ_KEYS_HISTOGRAM, REGION_WRITTEN_BYTES_HISTOGRAM, REGION_WRITTEN_KEYS_HISTOGRAM, }, - PdClient, }; use prometheus::local::LocalHistogram; use raftstore::store::{ - metrics::STORE_SNAPSHOT_TRAFFIC_GAUGE_VEC, UnsafeRecoveryExecutePlanSyncer, - UnsafeRecoveryForceLeaderSyncer, UnsafeRecoveryHandle, + UnsafeRecoveryExecutePlanSyncer, UnsafeRecoveryForceLeaderSyncer, UnsafeRecoveryHandle, + metrics::STORE_SNAPSHOT_TRAFFIC_GAUGE_VEC, }; use slog::{error, info, warn}; use tikv_util::{ diff --git a/components/raftstore-v2/src/worker/refresh_config.rs b/components/raftstore-v2/src/worker/refresh_config.rs index 797f5b821ab..fce6f94970a 100644 --- a/components/raftstore-v2/src/worker/refresh_config.rs +++ b/components/raftstore-v2/src/worker/refresh_config.rs @@ -3,16 +3,16 @@ use std::{sync::Arc, thread}; use batch_system::{BatchRouter, Fsm, FsmTypes, HandlerBuilder, Poller, PoolState, Priority}; -use file_system::{set_io_type, IoType}; +use file_system::{IoType, set_io_type}; use raftstore::store::{BatchComponent, RefreshConfigTask, Transport, WriterContoller}; -use slog::{error, info, warn, Logger}; +use slog::{Logger, error, info, warn}; use tikv_util::{ sys::thread::StdThreadBuildWrapper, thd_name, worker::Runnable, yatp_pool::FuturePool, }; use crate::{ - fsm::{PeerFsm, StoreFsm}, StoreRouter, + fsm::{PeerFsm, StoreFsm}, }; pub struct PoolController> { diff --git a/components/raftstore-v2/src/worker/tablet.rs b/components/raftstore-v2/src/worker/tablet.rs index 88bbe5f4ef7..aa5930c9522 100644 --- a/components/raftstore-v2/src/worker/tablet.rs +++ b/components/raftstore-v2/src/worker/tablet.rs @@ -9,21 +9,21 @@ use std::{ use collections::HashMap; use engine_traits::{ - CfName, DeleteStrategy, KvEngine, ManualCompactionOptions, Range, TabletContext, - TabletRegistry, WriteOptions, DATA_CFS, + CfName, DATA_CFS, DeleteStrategy, KvEngine, ManualCompactionOptions, Range, TabletContext, + TabletRegistry, WriteOptions, }; use fail::fail_point; use kvproto::{import_sstpb::SstMeta, metapb::Region}; use raftstore::store::{TabletSnapKey, TabletSnapManager}; -use slog::{debug, error, info, warn, Logger}; +use slog::{Logger, debug, error, info, warn}; use sst_importer::SstImporter; use tikv_util::{ + Either, config::ReadableDuration, slog_panic, time::Instant, worker::{Runnable, RunnableWithTimer}, yatp_pool::{DefaultTicker, FuturePool, YatpPoolBuilder}, - Either, }; const DEFAULT_HIGH_PRI_POOL_SIZE: usize = 2; diff --git a/components/raftstore-v2/tests/failpoints/test_basic_write.rs b/components/raftstore-v2/tests/failpoints/test_basic_write.rs index 1b1dba04e55..013b5e66a3f 100644 --- a/components/raftstore-v2/tests/failpoints/test_basic_write.rs +++ b/components/raftstore-v2/tests/failpoints/test_basic_write.rs @@ -3,11 +3,11 @@ use std::{assert_matches::assert_matches, time::Duration}; use engine_traits::{ - CompactExt, DbOptionsExt, ManualCompactionOptions, MiscExt, Peekable, RaftEngineReadOnly, - CF_DEFAULT, CF_RAFT, CF_WRITE, + CF_DEFAULT, CF_RAFT, CF_WRITE, CompactExt, DbOptionsExt, ManualCompactionOptions, MiscExt, + Peekable, RaftEngineReadOnly, }; use futures::executor::block_on; -use raftstore_v2::{router::PeerMsg, SimpleWriteEncoder}; +use raftstore_v2::{SimpleWriteEncoder, router::PeerMsg}; use crate::cluster::Cluster; diff --git a/components/raftstore-v2/tests/failpoints/test_bucket.rs b/components/raftstore-v2/tests/failpoints/test_bucket.rs index f136cf6dc53..36494615fb3 100644 --- a/components/raftstore-v2/tests/failpoints/test_bucket.rs +++ b/components/raftstore-v2/tests/failpoints/test_bucket.rs @@ -6,7 +6,7 @@ use engine_traits::RaftEngineReadOnly; use raftstore::store::RAFT_INIT_LOG_INDEX; use tikv_util::store::new_peer; -use crate::cluster::{split_helper::split_region_and_refresh_bucket, Cluster}; +use crate::cluster::{Cluster, split_helper::split_region_and_refresh_bucket}; /// Test refresh bucket. #[test] diff --git a/components/raftstore-v2/tests/failpoints/test_life.rs b/components/raftstore-v2/tests/failpoints/test_life.rs index ed05c1c6fad..46df7cd84e3 100644 --- a/components/raftstore-v2/tests/failpoints/test_life.rs +++ b/components/raftstore-v2/tests/failpoints/test_life.rs @@ -6,10 +6,10 @@ use engine_traits::CF_DEFAULT; use futures::executor::block_on; use kvproto::raft_serverpb::RaftMessage; use raft::prelude::MessageType; -use raftstore_v2::{router::PeerMsg, SimpleWriteEncoder}; +use raftstore_v2::{SimpleWriteEncoder, router::PeerMsg}; use tikv_util::store::new_peer; -use crate::cluster::{life_helper::assert_peer_not_exist, Cluster}; +use crate::cluster::{Cluster, life_helper::assert_peer_not_exist}; /// Test if a peer can be destroyed when it's applying entries #[test] diff --git a/components/raftstore-v2/tests/failpoints/test_merge.rs b/components/raftstore-v2/tests/failpoints/test_merge.rs index 11fe666b49b..a5cc2d2e940 100644 --- a/components/raftstore-v2/tests/failpoints/test_merge.rs +++ b/components/raftstore-v2/tests/failpoints/test_merge.rs @@ -1,7 +1,7 @@ // Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0. use std::{ - sync::{mpsc, Mutex}, + sync::{Mutex, mpsc}, time::Duration, }; @@ -10,10 +10,10 @@ use raftstore_v2::router::{PeerMsg, PeerTick}; use tikv_util::{config::ReadableDuration, info, store::new_peer}; use crate::cluster::{ + Cluster, life_helper::assert_peer_not_exist, merge_helper::merge_region, split_helper::{put, split_region}, - Cluster, }; #[test] diff --git a/components/raftstore-v2/tests/failpoints/test_pd_heartbeat.rs b/components/raftstore-v2/tests/failpoints/test_pd_heartbeat.rs index f175e3cd5c9..6ae9fb34b20 100644 --- a/components/raftstore-v2/tests/failpoints/test_pd_heartbeat.rs +++ b/components/raftstore-v2/tests/failpoints/test_pd_heartbeat.rs @@ -5,7 +5,7 @@ use pd_client::PdClient; use raftstore_v2::router::{StoreMsg, StoreTick}; use tikv_util::config::ReadableDuration; -use crate::cluster::{v2_default_config, Cluster}; +use crate::cluster::{Cluster, v2_default_config}; #[test] fn test_fake_store_heartbeat() { diff --git a/components/raftstore-v2/tests/failpoints/test_split.rs b/components/raftstore-v2/tests/failpoints/test_split.rs index e67041ab181..7255e6eee24 100644 --- a/components/raftstore-v2/tests/failpoints/test_split.rs +++ b/components/raftstore-v2/tests/failpoints/test_split.rs @@ -5,12 +5,12 @@ use std::{ time::{Duration, Instant}, }; -use engine_traits::{RaftEngineReadOnly, CF_DEFAULT}; +use engine_traits::{CF_DEFAULT, RaftEngineReadOnly}; use futures::executor::block_on; use raftstore::store::RAFT_INIT_LOG_INDEX; -use raftstore_v2::{router::PeerMsg, SimpleWriteEncoder}; +use raftstore_v2::{SimpleWriteEncoder, router::PeerMsg}; -use crate::cluster::{split_helper::split_region, Cluster}; +use crate::cluster::{Cluster, split_helper::split_region}; /// If a node is restarted after metadata is persisted before tablet is not /// installed, it should resume install the tablet. diff --git a/components/raftstore-v2/tests/integrations/cluster.rs b/components/raftstore-v2/tests/integrations/cluster.rs index 0cc14abf2c2..8c5ae40e38f 100644 --- a/components/raftstore-v2/tests/integrations/cluster.rs +++ b/components/raftstore-v2/tests/integrations/cluster.rs @@ -4,8 +4,8 @@ use std::{ ops::{Deref, DerefMut}, path::Path, sync::{ - atomic::{AtomicUsize, Ordering}, Arc, + atomic::{AtomicUsize, Ordering}, }, thread, time::{Duration, Instant}, @@ -15,13 +15,13 @@ use causal_ts::CausalTsProviderImpl; use collections::HashSet; use concurrency_manager::ConcurrencyManager; use crossbeam::channel::{self, Receiver, Sender, TrySendError}; -use encryption_export::{data_key_manager_from_config, DataKeyImporter}; +use encryption_export::{DataKeyImporter, data_key_manager_from_config}; use engine_test::{ ctor::{CfOptions, DbOptions}, kv::{KvTestEngine, KvTestSnapshot, TestTabletFactory}, raft::RaftTestEngine, }; -use engine_traits::{TabletContext, TabletRegistry, DATA_CFS}; +use engine_traits::{DATA_CFS, TabletContext, TabletRegistry}; use futures::executor::block_on; use kvproto::{ kvrpcpb::ApiVersion, @@ -34,20 +34,19 @@ use raft::eraftpb::MessageType; use raftstore::{ coprocessor::{Config as CopConfig, CoprocessorHost, StoreHandle}, store::{ + AutoSplitController, Bucket, Config, RAFT_INIT_LOG_INDEX, RegionSnapshot, TabletSnapKey, + TabletSnapManager, Transport, region_meta::{RegionLocalState, RegionMeta}, - AutoSplitController, Bucket, Config, RegionSnapshot, TabletSnapKey, TabletSnapManager, - Transport, RAFT_INIT_LOG_INDEX, }, }; use raftstore_v2::{ - create_store_batch_system, + Bootstrap, SimpleWriteEncoder, StateStorage, StoreSystem, create_store_batch_system, router::{DebugInfoChannel, FlushChannel, PeerMsg, QueryResult, RaftRouter, StoreMsg}, - Bootstrap, SimpleWriteEncoder, StateStorage, StoreSystem, }; use resource_control::{ResourceController, ResourceGroupManager}; use resource_metering::CollectorRegHandle; use service::service_manager::GrpcServiceManager; -use slog::{debug, o, Logger}; +use slog::{Logger, debug, o}; use sst_importer::SstImporter; use tempfile::TempDir; use test_pd::mocker::Service; @@ -62,7 +61,7 @@ pub fn check_skip_wal(path: &str) { let mut found = false; for f in std::fs::read_dir(path).unwrap() { let e = f.unwrap(); - if e.path().extension().map_or(false, |ext| ext == "log") { + if e.path().extension().is_some_and(|ext| ext == "log") { found = true; assert_eq!(e.metadata().unwrap().len(), 0, "{}", e.path().display()); } @@ -720,7 +719,7 @@ pub mod split_helper { raft_cmdpb::{AdminCmdType, AdminRequest, RaftCmdRequest, RaftCmdResponse, SplitRequest}, }; use raftstore::store::Bucket; - use raftstore_v2::{router::PeerMsg, SimpleWriteEncoder}; + use raftstore_v2::{SimpleWriteEncoder, router::PeerMsg}; use super::TestRouter; diff --git a/components/raftstore-v2/tests/integrations/test_basic_write.rs b/components/raftstore-v2/tests/integrations/test_basic_write.rs index cb8d71840cf..08351debd93 100644 --- a/components/raftstore-v2/tests/integrations/test_basic_write.rs +++ b/components/raftstore-v2/tests/integrations/test_basic_write.rs @@ -2,14 +2,14 @@ use std::{assert_matches::assert_matches, time::Duration}; -use engine_traits::{Peekable, CF_DEFAULT}; +use engine_traits::{CF_DEFAULT, Peekable}; use futures::executor::block_on; use kvproto::raft_serverpb::RaftMessage; use raftstore::store::{INIT_EPOCH_CONF_VER, INIT_EPOCH_VER}; -use raftstore_v2::{router::PeerMsg, SimpleWriteEncoder}; +use raftstore_v2::{SimpleWriteEncoder, router::PeerMsg}; use tikv_util::store::new_peer; -use crate::cluster::{check_skip_wal, Cluster}; +use crate::cluster::{Cluster, check_skip_wal}; /// Test basic write flow. #[test] diff --git a/components/raftstore-v2/tests/integrations/test_conf_change.rs b/components/raftstore-v2/tests/integrations/test_conf_change.rs index c1c7861fd54..017225965a7 100644 --- a/components/raftstore-v2/tests/integrations/test_conf_change.rs +++ b/components/raftstore-v2/tests/integrations/test_conf_change.rs @@ -2,7 +2,7 @@ use std::{self, time::Duration}; -use engine_traits::{Peekable, RaftEngineReadOnly, CF_DEFAULT}; +use engine_traits::{CF_DEFAULT, Peekable, RaftEngineReadOnly}; use futures::executor::block_on; use kvproto::{ raft_cmdpb::{AdminCmdType, RaftCmdRequest}, @@ -10,12 +10,12 @@ use kvproto::{ }; use raft::prelude::{ConfChangeType, MessageType}; use raftstore_v2::{ - router::{PeerMsg, PeerTick}, SimpleWriteEncoder, + router::{PeerMsg, PeerTick}, }; use tikv_util::store::{new_learner_peer, new_peer}; -use crate::cluster::{check_skip_wal, Cluster}; +use crate::cluster::{Cluster, check_skip_wal}; #[test] fn test_simple_change() { diff --git a/components/raftstore-v2/tests/integrations/test_life.rs b/components/raftstore-v2/tests/integrations/test_life.rs index 373763a53ef..50052dd9b01 100644 --- a/components/raftstore-v2/tests/integrations/test_life.rs +++ b/components/raftstore-v2/tests/integrations/test_life.rs @@ -2,21 +2,21 @@ use std::time::Duration; -use engine_traits::{RaftEngineReadOnly, CF_DEFAULT}; +use engine_traits::{CF_DEFAULT, RaftEngineReadOnly}; use futures::executor::block_on; use kvproto::{raft_cmdpb::AdminCmdType, raft_serverpb::RaftMessage}; use raft::prelude::{ConfChangeType, MessageType}; use raftstore_v2::{ - router::{PeerMsg, PeerTick}, SimpleWriteEncoder, + router::{PeerMsg, PeerTick}, }; use tikv_util::store::{new_learner_peer, new_peer}; use crate::cluster::{ + Cluster, life_helper::{ assert_peer_not_exist, assert_tombstone, assert_tombstone_msg, assert_valid_report, }, - Cluster, }; /// Test a peer can be created by general raft message and destroyed tombstone diff --git a/components/raftstore-v2/tests/integrations/test_merge.rs b/components/raftstore-v2/tests/integrations/test_merge.rs index 7d9dbef720e..b00c0113afd 100644 --- a/components/raftstore-v2/tests/integrations/test_merge.rs +++ b/components/raftstore-v2/tests/integrations/test_merge.rs @@ -7,7 +7,7 @@ use kvproto::metapb::{Peer, Region}; use raftstore::store::RAFT_INIT_LOG_INDEX; use tikv_util::store::new_peer; -use crate::cluster::{merge_helper::merge_region, split_helper::split_region, Cluster, TestRouter}; +use crate::cluster::{Cluster, TestRouter, merge_helper::merge_region, split_helper::split_region}; #[test] fn test_merge() { diff --git a/components/raftstore-v2/tests/integrations/test_pd_heartbeat.rs b/components/raftstore-v2/tests/integrations/test_pd_heartbeat.rs index 679183735b6..e9d92cb9b07 100644 --- a/components/raftstore-v2/tests/integrations/test_pd_heartbeat.rs +++ b/components/raftstore-v2/tests/integrations/test_pd_heartbeat.rs @@ -2,18 +2,18 @@ use std::time::Duration; -use engine_traits::{MiscExt, CF_DEFAULT}; +use engine_traits::{CF_DEFAULT, MiscExt}; use futures::executor::block_on; use kvproto::raft_cmdpb::{RaftCmdRequest, StatusCmdType}; use pd_client::PdClient; use raftstore::coprocessor::Config as CopConfig; use raftstore_v2::{ - router::{PeerMsg, PeerTick, StoreMsg, StoreTick}, SimpleWriteEncoder, + router::{PeerMsg, PeerTick, StoreMsg, StoreTick}, }; use tikv_util::{config::ReadableSize, store::new_peer}; -use crate::cluster::{v2_default_config, Cluster}; +use crate::cluster::{Cluster, v2_default_config}; #[test] fn test_region_heartbeat() { @@ -210,8 +210,6 @@ fn test_report_buckets() { router.send(region_id, msg).unwrap(); let _resp = block_on(sub.result()).unwrap(); } - ((suffix.as_bytes().len() + 10) * repeat) - .try_into() - .unwrap() + ((suffix.len() + 10) * repeat) as u64 } } diff --git a/components/raftstore-v2/tests/integrations/test_read.rs b/components/raftstore-v2/tests/integrations/test_read.rs index f9575ff8da1..8e7dea81a54 100644 --- a/components/raftstore-v2/tests/integrations/test_read.rs +++ b/components/raftstore-v2/tests/integrations/test_read.rs @@ -3,11 +3,11 @@ use engine_traits::CF_DEFAULT; use futures::executor::block_on; use kvproto::raft_cmdpb::{CmdType, Request}; -use raftstore_v2::{router::PeerMsg, SimpleWriteEncoder}; +use raftstore_v2::{SimpleWriteEncoder, router::PeerMsg}; use tikv_util::{config::ReadableDuration, store::new_peer}; use txn_types::WriteBatchFlags; -use crate::cluster::{v2_default_config, Cluster}; +use crate::cluster::{Cluster, v2_default_config}; #[test] fn test_read_index() { diff --git a/components/raftstore-v2/tests/integrations/test_split.rs b/components/raftstore-v2/tests/integrations/test_split.rs index 9dab98be598..8c9ff35d030 100644 --- a/components/raftstore-v2/tests/integrations/test_split.rs +++ b/components/raftstore-v2/tests/integrations/test_split.rs @@ -2,12 +2,12 @@ use std::time::Duration; -use engine_traits::{Peekable, RaftEngineReadOnly, CF_RAFT}; +use engine_traits::{CF_RAFT, Peekable, RaftEngineReadOnly}; use raftstore::store::{INIT_EPOCH_VER, RAFT_INIT_LOG_INDEX}; use tikv_util::store::new_peer; use txn_types::{Key, TimeStamp}; -use crate::cluster::{split_helper::split_region, Cluster}; +use crate::cluster::{Cluster, split_helper::split_region}; #[test] fn test_split() { diff --git a/components/raftstore-v2/tests/integrations/test_trace_apply.rs b/components/raftstore-v2/tests/integrations/test_trace_apply.rs index 71682ff52a4..c484fb35ff0 100644 --- a/components/raftstore-v2/tests/integrations/test_trace_apply.rs +++ b/components/raftstore-v2/tests/integrations/test_trace_apply.rs @@ -2,10 +2,10 @@ use std::{path::Path, time::Duration}; -use engine_traits::{DbOptionsExt, MiscExt, Peekable, CF_DEFAULT, CF_LOCK, CF_WRITE, DATA_CFS}; +use engine_traits::{CF_DEFAULT, CF_LOCK, CF_WRITE, DATA_CFS, DbOptionsExt, MiscExt, Peekable}; use futures::executor::block_on; use raftstore::store::RAFT_INIT_LOG_INDEX; -use raftstore_v2::{router::PeerMsg, SimpleWriteEncoder}; +use raftstore_v2::{SimpleWriteEncoder, router::PeerMsg}; use crate::cluster::Cluster; @@ -21,7 +21,7 @@ fn count_file(path: &Path, pat: impl Fn(&Path) -> bool) -> usize { fn count_sst(path: &Path) -> usize { count_file(path, |path| { - path.extension().map_or(false, |ext| ext == "sst") + path.extension().is_some_and(|ext| ext == "sst") }) } diff --git a/components/raftstore-v2/tests/integrations/test_transfer_leader.rs b/components/raftstore-v2/tests/integrations/test_transfer_leader.rs index 3b0feefc406..ed8200737e5 100644 --- a/components/raftstore-v2/tests/integrations/test_transfer_leader.rs +++ b/components/raftstore-v2/tests/integrations/test_transfer_leader.rs @@ -2,7 +2,7 @@ use std::{assert_matches::assert_matches, time::Duration}; -use engine_traits::{Peekable, CF_DEFAULT}; +use engine_traits::{CF_DEFAULT, Peekable}; use futures::executor::block_on; use kvproto::{ metapb, @@ -10,8 +10,8 @@ use kvproto::{ }; use raft::prelude::ConfChangeType; use raftstore_v2::{ - router::{PeerMsg, PeerTick}, SimpleWriteEncoder, + router::{PeerMsg, PeerTick}, }; use tikv_util::{store::new_peer, time::Instant}; diff --git a/components/raftstore/Cargo.toml b/components/raftstore/Cargo.toml index e840c9a4e61..28ffae46600 100644 --- a/components/raftstore/Cargo.toml +++ b/components/raftstore/Cargo.toml @@ -31,7 +31,7 @@ collections = { workspace = true } concurrency_manager = { workspace = true } crc32fast = "1.2" crossbeam = { workspace = true } -derivative = "2" +derive_more = { version = "2.0", features = ["debug"] } encryption = { workspace = true } engine_rocks = { workspace = true, optional = true } engine_traits = { workspace = true } diff --git a/components/raftstore/src/compacted_event_sender.rs b/components/raftstore/src/compacted_event_sender.rs index 736332b52c5..3f963403e0b 100644 --- a/components/raftstore/src/compacted_event_sender.rs +++ b/components/raftstore/src/compacted_event_sender.rs @@ -5,7 +5,7 @@ use engine_rocks::{CompactedEventSender, RocksCompactedEvent}; use engine_traits::{KvEngine, RaftEngine}; use tikv_util::error_unknown; -use crate::store::{fsm::store::RaftRouter, StoreMsg}; +use crate::store::{StoreMsg, fsm::store::RaftRouter}; // raftstore v1's implementation pub struct RaftRouterCompactedEventSender diff --git a/components/raftstore/src/coprocessor/config.rs b/components/raftstore/src/coprocessor/config.rs index 7eb13210ee1..d2d904f316f 100644 --- a/components/raftstore/src/coprocessor/config.rs +++ b/components/raftstore/src/coprocessor/config.rs @@ -1,6 +1,6 @@ // Copyright 2017 TiKV Project Authors. Licensed under Apache-2.0. -use engine_traits::{perf_level_serde, PerfLevel}; +use engine_traits::{PerfLevel, perf_level_serde}; use online_config::{ConfigChange, ConfigManager, OnlineConfig}; use serde::{Deserialize, Serialize}; use tikv_util::{box_err, config::ReadableSize, worker::Scheduler}; diff --git a/components/raftstore/src/coprocessor/consistency_check.rs b/components/raftstore/src/coprocessor/consistency_check.rs index 2ebf27c963f..43c0aaddd37 100644 --- a/components/raftstore/src/coprocessor/consistency_check.rs +++ b/components/raftstore/src/coprocessor/consistency_check.rs @@ -2,12 +2,12 @@ use std::marker::PhantomData; -use engine_traits::{KvEngine, Snapshot, CF_RAFT}; +use engine_traits::{CF_RAFT, KvEngine, Snapshot}; use kvproto::metapb::Region; use crate::{ - coprocessor::{ConsistencyCheckMethod, Coprocessor}, Result, + coprocessor::{ConsistencyCheckMethod, Coprocessor}, }; pub trait ConsistencyCheckObserver: Coprocessor { diff --git a/components/raftstore/src/coprocessor/metrics.rs b/components/raftstore/src/coprocessor/metrics.rs index 79f76820f12..47ba6b5579b 100644 --- a/components/raftstore/src/coprocessor/metrics.rs +++ b/components/raftstore/src/coprocessor/metrics.rs @@ -2,7 +2,7 @@ use lazy_static::lazy_static; use prometheus::{ - exponential_buckets, register_histogram, register_int_gauge_vec, Histogram, IntGaugeVec, + Histogram, IntGaugeVec, exponential_buckets, register_histogram, register_int_gauge_vec, }; lazy_static! { diff --git a/components/raftstore/src/coprocessor/mod.rs b/components/raftstore/src/coprocessor/mod.rs index fc162498152..c2a08556877 100644 --- a/components/raftstore/src/coprocessor/mod.rs +++ b/components/raftstore/src/coprocessor/mod.rs @@ -3,8 +3,8 @@ use std::{ fmt::{self, Debug, Formatter}, sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering}, Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, }, vec::IntoIter, }; @@ -20,7 +20,7 @@ use kvproto::{ raft_serverpb::RaftApplyState, }; use pd_client::RegionStat; -use raft::{eraftpb, StateRole}; +use raft::{StateRole, eraftpb}; pub mod config; mod consistency_check; @@ -52,9 +52,9 @@ pub use self::{ RegionInfoProvider, SeekRegionCallback, }, split_check::{ - get_region_approximate_keys, get_region_approximate_middle, get_region_approximate_size, HalfCheckObserver, Host as SplitCheckerHost, KeysCheckObserver, SizeCheckObserver, - TableCheckObserver, + TableCheckObserver, get_region_approximate_keys, get_region_approximate_middle, + get_region_approximate_size, }, }; pub use crate::store::{Bucket, KeyEntry}; diff --git a/components/raftstore/src/coprocessor/read_write/write_batch.rs b/components/raftstore/src/coprocessor/read_write/write_batch.rs index 72c98523086..a0a806cc9fb 100644 --- a/components/raftstore/src/coprocessor/read_write/write_batch.rs +++ b/components/raftstore/src/coprocessor/read_write/write_batch.rs @@ -2,7 +2,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; -use engine_traits::{Mutable, Result, WriteBatch, WriteOptions, CF_DEFAULT}; +use engine_traits::{CF_DEFAULT, Mutable, Result, WriteBatch, WriteOptions}; use kvproto::metapb; pub trait WriteBatchObserver: Send { diff --git a/components/raftstore/src/coprocessor/region_info_accessor.rs b/components/raftstore/src/coprocessor/region_info_accessor.rs index 07caa3aa456..63ad5ce4ba1 100644 --- a/components/raftstore/src/coprocessor/region_info_accessor.rs +++ b/components/raftstore/src/coprocessor/region_info_accessor.rs @@ -7,7 +7,7 @@ use std::{ }, fmt::{Display, Formatter, Result as FmtResult}, num::NonZeroUsize, - sync::{mpsc, Arc, Mutex, RwLock}, + sync::{Arc, Mutex, RwLock, mpsc}, time::Duration, }; @@ -23,9 +23,9 @@ use tikv_util::{ }; use super::{ - dispatcher::BoxRegionHeartbeatObserver, metrics::*, BoxRegionChangeObserver, BoxRoleObserver, - Coprocessor, CoprocessorHost, ObserverContext, RegionChangeEvent, RegionChangeObserver, - RegionHeartbeatObserver, Result, RoleChange, RoleObserver, + BoxRegionChangeObserver, BoxRoleObserver, Coprocessor, CoprocessorHost, ObserverContext, + RegionChangeEvent, RegionChangeObserver, RegionHeartbeatObserver, Result, RoleChange, + RoleObserver, dispatcher::BoxRegionHeartbeatObserver, metrics::*, }; // TODO(SpadeA): this 100 may be adjusted by observing more workloads. @@ -48,7 +48,7 @@ const ITERATED_COUNT_FILTER_FACTOR: usize = 100; /// perfectly precise. Some regions may be temporarily absent while merging or /// splitting is in progress. Also, `RegionInfoAccessor`'s information may /// slightly lag the actual regions on the TiKV. - +/// /// `RaftStoreEvent` Represents events dispatched from raftstore coprocessor. #[derive(Debug)] pub enum RaftStoreEvent { @@ -1210,8 +1210,7 @@ mod tests { if is_regions_equal { for (expect_region, expect_role) in regions { is_regions_equal = is_regions_equal - && c.regions.get(&expect_region.get_id()).map_or( - false, + && c.regions.get(&expect_region.get_id()).is_some_and( |RegionInfo { region, role, .. }| { expect_region == region && expect_role == role }, @@ -1259,7 +1258,7 @@ mod tests { } fn must_create_region(c: &mut RegionCollector, region: &Region, role: StateRole) { - assert!(c.regions.get(®ion.get_id()).is_none()); + assert!(!c.regions.contains_key(®ion.get_id())); c.handle_raftstore_event(RaftStoreEvent::CreateRegion { region: region.clone(), @@ -1307,7 +1306,7 @@ mod tests { assert!( c.region_ranges .get(&RangeKey::from_end_key(old_end_key)) - .map_or(true, |id| *id != region.get_id()) + .is_none_or(|id| *id != region.get_id()) ); } } @@ -1329,14 +1328,14 @@ mod tests { c.handle_raftstore_event(RaftStoreEvent::DestroyRegion { region }); - assert!(c.regions.get(&id).is_none()); + assert!(!c.regions.contains_key(&id)); // If the region_id corresponding to the end_key doesn't equals to `id`, it // shouldn't be removed since it was used by another region. if let Some(end_key) = end_key { assert!( c.region_ranges .get(&RangeKey::from_end_key(end_key)) - .map_or(true, |r| *r != id) + .is_none_or(|r| *r != id) ); } } diff --git a/components/raftstore/src/coprocessor/split_check/half.rs b/components/raftstore/src/coprocessor/split_check/half.rs index 1f4527128d8..dfa88da64cf 100644 --- a/components/raftstore/src/coprocessor/split_check/half.rs +++ b/components/raftstore/src/coprocessor/split_check/half.rs @@ -6,7 +6,7 @@ use tikv_util::{box_try, config::ReadableSize}; use super::{ super::{ - error::Result, Coprocessor, KeyEntry, ObserverContext, SplitCheckObserver, SplitChecker, + Coprocessor, KeyEntry, ObserverContext, SplitCheckObserver, SplitChecker, error::Result, }, Host, }; @@ -126,7 +126,7 @@ mod tests { use std::{iter, sync::mpsc}; use engine_test::ctor::{CfOptions, DbOptions}; - use engine_traits::{MiscExt, SyncMutable, ALL_CFS, CF_DEFAULT, LARGE_CFS}; + use engine_traits::{ALL_CFS, CF_DEFAULT, LARGE_CFS, MiscExt, SyncMutable}; use kvproto::{ metapb::{Peer, Region}, pdpb::CheckPolicy, @@ -140,7 +140,7 @@ mod tests { *, }; use crate::{ - coprocessor::{dispatcher::SchedTask, Config, CoprocessorHost}, + coprocessor::{Config, CoprocessorHost, dispatcher::SchedTask}, store::{BucketRange, SplitCheckRunner, SplitCheckTask}, }; @@ -488,7 +488,7 @@ mod tests { let engine = engine_test::kv::new_engine_opt(path, db_opts, cfs_opts).unwrap(); let mut big_value = Vec::with_capacity(256); - big_value.extend(iter::repeat(b'v').take(256)); + big_value.extend(iter::repeat_n(b'v', 256)); for i in 0..100 { let k = format!("key_{:03}", i).into_bytes(); let k = keys::data_key(Key::from_raw(&k).as_encoded()); diff --git a/components/raftstore/src/coprocessor/split_check/keys.rs b/components/raftstore/src/coprocessor/split_check/keys.rs index d6a49175441..ae87d705fc5 100644 --- a/components/raftstore/src/coprocessor/split_check/keys.rs +++ b/components/raftstore/src/coprocessor/split_check/keys.rs @@ -7,12 +7,11 @@ use tikv_util::{box_try, debug, info, warn}; use super::{ super::{ - error::Result, metrics::*, Coprocessor, KeyEntry, ObserverContext, SplitCheckObserver, - SplitChecker, + Coprocessor, KeyEntry, ObserverContext, SplitCheckObserver, SplitChecker, error::Result, + metrics::*, }, - calc_split_keys_count, + Host, calc_split_keys_count, size::get_approximate_split_keys, - Host, }; use crate::coprocessor::dispatcher::StoreHandle; @@ -209,10 +208,10 @@ pub fn get_region_approximate_keys( #[cfg(test)] mod tests { - use std::{cmp, sync::mpsc, u64}; + use std::{cmp, sync::mpsc}; use engine_test::ctor::{CfOptions, DbOptions}; - use engine_traits::{KvEngine, MiscExt, SyncMutable, ALL_CFS, CF_DEFAULT, CF_WRITE, LARGE_CFS}; + use engine_traits::{ALL_CFS, CF_DEFAULT, CF_WRITE, KvEngine, LARGE_CFS, MiscExt, SyncMutable}; use kvproto::{ metapb::{Peer, Region}, pdpb::CheckPolicy, @@ -232,7 +231,7 @@ mod tests { *, }; use crate::{ - coprocessor::{dispatcher::SchedTask, Config, CoprocessorHost}, + coprocessor::{Config, CoprocessorHost, dispatcher::SchedTask}, store::{SplitCheckRunner, SplitCheckTask}, }; diff --git a/components/raftstore/src/coprocessor/split_check/mod.rs b/components/raftstore/src/coprocessor/split_check/mod.rs index e92000f2c95..bf8aecb1a74 100644 --- a/components/raftstore/src/coprocessor/split_check/mod.rs +++ b/components/raftstore/src/coprocessor/split_check/mod.rs @@ -9,12 +9,12 @@ use kvproto::{metapb::Region, pdpb::CheckPolicy}; use tikv_util::box_try; pub use self::{ - half::{get_region_approximate_middle, HalfCheckObserver}, - keys::{get_region_approximate_keys, KeysCheckObserver}, - size::{get_region_approximate_size, SizeCheckObserver}, + half::{HalfCheckObserver, get_region_approximate_middle}, + keys::{KeysCheckObserver, get_region_approximate_keys}, + size::{SizeCheckObserver, get_region_approximate_size}, table::TableCheckObserver, }; -use super::{config::Config, error::Result, Bucket, KeyEntry, ObserverContext, SplitChecker}; +use super::{Bucket, KeyEntry, ObserverContext, SplitChecker, config::Config, error::Result}; pub struct Host<'a, E> { checkers: Vec>>, diff --git a/components/raftstore/src/coprocessor/split_check/size.rs b/components/raftstore/src/coprocessor/split_check/size.rs index e5048a83826..dbafb2ad0fd 100644 --- a/components/raftstore/src/coprocessor/split_check/size.rs +++ b/components/raftstore/src/coprocessor/split_check/size.rs @@ -7,10 +7,10 @@ use tikv_util::{box_try, debug, info, warn}; use super::{ super::{ - error::Result, metrics::*, Coprocessor, KeyEntry, ObserverContext, SplitCheckObserver, - SplitChecker, + Coprocessor, KeyEntry, ObserverContext, SplitCheckObserver, SplitChecker, error::Result, + metrics::*, }, - calc_split_keys_count, Host, + Host, calc_split_keys_count, }; use crate::coprocessor::dispatcher::StoreHandle; @@ -234,7 +234,7 @@ pub fn get_approximate_split_keys( #[cfg(test)] pub mod tests { - use std::{assert_matches::assert_matches, iter, sync::mpsc, u64}; + use std::{assert_matches::assert_matches, iter, sync::mpsc}; use collections::HashSet; use engine_test::{ @@ -242,7 +242,7 @@ pub mod tests { kv::KvTestEngine, }; use engine_traits::{ - CfName, MiscExt, SyncMutable, ALL_CFS, CF_DEFAULT, CF_LOCK, CF_WRITE, LARGE_CFS, + ALL_CFS, CF_DEFAULT, CF_LOCK, CF_WRITE, CfName, LARGE_CFS, MiscExt, SyncMutable, }; use kvproto::{ metapb::{Peer, Region}, @@ -255,7 +255,7 @@ pub mod tests { use super::{Checker, *}; use crate::{ coprocessor::{ - dispatcher::SchedTask, Config, CoprocessorHost, ObserverContext, SplitChecker, + Config, CoprocessorHost, ObserverContext, SplitChecker, dispatcher::SchedTask, }, store::{BucketRange, KeyEntry, SplitCheckRunner, SplitCheckTask}, }; @@ -896,7 +896,7 @@ pub mod tests { ); let mut big_value = Vec::with_capacity(256); - big_value.extend(iter::repeat(b'v').take(256)); + big_value.extend(iter::repeat_n(b'v', 256)); for i in 0..100 { let k = format!("key_{:03}", i).into_bytes(); let k = keys::data_key(Key::from_raw(&k).as_encoded()); @@ -923,7 +923,7 @@ pub mod tests { let engine = engine_test::kv::new_engine_opt(path, db_opts, cfs_opts).unwrap(); let mut big_value = Vec::with_capacity(256); - big_value.extend(iter::repeat(b'v').take(256)); + big_value.extend(iter::repeat_n(b'v', 256)); for i in 0..4 { let k = format!("key_{:03}", i).into_bytes(); diff --git a/components/raftstore/src/coprocessor/split_check/table.rs b/components/raftstore/src/coprocessor/split_check/table.rs index df825bc2641..5573eb73f22 100644 --- a/components/raftstore/src/coprocessor/split_check/table.rs +++ b/components/raftstore/src/coprocessor/split_check/table.rs @@ -2,7 +2,7 @@ use std::cmp::Ordering; -use engine_traits::{IterOptions, Iterator, KvEngine, CF_WRITE}; +use engine_traits::{CF_WRITE, IterOptions, Iterator, KvEngine}; use error_code::ErrorCodeExt; use kvproto::{metapb::Region, pdpb::CheckPolicy}; use tidb_query_datatype::codec::table as table_codec; @@ -229,7 +229,7 @@ mod tests { use std::{io::Write, sync::mpsc}; use engine_test::kv::new_engine; - use engine_traits::{SyncMutable, ALL_CFS}; + use engine_traits::{ALL_CFS, SyncMutable}; use kvproto::{metapb::Peer, pdpb::CheckPolicy}; use tempfile::Builder; use tidb_query_datatype::codec::table::{TABLE_PREFIX, TABLE_PREFIX_KEY_LEN}; @@ -238,7 +238,7 @@ mod tests { use super::*; use crate::{ - coprocessor::{dispatcher::SchedTask, Config, CoprocessorHost}, + coprocessor::{Config, CoprocessorHost, dispatcher::SchedTask}, store::{SplitCheckRunner, SplitCheckTask}, }; diff --git a/components/raftstore/src/coprocessor/split_observer.rs b/components/raftstore/src/coprocessor/split_observer.rs index e84058d41dc..2c18cc046af 100644 --- a/components/raftstore/src/coprocessor/split_observer.rs +++ b/components/raftstore/src/coprocessor/split_observer.rs @@ -8,7 +8,7 @@ use kvproto::{ use tikv_util::{box_err, box_try, codec::bytes, error, warn}; use super::{AdminObserver, Coprocessor, ObserverContext, Result as CopResult}; -use crate::{store::util, Error}; +use crate::{Error, store::util}; pub const NO_VALID_SPLIT_KEY: &str = "no valid key found for split."; @@ -168,7 +168,7 @@ mod tests { raft_cmdpb::{AdminCmdType, AdminRequest, SplitRequest}, }; use tidb_query_datatype::{ - codec::{datum, table, Datum}, + codec::{Datum, datum, table}, expr::EvalContext, }; use tikv_util::codec::bytes::encode_bytes; diff --git a/components/raftstore/src/errors.rs b/components/raftstore/src/errors.rs index fce8eb2ef16..f5e830ff4a6 100644 --- a/components/raftstore/src/errors.rs +++ b/components/raftstore/src/errors.rs @@ -9,7 +9,7 @@ use protobuf::ProtobufError; use thiserror::Error; use tikv_util::{ codec, - deadline::{set_deadline_exceeded_busy_error, DeadlineError}, + deadline::{DeadlineError, set_deadline_exceeded_busy_error}, }; use super::{coprocessor::Error as CopError, store::SnapError}; diff --git a/components/raftstore/src/lib.rs b/components/raftstore/src/lib.rs index b8fbd2ac9af..5a3ce12eb89 100644 --- a/components/raftstore/src/lib.rs +++ b/components/raftstore/src/lib.rs @@ -2,10 +2,8 @@ #![cfg_attr(test, feature(test))] #![feature(cell_update)] -#![feature(div_duration)] #![feature(min_specialization)] #![feature(box_patterns)] -#![feature(hash_extract_if)] #![feature(let_chains)] #![feature(assert_matches)] #![feature(type_alias_impl_trait)] @@ -14,8 +12,6 @@ #[cfg(test)] extern crate test; -#[macro_use] -extern crate derivative; #[cfg(feature = "engine_rocks")] pub mod compacted_event_sender; diff --git a/components/raftstore/src/router.rs b/components/raftstore/src/router.rs index afa8b831ba2..2be6566d3a8 100644 --- a/components/raftstore/src/router.rs +++ b/components/raftstore/src/router.rs @@ -15,13 +15,13 @@ use slog_global::warn; use tikv_util::time::ThreadReadId; use crate::{ + DiscardReason, Error as RaftStoreError, Result as RaftStoreResult, store::{ - fsm::{ChangeObserver, RaftRouter}, - transport::{CasualRouter, ProposalRouter, SignificantRouter}, Callback, CasualMessage, LocalReader, PeerMsg, RaftCmdExtraOpts, RaftCommand, SignificantMsg, StoreMsg, StoreRouter, + fsm::{ChangeObserver, RaftRouter}, + transport::{CasualRouter, ProposalRouter, SignificantRouter}, }, - DiscardReason, Error as RaftStoreError, Result as RaftStoreResult, }; /// Routes messages to the raftstore. pub trait RaftStoreRouter: diff --git a/components/raftstore/src/store/async_io/read.rs b/components/raftstore/src/store/async_io/read.rs index 006fe0eb24c..cddcce2ea3f 100644 --- a/components/raftstore/src/store/async_io/read.rs +++ b/components/raftstore/src/store/async_io/read.rs @@ -4,8 +4,8 @@ use std::{ fmt, marker::PhantomData, sync::{ - atomic::{AtomicBool, Ordering}, Arc, + atomic::{AtomicBool, Ordering}, }, }; @@ -14,15 +14,15 @@ use fail::fail_point; use file_system::{IoType, WithIoType}; use kvproto::raft_serverpb::{PeerState, RaftSnapshotData, RegionLocalState}; use protobuf::Message; -use raft::{eraftpb::Snapshot, GetEntriesContext}; +use raft::{GetEntriesContext, eraftpb::Snapshot}; use tikv_util::{error, info, time::Instant, worker::Runnable}; use crate::store::{ + MAX_INIT_ENTRY_COUNT, RaftlogFetchResult, TabletSnapKey, TabletSnapManager, metrics::{SNAPSHOT_KV_COUNT_HISTOGRAM, SNAPSHOT_SIZE_HISTOGRAM}, snap::TABLET_SNAPSHOT_VERSION, util, worker::metrics::{SNAP_COUNTER, SNAP_HISTOGRAM}, - RaftlogFetchResult, TabletSnapKey, TabletSnapManager, MAX_INIT_ENTRY_COUNT, }; pub enum ReadTask { diff --git a/components/raftstore/src/store/async_io/write.rs b/components/raftstore/src/store/async_io/write.rs index 7f32b1aaf71..9f0a21159cd 100644 --- a/components/raftstore/src/store/async_io/write.rs +++ b/components/raftstore/src/store/async_io/write.rs @@ -21,7 +21,7 @@ use engine_traits::{ }; use error_code::ErrorCodeExt; use fail::fail_point; -use file_system::{set_io_type, IoType}; +use file_system::{IoType, set_io_type}; use health_controller::types::LatencyInspector; use kvproto::{ metapb::RegionEpoch, @@ -31,8 +31,8 @@ use parking_lot::Mutex; use protobuf::Message; use raft::eraftpb::Entry; use resource_control::{ - channel::{bounded, Receiver}, ResourceConsumeType, ResourceController, ResourceMetered, + channel::{Receiver, bounded}, }; use tikv_util::{ box_err, @@ -40,22 +40,23 @@ use tikv_util::{ debug, info, slow_log, sys::thread::StdThreadBuildWrapper, thd_name, - time::{duration_to_sec, setup_for_spin_interval, spin_at_least, Duration, Instant}, + time::{Duration, Instant, duration_to_sec, setup_for_spin_interval, spin_at_least}, warn, }; use tracker::TrackerTokenArray; use super::write_router::{SharedSenders, WriteSenders}; use crate::{ + Result, store::{ + PeerMsg, config::Config, fsm::RaftRouter, local_metrics::{RaftSendMessageMetrics, StoreWriteMetrics, TimeTracker}, metrics::*, transport::Transport, - util, PeerMsg, + util, }, - Result, }; const KV_WB_SHRINK_SIZE: usize = 1024 * 1024; @@ -237,7 +238,7 @@ where !(self.raft_state.is_none() && self.entries.is_empty() && self.extra_write.is_empty() - && self.raft_wb.as_ref().map_or(true, |wb| wb.is_empty())) + && self.raft_wb.as_ref().is_none_or(|wb| wb.is_empty())) } /// Append continous entries. diff --git a/components/raftstore/src/store/async_io/write_router.rs b/components/raftstore/src/store/async_io/write_router.rs index 3669fddd613..0747c63a441 100644 --- a/components/raftstore/src/store/async_io/write_router.rs +++ b/components/raftstore/src/store/async_io/write_router.rs @@ -7,8 +7,8 @@ use std::{ mem, ops::Index, sync::{ - atomic::{AtomicUsize, Ordering}, Arc, + atomic::{AtomicUsize, Ordering}, }, time::Duration, }; @@ -123,7 +123,7 @@ where ctx: &mut C, persisted_number: u64, ) { - if self.last_unpersisted.map_or(true, |n| n > persisted_number) { + if self.last_unpersisted.is_none_or(|n| n > persisted_number) { return; } // The peer must be destroyed after all previous write tasks have been finished. @@ -354,7 +354,7 @@ pub(crate) mod tests { use std::thread; use engine_test::{kv::KvTestEngine, raft::RaftTestEngine}; - use resource_control::channel::{bounded, Receiver}; + use resource_control::channel::{Receiver, bounded}; use tikv_util::config::ReadableDuration; use super::*; diff --git a/components/raftstore/src/store/async_io/write_tests.rs b/components/raftstore/src/store/async_io/write_tests.rs index aa6bd8aca3c..7aaa578c9e3 100644 --- a/components/raftstore/src/store/async_io/write_tests.rs +++ b/components/raftstore/src/store/async_io/write_tests.rs @@ -3,7 +3,7 @@ use std::{sync::mpsc, time::Duration}; use collections::HashSet; -use crossbeam::channel::{unbounded, Receiver, Sender}; +use crossbeam::channel::{Receiver, Sender, unbounded}; use engine_test::{kv::KvTestEngine, new_temp_engine, raft::RaftTestEngine}; use engine_traits::{Engines, Mutable, Peekable, RaftEngineReadOnly, WriteBatchExt}; use kvproto::{ @@ -16,11 +16,11 @@ use tempfile::Builder; use super::*; use crate::{ + Result, store::{ - async_io::write_router::tests::TestContext, local_metrics::RaftMetrics, - peer_storage::tests::new_entry, Config, Transport, WriteRouter, + Config, Transport, WriteRouter, async_io::write_router::tests::TestContext, + local_metrics::RaftMetrics, peer_storage::tests::new_entry, }, - Result, }; type TestKvWriteBatch = ::WriteBatch; type TestRaftLogBatch = ::LogBatch; diff --git a/components/raftstore/src/store/bootstrap.rs b/components/raftstore/src/store/bootstrap.rs index 249ae4b704f..f959e754788 100644 --- a/components/raftstore/src/store/bootstrap.rs +++ b/components/raftstore/src/store/bootstrap.rs @@ -1,6 +1,6 @@ // Copyright 2016 TiKV Project Authors. Licensed under Apache-2.0. -use engine_traits::{Engines, KvEngine, Mutable, RaftEngine, WriteBatch, CF_DEFAULT, CF_RAFT}; +use engine_traits::{CF_DEFAULT, CF_RAFT, Engines, KvEngine, Mutable, RaftEngine, WriteBatch}; use kvproto::{ metapb, raft_serverpb::{RaftLocalState, RegionLocalState, StoreIdent}, @@ -8,7 +8,7 @@ use kvproto::{ use tikv_util::{box_err, box_try, store::new_peer}; use super::peer_storage::{ - write_initial_apply_state, write_initial_raft_state, INIT_EPOCH_CONF_VER, INIT_EPOCH_VER, + INIT_EPOCH_CONF_VER, INIT_EPOCH_VER, write_initial_apply_state, write_initial_raft_state, }; use crate::Result; @@ -123,7 +123,7 @@ pub fn clear_prepare_bootstrap_key( #[cfg(test)] mod tests { use engine_traits::{ - Engines, Peekable, RaftEngineDebug, RaftEngineReadOnly, RaftLogBatch, CF_DEFAULT, + CF_DEFAULT, Engines, Peekable, RaftEngineDebug, RaftEngineReadOnly, RaftLogBatch, }; use tempfile::Builder; diff --git a/components/raftstore/src/store/compaction_guard.rs b/components/raftstore/src/store/compaction_guard.rs index 992f0856556..2f6aea36df5 100644 --- a/components/raftstore/src/store/compaction_guard.rs +++ b/components/raftstore/src/store/compaction_guard.rs @@ -3,15 +3,15 @@ use std::ffi::CString; use engine_traits::{ - CfName, SstPartitioner, SstPartitionerContext, SstPartitionerFactory, SstPartitionerRequest, - SstPartitionerResult, CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE, + CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE, CfName, SstPartitioner, SstPartitionerContext, + SstPartitionerFactory, SstPartitionerRequest, SstPartitionerResult, }; use keys::{data_end_key, origin_key}; use lazy_static::lazy_static; use tikv_util::warn; use super::metrics::*; -use crate::{coprocessor::RegionInfoProvider, Error, Result}; +use crate::{Error, Result, coprocessor::RegionInfoProvider}; const COMPACTION_GUARD_MAX_POS_SKIP: u32 = 10; @@ -270,13 +270,13 @@ mod tests { use collections::HashMap; use engine_rocks::{ + RocksCfOptions, RocksDbOptions, RocksEngine, RocksSstPartitionerFactory, RocksSstReader, raw::{BlockBasedOptions, DBCompressionType}, util::new_engine_opt, - RocksCfOptions, RocksDbOptions, RocksEngine, RocksSstPartitionerFactory, RocksSstReader, }; use engine_traits::{ - CompactExt, IterOptions, Iterator, ManualCompactionOptions, MiscExt, RefIterable, - SstReader, SyncMutable, CF_DEFAULT, + CF_DEFAULT, CompactExt, IterOptions, Iterator, ManualCompactionOptions, MiscExt, + RefIterable, SstReader, SyncMutable, }; use keys::DATA_PREFIX_KEY; use kvproto::metapb::Region; diff --git a/components/raftstore/src/store/config.rs b/components/raftstore/src/store/config.rs index ebc57653ecc..b11cc8c600e 100644 --- a/components/raftstore/src/store/config.rs +++ b/components/raftstore/src/store/config.rs @@ -1,9 +1,9 @@ // Copyright 2016 TiKV Project Authors. Licensed under Apache-2.0. -use std::{cmp::min, collections::HashMap, sync::Arc, time::Duration, u64}; +use std::{cmp::min, collections::HashMap, sync::Arc, time::Duration}; use batch_system::Config as BatchSystemConfig; -use engine_traits::{perf_level_serde, PerfLevel}; +use engine_traits::{PerfLevel, perf_level_serde}; use lazy_static::lazy_static; use online_config::{ConfigChange, ConfigManager, ConfigValue, OnlineConfig}; use prometheus::register_gauge_vec; @@ -20,7 +20,7 @@ use tikv_util::{ use time::Duration as TimeDuration; use super::worker::{RaftStoreBatchComponent, RefreshConfigTask}; -use crate::{coprocessor::config::RAFTSTORE_V2_SPLIT_SIZE, Result}; +use crate::{Result, coprocessor::config::RAFTSTORE_V2_SPLIT_SIZE}; lazy_static! { pub static ref CONFIG_RAFTSTORE_GAUGE: prometheus::GaugeVec = register_gauge_vec!( @@ -507,12 +507,12 @@ impl Default for Config { leader_transfer_max_log_lag: 128, snap_apply_batch_size: ReadableSize::mb(10), snap_apply_copy_symlink: false, - region_worker_tick_interval: if cfg!(feature = "test") { + region_worker_tick_interval: if cfg!(test) { ReadableDuration::millis(200) } else { ReadableDuration::millis(1000) }, - clean_stale_ranges_tick: if cfg!(feature = "test") { 1 } else { 10 }, + clean_stale_ranges_tick: if cfg!(test) { 1 } else { 10 }, lock_cf_compact_interval: ReadableDuration::minutes(10), lock_cf_compact_bytes_threshold: ReadableSize::mb(256), // Disable consistency check by default as it will hurt performance. diff --git a/components/raftstore/src/store/entry_storage.rs b/components/raftstore/src/store/entry_storage.rs index d43de79a2cf..043874f033c 100644 --- a/components/raftstore/src/store/entry_storage.rs +++ b/components/raftstore/src/store/entry_storage.rs @@ -15,7 +15,7 @@ use std::{ }; use collections::HashMap; -use engine_traits::{KvEngine, RaftEngine, RAFT_LOG_MULTI_GET_CNT}; +use engine_traits::{KvEngine, RAFT_LOG_MULTI_GET_CNT, RaftEngine}; use fail::fail_point; use kvproto::{ metapb, @@ -23,15 +23,15 @@ use kvproto::{ }; use prometheus::local::LocalHistogram; use protobuf::Message; -use raft::{prelude::*, util::limit_size, GetEntriesContext, StorageError}; +use raft::{GetEntriesContext, StorageError, prelude::*, util::limit_size}; use tikv_alloc::TraceEvent; use tikv_util::{box_err, debug, error, info, time::Instant, warn, worker::Scheduler}; use super::{ - local_metrics::RaftMetrics, metrics::*, peer_storage::storage_error, WriteTask, - MEMTRACE_ENTRY_CACHE, RAFT_INIT_LOG_INDEX, RAFT_INIT_LOG_TERM, + MEMTRACE_ENTRY_CACHE, RAFT_INIT_LOG_INDEX, RAFT_INIT_LOG_TERM, WriteTask, + local_metrics::RaftMetrics, metrics::*, peer_storage::storage_error, }; -use crate::{bytes_capacity, store::ReadTask, Result}; +use crate::{Result, bytes_capacity, store::ReadTask}; const MAX_ASYNC_FETCH_TRY_CNT: usize = 3; const SHRINK_CACHE_CAPACITY: usize = 64; @@ -482,7 +482,7 @@ fn validate_states( // If so, forward the commit index. if commit_index < recorded_commit_index { let entry = raft_engine.get_entry(region_id, recorded_commit_index)?; - if entry.map_or(true, |e| e.get_term() != apply_state.get_commit_term()) { + if entry.is_none_or(|e| e.get_term() != apply_state.get_commit_term()) { return Err(box_err!( "log at recorded commit index [{}] {} doesn't exist, may lose data, {}", apply_state.get_commit_term(), @@ -1298,7 +1298,7 @@ pub mod tests { use protobuf::Message; use raft::{GetEntriesContext, StorageError}; use tempfile::Builder; - use tikv_util::worker::{dummy_scheduler, LazyWorker, Worker}; + use tikv_util::worker::{LazyWorker, Worker, dummy_scheduler}; use super::*; use crate::store::peer_storage::tests::{append_ents, new_entry, new_storage_from_ents}; @@ -1444,7 +1444,7 @@ pub mod tests { let mut store = new_storage_from_ents(region_scheduler, dummy_scheduler, &td, &ents); - let max_u64 = u64::max_value(); + let max_u64 = u64::MAX; let mut tests = vec![ // already compacted ( @@ -1708,7 +1708,7 @@ pub mod tests { append_ents(&mut store, &entries); let li = store.last_index().unwrap(); let actual_entries = store - .entries(4, li + 1, u64::max_value(), GetEntriesContext::empty(false)) + .entries(4, li + 1, u64::MAX, GetEntriesContext::empty(false)) .unwrap(); if actual_entries != wentries { panic!("#{}: want {:?}, got {:?}", i, wentries, actual_entries); @@ -1727,7 +1727,7 @@ pub mod tests { store.cache.cache.clear(); // empty cache should fetch data from rocksdb directly. let mut res = store - .entries(4, 6, u64::max_value(), GetEntriesContext::empty(false)) + .entries(4, 6, u64::MAX, GetEntriesContext::empty(false)) .unwrap(); assert_eq!(*res, ents[1..]); @@ -1737,7 +1737,7 @@ pub mod tests { // direct cache access res = store - .entries(6, 8, u64::max_value(), GetEntriesContext::empty(false)) + .entries(6, 8, u64::MAX, GetEntriesContext::empty(false)) .unwrap(); assert_eq!(res, entries); @@ -1765,7 +1765,7 @@ pub mod tests { for low in 4..9 { for high in low..9 { let res = store - .entries(low, high, u64::max_value(), GetEntriesContext::empty(false)) + .entries(low, high, u64::MAX, GetEntriesContext::empty(false)) .unwrap(); assert_eq!(*res, exp_res[low as usize - 4..high as usize - 4]); } diff --git a/components/raftstore/src/store/fsm/apply.rs b/components/raftstore/src/store/fsm/apply.rs index 21b6a36b088..893fc03d52e 100644 --- a/components/raftstore/src/store/fsm/apply.rs +++ b/components/raftstore/src/store/fsm/apply.rs @@ -13,12 +13,11 @@ use std::{ mem, ops::{Deref, DerefMut, Range as StdRange}, sync::{ + Arc, Mutex, atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, mpsc::SyncSender, - Arc, Mutex, }, time::Duration, - usize, vec::Drain, }; @@ -28,10 +27,11 @@ use batch_system::{ }; use collections::{HashMap, HashMapEntry, HashSet}; use crossbeam::channel::{TryRecvError, TrySendError}; +use derive_more::Debug as DebugMore; use engine_traits::{ - util::SequenceNumber, DeleteStrategy, KvEngine, Mutable, PerfContext, PerfContextKind, - RaftEngine, RaftEngineReadOnly, Range as EngineRange, Snapshot, SstMetaInfo, WriteBatch, - WriteOptions, ALL_CFS, CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE, + ALL_CFS, CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE, DeleteStrategy, KvEngine, Mutable, + PerfContext, PerfContextKind, RaftEngine, RaftEngineReadOnly, Range as EngineRange, Snapshot, + SstMetaInfo, WriteBatch, WriteOptions, util::SequenceNumber, }; use fail::fail_point; use health_controller::types::LatencyInspector; @@ -47,42 +47,41 @@ use kvproto::{ }; use pd_client::{BucketMeta, BucketStat}; use prometheus::local::LocalHistogram; -use protobuf::{wire_format::WireType, CodedInputStream, Message}; +use protobuf::{CodedInputStream, Message, wire_format::WireType}; use raft::eraftpb::{ ConfChange, ConfChangeType, ConfChangeV2, Entry, EntryType, Snapshot as RaftSnapshot, }; use raft_proto::ConfChangeI; use resource_control::{ResourceConsumeType, ResourceController, ResourceMetered}; -use smallvec::{smallvec, SmallVec}; +use smallvec::{SmallVec, smallvec}; use sst_importer::SstImporter; use tikv_alloc::trace::TraceEvent; use tikv_util::{ - box_err, box_try, + Either, MustConsumeVec, box_err, box_try, config::{Tracker, VersionTrack}, debug, error, info, memory::HeapSize, - mpsc::{loose_bounded, LooseBoundedSender, Receiver}, + mpsc::{LooseBoundedSender, Receiver, loose_bounded}, safe_panic, slow_log, store::{find_peer, find_peer_by_id, find_peer_mut, is_learner, remove_peer}, - time::{duration_to_sec, Instant}, + time::{Instant, duration_to_sec}, warn, worker::Scheduler, - Either, MustConsumeVec, }; use time::Timespec; -use tracker::{TrackerToken, TrackerTokenArray, GLOBAL_TRACKERS}; +use tracker::{GLOBAL_TRACKERS, TrackerToken, TrackerTokenArray}; use uuid::Builder as UuidBuilder; use self::memtrace::*; use super::metrics::*; use crate::{ - bytes_capacity, + Error, Result, bytes_capacity, coprocessor::{ ApplyCtxInfo, Cmd, CmdBatch, CmdObserveInfo, CoprocessorHost, ObserveHandle, ObserveLevel, RegionState, WriteBatchWrapper, }, store::{ - cmd_resp, + Config, RegionSnapshot, SnapGenTask, WriteCallback, cmd_resp, entry_storage::{self, CachedEntries}, fsm::RaftPollerBuilder, local_metrics::RaftMetrics, @@ -92,12 +91,10 @@ use crate::{ peer::Peer, peer_storage::{write_initial_apply_state, write_peer_state}, util::{ - self, admin_cmd_epoch_lookup, check_flashback_state, check_req_region_epoch, - compare_region_epoch, ChangePeerI, ConfChangeKind, KeysInfoFormatter, + self, ChangePeerI, ConfChangeKind, KeysInfoFormatter, admin_cmd_epoch_lookup, + check_flashback_state, check_req_region_epoch, compare_region_epoch, }, - Config, RegionSnapshot, SnapGenTask, WriteCallback, }, - Error, Result, }; // These consts are shared in both v1 and v2. @@ -201,7 +198,7 @@ impl PendingCmdQueue { fn pop_compact(&mut self, index: u64) -> Option> { let mut front = None; - while self.compacts.front().map_or(false, |c| c.index < index) { + while self.compacts.front().is_some_and(|c| c.index < index) { front = self.compacts.pop_front(); front.as_mut().unwrap().cb.take().unwrap(); } @@ -1013,8 +1010,7 @@ pub struct NewSplitPeer { /// Region. The apply worker receives all the apply tasks of different Regions /// located at this store, and it will get the corresponding apply delegate to /// handle the apply task to make the code logic more clear. -#[derive(Derivative)] -#[derivative(Debug)] +#[derive(DebugMore)] pub struct ApplyDelegate where EK: KvEngine, @@ -1084,7 +1080,7 @@ where priority: Priority, /// To fetch Raft entries for applying if necessary. - #[derivative(Debug = "ignore")] + #[debug(skip)] raft_engine: Box, trace: ApplyMemoryTrace, @@ -1460,8 +1456,8 @@ where /// /// An apply operation can fail in the following situations: /// - it encounters an error that will occur on all stores, it can - /// continue applying next entry safely, like epoch not match for - /// example; + /// continue applying next entry safely, like epoch not match for + /// example; /// - it encounters an error that may not occur on all stores, in this /// case we should try to apply the entry again or panic. Considering /// that this usually due to disk operation fail, which is rare, so just @@ -1792,7 +1788,7 @@ where ) -> Result<(RaftCmdResponse, ApplyResult)> { fail_point!( "on_apply_write_cmd", - cfg!(release) || self.id() == 3, + !cfg!(debug_assertions) || self.id() == 3, |_| { unimplemented!(); } @@ -1990,7 +1986,7 @@ where if cf.is_empty() { cf = CF_DEFAULT; } - if !ALL_CFS.iter().any(|x| *x == cf) { + if !ALL_CFS.contains(&cf) { return Err(box_err!("invalid delete range command, cf: {:?}", cf)); } @@ -2729,7 +2725,7 @@ where { Ok(None) => (), Ok(Some(state)) => { - if replace_regions.get(region_id).is_some() { + if replace_regions.contains(region_id) { // It's marked replaced, then further destroy will skip cleanup, so there // should be no region local state. panic!( @@ -4649,7 +4645,7 @@ pub struct ControlFsm { impl ControlFsm { pub fn new() -> (LooseBoundedSender, Box) { - let (tx, rx) = loose_bounded(std::usize::MAX); + let (tx, rx) = loose_bounded(usize::MAX); let fsm = Box::new(ControlFsm { stopped: false, receiver: rx, @@ -5184,7 +5180,7 @@ mod tests { use bytes::Bytes; use engine_panic::PanicEngine; - use engine_test::kv::{new_engine, KvTestEngine, KvTestSnapshot}; + use engine_test::kv::{KvTestEngine, KvTestSnapshot, new_engine}; use engine_traits::{Peekable as PeekableTrait, SyncMutable, WriteBatchExt}; use kvproto::{ kvrpcpb::ApiVersion, @@ -5208,10 +5204,10 @@ mod tests { use crate::{ coprocessor::*, store::{ + Config, SnapGenTask, msg::WriteResponse, peer_storage::RAFT_INIT_LOG_INDEX, simple_write::{SimpleWriteEncoder, SimpleWriteReqEncoder}, - Config, SnapGenTask, }, }; @@ -6005,18 +6001,15 @@ mod tests { _: &RegionState, apply_info: &mut ApplyCtxInfo<'_>, ) -> bool { - match apply_info.pending_handle_ssts { - Some(v) => { - // If it is a ingest sst - let mut ssts = std::mem::take(v); - assert_ne!(ssts.len(), 0); - if self.delay_remove_ssts.load(Ordering::SeqCst) { - apply_info.pending_delete_ssts.append(&mut ssts); - } else { - apply_info.delete_ssts.append(&mut ssts); - } + if let Some(v) = apply_info.pending_handle_ssts { + // If it is a ingest sst + let mut ssts = std::mem::take(v); + assert_ne!(ssts.len(), 0); + if self.delay_remove_ssts.load(Ordering::SeqCst) { + apply_info.pending_delete_ssts.append(&mut ssts); + } else { + apply_info.delete_ssts.append(&mut ssts); } - None => (), } self.last_delete_sst_count .store(apply_info.delete_ssts.len() as u64, Ordering::SeqCst); diff --git a/components/raftstore/src/store/fsm/life.rs b/components/raftstore/src/store/fsm/life.rs index e95f8978338..4cbbf194739 100644 --- a/components/raftstore/src/store/fsm/life.rs +++ b/components/raftstore/src/store/fsm/life.rs @@ -3,7 +3,7 @@ //! This module contains functions that relates to peer liftime management and //! are shared with raftstore and raftstore v2. -use engine_traits::{KvEngine, CF_RAFT}; +use engine_traits::{CF_RAFT, KvEngine}; use kvproto::raft_serverpb::{ExtraMessageType, PeerState, RaftMessage, RegionLocalState}; use tikv_util::warn; diff --git a/components/raftstore/src/store/fsm/metrics.rs b/components/raftstore/src/store/fsm/metrics.rs index 6ee346bfd75..89c0c6f6a4b 100644 --- a/components/raftstore/src/store/fsm/metrics.rs +++ b/components/raftstore/src/store/fsm/metrics.rs @@ -2,12 +2,12 @@ // #[PerformanceCriticalPath] use std::sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, }; use lazy_static::lazy_static; -use prometheus::{exponential_buckets, register_histogram, Histogram}; +use prometheus::{Histogram, exponential_buckets, register_histogram}; use tikv_util::store::QueryStats; lazy_static! { diff --git a/components/raftstore/src/store/fsm/mod.rs b/components/raftstore/src/store/fsm/mod.rs index e0cd7235d97..811e893b7e4 100644 --- a/components/raftstore/src/store/fsm/mod.rs +++ b/components/raftstore/src/store/fsm/mod.rs @@ -12,14 +12,14 @@ pub mod store; pub use self::{ apply::{ - check_sst_for_ingestion, create_apply_batch_system, Apply, ApplyBatchSystem, ApplyMetrics, - ApplyRes, ApplyRouter, Builder as ApplyPollerBuilder, CatchUpLogs, ChangeObserver, - ChangePeer, ExecResult, GenSnapTask, Msg as ApplyTask, Notifier as ApplyNotifier, Proposal, - Registration, SwitchWitness, TaskRes as ApplyTaskRes, + Apply, ApplyBatchSystem, ApplyMetrics, ApplyRes, ApplyRouter, + Builder as ApplyPollerBuilder, CatchUpLogs, ChangeObserver, ChangePeer, ExecResult, + GenSnapTask, Msg as ApplyTask, Notifier as ApplyNotifier, Proposal, Registration, + SwitchWitness, TaskRes as ApplyTaskRes, check_sst_for_ingestion, create_apply_batch_system, }, metrics::{GlobalStoreStat, LocalStoreStat}, peer::{ - new_admin_request, new_read_index_request, DestroyPeerJob, PeerFsm, MAX_PROPOSAL_SIZE_RATIO, + DestroyPeerJob, MAX_PROPOSAL_SIZE_RATIO, PeerFsm, new_admin_request, new_read_index_request, }, - store::{create_raft_batch_system, RaftBatchSystem, RaftPollerBuilder, RaftRouter, StoreMeta}, + store::{RaftBatchSystem, RaftPollerBuilder, RaftRouter, StoreMeta, create_raft_batch_system}, }; diff --git a/components/raftstore/src/store/fsm/peer.rs b/components/raftstore/src/store/fsm/peer.rs index 49814a14ffd..099b1667bf2 100644 --- a/components/raftstore/src/store/fsm/peer.rs +++ b/components/raftstore/src/store/fsm/peer.rs @@ -11,15 +11,14 @@ use std::{ }, iter::Iterator, mem, - sync::{atomic::Ordering, Arc, Mutex}, + sync::{Arc, Mutex, atomic::Ordering}, time::{Duration, Instant}, - u64, }; use batch_system::{BasicMailbox, Fsm, FsmType}; use collections::{HashMap, HashSet}; use engine_traits::{ - Engines, KvEngine, RaftEngine, RaftLogBatch, SstMetaInfo, WriteBatchExt, CF_LOCK, CF_RAFT, + CF_LOCK, CF_RAFT, Engines, KvEngine, RaftEngine, RaftLogBatch, SstMetaInfo, WriteBatchExt, }; use error_code::ErrorCodeExt; use fail::fail_point; @@ -47,23 +46,22 @@ use parking_lot::RwLockWriteGuard; use pd_client::BucketMeta; use protobuf::Message; use raft::{ - self, + self, GetEntriesContext, INVALID_INDEX, NO_LIMIT, Progress, ReadState, SnapshotStatus, + StateRole, eraftpb::{self, ConfChangeType, MessageType}, - GetEntriesContext, Progress, ReadState, SnapshotStatus, StateRole, INVALID_INDEX, NO_LIMIT, }; use smallvec::SmallVec; use strum::{EnumCount, VariantNames}; use tikv_alloc::trace::TraceEvent; use tikv_util::{ - box_err, debug, defer, error, escape, info, info_or_debug, is_zero_duration, + Either, box_err, debug, defer, error, escape, info, info_or_debug, is_zero_duration, mpsc::{self, LooseBoundedSender, Receiver}, slow_log, store::{find_peer, find_peer_by_id, is_learner, region_on_same_stores}, sys::disk::DiskUsage, - time::{monotonic_raw_now, Instant as TiInstant, SlowTimer}, + time::{Instant as TiInstant, SlowTimer, monotonic_raw_now}, trace, warn, worker::{ScheduleError, Scheduler}, - Either, }; use tracker::GLOBAL_TRACKERS; use txn_types::WriteBatchFlags; @@ -73,15 +71,18 @@ use super::life::forward_destroy_to_source_peer; #[cfg(any(test, feature = "testexport"))] use crate::store::PeerInternalStat; use crate::{ + Error, Result, coprocessor::{RegionChangeEvent, RegionChangeReason}, store::{ + CasualMessage, Config, LocksStatus, MergeResultKind, PdTask, PeerMsg, PeerTick, + ProposalContext, RAFT_INIT_LOG_INDEX, RaftCmdExtraOpts, RaftCommand, RaftlogFetchResult, + ReadCallback, ReadIndexContext, ReadTask, SignificantMsg, SnapKey, StoreMsg, WriteCallback, cmd_resp::{bind_term, new_error}, demote_failed_voters_request, fsm::{ - apply, - store::{PollContext, StoreMeta}, ApplyMetrics, ApplyTask, ApplyTaskRes, CatchUpLogs, ChangeObserver, ChangePeer, - ExecResult, SwitchWitness, + ExecResult, SwitchWitness, apply, + store::{PollContext, StoreMeta}, }, hibernate_state::{GroupState, HibernateState}, local_metrics::{RaftMetrics, TimeTracker}, @@ -93,21 +94,16 @@ use crate::{ snapshot_backup::{AbortReason, SnapshotBrState, SnapshotBrWaitApplyRequest}, transport::Transport, unsafe_recovery::{ - exit_joint_request, ForceLeaderState, UnsafeRecoveryExecutePlanSyncer, - UnsafeRecoveryFillOutReportSyncer, UnsafeRecoveryForceLeaderSyncer, - UnsafeRecoveryState, UnsafeRecoveryWaitApplySyncer, + ForceLeaderState, UnsafeRecoveryExecutePlanSyncer, UnsafeRecoveryFillOutReportSyncer, + UnsafeRecoveryForceLeaderSyncer, UnsafeRecoveryState, UnsafeRecoveryWaitApplySyncer, + exit_joint_request, }, - util::{self, compare_region_epoch, KeysInfoFormatter, LeaseState}, + util::{self, KeysInfoFormatter, LeaseState, compare_region_epoch}, worker::{ Bucket, BucketRange, CleanupTask, ConsistencyCheckTask, GcSnapshotTask, RaftlogGcTask, ReadDelegate, ReadProgress, RegionTask, SplitCheckTask, }, - CasualMessage, Config, LocksStatus, MergeResultKind, PdTask, PeerMsg, PeerTick, - ProposalContext, RaftCmdExtraOpts, RaftCommand, RaftlogFetchResult, ReadCallback, - ReadIndexContext, ReadTask, SignificantMsg, SnapKey, StoreMsg, WriteCallback, - RAFT_INIT_LOG_INDEX, }, - Error, Result, }; #[derive(Clone, Copy, Debug)] @@ -2052,7 +2048,7 @@ where .peer .uncampaigned_new_regions .as_ref() - .map_or(false, |r| r.is_empty()); + .is_some_and(|r| r.is_empty()); // If the peer has any uncleared records in the uncampaigned_new_regions list, // and there has valid leader in the region, it's safely to clear the records. if has_uncompaigned_regions && self.fsm.peer.has_valid_leader() { @@ -2650,10 +2646,7 @@ where } let disk_full_peers = &self.fsm.peer.disk_full_peers; - disk_full_peers.is_empty() - || disk_full_peers - .get(peer_id) - .map_or(true, |x| x != msg.disk_usage) + disk_full_peers.is_empty() || (disk_full_peers.get(peer_id) != Some(msg.disk_usage)) }; if refill_disk_usages || self.fsm.peer.has_region_merge_proposal { let prev = self.fsm.peer.disk_full_peers.get(peer_id); @@ -4051,7 +4044,7 @@ where if self .fsm .delayed_destroy - .map_or(false, |delay| delay.reason == reason) + .is_some_and(|delay| delay.reason == reason) { panic!( "{} destroy peer twice with same delay reason, original {:?}, now {}", @@ -4239,7 +4232,7 @@ where // peer decide to destroy by itself. Without target, the // `pending_merge_targets` for target won't be removed, so here source peer help // target to clear. - if meta.regions.get(&target).is_none() + if !meta.regions.contains_key(&target) && meta.pending_merge_targets.get(&target).unwrap().is_empty() { meta.pending_merge_targets.remove(&target); @@ -5273,16 +5266,12 @@ where target: metapb::Peer, result: MergeResultKind, ) { - let exists = self - .fsm - .peer - .pending_merge_state - .as_ref() - .map_or(true, |s| { - s.get_target().get_peers().iter().any(|p| { - p.get_store_id() == target.get_store_id() && p.get_id() <= target.get_id() - }) - }); + let exists = self.fsm.peer.pending_merge_state.as_ref().is_none_or(|s| { + s.get_target() + .get_peers() + .iter() + .any(|p| p.get_store_id() == target.get_store_id() && p.get_id() <= target.get_id()) + }); if !exists { panic!( "{} unexpected merge result: {:?} {:?} {:?}", @@ -6302,7 +6291,7 @@ where #[inline] fn region_split_skip_max_count(&self) -> usize { - fail_point!("region_split_skip_max_count", |_| { usize::max_value() }); + fail_point!("region_split_skip_max_count", |_| { usize::MAX }); REGION_SPLIT_SKIP_MAX_COUNT } @@ -7528,8 +7517,8 @@ mod memtrace { #[cfg(test)] mod tests { use std::sync::{ - atomic::{AtomicBool, Ordering}, Arc, + atomic::{AtomicBool, Ordering}, }; use engine_test::kv::KvTestEngine; diff --git a/components/raftstore/src/store/fsm/store.rs b/components/raftstore/src/store/fsm/store.rs index 99a03b2d2cc..8b9ae185747 100644 --- a/components/raftstore/src/store/fsm/store.rs +++ b/components/raftstore/src/store/fsm/store.rs @@ -11,11 +11,10 @@ use std::{ mem, ops::{Deref, DerefMut}, sync::{ - atomic::{AtomicU64, Ordering}, Arc, Mutex, + atomic::{AtomicU64, Ordering}, }, time::{Duration, Instant, SystemTime}, - u64, }; use batch_system::{ @@ -27,15 +26,15 @@ use collections::{HashMap, HashMapEntry, HashSet}; use concurrency_manager::ConcurrencyManager; use crossbeam::channel::{TryRecvError, TrySendError}; use engine_traits::{ - CompactedEvent, DeleteStrategy, Engines, KvEngine, Mutable, PerfContextKind, RaftEngine, - RaftLogBatch, Range, WriteBatch, WriteOptions, CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE, + CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE, CompactedEvent, DeleteStrategy, Engines, KvEngine, + Mutable, PerfContextKind, RaftEngine, RaftLogBatch, Range, WriteBatch, WriteOptions, }; use fail::fail_point; use file_system::{IoType, WithIoType}; -use futures::{compat::Future01CompatExt, FutureExt}; +use futures::{FutureExt, compat::Future01CompatExt}; use health_controller::{ - types::{InspectFactor, LatencyInspector}, HealthController, + types::{InspectFactor, LatencyInspector}, }; use itertools::Itertools; use keys::{self, data_end_key, data_key, enc_end_key, enc_start_key}; @@ -49,14 +48,14 @@ use kvproto::{ use pd_client::{Feature, FeatureGate, PdClient}; use protobuf::Message; use raft::StateRole; -use resource_control::{channel::unbounded, ResourceGroupManager}; +use resource_control::{ResourceGroupManager, channel::unbounded}; use resource_metering::CollectorRegHandle; use service::service_manager::GrpcServiceManager; use sst_importer::SstImporter; use strum::{EnumCount, VariantNames}; use tikv_alloc::trace::TraceEvent; use tikv_util::{ - box_try, + Either, GLOBAL_SERVER_READINESS, RingQueue, box_try, config::{Tracker, VersionTrack}, debug, defer, error, future::poll_future_notify, @@ -67,21 +66,23 @@ use tikv_util::{ sys::{ self as sys_util, cpu_time::ProcessStat, - disk::{get_disk_status, DiskUsage}, + disk::{DiskUsage, get_disk_status}, }, - time::{duration_to_sec, monotonic_raw_now, Instant as TiInstant, SlowTimer}, + time::{Instant as TiInstant, SlowTimer, duration_to_sec, monotonic_raw_now}, timer::SteadyTimer, warn, worker::{Builder as WorkerBuilder, LazyWorker, Scheduler, Worker}, yatp_pool::FuturePool, - Either, RingQueue, GLOBAL_SERVER_READINESS, }; use time::{self, Timespec}; use crate::{ - bytes_capacity, + Error, Result, bytes_capacity, coprocessor::{CoprocessorHost, RegionChangeEvent, RegionChangeReason}, store::{ + Callback, CasualMessage, CompactThreshold, FullCompactController, GlobalReplicationState, + InspectedRaftMessage, MergeResultKind, PdTask, PeerMsg, PeerTick, RaftCommand, + SignificantMsg, SnapManager, StoreMsg, StoreTick, async_io::{ read::{ReadRunner, ReadTask}, write::{StoreWriters, StoreWritersContext, Worker as WriteWorker, WriteMsg}, @@ -89,14 +90,13 @@ use crate::{ }, config::Config, fsm::{ - create_apply_batch_system, + ApplyBatchSystem, ApplyNotifier, ApplyPollerBuilder, ApplyRes, ApplyRouter, + ApplyTaskRes, create_apply_batch_system, life::handle_tombstone_message_on_learner, metrics::*, peer::{ - maybe_destroy_source, new_admin_request, PeerFsm, PeerFsmDelegate, SenderFsmPair, + PeerFsm, PeerFsmDelegate, SenderFsmPair, maybe_destroy_source, new_admin_request, }, - ApplyBatchSystem, ApplyNotifier, ApplyPollerBuilder, ApplyRes, ApplyRouter, - ApplyTaskRes, }, local_metrics::{IoType as InspectIoType, RaftMetrics}, memory::*, @@ -104,21 +104,17 @@ use crate::{ peer_storage, transport::Transport, util, - util::{is_initial_msg, RegionReadProgressRegistry}, + util::{RegionReadProgressRegistry, is_initial_msg}, worker::{ AutoSplitController, CleanupRunner, CleanupSstRunner, CleanupSstTask, CleanupTask, CompactRunner, CompactTask, ConsistencyCheckRunner, ConsistencyCheckTask, DiskCheckRunner, DiskCheckTask, GcSnapshotRunner, GcSnapshotTask, PdRunner, RaftlogGcRunner, RaftlogGcTask, ReadDelegate, RefreshConfigRunner, RefreshConfigTask, - RegionRunner, RegionTask, SnapGenRunner, SnapGenTask, SplitCheckTask, - SNAP_GENERATOR_MAX_POOL_SIZE, + RegionRunner, RegionTask, SNAP_GENERATOR_MAX_POOL_SIZE, SnapGenRunner, SnapGenTask, + SplitCheckTask, }, worker_metrics::PROCESS_STAT_CPU_USAGE, - Callback, CasualMessage, CompactThreshold, FullCompactController, GlobalReplicationState, - InspectedRaftMessage, MergeResultKind, PdTask, PeerMsg, PeerTick, RaftCommand, - SignificantMsg, SnapManager, StoreMsg, StoreTick, }, - Error, Result, }; type Key = Vec; @@ -269,7 +265,7 @@ impl StoreMeta { reason: RegionChangeReason, ) { let prev = self.regions.insert(region.get_id(), region.clone()); - if prev.map_or(true, |r| r.get_id() != region.get_id()) { + if prev.is_none_or(|r| r.get_id() != region.get_id()) { // TODO: may not be a good idea to panic when holding a lock. panic!("{} region corrupted", peer.tag); } @@ -2683,15 +2679,15 @@ impl<'a, EK: KvEngine, ER: RaftEngine, T: Transport> StoreFsmDelegate<'a, EK, ER /// Returns a predicate `Fn` which is evaluated: /// 1. Before full compaction runs: if `false`, we return and wait for the - /// next full compaction tick - /// (`PERIODIC_FULL_COMPACT_TICK_INTERVAL_DURATION`) before starting. If - /// true, we begin full compaction, which means the first incremental range - /// will be compactecd. See: ``StoreFsmDelegate::on_full_compact_tick`` - /// in this file. + /// next full compaction tick + /// (`PERIODIC_FULL_COMPACT_TICK_INTERVAL_DURATION`) before starting. If + /// true, we begin full compaction, which means the first incremental + /// range will be compactecd. See: + /// ``StoreFsmDelegate::on_full_compact_tick`` in this file. /// /// 2. After each incremental range finishes and before next one (if any) - /// starts. If `false`, we pause compaction and wait. See: - /// `CompactRunner::full_compact` in `worker/compact.rs`. + /// starts. If `false`, we pause compaction and wait. See: + /// `CompactRunner::full_compact` in `worker/compact.rs`. fn is_low_load_for_full_compact(&self) -> impl Fn() -> bool { let max_start_cpu_usage = self.ctx.cfg.periodic_full_compact_start_max_cpu; let global_stat = self.ctx.global_stat.clone(); diff --git a/components/raftstore/src/store/local_metrics.rs b/components/raftstore/src/store/local_metrics.rs index 2f8f36a5634..02b16850597 100644 --- a/components/raftstore/src/store/local_metrics.rs +++ b/components/raftstore/src/store/local_metrics.rs @@ -7,7 +7,7 @@ use collections::HashSet; use prometheus::local::{LocalHistogram, LocalIntCounter}; use raft::eraftpb::MessageType; use tikv_util::time::{Duration, Instant}; -use tracker::{Tracker, TrackerToken, GLOBAL_TRACKERS, INVALID_TRACKER_TOKEN}; +use tracker::{GLOBAL_TRACKERS, INVALID_TRACKER_TOKEN, Tracker, TrackerToken}; use super::metrics::*; diff --git a/components/raftstore/src/store/mod.rs b/components/raftstore/src/store/mod.rs index c28d8806f9a..a971be2e630 100644 --- a/components/raftstore/src/store/mod.rs +++ b/components/raftstore/src/store/mod.rs @@ -35,8 +35,8 @@ pub use self::{ async_io::{ read::{AsyncReadNotifier, FetchedLogs, GenSnapRes, ReadRunner, ReadTask}, write::{ - write_to_db_for_test, PersistedNotifier, StoreWriters, StoreWritersContext, - Worker as WriteWorker, WriteMsg, WriteTask, + PersistedNotifier, StoreWriters, StoreWritersContext, Worker as WriteWorker, WriteMsg, + WriteTask, write_to_db_for_test, }, write_router::{WriteRouter, WriteRouterContext, WriteSenders}, }, @@ -46,8 +46,8 @@ pub use self::{ }, compaction_guard::CompactionGuardGeneratorFactory, config::Config, - entry_storage::{EntryStorage, RaftlogFetchResult, MAX_INIT_ENTRY_COUNT}, - fsm::{check_sst_for_ingestion, DestroyPeerJob, RaftRouter}, + entry_storage::{EntryStorage, MAX_INIT_ENTRY_COUNT, RaftlogFetchResult}, + fsm::{DestroyPeerJob, RaftRouter, check_sst_for_ingestion}, hibernate_state::{GroupState, HibernateState}, memory::*, metrics::RAFT_ENTRY_FETCHES_VEC, @@ -57,45 +57,45 @@ pub use self::{ StoreMsg, StoreTick, WriteCallback, WriteResponse, }, peer::{ - can_amend_read, get_sync_log_from_request, make_transfer_leader_response, - propose_read_index, should_renew_lease, DiskFullPeers, Peer, PeerStat, ProposalContext, - ProposalQueue, RequestInspector, RequestPolicy, TransferLeaderContext, TransferLeaderState, + DiskFullPeers, Peer, PeerStat, ProposalContext, ProposalQueue, RequestInspector, + RequestPolicy, TransferLeaderContext, TransferLeaderState, can_amend_read, + get_sync_log_from_request, make_transfer_leader_response, propose_read_index, + should_renew_lease, }, peer_storage::{ - clear_meta, do_snapshot, write_initial_apply_state, write_initial_raft_state, - write_peer_state, PeerStorage, SnapState, INIT_EPOCH_CONF_VER, INIT_EPOCH_VER, - RAFT_INIT_LOG_INDEX, RAFT_INIT_LOG_TERM, + INIT_EPOCH_CONF_VER, INIT_EPOCH_VER, PeerStorage, RAFT_INIT_LOG_INDEX, RAFT_INIT_LOG_TERM, + SnapState, clear_meta, do_snapshot, write_initial_apply_state, write_initial_raft_state, + write_peer_state, }, read_queue::{ReadIndexContext, ReadIndexQueue, ReadIndexRequest}, region_snapshot::{RegionIterator, RegionSnapshot}, replication_mode::{GlobalReplicationState, StoreGroup}, snap::{ - check_abort, copy_snapshot, - snap_io::{apply_sst_cf_files_by_ingest, build_sst_cf_file_list}, ApplyOptions, CfFile, Error as SnapError, SnapEntry, SnapKey, SnapManager, SnapManagerBuilder, Snapshot, SnapshotStatistics, TabletSnapKey, TabletSnapManager, + check_abort, copy_snapshot, + snap_io::{apply_sst_cf_files_by_ingest, build_sst_cf_file_list}, }, snapshot_backup::SnapshotBrWaitApplySyncer, transport::{CasualRouter, ProposalRouter, SignificantRouter, StoreRouter, Transport}, txn_ext::{LocksStatus, PeerPessimisticLocks, PessimisticLockPair, TxnExt}, unsafe_recovery::{ - demote_failed_voters_request, exit_joint_request, ForceLeaderState, - UnsafeRecoveryExecutePlanSyncer, UnsafeRecoveryFillOutReportSyncer, + ForceLeaderState, UnsafeRecoveryExecutePlanSyncer, UnsafeRecoveryFillOutReportSyncer, UnsafeRecoveryForceLeaderSyncer, UnsafeRecoveryHandle, UnsafeRecoveryState, - UnsafeRecoveryWaitApplySyncer, + UnsafeRecoveryWaitApplySyncer, demote_failed_voters_request, exit_joint_request, }, util::{RegionReadProgress, RegionReadProgressRegistry}, worker::{ - metrics as worker_metrics, need_compact, AutoSplitController, BatchComponent, Bucket, + AutoSplitController, BIG_REGION_CPU_OVERLOAD_THRESHOLD_RATIO, BatchComponent, Bucket, BucketRange, BucketStatsInfo, CachedReadDelegate, CheckLeaderRunner, CheckLeaderTask, - CompactThreshold, DiskCheckRunner, FlowStatistics, FlowStatsReporter, - FullCompactController, KeyEntry, LocalReadContext, LocalReader, LocalReaderCore, - PdStatsMonitor, PdTask, ReadDelegate, ReadExecutor, ReadExecutorProvider, ReadProgress, - ReadStats, RefreshConfigTask, RegionTask, SnapGenTask, SplitCheckRunner, SplitCheckTask, - SplitConfig, SplitConfigManager, SplitInfo, StoreMetaDelegate, StoreStatsReporter, - TrackVer, WriteStats, WriterContoller, BIG_REGION_CPU_OVERLOAD_THRESHOLD_RATIO, - DEFAULT_BIG_REGION_BYTE_THRESHOLD, DEFAULT_BIG_REGION_QPS_THRESHOLD, - DEFAULT_BYTE_THRESHOLD, DEFAULT_QPS_THRESHOLD, NUM_COLLECT_STORE_INFOS_PER_HEARTBEAT, - REGION_CPU_OVERLOAD_THRESHOLD_RATIO, + CompactThreshold, DEFAULT_BIG_REGION_BYTE_THRESHOLD, DEFAULT_BIG_REGION_QPS_THRESHOLD, + DEFAULT_BYTE_THRESHOLD, DEFAULT_QPS_THRESHOLD, DiskCheckRunner, FlowStatistics, + FlowStatsReporter, FullCompactController, KeyEntry, LocalReadContext, LocalReader, + LocalReaderCore, NUM_COLLECT_STORE_INFOS_PER_HEARTBEAT, PdStatsMonitor, PdTask, + REGION_CPU_OVERLOAD_THRESHOLD_RATIO, ReadDelegate, ReadExecutor, ReadExecutorProvider, + ReadProgress, ReadStats, RefreshConfigTask, RegionTask, SnapGenTask, SplitCheckRunner, + SplitCheckTask, SplitConfig, SplitConfigManager, SplitInfo, StoreMetaDelegate, + StoreStatsReporter, TrackVer, WriteStats, WriterContoller, metrics as worker_metrics, + need_compact, }, }; diff --git a/components/raftstore/src/store/msg.rs b/components/raftstore/src/store/msg.rs index dc88e881b20..62c5a655366 100644 --- a/components/raftstore/src/store/msg.rs +++ b/components/raftstore/src/store/msg.rs @@ -23,16 +23,17 @@ use kvproto::{ use pd_client::BucketMeta; use raft::SnapshotStatus; use resource_control::ResourceMetered; -use smallvec::{smallvec, SmallVec}; +use smallvec::{SmallVec, smallvec}; use strum::{EnumCount, EnumVariantNames}; use tikv_util::{deadline::Deadline, escape, memory::HeapSize, time::Instant}; -use tracker::{get_tls_tracker_token, TrackerToken}; +use tracker::{TrackerToken, get_tls_tracker_token}; use super::{ - local_metrics::TimeTracker, region_meta::RegionMeta, - snapshot_backup::SnapshotBrWaitApplyRequest, FetchedLogs, RegionSnapshot, + FetchedLogs, RegionSnapshot, local_metrics::TimeTracker, region_meta::RegionMeta, + snapshot_backup::SnapshotBrWaitApplyRequest, }; use crate::store::{ + SnapKey, fsm::apply::{CatchUpLogs, ChangeObserver, TaskRes as ApplyTaskRes}, metrics::RaftEventDurationType, unsafe_recovery::{ @@ -41,7 +42,6 @@ use crate::store::{ }, util::KeysInfoFormatter, worker::{Bucket, BucketRange}, - SnapKey, }; #[derive(Debug)] diff --git a/components/raftstore/src/store/peer.rs b/components/raftstore/src/store/peer.rs index d11f89b9722..51713293e17 100644 --- a/components/raftstore/src/store/peer.rs +++ b/components/raftstore/src/store/peer.rs @@ -7,11 +7,10 @@ use std::{ collections::VecDeque, mem, sync::{ - atomic::{AtomicUsize, Ordering}, Arc, Mutex, + atomic::{AtomicUsize, Ordering}, }, time::{Duration, Instant}, - u64, usize, }; use bitflags::bitflags; @@ -23,8 +22,8 @@ use codec::{ use collections::{HashMap, HashSet}; use crossbeam::{atomic::AtomicCell, channel::TrySendError}; use engine_traits::{ - Engines, KvEngine, PerfContext, RaftEngine, Snapshot, WriteBatch, WriteOptions, CF_DEFAULT, - CF_LOCK, CF_WRITE, + CF_DEFAULT, CF_LOCK, CF_WRITE, Engines, KvEngine, PerfContext, RaftEngine, Snapshot, + WriteBatch, WriteOptions, }; use error_code::ErrorCodeExt; use fail::fail_point; @@ -50,78 +49,75 @@ use parking_lot::RwLockUpgradableReadGuard; use pd_client::{Feature, INVALID_ID}; use protobuf::Message; use raft::{ - self, + self, GetEntriesContext, INVALID_INDEX, LightReady, NO_LIMIT, ProgressState, RawNode, Ready, + SnapshotStatus, StateRole, eraftpb::{self, Entry, EntryType, MessageType}, - GetEntriesContext, LightReady, ProgressState, RawNode, Ready, SnapshotStatus, StateRole, - INVALID_INDEX, NO_LIMIT, }; use rand::seq::SliceRandom; use smallvec::SmallVec; use tikv_alloc::trace::TraceEvent; use tikv_util::{ - box_err, box_try, + Either, box_err, box_try, codec::number::decode_u64, debug, error, info, store::{find_peer_by_id, is_learner}, sys::disk::DiskUsage, - time::{duration_to_sec, monotonic_raw_now, Instant as TiInstant, InstantExt}, + time::{Instant as TiInstant, InstantExt, duration_to_sec, monotonic_raw_now}, warn, worker::Scheduler, - Either, }; use time::{Duration as TimeDuration, Timespec}; -use tracker::{TrackerTokenArray, GLOBAL_TRACKERS}; +use tracker::{GLOBAL_TRACKERS, TrackerTokenArray}; use txn_types::{TimeStamp, WriteBatchFlags}; use uuid::Uuid; use super::{ - cmd_resp, + DestroyPeerJob, LocalReadContext, cmd_resp, local_metrics::{IoType, RaftMetrics}, metrics::*, peer_storage::{ - write_peer_state, CheckApplyingSnapStatus, HandleReadyResult, PeerStorage, - RAFT_INIT_LOG_TERM, + CheckApplyingSnapStatus, HandleReadyResult, PeerStorage, RAFT_INIT_LOG_TERM, + write_peer_state, }, read_queue::{ReadIndexQueue, ReadIndexRequest}, transport::Transport, util::{ - self, check_req_region_epoch, is_initial_msg, AdminCmdEpochState, ChangePeerI, - ConfChangeKind, Lease, LeaseState, NORMAL_REQ_CHECK_CONF_VER, NORMAL_REQ_CHECK_VER, + self, AdminCmdEpochState, ChangePeerI, ConfChangeKind, Lease, LeaseState, + NORMAL_REQ_CHECK_CONF_VER, NORMAL_REQ_CHECK_VER, check_req_region_epoch, is_initial_msg, }, worker::BucketStatsInfo, - DestroyPeerJob, LocalReadContext, }; use crate::{ + Error, Result, coprocessor::{ - split_observer::NO_VALID_SPLIT_KEY, CoprocessorHost, RegionChangeEvent, RegionChangeReason, - RoleChange, TransferLeaderCustomContext, + CoprocessorHost, RegionChangeEvent, RegionChangeReason, RoleChange, + TransferLeaderCustomContext, split_observer::NO_VALID_SPLIT_KEY, }, errors::RAFTSTORE_IS_BUSY, router::{RaftStoreRouter, ReadContext}, store::{ + Callback, Config, GlobalReplicationState, PdTask, PeerMsg, RAFT_INIT_LOG_INDEX, + ReadCallback, ReadIndexContext, ReadResponse, TxnExt, WriteCallback, async_io::{read::ReadTask, write::WriteMsg, write_router::WriteRouter}, entry_storage::CacheWarmupState, fsm::{ + Apply, ApplyMetrics, ApplyTask, Proposal, apply::{self, CatchUpLogs}, store::PollContext, - Apply, ApplyMetrics, ApplyTask, Proposal, }, hibernate_state::GroupState, - memory::{needs_evict_entry_cache, MEMTRACE_RAFT_ENTRIES}, + memory::{MEMTRACE_RAFT_ENTRIES, needs_evict_entry_cache}, msg::{CampaignType, CasualMessage, ErrorCallback, RaftCommand}, peer_storage::HandleSnapshotResult, snapshot_backup::{AbortReason, SnapshotBrState}, txn_ext::LocksStatus, unsafe_recovery::{ForceLeaderState, UnsafeRecoveryState}, - util::{admin_cmd_epoch_lookup, RegionReadProgress}, + util::{RegionReadProgress, admin_cmd_epoch_lookup}, worker::{ CleanupTask, CompactTask, HeartbeatTask, RaftlogGcTask, ReadDelegate, ReadExecutor, ReadProgress, RegionTask, SplitCheckTask, }, - Callback, Config, GlobalReplicationState, PdTask, PeerMsg, ReadCallback, ReadIndexContext, - ReadResponse, TxnExt, WriteCallback, RAFT_INIT_LOG_INDEX, }, - Error, Result, }; const SHRINK_CACHE_CAPACITY: usize = 64; @@ -1776,7 +1772,7 @@ where pub fn should_destroy_after_apply_snapshot(&self) -> bool { self.apply_snap_ctx .as_ref() - .map_or(false, |ctx| ctx.destroy_peer_after_apply) + .is_some_and(|ctx| ctx.destroy_peer_after_apply) } /// Returns `true` if the raft group has replicated a snapshot but not @@ -2628,8 +2624,8 @@ where /// `Peer::on_persist_ready` /// - Schedule the snapshot task to region worker through /// `schedule_applying_snapshot` - /// - Wait for applying snapshot to complete(`check_snap_status`) - /// Then it's valid to handle the next ready. + /// - Wait for applying snapshot to complete(`check_snap_status`) Then it's + /// valid to handle the next ready. fn check_snap_status(&mut self, ctx: &mut PollContext) -> bool { if let Some(snap_ctx) = self.apply_snap_ctx.as_ref() { if !snap_ctx.scheduled { @@ -2763,7 +2759,7 @@ where None => return, }; let cancel_by_unreachable_store = - unreachable_store_id.map_or(false, |s| to_peer.get_store_id() == s); + unreachable_store_id.is_some_and(|s| to_peer.get_store_id() == s); let cancel_by_peer_not_found = find_peer_by_id(self.region(), to_peer.get_id()).is_none(); if cancel_by_unreachable_store || cancel_by_peer_not_found { self.get_store().cancel_generating_snap(None); @@ -4128,7 +4124,7 @@ where pub fn pre_read_index(&self) -> Result<()> { fail_point!( "before_propose_readindex", - |s| if s.map_or(true, |s| s.parse().unwrap_or(true)) { + |s| if s.is_none_or(|s| s.parse().unwrap_or(true)) { Ok(()) } else { Err(box_err!( @@ -4577,7 +4573,7 @@ where poll_ctx .coprocessor_host .pre_propose(self.region(), req) - .map_err(|e| { + .inspect_err(|e| { // If the error of prepropose contains str `NO_VALID_SPLIT_KEY`, it may mean the // split_key of the split request is the region start key which // means we may have so many potential duplicate mvcc versions @@ -4596,7 +4592,7 @@ where "region_id" => self.region_id, "safe_point" => safe_ts, ); - return e; + return; } let start_key = enc_start_key(self.region()); @@ -4637,7 +4633,6 @@ where self.last_record_safe_point = safe_ts; } } - e })?; let mut ctx = ProposalContext::empty(); @@ -5546,7 +5541,7 @@ where } for peer in self.get_store().region().get_peers() { let (peer_id, store_id) = (peer.get_id(), peer.get_store_id()); - if self.disk_full_peers.peers.get(&peer_id).is_some() { + if self.disk_full_peers.peers.contains_key(&peer_id) { disk_full_stores.push(store_id); } } @@ -6111,17 +6106,17 @@ where None => return false, }; let max_lease = ctx.cfg.raft_store_max_leader_lease(); - let has_overlapped_reads = self.pending_reads.back().map_or(false, |read| { + let has_overlapped_reads = self.pending_reads.back().is_some_and(|read| { // If there is any read index whose lease can cover till next heartbeat // then we don't need to propose a new one read.propose_time + max_lease > renew_bound }); - let has_overlapped_writes = self.proposals.back().map_or(false, |proposal| { + let has_overlapped_writes = self.proposals.back().is_some_and(|proposal| { // If there is any write whose lease can cover till next heartbeat // then we don't need to propose a new one proposal .propose_time - .map_or(false, |propose_time| propose_time + max_lease > renew_bound) + .is_some_and(|propose_time| propose_time + max_lease > renew_bound) }); !has_overlapped_reads && !has_overlapped_writes } diff --git a/components/raftstore/src/store/peer_storage.rs b/components/raftstore/src/store/peer_storage.rs index 5fba934b462..7e6615951d5 100644 --- a/components/raftstore/src/store/peer_storage.rs +++ b/components/raftstore/src/store/peer_storage.rs @@ -6,14 +6,13 @@ use std::{ error, ops::{Deref, DerefMut}, sync::{ + Arc, atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, mpsc::{self, Receiver, TryRecvError}, - Arc, }, - u64, }; -use engine_traits::{Engines, KvEngine, Mutable, Peekable, RaftEngine, RaftLogBatch, CF_RAFT}; +use engine_traits::{CF_RAFT, Engines, KvEngine, Mutable, Peekable, RaftEngine, RaftLogBatch}; use fail::fail_point; use into_other::into_other; use keys::{self, enc_end_key, enc_start_key}; @@ -25,9 +24,8 @@ use kvproto::{ }; use protobuf::Message; use raft::{ - self, + self, Error as RaftError, GetEntriesContext, RaftState, Ready, Storage, StorageError, eraftpb::{self, ConfState, Entry, HardState, Snapshot}, - Error as RaftError, GetEntriesContext, RaftState, Ready, Storage, StorageError, }; use rand::Rng; use tikv_util::{ @@ -39,9 +37,10 @@ use tikv_util::{ }; use super::{ - local_metrics::RaftMetrics, metrics::*, worker::RegionTask, SnapEntry, SnapKey, SnapManager, + SnapEntry, SnapKey, SnapManager, local_metrics::RaftMetrics, metrics::*, worker::RegionTask, }; use crate::{ + Error, Result, store::{ async_io::{read::ReadTask, write::WriteTask}, entry_storage::{CacheWarmupState, EntryStorage}, @@ -49,7 +48,6 @@ use crate::{ peer::PersistSnapshotResult, util, }, - Error, Result, }; // The maximum tick interval between precheck requests. The tick interval helps @@ -488,7 +486,7 @@ where )); } - if find_peer_by_id(&self.region, to).map_or(false, |p| p.is_witness) { + if find_peer_by_id(&self.region, to).is_some_and(|p| p.is_witness) { // Although we always sending snapshot task behind apply task to get latest // snapshot, we can't use `last_applying_idx` here, as below the judgment // condition will generate an witness snapshot directly, the new non-witness @@ -1270,31 +1268,31 @@ pub mod tests { raft::RaftTestEngine, }; use engine_traits::{ - Engines, Iterable, RaftEngineDebug, RaftEngineReadOnly, SyncMutable, WriteBatch, - WriteBatchExt, ALL_CFS, CF_DEFAULT, + ALL_CFS, CF_DEFAULT, Engines, Iterable, RaftEngineDebug, RaftEngineReadOnly, SyncMutable, + WriteBatch, WriteBatchExt, }; use kvproto::raft_serverpb::RaftSnapshotData; use metapb::{Peer, Store, StoreLabel}; use pd_client::PdClient; use raft::{ - eraftpb::{ConfState, Entry, HardState}, Error as RaftError, GetEntriesContext, StorageError, + eraftpb::{ConfState, Entry, HardState}, }; use tempfile::{Builder, TempDir}; use tikv_util::{ store::{new_peer, new_witness_peer}, - worker::{dummy_scheduler, LazyWorker, Scheduler, Worker}, + worker::{LazyWorker, Scheduler, Worker, dummy_scheduler}, }; use super::*; use crate::store::{ + AsyncReadNotifier, FetchedLogs, GenSnapRes, async_io::{read::ReadRunner, write::write_to_db_for_test}, bootstrap_store, entry_storage::tests::validate_cache, fsm::apply::compact_raft_log, initial_region, prepare_bootstrap_cluster, worker::{RegionTask, SnapGenRunner, SnapGenTask}, - AsyncReadNotifier, FetchedLogs, GenSnapRes, }; fn new_storage( @@ -1546,7 +1544,7 @@ pub mod tests { new_entry(5, 5), new_entry(6, 6), ]; - let max_u64 = u64::max_value(); + let max_u64 = u64::MAX; let mut tests = vec![ ( 2, diff --git a/components/raftstore/src/store/read_queue.rs b/components/raftstore/src/store/read_queue.rs index 34e2b457ae6..73acfd703d5 100644 --- a/components/raftstore/src/store/read_queue.rs +++ b/components/raftstore/src/store/read_queue.rs @@ -1,7 +1,7 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. // #[PerformanceCriticalPath] -use std::{cmp, collections::VecDeque, mem, u64, usize}; +use std::{cmp, collections::VecDeque, mem}; use collections::HashMap; use kvproto::{ @@ -10,20 +10,19 @@ use kvproto::{ }; use protobuf::Message; use tikv_util::{ - box_err, - codec::number::{NumberEncoder, MAX_VAR_U64_LEN}, + MustConsumeVec, box_err, + codec::number::{MAX_VAR_U64_LEN, NumberEncoder}, debug, error, memory::HeapSize, time::{duration_to_sec, monotonic_raw_now}, - MustConsumeVec, }; use time::Timespec; use uuid::Uuid; use super::msg::ErrorCallback; use crate::{ - store::{fsm::apply, metrics::*, Config}, Result, + store::{Config, fsm::apply, metrics::*}, }; const READ_QUEUE_SHRINK_SIZE: usize = 64; diff --git a/components/raftstore/src/store/region_snapshot.rs b/components/raftstore/src/store/region_snapshot.rs index f19c725a569..48b59e933b2 100644 --- a/components/raftstore/src/store/region_snapshot.rs +++ b/components/raftstore/src/store/region_snapshot.rs @@ -5,14 +5,14 @@ use std::{ fmt, num::NonZeroU64, sync::{ - atomic::{AtomicU64, Ordering}, Arc, Mutex, + atomic::{AtomicU64, Ordering}, }, }; use engine_traits::{ - util::check_key_in_range, Error as EngineError, IterOptions, Iterable, Iterator, KvEngine, - MetricsExt, Peekable, RaftEngine, ReadOptions, Result as EngineResult, Snapshot, CF_RAFT, + CF_RAFT, Error as EngineError, IterOptions, Iterable, Iterator, KvEngine, MetricsExt, Peekable, + RaftEngine, ReadOptions, Result as EngineResult, Snapshot, util::check_key_in_range, }; use fail::fail_point; use keys::DATA_PREFIX_KEY; @@ -24,9 +24,9 @@ use tikv_util::{ }; use crate::{ - coprocessor::ObservedSnapshot, - store::{util, PeerStorage, TxnExt}, Error, Result, + coprocessor::ObservedSnapshot, + store::{PeerStorage, TxnExt, util}, }; /// Snapshot of a region. @@ -452,7 +452,7 @@ fn handle_check_key_in_region_error(e: crate::Error) -> Result<()> { #[cfg(test)] mod tests { use engine_test::{kv::KvTestSnapshot, new_temp_engine}; - use engine_traits::{Engines, KvEngine, Peekable, RaftEngine, SyncMutable, CF_DEFAULT}; + use engine_traits::{CF_DEFAULT, Engines, KvEngine, Peekable, RaftEngine, SyncMutable}; use keys::data_key; use kvproto::metapb::{Peer, Region}; use tempfile::Builder; @@ -460,8 +460,8 @@ mod tests { use super::*; use crate::{ - store::{local_metrics::RaftMetrics, PeerStorage}, Result, + store::{PeerStorage, local_metrics::RaftMetrics}, }; type DataSet = Vec<(Vec, Vec)>; diff --git a/components/raftstore/src/store/replication_mode.rs b/components/raftstore/src/store/replication_mode.rs index b83aff3d991..3a6f91873df 100644 --- a/components/raftstore/src/store/replication_mode.rs +++ b/components/raftstore/src/store/replication_mode.rs @@ -31,7 +31,7 @@ impl StoreGroup { if self .labels .get(&store.id) - .map_or(false, |l| store.get_labels() == &**l) + .is_some_and(|l| store.get_labels() == &**l) { return; } diff --git a/components/raftstore/src/store/simple_write.rs b/components/raftstore/src/store/simple_write.rs index 6f2901402a6..a5e15a4096a 100644 --- a/components/raftstore/src/store/simple_write.rs +++ b/components/raftstore/src/store/simple_write.rs @@ -11,7 +11,7 @@ use protobuf::{CodedInputStream, Message}; use slog::Logger; use tikv_util::slog_panic; -use crate::store::{msg::ErrorCallback, WriteCallback}; +use crate::store::{WriteCallback, msg::ErrorCallback}; // MAGIC number to hint simple write codec is used. If it's a protobuf message, // the first one or several bytes are for field tag, which can't be zero. diff --git a/components/raftstore/src/store/snap.rs b/components/raftstore/src/store/snap.rs index f53de9a37ed..69833fbfd89 100644 --- a/components/raftstore/src/store/snap.rs +++ b/components/raftstore/src/store/snap.rs @@ -8,22 +8,21 @@ use std::{ path::{Path, PathBuf}, result, str, sync::{ - atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, Arc, Mutex, RwLock, + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, }, thread, time::{self, Duration}, - u64, }; use collections::{HashMap, HashMapEntry as Entry}; -use encryption::{create_aes_ctr_crypter, DataKeyManager, Iv}; -use engine_traits::{CfName, KvEngine, CF_DEFAULT, CF_LOCK, CF_WRITE}; +use encryption::{DataKeyManager, Iv, create_aes_ctr_crypter}; +use engine_traits::{CF_DEFAULT, CF_LOCK, CF_WRITE, CfName, KvEngine}; use error_code::{self, ErrorCode, ErrorCodeExt}; use fail::fail_point; use file_system::{ - calc_crc32, calc_crc32_and_size, delete_dir_if_exist, delete_file_if_exist, file_exists, - get_file_size, sync_dir, File, Metadata, OpenOptions, + File, Metadata, OpenOptions, calc_crc32, calc_crc32_and_size, delete_dir_if_exist, + delete_file_if_exist, file_exists, get_file_size, sync_dir, }; use keys::{enc_end_key, enc_start_key}; use kvproto::{ @@ -37,17 +36,17 @@ use protobuf::Message; use raft::eraftpb::Snapshot as RaftSnapshot; use thiserror::Error; use tikv_util::{ - box_err, box_try, + HandyRwLock, box_err, box_try, config::ReadableSize, debug, error, info, - time::{duration_to_sec, Instant, Limiter, UnixSecs}, - warn, HandyRwLock, + time::{Instant, Limiter, UnixSecs, duration_to_sec}, + warn, }; use crate::{ + Error as RaftStoreError, Result as RaftStoreResult, coprocessor::CoprocessorHost, store::{metrics::*, peer_storage::JOB_STATUS_CANCELLING}, - Error as RaftStoreError, Result as RaftStoreResult, }; #[path = "snap/io.rs"] @@ -162,7 +161,7 @@ impl SnapKey { pub fn from_snap(snap: &RaftSnapshot) -> io::Result { let mut snap_data = RaftSnapshotData::default(); if let Err(e) = snap_data.merge_from_bytes(snap.get_data()) { - return Err(io::Error::new(ErrorKind::Other, e)); + return Err(io::Error::other(e)); } Ok(SnapKey::from_region_snap( snap_data.get_region().get_id(), @@ -230,7 +229,7 @@ fn retry_delete_snapshot(mgr: &SnapManagerCore, key: &SnapKey, snap: &Snapshot) pub fn gen_snapshot_meta(cf_files: &[CfFile], for_balance: bool) -> RaftStoreResult { let mut meta = Vec::with_capacity(cf_files.len()); for cf_file in cf_files { - if !SNAPSHOT_CFS.iter().any(|cf| cf_file.cf == *cf) { + if !SNAPSHOT_CFS.contains(&cf_file.cf) { return Err(box_err!( "failed to encode invalid snapshot cf {}", cf_file.cf @@ -810,10 +809,7 @@ impl Snapshot { self.cf_index = index; Ok(()) } - None => Err(io::Error::new( - ErrorKind::Other, - format!("fail to find cf {}", cf), - )), + None => Err(io::Error::other(format!("fail to find cf {}", cf))), } } @@ -848,10 +844,7 @@ impl Snapshot { region: &Region, allow_multi_files_snapshot: bool, for_balance: bool, - ) -> RaftStoreResult<()> - where - EK: KvEngine, - { + ) -> RaftStoreResult<()> { fail_point!("snapshot_enter_do_build"); if self.exists() { match self.validate(|_, _| -> RaftStoreResult<()> { Ok(()) }) { @@ -1515,10 +1508,10 @@ impl SnapManager { return Ok(()); } if !path.is_dir() { - return Err(io::Error::new( - ErrorKind::Other, - format!("{} should be a directory", path.display()), - )); + return Err(io::Error::other(format!( + "{} should be a directory", + path.display() + ))); } for f in file_system::read_dir(path)? { let p = f?; @@ -2359,10 +2352,10 @@ impl TabletSnapManager { file_system::create_dir_all(&path)?; } if !path.is_dir() { - return Err(io::Error::new( - ErrorKind::Other, - format!("{} should be a directory", path.display()), - )); + return Err(io::Error::other(format!( + "{} should be a directory", + path.display() + ))); } encryption::clean_up_dir(&path, SNAP_GEN_PREFIX, key_manager.as_deref())?; encryption::clean_up_trash(&path, key_manager.as_deref())?; @@ -2452,9 +2445,11 @@ impl TabletSnapManager { }; let path = entry.path(); - if path.file_name().and_then(|n| n.to_str()).map_or(true, |n| { - !n.starts_with(SNAP_GEN_PREFIX) || n.ends_with(TMP_FILE_SUFFIX) - }) { + if path + .file_name() + .and_then(|n| n.to_str()) + .is_none_or(|n| !n.starts_with(SNAP_GEN_PREFIX) || n.ends_with(TMP_FILE_SUFFIX)) + { continue; } paths.push(path); @@ -2476,7 +2471,7 @@ impl TabletSnapManager { if !path .file_name() .and_then(|n| n.to_str()) - .map_or(true, |n| n.starts_with(SNAP_REV_PREFIX)) + .is_none_or(|n| n.starts_with(SNAP_REV_PREFIX)) { continue; } @@ -2539,8 +2534,8 @@ pub mod tests { io::{self, Read, Seek, SeekFrom, Write}, path::{Path, PathBuf}, sync::{ - atomic::{AtomicBool, AtomicU64, AtomicUsize}, Arc, + atomic::{AtomicBool, AtomicU64, AtomicUsize}, }, }; @@ -2552,9 +2547,9 @@ pub mod tests { raft::RaftTestEngine, }; use engine_traits::{ - Engines, ExternalSstFileInfo, KvEngine, RaftEngine, RaftLogBatch, - Snapshot as EngineSnapshot, SstExt, SstWriter, SstWriterBuilder, SyncMutable, ALL_CFS, - CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE, + ALL_CFS, CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE, Engines, ExternalSstFileInfo, KvEngine, + RaftEngine, RaftLogBatch, Snapshot as EngineSnapshot, SstExt, SstWriter, SstWriterBuilder, + SyncMutable, }; use kvproto::{ encryptionpb::EncryptionMethod, @@ -2568,9 +2563,9 @@ pub mod tests { use super::*; use crate::{ - coprocessor::CoprocessorHost, - store::{peer_storage::JOB_STATUS_RUNNING, INIT_EPOCH_CONF_VER, INIT_EPOCH_VER}, Result, + coprocessor::CoprocessorHost, + store::{INIT_EPOCH_CONF_VER, INIT_EPOCH_VER, peer_storage::JOB_STATUS_RUNNING}, }; const TEST_STORE_ID: u64 = 1; @@ -3086,7 +3081,7 @@ pub mod tests { f.seek(pos).unwrap(); let mut buf = [0; BYTE_SIZE]; f.read_exact(&mut buf[..]).unwrap(); - buf[0] ^= u8::max_value(); + buf[0] ^= u8::MAX; f.seek(pos).unwrap(); f.write_all(&buf[..]).unwrap(); total += 1; @@ -3446,7 +3441,7 @@ pub mod tests { s.save().unwrap(); let snap_size = snap_mgr.get_total_snap_size().unwrap(); - let max_snap_count = (max_total_size + snap_size - 1) / snap_size; + let max_snap_count = max_total_size.div_ceil(snap_size); for (i, region_id) in regions.into_iter().enumerate() { let key = SnapKey::new(region_id, 1, 1); let region = gen_test_region(region_id, 1, 1); diff --git a/components/raftstore/src/store/snap/io.rs b/components/raftstore/src/store/snap/io.rs index aec9ce1d0d6..dea922a349c 100644 --- a/components/raftstore/src/store/snap/io.rs +++ b/components/raftstore/src/store/snap/io.rs @@ -4,7 +4,6 @@ use std::{ fs, io::{self, BufReader, Read, Write}, sync::Arc, - usize, }; use encryption::{DataKeyManager, DecrypterReader, EncrypterWriter, Iv}; @@ -243,7 +242,7 @@ where as usize; } Err(e) => { - let io_error = io::Error::new(io::ErrorKind::Other, e); + let io_error = io::Error::other(e); return Err(io_error.into()); } } @@ -260,7 +259,7 @@ where } if let Err(e) = sst_writer.borrow_mut().put(key, value) { - let io_error = io::Error::new(io::ErrorKind::Other, e); + let io_error = io::Error::other(e); return Err(io_error.into()); } file_length += entry_len; @@ -527,7 +526,7 @@ mod tests { use tikv_util::time::Limiter; use super::*; - use crate::store::snap::{tests::*, SNAPSHOT_CFS, SST_FILE_SUFFIX}; + use crate::store::snap::{SNAPSHOT_CFS, SST_FILE_SUFFIX, tests::*}; struct TestStaleDetector; impl StaleDetector for TestStaleDetector { diff --git a/components/raftstore/src/store/snapshot_backup.rs b/components/raftstore/src/store/snapshot_backup.rs index 1f5a3d65cf9..ae0b8d6785b 100644 --- a/components/raftstore/src/store/snapshot_backup.rs +++ b/components/raftstore/src/store/snapshot_backup.rs @@ -2,8 +2,8 @@ use std::{ sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, Arc, Mutex, + atomic::{AtomicBool, AtomicU64, Ordering}, }, time::Duration, }; @@ -14,11 +14,11 @@ use kvproto::{brpb::CheckAdminResponse, metapb::RegionEpoch, raft_cmdpb::AdminCm use tikv_util::{info, warn}; use tokio::sync::oneshot; -use super::{metrics, PeerMsg, RaftRouter, SignificantMsg, SignificantRouter}; +use super::{PeerMsg, RaftRouter, SignificantMsg, SignificantRouter, metrics}; use crate::coprocessor::{ - dispatcher::BoxTransferLeaderObserver, AdminObserver, BoxAdminObserver, BoxQueryObserver, - Coprocessor, CoprocessorHost, Error as CopError, QueryObserver, TransferLeaderCustomContext, - TransferLeaderObserver, + AdminObserver, BoxAdminObserver, BoxQueryObserver, Coprocessor, CoprocessorHost, + Error as CopError, QueryObserver, TransferLeaderCustomContext, TransferLeaderObserver, + dispatcher::BoxTransferLeaderObserver, }; fn epoch_second_coarse() -> u64 { diff --git a/components/raftstore/src/store/transport.rs b/components/raftstore/src/store/transport.rs index 35761aa5d18..f17258cb495 100644 --- a/components/raftstore/src/store/transport.rs +++ b/components/raftstore/src/store/transport.rs @@ -1,7 +1,7 @@ // Copyright 2016 TiKV Project Authors. Licensed under Apache-2.0. // #[PerformanceCriticalPath] -use std::sync::{mpsc, Mutex}; +use std::sync::{Mutex, mpsc}; use crossbeam::channel::{SendError, TrySendError}; use engine_traits::{KvEngine, RaftEngine, Snapshot}; @@ -10,8 +10,8 @@ use tikv_util::{error, warn}; use super::{AsyncReadNotifier, FetchedLogs, GenSnapRes}; use crate::{ - store::{CasualMessage, PeerMsg, RaftCommand, RaftRouter, SignificantMsg, StoreMsg}, DiscardReason, Error, Result, + store::{CasualMessage, PeerMsg, RaftCommand, RaftRouter, SignificantMsg, StoreMsg}, }; /// Transports messages between different Raft peers. @@ -46,7 +46,7 @@ where fn significant_send(&self, region_id: u64, msg: SignificantMsg) -> Result<()>; } -impl<'a, T: SignificantRouter, EK: KvEngine> SignificantRouter for &'a Mutex { +impl, EK: KvEngine> SignificantRouter for &Mutex { #[inline] fn significant_send(&self, region_id: u64, msg: SignificantMsg) -> Result<()> { Mutex::lock(self).unwrap().significant_send(region_id, msg) @@ -89,7 +89,7 @@ where } } -impl<'a, EK: KvEngine, T: CasualRouter> CasualRouter for &'a Mutex { +impl> CasualRouter for &Mutex { #[inline] fn send(&self, region_id: u64, msg: CasualMessage) -> Result<()> { CasualRouter::send(&*Mutex::lock(self).unwrap(), region_id, msg) diff --git a/components/raftstore/src/store/txn_ext.rs b/components/raftstore/src/store/txn_ext.rs index e791b520c05..1097ee753eb 100644 --- a/components/raftstore/src/store/txn_ext.rs +++ b/components/raftstore/src/store/txn_ext.rs @@ -9,7 +9,7 @@ use std::{ use kvproto::metapb; use lazy_static::lazy_static; use parking_lot::RwLock; -use prometheus::{register_int_gauge, IntGauge}; +use prometheus::{IntGauge, register_int_gauge}; use txn_types::{Key, Lock, PessimisticLock}; /// Transaction extensions related to a peer. diff --git a/components/raftstore/src/store/unsafe_recovery.rs b/components/raftstore/src/store/unsafe_recovery.rs index 876f181807b..da1a037fbae 100644 --- a/components/raftstore/src/store/unsafe_recovery.rs +++ b/components/raftstore/src/store/unsafe_recovery.rs @@ -18,8 +18,8 @@ use raft::eraftpb::ConfChangeType; use tikv_util::{box_err, error, info, time::Instant as TiInstant, warn}; use super::{ - fsm::new_admin_request, worker::new_change_peer_v2_request, PeerMsg, RaftRouter, - SignificantMsg, SignificantRouter, StoreMsg, + PeerMsg, RaftRouter, SignificantMsg, SignificantRouter, StoreMsg, fsm::new_admin_request, + worker::new_change_peer_v2_request, }; use crate::Result; @@ -246,6 +246,7 @@ pub fn start_unsafe_recovery_report( } #[derive(Clone, Debug)] +#[allow(dead_code)] pub struct UnsafeRecoveryForceLeaderSyncer(Arc); impl UnsafeRecoveryForceLeaderSyncer { diff --git a/components/raftstore/src/store/util.rs b/components/raftstore/src/store/util.rs index a2d8bbaec8e..f27aad7f50e 100644 --- a/components/raftstore/src/store/util.rs +++ b/components/raftstore/src/store/util.rs @@ -8,10 +8,9 @@ use std::{ fmt::{Debug, Display}, option::Option, sync::{ - atomic::{AtomicBool, AtomicU64, Ordering as AtomicOrdering}, Arc, Mutex, MutexGuard, + atomic::{AtomicBool, AtomicU64, Ordering as AtomicOrdering}, }, - u64, }; use collections::HashSet; @@ -26,27 +25,26 @@ use kvproto::{ }; use protobuf::{self, CodedInputStream, Message}; use raft::{ + Changer, INVALID_INDEX, RawNode, eraftpb::{self, ConfChangeType, ConfState, Entry, EntryType, MessageType, Snapshot}, - Changer, RawNode, INVALID_INDEX, }; use raft_proto::ConfChangeI; use tikv_util::{ - box_err, - codec::number::{decode_u64, NumberEncoder}, + Either, box_err, + codec::number::{NumberEncoder, decode_u64}, debug, info, store::{find_peer_by_id, region}, - time::{monotonic_raw_now, Instant}, - Either, + time::{Instant, monotonic_raw_now}, }; use time::{Duration, Timespec}; use tokio::sync::Notify; use txn_types::WriteBatchFlags; -use super::{metrics::PEER_ADMIN_CMD_COUNTER_VEC, peer_storage, Config}; +use super::{Config, metrics::PEER_ADMIN_CMD_COUNTER_VEC, peer_storage}; use crate::{ + Error, Result, coprocessor::CoprocessorHost, store::{simple_write::SimpleWriteReqDecoder, snap::SNAPSHOT_VERSION}, - Error, Result, }; const INVALID_TIMESTAMP: u64 = u64::MAX; @@ -941,7 +939,7 @@ pub trait ChangePeerI { fn to_confchange(&self, _: Vec) -> Self::CC; } -impl<'a> ChangePeerI for &'a ChangePeerRequest { +impl ChangePeerI for &ChangePeerRequest { type CC = eraftpb::ConfChange; type CP = Vec; @@ -1055,7 +1053,7 @@ pub fn check_conf_change( .get_peers() .iter() .find(|p| p.get_id() == peer.get_id()) - .map_or(false, |p| p.get_is_witness() != peer.get_is_witness()) + .is_some_and(|p| p.get_is_witness() != peer.get_is_witness()) { return Err(box_err!( "invalid conf change request: {:?}, can not switch witness in conf change", @@ -1151,9 +1149,7 @@ fn check_availability_by_last_heartbeats( .get_peers() .iter() .find(|p| p.get_id() == *id) - .map_or(false, |p| { - p.role == PeerRole::Voter || p.role == PeerRole::IncomingVoter - }) + .is_some_and(|p| p.role == PeerRole::Voter || p.role == PeerRole::IncomingVoter) { // leader itself is not a slow peer if *id == leader_id || last_heartbeat.elapsed() <= slow_voter_threshold { @@ -1178,9 +1174,7 @@ fn check_availability_by_last_heartbeats( .get_peers() .iter() .find(|p| p.get_id() == peer.get_id()) - .map_or(false, |p| { - p.role == PeerRole::Voter || p.role == PeerRole::IncomingVoter - }); + .is_some_and(|p| p.role == PeerRole::Voter || p.role == PeerRole::IncomingVoter); if !is_voter && change_type == ConfChangeType::AddNode { // exiting peers, promoting from learner to voter if let Some(last_heartbeat) = peer_heartbeats.get(&peer.get_id()) { @@ -1678,7 +1672,7 @@ impl RegionReadProgressCore { peer_id: u64, ) -> RegionReadProgressCore { // forbids stale read for witness - let is_witness = find_peer_by_id(region, peer_id).map_or(false, |p| p.is_witness); + let is_witness = find_peer_by_id(region, peer_id).is_some_and(|p| p.is_witness); RegionReadProgressCore { peer_id, region_id: region.get_id(), diff --git a/components/raftstore/src/store/worker/cleanup_snapshot.rs b/components/raftstore/src/store/worker/cleanup_snapshot.rs index 78167a726d2..a0bc1ff0ffa 100644 --- a/components/raftstore/src/store/worker/cleanup_snapshot.rs +++ b/components/raftstore/src/store/worker/cleanup_snapshot.rs @@ -8,10 +8,10 @@ use fail::fail_point; use tikv_util::{debug, error, info, warn, worker::Runnable}; use crate::{ + Result, store::{ CasualMessage, PeerMsg, RaftRouter, SnapKey, SnapManager, Snapshot, StoreMsg, StoreRouter, }, - Result, }; pub enum Task { diff --git a/components/raftstore/src/store/worker/compact.rs b/components/raftstore/src/store/worker/compact.rs index 3bf6b1358e4..a17c20ba082 100644 --- a/components/raftstore/src/store/worker/compact.rs +++ b/components/raftstore/src/store/worker/compact.rs @@ -8,7 +8,7 @@ use std::{ time::Duration, }; -use engine_traits::{KvEngine, ManualCompactionOptions, RangeStats, CF_LOCK, CF_WRITE}; +use engine_traits::{CF_LOCK, CF_WRITE, KvEngine, ManualCompactionOptions, RangeStats}; use fail::fail_point; use futures_util::compat::Future01CompatExt; use thiserror::Error; @@ -510,11 +510,11 @@ mod tests { use engine_test::{ ctor::{CfOptions, DbOptions}, - kv::{new_engine, new_engine_opt, KvTestEngine}, + kv::{KvTestEngine, new_engine, new_engine_opt}, }; use engine_traits::{ - CompactExt, MiscExt, Mutable, SyncMutable, WriteBatch, WriteBatchExt, CF_DEFAULT, CF_LOCK, - CF_RAFT, CF_WRITE, + CF_DEFAULT, CF_LOCK, CF_RAFT, CF_WRITE, CompactExt, MiscExt, Mutable, SyncMutable, + WriteBatch, WriteBatchExt, }; use keys::data_key; use tempfile::Builder; diff --git a/components/raftstore/src/store/worker/consistency_check.rs b/components/raftstore/src/store/worker/consistency_check.rs index fef2bae332c..4eb6a75810c 100644 --- a/components/raftstore/src/store/worker/consistency_check.rs +++ b/components/raftstore/src/store/worker/consistency_check.rs @@ -9,7 +9,7 @@ use tikv_util::{error, info, warn, worker::Runnable}; use super::metrics::*; use crate::{ - coprocessor::{dispatcher::StoreHandle, CoprocessorHost}, + coprocessor::{CoprocessorHost, dispatcher::StoreHandle}, store::metrics::*, }; @@ -117,16 +117,16 @@ mod tests { use std::{assert_matches::assert_matches, sync::mpsc, time::Duration}; use byteorder::{BigEndian, WriteBytesExt}; - use engine_test::kv::{new_engine, KvTestEngine}; - use engine_traits::{KvEngine, SyncMutable, ALL_CFS}; + use engine_test::kv::{KvTestEngine, new_engine}; + use engine_traits::{ALL_CFS, KvEngine, SyncMutable}; use kvproto::metapb::*; use tempfile::Builder; use tikv_util::worker::Runnable; use super::*; use crate::coprocessor::{ - dispatcher::SchedTask, BoxConsistencyCheckObserver, ConsistencyCheckMethod, - RawConsistencyCheckObserver, + BoxConsistencyCheckObserver, ConsistencyCheckMethod, RawConsistencyCheckObserver, + dispatcher::SchedTask, }; #[test] diff --git a/components/raftstore/src/store/worker/disk_check.rs b/components/raftstore/src/store/worker/disk_check.rs index 3f497102b08..0f0046a4d92 100644 --- a/components/raftstore/src/store/worker/disk_check.rs +++ b/components/raftstore/src/store/worker/disk_check.rs @@ -7,7 +7,7 @@ use std::{ time::Duration, }; -use crossbeam::channel::{bounded, Receiver, Sender}; +use crossbeam::channel::{Receiver, Sender, bounded}; use health_controller::types::LatencyInspector; use tikv_util::{ time::Instant, diff --git a/components/raftstore/src/store/worker/mod.rs b/components/raftstore/src/store/worker/mod.rs index 3cd4534b8f0..a682a3dec82 100644 --- a/components/raftstore/src/store/worker/mod.rs +++ b/components/raftstore/src/store/worker/mod.rs @@ -24,15 +24,15 @@ pub use self::{ cleanup_snapshot::{Runner as GcSnapshotRunner, Task as GcSnapshotTask}, cleanup_sst::{Runner as CleanupSstRunner, Task as CleanupSstTask}, compact::{ - need_compact, CompactThreshold, FullCompactController, Runner as CompactRunner, - Task as CompactTask, + CompactThreshold, FullCompactController, Runner as CompactRunner, Task as CompactTask, + need_compact, }, consistency_check::{Runner as ConsistencyCheckRunner, Task as ConsistencyCheckTask}, disk_check::{Runner as DiskCheckRunner, Task as DiskCheckTask}, pd::{ - new_change_peer_v2_request, FlowStatistics, FlowStatsReporter, HeartbeatTask, + FlowStatistics, FlowStatsReporter, HeartbeatTask, NUM_COLLECT_STORE_INFOS_PER_HEARTBEAT, Runner as PdRunner, StatsMonitor as PdStatsMonitor, StoreStatsReporter, Task as PdTask, - NUM_COLLECT_STORE_INFOS_PER_HEARTBEAT, + new_change_peer_v2_request, }, raftlog_gc::{Runner as RaftlogGcRunner, Task as RaftlogGcTask}, read::{ @@ -45,15 +45,15 @@ pub use self::{ Task as RefreshConfigTask, WriterContoller, }, region::{Runner as RegionRunner, Task as RegionTask}, - snap_gen::{Runner as SnapGenRunner, Task as SnapGenTask, SNAP_GENERATOR_MAX_POOL_SIZE}, + snap_gen::{Runner as SnapGenRunner, SNAP_GENERATOR_MAX_POOL_SIZE, Task as SnapGenTask}, split_check::{ Bucket, BucketRange, BucketStatsInfo, KeyEntry, Runner as SplitCheckRunner, Task as SplitCheckTask, }, split_config::{ - SplitConfig, SplitConfigManager, BIG_REGION_CPU_OVERLOAD_THRESHOLD_RATIO, - DEFAULT_BIG_REGION_BYTE_THRESHOLD, DEFAULT_BIG_REGION_QPS_THRESHOLD, - DEFAULT_BYTE_THRESHOLD, DEFAULT_QPS_THRESHOLD, REGION_CPU_OVERLOAD_THRESHOLD_RATIO, + BIG_REGION_CPU_OVERLOAD_THRESHOLD_RATIO, DEFAULT_BIG_REGION_BYTE_THRESHOLD, + DEFAULT_BIG_REGION_QPS_THRESHOLD, DEFAULT_BYTE_THRESHOLD, DEFAULT_QPS_THRESHOLD, + REGION_CPU_OVERLOAD_THRESHOLD_RATIO, SplitConfig, SplitConfigManager, }, split_controller::{AutoSplitController, ReadStats, SplitConfigChange, SplitInfo, WriteStats}, }; diff --git a/components/raftstore/src/store/worker/pd.rs b/components/raftstore/src/store/worker/pd.rs index 5d435eb7390..f225edbf600 100644 --- a/components/raftstore/src/store/worker/pd.rs +++ b/components/raftstore/src/store/worker/pd.rs @@ -6,9 +6,9 @@ use std::{ fmt::{self, Display, Formatter}, io, mem, sync::{ + Arc, Mutex, atomic::Ordering, mpsc::{self, Receiver, Sender, SyncSender}, - Arc, Mutex, }, thread::{Builder, JoinHandle}, time::{Duration, Instant}, @@ -19,11 +19,11 @@ use collections::{HashMap, HashSet}; use concurrency_manager::ConcurrencyManager; use engine_traits::{KvEngine, RaftEngine}; use fail::fail_point; -use futures::{compat::Future01CompatExt, FutureExt}; +use futures::{FutureExt, compat::Future01CompatExt}; use health_controller::{ + HealthController, reporters::{RaftstoreReporter, RaftstoreReporterConfig}, types::{InspectFactor, LatencyInspector, RaftstoreDuration}, - HealthController, }; use kvproto::{ kvrpcpb::DiskFullOpt, @@ -35,23 +35,22 @@ use kvproto::{ raft_serverpb::RaftMessage, replication_modepb::{RegionReplicationStatus, StoreDrAutoSyncStatus}, }; -use pd_client::{metrics::*, BucketStat, Error, PdClient, RegionStat, RegionWriteCfCopDetail}; +use pd_client::{BucketStat, Error, PdClient, RegionStat, RegionWriteCfCopDetail, metrics::*}; use prometheus::local::LocalHistogram; use raft::eraftpb::ConfChangeType; use resource_metering::{Collector, CollectorGuard, CollectorRegHandle, RawRecords}; use service::service_manager::GrpcServiceManager; use tikv_util::{ - box_err, debug, error, info, + GLOBAL_SERVER_READINESS, box_err, debug, error, info, metrics::ThreadInfoStatistics, store::QueryStats, - sys::{disk, thread::StdThreadBuildWrapper, SysQuota}, + sys::{SysQuota, disk, thread::StdThreadBuildWrapper}, thd_name, time::{Instant as TiInstant, UnixSecs}, timer::GLOBAL_TIMER_HANDLE, topn::TopN, warn, worker::{Runnable, ScheduleError, Scheduler}, - GLOBAL_SERVER_READINESS, }; use yatp::Remote; @@ -60,18 +59,18 @@ use crate::{ coprocessor::CoprocessorHost, router::RaftStoreRouter, store::{ + Callback, CasualMessage, Config, PeerMsg, RaftCmdExtraOpts, RaftCommand, RaftRouter, + SnapManager, StoreMsg, TxnExt, cmd_resp::new_error, metrics::*, unsafe_recovery::{ UnsafeRecoveryExecutePlanSyncer, UnsafeRecoveryForceLeaderSyncer, UnsafeRecoveryHandle, }, - util::{is_epoch_stale, KeysInfoFormatter}, + util::{KeysInfoFormatter, is_epoch_stale}, worker::{ - split_controller::{SplitInfo, TOP_N}, AutoSplitController, ReadStats, SplitConfigChange, WriteStats, + split_controller::{SplitInfo, TOP_N}, }, - Callback, CasualMessage, Config, PeerMsg, RaftCmdExtraOpts, RaftCommand, RaftRouter, - SnapManager, StoreMsg, TxnExt, }, }; @@ -2533,7 +2532,7 @@ mod tests { use std::thread::sleep; use kvproto::{kvrpcpb, pdpb::QueryKind}; - use pd_client::{new_bucket_stats, BucketMeta}; + use pd_client::{BucketMeta, new_bucket_stats}; use tikv_util::worker::LazyWorker; use super::*; diff --git a/components/raftstore/src/store/worker/raftlog_gc.rs b/components/raftstore/src/store/worker/raftlog_gc.rs index 3edabae71a0..d5b643734e0 100644 --- a/components/raftstore/src/store/worker/raftlog_gc.rs +++ b/components/raftstore/src/store/worker/raftlog_gc.rs @@ -187,7 +187,7 @@ where mod tests { use std::time::Duration; - use engine_traits::{RaftEngine, RaftLogBatch, ALL_CFS}; + use engine_traits::{ALL_CFS, RaftEngine, RaftLogBatch}; use raft::eraftpb::Entry; use tempfile::Builder; diff --git a/components/raftstore/src/store/worker/read.rs b/components/raftstore/src/store/worker/read.rs index ab4d0c18244..8349c96df08 100644 --- a/components/raftstore/src/store/worker/read.rs +++ b/components/raftstore/src/store/worker/read.rs @@ -6,8 +6,8 @@ use std::{ fmt::{self, Display, Formatter}, ops::Deref, sync::{ - atomic::{self, AtomicU64, Ordering}, Arc, Mutex, + atomic::{self, AtomicU64, Ordering}, }, }; @@ -26,7 +26,7 @@ use tikv_util::{ debug, error, lru::LruCache, store::find_peer_by_id, - time::{monotonic_raw_now, ThreadReadId}, + time::{ThreadReadId, monotonic_raw_now}, }; use time::Timespec; use tracker::GLOBAL_TRACKERS; @@ -34,17 +34,16 @@ use txn_types::{TimeStamp, WriteBatchFlags}; use super::metrics::*; use crate::{ + Error, Result, coprocessor::CoprocessorHost, errors::RAFTSTORE_IS_BUSY, router::ReadContext, store::{ - cmd_resp, + Callback, CasualMessage, CasualRouter, Peer, ProposalRouter, RaftCommand, ReadCallback, + ReadResponse, RegionSnapshot, RequestInspector, RequestPolicy, TxnExt, cmd_resp, fsm::store::StoreMeta, util::{self, LeaseState, RegionReadProgress, RemoteLease}, - Callback, CasualMessage, CasualRouter, Peer, ProposalRouter, RaftCommand, ReadCallback, - ReadResponse, RegionSnapshot, RequestInspector, RequestPolicy, TxnExt, }, - Error, Result, }; /// #[RaftstoreCommon] @@ -1157,7 +1156,7 @@ where // local peer is a valid leader. let allow_fallback_leader_read = inspector .inspect(&req) - .map_or(false, |r| r == RequestPolicy::ReadLocal); + .is_ok_and(|r| r == RequestPolicy::ReadLocal); if !allow_fallback_leader_read { cb.set_result(ReadResponse { response: err_resp, @@ -1351,7 +1350,7 @@ mod tests { use crossbeam::channel::TrySendError; use engine_test::kv::{KvTestEngine, KvTestSnapshot}; - use engine_traits::{MiscExt, Peekable, SyncMutable, ALL_CFS}; + use engine_traits::{ALL_CFS, MiscExt, Peekable, SyncMutable}; use kvproto::{metapb::RegionEpoch, raft_cmdpb::*}; use tempfile::{Builder, TempDir}; use tikv_util::{codec::number::NumberEncoder, time::monotonic_raw_now}; @@ -1359,7 +1358,7 @@ mod tests { use txn_types::WriteBatchFlags; use super::*; - use crate::store::{util::Lease, Callback}; + use crate::store::{Callback, util::Lease}; struct MockRouter { p_router: SyncSender>, @@ -1523,7 +1522,7 @@ mod tests { TLS_LOCAL_READ_METRICS.with(|m| m.borrow().reject_reason.cache_miss.get()), 1 ); - assert!(reader.local_reader.delegates.get(&1).is_none()); + assert!(!reader.local_reader.delegates.contains_key(&1)); // Register region 1 lease.renew(monotonic_raw_now()); diff --git a/components/raftstore/src/store/worker/refresh_config.rs b/components/raftstore/src/store/worker/refresh_config.rs index 066b463e75e..9fa99abe774 100644 --- a/components/raftstore/src/store/worker/refresh_config.rs +++ b/components/raftstore/src/store/worker/refresh_config.rs @@ -6,21 +6,21 @@ use std::{ }; use batch_system::{BatchRouter, Fsm, FsmTypes, HandlerBuilder, Poller, PoolState, Priority}; -use file_system::{set_io_type, IoType}; +use file_system::{IoType, set_io_type}; use tikv_util::{ debug, error, info, safe_panic, sys::thread::StdThreadBuildWrapper, thd_name, warn, worker::Runnable, yatp_pool::FuturePool, }; use crate::store::{ + PersistedNotifier, async_io::write::{StoreWriters, StoreWritersContext}, fsm::{ + PeerFsm, apply::{ApplyFsm, ControlFsm}, store::{RaftRouter, StoreFsm}, - PeerFsm, }, transport::Transport, - PersistedNotifier, }; pub struct PoolController> { diff --git a/components/raftstore/src/store/worker/region.rs b/components/raftstore/src/store/worker/region.rs index c2e70ef6737..0d898928902 100644 --- a/components/raftstore/src/store/worker/region.rs +++ b/components/raftstore/src/store/worker/region.rs @@ -8,15 +8,14 @@ use std::{ }, fmt::{self, Display, Formatter}, sync::{ - atomic::{AtomicUsize, Ordering}, Arc, + atomic::{AtomicUsize, Ordering}, }, time::Duration, - u64, }; use engine_traits::{ - DeleteStrategy, KvEngine, Mutable, Range, WriteBatch, WriteOptions, CF_LOCK, CF_RAFT, + CF_LOCK, CF_RAFT, DeleteStrategy, KvEngine, Mutable, Range, WriteBatch, WriteOptions, }; use fail::fail_point; use kvproto::raft_serverpb::{PeerState, RaftApplyState, RegionLocalState}; @@ -33,14 +32,13 @@ use super::metrics::*; use crate::{ coprocessor::CoprocessorHost, store::{ - check_abort, + ApplyOptions, CasualMessage, Config, SnapEntry, SnapKey, SnapManager, check_abort, peer_storage::{ JOB_STATUS_CANCELLED, JOB_STATUS_CANCELLING, JOB_STATUS_FAILED, JOB_STATUS_FINISHED, JOB_STATUS_PENDING, JOB_STATUS_RUNNING, }, - snap::{plain_file_used, Error, Result, SNAPSHOT_CFS}, + snap::{Error, Result, SNAPSHOT_CFS, plain_file_used}, transport::CasualRouter, - ApplyOptions, CasualMessage, Config, SnapEntry, SnapKey, SnapManager, }, }; @@ -805,8 +803,9 @@ pub(crate) mod tests { use std::{ io, sync::{ + Arc, atomic::{AtomicBool, AtomicUsize}, - mpsc, Arc, + mpsc, }, thread, time::Duration, @@ -814,8 +813,8 @@ pub(crate) mod tests { use engine_test::{ctor::CfOptions, kv::KvTestEngine}; use engine_traits::{ - CompactExt, FlowControlFactorsExt, KvEngine, MiscExt, Mutable, Peekable, - RaftEngineReadOnly, SyncMutable, WriteBatch, WriteBatchExt, CF_DEFAULT, CF_WRITE, + CF_DEFAULT, CF_WRITE, CompactExt, FlowControlFactorsExt, KvEngine, MiscExt, Mutable, + Peekable, RaftEngineReadOnly, SyncMutable, WriteBatch, WriteBatchExt, }; use keys::data_key; use kvproto::raft_serverpb::{PeerState, RaftApplyState, RaftSnapshotData, RegionLocalState}; @@ -834,10 +833,10 @@ pub(crate) mod tests { ObserverContext, }, store::{ + CasualMessage, SnapKey, SnapManager, peer_storage::JOB_STATUS_PENDING, snap::tests::get_test_db_for_regions, worker::{RegionRunner, SnapGenRunner, SnapGenTask}, - CasualMessage, SnapKey, SnapManager, }, }; diff --git a/components/raftstore/src/store/worker/snap_gen.rs b/components/raftstore/src/store/worker/snap_gen.rs index b791ff6e9e1..f5a8c682aae 100644 --- a/components/raftstore/src/store/worker/snap_gen.rs +++ b/components/raftstore/src/store/worker/snap_gen.rs @@ -3,11 +3,10 @@ use std::{ fmt::{self, Display, Formatter}, sync::{ + Arc, atomic::{AtomicBool, Ordering}, mpsc::SyncSender, - Arc, }, - u64, }; use collections::HashMap; @@ -25,7 +24,7 @@ use tikv_util::{ }; use super::metrics::*; -use crate::store::{self, snap::Result, transport::CasualRouter, CasualMessage, SnapManager}; +use crate::store::{self, CasualMessage, SnapManager, snap::Result, transport::CasualRouter}; pub const SNAP_GENERATOR_MAX_POOL_SIZE: usize = 16; @@ -221,7 +220,7 @@ where if let Some(is_tiflash) = self.tiflash_stores.get(&to_store_id) { allow_multi_files_snapshot = !is_tiflash; } else { - let is_tiflash = self.pd_client.as_ref().map_or(false, |pd_client| { + let is_tiflash = self.pd_client.as_ref().is_some_and(|pd_client| { if let Ok(s) = pd_client.get_store(to_store_id) { return s.get_labels().iter().any(|label| { label.get_key().to_lowercase() == ENGINE diff --git a/components/raftstore/src/store/worker/split_check.rs b/components/raftstore/src/store/worker/split_check.rs index ce2b2a6d10f..98685f1edd7 100644 --- a/components/raftstore/src/store/worker/split_check.rs +++ b/components/raftstore/src/store/worker/split_check.rs @@ -9,7 +9,7 @@ use std::{ }; use engine_traits::{ - CfName, IterOptions, Iterable, Iterator, KvEngine, TabletRegistry, CF_WRITE, LARGE_CFS, + CF_WRITE, CfName, IterOptions, Iterable, Iterator, KvEngine, LARGE_CFS, TabletRegistry, }; use file_system::{IoType, WithIoType}; use itertools::Itertools; @@ -20,18 +20,18 @@ use kvproto::{ use online_config::{ConfigChange, OnlineConfig}; use pd_client::{BucketMeta, BucketStat}; use tikv_util::{ - box_err, debug, error, info, keybuilder::KeyBuilder, warn, worker::Runnable, Either, + Either, box_err, debug, error, info, keybuilder::KeyBuilder, warn, worker::Runnable, }; use txn_types::Key; use super::metrics::*; use crate::{ + Result, coprocessor::{ + Config, CoprocessorHost, SplitCheckerHost, dispatcher::StoreHandle, split_observer::{is_valid_split_key, strip_timestamp_if_exists}, - Config, CoprocessorHost, SplitCheckerHost, }, - Result, }; #[derive(PartialEq, Eq)] diff --git a/components/raftstore/src/store/worker/split_config.rs b/components/raftstore/src/store/worker/split_config.rs index 2d29bd21a89..247a7c4754a 100644 --- a/components/raftstore/src/store/worker/split_config.rs +++ b/components/raftstore/src/store/worker/split_config.rs @@ -235,8 +235,8 @@ mod tests { use tikv_util::config::VersionTrack; use crate::store::{ - worker::split_config::{get_sample_num, DEFAULT_SAMPLE_NUM}, SplitConfig, SplitConfigManager, + worker::split_config::{DEFAULT_SAMPLE_NUM, get_sample_num}, }; #[test] diff --git a/components/raftstore/src/store/worker/split_controller.rs b/components/raftstore/src/store/worker/split_controller.rs index 0a6a19acc91..b08c81e2624 100644 --- a/components/raftstore/src/store/worker/split_controller.rs +++ b/components/raftstore/src/store/worker/split_controller.rs @@ -1,10 +1,10 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. use std::{ - cmp::{min, Ordering}, + cmp::{Ordering, min}, collections::{BinaryHeap, HashSet}, slice::{Iter, IterMut}, - sync::{mpsc::Receiver, Arc}, + sync::{Arc, mpsc::Receiver}, time::{Duration, SystemTime}, }; @@ -21,7 +21,7 @@ use tikv_util::{ config::Tracker, debug, info, metrics::ThreadInfoStatistics, - store::{is_read_query, QueryStats}, + store::{QueryStats, is_read_query}, time::Instant, warn, }; @@ -29,7 +29,7 @@ use tikv_util::{ use crate::store::{ metrics::*, util::build_key_range, - worker::{split_config::get_sample_num, FlowStatistics, SplitConfig, SplitConfigManager}, + worker::{FlowStatistics, SplitConfig, SplitConfigManager, split_config::get_sample_num}, }; const DEFAULT_MAX_SAMPLE_LOOP_COUNT: usize = 10000; diff --git a/components/resolved_ts/src/advance.rs b/components/resolved_ts/src/advance.rs index a20e5c6b80d..00c69c89ad0 100644 --- a/components/resolved_ts/src/advance.rs +++ b/components/resolved_ts/src/advance.rs @@ -4,8 +4,8 @@ use std::{ cmp, ffi::CString, sync::{ - atomic::{AtomicI32, Ordering}, Arc, + atomic::{AtomicI32, Ordering}, }, time::Duration, }; @@ -14,7 +14,7 @@ use collections::{HashMap, HashSet}; use concurrency_manager::ConcurrencyManager; use engine_traits::KvEngine; use fail::fail_point; -use futures::{compat::Future01CompatExt, future::select_all, FutureExt, TryFutureExt}; +use futures::{FutureExt, TryFutureExt, compat::Future01CompatExt, future::select_all}; use grpcio::{ ChannelBuilder, CompressionAlgorithms, Environment, Error as GrpcError, RpcStatusCode, }; @@ -43,7 +43,7 @@ use tokio::{ }; use txn_types::TimeStamp; -use crate::{endpoint::Task, metrics::*, TsSource}; +use crate::{TsSource, endpoint::Task, metrics::*}; pub(crate) const DEFAULT_CHECK_LEADER_TIMEOUT_DURATION: Duration = Duration::from_secs(5); // 5s const DEFAULT_GRPC_GZIP_COMPRESSION_LEVEL: usize = 2; @@ -590,8 +590,8 @@ impl RegionProgress { mod tests { use std::{ sync::{ - mpsc::{channel, Receiver, Sender}, Arc, + mpsc::{Receiver, Sender, channel}, }, time::Duration, }; diff --git a/components/resolved_ts/src/cmd.rs b/components/resolved_ts/src/cmd.rs index 7f6a36d9184..778d6d80643 100644 --- a/components/resolved_ts/src/cmd.rs +++ b/components/resolved_ts/src/cmd.rs @@ -336,18 +336,18 @@ mod tests { raft_cmdpb::{CmdType, Request}, }; use tikv::storage::{ + Engine, kv::{MockEngineBuilder, TestEngineBuilder}, - mvcc::{tests::write, Mutation, MvccTxn, SnapshotReader}, + mvcc::{Mutation, MvccTxn, SnapshotReader, tests::write}, txn::{ - commands::one_pc_commit, prewrite, tests::*, CommitKind, TransactionKind, - TransactionProperties, + CommitKind, TransactionKind, TransactionProperties, commands::one_pc_commit, prewrite, + tests::*, }, - Engine, }; use tikv_kv::Modify; use txn_types::{Key, LockType, WriteType}; - use super::{group_row_changes, ChangeLog, ChangeRow}; + use super::{ChangeLog, ChangeRow, group_row_changes}; #[test] fn test_cmd_encode() { diff --git a/components/resolved_ts/src/endpoint.rs b/components/resolved_ts/src/endpoint.rs index a02cdc8877e..da200b0b7f6 100644 --- a/components/resolved_ts/src/endpoint.rs +++ b/components/resolved_ts/src/endpoint.rs @@ -11,7 +11,7 @@ use std::{ use concurrency_manager::ConcurrencyManager; use engine_traits::KvEngine; -use futures::channel::oneshot::{channel, Receiver, Sender}; +use futures::channel::oneshot::{Receiver, Sender, channel}; use grpcio::Environment; use kvproto::{kvrpcpb::LeaderInfo, metapb::Region, raft_cmdpb::AdminCmdType}; use online_config::{self, ConfigChange, ConfigManager, OnlineConfig}; @@ -37,12 +37,12 @@ use tokio::sync::{Notify, Semaphore}; use txn_types::{Key, TimeStamp}; use crate::{ - advance::{AdvanceTsWorker, LeadershipResolver, DEFAULT_CHECK_LEADER_TIMEOUT_DURATION}, + Error, ON_DROP_WARN_HEAP_SIZE, Result, TsSource, TxnLocks, + advance::{AdvanceTsWorker, DEFAULT_CHECK_LEADER_TIMEOUT_DURATION, LeadershipResolver}, cmd::{ChangeLog, ChangeRow}, metrics::*, resolver::{LastAttempt, Resolver}, scanner::{ScanEntries, ScanTask, ScannerPool}, - Error, Result, TsSource, TxnLocks, ON_DROP_WARN_HEAP_SIZE, }; /// grace period for identifying slow resolved-ts and safe-ts. @@ -144,9 +144,8 @@ impl ResolverStatus { let locks = std::mem::take(locks); ( *tracked_index, - locks.into_iter().map(|lock| { + locks.into_iter().inspect(|lock| { memory_quota.free(lock.approximate_heap_size()); - lock }), ) } @@ -479,7 +478,7 @@ where } stats.unresolved_count += 1; } - ResolverStatus::Ready { .. } => { + ResolverStatus::Ready => { stats.heap_size += observed_region.resolver.approximate_heap_bytes() as i64; stats.resolved_count += 1; } @@ -728,7 +727,7 @@ where fn register_region(&mut self, region: Region, backoff: Option) { let region_id = region.get_id(); - assert!(self.regions.get(®ion_id).is_none()); + assert!(!self.regions.contains_key(®ion_id)); let Some(read_progress) = self.region_read_progress.get(®ion_id) else { warn!("try register nonexistent region"; "region" => ?region); return; diff --git a/components/resolved_ts/src/observer.rs b/components/resolved_ts/src/observer.rs index 7421beaad85..48fad04a2b8 100644 --- a/components/resolved_ts/src/observer.rs +++ b/components/resolved_ts/src/observer.rs @@ -141,7 +141,7 @@ mod test { use engine_traits::{CF_DEFAULT, CF_LOCK, CF_WRITE}; use kvproto::raft_cmdpb::*; use tikv::storage::kv::TestEngineBuilder; - use tikv_util::worker::{dummy_scheduler, ReceiverWrapper}; + use tikv_util::worker::{ReceiverWrapper, dummy_scheduler}; use super::*; diff --git a/components/resolved_ts/src/resolver.rs b/components/resolved_ts/src/resolver.rs index a90df0f5a1d..48599c45a88 100644 --- a/components/resolved_ts/src/resolver.rs +++ b/components/resolved_ts/src/resolver.rs @@ -704,7 +704,7 @@ mod tests { ]; for (i, case) in cases.into_iter().enumerate() { - let memory_quota = Arc::new(MemoryQuota::new(std::usize::MAX)); + let memory_quota = Arc::new(MemoryQuota::new(usize::MAX)); let mut resolver = Resolver::new_for_test(1, memory_quota); for e in case.clone() { match e { @@ -754,7 +754,7 @@ mod tests { #[test] fn test_untrack_lock_shrink_ratio() { - let memory_quota = Arc::new(MemoryQuota::new(std::usize::MAX)); + let memory_quota = Arc::new(MemoryQuota::new(usize::MAX)); let mut resolver = Resolver::new_for_test(1, memory_quota); let mut key = vec![0; 16]; let mut ts = TimeStamp::default(); @@ -809,7 +809,7 @@ mod tests { #[test] fn test_idempotent_track_and_untrack_lock() { - let memory_quota = Arc::new(MemoryQuota::new(std::usize::MAX)); + let memory_quota = Arc::new(MemoryQuota::new(usize::MAX)); let mut resolver = Resolver::new_for_test(1, memory_quota); let mut key = vec![0; 16]; @@ -883,7 +883,7 @@ mod tests { #[test] fn test_large_txn_tracking() { - let memory_quota = Arc::new(MemoryQuota::new(std::usize::MAX)); + let memory_quota = Arc::new(MemoryQuota::new(usize::MAX)); let txn_status_cache = Arc::new(TxnStatusCache::new(100)); let mut resolver = Resolver::new(1, memory_quota, txn_status_cache.clone()); let key1: Vec = vec![1, 2, 3, 4]; @@ -933,7 +933,7 @@ mod tests { // A later commit_ts must be strictly larger than resolved-ts. Equality is not // allowed. The case may not happen in real implementation, but we want // to ensure the correctness and robustness of every submodule. - let memory_quota = Arc::new(MemoryQuota::new(std::usize::MAX)); + let memory_quota = Arc::new(MemoryQuota::new(usize::MAX)); let txn_status_cache = Arc::new(TxnStatusCache::new(100)); let mut resolver = Resolver::new(1, memory_quota, txn_status_cache.clone()); let key: Vec = vec![1, 2, 3, 4]; diff --git a/components/resolved_ts/src/scanner.rs b/components/resolved_ts/src/scanner.rs index 1a98ac001dd..9cdcf389cba 100644 --- a/components/resolved_ts/src/scanner.rs +++ b/components/resolved_ts/src/scanner.rs @@ -3,12 +3,12 @@ use std::{marker::PhantomData, sync::Arc, time::Duration}; use engine_traits::KvEngine; -use futures::{channel::oneshot::Receiver, compat::Future01CompatExt, FutureExt}; +use futures::{FutureExt, channel::oneshot::Receiver, compat::Future01CompatExt}; use kvproto::metapb::Region; use raftstore::{ coprocessor::ObserveHandle, router::CdcHandle, - store::{fsm::ChangeObserver, msg::Callback, RegionSnapshot}, + store::{RegionSnapshot, fsm::ChangeObserver, msg::Callback}, }; use tikv::storage::{ kv::{ScanMode as MvccScanMode, Snapshot}, @@ -24,9 +24,9 @@ use tokio::{ use txn_types::{Key, Lock, LockType, TimeStamp}; use crate::{ + Task, errors::{Error, Result}, metrics::*, - Task, }; const DEFAULT_SCAN_BATCH_SIZE: usize = 128; diff --git a/components/resolved_ts/tests/failpoints/mod.rs b/components/resolved_ts/tests/failpoints/mod.rs index 64b58e0ed22..a44956f9f93 100644 --- a/components/resolved_ts/tests/failpoints/mod.rs +++ b/components/resolved_ts/tests/failpoints/mod.rs @@ -3,7 +3,7 @@ #[path = "../mod.rs"] mod testsuite; use std::{ - sync::{mpsc::channel, Mutex}, + sync::{Mutex, mpsc::channel}, time::Duration, }; diff --git a/components/resolved_ts/tests/integrations/mod.rs b/components/resolved_ts/tests/integrations/mod.rs index 75338517545..a2fee9d7fa8 100644 --- a/components/resolved_ts/tests/integrations/mod.rs +++ b/components/resolved_ts/tests/integrations/mod.rs @@ -9,7 +9,7 @@ use kvproto::{kvrpcpb::*, metapb::RegionEpoch}; use pd_client::PdClient; use resolved_ts::Task; use tempfile::Builder; -use test_raftstore::{sleep_ms, IsolationFilterFactory}; +use test_raftstore::{IsolationFilterFactory, sleep_ms}; use test_sst_importer::*; pub use testsuite::*; use tikv_util::store::new_peer; diff --git a/components/resolved_ts/tests/mod.rs b/components/resolved_ts/tests/mod.rs index 90592de582b..96bb5f83770 100644 --- a/components/resolved_ts/tests/mod.rs +++ b/components/resolved_ts/tests/mod.rs @@ -8,7 +8,7 @@ use kvproto::{import_sstpb_grpc::ImportSstClient, kvrpcpb::*, tikvpb::TikvClient use online_config::ConfigValue; use resolved_ts::Task; use test_raftstore::*; -use tikv_util::{config::ReadableDuration, HandyRwLock}; +use tikv_util::{HandyRwLock, config::ReadableDuration}; use txn_types::TimeStamp; static INIT: Once = Once::new(); diff --git a/components/resource_control/src/channel.rs b/components/resource_control/src/channel.rs index eec0accc259..6bdb962707d 100644 --- a/components/resource_control/src/channel.rs +++ b/components/resource_control/src/channel.rs @@ -169,7 +169,7 @@ impl Receiver { #[cfg(test)] mod tests { - use std::{thread, usize}; + use std::thread; use test::Bencher; diff --git a/components/resource_control/src/future.rs b/components/resource_control/src/future.rs index a6eeea04f9f..8c53aec3cf7 100644 --- a/components/resource_control/src/future.rs +++ b/components/resource_control/src/future.rs @@ -207,7 +207,7 @@ pub async fn with_resource_limiter( #[cfg(all(test, feature = "failpoints"))] mod tests { - use std::sync::mpsc::{channel, Sender}; + use std::sync::mpsc::{Sender, channel}; use tikv_util::yatp_pool::{DefaultTicker, FuturePool, YatpPoolBuilder}; diff --git a/components/resource_control/src/lib.rs b/components/resource_control/src/lib.rs index 55b0ee10f17..bc0d8d178fa 100644 --- a/components/resource_control/src/lib.rs +++ b/components/resource_control/src/lib.rs @@ -8,12 +8,12 @@ use pd_client::RpcClient; mod resource_group; pub use resource_group::{ - ResourceConsumeType, ResourceController, ResourceGroupManager, MIN_PRIORITY_UPDATE_INTERVAL, + MIN_PRIORITY_UPDATE_INTERVAL, ResourceConsumeType, ResourceController, ResourceGroupManager, }; pub use tikv_util::resource_control::*; mod future; -pub use future::{with_resource_limiter, ControlledFuture}; +pub use future::{ControlledFuture, with_resource_limiter}; #[cfg(test)] extern crate test; @@ -30,7 +30,7 @@ mod resource_limiter; pub use resource_limiter::ResourceLimiter; use tikv_util::worker::Worker; use worker::{ - GroupQuotaAdjustWorker, PriorityLimiterAdjustWorker, BACKGROUND_LIMIT_ADJUST_DURATION, + BACKGROUND_LIMIT_ADJUST_DURATION, GroupQuotaAdjustWorker, PriorityLimiterAdjustWorker, }; mod metrics; diff --git a/components/resource_control/src/resource_group.rs b/components/resource_control/src/resource_group.rs index 1332ed1950d..4fea0fd0033 100644 --- a/components/resource_control/src/resource_group.rs +++ b/components/resource_control/src/resource_group.rs @@ -5,14 +5,14 @@ use std::{ cmp::{max, min}, collections::HashSet, sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, Arc, Mutex, + atomic::{AtomicBool, AtomicU64, Ordering}, }, time::Duration, }; use collections::HashMap; -use dashmap::{mapref::one::Ref, DashMap}; +use dashmap::{DashMap, mapref::one::Ref}; use fail::fail_point; use kvproto::{ kvrpcpb::{CommandPri, ResourceControlContext}, @@ -22,7 +22,7 @@ use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard}; use tikv_util::{ config::VersionTrack, info, - resource_control::{TaskMetadata, TaskPriority, DEFAULT_RESOURCE_GROUP_NAME}, + resource_control::{DEFAULT_RESOURCE_GROUP_NAME, TaskMetadata, TaskPriority}, time::Instant, }; use yatp::queue::priority::TaskPriorityProvider; @@ -1086,7 +1086,7 @@ pub(crate) mod tests { #[cfg(feature = "failpoints")] #[test] fn test_reset_resource_group_vt_overflow() { - use rand::{thread_rng, RngCore}; + use rand::{RngCore, thread_rng}; let resource_manager = ResourceGroupManager::default(); let resource_ctl = resource_manager.derive_controller("test_write".into(), false); let mut rng = thread_rng(); diff --git a/components/resource_control/src/service.rs b/components/resource_control/src/service.rs index 0436ce71747..9cd71d0cf1f 100644 --- a/components/resource_control/src/service.rs +++ b/components/resource_control/src/service.rs @@ -6,20 +6,20 @@ use std::{ time::Duration, }; -use futures::{compat::Future01CompatExt, stream, StreamExt}; +use futures::{StreamExt, compat::Future01CompatExt, stream}; use kvproto::{ meta_storagepb::EventEventType, resource_manager::{ResourceGroup, TokenBucketRequest, TokenBucketsRequest}, }; use pd_client::{ + Error as PdError, PdClient, RESOURCE_CONTROL_CONFIG_PATH, + RESOURCE_CONTROL_CONTROLLER_CONFIG_PATH, RpcClient, meta_storage::{Checked, Get, MetaStorageClient, Sourced, Watch}, - Error as PdError, PdClient, RpcClient, RESOURCE_CONTROL_CONFIG_PATH, - RESOURCE_CONTROL_CONTROLLER_CONFIG_PATH, }; use serde::{Deserialize, Serialize}; use tikv_util::{error, info, timer::GLOBAL_TIMER_HANDLE}; -use crate::{resource_limiter::ResourceType, ResourceGroupManager}; +use crate::{ResourceGroupManager, resource_limiter::ResourceType}; #[derive(Clone)] pub struct ResourceManagerService { @@ -335,11 +335,11 @@ pub mod tests { use file_system::IoBytes; use futures::executor::block_on; use pd_client::{ - meta_storage::{Delete, Put}, RpcClient, + meta_storage::{Delete, Put}, }; use protobuf::Message; - use test_pd::{mocker::MetaStorage, util::*, Server as MockServer}; + use test_pd::{Server as MockServer, mocker::MetaStorage, util::*}; use tikv_util::{config::ReadableDuration, worker::Builder}; use crate::resource_group::tests::{ diff --git a/components/resource_control/src/worker.rs b/components/resource_control/src/worker.rs index 4d85b147fbc..f2977862804 100644 --- a/components/resource_control/src/worker.rs +++ b/components/resource_control/src/worker.rs @@ -8,13 +8,13 @@ use std::{ time::Duration, }; -use file_system::{fetch_io_bytes, IoBytes, IoType}; +use file_system::{IoBytes, IoType, fetch_io_bytes}; use prometheus::Histogram; use strum::EnumCount; use tikv_util::{ debug, - resource_control::{TaskPriority, DEFAULT_RESOURCE_GROUP_NAME}, - sys::{cpu_time::ProcessStat, SysQuota}, + resource_control::{DEFAULT_RESOURCE_GROUP_NAME, TaskPriority}, + sys::{SysQuota, cpu_time::ProcessStat}, time::Instant, warn, yatp_pool::metrics::YATP_POOL_SCHEDULE_WAIT_DURATION_VEC, diff --git a/components/resource_metering/src/config.rs b/components/resource_metering/src/config.rs index 090768a9493..cf4a495ee63 100644 --- a/components/resource_metering/src/config.rs +++ b/components/resource_metering/src/config.rs @@ -7,8 +7,8 @@ use serde_derive::{Deserialize, Serialize}; use tikv_util::config::ReadableDuration; use crate::{ - recorder::ConfigChangeNotifier as RecorderConfigChangeNotifier, - reporter::ConfigChangeNotifier as ReporterConfigChangeNotifier, AddressChangeNotifier, + AddressChangeNotifier, recorder::ConfigChangeNotifier as RecorderConfigChangeNotifier, + reporter::ConfigChangeNotifier as ReporterConfigChangeNotifier, }; const MIN_PRECISION: ReadableDuration = ReadableDuration::millis(100); diff --git a/components/resource_metering/src/lib.rs b/components/resource_metering/src/lib.rs index 3d837b35ba2..e014d84bda1 100644 --- a/components/resource_metering/src/lib.rs +++ b/components/resource_metering/src/lib.rs @@ -2,7 +2,6 @@ // TODO(mornyx): crate doc. -#![feature(hash_extract_if)] #![allow(internal_features)] #![feature(core_intrinsics)] @@ -10,8 +9,8 @@ use std::{ intrinsics::unlikely, pin::Pin, sync::{ - atomic::Ordering::{Relaxed, SeqCst}, Arc, + atomic::Ordering::{Relaxed, SeqCst}, }, task::{Context, Poll}, }; @@ -20,18 +19,18 @@ pub use collector::Collector; pub use config::{Config, ConfigManager}; pub use model::*; pub use recorder::{ - init_recorder, record_read_keys, record_write_keys, CollectorGuard, CollectorId, - CollectorRegHandle, ConfigChangeNotifier as RecorderConfigChangeNotifier, CpuRecorder, - Recorder, RecorderBuilder, SummaryRecorder, + CollectorGuard, CollectorId, CollectorRegHandle, + ConfigChangeNotifier as RecorderConfigChangeNotifier, CpuRecorder, Recorder, RecorderBuilder, + SummaryRecorder, init_recorder, record_read_keys, record_write_keys, }; use recorder::{LocalStorage, LocalStorageRef, STORAGE}; pub use reporter::{ + ConfigChangeNotifier as ReporterConfigChangeNotifier, Reporter, Task, data_sink::DataSink, data_sink_reg::DataSinkRegHandle, init_reporter, pubsub::PubSubService, - single_target::{init_single_target, AddressChangeNotifier, SingleTargetDataSink}, - ConfigChangeNotifier as ReporterConfigChangeNotifier, Reporter, Task, + single_target::{AddressChangeNotifier, SingleTargetDataSink, init_single_target}, }; use tikv_util::{ memory::HeapSize, diff --git a/components/resource_metering/src/model.rs b/components/resource_metering/src/model.rs index 07396bbec50..11bff95b575 100644 --- a/components/resource_metering/src/model.rs +++ b/components/resource_metering/src/model.rs @@ -3,8 +3,8 @@ use std::{ cell::Cell, sync::{ - atomic::{AtomicU32, Ordering::Relaxed}, Arc, + atomic::{AtomicU32, Ordering::Relaxed}, }, time::{Duration, SystemTime, UNIX_EPOCH}, }; diff --git a/components/resource_metering/src/recorder/collector_reg.rs b/components/resource_metering/src/recorder/collector_reg.rs index f166101dfe5..6d79a09c743 100644 --- a/components/resource_metering/src/recorder/collector_reg.rs +++ b/components/resource_metering/src/recorder/collector_reg.rs @@ -7,7 +7,7 @@ use tikv_util::{ worker::{Scheduler, Worker}, }; -use crate::{recorder::Task, Collector}; +use crate::{Collector, recorder::Task}; /// `CollectorRegHandle` accepts registrations of [Collector]. /// diff --git a/components/resource_metering/src/recorder/localstorage.rs b/components/resource_metering/src/recorder/localstorage.rs index c9f0b25b478..b8de99ebd17 100644 --- a/components/resource_metering/src/recorder/localstorage.rs +++ b/components/resource_metering/src/recorder/localstorage.rs @@ -2,14 +2,14 @@ use std::{ cell::RefCell, - sync::{atomic::AtomicBool, Arc, Mutex}, + sync::{Arc, Mutex, atomic::AtomicBool}, }; use collections::HashMap; use crossbeam::atomic::AtomicCell; use tikv_util::sys::thread::Pid; -use crate::{model::SummaryRecord, TagInfos}; +use crate::{TagInfos, model::SummaryRecord}; thread_local! { /// `STORAGE` is a thread-localized instance of [LocalStorage]. diff --git a/components/resource_metering/src/recorder/mod.rs b/components/resource_metering/src/recorder/mod.rs index f0b2e88ee4e..8acd0e59a22 100644 --- a/components/resource_metering/src/recorder/mod.rs +++ b/components/resource_metering/src/recorder/mod.rs @@ -15,7 +15,7 @@ use tikv_util::{ }; use self::{collector_reg::CollectorReg, sub_recorder::SubRecorder}; -use crate::{collector::Collector, Config, RawRecords, ResourceTagFactory}; +use crate::{Config, RawRecords, ResourceTagFactory, collector::Collector}; mod collector_reg; mod localstorage; @@ -26,7 +26,7 @@ pub use self::{ localstorage::{LocalStorage, LocalStorageRef, STORAGE}, sub_recorder::{ cpu::CpuRecorder, - summary::{record_read_keys, record_write_keys, SummaryRecorder}, + summary::{SummaryRecorder, record_read_keys, record_write_keys}, }, }; @@ -328,8 +328,8 @@ pub fn init_recorder( mod tests { use std::{ sync::{ - atomic::{AtomicUsize, Ordering::SeqCst}, Mutex, + atomic::{AtomicUsize, Ordering::SeqCst}, }, thread::sleep, }; @@ -338,8 +338,8 @@ mod tests { use super::*; use crate::{ - recorder::localstorage::{LocalStorage, LocalStorageRef}, TagInfos, + recorder::localstorage::{LocalStorage, LocalStorageRef}, }; #[derive(Clone, Default)] diff --git a/components/resource_metering/src/recorder/sub_recorder/cpu.rs b/components/resource_metering/src/recorder/sub_recorder/cpu.rs index 08675bb6153..94aae4075f2 100644 --- a/components/resource_metering/src/recorder/sub_recorder/cpu.rs +++ b/components/resource_metering/src/recorder/sub_recorder/cpu.rs @@ -4,12 +4,12 @@ use collections::HashMap; use tikv_util::sys::thread::{self, Pid}; use crate::{ + RawRecords, metrics::STAT_TASK_COUNT, recorder::{ - localstorage::{LocalStorage, SharedTagInfos}, SubRecorder, + localstorage::{LocalStorage, SharedTagInfos}, }, - RawRecords, }; /// An implementation of [SubRecorder] for collecting cpu statistics. diff --git a/components/resource_metering/src/recorder/sub_recorder/mod.rs b/components/resource_metering/src/recorder/sub_recorder/mod.rs index 42647f3486d..ef6cd500b5b 100644 --- a/components/resource_metering/src/recorder/sub_recorder/mod.rs +++ b/components/resource_metering/src/recorder/sub_recorder/mod.rs @@ -3,7 +3,7 @@ use collections::HashMap; use tikv_util::sys::thread::Pid; -use crate::{recorder::localstorage::LocalStorage, RawRecords}; +use crate::{RawRecords, recorder::localstorage::LocalStorage}; pub mod cpu; pub mod summary; diff --git a/components/resource_metering/src/recorder/sub_recorder/summary.rs b/components/resource_metering/src/recorder/sub_recorder/summary.rs index 93ba95080e3..8c0b5afcbf9 100644 --- a/components/resource_metering/src/recorder/sub_recorder/summary.rs +++ b/components/resource_metering/src/recorder/sub_recorder/summary.rs @@ -6,11 +6,11 @@ use collections::HashMap; use tikv_util::sys::thread::Pid; use crate::{ + RawRecords, recorder::{ - localstorage::{LocalStorage, STORAGE}, SubRecorder, + localstorage::{LocalStorage, STORAGE}, }, - RawRecords, }; /// Records how many keys have been read in the current context. diff --git a/components/resource_metering/src/reporter/collector_impl.rs b/components/resource_metering/src/reporter/collector_impl.rs index 5bba601807c..53d9b93befc 100644 --- a/components/resource_metering/src/reporter/collector_impl.rs +++ b/components/resource_metering/src/reporter/collector_impl.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use tikv_util::{warn, worker::Scheduler}; -use crate::{metrics::IGNORED_DATA_COUNTER, Collector, RawRecords, Task}; +use crate::{Collector, RawRecords, Task, metrics::IGNORED_DATA_COUNTER}; /// A [Collector] implementation for scheduling [RawRecords]. /// diff --git a/components/resource_metering/src/reporter/mod.rs b/components/resource_metering/src/reporter/mod.rs index 721fb570b22..f7620ba3af9 100644 --- a/components/resource_metering/src/reporter/mod.rs +++ b/components/resource_metering/src/reporter/mod.rs @@ -20,12 +20,12 @@ use tikv_util::{ }; use crate::{ + Config, DataSink, RawRecords, Records, recorder::{CollectorGuard, CollectorRegHandle}, reporter::{ collector_impl::CollectorImpl, data_sink_reg::{DataSinkId, DataSinkReg, DataSinkRegHandle}, }, - Config, DataSink, RawRecords, Records, }; /// A structure for reporting statistics through [Client]. @@ -244,7 +244,7 @@ mod tests { }; use super::*; - use crate::{error::Result, RawRecord, TagInfos}; + use crate::{RawRecord, TagInfos, error::Result}; #[derive(Default, Clone)] struct MockDataSink { diff --git a/components/resource_metering/src/reporter/pubsub.rs b/components/resource_metering/src/reporter/pubsub.rs index 62144ec920c..2dc3cc4376f 100644 --- a/components/resource_metering/src/reporter/pubsub.rs +++ b/components/resource_metering/src/reporter/pubsub.rs @@ -3,8 +3,8 @@ use std::sync::Arc; use futures::{ - channel::mpsc::{channel, Sender}, SinkExt, StreamExt, + channel::mpsc::{Sender, channel}, }; use grpcio::{RpcContext, ServerStreamingSink, WriteFlags}; use kvproto::{ @@ -15,9 +15,9 @@ use tikv_util::{info, warn}; use super::DataSinkRegHandle; use crate::{ + DataSink, error::Result, metrics::{IGNORED_DATA_COUNTER, REPORT_DATA_COUNTER, REPORT_DURATION_HISTOGRAM}, - DataSink, }; /// `PubSubService` implements [ResourceMeteringPubSub]. diff --git a/components/resource_metering/src/reporter/single_target.rs b/components/resource_metering/src/reporter/single_target.rs index 09609b84462..c6113c51778 100644 --- a/components/resource_metering/src/reporter/single_target.rs +++ b/components/resource_metering/src/reporter/single_target.rs @@ -3,8 +3,8 @@ use std::{ fmt::{self, Display, Formatter}, sync::{ - atomic::{AtomicBool, Ordering}, Arc, + atomic::{AtomicBool, Ordering}, }, time::Duration, }; diff --git a/components/resource_metering/tests/recorder_test.rs b/components/resource_metering/tests/recorder_test.rs index 9f0ec504917..0d75b6b7566 100644 --- a/components/resource_metering/tests/recorder_test.rs +++ b/components/resource_metering/tests/recorder_test.rs @@ -8,10 +8,10 @@ mod tests { time::Duration, }; + use Operation::*; use collections::HashMap; - use resource_metering::{init_recorder, Collector, RawRecord, RawRecords, ResourceTagFactory}; + use resource_metering::{Collector, RawRecord, RawRecords, ResourceTagFactory, init_recorder}; use tikv_util::sys::thread; - use Operation::*; enum Operation { SetContext(&'static str), diff --git a/components/resource_metering/tests/summary_test.rs b/components/resource_metering/tests/summary_test.rs index ae647055206..c99ff29111a 100644 --- a/components/resource_metering/tests/summary_test.rs +++ b/components/resource_metering/tests/summary_test.rs @@ -8,7 +8,7 @@ use std::{ use collections::HashMap; use kvproto::{kvrpcpb::Context, resource_usage_agent::ResourceUsageRecord}; -use resource_metering::{error::Result, init_recorder, init_reporter, Config, DataSink}; +use resource_metering::{Config, DataSink, error::Result, init_recorder, init_reporter}; use tikv_util::config::ReadableDuration; const PRECISION_MS: u64 = 1000; diff --git a/components/server/src/common.rs b/components/server/src/common.rs index 89769de17d4..0c6fe8320ae 100644 --- a/components/server/src/common.rs +++ b/components/server/src/common.rs @@ -8,30 +8,29 @@ use std::{ net::SocketAddr, path::{Path, PathBuf}, sync::{ + Arc, atomic::{AtomicU32, Ordering}, - mpsc, Arc, + mpsc, }, time::Duration, - u64, }; -use encryption_export::{data_key_manager_from_config, DataKeyManager}; +use encryption_export::{DataKeyManager, data_key_manager_from_config}; use engine_rocks::{ - flush_engine_statistics, + FlowInfo, RocksEngine, RocksStatistics, flush_engine_statistics, raw::{Cache, Env}, - FlowInfo, RocksEngine, RocksStatistics, }; use engine_traits::{ - data_cf_offset, CachedTablet, CfOptions, CfOptionsExt, FlowControlFactorsExt, KvEngine, - RaftEngine, RegionCacheEngine, StatisticsReporter, TabletRegistry, CF_DEFAULT, DATA_CFS, + CF_DEFAULT, CachedTablet, CfOptions, CfOptionsExt, DATA_CFS, FlowControlFactorsExt, KvEngine, + RaftEngine, RegionCacheEngine, StatisticsReporter, TabletRegistry, data_cf_offset, }; use error_code::ErrorCodeExt; -use file_system::{get_io_rate_limiter, set_io_rate_limiter, BytesFetcher, File, IoBudgetAdjustor}; +use file_system::{BytesFetcher, File, IoBudgetAdjustor, get_io_rate_limiter, set_io_rate_limiter}; use grpcio::Environment; use hybrid_engine::HybridEngine; use in_memory_engine::{ - flush_in_memory_engine_statistics, InMemoryEngineContext, InMemoryEngineStatistics, - RegionCacheMemoryEngine, + InMemoryEngineContext, InMemoryEngineStatistics, RegionCacheMemoryEngine, + flush_in_memory_engine_statistics, }; use pd_client::{PdClient, RpcClient}; use raft_log_engine::RaftLogEngine; @@ -40,15 +39,15 @@ use security::SecurityManager; use tikv::{ config::{ConfigController, DbConfigManger, DbType, TikvConfig}, server::{ - gc_worker::compaction_filter::GC_CONTEXT, status_server::StatusServer, DEFAULT_CLUSTER_ID, + DEFAULT_CLUSTER_ID, gc_worker::compaction_filter::GC_CONTEXT, status_server::StatusServer, }, }; use tikv_util::{ - config::{ensure_dir_exist, RaftDataStateMachine}, + config::{RaftDataStateMachine, ensure_dir_exist}, math::MovingAvgU32, metrics::INSTANCE_BACKEND_CPU_QUOTA, quota_limiter::QuotaLimiter, - sys::{cpu_time::ProcessStat, disk, path_in_diff_mount_point, SysQuota}, + sys::{SysQuota, cpu_time::ProcessStat, disk, path_in_diff_mount_point}, time::Instant, worker::{LazyWorker, Worker}, }; @@ -646,7 +645,7 @@ impl EnginesResourceInfo { ); } - #[cfg(any(test, feature = "testexport"))] + #[cfg(test)] pub fn latest_normalized_pending_bytes(&self) -> u32 { self.latest_normalized_pending_bytes.load(Ordering::Relaxed) } @@ -1021,7 +1020,7 @@ impl DiskUsageChecker { // If the auxiliary directory of raft engine is not separated from // kv engine, returns u64::MAX to indicate that the disk space of // the raft engine should not be checked. - (std::u64::MAX, std::u64::MAX) + (u64::MAX, u64::MAX) } else if self.separated_raft_auxiliary_and_kvdb_mount_path { // If the auxiliary directory of raft engine is separated from kv // engine and the main directory of diff --git a/components/server/src/raft_engine_switch.rs b/components/server/src/raft_engine_switch.rs index 729029d4c8f..52320a36ea8 100644 --- a/components/server/src/raft_engine_switch.rs +++ b/components/server/src/raft_engine_switch.rs @@ -1,13 +1,13 @@ // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. use std::sync::{ - atomic::{AtomicUsize, Ordering}, Arc, + atomic::{AtomicUsize, Ordering}, }; -use crossbeam::channel::{unbounded, Receiver}; +use crossbeam::channel::{Receiver, unbounded}; use engine_rocks::{self, RocksEngine}; -use engine_traits::{Iterable, Iterator, RaftEngine, RaftEngineReadOnly, RaftLogBatch, CF_DEFAULT}; +use engine_traits::{CF_DEFAULT, Iterable, Iterator, RaftEngine, RaftEngineReadOnly, RaftLogBatch}; use kvproto::raft_serverpb::RaftLocalState; use protobuf::Message; use raft::eraftpb::Entry; diff --git a/components/server/src/server.rs b/components/server/src/server.rs index 15542321873..9576f876537 100644 --- a/components/server/src/server.rs +++ b/components/server/src/server.rs @@ -16,28 +16,27 @@ use std::{ convert::TryFrom, path::{Path, PathBuf}, str::FromStr, - sync::{atomic::AtomicU64, mpsc, Arc, Mutex}, + sync::{Arc, Mutex, atomic::AtomicU64, mpsc}, time::Duration, - u64, }; -use api_version::{dispatch_api_version, KvFormat}; +use api_version::{KvFormat, dispatch_api_version}; use backup_stream::{ - config::BackupStreamConfigManager, metadata::store::PdStore, observer::BackupStreamObserver, - BackupStreamResolver, + BackupStreamResolver, config::BackupStreamConfigManager, metadata::store::PdStore, + observer::BackupStreamObserver, }; use causal_ts::CausalTsProviderImpl; use cdc::CdcConfigManager; use concurrency_manager::{ConcurrencyManager, LIMIT_VALID_TIME_MULTIPLIER}; use engine_rocks::{ - from_rocks_compression_type, RocksCompactedEvent, RocksEngine, RocksStatistics, + RocksCompactedEvent, RocksEngine, RocksStatistics, from_rocks_compression_type, }; -use engine_rocks_helper::sst_recovery::{RecoveryRunner, DEFAULT_CHECK_INTERVAL}; +use engine_rocks_helper::sst_recovery::{DEFAULT_CHECK_INTERVAL, RecoveryRunner}; use engine_traits::{ - Engines, KvEngine, MiscExt, RaftEngine, SingletonFactory, TabletContext, TabletRegistry, - CF_DEFAULT, CF_WRITE, + CF_DEFAULT, CF_WRITE, Engines, KvEngine, MiscExt, RaftEngine, SingletonFactory, TabletContext, + TabletRegistry, }; -use file_system::{get_io_rate_limiter, BytesFetcher, MetricsManager as IoMetricsManager}; +use file_system::{BytesFetcher, MetricsManager as IoMetricsManager, get_io_rate_limiter}; use futures::executor::block_on; use grpcio::{EnvBuilder, Environment}; use health_controller::HealthController; @@ -46,8 +45,8 @@ use hybrid_engine::observer::{ RegionCacheWriteBatchObserver, }; use in_memory_engine::{ - config::InMemoryEngineConfigManager, InMemoryEngineContext, InMemoryEngineStatistics, - RegionCacheMemoryEngine, + InMemoryEngineContext, InMemoryEngineStatistics, RegionCacheMemoryEngine, + config::InMemoryEngineConfigManager, }; use kvproto::{ brpb::create_backup, cdcpb::create_change_data, deadlock::create_deadlock, @@ -56,32 +55,32 @@ use kvproto::{ resource_usage_agent::create_resource_metering_pub_sub, }; use pd_client::{ + PdClient, RpcClient, meta_storage::{Checked, Sourced}, metrics::STORE_SIZE_EVENT_INT_VEC, - PdClient, RpcClient, }; use raft_log_engine::RaftLogEngine; use raftstore::{ + RaftRouterCompactedEventSender, coprocessor::{ - config::SplitCheckConfigManager, BoxConsistencyCheckObserver, ConsistencyCheckMethod, - CoprocessorHost, RawConsistencyCheckObserver, RegionInfoAccessor, + BoxConsistencyCheckObserver, ConsistencyCheckMethod, CoprocessorHost, + RawConsistencyCheckObserver, RegionInfoAccessor, config::SplitCheckConfigManager, }, router::{CdcRaftRouter, ServerRaftStoreRouter}, store::{ + AutoSplitController, CheckLeaderRunner, DiskCheckRunner, LocalReader, SnapManager, + SnapManagerBuilder, SplitCheckRunner, SplitConfigManager, StoreMetaDelegate, config::RaftstoreConfigManager, fsm, fsm::store::{ - RaftBatchSystem, RaftRouter, StoreMeta, MULTI_FILES_SNAPSHOT_FEATURE, PENDING_MSG_CAP, + MULTI_FILES_SNAPSHOT_FEATURE, PENDING_MSG_CAP, RaftBatchSystem, RaftRouter, StoreMeta, }, memory::MEMTRACE_ROOT as MEMTRACE_RAFTSTORE, snapshot_backup::PrepareDiskSnapObserver, - AutoSplitController, CheckLeaderRunner, DiskCheckRunner, LocalReader, SnapManager, - SnapManagerBuilder, SplitCheckRunner, SplitConfigManager, StoreMetaDelegate, }, - RaftRouterCompactedEventSender, }; use resolved_ts::{LeadershipResolver, Task}; -use resource_control::{config::ResourceContrlCfgMgr, ResourceGroupManager}; +use resource_control::{ResourceGroupManager, config::ResourceContrlCfgMgr}; use security::{SecurityConfigManager, SecurityManager}; use service::{service_event::ServiceEvent, service_manager::GrpcServiceManager}; use snap_recovery::RecoveryService; @@ -93,9 +92,11 @@ use tikv::{ coprocessor_v2, import::{ImportSstService, SstImporter}, read_pool::{ - build_yatp_read_pool, ReadPool, ReadPoolConfigManager, UPDATE_EWMA_TIME_SLICE_INTERVAL, + ReadPool, ReadPoolConfigManager, UPDATE_EWMA_TIME_SLICE_INTERVAL, build_yatp_read_pool, }, server::{ + CPU_CORES_QUOTA_GAUGE, GRPC_THREAD_PREFIX, KvEngineFactoryBuilder, MEMORY_LIMIT_GAUGE, + MultiRaftServer, RaftKv, Server, config::{Config as ServerConfig, ServerConfigManager}, debug::{Debugger, DebuggerImpl}, gc_worker::{AutoGcConfig, GcWorker}, @@ -106,11 +107,9 @@ use tikv::{ status_server::StatusServer, tablet_snap::NoSnapshotCache, ttl::TtlChecker, - KvEngineFactoryBuilder, MultiRaftServer, RaftKv, Server, CPU_CORES_QUOTA_GAUGE, - GRPC_THREAD_PREFIX, MEMORY_LIMIT_GAUGE, }, storage::{ - self, + self, Engine, Storage, config::EngineType, config_manager::StorageConfigManger, kv::LocalTablets, @@ -119,31 +118,29 @@ use tikv::{ flow_controller::{EngineFlowController, FlowController}, txn_status_cache::TxnStatusCache, }, - Engine, Storage, }, }; use tikv_alloc::{ add_thread_memory_accessor, remove_thread_memory_accessor, thread_allocate_exclusive_arena, }; use tikv_util::{ - check_environment_variables, + Either, check_environment_variables, config::VersionTrack, memory::MemoryQuota, mpsc as TikvMpsc, quota_limiter::{QuotaLimitConfigManager, QuotaLimiter}, - sys::{disk, path_in_diff_mount_point, register_memory_usage_high_water, SysQuota}, + sys::{SysQuota, disk, path_in_diff_mount_point, register_memory_usage_high_water}, thread_group::GroupProperties, time::{Instant, Monitor}, worker::{Builder as WorkerBuilder, LazyWorker, Scheduler, Worker}, yatp_pool::CleanupMethod, - Either, }; use tokio::runtime::Builder; use crate::{ common::{ - build_hybrid_engine, ConfiguredRaftEngine, DiskUsageChecker, EngineMetricsManager, - EnginesResourceInfo, TikvServerCore, + ConfiguredRaftEngine, DiskUsageChecker, EngineMetricsManager, EnginesResourceInfo, + TikvServerCore, build_hybrid_engine, }, memory::*, setup::*, @@ -1384,7 +1381,7 @@ where self.tablet_registry.clone().unwrap(), self.kv_statistics.clone(), self.in_memory_engine_statistics.clone(), - self.core.config.rocksdb.titan.enabled.map_or(false, |v| v), + self.core.config.rocksdb.titan.enabled.is_some_and(|v| v), self.engines.as_ref().unwrap().engines.raft.clone(), self.raft_statistics.clone(), ); @@ -1835,7 +1832,7 @@ mod test { use engine_rocks::raw::Env; use engine_traits::{ - FlowControlFactorsExt, MiscExt, SyncMutable, TabletContext, TabletRegistry, CF_DEFAULT, + CF_DEFAULT, FlowControlFactorsExt, MiscExt, SyncMutable, TabletContext, TabletRegistry, }; use tempfile::Builder; use tikv::{config::TikvConfig, server::KvEngineFactoryBuilder}; diff --git a/components/server/src/server2.rs b/components/server/src/server2.rs index af46c6bdc0e..d8f8e58ab3e 100644 --- a/components/server/src/server2.rs +++ b/components/server/src/server2.rs @@ -17,26 +17,25 @@ use std::{ path::{Path, PathBuf}, str::FromStr, sync::{ + Arc, atomic::AtomicU64, mpsc::{self, sync_channel}, - Arc, }, time::Duration, - u64, }; -use api_version::{dispatch_api_version, KvFormat}; +use api_version::{KvFormat, dispatch_api_version}; use backup::disk_snap::Env; use backup_stream::{ - config::BackupStreamConfigManager, metadata::store::PdStore, observer::BackupStreamObserver, - BackupStreamResolver, + BackupStreamResolver, config::BackupStreamConfigManager, metadata::store::PdStore, + observer::BackupStreamObserver, }; use causal_ts::CausalTsProviderImpl; use cdc::CdcConfigManager; use concurrency_manager::{ConcurrencyManager, LIMIT_VALID_TIME_MULTIPLIER}; -use engine_rocks::{from_rocks_compression_type, RocksEngine, RocksStatistics}; -use engine_traits::{Engines, KvEngine, MiscExt, RaftEngine, TabletRegistry, CF_DEFAULT, CF_WRITE}; -use file_system::{get_io_rate_limiter, BytesFetcher, MetricsManager as IoMetricsManager}; +use engine_rocks::{RocksEngine, RocksStatistics, from_rocks_compression_type}; +use engine_traits::{CF_DEFAULT, CF_WRITE, Engines, KvEngine, MiscExt, RaftEngine, TabletRegistry}; +use file_system::{BytesFetcher, MetricsManager as IoMetricsManager, get_io_rate_limiter}; use futures::executor::block_on; use grpcio::{EnvBuilder, Environment}; use health_controller::HealthController; @@ -48,42 +47,44 @@ use kvproto::{ resource_usage_agent::create_resource_metering_pub_sub, }; use pd_client::{ + PdClient, RpcClient, meta_storage::{Checked, Sourced}, metrics::STORE_SIZE_EVENT_INT_VEC, - PdClient, RpcClient, }; use raft_log_engine::RaftLogEngine; use raftstore::{ + RegionInfoAccessor, coprocessor::{ BoxConsistencyCheckObserver, ConsistencyCheckMethod, CoprocessorHost, RawConsistencyCheckObserver, }, store::{ - config::RaftstoreConfigManager, memory::MEMTRACE_ROOT as MEMTRACE_RAFTSTORE, AutoSplitController, CheckLeaderRunner, SplitConfigManager, TabletSnapManager, + config::RaftstoreConfigManager, memory::MEMTRACE_ROOT as MEMTRACE_RAFTSTORE, }, - RegionInfoAccessor, }; use raftstore_v2::{ - router::{DiskSnapBackupHandle, PeerMsg, RaftRouter}, StateStorage, + router::{DiskSnapBackupHandle, PeerMsg, RaftRouter}, }; use resolved_ts::Task; -use resource_control::{config::ResourceContrlCfgMgr, ResourceGroupManager}; +use resource_control::{ResourceGroupManager, config::ResourceContrlCfgMgr}; use security::{SecurityConfigManager, SecurityManager}; use service::{service_event::ServiceEvent, service_manager::GrpcServiceManager}; use tikv::{ config::{ - loop_registry, ConfigController, ConfigurableDb, DbConfigManger, DbType, LogConfigManager, - MemoryConfigManager, TikvConfig, + ConfigController, ConfigurableDb, DbConfigManger, DbType, LogConfigManager, + MemoryConfigManager, TikvConfig, loop_registry, }, coprocessor::{self, MEMTRACE_ROOT as MEMTRACE_COPROCESSOR}, coprocessor_v2, import::{ImportSstService, SstImporter}, read_pool::{ - build_yatp_read_pool, ReadPool, ReadPoolConfigManager, UPDATE_EWMA_TIME_SLICE_INTERVAL, + ReadPool, ReadPoolConfigManager, UPDATE_EWMA_TIME_SLICE_INTERVAL, build_yatp_read_pool, }, server::{ + CPU_CORES_QUOTA_GAUGE, GRPC_THREAD_PREFIX, KvEngineFactoryBuilder, MEMORY_LIMIT_GAUGE, + NodeV2, RaftKv2, Server, config::{Config as ServerConfig, ServerConfigManager}, debug::Debugger, debug2::DebuggerImplV2, @@ -93,11 +94,9 @@ use tikv::{ resolve, service::{DebugService, DefaultGrpcMessageFilter, DiagnosticsService}, status_server::StatusServer, - KvEngineFactoryBuilder, NodeV2, RaftKv2, Server, CPU_CORES_QUOTA_GAUGE, GRPC_THREAD_PREFIX, - MEMORY_LIMIT_GAUGE, }, storage::{ - self, + self, Engine, Storage, config::EngineType, config_manager::StorageConfigManger, kv::LocalTablets, @@ -106,24 +105,22 @@ use tikv::{ flow_controller::{FlowController, TabletFlowController}, txn_status_cache::TxnStatusCache, }, - Engine, Storage, }, }; use tikv_alloc::{ add_thread_memory_accessor, remove_thread_memory_accessor, thread_allocate_exclusive_arena, }; use tikv_util::{ - check_environment_variables, + Either, check_environment_variables, config::VersionTrack, memory::MemoryQuota, mpsc as TikvMpsc, quota_limiter::{QuotaLimitConfigManager, QuotaLimiter}, - sys::{disk, path_in_diff_mount_point, register_memory_usage_high_water, SysQuota}, + sys::{SysQuota, disk, path_in_diff_mount_point, register_memory_usage_high_water}, thread_group::GroupProperties, time::{Instant, Monitor}, worker::{Builder as WorkerBuilder, LazyWorker, Scheduler}, yatp_pool::CleanupMethod, - Either, }; use tokio::runtime::Builder; @@ -1171,7 +1168,7 @@ where self.tablet_registry.clone().unwrap(), self.kv_statistics.clone(), self.in_memory_engine_statistics.clone(), - self.core.config.rocksdb.titan.enabled.map_or(false, |v| v), + self.core.config.rocksdb.titan.enabled.is_some_and(|v| v), self.engines.as_ref().unwrap().raft_engine.clone(), self.raft_statistics.clone(), ); @@ -1669,7 +1666,7 @@ mod test { use engine_rocks::raw::Env; use engine_traits::{ - FlowControlFactorsExt, MiscExt, SyncMutable, TabletContext, TabletRegistry, CF_DEFAULT, + CF_DEFAULT, FlowControlFactorsExt, MiscExt, SyncMutable, TabletContext, TabletRegistry, }; use tempfile::Builder; use tikv::{config::TikvConfig, server::KvEngineFactoryBuilder}; diff --git a/components/service/src/service_manager.rs b/components/service/src/service_manager.rs index 0b69fc00c93..212117374c6 100644 --- a/components/service/src/service_manager.rs +++ b/components/service/src/service_manager.rs @@ -1,4 +1,4 @@ -use std::sync::{atomic::Ordering, Arc}; +use std::sync::{Arc, atomic::Ordering}; use atomic::Atomic; use crossbeam::channel::SendError; diff --git a/components/snap_recovery/Cargo.toml b/components/snap_recovery/Cargo.toml index 557fec0a2cf..aa48453e6bb 100644 --- a/components/snap_recovery/Cargo.toml +++ b/components/snap_recovery/Cargo.toml @@ -23,7 +23,7 @@ keys = { workspace = true } kvproto = { workspace = true } lazy_static = "1.4" pd_client = { workspace = true } -prometheus = { version = "0.13", default_features = false, features = ["nightly"] } +prometheus = { version = "0.13", default-features = false, features = ["nightly"] } prometheus-static-metric = "0.5" raft_log_engine = { workspace = true } raftstore = { workspace = true } diff --git a/components/snap_recovery/src/data_resolver.rs b/components/snap_recovery/src/data_resolver.rs index 90edb8d6348..810676b6cbf 100644 --- a/components/snap_recovery/src/data_resolver.rs +++ b/components/snap_recovery/src/data_resolver.rs @@ -11,8 +11,8 @@ use std::{ use engine_rocks::{RocksEngine, RocksEngineIterator, RocksWriteBatchVec}; use engine_traits::{ - IterOptions, Iterable, Iterator, Mutable, WriteBatch, WriteBatchExt, WriteOptions, CF_DEFAULT, - CF_LOCK, CF_WRITE, + CF_DEFAULT, CF_LOCK, CF_WRITE, IterOptions, Iterable, Iterator, Mutable, WriteBatch, + WriteBatchExt, WriteOptions, }; use futures::channel::mpsc::UnboundedSender; use kvproto::recoverdatapb::ResolveKvDataResponse; @@ -306,7 +306,7 @@ impl WriteResolverWorker { } #[cfg(test)] mod tests { - use engine_traits::{WriteBatch, WriteBatchExt, ALL_CFS, CF_LOCK}; + use engine_traits::{ALL_CFS, CF_LOCK, WriteBatch, WriteBatchExt}; use futures::channel::mpsc; use tempfile::Builder; use txn_types::{Lock, LockType, WriteType}; diff --git a/components/snap_recovery/src/init_cluster.rs b/components/snap_recovery/src/init_cluster.rs index c6a14c1e0d3..3d2cf85e596 100644 --- a/components/snap_recovery/src/init_cluster.rs +++ b/components/snap_recovery/src/init_cluster.rs @@ -1,6 +1,6 @@ // Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0. -use std::{cmp, error::Error as StdError, i32, result, sync::Arc, thread, time::Duration}; +use std::{cmp, error::Error as StdError, result, sync::Arc, thread, time::Duration}; use encryption_export::data_key_manager_from_config; use engine_rocks::util::new_engine_opt; @@ -12,7 +12,7 @@ use raftstore::store::initial_region; use thiserror::Error; use tikv::{ config::TikvConfig, - server::{config::Config as ServerConfig, KvEngineFactoryBuilder}, + server::{KvEngineFactoryBuilder, config::Config as ServerConfig}, }; use tikv_util::{ config::{ReadableDuration, ReadableSize, VersionTrack}, @@ -53,7 +53,7 @@ const MAX_SPLIT_KEY: u64 = 1 << 31; /// 2. peer valid during a recovery time even without leader in its region /// 3. PD can not put any peer into tombstone /// 4. must ensure all region data with ts less than backup ts (below commit -/// index) are safe +/// index) are safe pub fn enter_snap_recovery_mode(config: &mut TikvConfig) { // TOOD: if we do not have to restart TiKV, then, we need exit the recovery mode // and bring the following parameter back. diff --git a/components/snap_recovery/src/leader_keeper.rs b/components/snap_recovery/src/leader_keeper.rs index 3c046f6d7dd..0e5e3ee882b 100644 --- a/components/snap_recovery/src/leader_keeper.rs +++ b/components/snap_recovery/src/leader_keeper.rs @@ -12,7 +12,7 @@ use futures::compat::Future01CompatExt; use raftstore::{ errors::{Error, Result}, store::{ - msg::CampaignType, Callback, CasualMessage, CasualRouter, SignificantMsg, SignificantRouter, + Callback, CasualMessage, CasualRouter, SignificantMsg, SignificantRouter, msg::CampaignType, }, }; use tikv_util::{future::paired_future_callback, timer::GLOBAL_TIMER_HANDLE}; diff --git a/components/snap_recovery/src/region_meta_collector.rs b/components/snap_recovery/src/region_meta_collector.rs index 3a88931fae4..929689ce6fb 100644 --- a/components/snap_recovery/src/region_meta_collector.rs +++ b/components/snap_recovery/src/region_meta_collector.rs @@ -2,7 +2,7 @@ use std::{cell::RefCell, error::Error as StdError, result, thread::JoinHandle}; -use engine_traits::{Engines, KvEngine, RaftEngine, CF_RAFT}; +use engine_traits::{CF_RAFT, Engines, KvEngine, RaftEngine}; use futures::channel::mpsc::UnboundedSender; use kvproto::{ raft_serverpb::{PeerState, RaftApplyState, RaftLocalState, RegionLocalState}, diff --git a/components/snap_recovery/src/services.rs b/components/snap_recovery/src/services.rs index 931870a8ae4..6dec1f454bd 100644 --- a/components/snap_recovery/src/services.rs +++ b/components/snap_recovery/src/services.rs @@ -6,24 +6,24 @@ use std::{ future::Future, result, sync::{ - atomic::{AtomicBool, Ordering}, Arc, Mutex, + atomic::{AtomicBool, Ordering}, }, thread::Builder, time::Instant, }; use engine_rocks::{ + RocksEngine, raw::{CompactOptions, DBBottommostLevelCompaction}, util::get_cf_handle, - RocksEngine, }; use engine_traits::{CfNamesExt, CfOptionsExt, Engines, KvEngine, RaftEngine}; use futures::{ + FutureExt, SinkExt, StreamExt, channel::mpsc, executor::{ThreadPool, ThreadPoolBuilder}, stream::{AbortHandle, Aborted}, - FutureExt, SinkExt, StreamExt, }; use grpcio::{ ClientStreamingSink, RequestStream, RpcContext, RpcStatus, RpcStatusCode, ServerStreamingSink, @@ -33,11 +33,11 @@ use kvproto::{raft_serverpb::StoreIdent, recoverdatapb::*}; use raftstore::{ router::RaftStoreRouter, store::{ + SnapshotBrWaitApplySyncer, fsm::RaftRouter, msg::{PeerMsg, SignificantMsg}, snapshot_backup::{SnapshotBrWaitApplyRequest, SyncReport}, transport::SignificantRouter, - SnapshotBrWaitApplySyncer, }, }; use thiserror::Error; diff --git a/components/sst_importer/src/caching/cache_map.rs b/components/sst_importer/src/caching/cache_map.rs index e88e5c3545d..f275f84cbdb 100644 --- a/components/sst_importer/src/caching/cache_map.rs +++ b/components/sst_importer/src/caching/cache_map.rs @@ -2,13 +2,13 @@ use std::{ sync::{ - atomic::{AtomicUsize, Ordering}, Arc, + atomic::{AtomicUsize, Ordering}, }, time::Duration, }; -use dashmap::{mapref::entry::Entry, DashMap}; +use dashmap::{DashMap, mapref::entry::Entry}; use futures::Future; use crate::metrics::EXT_STORAGE_CACHE_COUNT; diff --git a/components/sst_importer/src/config.rs b/components/sst_importer/src/config.rs index 741c90c8e65..61e01aa03af 100644 --- a/components/sst_importer/src/config.rs +++ b/components/sst_importer/src/config.rs @@ -6,7 +6,7 @@ use std::{ }; use online_config::{self, OnlineConfig}; -use tikv_util::{config::ReadableDuration, resizable_threadpool::ResizableRuntime, HandyRwLock}; +use tikv_util::{HandyRwLock, config::ReadableDuration, resizable_threadpool::ResizableRuntime}; #[derive(Clone, Serialize, Deserialize, PartialEq, Debug, OnlineConfig)] #[serde(default)] diff --git a/components/sst_importer/src/import_file.rs b/components/sst_importer/src/import_file.rs index a7770ddbad5..71aace3ab88 100644 --- a/components/sst_importer/src/import_file.rs +++ b/components/sst_importer/src/import_file.rs @@ -12,14 +12,14 @@ use std::{ use api_version::api_v2::TIDB_RANGES_COMPLEMENT; use encryption::{DataKeyManager, EncrypterWriter}; -use engine_traits::{iter_option, Iterator, KvEngine, RefIterable, SstMetaInfo, SstReader}; -use file_system::{sync_dir, File, OpenOptions}; +use engine_traits::{Iterator, KvEngine, RefIterable, SstMetaInfo, SstReader, iter_option}; +use file_system::{File, OpenOptions, sync_dir}; use keys::data_key; use kvproto::{import_sstpb::*, kvrpcpb::ApiVersion}; use tikv_util::time::Instant; use uuid::{Builder as UuidBuilder, Uuid}; -use crate::{metrics::*, Error, Result}; +use crate::{Error, Result, metrics::*}; // `SyncableWrite` extends io::Write with sync trait SyncableWrite: io::Write + Send { diff --git a/components/sst_importer/src/import_mode.rs b/components/sst_importer/src/import_mode.rs index 0070462892d..756e86f565e 100644 --- a/components/sst_importer/src/import_mode.rs +++ b/components/sst_importer/src/import_mode.rs @@ -2,8 +2,8 @@ use std::{ sync::{ - atomic::{AtomicBool, Ordering}, Arc, Mutex, + atomic::{AtomicBool, Ordering}, }, time::{Duration, Instant}, }; @@ -242,7 +242,7 @@ impl ImportModeCfOptions { mod tests { use std::thread; - use engine_traits::{KvEngine, CF_DEFAULT}; + use engine_traits::{CF_DEFAULT, KvEngine}; use tempfile::Builder; use test_sst_importer::{new_test_engine, new_test_engine_with_options}; use tikv_util::{config::ReadableDuration, resizable_threadpool::ResizableRuntime}; diff --git a/components/sst_importer/src/lib.rs b/components/sst_importer/src/lib.rs index 3a7a54a650c..f69093de36a 100644 --- a/components/sst_importer/src/lib.rs +++ b/components/sst_importer/src/lib.rs @@ -24,8 +24,8 @@ pub mod sst_importer; pub use self::{ config::{Config, ConfigManager}, - errors::{error_inc, Error, Result}, - import_file::{sst_meta_to_path, API_VERSION_2}, + errors::{Error, Result, error_inc}, + import_file::{API_VERSION_2, sst_meta_to_path}, import_mode2::range_overlaps, sst_importer::SstImporter, sst_writer::{RawSstWriter, TxnSstWriter}, diff --git a/components/sst_importer/src/metrics.rs b/components/sst_importer/src/metrics.rs index f8785f721b2..2519a91c978 100644 --- a/components/sst_importer/src/metrics.rs +++ b/components/sst_importer/src/metrics.rs @@ -1,5 +1,6 @@ // Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0. +use lazy_static::lazy_static; use prometheus::*; lazy_static! { diff --git a/components/sst_importer/src/sst_importer.rs b/components/sst_importer/src/sst_importer.rs index 90e8207865e..f01e9ca76b4 100644 --- a/components/sst_importer/src/sst_importer.rs +++ b/components/sst_importer/src/sst_importer.rs @@ -12,17 +12,17 @@ use std::{ }; use collections::HashSet; -use dashmap::{mapref::entry::Entry, DashMap}; +use dashmap::{DashMap, mapref::entry::Entry}; use encryption::{DataKeyManager, FileEncryptionInfo, MultiMasterKeyBackend}; use encryption_export::create_async_backend; use engine_traits::{ - name_to_cf, util::check_key_in_range, CfName, IterOptions, Iterator, KvEngine, RefIterable, - SstCompressionType, SstExt, SstMetaInfo, SstReader, SstWriter, SstWriterBuilder, CF_DEFAULT, - CF_WRITE, + CF_DEFAULT, CF_WRITE, CfName, IterOptions, Iterator, KvEngine, RefIterable, SstCompressionType, + SstExt, SstMetaInfo, SstReader, SstWriter, SstWriterBuilder, name_to_cf, + util::check_key_in_range, }; use external_storage::{ - compression_reader_dispatcher, encrypt_wrap_reader, wrap_with_checksum_reader_if_needed, - ExternalStorage, RestoreConfig, + ExternalStorage, RestoreConfig, compression_reader_dispatcher, encrypt_wrap_reader, + wrap_with_checksum_reader_if_needed, }; use file_system::{IoType, OpenOptions}; use kvproto::{ @@ -33,6 +33,7 @@ use kvproto::{ metapb::Region, }; use tikv_util::{ + Either, HandyRwLock, codec::{ bytes::{decode_bytes_in_place, encode_bytes}, stream_event::{EventEncoder, EventIterator, Iterator as EIterator}, @@ -40,21 +41,21 @@ use tikv_util::{ future::RescheduleChecker, memory::{MemoryQuota, OwnedAllocated}, resizable_threadpool::DeamonRuntimeHandle, - sys::{thread::ThreadBuildWrapper, SysQuota}, + sys::{SysQuota, thread::ThreadBuildWrapper}, time::{Instant, Limiter}, - Either, HandyRwLock, }; use tokio::{runtime::Runtime, sync::OnceCell}; use txn_types::{Key, TimeStamp, WriteRef}; use crate::{ + Config, ConfigManager as ImportConfigManager, Error, Result, caching::cache_map::{CacheMap, ShareOwned}, import_file::{ImportDir, ImportFile}, import_mode::{ImportModeSwitcher, RocksDbMetricsFn}, import_mode2::{HashRange, ImportModeSwitcherV2}, metrics::*, sst_writer::{RawSstWriter, TxnSstWriter}, - util, Config, ConfigManager as ImportConfigManager, Error, Result, + util, }; pub struct LoadedFile { @@ -110,10 +111,10 @@ pub enum CacheKvFile { /// returns an error on an invalid internal state. /// pass the error back to the client side for further debugging. fn error(message: impl std::fmt::Display) -> Error { - Error::Io(io::Error::new( - ErrorKind::Other, - format!("internal error in TiKV: {}", message), - )) + Error::Io(io::Error::other(format!( + "internal error in TiKV: {}", + message + ))) } impl CacheKvFile { @@ -1730,18 +1731,17 @@ mod tests { io::{self, Cursor}, ops::Sub, sync::{ - atomic::{AtomicUsize, Ordering}, Mutex, + atomic::{AtomicUsize, Ordering}, }, - usize, }; use async_compression::tokio::write::ZstdEncoder; use encryption::{EncrypterWriter, Iv}; use engine_rocks::get_env; use engine_traits::{ - collect, Error as TraitError, ExternalSstFileInfo, Iterable, Iterator, RefIterable, - SstCompressionType::Zstd, SstReader, SstWriter, CF_DEFAULT, DATA_CFS, + CF_DEFAULT, DATA_CFS, Error as TraitError, ExternalSstFileInfo, Iterable, Iterator, + RefIterable, SstCompressionType::Zstd, SstReader, SstWriter, collect, }; use external_storage::read_external_storage_info_buff; use file_system::Sha256Reader; @@ -2306,7 +2306,7 @@ mod tests { #[test] fn test_read_external_storage_into_file_timed_out() { - use futures_util::stream::{pending, TryStreamExt}; + use futures_util::stream::{TryStreamExt, pending}; let mut input = pending::>().into_async_read(); let mut output = Vec::new(); @@ -2393,7 +2393,7 @@ mod tests { #[test] fn test_read_external_storage_info_buff_timed_out() { - use futures_util::stream::{pending, TryStreamExt}; + use futures_util::stream::{TryStreamExt, pending}; let mut input = pending::>().into_async_read(); let err = block_on_external_io(read_external_storage_info_buff( diff --git a/components/sst_importer/src/sst_writer.rs b/components/sst_importer/src/sst_writer.rs index 75d0d312a9e..a7ba5b0262c 100644 --- a/components/sst_importer/src/sst_writer.rs +++ b/components/sst_importer/src/sst_writer.rs @@ -2,14 +2,14 @@ use std::sync::Arc; -use api_version::{dispatch_api_version, match_template_api_version, KeyMode, KvFormat, RawValue}; +use api_version::{KeyMode, KvFormat, RawValue, dispatch_api_version, match_template_api_version}; use encryption::DataKeyManager; -use engine_traits::{raw_ttl::ttl_to_expire_ts, KvEngine, SstWriter}; +use engine_traits::{KvEngine, SstWriter, raw_ttl::ttl_to_expire_ts}; use kvproto::{import_sstpb::*, kvrpcpb::ApiVersion}; use tikv_util::time::Instant; -use txn_types::{is_short_value, Key, TimeStamp, Write as KvWrite, WriteType}; +use txn_types::{Key, TimeStamp, Write as KvWrite, WriteType, is_short_value}; -use crate::{import_file::ImportPath, metrics::*, Error, Result}; +use crate::{Error, Result, import_file::ImportPath, metrics::*}; #[derive(Debug)] pub enum SstWriterType { diff --git a/components/sst_importer/src/util.rs b/components/sst_importer/src/util.rs index c20ef14ea28..fef7da3203b 100644 --- a/components/sst_importer/src/util.rs +++ b/components/sst_importer/src/util.rs @@ -123,12 +123,12 @@ mod tests { use encryption::DataKeyManager; use engine_rocks::{ - util::new_engine_opt, RocksCfOptions, RocksDbOptions, RocksEngine, RocksSstWriterBuilder, - RocksTitanDbOptions, + RocksCfOptions, RocksDbOptions, RocksEngine, RocksSstWriterBuilder, RocksTitanDbOptions, + util::new_engine_opt, }; use engine_traits::{ - CfName, CfOptions, DbOptions, ImportExt, Peekable, SstWriter, SstWriterBuilder, - TitanCfOptions, CF_DEFAULT, + CF_DEFAULT, CfName, CfOptions, DbOptions, ImportExt, Peekable, SstWriter, SstWriterBuilder, + TitanCfOptions, }; use tempfile::Builder; use test_util::encryption::new_test_key_manager; diff --git a/components/test_backup/src/disk_snap.rs b/components/test_backup/src/disk_snap.rs index 0f06efa53a8..8b5574b8b52 100644 --- a/components/test_backup/src/disk_snap.rs +++ b/components/test_backup/src/disk_snap.rs @@ -24,12 +24,12 @@ use kvproto::{ metapb::Region, raft_cmdpb::RaftCmdResponse, }; -use raftstore::store::{snapshot_backup::PrepareDiskSnapObserver, Callback, WriteResponse}; +use raftstore::store::{Callback, WriteResponse, snapshot_backup::PrepareDiskSnapObserver}; use test_raftstore::*; use tikv_util::{ + HandyRwLock, future::{block_on_timeout, paired_future_callback}, worker::dummy_scheduler, - HandyRwLock, }; pub struct Node { diff --git a/components/test_backup/src/lib.rs b/components/test_backup/src/lib.rs index cf68db05833..58b63f3b976 100644 --- a/components/test_backup/src/lib.rs +++ b/components/test_backup/src/lib.rs @@ -8,12 +8,12 @@ use std::{ time::Duration, }; -use api_version::{dispatch_api_version, keyspace::KvPair, ApiV1, KvFormat, RawValue}; +use api_version::{ApiV1, KvFormat, RawValue, dispatch_api_version, keyspace::KvPair}; use backup::Task; use collections::HashMap; // NOTE: Perhaps we'd better use test engine here. But it seems for now we cannot initialize a // mock cluster with `PanicEngine` and in our CI environment clippy will complain that. -use engine_traits::{CfName, IterOptions, CF_DEFAULT, CF_WRITE, DATA_KEY_PREFIX_LEN}; +use engine_traits::{CF_DEFAULT, CF_WRITE, CfName, DATA_KEY_PREFIX_LEN, IterOptions}; use external_storage::make_local_backend; use futures::{channel::mpsc as future_mpsc, executor::block_on}; use grpcio::{ChannelBuilder, Environment}; @@ -21,22 +21,22 @@ use kvproto::{brpb::*, kvrpcpb::*, tikvpb::TikvClient}; use rand::Rng; use test_raftstore::*; use tidb_query_common::storage::{ - scanner::{RangesScanner, RangesScannerOptions}, IntervalRange, Range, + scanner::{RangesScanner, RangesScannerOptions}, }; use tikv::{ config::BackupConfig, coprocessor::{checksum_crc64_xor, dag::TikvStorage}, storage::{ - kv::{Engine, LocalTablets, SnapContext}, SnapshotStore, + kv::{Engine, LocalTablets, SnapContext}, }, }; use tikv_util::{ + HandyRwLock, config::ReadableSize, time::Instant, worker::{LazyWorker, Worker}, - HandyRwLock, }; use txn_types::TimeStamp; diff --git a/components/test_coprocessor/src/column.rs b/components/test_coprocessor/src/column.rs index 0fb40aa871f..9162b83f08e 100644 --- a/components/test_coprocessor/src/column.rs +++ b/components/test_coprocessor/src/column.rs @@ -1,7 +1,7 @@ // Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0. use tidb_query_datatype::{ - codec::{datum, Datum}, + codec::{Datum, datum}, expr::EvalContext, }; use tipb::{ColumnInfo, FieldType}; diff --git a/components/test_coprocessor/src/dag.rs b/components/test_coprocessor/src/dag.rs index 07f7155b2f9..f3ddcf23705 100644 --- a/components/test_coprocessor/src/dag.rs +++ b/components/test_coprocessor/src/dag.rs @@ -5,7 +5,7 @@ use kvproto::{ kvrpcpb::Context, }; use protobuf::Message; -use tidb_query_datatype::codec::{datum, Datum}; +use tidb_query_datatype::codec::{Datum, datum}; use tikv::coprocessor::REQ_TYPE_DAG; use tikv_util::codec::number::NumberEncoder; use tipb::{ diff --git a/components/test_coprocessor/src/fixture.rs b/components/test_coprocessor/src/fixture.rs index 863e24b980f..acd39b8e5d0 100644 --- a/components/test_coprocessor/src/fixture.rs +++ b/components/test_coprocessor/src/fixture.rs @@ -6,15 +6,15 @@ use concurrency_manager::ConcurrencyManager; use kvproto::kvrpcpb::Context; use pd_client::PdClient; use resource_metering::ResourceTagFactory; -use tidb_query_datatype::codec::{row::v2::CODEC_VERSION, Datum}; +use tidb_query_datatype::codec::{Datum, row::v2::CODEC_VERSION}; use tikv::{ config::CoprReadPoolConfig, - coprocessor::{readpool_impl, Endpoint}, + coprocessor::{Endpoint, readpool_impl}, read_pool::ReadPool, server::Config, storage::{ - kv::RocksEngine, lock_manager::MockLockManager, Engine, TestEngineBuilder, - TestStorageBuilderApiV1, + Engine, TestEngineBuilder, TestStorageBuilderApiV1, kv::RocksEngine, + lock_manager::MockLockManager, }, }; use tikv_util::{quota_limiter::QuotaLimiter, thread_group::GroupProperties}; diff --git a/components/test_coprocessor/src/store.rs b/components/test_coprocessor/src/store.rs index 72d4c3ce157..de397bd5cc4 100644 --- a/components/test_coprocessor/src/store.rs +++ b/components/test_coprocessor/src/store.rs @@ -8,20 +8,21 @@ use pd_client::PdClient; use test_storage::SyncTestStorageApiV1; use tidb_query_datatype::{ codec::{ + Datum, data_type::ScalarValue, datum, row::v2::encoder_for_test::{Column as ColumnV2, RowEncoder}, - table, Datum, + table, }, expr::EvalContext, }; use tikv::{ server::gc_worker::GcConfig, storage::{ + SnapshotStore, StorageApiV1, TestStorageBuilderApiV1, kv::{Engine, RocksEngine}, lock_manager::MockLockManager, txn::FixtureStore, - SnapshotStore, StorageApiV1, TestStorageBuilderApiV1, }, }; use tikv_util::future::block_on_timeout; diff --git a/components/test_pd/src/lib.rs b/components/test_pd/src/lib.rs index bd768e58318..0313a91c1bb 100644 --- a/components/test_pd/src/lib.rs +++ b/components/test_pd/src/lib.rs @@ -1,6 +1,4 @@ // Copyright 2017 TiKV Project Authors. Licensed under Apache-2.0. -#![feature(slice_group_by)] - #[macro_use] extern crate tikv_util; #[macro_use] diff --git a/components/test_pd/src/mocker/etcd.rs b/components/test_pd/src/mocker/etcd.rs index d0fe3f43e68..651e3e2fc3c 100644 --- a/components/test_pd/src/mocker/etcd.rs +++ b/components/test_pd/src/mocker/etcd.rs @@ -45,7 +45,7 @@ impl Etcd { )) .collect::>() .as_slice() - .group_by(|item1, item2| item1.0.0 == item2.0.0) + .chunk_by(|item1, item2| item1.0.0 == item2.0.0) .filter_map(|group| { let (k, v) = group.last()?; match v { diff --git a/components/test_pd/src/mocker/meta_storage.rs b/components/test_pd/src/mocker/meta_storage.rs index 360e496f44e..9657a110c34 100644 --- a/components/test_pd/src/mocker/meta_storage.rs +++ b/components/test_pd/src/mocker/meta_storage.rs @@ -2,7 +2,7 @@ use std::sync::{Arc, Mutex}; -use futures::{executor::block_on, SinkExt, StreamExt}; +use futures::{SinkExt, StreamExt, executor::block_on}; use grpcio::{RpcStatus, RpcStatusCode}; use kvproto::meta_storagepb as mpb; @@ -123,16 +123,6 @@ impl PdMocker for MetaStorage { let mut resp = mpb::WatchResponse::default(); resp.set_events(vec![event].into()); sink.send((resp, Default::default())).await.unwrap(); - - #[cfg(feature = "failpoints")] - { - use futures::executor::block_on; - let cli_clone = cli.clone(); - fail_point!("watch_meta_storage_return", |_| { - block_on(async move { cli_clone.lock().await.clear_subs() }); - watcher.close(); - }); - } } }); true diff --git a/components/test_pd/src/mocker/service.rs b/components/test_pd/src/mocker/service.rs index 92a5107c873..05ae79983dd 100644 --- a/components/test_pd/src/mocker/service.rs +++ b/components/test_pd/src/mocker/service.rs @@ -1,8 +1,8 @@ // Copyright 2017 TiKV Project Authors. Licensed under Apache-2.0. use std::sync::{ - atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, Mutex, + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, }; use collections::HashMap; diff --git a/components/test_pd/src/server.rs b/components/test_pd/src/server.rs index 8c42ea642f5..577a6990bb3 100644 --- a/components/test_pd/src/server.rs +++ b/components/test_pd/src/server.rs @@ -2,21 +2,21 @@ use std::{ sync::{ - atomic::{AtomicI64, Ordering}, Arc, + atomic::{AtomicI64, Ordering}, }, thread, time::Duration, }; use fail::fail_point; -use futures::{future, SinkExt, TryFutureExt, TryStreamExt}; +use futures::{SinkExt, TryFutureExt, TryStreamExt, future}; use grpcio::{ ClientStreamingSink, DuplexSink, EnvBuilder, RequestStream, RpcContext, RpcStatus, RpcStatusCode, Server as GrpcServer, ServerBuilder, UnarySink, WriteFlags, }; use kvproto::{ - meta_storagepb_grpc::{create_meta_storage, MetaStorage}, + meta_storagepb_grpc::{MetaStorage, create_meta_storage}, pdpb::*, resource_manager, resource_manager_grpc::create_resource_manager, diff --git a/components/test_pd_client/src/pd.rs b/components/test_pd_client/src/pd.rs index e8e6ef0b715..ca2890725c6 100644 --- a/components/test_pd_client/src/pd.rs +++ b/components/test_pd_client/src/pd.rs @@ -7,8 +7,8 @@ use std::{ Bound::{Excluded, Unbounded}, }, sync::{ - atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, Arc, RwLock, + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, }, time::Duration, }; @@ -19,7 +19,7 @@ use futures::{ channel::mpsc::{self, UnboundedReceiver, UnboundedSender}, compat::Future01CompatExt, executor::block_on, - future::{err, ok, ready, BoxFuture, FutureExt}, + future::{BoxFuture, FutureExt, err, ok, ready}, stream, stream::StreamExt, }; @@ -40,13 +40,13 @@ use pd_client::{ }; use raft::eraftpb::ConfChangeType; use tikv_util::{ - store::{check_key_in_region, find_peer, find_peer_by_id, is_learner, new_peer, QueryStats}, + Either, HandyRwLock, + store::{QueryStats, check_key_in_region, find_peer, find_peer_by_id, is_learner, new_peer}, time::{Instant, UnixSecs}, timer::GLOBAL_TIMER_HANDLE, - Either, HandyRwLock, }; use tokio_timer::timer::Handle; -use txn_types::{TimeStamp, TSO_PHYSICAL_SHIFT_BITS}; +use txn_types::{TSO_PHYSICAL_SHIFT_BITS, TimeStamp}; use super::*; @@ -529,7 +529,7 @@ impl PdCluster { if self .stores .get(&store_id) - .map_or(true, |s| s.store.get_id() != 0) + .is_none_or(|s| s.store.get_id() != 0) { self.stores.insert( store_id, @@ -1079,10 +1079,10 @@ impl TestPdClient { }; let add = add_peers .iter() - .all(|peer| find_peer(®ion, peer.get_store_id()).map_or(false, |p| p == peer)); + .all(|peer| find_peer(®ion, peer.get_store_id()).is_some_and(|p| p == peer)); let remove = remove_peers .iter() - .all(|peer| find_peer(®ion, peer.get_store_id()).map_or(true, |p| p != peer)); + .all(|peer| find_peer(®ion, peer.get_store_id()) != Some(peer)); if add && remove { return; } @@ -1395,7 +1395,7 @@ impl TestPdClient { .rl() .leaders .get(®ion_id) - .map_or(false, |p| *p == peer) + .is_some_and(|p| *p == peer) } // check whether region is split by split_key or not. diff --git a/components/test_raftstore-v2/src/cluster.rs b/components/test_raftstore-v2/src/cluster.rs index cf61dbfdcdd..6e259f2c041 100644 --- a/components/test_raftstore-v2/src/cluster.rs +++ b/components/test_raftstore-v2/src/cluster.rs @@ -13,12 +13,12 @@ use encryption_export::DataKeyManager; use engine_rocks::{RocksSnapshot, RocksStatistics}; use engine_test::raft::RaftTestEngine; use engine_traits::{ - KvEngine, Peekable, RaftEngine, RaftEngineReadOnly, RaftLogBatch, ReadOptions, SyncMutable, - TabletRegistry, CF_DEFAULT, + CF_DEFAULT, KvEngine, Peekable, RaftEngine, RaftEngineReadOnly, RaftLogBatch, ReadOptions, + SyncMutable, TabletRegistry, }; use file_system::IoRateLimiter; -use futures::{executor::block_on, future::BoxFuture, Future}; -use keys::{data_key, validate_data_key, DATA_PREFIX_KEY}; +use futures::{Future, executor::block_on, future::BoxFuture}; +use keys::{DATA_PREFIX_KEY, data_key, validate_data_key}; use kvproto::{ errorpb::Error as PbError, kvrpcpb::ApiVersion, @@ -35,37 +35,37 @@ use kvproto::{ }; use pd_client::PdClient; use raftstore::{ + Error, Result, store::{ - cmd_resp, initial_region, region_meta::RegionMeta, util::check_key_in_region, Bucket, - BucketRange, Callback, RaftCmdExtraOpts, RegionSnapshot, TabletSnapManager, WriteResponse, - INIT_EPOCH_CONF_VER, INIT_EPOCH_VER, + Bucket, BucketRange, Callback, INIT_EPOCH_CONF_VER, INIT_EPOCH_VER, RaftCmdExtraOpts, + RegionSnapshot, TabletSnapManager, WriteResponse, cmd_resp, initial_region, + region_meta::RegionMeta, util::check_key_in_region, }, - Error, Result, }; use raftstore_v2::{ + SimpleWriteEncoder, StoreMeta, StoreRouter, router::{DebugInfoChannel, PeerMsg, QueryResult, StoreMsg, StoreTick}, - write_initial_states, SimpleWriteEncoder, StoreMeta, StoreRouter, + write_initial_states, }; use resource_control::ResourceGroupManager; use tempfile::TempDir; use test_pd_client::TestPdClient; use test_raftstore::{ - check_raft_cmd_request, is_error_response, new_admin_request, new_delete_cmd, - new_delete_range_cmd, new_get_cf_cmd, new_peer, new_prepare_merge, new_put_cf_cmd, new_put_cmd, - new_region_detail_cmd, new_region_leader_cmd, new_request, new_status_request, new_store, - new_tikv_config_with_api_ver, new_transfer_leader_cmd, sleep_ms, Config, Filter, FilterFactory, - PartitionFilterFactory, RawEngine, + Config, Filter, FilterFactory, PartitionFilterFactory, RawEngine, check_raft_cmd_request, + is_error_response, new_admin_request, new_delete_cmd, new_delete_range_cmd, new_get_cf_cmd, + new_peer, new_prepare_merge, new_put_cf_cmd, new_put_cmd, new_region_detail_cmd, + new_region_leader_cmd, new_request, new_status_request, new_store, + new_tikv_config_with_api_ver, new_transfer_leader_cmd, sleep_ms, }; use tikv::{config::TikvConfig, server::Result as ServerResult, storage::config::EngineType}; use tikv_util::{ - box_err, box_try, debug, error, + HandyRwLock, box_err, box_try, debug, error, future::block_on_timeout, safe_panic, thread_group::GroupProperties, time::{Instant, ThreadReadId}, warn, worker::LazyWorker, - HandyRwLock, }; use txn_types::WriteBatchFlags; diff --git a/components/test_raftstore-v2/src/node.rs b/components/test_raftstore-v2/src/node.rs index 2bb96622066..24a13cd6451 100644 --- a/components/test_raftstore-v2/src/node.rs +++ b/components/test_raftstore-v2/src/node.rs @@ -17,19 +17,19 @@ use kvproto::{ raft_cmdpb::{RaftCmdRequest, RaftCmdResponse}, raft_serverpb::RaftMessage, }; -use raft::{prelude::MessageType, SnapshotStatus}; +use raft::{SnapshotStatus, prelude::MessageType}; use raftstore::{ + Result, coprocessor::CoprocessorHost, errors::Error as RaftError, store::{ - config::RaftstoreConfigManager, AutoSplitController, GlobalReplicationState, - RegionSnapshot, SplitConfigManager, TabletSnapKey, TabletSnapManager, Transport, + AutoSplitController, GlobalReplicationState, RegionSnapshot, SplitConfigManager, + TabletSnapKey, TabletSnapManager, Transport, config::RaftstoreConfigManager, }, - Result, }; use raftstore_v2::{ - router::{PeerMsg, RaftRouter}, StateStorage, StoreMeta, StoreRouter, + router::{PeerMsg, RaftRouter}, }; use resource_control::ResourceGroupManager; use resource_metering::CollectorRegHandle; @@ -41,8 +41,8 @@ use tikv::{ config::{ConfigController, Module}, import::SstImporter, server::{ - raftkv::ReplicaReadLockChecker, tablet_snap::copy_tablet_snapshot, NodeV2, - Result as ServerResult, + NodeV2, Result as ServerResult, raftkv::ReplicaReadLockChecker, + tablet_snap::copy_tablet_snapshot, }, }; use tikv_util::{ diff --git a/components/test_raftstore-v2/src/server.rs b/components/test_raftstore-v2/src/server.rs index 8be0cc57dc4..952fbba317f 100644 --- a/components/test_raftstore-v2/src/server.rs +++ b/components/test_raftstore-v2/src/server.rs @@ -7,7 +7,7 @@ use std::{ time::Duration, }; -use api_version::{dispatch_api_version, KvFormat}; +use api_version::{KvFormat, dispatch_api_version}; use causal_ts::CausalTsProviderImpl; use collections::{HashMap, HashSet}; use concurrency_manager::ConcurrencyManager; @@ -15,13 +15,13 @@ use encryption_export::DataKeyManager; use engine_rocks::RocksEngine; use engine_test::raft::RaftTestEngine; use engine_traits::{KvEngine, RaftEngine, TabletRegistry}; -use futures::{executor::block_on, future::BoxFuture, Future}; +use futures::{Future, executor::block_on, future::BoxFuture}; use grpcio::{ChannelBuilder, EnvBuilder, Environment, Error as GrpcError, Service}; use grpcio_health::HealthService; use health_controller::HealthController; use kvproto::{ deadlock_grpc::create_deadlock, - debugpb_grpc::{create_debug, DebugClient}, + debugpb_grpc::{DebugClient, create_debug}, diagnosticspb_grpc::create_diagnostics, import_sstpb_grpc::create_import_sst, kvrpcpb::{ApiVersion, Context}, @@ -32,15 +32,15 @@ use kvproto::{ }; use pd_client::PdClient; use raftstore::{ + RegionInfoAccessor, coprocessor::CoprocessorHost, errors::Error as RaftError, store::{ - region_meta, AutoSplitController, CheckLeaderRunner, FlowStatsReporter, ReadStats, - RegionSnapshot, TabletSnapManager, WriteStats, + AutoSplitController, CheckLeaderRunner, FlowStatsReporter, ReadStats, RegionSnapshot, + TabletSnapManager, WriteStats, region_meta, }, - RegionInfoAccessor, }; -use raftstore_v2::{router::RaftRouter, StateStorage, StoreMeta, StoreRouter}; +use raftstore_v2::{StateStorage, StoreMeta, StoreRouter, router::RaftRouter}; use resource_control::ResourceGroupManager; use resource_metering::{CollectorRegHandle, ResourceTagFactory}; use security::SecurityManager; @@ -48,13 +48,15 @@ use service::service_manager::GrpcServiceManager; use slog_global::debug; use tempfile::TempDir; use test_pd_client::TestPdClient; -use test_raftstore::{filter_send, AddressMap, Config, Filter}; +use test_raftstore::{AddressMap, Config, Filter, filter_send}; use tikv::{ config::ConfigController, coprocessor, coprocessor_v2, import::{ImportSstService, SstImporter}, read_pool::ReadPool, server::{ + ConnectionBuilder, Error, Extension, NodeV2, PdStoreAddrResolver, RaftClient, RaftKv2, + Result as ServerResult, Server, ServerTransport, debug2::DebuggerImplV2, gc_worker::GcWorker, load_statistics::ThreadLoadPool, @@ -62,27 +64,23 @@ use tikv::{ raftkv::ReplicaReadLockChecker, resolve, service::{DebugService, DefaultGrpcMessageFilter, DiagnosticsService}, - ConnectionBuilder, Error, Extension, NodeV2, PdStoreAddrResolver, RaftClient, RaftKv2, - Result as ServerResult, Server, ServerTransport, }, storage::{ - self, + self, Engine, Storage, kv::{FakeExtension, LocalTablets, RaftExtension, SnapContext}, txn::{ flow_controller::{EngineFlowController, FlowController}, txn_status_cache::TxnStatusCache, }, - Engine, Storage, }, }; use tikv_util::{ - box_err, + Either, HandyRwLock, box_err, config::VersionTrack, quota_limiter::QuotaLimiter, sys::thread::ThreadBuildWrapper, thd_name, worker::{Builder as WorkerBuilder, LazyWorker}, - Either, HandyRwLock, }; use tokio::runtime::{Builder as TokioBuilder, Handle}; use txn_types::TxnExtraScheduler; diff --git a/components/test_raftstore-v2/src/transport_simulate.rs b/components/test_raftstore-v2/src/transport_simulate.rs index 995662ac484..ca369a5706c 100644 --- a/components/test_raftstore-v2/src/transport_simulate.rs +++ b/components/test_raftstore-v2/src/transport_simulate.rs @@ -10,12 +10,12 @@ use kvproto::{ }; use raft::SnapshotStatus; use raftstore::{ + Result, Result as RaftStoreResult, router::handle_send_error, store::{RegionSnapshot, Transport}, - Result, Result as RaftStoreResult, }; use raftstore_v2::router::{PeerMsg, RaftRouter}; -use test_raftstore::{filter_send, Filter}; +use test_raftstore::{Filter, filter_send}; use tikv_util::HandyRwLock; #[derive(Clone)] diff --git a/components/test_raftstore-v2/src/util.rs b/components/test_raftstore-v2/src/util.rs index 5d87a3f5b02..5364cc4d1f8 100644 --- a/components/test_raftstore-v2/src/util.rs +++ b/components/test_raftstore-v2/src/util.rs @@ -7,10 +7,10 @@ use std::{ time::{Duration, Instant}, }; -use encryption_export::{data_key_manager_from_config, DataKeyManager}; +use encryption_export::{DataKeyManager, data_key_manager_from_config}; use engine_rocks::{RocksEngine, RocksStatistics}; use engine_test::raft::RaftTestEngine; -use engine_traits::{CfName, KvEngine, TabletRegistry, CF_DEFAULT}; +use engine_traits::{CF_DEFAULT, CfName, KvEngine, TabletRegistry}; use file_system::IoRateLimiter; use futures::future::BoxFuture; use grpcio::{ChannelBuilder, Environment}; @@ -21,26 +21,27 @@ use kvproto::{ raft_cmdpb::{CmdType, RaftCmdRequest, RaftCmdResponse}, tikvpb::TikvClient, }; -use raftstore::{store::ReadResponse, Result}; -use rand::{prelude::SliceRandom, RngCore}; +use raftstore::{Result, store::ReadResponse}; +use rand::{RngCore, prelude::SliceRandom}; use server::common::ConfiguredRaftEngine; use tempfile::TempDir; use test_pd_client::TestPdClient; -use test_raftstore::{new_get_cmd, new_put_cf_cmd, new_request, new_snap_cmd, sleep_ms, Config}; +use test_raftstore::{Config, new_get_cmd, new_put_cf_cmd, new_request, new_snap_cmd, sleep_ms}; use tikv::{ server::KvEngineFactoryBuilder, storage::{ + Engine, Snapshot, kv::{SnapContext, SnapshotExt}, - point_key_range, Engine, Snapshot, + point_key_range, }, }; use tikv_util::{ - config::ReadableDuration, escape, future::block_on_timeout, time::InstantExt, - worker::LazyWorker, HandyRwLock, + HandyRwLock, config::ReadableDuration, escape, future::block_on_timeout, time::InstantExt, + worker::LazyWorker, }; use txn_types::Key; -use crate::{bootstrap_store, cluster::Cluster, ServerCluster, Simulator}; +use crate::{ServerCluster, Simulator, bootstrap_store, cluster::Cluster}; pub fn create_test_engine( // TODO: pass it in for all cases. @@ -459,7 +460,7 @@ pub fn wait_down_peers, EK: KvEngine>( ) { let mut peers = cluster.get_down_peers(); for _ in 1..1000 { - if peers.len() == count as usize && peer.as_ref().map_or(true, |p| peers.contains_key(p)) { + if peers.len() == count as usize && peer.as_ref().is_none_or(|p| peers.contains_key(p)) { return; } std::thread::sleep(Duration::from_millis(10)); diff --git a/components/test_raftstore/src/cluster.rs b/components/test_raftstore/src/cluster.rs index 374533b427c..b462ee9138e 100644 --- a/components/test_raftstore/src/cluster.rs +++ b/components/test_raftstore/src/cluster.rs @@ -4,7 +4,7 @@ use std::{ collections::hash_map::Entry as MapEntry, error::Error as StdError, result, - sync::{mpsc, Arc, Mutex, RwLock}, + sync::{Arc, Mutex, RwLock, mpsc}, thread, time::Duration, }; @@ -15,11 +15,11 @@ use encryption_export::DataKeyManager; use engine_rocks::{RocksEngine, RocksSnapshot, RocksStatistics}; use engine_test::raft::RaftTestEngine; use engine_traits::{ - CompactExt, Engines, Iterable, ManualCompactionOptions, MiscExt, Mutable, Peekable, - RaftEngineReadOnly, SyncMutable, WriteBatch, WriteBatchExt, CF_DEFAULT, CF_RAFT, + CF_DEFAULT, CF_RAFT, CompactExt, Engines, Iterable, ManualCompactionOptions, MiscExt, Mutable, + Peekable, RaftEngineReadOnly, SyncMutable, WriteBatch, WriteBatchExt, }; use file_system::IoRateLimiter; -use futures::{self, channel::oneshot, executor::block_on, future::BoxFuture, StreamExt}; +use futures::{self, StreamExt, channel::oneshot, executor::block_on, future::BoxFuture}; use kvproto::{ errorpb::Error as PbError, kvrpcpb::{ApiVersion, Context, DiskFullOpt}, @@ -34,27 +34,26 @@ use kvproto::{ use pd_client::{BucketStat, PdClient}; use raft::eraftpb::ConfChangeType; use raftstore::{ + Error, Result, router::RaftStoreRouter, store::{ fsm::{ - create_raft_batch_system, - store::{StoreMeta, PENDING_MSG_CAP}, - ApplyRouter, RaftBatchSystem, RaftRouter, + ApplyRouter, RaftBatchSystem, RaftRouter, create_raft_batch_system, + store::{PENDING_MSG_CAP, StoreMeta}, }, transport::CasualRouter, *, }, - Error, Result, }; use resource_control::ResourceGroupManager; use tempfile::TempDir; use test_pd_client::TestPdClient; use tikv::{config::TikvConfig, server::Result as ServerResult}; use tikv_util::{ + HandyRwLock, thread_group::GroupProperties, time::{Instant, ThreadReadId}, worker::LazyWorker, - HandyRwLock, }; use txn_types::WriteBatchFlags; diff --git a/components/test_raftstore/src/node.rs b/components/test_raftstore/src/node.rs index 68fd05134f6..a7a4305cf66 100644 --- a/components/test_raftstore/src/node.rs +++ b/components/test_raftstore/src/node.rs @@ -2,7 +2,7 @@ use std::{ path::{Path, PathBuf}, - sync::{atomic::AtomicU64, Arc, Mutex, RwLock}, + sync::{Arc, Mutex, RwLock, atomic::AtomicU64}, }; use collections::{HashMap, HashSet}; @@ -19,17 +19,18 @@ use kvproto::{ raft_serverpb::{self, RaftMessage}, }; use protobuf::Message; -use raft::{eraftpb::MessageType, SnapshotStatus}; +use raft::{SnapshotStatus, eraftpb::MessageType}; use raftstore::{ - coprocessor::{config::SplitCheckConfigManager, CoprocessorHost}, + Result, + coprocessor::{CoprocessorHost, config::SplitCheckConfigManager}, errors::Error as RaftError, router::{LocalReadRouter, RaftStoreRouter, ReadContext, ServerRaftStoreRouter}, store::{ + SnapManagerBuilder, config::RaftstoreConfigManager, - fsm::{store::StoreMeta, ApplyRouter, RaftBatchSystem, RaftRouter}, - SnapManagerBuilder, *, + fsm::{ApplyRouter, RaftBatchSystem, RaftRouter, store::StoreMeta}, + *, }, - Result, }; use resource_control::ResourceGroupManager; use resource_metering::CollectorRegHandle; @@ -39,7 +40,7 @@ use test_pd_client::TestPdClient; use tikv::{ config::{ConfigController, Module}, import::SstImporter, - server::{raftkv::ReplicaReadLockChecker, MultiRaftServer, Result as ServerResult}, + server::{MultiRaftServer, Result as ServerResult, raftkv::ReplicaReadLockChecker}, }; use tikv_util::{ config::VersionTrack, diff --git a/components/test_raftstore/src/router.rs b/components/test_raftstore/src/router.rs index d6a135c9f9a..1bfb1752ba6 100644 --- a/components/test_raftstore/src/router.rs +++ b/components/test_raftstore/src/router.rs @@ -8,13 +8,13 @@ use engine_rocks::{RocksEngine, RocksSnapshot}; use kvproto::raft_serverpb::RaftMessage; use raftstore::{ errors::{Error as RaftStoreError, Result as RaftStoreResult}, - router::{handle_send_error, RaftStoreRouter}, + router::{RaftStoreRouter, handle_send_error}, store::{ - msg::{CasualMessage, PeerMsg, SignificantMsg}, CasualRouter, ProposalRouter, RaftCommand, SignificantRouter, StoreMsg, StoreRouter, + msg::{CasualMessage, PeerMsg, SignificantMsg}, }, }; -use tikv_util::mpsc::{loose_bounded, LooseBoundedSender, Receiver}; +use tikv_util::mpsc::{LooseBoundedSender, Receiver, loose_bounded}; #[derive(Clone)] #[allow(clippy::type_complexity)] diff --git a/components/test_raftstore/src/server.rs b/components/test_raftstore/src/server.rs index 3e8220271f4..c617f01c841 100644 --- a/components/test_raftstore/src/server.rs +++ b/components/test_raftstore/src/server.rs @@ -2,14 +2,13 @@ use std::{ path::Path, - sync::{atomic::AtomicU64, mpsc::Receiver, Arc, Mutex, RwLock}, + sync::{Arc, Mutex, RwLock, atomic::AtomicU64, mpsc::Receiver}, thread, time::Duration, - usize, }; use ::server::common::build_hybrid_engine; -use api_version::{dispatch_api_version, KvFormat}; +use api_version::{KvFormat, dispatch_api_version}; use causal_ts::CausalTsProviderImpl; use collections::{HashMap, HashSet}; use concurrency_manager::ConcurrencyManager; @@ -26,7 +25,7 @@ use hybrid_engine::observer::{ use in_memory_engine::{InMemoryEngineConfig, InMemoryEngineContext, RegionCacheMemoryEngine}; use kvproto::{ deadlock::create_deadlock, - debugpb::{create_debug, DebugClient}, + debugpb::{DebugClient, create_debug}, import_sstpb::create_import_sst, kvrpcpb::{ApiVersion, Context}, metapb, @@ -36,17 +35,17 @@ use kvproto::{ }; use pd_client::PdClient; use raftstore::{ + Result, coprocessor::{CoprocessorHost, RegionInfoAccessor}, errors::Error as RaftError, router::{CdcRaftRouter, LocalReadRouter, RaftStoreRouter, ReadContext, ServerRaftStoreRouter}, store::{ - fsm::{store::StoreMeta, ApplyRouter, RaftBatchSystem, RaftRouter}, - msg::RaftCmdExtraOpts, AutoSplitController, Callback, CheckLeaderRunner, DiskCheckRunner, LocalReader, RegionSnapshot, SnapManager, SnapManagerBuilder, SplitCheckRunner, SplitConfigManager, StoreMetaDelegate, + fsm::{ApplyRouter, RaftBatchSystem, RaftRouter, store::StoreMeta}, + msg::RaftCmdExtraOpts, }, - Result, }; use resource_control::ResourceGroupManager; use resource_metering::{CollectorRegHandle, ResourceTagFactory}; @@ -61,6 +60,8 @@ use tikv::{ import::{ImportSstService, SstImporter}, read_pool::ReadPool, server::{ + ConnectionBuilder, Error, MultiRaftServer, PdStoreAddrResolver, RaftClient, RaftKv, + Result as ServerResult, Server, ServerTransport, debug::DebuggerImpl, gc_worker::GcWorker, load_statistics::ThreadLoadPool, @@ -69,26 +70,23 @@ use tikv::{ resolve::{self, StoreAddrResolver}, service::{DebugService, DefaultGrpcMessageFilter}, tablet_snap::NoSnapshotCache, - ConnectionBuilder, Error, MultiRaftServer, PdStoreAddrResolver, RaftClient, RaftKv, - Result as ServerResult, Server, ServerTransport, }, storage::{ - self, + self, Engine, Storage, kv::{FakeExtension, LocalTablets, SnapContext}, txn::{ flow_controller::{EngineFlowController, FlowController}, txn_status_cache::TxnStatusCache, }, - Engine, Storage, }, }; use tikv_util::{ + HandyRwLock, config::VersionTrack, quota_limiter::QuotaLimiter, sys::thread::ThreadBuildWrapper, time::ThreadReadId, worker::{Builder as WorkerBuilder, LazyWorker, Scheduler}, - HandyRwLock, }; use tokio::runtime::Builder as TokioBuilder; use txn_types::TxnExtraScheduler; diff --git a/components/test_raftstore/src/transport_simulate.rs b/components/test_raftstore/src/transport_simulate.rs index 050f735aac3..d68d057d919 100644 --- a/components/test_raftstore/src/transport_simulate.rs +++ b/components/test_raftstore/src/transport_simulate.rs @@ -3,10 +3,9 @@ use std::{ marker::PhantomData, mem, - sync::{atomic::*, mpsc::Sender, Arc, Mutex, RwLock}, + sync::{Arc, Mutex, RwLock, atomic::*, mpsc::Sender}, thread, time, time::Duration, - usize, }; use collections::{HashMap, HashSet}; @@ -18,12 +17,12 @@ use kvproto::{ }; use raft::eraftpb::MessageType; use raftstore::{ + DiscardReason, Error, Result as RaftStoreResult, Result, router::{LocalReadRouter, RaftStoreRouter, ReadContext}, store::{ Callback, CasualMessage, CasualRouter, PeerMsg, ProposalRouter, RaftCommand, SignificantMsg, SignificantRouter, StoreMsg, StoreRouter, Transport, }, - DiscardReason, Error, Result as RaftStoreResult, Result, }; use tikv_util::{Either, HandyRwLock}; @@ -748,7 +747,7 @@ impl Filter for LeadingDuplicatedSnapshotFilter { let mut to_send = vec![]; for msg in msgs.drain(..) { if msg.get_message().get_msg_type() == MessageType::MsgSnapshot && !stale { - if last_msg.as_ref().map_or(false, |l| l != &msg) { + if last_msg.as_ref().is_some_and(|l| l != &msg) { to_send.push(last_msg.take().unwrap()); if self.together { to_send.push(msg); diff --git a/components/test_raftstore/src/util.rs b/components/test_raftstore/src/util.rs index 523608e938f..0e50df315d4 100644 --- a/components/test_raftstore/src/util.rs +++ b/components/test_raftstore/src/util.rs @@ -4,24 +4,24 @@ use std::{ fmt::Write, path::Path, str::FromStr, - sync::{mpsc, Arc, Mutex}, + sync::{Arc, Mutex, mpsc}, thread, time::Duration, }; use collections::HashMap; use encryption_export::{ - data_key_manager_from_config, DataKeyManager, FileConfig, MasterKeyConfig, + DataKeyManager, FileConfig, MasterKeyConfig, data_key_manager_from_config, }; -use engine_rocks::{config::BlobRunMode, RocksEngine, RocksSnapshot, RocksStatistics}; +use engine_rocks::{RocksEngine, RocksSnapshot, RocksStatistics, config::BlobRunMode}; use engine_test::raft::RaftTestEngine; use engine_traits::{ - CfName, CfNamesExt, Engines, Iterable, KvEngine, Peekable, RaftEngineDebug, RaftEngineReadOnly, - CF_DEFAULT, CF_RAFT, CF_WRITE, + CF_DEFAULT, CF_RAFT, CF_WRITE, CfName, CfNamesExt, Engines, Iterable, KvEngine, Peekable, + RaftEngineDebug, RaftEngineReadOnly, }; use fail::fail_point; use file_system::IoRateLimiter; -use futures::{executor::block_on, future::BoxFuture, StreamExt}; +use futures::{StreamExt, executor::block_on, future::BoxFuture}; use grpcio::{ChannelBuilder, Environment}; use hybrid_engine::HybridEngine; use in_memory_engine::RegionCacheMemoryEngine; @@ -38,14 +38,15 @@ use kvproto::{ }, tikvpb::TikvClient, }; +use lazy_static::lazy_static; use pd_client::PdClient; use protobuf::RepeatedField; use raft::eraftpb::ConfChangeType; use raftstore::{ - store::{fsm::RaftRouter, util::encode_start_ts_into_flag_data, *}, RaftRouterCompactedEventSender, Result, + store::{fsm::RaftRouter, util::encode_start_ts_into_flag_data, *}, }; -use rand::{seq::SliceRandom, RngCore}; +use rand::{RngCore, seq::SliceRandom}; use server::common::ConfiguredRaftEngine; use tempfile::TempDir; use test_pd_client::TestPdClient; @@ -54,19 +55,20 @@ use tikv::{ config::*, server::KvEngineFactoryBuilder, storage::{ + Engine, Snapshot, kv::{SnapContext, SnapshotExt}, - point_key_range, Engine, Snapshot, + point_key_range, }, }; pub use tikv_util::store::{find_peer, new_learner_peer, new_peer}; use tikv_util::{ + HandyRwLock, config::*, escape, future::block_on_timeout, mpsc::future, time::{Instant, ThreadReadId}, worker::LazyWorker, - HandyRwLock, }; use txn_types::Key; @@ -1867,7 +1869,7 @@ pub fn put_with_timeout( pub fn wait_down_peers(cluster: &Cluster, count: u64, peer: Option) { let mut peers = cluster.get_down_peers(); for _ in 1..1000 { - if peers.len() == count as usize && peer.as_ref().map_or(true, |p| peers.contains_key(p)) { + if peers.len() == count as usize && peer.as_ref().is_none_or(|p| peers.contains_key(p)) { return; } std::thread::sleep(Duration::from_millis(10)); diff --git a/components/test_raftstore_macro/src/lib.rs b/components/test_raftstore_macro/src/lib.rs index 39db5427bc2..982de63fce4 100644 --- a/components/test_raftstore_macro/src/lib.rs +++ b/components/test_raftstore_macro/src/lib.rs @@ -2,8 +2,8 @@ use proc_macro::TokenStream; use proc_macro2::{TokenStream as TokenStream2, TokenTree}; -use quote::{quote, ToTokens}; -use syn::{parse_macro_input, parse_quote, Ident, ItemFn, Path}; +use quote::{ToTokens, quote}; +use syn::{Ident, ItemFn, Path, parse_macro_input, parse_quote}; /// test_case generate test cases using cluster creation method provided. /// It also import the package related util module, which means we should locate diff --git a/components/test_sst_importer/src/lib.rs b/components/test_sst_importer/src/lib.rs index 721add2b58d..06c5c8f5664 100644 --- a/components/test_sst_importer/src/lib.rs +++ b/components/test_sst_importer/src/lib.rs @@ -3,9 +3,9 @@ use std::{collections::HashMap, fs, path::Path, sync::Arc}; use engine_rocks::{ + RocksCfOptions, RocksDbOptions, RocksEngine, RocksSstReader, RocksSstWriterBuilder, raw::{DBEntryType, Env, TablePropertiesCollector, TablePropertiesCollectorFactory}, util::new_engine_opt, - RocksCfOptions, RocksDbOptions, RocksEngine, RocksSstReader, RocksSstWriterBuilder, }; pub use engine_rocks::{RocksEngine as TestEngine, RocksSstWriter}; use engine_traits::{KvEngine, SstWriter, SstWriterBuilder}; diff --git a/components/test_sst_importer/src/util.rs b/components/test_sst_importer/src/util.rs index bc954279b1f..2a1344728f4 100644 --- a/components/test_sst_importer/src/util.rs +++ b/components/test_sst_importer/src/util.rs @@ -9,7 +9,7 @@ use std::{ use engine_traits::CF_DEFAULT; use external_storage::{ExternalStorage, UnpinReader}; -use futures::{executor::block_on, io::Cursor as AsyncCursor, stream, SinkExt}; +use futures::{SinkExt, executor::block_on, io::Cursor as AsyncCursor, stream}; use grpcio::{Result, WriteFlags}; use kvproto::{ brpb::{Local, StorageBackend}, diff --git a/components/test_storage/src/assert_storage.rs b/components/test_storage/src/assert_storage.rs index 3a641a322a2..21986f54c70 100644 --- a/components/test_storage/src/assert_storage.rs +++ b/components/test_storage/src/assert_storage.rs @@ -7,11 +7,10 @@ use kvproto::{ }; use test_raftstore::{Cluster, ServerCluster, SimulateEngine}; use tikv::storage::{ - self, + self, Engine, Error as StorageError, ErrorInner as StorageErrorInner, TxnStatus, kv::{Error as KvError, ErrorInner as KvErrorInner, RocksEngine}, mvcc::{Error as MvccError, ErrorInner as MvccErrorInner, MAX_TXN_WRITE_SIZE}, txn::{Error as TxnError, ErrorInner as TxnErrorInner}, - Engine, Error as StorageError, ErrorInner as StorageErrorInner, TxnStatus, }; use tikv_util::HandyRwLock; use txn_types::{self, Key, KvPair, Mutation, TimeStamp, Value}; diff --git a/components/test_storage/src/lib.rs b/components/test_storage/src/lib.rs index 40cf5b3a337..da8c4245c76 100644 --- a/components/test_storage/src/lib.rs +++ b/components/test_storage/src/lib.rs @@ -1,7 +1,6 @@ // Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0. #![feature(box_patterns)] -#![feature(associated_type_bounds)] #[macro_use] extern crate tikv_util; diff --git a/components/test_storage/src/sync_storage.rs b/components/test_storage/src/sync_storage.rs index b562d8fc9b6..63685532d77 100644 --- a/components/test_storage/src/sync_storage.rs +++ b/components/test_storage/src/sync_storage.rs @@ -2,7 +2,7 @@ use std::{ marker::PhantomData, - sync::{atomic::AtomicU64, Arc}, + sync::{Arc, atomic::AtomicU64}, }; use api_version::{ApiV1, KvFormat}; @@ -14,14 +14,14 @@ use kvproto::{ metapb, }; use raftstore::coprocessor::{ - region_info_accessor::MockRegionInfoProvider, CoprocessorHost, RegionInfoProvider, + CoprocessorHost, RegionInfoProvider, region_info_accessor::MockRegionInfoProvider, }; use tikv::{ server::gc_worker::{AutoGcConfig, GcConfig, GcSafePointProvider, GcWorker}, storage::{ - config::Config, kv::RocksEngine, lock_manager::MockLockManager, test_util::GetConsumer, - txn::commands, Engine, KvGetStatistics, PrewriteResult, Result, Storage, TestEngineBuilder, - TestStorageBuilder, TxnStatus, + Engine, KvGetStatistics, PrewriteResult, Result, Storage, TestEngineBuilder, + TestStorageBuilder, TxnStatus, config::Config, kv::RocksEngine, + lock_manager::MockLockManager, test_util::GetConsumer, txn::commands, }, }; use tikv_util::time::Instant; diff --git a/components/test_storage/src/util.rs b/components/test_storage/src/util.rs index e91125ba001..2be7bffb8ae 100644 --- a/components/test_storage/src/util.rs +++ b/components/test_storage/src/util.rs @@ -2,7 +2,7 @@ use api_version::KvFormat; use kvproto::kvrpcpb::Context; -use test_raftstore::{new_server_cluster, Cluster, ServerCluster, SimulateEngine}; +use test_raftstore::{Cluster, ServerCluster, SimulateEngine, new_server_cluster}; use tikv_util::HandyRwLock; use super::*; diff --git a/components/test_util/src/encryption.rs b/components/test_util/src/encryption.rs index 3f1691b3d21..d03a313eea4 100644 --- a/components/test_util/src/encryption.rs +++ b/components/test_util/src/encryption.rs @@ -3,8 +3,8 @@ use std::{fs::File, io::Write, path::Path, time::Duration}; use encryption_export::{ - create_backend, DataKeyManager, DataKeyManagerArgs, EncryptionConfig, FileConfig, - MasterKeyConfig, Result, + DataKeyManager, DataKeyManagerArgs, EncryptionConfig, FileConfig, MasterKeyConfig, Result, + create_backend, }; use kvproto::encryptionpb::EncryptionMethod; use tikv_util::config::ReadableDuration; diff --git a/components/test_util/src/lib.rs b/components/test_util/src/lib.rs index 653d246e0fb..1435990aea6 100644 --- a/components/test_util/src/lib.rs +++ b/components/test_util/src/lib.rs @@ -28,7 +28,7 @@ pub use crate::{ encryption::*, kv_generator::*, logging::*, - runner::{clear_failpoints, run_failpoint_tests, run_test_with_hook, run_tests, TestHook}, + runner::{TestHook, clear_failpoints, run_failpoint_tests, run_test_with_hook, run_tests}, security::*, }; diff --git a/components/test_util/src/logging.rs b/components/test_util/src/logging.rs index 4212102df38..e1ee5f90fce 100644 --- a/components/test_util/src/logging.rs +++ b/components/test_util/src/logging.rs @@ -9,7 +9,7 @@ use std::{ time::SystemTime, }; -use chrono::{offset::Local, DateTime}; +use chrono::{DateTime, offset::Local}; use slog::{self, Drain, OwnedKVList, Record}; struct Serializer<'a>(&'a mut dyn std::io::Write); diff --git a/components/tidb_query_aggr/src/impl_avg.rs b/components/tidb_query_aggr/src/impl_avg.rs index c67a72ee9a3..b48503f1a30 100644 --- a/components/tidb_query_aggr/src/impl_avg.rs +++ b/components/tidb_query_aggr/src/impl_avg.rs @@ -3,8 +3,8 @@ use tidb_query_codegen::AggrFunction; use tidb_query_common::Result; use tidb_query_datatype::{ - builder::FieldTypeBuilder, codec::data_type::*, expr::EvalContext, EvalType, FieldTypeFlag, - FieldTypeTp, + EvalType, FieldTypeFlag, FieldTypeTp, builder::FieldTypeBuilder, codec::data_type::*, + expr::EvalContext, }; use tidb_query_expr::RpnExpression; use tipb::{Expr, ExprType, FieldType}; @@ -312,8 +312,8 @@ mod tests { use std::sync::Arc; use tidb_query_datatype::{ - codec::batch::{LazyBatchColumn, LazyBatchColumnVec}, FieldTypeAccessor, + codec::batch::{LazyBatchColumn, LazyBatchColumnVec}, }; use tikv_util::buffer_vec::BufferVec; use tipb_helper::ExprDefBuilder; diff --git a/components/tidb_query_aggr/src/impl_bit_op.rs b/components/tidb_query_aggr/src/impl_bit_op.rs index e69f7fff2b6..5f1fbf8dae3 100644 --- a/components/tidb_query_aggr/src/impl_bit_op.rs +++ b/components/tidb_query_aggr/src/impl_bit_op.rs @@ -132,8 +132,8 @@ impl super::ConcreteAggrFunctionState for AggrFnStateBitOp { #[cfg(test)] mod tests { use tidb_query_datatype::{ - codec::batch::{LazyBatchColumn, LazyBatchColumnVec}, EvalType, FieldTypeAccessor, FieldTypeTp, + codec::batch::{LazyBatchColumn, LazyBatchColumnVec}, }; use tipb_helper::ExprDefBuilder; diff --git a/components/tidb_query_aggr/src/impl_count.rs b/components/tidb_query_aggr/src/impl_count.rs index 3d49d8b25af..15254322f3b 100644 --- a/components/tidb_query_aggr/src/impl_count.rs +++ b/components/tidb_query_aggr/src/impl_count.rs @@ -3,7 +3,7 @@ use tidb_query_codegen::AggrFunction; use tidb_query_common::Result; use tidb_query_datatype::{ - builder::FieldTypeBuilder, codec::data_type::*, expr::EvalContext, FieldTypeFlag, FieldTypeTp, + FieldTypeFlag, FieldTypeTp, builder::FieldTypeBuilder, codec::data_type::*, expr::EvalContext, }; use tidb_query_expr::RpnExpression; use tipb::{Expr, ExprType, FieldType}; diff --git a/components/tidb_query_aggr/src/impl_first.rs b/components/tidb_query_aggr/src/impl_first.rs index fbd23822f1f..f9fb0c76ef7 100644 --- a/components/tidb_query_aggr/src/impl_first.rs +++ b/components/tidb_query_aggr/src/impl_first.rs @@ -4,7 +4,7 @@ use std::marker::PhantomData; use tidb_query_codegen::AggrFunction; use tidb_query_common::Result; -use tidb_query_datatype::{codec::data_type::*, expr::EvalContext, EvalType}; +use tidb_query_datatype::{EvalType, codec::data_type::*, expr::EvalContext}; use tidb_query_expr::RpnExpression; use tipb::{Expr, ExprType, FieldType}; @@ -297,6 +297,7 @@ mod tests { let mut state = function.create_state(); let mut result = [VectorValue::with_capacity(0, EvalType::Int)]; + #[allow(clippy::zero_repeat_side_effects)] update_vector!( state, &mut ctx, diff --git a/components/tidb_query_aggr/src/impl_max_min.rs b/components/tidb_query_aggr/src/impl_max_min.rs index aada14e7bfa..4714ae1ffdd 100644 --- a/components/tidb_query_aggr/src/impl_max_min.rs +++ b/components/tidb_query_aggr/src/impl_max_min.rs @@ -5,9 +5,10 @@ use std::{cmp::Ordering, convert::TryFrom}; use tidb_query_codegen::AggrFunction; use tidb_query_common::Result; use tidb_query_datatype::{ + Collation, EvalType, FieldTypeAccessor, FieldTypeFlag, codec::{collation::Collator, data_type::*}, expr::EvalContext, - match_template_collator, Collation, EvalType, FieldTypeAccessor, FieldTypeFlag, + match_template_collator, }; use tidb_query_expr::RpnExpression; use tipb::{Expr, ExprType, FieldType}; @@ -259,10 +260,12 @@ where _ctx: &mut EvalContext, value: Option>, ) -> Result<()> { - let extreme_ref = self - .extremum - .as_ref() - .map(|x| EnumRef::from_owned_value(unsafe { std::mem::transmute(x) })); + let extreme_ref = self.extremum.as_ref().map(|x| { + EnumRef::from_owned_value(unsafe { + #[allow(clippy::missing_transmute_annotations)] + std::mem::transmute(x) + }) + }); if value.is_some() && (self.extremum.is_none() @@ -344,10 +347,12 @@ where /// ref: https://dev.mysql.com/doc/refman/5.7/en/aggregate-functions.html#function_max #[inline] fn update_concrete(&mut self, _ctx: &mut EvalContext, value: Option>) -> Result<()> { - let extreme_ref = self - .extremum - .as_ref() - .map(|x| SetRef::from_owned_value(unsafe { std::mem::transmute(x) })); + let extreme_ref = self.extremum.as_ref().map(|x| { + SetRef::from_owned_value(unsafe { + #[allow(clippy::missing_transmute_annotations)] + std::mem::transmute(x) + }) + }); if value.is_some() && (self.extremum.is_none() @@ -434,10 +439,12 @@ where where TT: EvaluableRef<'a, EvaluableType = T::EvaluableType> + Ord, { - let extreme_ref = self - .extremum_value - .as_ref() - .map(|x| TT::from_owned_value(unsafe { std::mem::transmute(x) })); + let extreme_ref = self.extremum_value.as_ref().map(|x| { + TT::from_owned_value(unsafe { + #[allow(clippy::missing_transmute_annotations)] + std::mem::transmute(x) + }) + }); if value.is_some() && (self.extremum_value.is_none() || extreme_ref.cmp(&value) == E::ORD) { self.extremum_value = value.map(|x| x.into_owned_value()); } @@ -553,14 +560,14 @@ mod tests { use std::sync::Arc; use tidb_query_datatype::{ - codec::batch::{LazyBatchColumn, LazyBatchColumnVec}, EvalType, FieldTypeAccessor, FieldTypeTp, + codec::batch::{LazyBatchColumn, LazyBatchColumnVec}, }; use tikv_util::buffer_vec::BufferVec; use tipb_helper::ExprDefBuilder; use super::*; - use crate::{parser::AggrDefinitionParser, AggrFunction}; + use crate::{AggrFunction, parser::AggrDefinitionParser}; #[test] fn test_max() { diff --git a/components/tidb_query_aggr/src/impl_sum.rs b/components/tidb_query_aggr/src/impl_sum.rs index be2affd304b..5ee8d2f5a82 100644 --- a/components/tidb_query_aggr/src/impl_sum.rs +++ b/components/tidb_query_aggr/src/impl_sum.rs @@ -2,7 +2,7 @@ use tidb_query_codegen::AggrFunction; use tidb_query_common::Result; -use tidb_query_datatype::{codec::data_type::*, expr::EvalContext, EvalType}; +use tidb_query_datatype::{EvalType, codec::data_type::*, expr::EvalContext}; use tidb_query_expr::RpnExpression; use tipb::{Expr, ExprType, FieldType}; @@ -306,8 +306,8 @@ mod tests { use std::sync::Arc; use tidb_query_datatype::{ - codec::batch::{LazyBatchColumn, LazyBatchColumnVec}, FieldTypeAccessor, FieldTypeTp, + codec::batch::{LazyBatchColumn, LazyBatchColumnVec}, }; use tikv_util::buffer_vec::BufferVec; use tipb_helper::ExprDefBuilder; diff --git a/components/tidb_query_aggr/src/impl_variance.rs b/components/tidb_query_aggr/src/impl_variance.rs index 40b85e07f23..50c44c9499b 100644 --- a/components/tidb_query_aggr/src/impl_variance.rs +++ b/components/tidb_query_aggr/src/impl_variance.rs @@ -3,8 +3,8 @@ use tidb_query_codegen::AggrFunction; use tidb_query_common::Result; use tidb_query_datatype::{ - builder::FieldTypeBuilder, codec::data_type::*, expr::EvalContext, EvalType, FieldTypeFlag, - FieldTypeTp, + EvalType, FieldTypeFlag, FieldTypeTp, builder::FieldTypeBuilder, codec::data_type::*, + expr::EvalContext, }; use tidb_query_expr::RpnExpression; use tipb::{Expr, ExprType, FieldType}; @@ -453,8 +453,8 @@ mod tests { use std::sync::Arc; use tidb_query_datatype::{ - codec::batch::{LazyBatchColumn, LazyBatchColumnVec}, FieldTypeAccessor, FieldTypeTp, + codec::batch::{LazyBatchColumn, LazyBatchColumnVec}, }; use tikv_util::buffer_vec::BufferVec; use tipb_helper::ExprDefBuilder; diff --git a/components/tidb_query_aggr/src/lib.rs b/components/tidb_query_aggr/src/lib.rs index 8b53c8a078d..ce1133250a3 100644 --- a/components/tidb_query_aggr/src/lib.rs +++ b/components/tidb_query_aggr/src/lib.rs @@ -3,8 +3,10 @@ //! This crate provides aggregate functions for batch executors. #![allow(incomplete_features)] +#![allow(clippy::macro_metavars_in_unsafe)] #![feature(proc_macro_hygiene)] #![feature(specialization)] +#![feature(stmt_expr_attributes)] #[macro_use(box_try)] extern crate tikv_util; @@ -35,13 +37,13 @@ pub use self::parser::{AggrDefinitionParser, AllAggrDefinitionParser}; /// incremental update model: /// /// - Each aggregate function associates a state structure, storing partially -/// computed aggregate results. +/// computed aggregate results. /// /// - The caller calls `update()` or `update_vector()` for each row to update -/// the state. +/// the state. /// /// - The caller finally calls `push_result()` to aggregate a summary value and -/// push it into the given data container. +/// push it into the given data container. /// /// This trait can be auto derived by using `tidb_query_codegen::AggrFunction`. pub trait AggrFunction: std::fmt::Debug + Send + 'static { @@ -129,6 +131,7 @@ macro_rules! update_vector { } #[macro_export] +#[allow(clippy::macro_metavars_in_unsafe)] macro_rules! update_repeat { ($state:expr, $ctx:expr, $value:expr, $repeat_times:expr) => { unsafe { $state.update_repeat_unsafe($ctx, $value.unsafe_into(), $repeat_times) } @@ -138,7 +141,10 @@ macro_rules! update_repeat { #[macro_export] macro_rules! update { ($state:expr, $ctx:expr, $value:expr) => { - unsafe { $state.update_unsafe($ctx, $value.unsafe_into()) } + #[allow(clippy::macro_metavars_in_unsafe)] + unsafe { + $state.update_unsafe($ctx, $value.unsafe_into()) + } }; } diff --git a/components/tidb_query_aggr/src/parser.rs b/components/tidb_query_aggr/src/parser.rs index 600326edb2f..78558912623 100644 --- a/components/tidb_query_aggr/src/parser.rs +++ b/components/tidb_query_aggr/src/parser.rs @@ -5,7 +5,7 @@ use tidb_query_datatype::expr::EvalContext; use tidb_query_expr::{RpnExpression, RpnExpressionBuilder}; use tipb::{Expr, ExprType, FieldType}; -use crate::{impl_bit_op::*, impl_max_min::*, impl_variance::*, AggrFunction}; +use crate::{AggrFunction, impl_bit_op::*, impl_max_min::*, impl_variance::*}; /// Parse a specific aggregate function definition from protobuf. /// diff --git a/components/tidb_query_aggr/src/summable.rs b/components/tidb_query_aggr/src/summable.rs index dd8b802644a..1f304a2bd4e 100644 --- a/components/tidb_query_aggr/src/summable.rs +++ b/components/tidb_query_aggr/src/summable.rs @@ -13,6 +13,7 @@ pub trait Summable: Evaluable + EvaluableRet { /// Adds assign another value. fn add_assign(&mut self, ctx: &mut EvalContext, other: &Self) -> Result<()>; + #[allow(dead_code)] fn add(&self, other: &Self) -> Result; fn sub(&self, other: &Self) -> Result; fn mul(&self, other: &Self) -> Result; diff --git a/components/tidb_query_aggr/src/util.rs b/components/tidb_query_aggr/src/util.rs index c4361e685a5..0d83cfaac98 100644 --- a/components/tidb_query_aggr/src/util.rs +++ b/components/tidb_query_aggr/src/util.rs @@ -1,8 +1,8 @@ // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. use tidb_query_common::Result; -use tidb_query_datatype::{builder::FieldTypeBuilder, EvalType, FieldTypeAccessor, FieldTypeTp}; -use tidb_query_expr::{impl_cast::get_cast_fn_rpn_node, RpnExpression, RpnExpressionBuilder}; +use tidb_query_datatype::{EvalType, FieldTypeAccessor, FieldTypeTp, builder::FieldTypeBuilder}; +use tidb_query_expr::{RpnExpression, RpnExpressionBuilder, impl_cast::get_cast_fn_rpn_node}; use tipb::{Expr, FieldType}; /// Checks whether or not there is only one child and the child expression is diff --git a/components/tidb_query_codegen/src/lib.rs b/components/tidb_query_codegen/src/lib.rs index feee1c6afb3..9053fa8c9ff 100644 --- a/components/tidb_query_codegen/src/lib.rs +++ b/components/tidb_query_codegen/src/lib.rs @@ -20,7 +20,7 @@ mod rpn_function; use darling::FromDeriveInput; use proc_macro::TokenStream; -use syn::{parse_macro_input, DeriveInput}; +use syn::{DeriveInput, parse_macro_input}; /// The `AggrFunction` custom derive. #[proc_macro_derive(AggrFunction, attributes(aggr_function))] diff --git a/components/tidb_query_codegen/src/rpn_function.rs b/components/tidb_query_codegen/src/rpn_function.rs index 8b414b64a63..00494f2f216 100644 --- a/components/tidb_query_codegen/src/rpn_function.rs +++ b/components/tidb_query_codegen/src/rpn_function.rs @@ -200,7 +200,7 @@ //! ``` use heck::CamelCase; use proc_macro2::{Span, TokenStream}; -use quote::{quote, ToTokens}; +use quote::{ToTokens, quote}; use syn::{punctuated::Punctuated, *}; /// Entry point for the `rpn_fn` attribute. diff --git a/components/tidb_query_common/src/storage/scanner.rs b/components/tidb_query_common/src/storage/scanner.rs index d0d2345a09e..75e5e6d180f 100644 --- a/components/tidb_query_common/src/storage/scanner.rs +++ b/components/tidb_query_common/src/storage/scanner.rs @@ -6,7 +6,7 @@ use api_version::KvFormat; use tikv_util::time::Instant; use yatp::task::future::reschedule; -use super::{range::*, ranges_iter::*, OwnedKvPair, Storage}; +use super::{OwnedKvPair, Storage, range::*, ranges_iter::*}; use crate::error::StorageError; const KEY_BUFFER_CAPACITY: usize = 64; @@ -292,11 +292,11 @@ impl RangesScanner { #[cfg(test)] mod tests { - use api_version::{keyspace::KvPair, ApiV1}; + use api_version::{ApiV1, keyspace::KvPair}; use futures::executor::block_on; use super::*; - use crate::storage::{test_fixture::FixtureStorage, IntervalRange, PointRange, Range}; + use crate::storage::{IntervalRange, PointRange, Range, test_fixture::FixtureStorage}; fn create_storage() -> FixtureStorage { let data: &[(&'static [u8], &'static [u8])] = &[ diff --git a/components/tidb_query_common/src/storage/test_fixture.rs b/components/tidb_query_common/src/storage/test_fixture.rs index 305bc5bf168..f7ac5bbc9e0 100644 --- a/components/tidb_query_common/src/storage/test_fixture.rs +++ b/components/tidb_query_common/src/storage/test_fixture.rs @@ -1,11 +1,11 @@ // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. use std::{ - collections::{btree_map, BTreeMap}, + collections::{BTreeMap, btree_map}, sync::Arc, }; -use super::{range::*, Result}; +use super::{Result, range::*}; type ErrorBuilder = Box crate::error::StorageError>; @@ -61,8 +61,12 @@ impl super::Storage for FixtureStorage { let data_view = self .data .range(range.lower_inclusive..range.upper_exclusive); + // Erase the lifetime to be 'static. - self.data_view_unsafe = unsafe { Some(std::mem::transmute(data_view)) }; + self.data_view_unsafe = unsafe { + #[allow(clippy::missing_transmute_annotations)] + Some(std::mem::transmute(data_view)) + }; self.is_backward_scan = is_backward_scan; self.is_key_only = is_key_only; Ok(()) diff --git a/components/tidb_query_datatype/Cargo.toml b/components/tidb_query_datatype/Cargo.toml index ccfb82750c0..931508d062d 100644 --- a/components/tidb_query_datatype/Cargo.toml +++ b/components/tidb_query_datatype/Cargo.toml @@ -28,7 +28,7 @@ log_wrappers = { workspace = true } match-template = "0.0.1" nom = { version = "7.1.0", default-features = false, features = ["std"] } num = { version = "0.3", default-features = false } -num-derive = "0.3" +num-derive = "0.4" num-traits = "0.2" ordered-float = "2.0" protobuf = "2" diff --git a/components/tidb_query_datatype/benches/bench_vector_distance.rs b/components/tidb_query_datatype/benches/bench_vector_distance.rs index 62cf2b2ad98..4a2675e3e9f 100644 --- a/components/tidb_query_datatype/benches/bench_vector_distance.rs +++ b/components/tidb_query_datatype/benches/bench_vector_distance.rs @@ -1,6 +1,6 @@ // Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0. -use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use criterion::{Criterion, black_box, criterion_group, criterion_main}; use tidb_query_datatype::codec::mysql::VectorFloat32Ref; fn bench_l1_distance_3d(c: &mut Criterion) { diff --git a/components/tidb_query_datatype/src/codec/batch/lazy_column.rs b/components/tidb_query_datatype/src/codec/batch/lazy_column.rs index b95b892e3f0..03c20162771 100644 --- a/components/tidb_query_datatype/src/codec/batch/lazy_column.rs +++ b/components/tidb_query_datatype/src/codec/batch/lazy_column.rs @@ -4,14 +4,15 @@ use tikv_util::buffer_vec::BufferVec; use tipb::FieldType; use crate::{ + EvalType, FieldTypeAccessor, codec::{ + Result, chunk::{ChunkColumnEncoder, Column}, data_type::{ChunkedVec, LogicalRows, VectorValue}, datum_codec::RawDatumDecoder, - Result, }, expr::EvalContext, - match_template_evaltype, EvalType, FieldTypeAccessor, + match_template_evaltype, }; /// A container stores an array of datums, which can be either raw (not @@ -412,8 +413,8 @@ mod benches { #[bench] fn bench_lazy_batch_column_clone_decoded(b: &mut test::Bencher) { use crate::{ - codec::datum::{Datum, DatumEncoder}, FieldTypeTp, + codec::datum::{Datum, DatumEncoder}, }; let mut column = LazyBatchColumn::raw_with_capacity(1000); @@ -445,8 +446,8 @@ mod benches { #[bench] fn bench_lazy_batch_column_clone_and_decode(b: &mut test::Bencher) { use crate::{ - codec::datum::{Datum, DatumEncoder}, FieldTypeTp, + codec::datum::{Datum, DatumEncoder}, }; let mut ctx = EvalContext::default(); @@ -482,8 +483,8 @@ mod benches { #[bench] fn bench_lazy_batch_column_clone_and_decode_decoded(b: &mut test::Bencher) { use crate::{ - codec::datum::{Datum, DatumEncoder}, FieldTypeTp, + codec::datum::{Datum, DatumEncoder}, }; let mut column = LazyBatchColumn::raw_with_capacity(1000); diff --git a/components/tidb_query_datatype/src/codec/batch/lazy_column_vec.rs b/components/tidb_query_datatype/src/codec/batch/lazy_column_vec.rs index 2bd5ebaa99a..a6f5b97b5cb 100644 --- a/components/tidb_query_datatype/src/codec/batch/lazy_column_vec.rs +++ b/components/tidb_query_datatype/src/codec/batch/lazy_column_vec.rs @@ -6,7 +6,7 @@ use tipb::FieldType; use super::LazyBatchColumn; use crate::{ - codec::{data_type::VectorValue, Result}, + codec::{Result, data_type::VectorValue}, expr::EvalContext, }; diff --git a/components/tidb_query_datatype/src/codec/chunk/chunk.rs b/components/tidb_query_datatype/src/codec/chunk/chunk.rs index b4478c8a4d3..acd8d9a35e7 100644 --- a/components/tidb_query_datatype/src/codec/chunk/chunk.rs +++ b/components/tidb_query_datatype/src/codec/chunk/chunk.rs @@ -4,10 +4,10 @@ use codec::buffer::BufferWriter; use tipb::FieldType; use super::{ - column::{ChunkColumnEncoder, Column}, Result, + column::{ChunkColumnEncoder, Column}, }; -use crate::{codec::Datum, FieldTypeAccessor}; +use crate::{FieldTypeAccessor, codec::Datum}; /// `Chunk` stores multiple rows of data. /// Values are appended in compact format and can be directly accessed without @@ -166,17 +166,17 @@ impl<'a> Iterator for RowIterator<'a> { #[cfg(test)] mod tests { - use test::{black_box, Bencher}; + use test::{Bencher, black_box}; use super::*; use crate::{ + FieldTypeTp, codec::{ batch::LazyBatchColumn, datum::{Datum, DatumEncoder}, mysql::*, }, expr::EvalContext, - FieldTypeTp, }; #[test] diff --git a/components/tidb_query_datatype/src/codec/chunk/column.rs b/components/tidb_query_datatype/src/codec/chunk/column.rs index 1068f0d7ba5..1ce69616466 100644 --- a/components/tidb_query_datatype/src/codec/chunk/column.rs +++ b/components/tidb_query_datatype/src/codec/chunk/column.rs @@ -9,14 +9,16 @@ use tipb::FieldType; use super::{Error, Result}; use crate::{ + EvalType, FieldTypeFlag, FieldTypeTp, codec::{ + Datum, data_type::{ChunkRef, VectorFloat32Ref, VectorValue}, datum, datum_codec::DatumPayloadDecoder, mysql::{ decimal::{ - Decimal, DecimalDatumPayloadChunkEncoder, DecimalDecoder, DecimalEncoder, - DECIMAL_STRUCT_SIZE, + DECIMAL_STRUCT_SIZE, Decimal, DecimalDatumPayloadChunkEncoder, DecimalDecoder, + DecimalEncoder, }, duration::{ Duration, DurationDatumPayloadChunkEncoder, DurationDecoder, DurationEncoder, @@ -29,11 +31,9 @@ use crate::{ VectorFloat32Encoder, }, }, - Datum, }, expr::EvalContext, prelude::*, - EvalType, FieldTypeFlag, FieldTypeTp, }; /// `Column` stores the same column data of multi rows in one chunk. @@ -1023,7 +1023,7 @@ impl Column { let mut col = Column::new(tp, length); col.length = length; col.null_cnt = buf.read_u32_le()? as usize; - let null_length = (col.length + 7) / 8_usize; + let null_length = col.length.div_ceil(8); if col.null_cnt > 0 { col.null_bitmap = buf.read_bytes(null_length)?.to_vec(); } else { @@ -1053,7 +1053,7 @@ pub trait ChunkColumnEncoder: NumberEncoder { self.write_u32_le(col.null_cnt as u32)?; // bitmap if col.null_cnt > 0 { - let length = (col.length + 7) / 8; + let length = col.length.div_ceil(8); self.write_bytes(&col.null_bitmap[0..length])?; } // offsets @@ -1073,7 +1073,7 @@ impl ChunkColumnEncoder for T {} #[cfg(test)] mod tests { - use std::{f64, u64}; + use std::f64; use tipb::FieldType; diff --git a/components/tidb_query_datatype/src/codec/collation/collator/gb18030_collation.rs b/components/tidb_query_datatype/src/codec/collation/collator/gb18030_collation.rs index ff21559ded6..cc89e986cc6 100644 --- a/components/tidb_query_datatype/src/codec/collation/collator/gb18030_collation.rs +++ b/components/tidb_query_datatype/src/codec/collation/collator/gb18030_collation.rs @@ -148,8 +148,8 @@ const GB18030_CHINESE_CI_TABLE: &[u8; TABLE_SIZE_FOR_GB18030] = #[cfg(test)] mod tests { use crate::codec::collation::{ - collator::{CollatorGb18030Bin, CollatorGb18030ChineseCi}, Collator, + collator::{CollatorGb18030Bin, CollatorGb18030ChineseCi}, }; #[test] diff --git a/components/tidb_query_datatype/src/codec/collation/collator/latin1_bin.rs b/components/tidb_query_datatype/src/codec/collation/collator/latin1_bin.rs index a63a5aed7e2..f4ba576340a 100644 --- a/components/tidb_query_datatype/src/codec/collation/collator/latin1_bin.rs +++ b/components/tidb_query_datatype/src/codec/collation/collator/latin1_bin.rs @@ -1,6 +1,6 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. -use bstr::{ByteSlice, B}; +use bstr::{B, ByteSlice}; use super::*; diff --git a/components/tidb_query_datatype/src/codec/collation/collator/mod.rs b/components/tidb_query_datatype/src/codec/collation/collator/mod.rs index 4385f83e74b..32501798f4e 100644 --- a/components/tidb_query_datatype/src/codec/collation/collator/mod.rs +++ b/components/tidb_query_datatype/src/codec/collation/collator/mod.rs @@ -23,14 +23,14 @@ pub use utf8mb4_binary::*; pub use utf8mb4_general_ci::*; pub use utf8mb4_uca::*; -use super::{charset::*, Collator}; +use super::{Collator, charset::*}; use crate::codec::Result; pub const PADDING_SPACE: char = 0x20 as char; #[cfg(test)] mod tests { - use crate::{codec::collation::Collator, match_template_collator, Collation}; + use crate::{Collation, codec::collation::Collator, match_template_collator}; #[test] #[allow(clippy::string_lit_as_bytes)] diff --git a/components/tidb_query_datatype/src/codec/collation/encoding/gb18030.rs b/components/tidb_query_datatype/src/codec/collation/encoding/gb18030.rs index b66512af0f7..44077975064 100644 --- a/components/tidb_query_datatype/src/codec/collation/encoding/gb18030.rs +++ b/components/tidb_query_datatype/src/codec/collation/encoding/gb18030.rs @@ -275,7 +275,7 @@ impl Encoding for EncodingGb18030 { mod tests { use bstr::ByteSlice; - use crate::codec::collation::{encoding::EncodingGb18030, Encoding}; + use crate::codec::collation::{Encoding, encoding::EncodingGb18030}; #[test] fn test_encode() { diff --git a/components/tidb_query_datatype/src/codec/collation/encoding/mod.rs b/components/tidb_query_datatype/src/codec/collation/encoding/mod.rs index ba7d5d75ec2..5a713248b56 100644 --- a/components/tidb_query_datatype/src/codec/collation/encoding/mod.rs +++ b/components/tidb_query_datatype/src/codec/collation/encoding/mod.rs @@ -17,8 +17,8 @@ pub use utf8::*; use super::Encoding; use crate::codec::{ - data_type::{Bytes, BytesRef}, Error, Result, + data_type::{Bytes, BytesRef}, }; fn format_invalid_char(data: BytesRef<'_>) -> String { diff --git a/components/tidb_query_datatype/src/codec/collation/mod.rs b/components/tidb_query_datatype/src/codec/collation/mod.rs index 98284ac5463..159245099be 100644 --- a/components/tidb_query_datatype/src/codec/collation/mod.rs +++ b/components/tidb_query_datatype/src/codec/collation/mod.rs @@ -15,8 +15,8 @@ use codec::prelude::*; use num::Unsigned; use crate::codec::{ - data_type::{Bytes, BytesGuard, BytesRef, BytesWriter}, Result, + data_type::{Bytes, BytesGuard, BytesRef, BytesWriter}, }; #[macro_export] @@ -197,7 +197,10 @@ where #[allow(clippy::transmute_ptr_to_ptr)] pub fn new_ref(inner: &T) -> Result<&Self> { C::Charset::validate(inner.as_ref())?; - Ok(unsafe { std::mem::transmute(inner) }) + Ok(unsafe { + #[allow(clippy::missing_transmute_annotations)] + std::mem::transmute(inner) + }) } #[inline] @@ -206,7 +209,10 @@ where if let Some(inner) = inner { C::Charset::validate(inner.as_ref())?; } - Ok(unsafe { std::mem::transmute(inner) }) + Ok(unsafe { + #[allow(clippy::missing_transmute_annotations)] + std::mem::transmute(inner) + }) } #[inline] diff --git a/components/tidb_query_datatype/src/codec/convert.rs b/components/tidb_query_datatype/src/codec/convert.rs index 94fd1fbe253..1ac16cd6da3 100644 --- a/components/tidb_query_datatype/src/codec/convert.rs +++ b/components/tidb_query_datatype/src/codec/convert.rs @@ -5,18 +5,18 @@ use std::{borrow::Cow, fmt::Display}; use tipb::FieldType; use super::{ - mysql::{charset::MULTI_BYTES_CHARSETS, RoundMode, DEFAULT_FSP}, Error, Result, + mysql::{DEFAULT_FSP, RoundMode, charset::MULTI_BYTES_CHARSETS}, }; // use crate::{self, FieldTypeTp, UNSPECIFIED_LENGTH}; use crate::{ + Collation, FieldTypeAccessor, FieldTypeTp, UNSPECIFIED_LENGTH, codec::{ data_type::*, error::ERR_DATA_OUT_OF_RANGE, - mysql::{decimal::max_or_min_dec, Res}, + mysql::{Res, decimal::max_or_min_dec}, }, expr::{EvalContext, Flag}, - Collation, FieldTypeAccessor, FieldTypeTp, UNSPECIFIED_LENGTH, }; /// A trait for converting a value to an `Int`. @@ -27,6 +27,20 @@ pub trait ToInt { fn to_uint(&self, ctx: &mut EvalContext, tp: FieldTypeTp) -> Result; } +/// A trait for converting a value to a `String`. +/// +/// NOTE: we can't resue the `ToString` trait becuase this trait may return +/// different value with `fmt::Display`. +pub trait ToStringValue { + fn to_string_value(&self) -> String; +} + +impl ToStringValue for T { + default fn to_string_value(&self) -> String { + self.to_string() + } +} + /// A trait for converting a value to `T` pub trait ConvertTo { /// Converts the given value to `T` value @@ -78,22 +92,22 @@ where impl ConvertTo for T where - T: ToString + EvaluableRet, + T: ToStringValue + EvaluableRet, { #[inline] fn convert(&self, _: &mut EvalContext) -> Result { // FIXME: There is an additional step `ProduceStrWithSpecifiedTp` in TiDB. - Ok(self.to_string()) + Ok(self.to_string_value()) } } impl ConvertTo for T where - T: ToString + EvaluableRet, + T: ToStringValue + EvaluableRet, { #[inline] fn convert(&self, _: &mut EvalContext) -> Result { - Ok(self.to_string().into_bytes()) + Ok(self.to_string_value().into_bytes()) } } @@ -110,21 +124,21 @@ impl<'a> ConvertTo for JsonRef<'a> { #[inline] fn convert(&self, _: &mut EvalContext) -> Result { // FIXME: There is an additional step `ProduceStrWithSpecifiedTp` in TiDB. - Ok(self.to_string()) + Ok(self.to_string_value()) } } impl<'a> ConvertTo for JsonRef<'a> { #[inline] fn convert(&self, _: &mut EvalContext) -> Result { - Ok(self.to_string().into_bytes()) + Ok(self.to_string_value().into_bytes()) } } impl<'a> ConvertTo for EnumRef<'a> { #[inline] fn convert(&self, _: &mut EvalContext) -> Result { - Ok(self.to_string().into_bytes()) + Ok(self.to_string_value().into_bytes()) } } @@ -516,7 +530,10 @@ impl<'a> ToInt for JsonRef<'a> { JsonType::Double => self.get_double().to_int(ctx, tp), JsonType::String => self.get_str_bytes()?.to_int(ctx, tp), _ => Ok(ctx - .handle_truncate_err(Error::truncated_wrong_val("Integer", self.to_string())) + .handle_truncate_err(Error::truncated_wrong_val( + "Integer", + self.to_string_value(), + )) .map(|_| 0)?), }?; val.to_int(ctx, tp) @@ -532,7 +549,10 @@ impl<'a> ToInt for JsonRef<'a> { JsonType::Double => self.get_double().to_uint(ctx, tp), JsonType::String => self.get_str_bytes()?.to_uint(ctx, tp), _ => Ok(ctx - .handle_truncate_err(Error::truncated_wrong_val("Integer", self.to_string())) + .handle_truncate_err(Error::truncated_wrong_val( + "Integer", + self.to_string_value(), + )) .map(|_| 0)?), }?; val.to_uint(ctx, tp) @@ -1011,7 +1031,7 @@ fn exp_float_str_to_int_str<'a>( let int_cnt: i64; match dot_idx { None => { - digits.extend_from_slice(valid_float[..e_idx].as_bytes()); + digits.extend_from_slice(&valid_float.as_bytes()[..e_idx]); // if digits.len() > i64::MAX, // then the input str has at least 9223372036854775808 chars, // which make the str >= 8388608.0 TB, @@ -1019,9 +1039,9 @@ fn exp_float_str_to_int_str<'a>( int_cnt = digits.len() as i64; } Some(dot_idx) => { - digits.extend_from_slice(valid_float[..dot_idx].as_bytes()); + digits.extend_from_slice(&valid_float.as_bytes()[..dot_idx]); int_cnt = digits.len() as i64; - digits.extend_from_slice(valid_float[(dot_idx + 1)..e_idx].as_bytes()); + digits.extend_from_slice(&valid_float.as_bytes()[(dot_idx + 1)..e_idx]); } } // make `digits` immutable @@ -1138,15 +1158,15 @@ mod tests { use super::*; use crate::{ + Collation, FieldTypeFlag, codec::{ error::{ ERR_DATA_OUT_OF_RANGE, ERR_M_BIGGER_THAN_D, ERR_TRUNCATE_WRONG_VALUE, WARN_DATA_TRUNCATED, }, - mysql::{charset, Res, UNSPECIFIED_FSP}, + mysql::{Res, UNSPECIFIED_FSP, charset}, }, expr::{EvalConfig, EvalContext, Flag}, - Collation, FieldTypeFlag, }; #[test] @@ -2383,7 +2403,11 @@ mod tests { nd.frac_cnt(), nd.result_frac_cnt() ); - assert_eq!(nd, want, "{}, {}, {}, {}, {}", dec, nd, want, flen, decimal); + assert_eq!( + nd, want, + "{:?}, {:?}, {:?}, {}, {}", + dec, nd, want, flen, decimal + ); } } @@ -2771,8 +2795,8 @@ mod tests { let r = produce_dec_with_specified_tp(&mut ctx, input, &rft); // make log - let rs = r.as_ref().map(|x| x.to_string()); - let expect_str = expect.as_ref().map(|x| x.to_string()); + let rs = r.as_ref().map(|x| x.to_string_value()); + let expect_str = expect.as_ref().map(|x| x.to_string_value()); let log = format!( "input: {}, origin_flen: {}, origin_decimal: {}, \ res_flen: {}, res_decimal: {}, is_unsigned: {}, \ diff --git a/components/tidb_query_datatype/src/codec/data_type/chunked_vec_bytes.rs b/components/tidb_query_datatype/src/codec/data_type/chunked_vec_bytes.rs index c4f5abbc122..f5e0e37372d 100644 --- a/components/tidb_query_datatype/src/codec/data_type/chunked_vec_bytes.rs +++ b/components/tidb_query_datatype/src/codec/data_type/chunked_vec_bytes.rs @@ -1,6 +1,6 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. -use super::{bit_vec::BitVec, Bytes, BytesRef, ChunkRef, ChunkedVec, UnsafeRefInto}; +use super::{Bytes, BytesRef, ChunkRef, ChunkedVec, UnsafeRefInto, bit_vec::BitVec}; use crate::impl_chunked_vec_common; #[derive(Debug, PartialEq, Clone)] @@ -213,7 +213,7 @@ impl From>> for ChunkedVecBytes { } } -impl<'a> UnsafeRefInto<&'static ChunkedVecBytes> for &'a ChunkedVecBytes { +impl UnsafeRefInto<&'static ChunkedVecBytes> for &ChunkedVecBytes { unsafe fn unsafe_into(self) -> &'static ChunkedVecBytes { std::mem::transmute(self) } diff --git a/components/tidb_query_datatype/src/codec/data_type/chunked_vec_enum.rs b/components/tidb_query_datatype/src/codec/data_type/chunked_vec_enum.rs index 632adfedc29..33360b0db97 100644 --- a/components/tidb_query_datatype/src/codec/data_type/chunked_vec_enum.rs +++ b/components/tidb_query_datatype/src/codec/data_type/chunked_vec_enum.rs @@ -1,8 +1,8 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. -use super::{bit_vec::BitVec, ChunkRef, ChunkedVec, Enum, EnumRef, UnsafeRefInto}; +use super::{ChunkRef, ChunkedVec, Enum, EnumRef, UnsafeRefInto, bit_vec::BitVec}; use crate::{ - codec::data_type::{retain_lifetime_transmute, ChunkedVecBytes, ChunkedVecSized, Int}, + codec::data_type::{ChunkedVecBytes, ChunkedVecSized, Int, retain_lifetime_transmute}, impl_chunked_vec_common, }; @@ -150,7 +150,7 @@ impl From>> for ChunkedVecEnum { } } -impl<'a> UnsafeRefInto<&'static ChunkedVecEnum> for &'a ChunkedVecEnum { +impl UnsafeRefInto<&'static ChunkedVecEnum> for &ChunkedVecEnum { unsafe fn unsafe_into(self) -> &'static ChunkedVecEnum { std::mem::transmute(self) } diff --git a/components/tidb_query_datatype/src/codec/data_type/chunked_vec_json.rs b/components/tidb_query_datatype/src/codec/data_type/chunked_vec_json.rs index 4d6ac39c006..686fc12c093 100644 --- a/components/tidb_query_datatype/src/codec/data_type/chunked_vec_json.rs +++ b/components/tidb_query_datatype/src/codec/data_type/chunked_vec_json.rs @@ -1,6 +1,6 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. -use super::{bit_vec::BitVec, ChunkRef, ChunkedVec, Json, JsonRef, JsonType, UnsafeRefInto}; +use super::{ChunkRef, ChunkedVec, Json, JsonRef, JsonType, UnsafeRefInto, bit_vec::BitVec}; use crate::impl_chunked_vec_common; /// A vector storing `Option` with a compact layout. @@ -121,7 +121,7 @@ impl From>> for ChunkedVecJson { } } -impl<'a> UnsafeRefInto<&'static ChunkedVecJson> for &'a ChunkedVecJson { +impl UnsafeRefInto<&'static ChunkedVecJson> for &ChunkedVecJson { unsafe fn unsafe_into(self) -> &'static ChunkedVecJson { std::mem::transmute(self) } diff --git a/components/tidb_query_datatype/src/codec/data_type/chunked_vec_set.rs b/components/tidb_query_datatype/src/codec/data_type/chunked_vec_set.rs index 1a3f6838e96..a9dcb55e546 100644 --- a/components/tidb_query_datatype/src/codec/data_type/chunked_vec_set.rs +++ b/components/tidb_query_datatype/src/codec/data_type/chunked_vec_set.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use tikv_util::buffer_vec::BufferVec; -use super::{bit_vec::BitVec, ChunkRef, ChunkedVec, Set, SetRef, UnsafeRefInto}; +use super::{ChunkRef, ChunkedVec, Set, SetRef, UnsafeRefInto, bit_vec::BitVec}; use crate::impl_chunked_vec_common; /// `ChunkedVecSet` stores set in a compact way. @@ -142,7 +142,7 @@ impl From>> for ChunkedVecSet { } } -impl<'a> UnsafeRefInto<&'static ChunkedVecSet> for &'a ChunkedVecSet { +impl UnsafeRefInto<&'static ChunkedVecSet> for &ChunkedVecSet { unsafe fn unsafe_into(self) -> &'static ChunkedVecSet { std::mem::transmute(self) } diff --git a/components/tidb_query_datatype/src/codec/data_type/chunked_vec_sized.rs b/components/tidb_query_datatype/src/codec/data_type/chunked_vec_sized.rs index 4f614d00be0..8ea71e28e16 100644 --- a/components/tidb_query_datatype/src/codec/data_type/chunked_vec_sized.rs +++ b/components/tidb_query_datatype/src/codec/data_type/chunked_vec_sized.rs @@ -1,6 +1,6 @@ // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. -use super::{bit_vec::BitVec, ChunkRef, ChunkedVec, Evaluable, EvaluableRet, UnsafeRefInto}; +use super::{ChunkRef, ChunkedVec, Evaluable, EvaluableRet, UnsafeRefInto, bit_vec::BitVec}; use crate::impl_chunked_vec_common; /// A vector storing `Option` with a compact layout. @@ -105,7 +105,7 @@ impl From>> for ChunkedVecSized { } } -impl<'a, T: Evaluable> UnsafeRefInto<&'static ChunkedVecSized> for &'a ChunkedVecSized { +impl UnsafeRefInto<&'static ChunkedVecSized> for &ChunkedVecSized { unsafe fn unsafe_into(self) -> &'static ChunkedVecSized { std::mem::transmute(self) } diff --git a/components/tidb_query_datatype/src/codec/data_type/chunked_vec_vector_float32.rs b/components/tidb_query_datatype/src/codec/data_type/chunked_vec_vector_float32.rs index d065a8ec0ff..4fab500c778 100644 --- a/components/tidb_query_datatype/src/codec/data_type/chunked_vec_vector_float32.rs +++ b/components/tidb_query_datatype/src/codec/data_type/chunked_vec_vector_float32.rs @@ -1,7 +1,7 @@ // Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0. use super::{ - bit_vec::BitVec, ChunkRef, ChunkedVec, UnsafeRefInto, VectorFloat32, VectorFloat32Ref, + ChunkRef, ChunkedVec, UnsafeRefInto, VectorFloat32, VectorFloat32Ref, bit_vec::BitVec, }; use crate::{ codec::mysql::{VectorFloat32Decoder, VectorFloat32Encoder}, @@ -128,7 +128,7 @@ impl From>> for ChunkedVecVectorFloat32 { } } -impl<'a> UnsafeRefInto<&'static ChunkedVecVectorFloat32> for &'a ChunkedVecVectorFloat32 { +impl UnsafeRefInto<&'static ChunkedVecVectorFloat32> for &ChunkedVecVectorFloat32 { unsafe fn unsafe_into(self) -> &'static ChunkedVecVectorFloat32 { std::mem::transmute(self) } diff --git a/components/tidb_query_datatype/src/codec/data_type/mod.rs b/components/tidb_query_datatype/src/codec/data_type/mod.rs index 5a11a5f295e..68ffd388254 100644 --- a/components/tidb_query_datatype/src/codec/data_type/mod.rs +++ b/components/tidb_query_datatype/src/codec/data_type/mod.rs @@ -12,7 +12,7 @@ mod logical_rows; mod scalar; mod vector; -pub use logical_rows::{LogicalRows, BATCH_MAX_SIZE, IDENTICAL_LOGICAL_ROWS}; +pub use logical_rows::{BATCH_MAX_SIZE, IDENTICAL_LOGICAL_ROWS, LogicalRows}; #[macro_export] macro_rules! match_template_evaltype { @@ -48,10 +48,10 @@ pub use self::{ }; use super::Result; pub use crate::codec::mysql::{ - json::JsonRef, Decimal, Duration, Enum, EnumRef, Json, JsonType, Set, SetRef, Time as DateTime, - VectorFloat32, VectorFloat32Ref, + Decimal, Duration, Enum, EnumRef, Json, JsonType, Set, SetRef, Time as DateTime, VectorFloat32, + VectorFloat32Ref, json::JsonRef, }; -use crate::{codec::convert::ConvertTo, expr::EvalContext, EvalType}; +use crate::{EvalType, codec::convert::ConvertTo, expr::EvalContext}; /// A trait of evaluating current concrete eval type into a MySQL logic value, /// represented by Rust's `bool` type. @@ -74,7 +74,7 @@ impl AsMySqlBool for Real { } } -impl<'a, T: AsMySqlBool> AsMySqlBool for &'a T { +impl AsMySqlBool for &T { #[inline] fn as_mysql_bool(&self, context: &mut EvalContext) -> Result { (**self).as_mysql_bool(context) @@ -95,7 +95,7 @@ impl<'a> AsMySqlBool for BytesRef<'a> { } } -impl<'a, T> AsMySqlBool for Option<&'a T> +impl AsMySqlBool for Option<&T> where T: AsMySqlBool, { @@ -441,7 +441,7 @@ impl, B> UnsafeRefInto> for Option { } } -impl<'a, T: Evaluable + EvaluableRet> UnsafeRefInto<&'static T> for &'a T { +impl UnsafeRefInto<&'static T> for &T { unsafe fn unsafe_into(self) -> &'static T { std::mem::transmute(self) } @@ -711,7 +711,7 @@ impl<'a> EvaluableRef<'a> for SetRef<'a> { } } #[inline] - fn borrow_vector_value(v: &'a VectorValue) -> &ChunkedVecSet { + fn borrow_vector_value(v: &'a VectorValue) -> &'a ChunkedVecSet { match v { VectorValue::Set(x) => x, other => panic!( @@ -733,7 +733,6 @@ impl<'a> EvaluableRef<'a> for SetRef<'a> { #[cfg(test)] mod tests { - use std::f64; use super::*; diff --git a/components/tidb_query_datatype/src/codec/data_type/scalar.rs b/components/tidb_query_datatype/src/codec/data_type/scalar.rs index 397ffbe0619..3587802e565 100644 --- a/components/tidb_query_datatype/src/codec/data_type/scalar.rs +++ b/components/tidb_query_datatype/src/codec/data_type/scalar.rs @@ -7,8 +7,8 @@ use tipb::FieldType; use super::*; use crate::{ - codec::collation::Collator, match_template_collator, match_template_evaltype, Collation, - EvalType, FieldTypeAccessor, + Collation, EvalType, FieldTypeAccessor, codec::collation::Collator, match_template_collator, + match_template_evaltype, }; /// A scalar value container, a.k.a. datum, for all concrete eval types. diff --git a/components/tidb_query_datatype/src/codec/data_type/vector.rs b/components/tidb_query_datatype/src/codec/data_type/vector.rs index 9086ac3f7b9..2b52aa6c13f 100644 --- a/components/tidb_query_datatype/src/codec/data_type/vector.rs +++ b/components/tidb_query_datatype/src/codec/data_type/vector.rs @@ -2,8 +2,9 @@ use super::{scalar::ScalarValueRef, *}; use crate::{ - codec::{mysql::decimal::DECIMAL_STRUCT_SIZE, Result}, - match_template_collator, match_template_evaltype, EvalType, FieldTypeAccessor, + EvalType, FieldTypeAccessor, + codec::{Result, mysql::decimal::DECIMAL_STRUCT_SIZE}, + match_template_collator, match_template_evaltype, }; /// A vector value container, a.k.a. column, for all concrete eval types. @@ -482,8 +483,8 @@ impl VectorValue { output: &mut Vec, ) -> Result<()> { use crate::{ - codec::{collation::Collator, datum_codec::EvaluableDatumEncoder}, Collation, + codec::{collation::Collator, datum_codec::EvaluableDatumEncoder}, }; match self { diff --git a/components/tidb_query_datatype/src/codec/datum.rs b/components/tidb_query_datatype/src/codec/datum.rs index 9da537a2c09..e508c40d39c 100644 --- a/components/tidb_query_datatype/src/codec/datum.rs +++ b/components/tidb_query_datatype/src/codec/datum.rs @@ -4,7 +4,7 @@ use std::{ borrow::Cow, cmp::Ordering, fmt::{self, Debug, Display, Formatter}, - i64, str, + str, }; use codec::{ @@ -16,20 +16,20 @@ use mysql::VectorFloat32; use tikv_util::{codec::BytesSlice, escape}; use super::{ + Result, mysql::{ - self, parse_json_path_expr, Decimal, DecimalDecoder, DecimalEncoder, Duration, Enum, Json, - JsonDecoder, JsonEncoder, PathExpression, Set, Time, VectorFloat32Decoder, - VectorFloat32Encoder, DEFAULT_FSP, MAX_FSP, + self, DEFAULT_FSP, Decimal, DecimalDecoder, DecimalEncoder, Duration, Enum, Json, + JsonDecoder, JsonEncoder, MAX_FSP, PathExpression, Set, Time, VectorFloat32Decoder, + VectorFloat32Encoder, parse_json_path_expr, }, - Result, }; use crate::{ + FieldTypeTp, codec::{ - convert::{ConvertTo, ToInt}, + convert::{ConvertTo, ToInt, ToStringValue}, data_type::AsMySqlBool, }, expr::EvalContext, - FieldTypeTp, }; pub const NIL_FLAG: u8 = 0; @@ -409,9 +409,9 @@ impl Datum { Datum::Time(t) => format!("{}", t), Datum::Dur(ref d) => format!("{}", d), Datum::Dec(ref d) => format!("{}", d), - Datum::Json(ref d) => d.to_string(), - Datum::Enum(ref e) => e.to_string(), - Datum::Set(ref s) => s.to_string(), + Datum::Json(ref d) => d.to_string_value(), + Datum::Enum(ref e) => e.to_string_value(), + Datum::Set(ref s) => s.to_string_value(), ref d => return Err(invalid_type!("can't convert {} to string", d)), }; Ok(s) @@ -746,7 +746,7 @@ impl Datum { pub fn checked_rem(self, _: &mut EvalContext, d: Datum) -> Result { match d { Datum::I64(0) | Datum::U64(0) => return Ok(Datum::Null), - Datum::F64(f) if f == 0f64 => return Ok(Datum::Null), + Datum::F64(0f64) => return Ok(Datum::Null), _ => {} } match (self, d) { @@ -1176,14 +1176,11 @@ pub fn skip_n(buf: &mut &[u8], n: usize) -> Result<()> { #[cfg(test)] mod tests { - use std::{ - cmp::Ordering, i16, i32, i64, i8, slice::from_ref, str::FromStr, sync::Arc, u16, u32, u64, - u8, - }; + use std::{cmp::Ordering, slice::from_ref, str::FromStr, sync::Arc}; use super::*; use crate::{ - codec::mysql::{Decimal, Duration, Time, MAX_FSP}, + codec::mysql::{Decimal, Duration, MAX_FSP, Time}, expr::{EvalConfig, EvalContext}, }; diff --git a/components/tidb_query_datatype/src/codec/datum_codec.rs b/components/tidb_query_datatype/src/codec/datum_codec.rs index 930a9986a86..c4be8f4af3f 100644 --- a/components/tidb_query_datatype/src/codec/datum_codec.rs +++ b/components/tidb_query_datatype/src/codec/datum_codec.rs @@ -9,16 +9,15 @@ use tipb::FieldType; use super::data_type::*; use crate::{ + FieldTypeAccessor, FieldTypeTp, codec::{ - datum, + Error, Result, datum, mysql::{ DecimalDecoder, DecimalEncoder, DurationDecoder, EnumDecoder, EnumEncoder, JsonDecoder, JsonEncoder, TimeDecoder, VectorFloat32Decoder, VectorFloat32Encoder, }, - Error, Result, }, expr::EvalContext, - FieldTypeAccessor, FieldTypeTp, }; /// A decoder to decode the payload part of a datum. @@ -606,49 +605,49 @@ pub trait RawDatumDecoder { fn decode(self, field_type: &FieldType, ctx: &mut EvalContext) -> Result>; } -impl<'a> RawDatumDecoder for &'a [u8] { +impl RawDatumDecoder for &[u8] { fn decode(self, _field_type: &FieldType, _ctx: &mut EvalContext) -> Result> { decode_int_datum(self) } } -impl<'a> RawDatumDecoder for &'a [u8] { +impl RawDatumDecoder for &[u8] { fn decode(self, field_type: &FieldType, _ctx: &mut EvalContext) -> Result> { decode_real_datum(self, field_type) } } -impl<'a> RawDatumDecoder for &'a [u8] { +impl RawDatumDecoder for &[u8] { fn decode(self, _field_type: &FieldType, _ctx: &mut EvalContext) -> Result> { decode_decimal_datum(self) } } -impl<'a> RawDatumDecoder for &'a [u8] { +impl RawDatumDecoder for &[u8] { fn decode(self, _field_type: &FieldType, _ctx: &mut EvalContext) -> Result> { decode_bytes_datum(self) } } -impl<'a> RawDatumDecoder for &'a [u8] { +impl RawDatumDecoder for &[u8] { fn decode(self, field_type: &FieldType, ctx: &mut EvalContext) -> Result> { decode_date_time_datum(self, field_type, ctx) } } -impl<'a> RawDatumDecoder for &'a [u8] { +impl RawDatumDecoder for &[u8] { fn decode(self, field_type: &FieldType, _ctx: &mut EvalContext) -> Result> { decode_duration_datum(self, field_type) } } -impl<'a> RawDatumDecoder for &'a [u8] { +impl RawDatumDecoder for &[u8] { fn decode(self, _field_type: &FieldType, _ctx: &mut EvalContext) -> Result> { decode_json_datum(self) } } -impl<'a> RawDatumDecoder for &'a [u8] { +impl RawDatumDecoder for &[u8] { fn decode( self, _field_type: &FieldType, @@ -658,13 +657,13 @@ impl<'a> RawDatumDecoder for &'a [u8] { } } -impl<'a> RawDatumDecoder for &'a [u8] { +impl RawDatumDecoder for &[u8] { fn decode(self, field_type: &FieldType, _ctx: &mut EvalContext) -> Result> { decode_enum_datum(self, field_type) } } -impl<'a> RawDatumDecoder for &'a [u8] { +impl RawDatumDecoder for &[u8] { fn decode(self, _field_type: &FieldType, _ctx: &mut EvalContext) -> Result> { unimplemented!() } diff --git a/components/tidb_query_datatype/src/codec/mysql/binary_literal.rs b/components/tidb_query_datatype/src/codec/mysql/binary_literal.rs index 3ab44ad40df..bcde61f5823 100644 --- a/components/tidb_query_datatype/src/codec/mysql/binary_literal.rs +++ b/components/tidb_query_datatype/src/codec/mysql/binary_literal.rs @@ -2,11 +2,12 @@ use std::{ cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd}, + fmt, string::ToString, }; use crate::{ - codec::{error::Error, Result}, + codec::{Result, error::Error}, expr::EvalContext, }; @@ -171,12 +172,12 @@ impl BinaryLiteral { } } -impl ToString for BinaryLiteral { - fn to_string(&self) -> String { +impl fmt::Display for BinaryLiteral { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if self.0.is_empty() { - return String::new(); + return Ok(()); } - format!("0x{}", hex::encode(self.0.as_slice())) + write!(f, "0x{}", hex::encode(self.0.as_slice())) } } diff --git a/components/tidb_query_datatype/src/codec/mysql/decimal.rs b/components/tidb_query_datatype/src/codec/mysql/decimal.rs index 8d3e291fa4e..03be4558233 100644 --- a/components/tidb_query_datatype/src/codec/mysql/decimal.rs +++ b/components/tidb_query_datatype/src/codec/mysql/decimal.rs @@ -3,13 +3,12 @@ use std::{ cmp, cmp::Ordering, - fmt::{self, Display, Formatter}, + fmt, hash::{Hash, Hasher}, intrinsics::copy_nonoverlapping, mem, ops::{Add, Deref, DerefMut, Div, Mul, Neg, Rem, Sub}, str::{self, FromStr}, - string::ToString, }; use codec::prelude::*; @@ -17,10 +16,10 @@ use tikv_util::escape; use crate::{ codec::{ - convert::{self, ConvertTo}, + Error, Result, TEN_POW, + convert::{self, ConvertTo, ToStringValue}, data_type::*, mysql::DEFAULT_DIV_FRAC_INCR, - Error, Result, TEN_POW, }, expr::EvalContext, }; @@ -157,7 +156,11 @@ macro_rules! word_cnt { // when $len is negative and $t is unsigned 0 as $t } else { - ($len as $t + DIGITS_PER_WORD as $t - 1) / (DIGITS_PER_WORD as $t) + // feature `int_roundings` is not stable. + #[allow(clippy::manual_div_ceil)] + { + ($len as $t + DIGITS_PER_WORD as $t - 1) / (DIGITS_PER_WORD as $t) + } } }}; } @@ -996,9 +999,9 @@ impl Decimal { /// Given a precision count 'prec', get: /// 1. the index of first non-zero word in self.word_buf to hold the - /// leading 'prec' number of digits + /// leading 'prec' number of digits /// 2. the number of remained digits if we remove all leading zeros for the - /// leading 'prec' number of digits + /// leading 'prec' number of digits fn remove_leading_zeroes(&self, prec: u8) -> (usize, u8) { let mut cnt = prec; let mut i = ((cnt + DIGITS_PER_WORD - 1) % DIGITS_PER_WORD) + 1; @@ -1775,7 +1778,7 @@ impl ConvertTo for Decimal { /// Port from TiDB's MyDecimal::ToFloat64. #[inline] fn convert(&self, _: &mut EvalContext) -> Result { - let r = self.to_string().parse::(); + let r = self.to_string_value().parse::(); debug_assert!(r.is_ok()); Ok(r?) } @@ -1919,8 +1922,8 @@ impl FromStr for Decimal { } } -impl ToString for Decimal { - fn to_string(&self) -> String { +impl ToStringValue for Decimal { + fn to_string_value(&self) -> String { let (mut buf, word_start_idx, int_len, int_cnt, frac_cnt) = self.prepare_buf(); if self.negative { buf.push(b'-'); @@ -1964,13 +1967,13 @@ impl ToString for Decimal { } } -impl Display for Decimal { - fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { +impl fmt::Display for Decimal { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { let mut dec = *self; dec = dec .round(self.result_frac_cnt as i8, RoundMode::HalfEven) .unwrap(); - fmt.write_str(&dec.to_string()) + fmt.write_str(&dec.to_string_value()) } } @@ -2334,7 +2337,7 @@ impl Ord for Decimal { } } -impl<'a, 'b> Add<&'a Decimal> for &'b Decimal { +impl<'a> Add<&'a Decimal> for &Decimal { type Output = Res; fn add(self, rhs: &'a Decimal) -> Res { @@ -2349,7 +2352,7 @@ impl<'a, 'b> Add<&'a Decimal> for &'b Decimal { } } -impl<'a, 'b> Sub<&'a Decimal> for &'b Decimal { +impl<'a> Sub<&'a Decimal> for &Decimal { type Output = Res; fn sub(self, rhs: &'a Decimal) -> Res { @@ -2364,7 +2367,7 @@ impl<'a, 'b> Sub<&'a Decimal> for &'b Decimal { } } -impl<'a, 'b> Mul<&'a Decimal> for &'b Decimal { +impl<'a> Mul<&'a Decimal> for &Decimal { type Output = Res; fn mul(self, rhs: &'a Decimal) -> Res { @@ -2372,7 +2375,7 @@ impl<'a, 'b> Mul<&'a Decimal> for &'b Decimal { } } -impl<'a, 'b> Div<&'a Decimal> for &'b Decimal { +impl<'a> Div<&'a Decimal> for &Decimal { type Output = Option>; fn div(self, rhs: &'a Decimal) -> Self::Output { @@ -2389,7 +2392,7 @@ impl Rem for Decimal { } } -impl<'a, 'b> Rem<&'a Decimal> for &'b Decimal { +impl<'a> Rem<&'a Decimal> for &Decimal { type Output = Option>; fn rem(self, rhs: &'a Decimal) -> Self::Output { let result_frac_cnt = cmp::max(self.result_frac_cnt, rhs.result_frac_cnt); @@ -2460,7 +2463,7 @@ mod tests { for (num, exp) in cases { let dec: Decimal = num.into(); - let dec_str = format!("{}", dec); + let dec_str = dec.to_string_value(); assert_eq!(dec_str, exp); } } @@ -2475,7 +2478,7 @@ mod tests { for (num, exp) in cases { let dec: Decimal = num.into(); - let dec_str = format!("{}", dec); + let dec_str = dec.to_string_value(); assert_eq!(dec_str, exp); } } @@ -2521,8 +2524,8 @@ mod tests { let log = format!( "input: {}, expect: {:?}, output: {:?}", input, - expect.as_ref().map(|x| x.to_string()), - r.as_ref().map(|x| x.to_string()) + expect.as_ref().map(|x| x.to_string_value()), + r.as_ref().map(|x| x.to_string_value()) ); match expect { Err(e) => { @@ -2620,7 +2623,7 @@ mod tests { let mut ctx = EvalContext::default(); for (dec_str, exp) in cases { let dec = dec_str.parse::().unwrap(); - let res = format!("{}", dec); + let res = dec.to_string_value(); assert_eq!(res, dec_str); let f: f64 = dec.convert(&mut ctx).unwrap(); @@ -2929,7 +2932,7 @@ mod tests { .unwrap() .unwrap(); let shifted = dec.shift_with_word_buf_len(shift, word_buf_len); - let res = shifted.map(|d| d.to_string()); + let res = shifted.map(|d| d.to_string_value()); assert_eq!(res, exp.map(ToOwned::to_owned)); } } @@ -2982,15 +2985,15 @@ mod tests { let dec = dec_str.parse::().unwrap(); let round_dec = dec.round(scale, RoundMode::HalfEven); assert_eq!(round_dec.frac_cnt, round_dec.result_frac_cnt); - let res = round_dec.map(|d| d.to_string()); + let res = round_dec.map(|d| d.to_string_value()); assert_eq!(res, half_exp.map(|s| s.to_owned())); let round_dec = dec.round(scale, RoundMode::Truncate); assert_eq!(round_dec.frac_cnt, round_dec.result_frac_cnt); - let res = round_dec.map(|d| d.to_string()); + let res = round_dec.map(|d| d.to_string_value()); assert_eq!(res, trunc_exp.map(|s| s.to_owned())); let round_dec = dec.round(scale, RoundMode::Ceiling); assert_eq!(round_dec.frac_cnt, round_dec.result_frac_cnt); - let res = round_dec.map(|d| d.to_string()); + let res = round_dec.map(|d| d.to_string_value()); assert_eq!(res, ceil_exp.map(|s| s.to_owned())); } } @@ -3078,7 +3081,7 @@ mod tests { for (word_buf_len, dec, exp) in cases { let d = Decimal::from_bytes_with_word_buf(dec, word_buf_len).unwrap(); - let res = d.map(|d| d.to_string()); + let res = d.map(|d| d.to_string_value()); assert_eq!(res, exp.map(|s| s.to_owned())); } @@ -3140,7 +3143,7 @@ mod tests { let mut buf = vec![]; let res = buf.write_decimal(&dec, prec, frac).unwrap(); let decoded = buf.as_slice().read_decimal().unwrap(); - let res = res.map(|_| decoded.to_string()); + let res = res.map(|_| decoded.to_string_value()); assert_eq!(res, exp.map(|s| s.to_owned())); } } @@ -3251,7 +3254,7 @@ mod tests { for (prec, frac, exp) in cases { let dec = super::max_decimal(prec, frac); - let res = dec.to_string(); + let res = dec.to_string_value(); assert_eq!(&res, exp); } } @@ -3298,12 +3301,12 @@ mod tests { let rhs = rhs_str.parse::().unwrap(); let res_dec = &lhs + &rhs; - let res = res_dec.map(|s| s.to_string()); + let res = res_dec.map(|s| s.to_string_value()); let exp_str = exp.map(|s| s.to_owned()); assert_eq!(res, exp_str); let res_dec = &rhs + &lhs; - let res = res_dec.map(|s| s.to_string()); + let res = res_dec.map(|s| s.to_string_value()); assert_eq!(res, exp_str); } } @@ -3346,7 +3349,7 @@ mod tests { let lhs = lhs_str.parse::().unwrap(); let rhs = rhs_str.parse::().unwrap(); let res_dec = &lhs - &rhs; - let res = res_dec.map(|s| s.to_string()); + let res = res_dec.map(|s| s.to_string_value()); assert_eq!(res, exp.map(|s| s.to_owned())); } } @@ -3380,10 +3383,10 @@ mod tests { let lhs: Decimal = lhs_str.parse().unwrap(); let rhs: Decimal = rhs_str.parse().unwrap(); let exp = exp_str.map(|s| s.to_owned()); - let res = (&lhs * &rhs).map(|d| d.to_string()); + let res = (&lhs * &rhs).map(|d| d.to_string_value()); assert_eq!(res, exp); - let res = (&rhs * &lhs).map(|d| d.to_string()); + let res = (&rhs * &lhs).map(|d| d.to_string_value()); assert_eq!(res, exp); } } @@ -3405,11 +3408,11 @@ mod tests { let lhs: Decimal = lhs_str.parse().unwrap(); let rhs: Decimal = rhs_str.parse().unwrap(); let exp = exp_str.map(|s| s.to_owned()); - let res = (&lhs * &rhs).map(|d| d.to_string()); + let res = (&lhs * &rhs).map(|d| d.to_string_value()); assert_eq!(res, exp); let exp = rev_exp_str.map(|s| s.to_owned()); - let res = (&rhs * &lhs).map(|d| d.to_string()); + let res = (&rhs * &lhs).map(|d| d.to_string_value()); assert_eq!(res, exp); } } @@ -3602,12 +3605,12 @@ mod tests { for (frac_incr, lhs_str, rhs_str, div_exp, rem_exp) in cases { let lhs: Decimal = lhs_str.parse().unwrap(); let rhs: Decimal = rhs_str.parse().unwrap(); - let res = - super::do_div_mod(&lhs, &rhs, frac_incr, false).map(|d| d.unwrap().to_string()); + let res = super::do_div_mod(&lhs, &rhs, frac_incr, false) + .map(|d| d.unwrap().to_string_value()); assert_eq!(res, div_exp.map(|s| s.to_owned())); - let res = - super::do_div_mod(&lhs, &rhs, frac_incr, true).map(|d| d.unwrap().to_string()); + let res = super::do_div_mod(&lhs, &rhs, frac_incr, true) + .map(|d| d.unwrap().to_string_value()); assert_eq!(res, rem_exp.map(|s| s.to_owned())); } @@ -3624,7 +3627,7 @@ mod tests { for (lhs_str, rhs_str, div_exp) in div_cases { let lhs: Decimal = lhs_str.parse().unwrap(); let rhs: Decimal = rhs_str.parse().unwrap(); - let res = (&lhs / &rhs).unwrap().map(|d| d.to_string()); + let res = (&lhs / &rhs).unwrap().map(|d| d.to_string_value()); assert_eq!(res, div_exp.map(|s| s.to_owned())) } @@ -3632,7 +3635,7 @@ mod tests { for (lhs_str, rhs_str, rem_exp) in rem_cases { let lhs: Decimal = lhs_str.parse().unwrap(); let rhs: Decimal = rhs_str.parse().unwrap(); - let res = (lhs % rhs).unwrap().map(|d| d.to_string()); + let res = (lhs % rhs).unwrap().map(|d| d.to_string_value()); assert_eq!(res, rem_exp.map(|s| s.to_owned())) } } @@ -3653,12 +3656,12 @@ mod tests { for (pos, neg) in cases { let pos_dec: Decimal = pos.parse().unwrap(); let res = -pos_dec; - assert_eq!(format!("{}", res), neg); + assert_eq!(res.to_string_value(), neg); assert!((&pos_dec + &res).is_zero()); let neg_dec: Decimal = neg.parse().unwrap(); let res = -neg_dec; - assert_eq!(format!("{}", res), pos); + assert_eq!(res.to_string_value(), pos); assert!((&neg_dec + &res).is_zero()); } @@ -3690,12 +3693,12 @@ mod tests { for (prec, frac, exp) in cases { let positive = super::max_or_min_dec(false, prec, frac); - let res = positive.to_string(); + let res = positive.to_string_value(); assert_eq!(&res, exp); let negative = super::max_or_min_dec(true, prec, frac); let mut negative_exp = String::from("-"); negative_exp.push_str(exp); - let res = negative.to_string(); + let res = negative.to_string_value(); assert_eq!(res, negative_exp); } } @@ -3771,7 +3774,11 @@ mod tests { ]; for (s, expect) in cases { let got: Decimal = s.convert(&mut ctx).unwrap(); - assert_eq!(got, expect, "from {:?}, expect: {} got: {}", s, expect, got); + assert_eq!( + got, expect, + "from {:?}, expect: {:?} got: {:?}", + s, expect, got + ); } // OVERFLOWING @@ -3784,7 +3791,7 @@ mod tests { let mut ctx = EvalContext::new(Arc::new(EvalConfig::from_flag(Flag::OVERFLOW_AS_WARNING))); let val: Decimal = big.as_bytes().convert(&mut ctx).unwrap(); let max = max_decimal(WORD_BUF_LEN * DIGITS_PER_WORD, 0); - assert_eq!(val, max, "expect: {}, got: {}", val, max); + assert_eq!(val, max, "expect: {:?}, got: {:?}", val, max); assert_eq!(ctx.warnings.warning_cnt, 1); assert_eq!(ctx.warnings.warnings[0].get_code(), ERR_DATA_OUT_OF_RANGE); @@ -3814,7 +3821,11 @@ mod tests { EvalConfig::from_flag(Flag::TRUNCATE_AS_WARNING), )); let got: Decimal = s.convert(&mut truncate_as_warning_ctx).unwrap(); - assert_eq!(got, expect, "from {:?}, expect: {} got: {}", s, expect, got); + assert_eq!( + got, expect, + "from {:?}, expect: {:?} got: {:?}", + s, expect, got + ); assert_eq!(truncate_as_warning_ctx.warnings.warning_cnt, 1); } } diff --git a/components/tidb_query_datatype/src/codec/mysql/duration.rs b/components/tidb_query_datatype/src/codec/mysql/duration.rs index 6b08c756d24..719ee5522ec 100644 --- a/components/tidb_query_datatype/src/codec/mysql/duration.rs +++ b/components/tidb_query_datatype/src/codec/mysql/duration.rs @@ -8,16 +8,16 @@ use std::{ use codec::prelude::*; use tipb::FieldType; -use super::{check_fsp, Decimal, DEFAULT_FSP}; +use super::{DEFAULT_FSP, Decimal, check_fsp}; use crate::{ + FieldTypeAccessor, codec::{ + Error, Result, TEN_POW, convert::ConvertTo, error::{ERR_DATA_OUT_OF_RANGE, ERR_TRUNCATE_WRONG_VALUE}, - mysql::{Time as DateTime, TimeType, MAX_FSP, MIN_FSP}, - Error, Result, TEN_POW, + mysql::{MAX_FSP, MIN_FSP, Time as DateTime, TimeType}, }, expr::EvalContext, - FieldTypeAccessor, }; pub const NANOS_PER_MICRO: i64 = 1_000; @@ -95,9 +95,9 @@ fn check_nanos(nanos: i64) -> Result { mod parser { use nom::{ + IResult, character::complete::{anychar, char, digit0, digit1, space0, space1}, combinator::opt, - IResult, }; use super::*; diff --git a/components/tidb_query_datatype/src/codec/mysql/enums.rs b/components/tidb_query_datatype/src/codec/mysql/enums.rs index 6c39d7f8a95..5d080698606 100644 --- a/components/tidb_query_datatype/src/codec/mysql/enums.rs +++ b/components/tidb_query_datatype/src/codec/mysql/enums.rs @@ -9,9 +9,12 @@ use codec::prelude::*; use tipb::FieldType; use crate::{ - codec::{convert::ToInt, Result}, - expr::EvalContext, FieldTypeTp, + codec::{ + Result, + convert::{ToInt, ToStringValue}, + }, + expr::EvalContext, }; #[derive(Clone, Debug)] @@ -185,8 +188,8 @@ impl<'a> ToInt for EnumRef<'a> { } } -impl<'a> ToString for EnumRef<'a> { - fn to_string(&self) -> String { +impl<'a> ToStringValue for EnumRef<'a> { + fn to_string_value(&self) -> String { String::from_utf8_lossy(self.name).to_string() } } diff --git a/components/tidb_query_datatype/src/codec/mysql/json/binary.rs b/components/tidb_query_datatype/src/codec/mysql/json/binary.rs index 5028967ac59..4ac4fe6efec 100644 --- a/components/tidb_query_datatype/src/codec/mysql/json/binary.rs +++ b/components/tidb_query_datatype/src/codec/mysql/json/binary.rs @@ -4,8 +4,8 @@ use std::convert::TryInto; use codec::number::NumberCodec; -use super::{constants::*, JsonRef, JsonType, ERR_CONVERT_FAILED}; -use crate::codec::{mysql::json::path_expr::ArrayIndex, Result}; +use super::{ERR_CONVERT_FAILED, JsonRef, JsonType, constants::*}; +use crate::codec::{Result, convert::ToStringValue, mysql::json::path_expr::ArrayIndex}; impl<'a> JsonRef<'a> { /// Gets the index from the ArrayIndex @@ -136,7 +136,7 @@ impl<'a> JsonRef<'a> { _ => Err(invalid_type!( "{} from {} to literal", ERR_CONVERT_FAILED, - self.to_string() + self.to_string_value() )), } } diff --git a/components/tidb_query_datatype/src/codec/mysql/json/comparison.rs b/components/tidb_query_datatype/src/codec/mysql/json/comparison.rs index d9104385bc6..637cd972b18 100644 --- a/components/tidb_query_datatype/src/codec/mysql/json/comparison.rs +++ b/components/tidb_query_datatype/src/codec/mysql/json/comparison.rs @@ -5,7 +5,8 @@ use std::{ f64, }; -use super::{super::Result, constants::*, Json, JsonRef, JsonType, ERR_CONVERT_FAILED}; +use super::{super::Result, ERR_CONVERT_FAILED, Json, JsonRef, JsonType, constants::*}; +use crate::codec::convert::ToStringValue; fn compare(x: T, y: T) -> Ordering { x.cmp(&y) @@ -57,7 +58,7 @@ impl<'a> JsonRef<'a> { _ => Err(invalid_type!( "{} from {} to f64", ERR_CONVERT_FAILED, - self.to_string() + self.to_string_value() )), } } @@ -74,7 +75,7 @@ impl<'a> Ord for JsonRef<'a> { impl<'a> PartialEq for JsonRef<'a> { fn eq(&self, right: &JsonRef<'_>) -> bool { self.partial_cmp(right) - .map_or(false, |r| r == Ordering::Equal) + .is_some_and(|r| r == Ordering::Equal) } } impl<'a> PartialOrd for JsonRef<'a> { diff --git a/components/tidb_query_datatype/src/codec/mysql/json/jcodec.rs b/components/tidb_query_datatype/src/codec/mysql/json/jcodec.rs index f76b29790f9..746f26b4abd 100644 --- a/components/tidb_query_datatype/src/codec/mysql/json/jcodec.rs +++ b/components/tidb_query_datatype/src/codec/mysql/json/jcodec.rs @@ -4,10 +4,10 @@ use std::{collections::BTreeMap, convert::TryInto, f64, str}; use codec::{number::NumberCodec, prelude::*}; -use super::{constants::*, Json, JsonRef, JsonType}; +use super::{Json, JsonRef, JsonType, constants::*}; use crate::{ - codec::{Error, Result}, FieldTypeTp, + codec::{Error, Result}, }; impl<'a> JsonRef<'a> { diff --git a/components/tidb_query_datatype/src/codec/mysql/json/json_extract.rs b/components/tidb_query_datatype/src/codec/mysql/json/json_extract.rs index 7e619e74c32..c4bfe407f52 100644 --- a/components/tidb_query_datatype/src/codec/mysql/json/json_extract.rs +++ b/components/tidb_query_datatype/src/codec/mysql/json/json_extract.rs @@ -4,8 +4,8 @@ use collections::HashSet; use super::{ super::Result, - path_expr::{PathExpression, PathLeg}, Json, JsonRef, JsonType, + path_expr::{PathExpression, PathLeg}, }; use crate::codec::mysql::json::path_expr::{ArrayIndex, ArraySelection, KeySelection}; @@ -200,8 +200,8 @@ mod tests { use super::{ super::path_expr::{ - PathExpressionFlag, PATH_EXPRESSION_CONTAINS_ASTERISK, - PATH_EXPRESSION_CONTAINS_DOUBLE_ASTERISK, + PATH_EXPRESSION_CONTAINS_ASTERISK, PATH_EXPRESSION_CONTAINS_DOUBLE_ASTERISK, + PathExpressionFlag, }, *, }; diff --git a/components/tidb_query_datatype/src/codec/mysql/json/json_keys.rs b/components/tidb_query_datatype/src/codec/mysql/json/json_keys.rs index 68c361321ad..0d787a54109 100644 --- a/components/tidb_query_datatype/src/codec/mysql/json/json_keys.rs +++ b/components/tidb_query_datatype/src/codec/mysql/json/json_keys.rs @@ -2,7 +2,7 @@ use std::str; -use super::{super::Result, path_expr::PathExpression, Json, JsonRef, JsonType}; +use super::{super::Result, Json, JsonRef, JsonType, path_expr::PathExpression}; impl<'a> JsonRef<'a> { /// Evaluates a (possibly empty) list of values and returns a JSON array diff --git a/components/tidb_query_datatype/src/codec/mysql/json/json_length.rs b/components/tidb_query_datatype/src/codec/mysql/json/json_length.rs index f4c120d866f..96efc13665d 100644 --- a/components/tidb_query_datatype/src/codec/mysql/json/json_length.rs +++ b/components/tidb_query_datatype/src/codec/mysql/json/json_length.rs @@ -1,6 +1,6 @@ // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. -use super::{super::Result, path_expr::PathExpression, JsonRef, JsonType}; +use super::{super::Result, JsonRef, JsonType, path_expr::PathExpression}; impl<'a> JsonRef<'a> { fn len(&self) -> i64 { @@ -25,7 +25,7 @@ impl<'a> JsonRef<'a> { #[cfg(test)] mod tests { - use super::super::{path_expr::parse_json_path_expr, Json}; + use super::super::{Json, path_expr::parse_json_path_expr}; #[test] fn test_json_length() { let mut test_cases = vec![ diff --git a/components/tidb_query_datatype/src/codec/mysql/json/json_merge.rs b/components/tidb_query_datatype/src/codec/mysql/json/json_merge.rs index b8c82382d6a..518fe18a0d4 100644 --- a/components/tidb_query_datatype/src/codec/mysql/json/json_merge.rs +++ b/components/tidb_query_datatype/src/codec/mysql/json/json_merge.rs @@ -14,7 +14,7 @@ impl Json { /// 2. adjacent object are merged to a single object; /// 3. a scalar value is autowrapped as an array before merge; /// 4. an adjacent array and object are merged by autowrapping the object as - /// an array. + /// an array. /// /// See `MergeBinary()` in TiDB `json/binary_function.go` #[allow(clippy::comparison_chain)] diff --git a/components/tidb_query_datatype/src/codec/mysql/json/json_modify.rs b/components/tidb_query_datatype/src/codec/mysql/json/json_modify.rs index b359158d06b..f9d59021d8b 100644 --- a/components/tidb_query_datatype/src/codec/mysql/json/json_modify.rs +++ b/components/tidb_query_datatype/src/codec/mysql/json/json_modify.rs @@ -1,6 +1,6 @@ // Copyright 2017 TiKV Project Authors. Licensed under Apache-2.0. -use super::{super::Result, modifier::BinaryModifier, path_expr::PathExpression, Json, JsonRef}; +use super::{super::Result, Json, JsonRef, modifier::BinaryModifier, path_expr::PathExpression}; /// `ModifyType` is for modify a JSON. #[derive(Clone, Copy, Debug, PartialEq)] diff --git a/components/tidb_query_datatype/src/codec/mysql/json/json_remove.rs b/components/tidb_query_datatype/src/codec/mysql/json/json_remove.rs index bcb6fd01716..22d9b407781 100644 --- a/components/tidb_query_datatype/src/codec/mysql/json/json_remove.rs +++ b/components/tidb_query_datatype/src/codec/mysql/json/json_remove.rs @@ -1,6 +1,6 @@ // Copyright 2017 TiKV Project Authors. Licensed under Apache-2.0. -use super::{super::Result, modifier::BinaryModifier, path_expr::PathExpression, Json, JsonRef}; +use super::{super::Result, Json, JsonRef, modifier::BinaryModifier, path_expr::PathExpression}; impl<'a> JsonRef<'a> { /// Removes elements from Json, diff --git a/components/tidb_query_datatype/src/codec/mysql/json/json_unquote.rs b/components/tidb_query_datatype/src/codec/mysql/json/json_unquote.rs index f95c08cf958..5ed25e140df 100644 --- a/components/tidb_query_datatype/src/codec/mysql/json/json_unquote.rs +++ b/components/tidb_query_datatype/src/codec/mysql/json/json_unquote.rs @@ -1,8 +1,9 @@ // Copyright 2017 TiKV Project Authors. Licensed under Apache-2.0. -use std::{char, str, u32}; +use std::{char, str}; use super::{super::Result, JsonRef, JsonType}; +use crate::codec::convert::ToStringValue; const ESCAPED_UNICODE_BYTES_SIZE: usize = 4; @@ -29,12 +30,12 @@ impl<'a> JsonRef<'a> { | JsonType::Timestamp | JsonType::Time | JsonType::Opaque => { - let s = self.to_string(); + let s = self.to_string_value(); // Remove the quotes of output assert!(s.len() > 2); Ok(s[1..s.len() - 1].to_string()) } - _ => Ok(self.to_string()), + _ => Ok(self.to_string_value()), } } } diff --git a/components/tidb_query_datatype/src/codec/mysql/json/mod.rs b/components/tidb_query_datatype/src/codec/mysql/json/mod.rs index e6d2a391fae..fab33adbbb2 100644 --- a/components/tidb_query_datatype/src/codec/mysql/json/mod.rs +++ b/components/tidb_query_datatype/src/codec/mysql/json/mod.rs @@ -80,27 +80,27 @@ pub mod json_unquote; use std::{ collections::BTreeMap, convert::{TryFrom, TryInto}, - str, + fmt, str, }; -use codec::number::{NumberCodec, F64_SIZE, I64_SIZE}; +use codec::number::{F64_SIZE, I64_SIZE, NumberCodec}; use constants::{JSON_LITERAL_FALSE, JSON_LITERAL_NIL, JSON_LITERAL_TRUE}; use tikv_util::is_even; pub use self::{ jcodec::{JsonDatumPayloadChunkEncoder, JsonDecoder, JsonEncoder}, json_modify::ModifyType, - path_expr::{parse_json_path_expr, PathExpression}, + path_expr::{PathExpression, parse_json_path_expr}, }; -use super::super::{datum::Datum, Error, Result}; +use super::super::{Error, Result, datum::Datum}; use crate::{ + FieldTypeTp, codec::{ - convert::ConvertTo, + convert::{ConvertTo, ToStringValue}, data_type::{BytesRef, Decimal, Real}, mysql::{Duration, Time, TimeType}, }, expr::EvalContext, - FieldTypeTp, }; const ERR_CONVERT_FAILED: &str = "Can not covert from "; @@ -300,14 +300,12 @@ pub struct Json { pub value: Vec, } -use std::fmt::{Display, Formatter}; - use codec::prelude::NumberEncoder; use crate::codec::mysql::{TimeDecoder, TimeEncoder}; -impl Display for Json { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { +impl fmt::Display for Json { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let s = serde_json::to_string(&self.as_ref()).unwrap(); write!(f, "{}", s) } @@ -505,7 +503,7 @@ impl<'a> ConvertTo for JsonRef<'a> { .map_or(0f64, |x| if x { 1f64 } else { 0f64 }), JsonType::String => self.get_str_bytes()?.convert(ctx)?, _ => ctx - .handle_truncate_err(Error::truncated_wrong_val("Float", self.to_string())) + .handle_truncate_err(Error::truncated_wrong_val("Float", self.to_string_value())) .map(|_| 0f64)?, }; Ok(d) diff --git a/components/tidb_query_datatype/src/codec/mysql/json/modifier.rs b/components/tidb_query_datatype/src/codec/mysql/json/modifier.rs index 58fe8fbbbcb..dec36b853b6 100644 --- a/components/tidb_query_datatype/src/codec/mysql/json/modifier.rs +++ b/components/tidb_query_datatype/src/codec/mysql/json/modifier.rs @@ -6,10 +6,10 @@ use codec::number::NumberCodec; use super::{ super::Result, + Json, JsonRef, JsonType, constants::*, json_extract::extract_json, path_expr::{PathExpression, PathLeg}, - Json, JsonRef, JsonType, }; use crate::codec::mysql::json::path_expr::{ArraySelection, KeySelection}; @@ -30,7 +30,7 @@ pub struct BinaryModifier<'a> { impl<'a> BinaryModifier<'a> { /// Creates a new `BinaryModifier` from a `JsonRef` - pub fn new(old: JsonRef<'a>) -> BinaryModifier<'_> { + pub fn new(old: JsonRef<'a>) -> BinaryModifier<'a> { Self { // The initial offset is 0 by `as_ref()` call old, @@ -219,7 +219,7 @@ impl<'a> BinaryModifier<'a> { // Returns the old JSON's `JsonType` if the old is untouched or // returns the new appended JSON's `JsonType` if the old has been modified fn rebuild_to(&mut self, buf: &mut Vec) -> Result { - if self.to_be_modified_ptr == self.old.as_ptr() { + if ptr::eq(self.to_be_modified_ptr, self.old.as_ptr()) { // Replace the old directly self.to_be_modified_ptr = ptr::null(); buf.extend_from_slice(&self.new_value.as_ref().unwrap().value); diff --git a/components/tidb_query_datatype/src/codec/mysql/json/path_expr.rs b/components/tidb_query_datatype/src/codec/mysql/json/path_expr.rs index fb707887885..e9ced08558a 100644 --- a/components/tidb_query_datatype/src/codec/mysql/json/path_expr.rs +++ b/components/tidb_query_datatype/src/codec/mysql/json/path_expr.rs @@ -26,6 +26,7 @@ // ``` use nom::{ + IResult, branch::alt, bytes::complete::tag, character::{ @@ -35,7 +36,6 @@ use nom::{ combinator::{map, map_opt}, multi::{many0, many1}, sequence::{delimited, pair, tuple}, - IResult, }; use super::json_unquote::unquote_string; diff --git a/components/tidb_query_datatype/src/codec/mysql/json/serde.rs b/components/tidb_query_datatype/src/codec/mysql/json/serde.rs index 598e27347c8..27f1f026ecf 100644 --- a/components/tidb_query_datatype/src/codec/mysql/json/serde.rs +++ b/components/tidb_query_datatype/src/codec/mysql/json/serde.rs @@ -9,7 +9,7 @@ use serde::{ use serde_json::Serializer as JsonSerializer; use super::{Json, JsonRef, JsonType}; -use crate::codec::Error; +use crate::codec::{Error, convert::ToStringValue}; /// MySQL formatter follows the implementation in TiDB /// https://github.com/pingcap/tidb/blob/master/types/json/binary.go @@ -57,10 +57,10 @@ impl MySqlFormatter { } } -impl<'a> ToString for JsonRef<'a> { +impl<'a> ToStringValue for JsonRef<'a> { /// This function is a simple combination and rewrite of serde_json's /// `to_writer_pretty` - fn to_string(&self) -> String { + fn to_string_value(&self) -> String { let mut writer = Vec::with_capacity(128); let mut ser = JsonSerializer::with_formatter(&mut writer, MySqlFormatter::new()); self.serialize(&mut ser).unwrap(); @@ -144,9 +144,9 @@ impl<'a> Serialize for JsonRef<'a> { } } -impl ToString for Json { - fn to_string(&self) -> String { - self.as_ref().to_string() +impl ToStringValue for Json { + fn to_string_value(&self) -> String { + self.as_ref().to_string_value() } } @@ -255,7 +255,7 @@ mod tests { fn test_from_str_for_object() { let jstr1 = r#"{"a": [1, "2", {"aa": "bb"}, 4.0, null], "c": null,"b": true}"#; let j1: Json = jstr1.parse().unwrap(); - let jstr2 = j1.to_string(); + let jstr2 = j1.to_string_value(); let expect_str = r#"{"a": [1, "2", {"aa": "bb"}, 4.0, null], "b": true, "c": null}"#; assert_eq!(jstr2, expect_str); } @@ -336,7 +336,7 @@ mod tests { ]; for (json, json_str) in legal_cases { - assert_eq!(json.to_string(), json_str); + assert_eq!(json.to_string_value(), json_str); } } } diff --git a/components/tidb_query_datatype/src/codec/mysql/mod.rs b/components/tidb_query_datatype/src/codec/mysql/mod.rs index 2bcfeb8ee23..087e838d54f 100644 --- a/components/tidb_query_datatype/src/codec/mysql/mod.rs +++ b/components/tidb_query_datatype/src/codec/mysql/mod.rs @@ -36,12 +36,12 @@ pub mod time; pub mod vector; pub use self::{ - decimal::{dec_encoded_len, Decimal, DecimalDecoder, DecimalEncoder, Res, RoundMode}, + decimal::{Decimal, DecimalDecoder, DecimalEncoder, Res, RoundMode, dec_encoded_len}, duration::{Duration, DurationDecoder, DurationEncoder}, enums::{Enum, EnumDecoder, EnumEncoder, EnumRef}, json::{ - parse_json_path_expr, Json, JsonDatumPayloadChunkEncoder, JsonDecoder, JsonEncoder, - JsonType, ModifyType, PathExpression, + Json, JsonDatumPayloadChunkEncoder, JsonDecoder, JsonEncoder, JsonType, ModifyType, + PathExpression, parse_json_path_expr, }, set::{Set, SetRef}, time::{Time, TimeDecoder, TimeEncoder, TimeType, Tz}, diff --git a/components/tidb_query_datatype/src/codec/mysql/time/extension.rs b/components/tidb_query_datatype/src/codec/mysql/time/extension.rs index 9289625ad84..bd93e0117d2 100644 --- a/components/tidb_query_datatype/src/codec/mysql/time/extension.rs +++ b/components/tidb_query_datatype/src/codec/mysql/time/extension.rs @@ -2,7 +2,7 @@ use chrono::Weekday; -use super::{weekmode::WeekMode, Time}; +use super::{Time, weekmode::WeekMode}; pub trait WeekdayExtension { fn name(&self) -> &'static str; diff --git a/components/tidb_query_datatype/src/codec/mysql/time/interval.rs b/components/tidb_query_datatype/src/codec/mysql/time/interval.rs index 2f74c919fc0..13ce53db8d9 100644 --- a/components/tidb_query_datatype/src/codec/mysql/time/interval.rs +++ b/components/tidb_query_datatype/src/codec/mysql/time/interval.rs @@ -7,9 +7,9 @@ use regex::Regex; use crate::{ codec::{ - data_type::{BytesRef, Decimal, Real}, - mysql::{duration::*, RoundMode, DEFAULT_FSP, MAX_FSP, MIN_FSP}, Error, Result, + data_type::{BytesRef, Decimal, Real}, + mysql::{DEFAULT_FSP, MAX_FSP, MIN_FSP, RoundMode, duration::*}, }, expr::EvalContext, }; @@ -472,7 +472,7 @@ impl Interval { // Populate fields in reverse order for (i, &matched) in matches.iter().rev().enumerate() { - fields[index as usize - i] = &matched; + fields[index as usize - i] = matched; } // Helper to parse integer fields and handle errors diff --git a/components/tidb_query_datatype/src/codec/mysql/time/mod.rs b/components/tidb_query_datatype/src/codec/mysql/time/mod.rs index efc3d80afab..e5ad6a9d34e 100644 --- a/components/tidb_query_datatype/src/codec/mysql/time/mod.rs +++ b/components/tidb_query_datatype/src/codec/mysql/time/mod.rs @@ -21,14 +21,14 @@ use tipb::FieldType; pub use self::{extension::*, interval::IntervalUnit, tz::Tz, weekmode::WeekMode}; use crate::{ + FieldTypeAccessor, FieldTypeTp, codec::{ + Error, Result, TEN_POW, convert::ConvertTo, data_type::Real, - mysql::{check_fsp, duration::*, Decimal, Duration, Res, DEFAULT_FSP, MAX_FSP}, - Error, Result, TEN_POW, + mysql::{DEFAULT_FSP, Decimal, Duration, MAX_FSP, Res, check_fsp, duration::*}, }, expr::{EvalContext, Flag, SqlMode}, - FieldTypeAccessor, FieldTypeTp, }; const MIN_TIMESTAMP: i64 = 0; @@ -1485,7 +1485,7 @@ impl Time { round: bool, ) -> Result