degrade TiKV Error log level for false alarms(raftstore-proxy-8.5)

Makefile

@@ -226,7 +226,8 @@ unset-override:
 
 pre-format: unset-override
 	@rustup component add rustfmt
-	@cargo install --force --locked -q cargo-sort
+	@cargo install cargo-sort --version 1.0.9
+	#@cargo install --force --locked -q cargo-sort
 
 pre-format-fast: unset-override
 	@rustup component add rustfmt

components/engine_panic/Cargo.toml

@@ -10,9 +10,9 @@ license = "Apache-2.0"
 testexport = []
 
 [dependencies]
+encryption = { workspace = true }
 engine_traits = { workspace = true }
 kvproto = { workspace = true }
-encryption = { workspace = true }
 raft = { workspace = true }
 tracker = { workspace = true }
 txn_types = { workspace = true }

components/engine_rocks/Cargo.toml

@@ -61,6 +61,6 @@ package = "rocksdb"
 features = ["encryption"]
 
 [dev-dependencies]
+proptest = "1.0.0"
 rand = "0.8"
 toml = "0.5"
-proptest = "1.0.0"

components/engine_traits/Cargo.toml

@@ -11,6 +11,7 @@ testexport = []
 
 [dependencies]
 collections = { workspace = true }
+encryption = { workspace = true }
 error_code = { workspace = true }
 fail = "0.5"
 file_system = { workspace = true }
@@ -19,7 +20,6 @@ kvproto = { workspace = true }
 log_wrappers = { workspace = true }
 protobuf = "2"
 raft = { workspace = true }
-encryption = { workspace = true }
 serde = "1.0"
 slog = { workspace = true }
 slog-global = { workspace = true }

components/hybrid_engine/Cargo.toml

@@ -6,27 +6,27 @@ publish = false
 license = "Apache-2.0"
 
 [dependencies]
-engine_traits = { workspace = true }
-txn_types = { workspace = true }
-tikv_util = { workspace = true }
+crossbeam = { workspace = true }
 engine_rocks = { workspace = true }
-online_config = { workspace = true }
+engine_traits = { workspace = true }
 in_memory_engine = { workspace = true }
-slog = { workspace = true }
-slog-global = { workspace = true }
-tempfile = "3.0"
+keys = { workspace = true }
+kvproto = { workspace = true }
+lazy_static = "1.4.0"
+online_config = { workspace = true }
 prometheus = { version = "0.13", default-features = false, features = [
     "nightly",
 ] }
 prometheus-static-metric = "0.5"
-lazy_static = "1.4.0"
-crossbeam = { workspace = true }
-raftstore = { workspace = true }
 raft = { workspace = true }
-kvproto = { workspace = true }
-keys = { workspace = true }
+raftstore = { workspace = true }
+slog = { workspace = true }
+slog-global = { workspace = true }
+tempfile = "3.0"
+tikv_util = { workspace = true }
+txn_types = { workspace = true }
 
 [dev-dependencies]
+fail = { version = "0.5", features = ["failpoints"] }
 tempfile = "3.0"
 test_util = { workspace = true }
-fail = { version = "0.5", features = ["failpoints"] }

components/in_memory_engine/Cargo.toml

@@ -20,44 +20,44 @@ path = "benches/load_region.rs"
 harness = false
 
 [dependencies]
-engine_traits = { workspace = true }
-collections = { workspace = true }
-crossbeam-skiplist = { workspace = true }
 bytes = "1.0"
+collections = { workspace = true }
 crossbeam = { workspace = true }
+crossbeam-skiplist = { workspace = true }
+dashmap = "5.1"
+engine_rocks = { workspace = true }
+engine_traits = { workspace = true }
+fail = "0.5"
 futures = { version = "0.3", features = ["compat"] }
-tikv_util = { workspace = true }
-txn_types = { workspace = true }
+hex = "0.4"
+keys = { workspace = true }
 kvproto = { workspace = true }
+lazy_static = "1.4.0"
+libc = "0.2"
 log_wrappers = { workspace = true }
+online_config = { workspace = true }
+parking_lot = "0.12"
 pd_client = { workspace = true }
+prometheus = { version = "0.13", default-features = false, features = ["nightly"] }
+prometheus-static-metric = "0.5"
 raftstore = { workspace = true }
-dashmap = "5.1"
+rand = "0.8"
 security = { workspace = true }
 serde = "1.0"
 serde_json = "1.0"
-slog-global = { workspace = true }
 slog = { workspace = true }
+slog-global = { workspace = true }
+smallvec = "1.4"
 strum = { version = "0.20", features = ["derive"] }
-engine_rocks = { workspace = true }
-fail = "0.5"
-yatp = { workspace = true }
-parking_lot = "0.12"
-keys = { workspace = true }
-prometheus = { version = "0.13", default-features = false, features = ["nightly"] }
-prometheus-static-metric = "0.5"
-lazy_static = "1.4.0"
-hex = "0.4"
 thiserror = "1.0"
-online_config = { workspace = true }
-libc = "0.2"
-rand = "0.8"
+tikv_util = { workspace = true }
 tokio = { version = "1.5", features = ["rt-multi-thread"] }
-smallvec = "1.4"
+txn_types = { workspace = true }
+yatp = { workspace = true }
 
 [dev-dependencies]
 criterion = "0.3"
+proptest = "1.0.0"
 tempfile = "3.0"
 test_pd = { workspace = true }
 test_util = { workspace = true }
-proptest = "1.0.0"

components/in_memory_engine/src/region_label.rs

@@ -15,7 +15,7 @@ use pd_client::{
     Error as PdError, PdClient, RpcClient, REGION_LABEL_PATH_PREFIX,
 };
 use serde::{Deserialize, Serialize};
-use tikv_util::{error, info, timer::GLOBAL_TIMER_HANDLE};
+use tikv_util::{error, info, timer::GLOBAL_TIMER_HANDLE, warn};
 
 /// RegionLabel is the label of a region. This struct is partially copied from
 /// https://github.com/tikv/pd/blob/783d060861cef37c38cbdcab9777fe95c17907fe/server/schedule/labeler/rules.go#L31.
@@ -239,7 +239,7 @@ impl RegionLabelService {
                         });
                     }
                     Err(PdError::DataCompacted(msg)) => {
-                        error!("ime required revision has been compacted"; "err" => ?msg);
+                        warn!("ime required revision has been compacted"; "err" => ?msg);
                         self.reload_all_region_labels().await;
                         cancel.abort();
                         continue 'outer;

components/pd_client/src/client.rs

@@ -652,7 +652,7 @@ impl PdClient for RpcClient {
                             info!("cancel region heartbeat sender");
                         }
                         Err(e) => {
-                            error!(?e; "failed to send heartbeat");
+                            warn!("failed to send heartbeat"; "err" => ?e);
                         }
                     };
                 });

components/pd_client/src/util.rs

@@ -494,7 +494,7 @@ where
             Err(err) => {
                 // these errors are not caused by network, no need to retry
                 if err.retryable() && self.remain_request_count > 0 {
-                    error!(?*err; "request failed, retry");
+                    warn!("request failed, retry"; "err" => ?*err);
                     false
                 } else {
                     true
@@ -550,7 +550,7 @@ where
                 return Ok(r);
             }
             Err(e) => {
-                error!(?e; "request failed");
+                warn!("request failed"; "err" => ?e);
                 if retry == 0 {
                     return Err(e);
                 }
@@ -559,7 +559,7 @@ where
         // try reconnect
         retry -= 1;
         if let Err(e) = block_on(client.reconnect(true)) {
-            error!(?e; "reconnect failed");
+            warn!("reconnect failed"; "err" => ?e);
             thread::sleep(REQUEST_RECONNECT_INTERVAL);
         }
     }
@@ -718,7 +718,7 @@ impl PdConnector {
                         }
                     }
                     Err(e) => {
-                        error!("connect failed"; "endpoints" => ep, "error" => ?e);
+                        warn!("connect failed"; "endpoints" => ep, "error" => ?e);
                         continue;
                     }
                 }

components/raft_log_engine/Cargo.toml

@@ -9,9 +9,9 @@ license = "Apache-2.0"
 failpoints = ["raft-engine/failpoints"]
 
 [dependencies]
+codec = { workspace = true }
 encryption = { workspace = true }
 engine_traits = { workspace = true }
-codec = { workspace = true }
 file_system = { workspace = true }
 kvproto = { workspace = true }
 raft = { workspace = true }

components/raftstore-v2/src/operation/command/write/ingest.rs

@@ -9,7 +9,7 @@ use raftstore::{
     store::{check_sst_for_ingestion, metrics::PEER_WRITE_CMD_COUNTER, util},
     Result,
 };
-use slog::{error, info};
+use slog::{error, info, warn};
 use sst_importer::range_overlaps;
 use tikv_util::{box_try, slog_panic};
 
@@ -25,7 +25,7 @@ impl<'a, EK: KvEngine, ER: RaftEngine, T> StoreFsmDelegate<'a, EK, ER, T> {
     #[inline]
     pub fn on_cleanup_import_sst(&mut self) {
         if let Err(e) = self.fsm.store.on_cleanup_import_sst(self.store_ctx) {
-            error!(self.fsm.store.logger(), "cleanup import sst failed"; "error" => ?e);
+            warn!(self.fsm.store.logger(), "cleanup import sst failed"; "error" => ?e);
         }
         self.schedule_tick(
             StoreTick::CleanupImportSst,

components/raftstore-v2/src/operation/misc.rs

@@ -12,7 +12,7 @@ use raftstore::{
     store::{CompactThreshold, TabletSnapKey},
     Result,
 };
-use slog::{debug, error, info};
+use slog::{debug, error, info, warn};
 
 use crate::{
     batch::StoreContext,
@@ -117,7 +117,7 @@ impl<'a, EK: KvEngine, ER: RaftEngine, T> StoreFsmDelegate<'a, EK, ER, T> {
     #[inline]
     pub fn on_snapshot_gc(&mut self) {
         if let Err(e) = self.fsm.store.on_snapshot_gc(self.store_ctx) {
-            error!(self.fsm.store.logger(), "cleanup import sst failed"; "error" => ?e);
+            warn!(self.fsm.store.logger(), "cleanup import sst failed"; "error" => ?e);
         }
         self.schedule_tick(
             StoreTick::SnapGc,

components/raftstore-v2/src/worker/pd/mod.rs

@@ -523,7 +523,7 @@ mod requests {
             None => PeerMsg::admin_command(req).0,
         };
         if let Err(e) = router.send(region_id, msg) {
-            error!(
+            warn!(
                 logger,
                 "send request failed";
                 "region_id" => region_id, "cmd_type" => ?cmd_type, "err" => ?e,

components/raftstore-v2/src/worker/pd/store.rs

@@ -370,7 +370,7 @@ where
                     }
                 }
                 Err(e) => {
-                    error!(logger, "store heartbeat failed"; "err" => ?e);
+                    warn!(logger, "store heartbeat failed"; "err" => ?e);
                 }
             }
         };

components/raftstore/src/compacted_event_sender.rs

@@ -3,7 +3,7 @@ use std::sync::Mutex;
 
 use engine_rocks::{CompactedEventSender, RocksCompactedEvent};
 use engine_traits::{KvEngine, RaftEngine};
-use tikv_util::error_unknown;
+use tikv_util::warn;
 
 use crate::store::{fsm::store::RaftRouter, StoreMsg};
 
@@ -25,7 +25,10 @@ where
         let router = self.router.lock().unwrap();
         let event = StoreMsg::CompactedEvent(event);
         if let Err(e) = router.send_control(event) {
-            error_unknown!(?e; "send compaction finished event to raftstore failed");
+            warn!(
+                "send compaction finished event to raftstore failed";
+                "err" => ?e,
+            );
         }
     }
 }

components/raftstore/src/coprocessor/split_observer.rs

@@ -5,7 +5,7 @@ use kvproto::{
     metapb::Region,
     raft_cmdpb::{AdminCmdType, AdminRequest, SplitRequest},
 };
-use tikv_util::{box_err, box_try, codec::bytes, error, warn};
+use tikv_util::{box_err, box_try, codec::bytes, warn};
 
 use super::{AdminObserver, Coprocessor, ObserverContext, Result as CopResult};
 use crate::{store::util, Error};
@@ -124,7 +124,7 @@ impl AdminObserver for SplitObserver {
                 }
                 let mut request = vec![req.take_split()];
                 if let Err(e) = self.on_split(ctx, &mut request) {
-                    error!(
+                    warn!(
                         "failed to handle split req";
                         "region_id" => ctx.region().get_id(),
                         "err" => ?e,
@@ -145,7 +145,7 @@ impl AdminObserver for SplitObserver {
                 }
                 let mut requests = req.mut_splits().take_requests().into();
                 if let Err(e) = self.on_split(ctx, &mut requests) {
-                    error!(
+                    warn!(
                         "failed to handle split req";
                         "region_id" => ctx.region().get_id(),
                         "err" => ?e,

components/raftstore/src/store/entry_storage.rs

@@ -1140,7 +1140,7 @@ impl<EK: KvEngine, ER: RaftEngine> EntryStorage<EK, ER> {
             Ok(_) => {
                 // This should not happen, but it's OK :)
                 debug_assert!(false, "entries should not have been fetched");
-                error!("entries are fetched unexpectedly during warming up");
+                warn!("entries are fetched unexpectedly during warming up");
                 None
             }
             Err(raft::Error::Store(raft::StorageError::LogTemporarilyUnavailable)) => {
@@ -1190,7 +1190,7 @@ impl<EK: KvEngine, ER: RaftEngine> EntryStorage<EK, ER> {
                         // FIXME: the assertion below doesn't hold.
                         // assert!(is_valid, "the warmup range should still be valid");
                         if !is_valid {
-                            error!(
+                            warn!(
                                 "unexpected warmup state";
                                 "region_id" => self.region_id,
                                 "peer_id" => self.peer_id,