From 96422c06f385c1220684cb55b4b59c08c5a7298a Mon Sep 17 00:00:00 2001 From: RazHemo Date: Mon, 30 May 2022 18:30:23 +0300 Subject: [PATCH 01/21] clippy --- src/context.rs | 6 +++--- src/encoders.rs | 12 ++++++++++++ src/processor.rs | 12 ++++++------ src/processor_helper.rs | 36 +++++++++++++++--------------------- src/state_store.rs | 8 ++++---- src/test_utils.rs | 17 +++++++---------- 6 files changed, 47 insertions(+), 44 deletions(-) diff --git a/src/context.rs b/src/context.rs index b7a7653..16e0326 100644 --- a/src/context.rs +++ b/src/context.rs @@ -2,7 +2,7 @@ use serde::Serialize; pub struct Context { key: String, - origingal_state: Option, + original_state: Option, new_state: Option, sends: Vec, } @@ -21,7 +21,7 @@ where pub fn new(key: &str, state: Option) -> Self { Self { key: key.to_string(), - origingal_state: state, + original_state: state, new_state: None, sends: vec![], // deserializers: HashMap::new(), @@ -33,7 +33,7 @@ where } pub fn get_state(&self) -> &Option { - &self.origingal_state + &self.original_state } pub fn get_new_state(&self) -> &Option { diff --git a/src/encoders.rs b/src/encoders.rs index 94d215d..d508323 100644 --- a/src/encoders.rs +++ b/src/encoders.rs @@ -50,6 +50,12 @@ impl JsonEncoder { } } +impl Default for JsonEncoder { + fn default() -> Self { + Self::new() + } +} + impl Encoder for JsonEncoder where T: DeserializeOwned, @@ -67,6 +73,12 @@ impl JsonDecoder { Self {} } } + +impl Default for JsonDecoder { + fn default() -> Self { + Self::new() + } +} impl Decoder for JsonDecoder { fn decode(&self, data: &T) -> Vec where diff --git a/src/processor.rs b/src/processor.rs index 12a9abb..480b970 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -59,7 +59,7 @@ where pub async fn start(&mut self) { let input_topcis_set: HashSet = self.inputs.keys().cloned().collect(); - let output_topcis_set: HashSet = + let output_topics_set: HashSet = self.outputs.iter().map(|o| o.topic().to_string()).collect(); self.helper @@ -71,7 +71,7 @@ where let partitions_barrier = Arc::new(Barrier::new(num_partitions + 1)); for partition in 0..num_partitions as i32 { let inputs = self.inputs.clone(); - let output_topcis_set = output_topcis_set.clone(); + let output_topics_set = output_topics_set.clone(); let partition_handler = self.helper.create_partitioned_consumer(partition as i32); let partitions_barrier_clone = partitions_barrier.clone(); let gen = self.state_store_gen.clone(); @@ -80,7 +80,7 @@ where let mut state_store = gen(); let mut extra_state = gen2(); let stream_gens: Vec<_> = inputs.keys().into_iter().map(|topic| { - partition_handler.create_partitioned_topic_stream(&topic) + partition_handler.create_partitioned_topic_stream(topic) }).collect(); let mut topic_partition_offset_locks = HashMap::new(); let mut streams = select_all( @@ -112,16 +112,16 @@ where let msg_offset = msg.offset(); let helper_clone = partition_handler.clone(); - let output_topcis_set = output_topcis_set.clone(); + let output_topics_set = output_topics_set.clone(); let sends = ctx.to_send().into_iter().map(move |s| { - assert!(output_topcis_set.contains(&s.topic.to_string())); //TODO: Should we remove this assertion? + assert!(output_topics_set.contains(&s.topic)); //TODO: Should we remove this assertion? helper_clone .send_result(FutureRecord::to(&s.topic).key(&s.key).payload(&s.payload)) .unwrap() }); let mut helper_clone = partition_handler.clone(); - let offset_lock_for_partitioned_topic = topic_partition_offset_locks.entry(msg_topic.clone()).or_insert(Arc::new(Mutex::new(msg_offset))).clone(); + let offset_lock_for_partitioned_topic = topic_partition_offset_locks.entry(msg_topic.clone()).or_insert_with(|| Arc::new(Mutex::new(msg_offset))).clone(); tokio::spawn(async move { join_all(sends).await; diff --git a/src/processor_helper.rs b/src/processor_helper.rs index b135316..b7e0dfa 100644 --- a/src/processor_helper.rs +++ b/src/processor_helper.rs @@ -140,18 +140,14 @@ impl KafkaProcessorImplementor for KafkaProcessorHelper { } fn wait_to_finish(self) -> tokio::task::JoinHandle> { - let task = tokio::spawn(async move { - loop { - let message = self.stream_consumer.recv().await; - let err = format!( - "main stream consumer queue unexpectedly received message: {:?}", - message - ); - return Err(err); - } - }); - - task + tokio::spawn(async move { + let message = self.stream_consumer.recv().await; + let err = format!( + "main stream consumer queue unexpectedly received message: {:?}", + message + ); + Err(err) + }) } fn ensure_copartitioned(&mut self) -> Result { @@ -167,17 +163,15 @@ impl KafkaProcessorImplementor for KafkaProcessorHelper { .map(|topic| (topic.name().to_string(), topic.partitions().len())) .collect(); - if topics_partitions.len() < 1 { + if topics_partitions.is_empty() { Err(()) + } else if topics_partitions + .iter() + .all(|(_, partitions)| partitions.eq(&topics_partitions[0].1)) + { + Ok(topics_partitions[0].1) } else { - if topics_partitions - .iter() - .all(|(_, partitions)| partitions.eq(&topics_partitions[0].1)) - { - Ok(topics_partitions[0].1) - } else { - Err(()) - } + Err(()) } } } diff --git a/src/state_store.rs b/src/state_store.rs index 5bcb246..dea1e20 100644 --- a/src/state_store.rs +++ b/src/state_store.rs @@ -13,7 +13,7 @@ where { async fn get(&self, k: &Q) -> Option where - K: std::borrow::Borrow, + K: Borrow, Q: Eq + Hash; async fn set(&mut self, k: K, v: V) -> Option; } @@ -29,7 +29,7 @@ where K: Borrow, Q: Eq + Hash, { - self.get(k).map(|v| v.clone()) + self.get(k).cloned() } async fn set(&mut self, k: K, v: V) -> Option { self.insert(k, v) @@ -39,7 +39,7 @@ where #[async_trait::async_trait] impl StateStore for Arc>> where - K: Eq + std::hash::Hash + Sync + Send, + K: Eq + Hash + Sync + Send, V: Send + Sync + Clone, { async fn get(&self, k: &Q) -> Option @@ -47,7 +47,7 @@ where K: Borrow, Q: Eq + Hash, { - self.read().await.get(k).map(|v| v.clone()) + self.read().await.get(k).cloned() } async fn set(&mut self, k: K, v: V) -> Option { diff --git a/src/test_utils.rs b/src/test_utils.rs index 0a33008..f6e859a 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -37,7 +37,7 @@ impl Clone for TopicTestHelper { _ch_rx: self.ch_tx.subscribe(), sent: self.sent.clone(), recv: self.recv.clone(), - input: self.input.clone(), + input: self.input, } } } @@ -64,8 +64,7 @@ impl TestsProcessorHelper { pub fn new(topics: Vec<&str>) -> Self { let mut m = HashMap::new(); for topic in topics { - m.entry(topic.to_string()) - .or_insert(TopicTestHelper::default()); + m.entry(topic.to_string()).or_default(); } Self { topics: m } @@ -111,8 +110,8 @@ impl PartitionHelper for TestsPartitionProcessor { type Error = BroadcastStreamRecvError; type OwnedStreamableType = tokio::sync::broadcast::Sender; - fn create_partitioned_topic_stream<'a>( - &'a self, + fn create_partitioned_topic_stream( + &self, topic_name: &str, ) -> tokio::sync::broadcast::Sender { let rec = self.topics.get(topic_name).unwrap(); @@ -140,7 +139,7 @@ impl PartitionHelper for TestsPartitionProcessor { let rec = self.topics.get(record.topic).unwrap(); rec.sent.fetch_add(1, Ordering::Relaxed); rec.ch_tx.send(msg).unwrap(); - Ok(Box::pin(async move { () })) + Ok(Box::pin(async move {})) } fn store_offset(&mut self, topic: &str, _offset: i64) -> KafkaResult<()> { @@ -177,7 +176,7 @@ impl KafkaProcessorImplementor for TestsProcessorHelper { } fn wait_to_finish(self) -> tokio::task::JoinHandle> { - let task = tokio::spawn(async move { + tokio::spawn(async move { // TODO: This is ugly and we can implement better "wait" mechanism // TODO: Support output topics where we don't receive every message, only if it's inputs loop { @@ -197,9 +196,7 @@ impl KafkaProcessorImplementor for TestsProcessorHelper { } Ok(()) - }); - - task + }) } fn ensure_copartitioned(&mut self) -> Result { From 045eecdc5e6aa35234f67119266bfdcc7b652858 Mon Sep 17 00:00:00 2001 From: RazHemo Date: Mon, 30 May 2022 18:39:18 +0300 Subject: [PATCH 02/21] another lint --- bin/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/main.rs b/bin/main.rs index 99c4a64..e170948 100644 --- a/bin/main.rs +++ b/bin/main.rs @@ -109,7 +109,7 @@ mod tests { Input::new("c2".to_string(), JsonEncoder::new(), re_emit_clicks), ], vec![Output::new("c1".to_string())], - move || state_store_clone.clone(), + move || state_store_clone, || (), ); From db6fb151c4086deca618b1c7441fb433a51823c4 Mon Sep 17 00:00:00 2001 From: RazHemo Date: Wed, 1 Jun 2022 17:31:25 +0300 Subject: [PATCH 03/21] let handler functions get the store itself for custom operations make state mutation be returned rather than set on context --- Cargo.lock | 9 ++++--- Cargo.toml | 1 + bin/main.rs | 41 +++++++++++++++++----------- src/context.rs | 29 ++++++++++---------- src/input.rs | 51 ++++++++++++++++++----------------- src/processor.rs | 35 +++++++++++++++--------- src/state_store.rs | 67 ++++++++++++++++++++++++++-------------------- 7 files changed, 133 insertions(+), 100 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1e5f26..82dba3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -741,6 +741,7 @@ dependencies = [ "serde_json", "sled", "tempfile", + "thiserror", "tokio", "tokio-stream", "tracing", @@ -1791,18 +1792,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 062f75f..e0e0b1d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ async-trait = "0.1.53" google-bigtableadmin2 = "3.0.0" # bigtable depends on `yup-oauth2` ^6, but on hyper-rustls==0.22, but `yup-oauth2` 6.6.0 pulls hyper-rustls==0.23 :( yup-oauth2 = "=6.5.0" +thiserror = "1.0.31" [dev-dependencies] rstest = "*" diff --git a/bin/main.rs b/bin/main.rs index e170948..926d4c9 100644 --- a/bin/main.rs +++ b/bin/main.rs @@ -1,11 +1,12 @@ -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc}; use kast::{ context::Context, encoders::JsonEncoder, input::Input, output::Output, processor::Processor, - processor_helper::KafkaProcessorHelper, + processor_helper::KafkaProcessorHelper, state_store::StateStore, }; use rdkafka::ClientConfig; use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use tokio::sync::Mutex; #[derive(Debug, Clone, Deserialize, Serialize)] struct Click {} @@ -24,7 +25,10 @@ fn json_encoder(data: Option<&[u8]>) -> T { serde_json::from_slice(data.expect("empty message")).unwrap() } -async fn handle_click(ctx: &mut Context, _click: Click) { +async fn handle_click(ctx: &mut Context, _click: Click) -> Option +where + TStore: StateStore, +{ let mut clicks_per_user = match ctx.get_state() { Some(state) => state.clone(), None => ClicksPerUser { clicks: 0 }, @@ -33,14 +37,19 @@ async fn handle_click(ctx: &mut Context, _click: Click) { clicks_per_user.clicks += 1; println!("{:?}, {:?}", ctx.key(), clicks_per_user); - ctx.set_state(Some(clicks_per_user)) + Some(clicks_per_user) } -async fn re_emit_clicks(ctx: &mut Context, click: Click2) { +async fn re_emit_clicks(ctx: &mut Context, click: Click2) -> Option +where + TStore: StateStore, +{ let key = ctx.key().to_string(); for _ in 0..click.clicks { ctx.emit("c1", &key, &Click {}) } + + None } //TODO: Support outputs @@ -63,13 +72,14 @@ async fn main() { .clone(); let mut p = Processor::new( + "clicks", KafkaProcessorHelper::new(settings), vec![ Input::new("c1".to_string(), json_encoder, handle_click), Input::new("c2".to_string(), JsonEncoder::new(), re_emit_clicks), ], vec![Output::new("c1".to_string())], - HashMap::::new, + || Arc::new(Mutex::new(HashMap::::new())), || (), ); @@ -79,30 +89,27 @@ async fn main() { #[cfg(test)] mod tests { - use std::{collections::HashMap, sync::Arc}; + use super::*; + use std::{collections::HashMap}; use crate::{handle_click, json_encoder, re_emit_clicks, Click, Click2, ClicksPerUser}; use kast::{ encoders::{JsonDecoder, JsonEncoder}, - input::Input, - output::Output, - processor::Processor, - state_store::StateStore, test_utils::TestsProcessorHelper, }; use rstest::rstest; - use tokio::sync::RwLock; #[rstest] #[tokio::test] async fn test_single_store() { let mut t = TestsProcessorHelper::new(vec!["c1", "c2", "c3"]); - let state_store = Arc::new(RwLock::new(HashMap::new())); + let state_store = Arc::new(Mutex::new(HashMap::new())); let state_store_clone = state_store.clone(); let mut in1 = t.input("c1".to_string(), JsonDecoder::new()); let mut in2 = t.input("c2".to_string(), JsonDecoder::new()); let mut p = Processor::new( + "clicks", t, vec![ Input::new("c1".to_string(), json_encoder, handle_click), @@ -133,13 +140,15 @@ mod tests { p.join().await; - let final_state: ClicksPerUser = state_store.get("a").await.unwrap(); + let lock = state_store.lock().await; + + let final_state: &ClicksPerUser = lock.get("a").unwrap(); assert_eq!(final_state.clicks, 110000); - let final_state: ClicksPerUser = state_store.get("b").await.unwrap(); + let final_state: &ClicksPerUser = lock.get("b").unwrap(); assert_eq!(final_state.clicks, 10000); - let final_state: ClicksPerUser = state_store.get("c").await.unwrap(); + let final_state: &ClicksPerUser = lock.get("c").unwrap(); assert_eq!(final_state.clicks, 1); } } diff --git a/src/context.rs b/src/context.rs index 16e0326..815cd3d 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,10 +1,15 @@ +use std::sync::Arc; + use serde::Serialize; +use tokio::sync::Mutex; + +use crate::state_store::StateStore; -pub struct Context { +pub struct Context { key: String, original_state: Option, - new_state: Option, sends: Vec, + state_store: Arc>, } #[derive(Clone)] @@ -14,17 +19,17 @@ pub struct FutureDeliverableMessage { pub payload: Vec, } -impl Context +impl Context where T: Clone, + TStore: StateStore { - pub fn new(key: &str, state: Option) -> Self { + pub fn new(key: &str, state: Option, state_store: Arc>) -> Self { Self { key: key.to_string(), original_state: state, - new_state: None, sends: vec![], - // deserializers: HashMap::new(), + state_store, } } @@ -32,16 +37,12 @@ where &self.key } - pub fn get_state(&self) -> &Option { - &self.original_state - } - - pub fn get_new_state(&self) -> &Option { - &self.new_state + pub fn state_store(&self) -> Arc> { + self.state_store.clone() } - pub fn set_state(&mut self, state: Option) { - self.new_state = state + pub fn get_state(&self) -> &Option { + &self.original_state } pub fn emit(&mut self, topic: &str, key: &str, msg: &M) { diff --git a/src/input.rs b/src/input.rs index 071699a..4e8f2f3 100644 --- a/src/input.rs +++ b/src/input.rs @@ -5,16 +5,16 @@ use serde::de::DeserializeOwned; use std::marker::PhantomData; #[async_trait::async_trait] -pub trait GenericInput: DynClone + Sync + Send { +pub trait GenericInput: DynClone + Sync + Send { fn topic(&self) -> String; - async fn handle(&self, state: &mut S, ctx: &mut Context, data: Option<&[u8]>); + async fn handle(&self, state: &mut S, ctx: &mut Context, data: Option<&[u8]>) -> Option; } -dyn_clone::clone_trait_object!( GenericInput); +dyn_clone::clone_trait_object!( GenericInput); -pub trait Handler<'a, T, S, R, VArgs> { - type Future: Future + Send + 'a; +pub trait Handler<'a, T, S, R, TStore, VArgs> { + type Future: Future> + Send + 'a; - fn call(self, state: &'a mut S, ctx: &'a mut Context, req: R) -> Self::Future; + fn call(self, state: &'a mut S, ctx: &'a mut Context, req: R) -> Self::Future; } #[derive(Clone)] @@ -43,12 +43,13 @@ where } #[async_trait::async_trait] -impl GenericInput for Input +impl GenericInput for Input where - for<'a> F: Handler<'a, T, S, R, VArgs> + Send + Sync + Copy, + for<'a> F: Handler<'a, T, S, R, TStore, VArgs> + Send + Sync + Copy, E: Encoder + Sync + Clone + Send + 'static, R: Sync + Send + Clone + DeserializeOwned + 'static + std::fmt::Debug, T: Clone + Send + Sync + 'static, + TStore: Send + Sync, S: Send + Sync + Clone, VArgs: Clone + Send + Sync, { @@ -56,25 +57,25 @@ where self.topic.clone() } - async fn handle(&self, state: &mut S, ctx: &mut Context, data: Option<&[u8]>) { + async fn handle(&self, state: &mut S, ctx: &mut Context, data: Option<&[u8]>) -> Option { let msg = self.encoder.encode(data); - self.callback.call(state, ctx, msg).await; + self.callback.call(state, ctx, msg).await } } #[derive(Clone)] pub struct MessageOnly; -impl<'a, Fut, T, S, R, F> Handler<'a, T, S, R, MessageOnly> for F +impl<'a, Fut, T, S, R, TStore, F> Handler<'a, T, S, R, TStore, MessageOnly> for F where F: FnOnce(R) -> Fut + Send, - Fut: Future + Send + 'a, + Fut: Future> + Send + 'a, T: Send + 'static, S: Send + Sync + 'static, { type Future = Fut; - fn call(self, _state: &'a mut S, _ctx: &'a mut Context, req: R) -> Self::Future { + fn call(self, _state: &'a mut S, _ctx: &'a mut Context, req: R) -> Self::Future { (self)(req) } } @@ -82,16 +83,16 @@ where #[derive(Clone)] pub struct MessageAndState; -impl<'a, Fut, T, S, R, F> Handler<'a, T, S, R, MessageAndState> for F +impl<'a, Fut, T, S, R, TStore, F> Handler<'a, T, S, R, TStore, MessageAndState> for F where F: FnOnce(&'a mut S, R) -> Fut + Send, - Fut: Future + Send + 'a, + Fut: Future> + Send + 'a, T: Send + 'static, S: Send + Sync + 'static, { type Future = Fut; - fn call(self, state: &'a mut S, _ctx: &'a mut Context, req: R) -> Self::Future { + fn call(self, state: &'a mut S, _ctx: &'a mut Context, req: R) -> Self::Future { (self)(state, req) } } @@ -99,16 +100,17 @@ where #[derive(Clone)] pub struct MessageAndContext; -impl<'a, Fut, T, S, R, F> Handler<'a, T, S, R, MessageAndContext> for F +impl<'a, Fut, T, S, R, TStore, F> Handler<'a, T, S, R, TStore, MessageAndContext> for F where - F: FnOnce(&'a mut Context, R) -> Fut + Send, - Fut: Future + Send + 'a, + F: FnOnce(&'a mut Context, R) -> Fut + Send, + Fut: Future> + Send + 'a, T: Send + 'static, + TStore: 'a, S: Send + Sync + 'static, { type Future = Fut; - fn call(self, _state: &'a mut S, ctx: &'a mut Context, req: R) -> Self::Future { + fn call(self, _state: &'a mut S, ctx: &'a mut Context, req: R) -> Self::Future { (self)(ctx, req) } } @@ -116,16 +118,17 @@ where #[derive(Clone)] pub struct MessageAndContextAndState; -impl<'a, Fut, T, S, R, F> Handler<'a, T, S, R, MessageAndContextAndState> for F +impl<'a, Fut, T, S, R, TStore, F> Handler<'a, T, S, R, TStore, MessageAndContextAndState> for F where - F: FnOnce(&'a mut S, &'a mut Context, R) -> Fut + Send, - Fut: Future + Send + 'a, + F: FnOnce(&'a mut S, &'a mut Context, R) -> Fut + Send, + Fut: Future> + Send + 'a, T: Send + 'static, + TStore: 'a, S: Send + Sync + 'static, { type Future = Fut; - fn call(self, state: &'a mut S, ctx: &'a mut Context, req: R) -> Self::Future { + fn call(self, state: &'a mut S, ctx: &'a mut Context, req: R) -> Self::Future { (self)(state, ctx, req) } } diff --git a/src/processor.rs b/src/processor.rs index 480b970..193d487 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -13,12 +13,13 @@ use crate::{ input::GenericInput, output::Output, processor_helper::{IntoKafkaStream, KafkaProcessorImplementor, PartitionHelper}, - state_store::StateStore, + state_store::{StateStore, StateStoreError}, }; pub struct Processor { + state_namespace: String, helper: H, - inputs: HashMap>>, + inputs: HashMap>>, state_store_gen: F1, extra_state_gen: F2, outputs: Vec, @@ -28,16 +29,17 @@ pub struct Processor { impl Processor where TState: Clone + Send + Sync + 'static, - TStore: StateStore + Send + Sync, - F1: FnOnce() -> TStore + Send + Clone + 'static, + TStore: StateStore + Send + Sync + 'static, + F1: FnOnce() -> Arc> + Send + Clone + 'static, F2: FnOnce() -> TExtraState + Clone + Send + 'static, TExtraState: Send + 'static, H: KafkaProcessorImplementor + Sync + 'static, <<::PartitionHelperType as PartitionHelper>::DeliveryFutureType as futures::Future>::Output: std::marker::Send, { pub fn new( + state_namespace: &str, helper: H, - inputs: Vec>>, + inputs: Vec>>, outputs: Vec, state_store_gen: F1, extra_state_gen: F2, @@ -48,6 +50,7 @@ where .collect(); Processor { + state_namespace: state_namespace.to_string(), helper, inputs, outputs, @@ -76,8 +79,9 @@ where let partitions_barrier_clone = partitions_barrier.clone(); let gen = self.state_store_gen.clone(); let gen2 = self.extra_state_gen.clone(); + let state_namespace = self.state_namespace.clone(); tokio::spawn(async move { - let mut state_store = gen(); + let state_store = gen(); let mut extra_state = gen2(); let stream_gens: Vec<_> = inputs.keys().into_iter().map(|topic| { partition_handler.create_partitioned_topic_stream(topic) @@ -97,17 +101,22 @@ where //TODO: Key serializer? let key = std::str::from_utf8(msg.key().unwrap()).unwrap(); - let state = state_store.get(key).await.map(|s| s.clone()); - let mut ctx = Context::new(key, state); + let state = match state_store.lock().await.get(&state_namespace, key).await { + Ok(state) => Some(state), + Err(StateStoreError::MissingKey(_)) => None, + Err(e) => { + panic!("Failed to get state while handling message on topic {}: {:?}", msg.topic(), e); + }, + }; + let mut ctx = Context::new(key, state, Arc::clone(&state_store)); let handler = inputs.get(msg.topic()).expect("streams are created from inputs keys"); - handler.handle(&mut extra_state, &mut ctx, msg.payload()).await; - - if let Some(state) = &ctx.get_new_state() { - state_store.set(key.to_string(), state.clone()).await; + if let Some(state) = handler.handle(&mut extra_state, &mut ctx, msg.payload()).await { + if let Err(e) = state_store.lock().await.set(&state_namespace, key, state.clone()).await { + panic!("Error updating state store on topic {}: {:?}", msg.topic(), e); + } } - // let stream_consumer = stream_consumer.clone(); let msg_topic = msg.topic().to_string(); let msg_offset = msg.offset(); diff --git a/src/state_store.rs b/src/state_store.rs index dea1e20..38d0a02 100644 --- a/src/state_store.rs +++ b/src/state_store.rs @@ -1,56 +1,65 @@ -use std::borrow::Borrow; use std::collections::HashMap; -use std::hash::Hash; use std::sync::Arc; use tokio::sync::RwLock; +#[derive(thiserror::Error, Debug, Clone)] +pub enum StateStoreError { + #[error("State store has no key {0}")] + MissingKey(String), + + #[error("State store internal error: {0}")] + Internal(String), +} + + +/// A type that implements this trait is a KV store with a slight abstraction that allows us to restrict access to specific sub-keys. +/// For example, if we want to store both a small application state (e.g. keypoints progress in pairing), but also a much bigger +/// dataset that only a small slice of it is relevant for a given moment (e.g. a several hours worth of keypoints), we might +/// have two wrapper types `PairingState` and `KeypointsState`, both of which hide away a `StateStore` implementor, and expose different +/// methods for access. Those methods internally will provide the `namespace` value to `get` and `set` to determine where in the store +/// each kind of data is stored. #[async_trait::async_trait] -pub trait StateStore +pub trait StateStore where - K: Eq + Hash, V: Clone, { - async fn get(&self, k: &Q) -> Option - where - K: Borrow, - Q: Eq + Hash; - async fn set(&mut self, k: K, v: V) -> Option; + async fn get(&self, namespace: &str, k: &str) -> Result; + async fn set(&mut self, namespace: &str, k: &str, v: V) -> Result<(), StateStoreError>; } #[async_trait::async_trait] -impl StateStore for HashMap +impl StateStore for HashMap where - K: Eq + Hash + Sync + Send, V: Send + Sync + Clone, { - async fn get(&self, k: &Q) -> Option - where - K: Borrow, - Q: Eq + Hash, - { - self.get(k).cloned() + async fn get(&self, namespace: &str, k: &str) -> Result { + self.get(&format!("{}::{}", namespace, k)) + .cloned() + .ok_or_else(|| StateStoreError::MissingKey(k.to_string())) } - async fn set(&mut self, k: K, v: V) -> Option { - self.insert(k, v) + async fn set(&mut self, namespace: &str, k: &str, v: V) -> Result<(), StateStoreError> { + let _ = self.insert(format!("{}::{}", namespace, k), v); + Ok(()) } } #[async_trait::async_trait] -impl StateStore for Arc>> +impl StateStore for Arc>> where - K: Eq + Hash + Sync + Send, V: Send + Sync + Clone, { - async fn get(&self, k: &Q) -> Option - where - K: Borrow, - Q: Eq + Hash, - { - self.read().await.get(k).cloned() + async fn get(&self, namespace: &str, k: &str) -> Result +where { + self.read() + .await + .get(&format!("{}::{}", namespace, k)) + .cloned() + .ok_or_else(|| StateStoreError::MissingKey(k.to_string())) } - async fn set(&mut self, k: K, v: V) -> Option { - self.write().await.insert(k, v) + async fn set(&mut self, namespace: &str, k: &str, v: V) -> Result<(), StateStoreError> { + let _ = self.write().await.insert(format!("{}::{}", namespace, k), v); + Ok(()) } } From 81b2b0c1254a1022c8efa8f42940823cb61e444e Mon Sep 17 00:00:00 2001 From: RazHemo Date: Wed, 1 Jun 2022 17:34:08 +0300 Subject: [PATCH 04/21] fix test --- bin/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/main.rs b/bin/main.rs index 926d4c9..f1fbd24 100644 --- a/bin/main.rs +++ b/bin/main.rs @@ -142,13 +142,13 @@ mod tests { let lock = state_store.lock().await; - let final_state: &ClicksPerUser = lock.get("a").unwrap(); + let final_state: &ClicksPerUser = lock.get("clicks::a").unwrap(); assert_eq!(final_state.clicks, 110000); - let final_state: &ClicksPerUser = lock.get("b").unwrap(); + let final_state: &ClicksPerUser = lock.get("clicks::b").unwrap(); assert_eq!(final_state.clicks, 10000); - let final_state: &ClicksPerUser = lock.get("c").unwrap(); + let final_state: &ClicksPerUser = lock.get("clicks::c").unwrap(); assert_eq!(final_state.clicks, 1); } } From 9de6d7690725a6ed9e776151287fa4f848318098 Mon Sep 17 00:00:00 2001 From: RazHemo Date: Mon, 6 Jun 2022 15:45:29 +0300 Subject: [PATCH 05/21] more indicative errors --- src/processor.rs | 2 +- src/processor_helper.rs | 24 ++++++++++++++++++------ src/test_utils.rs | 4 ++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/processor.rs b/src/processor.rs index 193d487..94fb933 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -69,7 +69,7 @@ where .subscribe_inputs(&input_topcis_set) .unwrap(); - let num_partitions = self.helper.ensure_copartitioned().unwrap(); + let num_partitions = self.helper.ensure_copartitioned().expect("Not copartitioned"); let partitions_barrier = Arc::new(Barrier::new(num_partitions + 1)); for partition in 0..num_partitions as i32 { diff --git a/src/processor_helper.rs b/src/processor_helper.rs index b7e0dfa..7d7304d 100644 --- a/src/processor_helper.rs +++ b/src/processor_helper.rs @@ -46,12 +46,24 @@ pub trait PartitionHelper: Send + Clone { fn store_offset(&mut self, topic: &str, offset: i64) -> KafkaResult<()>; } +#[derive(Debug, thiserror::Error)] +pub enum EnsureCopartitionedError { + #[error("Connection to kafka broker failed")] + ConnectionFailure, + + #[error("No topics found in broker")] + NoTopics, + + #[error("Topics are not copartitioned: {0:?}")] + NotCopartitioned(Vec<(String, usize)>), +} + pub trait KafkaProcessorImplementor: Send { type PartitionHelperType: PartitionHelper + Sync; fn subscribe_inputs(&mut self, input_topics: &HashSet) -> Result<(), ()>; - fn ensure_copartitioned(&mut self) -> Result; + fn ensure_copartitioned(&mut self) -> Result; fn create_partitioned_consumer(&self, partition: i32) -> Self::PartitionHelperType; @@ -150,11 +162,11 @@ impl KafkaProcessorImplementor for KafkaProcessorHelper { }) } - fn ensure_copartitioned(&mut self) -> Result { + fn ensure_copartitioned(&mut self) -> Result { let shit = self .stream_consumer - .fetch_metadata(None, Duration::from_secs(1)) - .unwrap(); + .fetch_metadata(None, Duration::from_secs(5)) + .map_err(|_| EnsureCopartitionedError::ConnectionFailure)?; let topics_partitions: Vec<(String, usize)> = shit .topics() @@ -164,14 +176,14 @@ impl KafkaProcessorImplementor for KafkaProcessorHelper { .collect(); if topics_partitions.is_empty() { - Err(()) + Err(EnsureCopartitionedError::NoTopics) } else if topics_partitions .iter() .all(|(_, partitions)| partitions.eq(&topics_partitions[0].1)) { Ok(topics_partitions[0].1) } else { - Err(()) + Err(EnsureCopartitionedError::NotCopartitioned(topics_partitions)) } } } diff --git a/src/test_utils.rs b/src/test_utils.rs index f6e859a..2066f67 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -155,7 +155,7 @@ impl PartitionHelper for TestsPartitionProcessor { use crate::{ encoders::{Decoder, Encoder}, - processor_helper::{IntoKafkaStream, KafkaProcessorImplementor, PartitionHelper}, + processor_helper::{IntoKafkaStream, KafkaProcessorImplementor, PartitionHelper, EnsureCopartitionedError}, }; impl KafkaProcessorImplementor for TestsProcessorHelper { type PartitionHelperType = TestsPartitionProcessor; @@ -199,7 +199,7 @@ impl KafkaProcessorImplementor for TestsProcessorHelper { }) } - fn ensure_copartitioned(&mut self) -> Result { + fn ensure_copartitioned(&mut self) -> Result { Ok(1) } } From 29eb1ff60558f0f89ae71560b30ff6c12b1fe58a Mon Sep 17 00:00:00 2001 From: RazHemo Date: Mon, 6 Jun 2022 17:45:28 +0300 Subject: [PATCH 06/21] more intuitive api --- bin/main.rs | 11 ++++------- src/processor.rs | 5 +---- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/bin/main.rs b/bin/main.rs index f1fbd24..5e256a4 100644 --- a/bin/main.rs +++ b/bin/main.rs @@ -71,7 +71,7 @@ async fn main() { .set("max.poll.interval.ms", "1500") .clone(); - let mut p = Processor::new( + let p = Processor::new( "clicks", KafkaProcessorHelper::new(settings), vec![ @@ -83,8 +83,7 @@ async fn main() { || (), ); - p.start().await; - p.join().await + p.run_forever().await; } #[cfg(test)] @@ -108,7 +107,7 @@ mod tests { let mut in1 = t.input("c1".to_string(), JsonDecoder::new()); let mut in2 = t.input("c2".to_string(), JsonDecoder::new()); - let mut p = Processor::new( + let p = Processor::new( "clicks", t, vec![ @@ -120,8 +119,6 @@ mod tests { || (), ); - p.start().await; - for _i in 0..100000 { in1.send("a".to_string(), &Click {}).await.unwrap(); } @@ -138,7 +135,7 @@ mod tests { .await .unwrap(); - p.join().await; + p.run_forever().await; let lock = state_store.lock().await; diff --git a/src/processor.rs b/src/processor.rs index 94fb933..97c21a8 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -60,7 +60,7 @@ where } } - pub async fn start(&mut self) { + pub async fn run_forever(mut self) { let input_topcis_set: HashSet = self.inputs.keys().cloned().collect(); let output_topics_set: HashSet = self.outputs.iter().map(|o| o.topic().to_string()).collect(); @@ -147,9 +147,6 @@ where } partitions_barrier.wait().await; - } - - pub async fn join(self) { self.helper .wait_to_finish() .await From 7db915874d0c44090b57c12d34a0f22a5c9c2fed Mon Sep 17 00:00:00 2001 From: RazHemo Date: Tue, 7 Jun 2022 15:31:07 +0300 Subject: [PATCH 07/21] today is not opposite day --- src/encoders.rs | 22 +++++++++++----------- src/input.rs | 8 ++++---- src/test_utils.rs | 14 +++++++------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/encoders.rs b/src/encoders.rs index d508323..1e1bade 100644 --- a/src/encoders.rs +++ b/src/encoders.rs @@ -1,13 +1,13 @@ use serde::{de::DeserializeOwned, Serialize}; use std::marker::PhantomData; -pub trait Encoder { +pub trait Decoder { type In; fn encode(&self, data: Option<&[u8]>) -> Self::In; } -impl Encoder for F +impl Decoder for F where F: FnOnce(Option<&[u8]>) -> T + Copy, { @@ -18,7 +18,7 @@ where } } -pub trait Decoder { +pub trait Encoder { fn decode(&self, data: &T) -> Vec where T: Serialize; @@ -38,11 +38,11 @@ pub trait Decoder { // } #[derive(Clone)] -pub struct JsonEncoder { +pub struct JsonDecoder { _marker: PhantomData, } -impl JsonEncoder { +impl JsonDecoder { pub fn new() -> Self { Self { _marker: PhantomData, @@ -50,13 +50,13 @@ impl JsonEncoder { } } -impl Default for JsonEncoder { +impl Default for JsonDecoder { fn default() -> Self { Self::new() } } -impl Encoder for JsonEncoder +impl Decoder for JsonDecoder where T: DeserializeOwned, { @@ -67,19 +67,19 @@ where } } -pub struct JsonDecoder {} -impl JsonDecoder { +pub struct JsonEncoder {} +impl JsonEncoder { pub fn new() -> Self { Self {} } } -impl Default for JsonDecoder { +impl Default for JsonEncoder { fn default() -> Self { Self::new() } } -impl Decoder for JsonDecoder { +impl Encoder for JsonEncoder { fn decode(&self, data: &T) -> Vec where T: Serialize, diff --git a/src/input.rs b/src/input.rs index 4e8f2f3..732869e 100644 --- a/src/input.rs +++ b/src/input.rs @@ -1,4 +1,4 @@ -use crate::{context::Context, encoders::Encoder}; +use crate::{context::Context, encoders::Decoder}; use dyn_clone::DynClone; use futures::Future; use serde::de::DeserializeOwned; @@ -20,7 +20,7 @@ pub trait Handler<'a, T, S, R, TStore, VArgs> { #[derive(Clone)] pub struct Input where - E: Encoder, + E: Decoder, { topic: String, encoder: E, @@ -30,7 +30,7 @@ where impl Input where - E: Encoder, + E: Decoder, { pub fn new(topic: String, encoder: E, callback: F) -> Box { Box::new(Input { @@ -46,7 +46,7 @@ where impl GenericInput for Input where for<'a> F: Handler<'a, T, S, R, TStore, VArgs> + Send + Sync + Copy, - E: Encoder + Sync + Clone + Send + 'static, + E: Decoder + Sync + Clone + Send + 'static, R: Sync + Send + Clone + DeserializeOwned + 'static + std::fmt::Debug, T: Clone + Send + Sync + 'static, TStore: Send + Sync, diff --git a/src/test_utils.rs b/src/test_utils.rs index 2066f67..f1909a9 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -74,7 +74,7 @@ impl TestsProcessorHelper { impl TestsProcessorHelper { pub fn input(&mut self, topic: String, decoder: F) -> Sender where - F: Decoder, + F: Encoder, T: Serialize, { let input = self.topics.get_mut(&topic).unwrap(); @@ -83,7 +83,7 @@ impl TestsProcessorHelper { pub fn output(&mut self, topic: String, encoder: E) -> Receiver where - E: Encoder, + E: Decoder, { let output = self.topics.get_mut(&topic).unwrap(); Receiver::new(encoder, output.ch_tx.subscribe()) @@ -154,7 +154,7 @@ impl PartitionHelper for TestsPartitionProcessor { } use crate::{ - encoders::{Decoder, Encoder}, + encoders::{Encoder, Decoder}, processor_helper::{IntoKafkaStream, KafkaProcessorImplementor, PartitionHelper, EnsureCopartitionedError}, }; impl KafkaProcessorImplementor for TestsProcessorHelper { @@ -207,7 +207,7 @@ impl KafkaProcessorImplementor for TestsProcessorHelper { //Should we use with flat map on channels? pub struct Sender where - F: Decoder, + F: Encoder, { topic: String, decoder: F, @@ -218,7 +218,7 @@ where impl Sender where - F: Decoder, + F: Encoder, T: Serialize, { fn new( @@ -256,7 +256,7 @@ where pub struct Receiver where - E: Encoder, + E: Decoder, { encoder: E, rx: tokio::sync::broadcast::Receiver, @@ -264,7 +264,7 @@ where impl Receiver where - E: Encoder, + E: Decoder, { fn new(encoder: E, rx: tokio::sync::broadcast::Receiver) -> Self { Self { encoder, rx } From c553386cb82aadb75b0b9e956a265afecf2f557d Mon Sep 17 00:00:00 2001 From: RazHemo Date: Tue, 7 Jun 2022 16:46:28 +0300 Subject: [PATCH 08/21] better ergonomics --- bin/main.rs | 24 ++++++++++++------------ src/output.rs | 4 ++-- src/test_utils.rs | 6 +++--- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/bin/main.rs b/bin/main.rs index 5e256a4..638029f 100644 --- a/bin/main.rs +++ b/bin/main.rs @@ -1,11 +1,11 @@ use std::{collections::HashMap, sync::Arc}; use kast::{ - context::Context, encoders::JsonEncoder, input::Input, output::Output, processor::Processor, + context::Context, encoders::JsonDecoder, input::Input, output::Output, processor::Processor, processor_helper::KafkaProcessorHelper, state_store::StateStore, }; use rdkafka::ClientConfig; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; #[derive(Debug, Clone, Deserialize, Serialize)] @@ -75,10 +75,10 @@ async fn main() { "clicks", KafkaProcessorHelper::new(settings), vec![ - Input::new("c1".to_string(), json_encoder, handle_click), - Input::new("c2".to_string(), JsonEncoder::new(), re_emit_clicks), + Input::new("c1", JsonDecoder::new(), handle_click), + Input::new("c2", JsonDecoder::new(), re_emit_clicks), ], - vec![Output::new("c1".to_string())], + vec![Output::new("c1")], || Arc::new(Mutex::new(HashMap::::new())), || (), ); @@ -89,9 +89,9 @@ async fn main() { #[cfg(test)] mod tests { use super::*; - use std::{collections::HashMap}; + use std::collections::HashMap; - use crate::{handle_click, json_encoder, re_emit_clicks, Click, Click2, ClicksPerUser}; + use crate::{handle_click, re_emit_clicks, Click, Click2, ClicksPerUser}; use kast::{ encoders::{JsonDecoder, JsonEncoder}, test_utils::TestsProcessorHelper, @@ -104,17 +104,17 @@ mod tests { let mut t = TestsProcessorHelper::new(vec!["c1", "c2", "c3"]); let state_store = Arc::new(Mutex::new(HashMap::new())); let state_store_clone = state_store.clone(); - let mut in1 = t.input("c1".to_string(), JsonDecoder::new()); - let mut in2 = t.input("c2".to_string(), JsonDecoder::new()); + let mut in1 = t.input("c1", JsonEncoder::new()); + let mut in2 = t.input("c2", JsonEncoder::new()); let p = Processor::new( "clicks", t, vec![ - Input::new("c1".to_string(), json_encoder, handle_click), - Input::new("c2".to_string(), JsonEncoder::new(), re_emit_clicks), + Input::new("c1", JsonDecoder::new(), handle_click), + Input::new("c2", JsonDecoder::new(), re_emit_clicks), ], - vec![Output::new("c1".to_string())], + vec![Output::new("c1")], move || state_store_clone, || (), ); diff --git a/src/output.rs b/src/output.rs index a422cc6..2a85a54 100644 --- a/src/output.rs +++ b/src/output.rs @@ -6,8 +6,8 @@ pub struct Output { } impl Output { - pub fn new(topic: String) -> Self { - Output { topic } + pub fn new(topic: &str) -> Self { + Output { topic: topic.to_string() } } pub fn topic(&self) -> &str { diff --git a/src/test_utils.rs b/src/test_utils.rs index f1909a9..4b2ed3a 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -72,13 +72,13 @@ impl TestsProcessorHelper { } impl TestsProcessorHelper { - pub fn input(&mut self, topic: String, decoder: F) -> Sender + pub fn input(&mut self, topic: &str, decoder: F) -> Sender where F: Encoder, T: Serialize, { - let input = self.topics.get_mut(&topic).unwrap(); - Sender::new(topic, decoder, input.ch_tx.clone(), input.sent.clone()) + let input = self.topics.get_mut(topic).unwrap(); + Sender::new(topic.to_string(), decoder, input.ch_tx.clone(), input.sent.clone()) } pub fn output(&mut self, topic: String, encoder: E) -> Receiver From db2f9d0b6d8021190d227dbe0f4fcc0a31670d5e Mon Sep 17 00:00:00 2001 From: RazHemo Date: Tue, 7 Jun 2022 16:48:05 +0300 Subject: [PATCH 09/21] pass message headers to processors --- bin/main.rs | 20 ++++++++----- src/context.rs | 6 ++-- src/input.rs | 73 ++++++++---------------------------------------- src/processor.rs | 31 ++++++++++++++++++-- 4 files changed, 56 insertions(+), 74 deletions(-) diff --git a/bin/main.rs b/bin/main.rs index 638029f..e08ef30 100644 --- a/bin/main.rs +++ b/bin/main.rs @@ -21,11 +21,12 @@ struct ClicksPerUser { clicks: u32, } -fn json_encoder(data: Option<&[u8]>) -> T { - serde_json::from_slice(data.expect("empty message")).unwrap() -} - -async fn handle_click(ctx: &mut Context, _click: Click) -> Option +async fn handle_click( + _settings: &mut (), + ctx: &mut Context, + _click: Click, + _headers: HashMap, +) -> Option where TStore: StateStore, { @@ -40,13 +41,18 @@ where Some(clicks_per_user) } -async fn re_emit_clicks(ctx: &mut Context, click: Click2) -> Option +async fn re_emit_clicks( + _settings: &mut (), + ctx: &mut Context, + click: Click2, + _headers: HashMap, +) -> Option where TStore: StateStore, { let key = ctx.key().to_string(); for _ in 0..click.clicks { - ctx.emit("c1", &key, &Click {}) + ctx.emit("c1", &key, &Click {}, Default::default()) } None diff --git a/src/context.rs b/src/context.rs index 815cd3d..44cf495 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::{sync::Arc, collections::HashMap}; use serde::Serialize; use tokio::sync::Mutex; @@ -17,6 +17,7 @@ pub struct FutureDeliverableMessage { pub topic: String, pub key: String, pub payload: Vec, + pub headers: HashMap, } impl Context @@ -45,13 +46,14 @@ where &self.original_state } - pub fn emit(&mut self, topic: &str, key: &str, msg: &M) { + pub fn emit(&mut self, topic: &str, key: &str, msg: &M, headers: HashMap) { // let output = self.deserializers.get(topic).unwrap(); let data = serde_json::to_vec(msg).unwrap(); self.sends.push(FutureDeliverableMessage { topic: topic.to_string(), key: key.to_string(), payload: data, + headers, }); } diff --git a/src/input.rs b/src/input.rs index 732869e..b0adb05 100644 --- a/src/input.rs +++ b/src/input.rs @@ -1,20 +1,21 @@ use crate::{context::Context, encoders::Decoder}; use dyn_clone::DynClone; use futures::Future; + use serde::de::DeserializeOwned; -use std::marker::PhantomData; +use std::{marker::PhantomData, collections::HashMap}; #[async_trait::async_trait] pub trait GenericInput: DynClone + Sync + Send { fn topic(&self) -> String; - async fn handle(&self, state: &mut S, ctx: &mut Context, data: Option<&[u8]>) -> Option; + async fn handle(&self, state: &mut S, ctx: &mut Context, data: Option<&[u8]>, headers: HashMap) -> Option; } dyn_clone::clone_trait_object!( GenericInput); pub trait Handler<'a, T, S, R, TStore, VArgs> { type Future: Future> + Send + 'a; - fn call(self, state: &'a mut S, ctx: &'a mut Context, req: R) -> Self::Future; + fn call(self, state: &'a mut S, ctx: &'a mut Context, req: R, headers: HashMap) -> Self::Future; } #[derive(Clone)] @@ -57,70 +58,18 @@ where self.topic.clone() } - async fn handle(&self, state: &mut S, ctx: &mut Context, data: Option<&[u8]>) -> Option { + async fn handle(&self, state: &mut S, ctx: &mut Context, data: Option<&[u8]>, headers: HashMap) -> Option { let msg = self.encoder.encode(data); - self.callback.call(state, ctx, msg).await - } -} - -#[derive(Clone)] -pub struct MessageOnly; - -impl<'a, Fut, T, S, R, TStore, F> Handler<'a, T, S, R, TStore, MessageOnly> for F -where - F: FnOnce(R) -> Fut + Send, - Fut: Future> + Send + 'a, - T: Send + 'static, - S: Send + Sync + 'static, -{ - type Future = Fut; - - fn call(self, _state: &'a mut S, _ctx: &'a mut Context, req: R) -> Self::Future { - (self)(req) - } -} - -#[derive(Clone)] -pub struct MessageAndState; - -impl<'a, Fut, T, S, R, TStore, F> Handler<'a, T, S, R, TStore, MessageAndState> for F -where - F: FnOnce(&'a mut S, R) -> Fut + Send, - Fut: Future> + Send + 'a, - T: Send + 'static, - S: Send + Sync + 'static, -{ - type Future = Fut; - - fn call(self, state: &'a mut S, _ctx: &'a mut Context, req: R) -> Self::Future { - (self)(state, req) - } -} - -#[derive(Clone)] -pub struct MessageAndContext; - -impl<'a, Fut, T, S, R, TStore, F> Handler<'a, T, S, R, TStore, MessageAndContext> for F -where - F: FnOnce(&'a mut Context, R) -> Fut + Send, - Fut: Future> + Send + 'a, - T: Send + 'static, - TStore: 'a, - S: Send + Sync + 'static, -{ - type Future = Fut; - - fn call(self, _state: &'a mut S, ctx: &'a mut Context, req: R) -> Self::Future { - (self)(ctx, req) + self.callback.call(state, ctx, msg, headers).await } } #[derive(Clone)] -pub struct MessageAndContextAndState; +pub struct MessageAndContextAndStateAndHeaders; -impl<'a, Fut, T, S, R, TStore, F> Handler<'a, T, S, R, TStore, MessageAndContextAndState> for F +impl<'a, Fut, T, S, R, TStore, F> Handler<'a, T, S, R, TStore, MessageAndContextAndStateAndHeaders> for F where - F: FnOnce(&'a mut S, &'a mut Context, R) -> Fut + Send, + F: FnOnce(&'a mut S, &'a mut Context, R, HashMap) -> Fut + Send, Fut: Future> + Send + 'a, T: Send + 'static, TStore: 'a, @@ -128,7 +77,7 @@ where { type Future = Fut; - fn call(self, state: &'a mut S, ctx: &'a mut Context, req: R) -> Self::Future { - (self)(state, ctx, req) + fn call(self, state: &'a mut S, ctx: &'a mut Context, req: R, headers: HashMap) -> Self::Future { + (self)(state, ctx, req, headers) } } diff --git a/src/processor.rs b/src/processor.rs index 97c21a8..8f2b488 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -5,7 +5,7 @@ use std::{ }; use futures::{future::join_all, stream::select_all, StreamExt}; -use rdkafka::{producer::FutureRecord, Message}; +use rdkafka::{producer::FutureRecord, Message, message::{OwnedHeaders, Headers}}; use tokio::sync::{Barrier, Mutex}; use crate::{ @@ -111,7 +111,7 @@ where let mut ctx = Context::new(key, state, Arc::clone(&state_store)); let handler = inputs.get(msg.topic()).expect("streams are created from inputs keys"); - if let Some(state) = handler.handle(&mut extra_state, &mut ctx, msg.payload()).await { + if let Some(state) = handler.handle(&mut extra_state, &mut ctx, msg.payload(), get_headers(&msg)).await { if let Err(e) = state_store.lock().await.set(&state_namespace, key, state.clone()).await { panic!("Error updating state store on topic {}: {:?}", msg.topic(), e); } @@ -125,7 +125,7 @@ where let sends = ctx.to_send().into_iter().map(move |s| { assert!(output_topics_set.contains(&s.topic)); //TODO: Should we remove this assertion? helper_clone - .send_result(FutureRecord::to(&s.topic).key(&s.key).payload(&s.payload)) + .send_result(FutureRecord::to(&s.topic).key(&s.key).payload(&s.payload).headers(make_owned_headers(s.headers))) .unwrap() }); @@ -154,3 +154,28 @@ where .unwrap(); } } + +fn get_headers(message: &impl Message) -> HashMap { + let mut result = HashMap::new(); + + if let Some(headers) = message.headers() { + for (k, v) in (0..headers.count()) + .filter_map(|i| headers.get(i)) + .map(|(k, v)| (k.to_string(), String::from_utf8_lossy(v).to_string())) + { + result.insert(k, v); + } + } + + result +} + +fn make_owned_headers(headers: HashMap) -> OwnedHeaders { + let mut result = OwnedHeaders::new(); + + for (k, v) in headers.into_iter() { + result = result.add(&k, &v); + } + + result +} \ No newline at end of file From 5494532ee802f48edd5568a47dfef4dce067b527 Mon Sep 17 00:00:00 2001 From: RazHemo Date: Tue, 7 Jun 2022 16:48:16 +0300 Subject: [PATCH 10/21] pass string as ref --- src/input.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/input.rs b/src/input.rs index b0adb05..bebb12a 100644 --- a/src/input.rs +++ b/src/input.rs @@ -33,9 +33,9 @@ impl Input where E: Decoder, { - pub fn new(topic: String, encoder: E, callback: F) -> Box { + pub fn new(topic: &str, encoder: E, callback: F) -> Box { Box::new(Input { - topic, + topic: topic.to_string(), encoder, callback, _marker: PhantomData, From 66bbad3159bb6c897991ebe11bc898b89d0ceee0 Mon Sep 17 00:00:00 2001 From: RazHemo Date: Wed, 8 Jun 2022 18:55:21 +0300 Subject: [PATCH 11/21] restore the join() method but make it only relevant for tests --- Cargo.lock | 33 ++++++++++ Cargo.toml | 6 +- bin/main.rs | 13 ++-- src/lib.rs | 2 + src/processor.rs | 18 +++--- src/processor_helper.rs | 34 ++++++---- src/test_utils.rs | 139 ++++++++++++++++++++-------------------- 7 files changed, 151 insertions(+), 94 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82dba3f..45be4dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,17 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f" +[[package]] +name = "async-channel" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2114d64672151c0c5eaa5e131ec84a74f06e1e559830dabba01ca30605d66319" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + [[package]] name = "async-stream" version = "0.3.3" @@ -143,6 +154,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "cache-padded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" + [[package]] name = "cc" version = "1.0.73" @@ -200,6 +217,15 @@ dependencies = [ "cc", ] +[[package]] +name = "concurrent-queue" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + [[package]] name = "const_fn" version = "0.4.9" @@ -291,6 +317,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "event-listener" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77f3309417938f28bf8228fcff79a4a37103981e3e186d2ccd19c74b38f4eb71" + [[package]] name = "fastrand" version = "1.7.0" @@ -726,6 +758,7 @@ name = "kast" version = "0.1.0" dependencies = [ "anyhow", + "async-channel", "async-stream", "async-trait", "bigtable_rs", diff --git a/Cargo.toml b/Cargo.toml index e0e0b1d..75cd736 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,11 @@ crate-type = ["lib"] [[bin]] name = "main" path = "bin/main.rs" +required-features = ["testing"] +[features] +default = [] +testing = ["async-channel"] [dependencies] rdkafka = { version = "0.28", features = ["cmake-build"] } @@ -38,8 +42,8 @@ google-bigtableadmin2 = "3.0.0" # bigtable depends on `yup-oauth2` ^6, but on hyper-rustls==0.22, but `yup-oauth2` 6.6.0 pulls hyper-rustls==0.23 :( yup-oauth2 = "=6.5.0" thiserror = "1.0.31" +async-channel = { version = "*", optional = true } [dev-dependencies] rstest = "*" tempfile = "*" - diff --git a/bin/main.rs b/bin/main.rs index e08ef30..227be20 100644 --- a/bin/main.rs +++ b/bin/main.rs @@ -61,7 +61,7 @@ where //TODO: Support outputs //TODO: Support different state stores which arent hashmaps #[tokio::main] -async fn main() { +async fn main() -> Result<(), String> { let settings = ClientConfig::new() .set("bootstrap.servers", "localhost:9092") .set("partitioner", "murmur2") @@ -77,7 +77,7 @@ async fn main() { .set("max.poll.interval.ms", "1500") .clone(); - let p = Processor::new( + let mut p = Processor::new( "clicks", KafkaProcessorHelper::new(settings), vec![ @@ -89,7 +89,7 @@ async fn main() { || (), ); - p.run_forever().await; + p.run_forever().await } #[cfg(test)] @@ -113,7 +113,7 @@ mod tests { let mut in1 = t.input("c1", JsonEncoder::new()); let mut in2 = t.input("c2", JsonEncoder::new()); - let p = Processor::new( + let mut p = Processor::new( "clicks", t, vec![ @@ -125,6 +125,9 @@ mod tests { || (), ); + // We must run the processor before sending inputs, otherwise they will not reach the processor + p.run_forever().await.unwrap(); + for _i in 0..100000 { in1.send("a".to_string(), &Click {}).await.unwrap(); } @@ -141,7 +144,7 @@ mod tests { .await .unwrap(); - p.run_forever().await; + p.join().await.unwrap(); let lock = state_store.lock().await; diff --git a/src/lib.rs b/src/lib.rs index b094187..58c7f75 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,4 +5,6 @@ pub mod output; pub mod processor; pub mod processor_helper; pub mod state_store; + +#[cfg(feature = "testing")] pub mod test_utils; diff --git a/src/processor.rs b/src/processor.rs index 8f2b488..d8093be 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -60,13 +60,13 @@ where } } - pub async fn run_forever(mut self) { - let input_topcis_set: HashSet = self.inputs.keys().cloned().collect(); + pub async fn run_forever(&mut self) -> Result<(), String> { + let input_topics_set: HashSet = self.inputs.keys().cloned().collect(); let output_topics_set: HashSet = self.outputs.iter().map(|o| o.topic().to_string()).collect(); self.helper - .subscribe_inputs(&input_topcis_set) + .subscribe_inputs(&input_topics_set) .unwrap(); let num_partitions = self.helper.ensure_copartitioned().expect("Not copartitioned"); @@ -98,7 +98,7 @@ where let msg = msg.unwrap(); assert_eq!(msg.partition(), partition); - //TODO: Key serializer? + // TODO: Key serializer? let key = std::str::from_utf8(msg.key().unwrap()).unwrap(); let state = match state_store.lock().await.get(&state_namespace, key).await { @@ -147,11 +147,11 @@ where } partitions_barrier.wait().await; - self.helper - .wait_to_finish() - .await - .unwrap() - .unwrap(); + self.helper.start().await + } + + pub async fn join(&self) -> Result<(), String> { + self.helper.wait_to_finish().await } } diff --git a/src/processor_helper.rs b/src/processor_helper.rs index 7d7304d..7971997 100644 --- a/src/processor_helper.rs +++ b/src/processor_helper.rs @@ -58,6 +58,7 @@ pub enum EnsureCopartitionedError { NotCopartitioned(Vec<(String, usize)>), } +#[async_trait::async_trait] pub trait KafkaProcessorImplementor: Send { type PartitionHelperType: PartitionHelper + Sync; @@ -67,7 +68,13 @@ pub trait KafkaProcessorImplementor: Send { fn create_partitioned_consumer(&self, partition: i32) -> Self::PartitionHelperType; - fn wait_to_finish(self) -> tokio::task::JoinHandle>; + /// Kafka requires us to recv() on the main stream consumer in order for partition queues to receive messages, + /// which is an implementation detail that does not play well with the streaming abstractions. + /// To cope, we put that kind of special logic in this function. + async fn start(&self) -> Result<(), String>; + + /// This function is only relevant for testing - in real applications, the processor never finishes. + async fn wait_to_finish(&self) -> Result<(), String>; } pub struct KafkaProcessorHelper { @@ -125,6 +132,7 @@ impl PartitionHelper for KafkaPartitionProcessor { } } +#[async_trait::async_trait] impl KafkaProcessorImplementor for KafkaProcessorHelper { type PartitionHelperType = KafkaPartitionProcessor; @@ -151,15 +159,17 @@ impl KafkaProcessorImplementor for KafkaProcessorHelper { } } - fn wait_to_finish(self) -> tokio::task::JoinHandle> { - tokio::spawn(async move { - let message = self.stream_consumer.recv().await; - let err = format!( - "main stream consumer queue unexpectedly received message: {:?}", - message - ); - Err(err) - }) + async fn start(&self) -> Result<(), String> { + let message = self.stream_consumer.recv().await; + let err = format!( + "main stream consumer queue unexpectedly received message: {:?}", + message + ); + Err(err) + } + + async fn wait_to_finish(&self) -> Result<(), String> { + panic!("Cannot wait for a kafka processor to finish") } fn ensure_copartitioned(&mut self) -> Result { @@ -183,7 +193,9 @@ impl KafkaProcessorImplementor for KafkaProcessorHelper { { Ok(topics_partitions[0].1) } else { - Err(EnsureCopartitionedError::NotCopartitioned(topics_partitions)) + Err(EnsureCopartitionedError::NotCopartitioned( + topics_partitions, + )) } } } diff --git a/src/test_utils.rs b/src/test_utils.rs index 4b2ed3a..a61d2e2 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -9,7 +9,14 @@ use std::{ time::Duration, }; -use futures::{Stream, StreamExt}; +use crate::{ + encoders::{Decoder, Encoder}, + processor_helper::{ + EnsureCopartitionedError, IntoKafkaStream, KafkaProcessorImplementor, PartitionHelper, + }, +}; +use async_stream::stream; +use futures::Stream; use rdkafka::{ error::{KafkaError, KafkaResult}, message::{OwnedMessage, ToBytes}, @@ -17,38 +24,23 @@ use rdkafka::{ Message, }; use serde::Serialize; -use tokio::{sync::broadcast::error::SendError, time::sleep}; -use tokio_stream::wrappers::{errors::BroadcastStreamRecvError, BroadcastStream}; +#[derive(Clone)] pub struct TopicTestHelper { - ch_tx: tokio::sync::broadcast::Sender, + ch_tx: async_channel::Sender, sent: Arc, recv: Arc, input: bool, - // Note: we store _ch_rx so sends to a channel which isn't input - // Wont panic when there are no subscribers - _ch_rx: tokio::sync::broadcast::Receiver, -} - -impl Clone for TopicTestHelper { - fn clone(&self) -> Self { - Self { - ch_tx: self.ch_tx.clone(), - _ch_rx: self.ch_tx.subscribe(), - sent: self.sent.clone(), - recv: self.recv.clone(), - input: self.input, - } - } + ch_rx: async_channel::Receiver, } impl Default for TopicTestHelper { fn default() -> Self { - let (tx, rx) = tokio::sync::broadcast::channel(1_000_000); + let (tx, rx) = async_channel::bounded(1_000_000); Self { ch_tx: tx, - _ch_rx: rx, + ch_rx: rx, sent: Default::default(), recv: Default::default(), input: false, @@ -56,6 +48,7 @@ impl Default for TopicTestHelper { } } +#[derive(Clone)] pub struct TestsProcessorHelper { topics: HashMap, } @@ -78,7 +71,12 @@ impl TestsProcessorHelper { T: Serialize, { let input = self.topics.get_mut(topic).unwrap(); - Sender::new(topic.to_string(), decoder, input.ch_tx.clone(), input.sent.clone()) + Sender::new( + topic.to_string(), + decoder, + input.ch_tx.clone(), + input.sent.clone(), + ) } pub fn output(&mut self, topic: String, encoder: E) -> Receiver @@ -86,7 +84,7 @@ impl TestsProcessorHelper { E: Decoder, { let output = self.topics.get_mut(&topic).unwrap(); - Receiver::new(encoder, output.ch_tx.subscribe()) + Receiver::new(encoder, output.ch_rx.clone()) } } @@ -95,27 +93,27 @@ pub struct TestsPartitionProcessor { topics: HashMap, } -impl IntoKafkaStream for tokio::sync::broadcast::Sender { - type Error = BroadcastStreamRecvError; +impl IntoKafkaStream for TopicTestHelper { + type Error = async_channel::RecvError; fn stream<'a>( &'a self, ) -> Pin> + Send + 'a>> { - BroadcastStream::new(self.subscribe()).boxed() + Box::pin(stream! { + loop { + yield self.ch_rx.recv().await + } + }) } } impl PartitionHelper for TestsPartitionProcessor { type DeliveryFutureType = Pin + Send>>; - type Error = BroadcastStreamRecvError; - type OwnedStreamableType = tokio::sync::broadcast::Sender; + type Error = async_channel::RecvError; + type OwnedStreamableType = TopicTestHelper; - fn create_partitioned_topic_stream( - &self, - topic_name: &str, - ) -> tokio::sync::broadcast::Sender { - let rec = self.topics.get(topic_name).unwrap(); - rec.ch_tx.clone() + fn create_partitioned_topic_stream(&self, topic_name: &str) -> Self::OwnedStreamableType { + self.topics.get(topic_name).unwrap().clone() } fn send_result<'a, K, P>( @@ -138,8 +136,9 @@ impl PartitionHelper for TestsPartitionProcessor { let rec = self.topics.get(record.topic).unwrap(); rec.sent.fetch_add(1, Ordering::Relaxed); - rec.ch_tx.send(msg).unwrap(); - Ok(Box::pin(async move {})) + + let ch_tx_clone = rec.ch_tx.clone(); + Ok(Box::pin(async move { ch_tx_clone.send(msg).await.unwrap(); })) } fn store_offset(&mut self, topic: &str, _offset: i64) -> KafkaResult<()> { @@ -153,10 +152,7 @@ impl PartitionHelper for TestsPartitionProcessor { } } -use crate::{ - encoders::{Encoder, Decoder}, - processor_helper::{IntoKafkaStream, KafkaProcessorImplementor, PartitionHelper, EnsureCopartitionedError}, -}; +#[async_trait::async_trait] impl KafkaProcessorImplementor for TestsProcessorHelper { type PartitionHelperType = TestsPartitionProcessor; @@ -175,28 +171,31 @@ impl KafkaProcessorImplementor for TestsProcessorHelper { } } - fn wait_to_finish(self) -> tokio::task::JoinHandle> { - tokio::spawn(async move { - // TODO: This is ugly and we can implement better "wait" mechanism - // TODO: Support output topics where we don't receive every message, only if it's inputs - loop { - if self - .topics - .iter() - .filter(|(_, t)| t.input) - .all(|(_t, stats)| { - let r = stats.recv.load(Ordering::Relaxed); - let s = stats.sent.load(Ordering::Relaxed); - s == r - }) - { - break; - } - sleep(Duration::from_nanos(0)).await; + async fn start(&self) -> Result<(), String> { + Ok(()) + } + + async fn wait_to_finish(&self) -> Result<(), String> { + // TODO: This is ugly and we can implement better "wait" mechanism + // TODO: Support output topics where we don't receive every message, only if it's inputs + loop { + if self + .topics + .iter() + .filter(|(_, t)| t.input) + .all(|(_t, stats)| { + let r = stats.recv.load(Ordering::Relaxed); + let s = stats.sent.load(Ordering::Relaxed); + println!("{} {} {}", _t, s, r); + s == r && r != 0 + }) + { + break; } + tokio::time::sleep(Duration::from_millis(50)).await; + } - Ok(()) - }) + Ok(()) } fn ensure_copartitioned(&mut self) -> Result { @@ -211,7 +210,7 @@ where { topic: String, decoder: F, - tx: tokio::sync::broadcast::Sender, + tx: async_channel::Sender, sem: Arc, _marker: PhantomData, } @@ -224,7 +223,7 @@ where fn new( topic: String, decoder: F, - tx: tokio::sync::broadcast::Sender, + tx: async_channel::Sender, sem: Arc, ) -> Self { Self { @@ -236,7 +235,11 @@ where } } - pub async fn send(&mut self, key: String, msg: &T) -> Result> { + pub async fn send( + &mut self, + key: String, + msg: &T, + ) -> Result<(), async_channel::SendError> { let data = self.decoder.decode(msg); let msg = OwnedMessage::new( Some(data), @@ -250,7 +253,7 @@ where self.sem.fetch_add(1, Ordering::Relaxed); - self.tx.send(msg) + self.tx.send(msg).await } } @@ -259,23 +262,23 @@ where E: Decoder, { encoder: E, - rx: tokio::sync::broadcast::Receiver, + rx: async_channel::Receiver, } impl Receiver where E: Decoder, { - fn new(encoder: E, rx: tokio::sync::broadcast::Receiver) -> Self { + fn new(encoder: E, rx: async_channel::Receiver) -> Self { Self { encoder, rx } } - pub async fn recv(&mut self) -> Result { + pub async fn recv(&mut self) -> Result { let data = self.rx.recv().await; data.map(|d| self.encoder.encode(d.payload())) } - pub fn try_recv(&mut self) -> Result { + pub fn try_recv(&mut self) -> Result { let data = self.rx.try_recv(); data.map(|d| self.encoder.encode(d.payload())) } From a076683a03bb1e2a8f935c4d7c9a0f5405dab43b Mon Sep 17 00:00:00 2001 From: RazHemo Date: Thu, 7 Jul 2022 15:02:00 +0300 Subject: [PATCH 12/21] remove unused crates --- Cargo.lock | 1836 +++++++--------------------------------------------- Cargo.toml | 5 - 2 files changed, 219 insertions(+), 1622 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 45be4dc..ada783f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,15 +13,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" - -[[package]] -name = "arc-swap" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f" +checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704" [[package]] name = "async-channel" @@ -57,9 +51,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.53" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" +checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716" dependencies = [ "proc-macro2", "quote", @@ -72,65 +66,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "base-x" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc19a4937b4fbd3fe3379793130e42060d10627a360f2127802b10b87e7baf74" - -[[package]] -name = "base64" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" - -[[package]] -name = "bigtable_rs" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fc055036449674accbe58ae7a1638d2de7b71ee7022c894bb689fd61493feb" -dependencies = [ - "goauth", - "log 0.4.16", - "prost", - "prost-types", - "smpl_jwt", - "thiserror", - "tonic", - "tonic-build", -] - -[[package]] -name = "bindgen" -version = "0.59.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "lazy_static", - "lazycell", - "peeking_take_while", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", -] - [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bumpalo" -version = "3.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" - [[package]] name = "byteorder" version = "1.4.3" @@ -143,17 +84,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "cache-padded" version = "1.2.0" @@ -165,18 +95,6 @@ name = "cc" version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" -dependencies = [ - "jobserver", -] - -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] [[package]] name = "cfg-if" @@ -193,21 +111,10 @@ dependencies = [ "libc", "num-integer", "num-traits", - "time 0.1.44", + "time", "winapi", ] -[[package]] -name = "clang-sys" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cc00842eed744b858222c4c9faf7243aafc6d33f92f96935263ef4d8a41ce21" -dependencies = [ - "glob", - "libc", - "libloading", -] - [[package]] name = "cmake" version = "0.1.48" @@ -226,28 +133,6 @@ dependencies = [ "cache-padded", ] -[[package]] -name = "const_fn" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" - -[[package]] -name = "core-foundation" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" - [[package]] name = "crc32fast" version = "1.3.2" @@ -259,63 +144,33 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" +checksum = "07db9d94cbd326813772c968ccd25999e5f8ae22f4f8d1b11effa37ef6ce281d" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "lazy_static", "memoffset", + "once_cell", "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +checksum = "7d82ee10ce34d7bc12c2122495e7593a9c41347ecdd64185af4ecf72cb1a7f83" dependencies = [ "cfg-if", - "lazy_static", -] - -[[package]] -name = "ct-logs" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" -dependencies = [ - "sct", + "once_cell", ] -[[package]] -name = "discard" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" - [[package]] name = "dyn-clone" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21e50f3adc76d6a43f5ed73b698a87d0760ca74617f60f7c3b879003536fdd28" - -[[package]] -name = "either" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" - -[[package]] -name = "encoding_rs" -version = "0.8.31" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" -dependencies = [ - "cfg-if", -] +checksum = "140206b78fb2bc3edbcfc9b5ccbd0b30699cfe8d348b8b31b330e47df5291a5a" [[package]] name = "event-listener" @@ -332,43 +187,6 @@ dependencies = [ "instant", ] -[[package]] -name = "fixedbitset" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - -[[package]] -name = "form_urlencoded" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" -dependencies = [ - "matches", - "percent-encoding 2.1.0", -] - [[package]] name = "fs2" version = "0.4.3" @@ -450,6 +268,12 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + [[package]] name = "futures-util" version = "0.3.21" @@ -478,627 +302,208 @@ dependencies = [ ] [[package]] -name = "getrandom" -version = "0.2.6" +name = "hermit-abi" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" dependencies = [ - "cfg-if", "libc", - "wasi 0.10.0+wasi-snapshot-preview1", ] [[package]] -name = "glob" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" - -[[package]] -name = "goauth" -version = "0.9.0" +name = "instant" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d94101e84ede813c04773b0a43396c01b5a3a9376537dbce1125858ae090ae60" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "arc-swap", - "futures", - "log 0.4.16", - "reqwest", - "serde", - "serde_derive", - "serde_json", - "simpl", - "smpl_jwt", - "time 0.2.27", - "tokio", + "cfg-if", ] [[package]] -name = "google-bigtableadmin2" -version = "3.0.0+20220222" +name = "itoa" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d400c25a60cb48c6106045066a8a1a7984dd1502524ce37ac22511811eb061a1" -dependencies = [ - "hyper", - "hyper-rustls", - "itertools", - "mime 0.2.6", - "serde", - "serde_derive", - "serde_json", - "url 1.7.2", - "yup-oauth2", -] +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] -name = "h2" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +name = "kast" +version = "0.1.0" dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", + "anyhow", + "async-channel", + "async-stream", + "async-trait", + "chrono", + "dyn-clone", + "futures", + "pin-project", + "rdkafka", + "rstest", + "serde", + "serde_json", + "sled", + "tempfile", + "thiserror", "tokio", - "tokio-util 0.7.1", + "tokio-stream", "tracing", + "tracing-subscriber", ] [[package]] -name = "hashbrown" -version = "0.11.2" +name = "lazy_static" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] -name = "heck" -version = "0.3.3" +name = "libc" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] -name = "hermit-abi" -version = "0.1.19" +name = "libz-sys" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" dependencies = [ + "cc", "libc", + "pkg-config", + "vcpkg", ] [[package]] -name = "http" -version = "0.2.7" +name = "lock_api" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ - "bytes", - "fnv", - "itoa", + "autocfg", + "scopeguard", ] [[package]] -name = "http-body" -version = "0.4.4" +name = "log" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ - "bytes", - "http", - "pin-project-lite", + "cfg-if", ] [[package]] -name = "httparse" -version = "1.7.1" +name = "matchers" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] [[package]] -name = "httpdate" -version = "1.0.2" +name = "memchr" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] -name = "hyper" -version = "0.14.18" +name = "memoffset" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", + "autocfg", ] [[package]] -name = "hyper-rustls" -version = "0.22.1" +name = "mio" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" dependencies = [ - "ct-logs", - "futures-util", - "hyper", - "log 0.4.16", - "rustls", - "rustls-native-certs", - "tokio", - "tokio-rustls", - "webpki", + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys", ] [[package]] -name = "hyper-timeout" -version = "0.4.1" +name = "num-integer" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", + "autocfg", + "num-traits", ] [[package]] -name = "hyper-tls" -version = "0.5.0" +name = "num-traits" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", + "autocfg", ] [[package]] -name = "idna" -version = "0.1.5" +name = "num_cpus" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", + "hermit-abi", + "libc", ] [[package]] -name = "idna" -version = "0.2.3" +name = "num_enum" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +checksum = "cf5395665662ef45796a4ff5486c5d41d29e0c09640af4c5f17fd94ee2c119c9" dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", + "num_enum_derive", ] [[package]] -name = "indexmap" -version = "1.8.1" +name = "num_enum_derive" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" +checksum = "3b0498641e53dd6ac1a4f22547548caa6864cc4933784319cd1775271c5a46ce" dependencies = [ - "autocfg", - "hashbrown", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "instant" -version = "0.1.12" +name = "once_cell" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] +checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" [[package]] -name = "ipnet" -version = "2.5.0" +name = "parking_lot" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" - -[[package]] -name = "itertools" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" - -[[package]] -name = "jobserver" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" -dependencies = [ - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "kast" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-channel", - "async-stream", - "async-trait", - "bigtable_rs", - "chrono", - "dyn-clone", - "futures", - "google-bigtableadmin2", - "pin-project", - "rdkafka", - "rocksdb", - "rstest", - "serde", - "serde_json", - "sled", - "tempfile", - "thiserror", - "tokio", - "tokio-stream", - "tracing", - "tracing-subscriber", - "yup-oauth2", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - -[[package]] -name = "libc" -version = "0.2.125" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" - -[[package]] -name = "libloading" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" -dependencies = [ - "cfg-if", - "winapi", -] - -[[package]] -name = "librocksdb-sys" -version = "0.6.1+6.28.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc587013734dadb7cf23468e531aa120788b87243648be42e2d3a072186291" -dependencies = [ - "bindgen", - "bzip2-sys", - "cc", - "glob", - "libc", - "libz-sys", - "zstd-sys", -] - -[[package]] -name = "libz-sys" -version = "1.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e7e15d7610cce1d9752e137625f14e61a28cd45929b6e12e47b50fe154ee2e" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "lock_api" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" -dependencies = [ - "log 0.4.16", -] - -[[package]] -name = "log" -version = "0.4.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "matchers" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" -dependencies = [ - "regex-automata", -] - -[[package]] -name = "matches" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" - -[[package]] -name = "memchr" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" - -[[package]] -name = "memoffset" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" -dependencies = [ - "autocfg", -] - -[[package]] -name = "mime" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" -dependencies = [ - "log 0.3.9", -] - -[[package]] -name = "mime" -version = "0.3.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "mio" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" -dependencies = [ - "libc", - "log 0.4.16", - "miow", - "ntapi", - "wasi 0.11.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", -] - -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - -[[package]] -name = "native-tls" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" -dependencies = [ - "lazy_static", - "libc", - "log 0.4.16", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - -[[package]] -name = "nom" -version = "7.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "ntapi" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" -dependencies = [ - "winapi", -] - -[[package]] -name = "num-integer" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" -dependencies = [ - "autocfg", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" -dependencies = [ - "autocfg", -] - -[[package]] -name = "num_cpus" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "num_enum" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf5395665662ef45796a4ff5486c5d41d29e0c09640af4c5f17fd94ee2c119c9" -dependencies = [ - "num_enum_derive", -] - -[[package]] -name = "num_enum_derive" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0498641e53dd6ac1a4f22547548caa6864cc4933784319cd1775271c5a46ce" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "num_threads" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aba1801fb138d8e85e11d0fc70baf4fe1cdfffda7c6cd34a854905df588e5ed0" -dependencies = [ - "libc", -] - -[[package]] -name = "once_cell" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" - -[[package]] -name = "openssl" -version = "0.10.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-sys", -] - -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - -[[package]] -name = "openssl-sys" -version = "0.9.72" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" -dependencies = [ - "autocfg", - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.5", -] +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.5", +] [[package]] name = "parking_lot" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.2", + "parking_lot_core 0.9.3", ] [[package]] @@ -1117,9 +522,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if", "libc", @@ -1128,48 +533,20 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - -[[package]] -name = "percent-encoding" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" - -[[package]] -name = "percent-encoding" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" - -[[package]] -name = "petgraph" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7" -dependencies = [ - "fixedbitset", - "indexmap", -] - [[package]] name = "pin-project" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" +checksum = "78203e83c48cffbe01e4a2d35d566ca4de445d79a85372fc64e378bfc812a260" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" +checksum = "710faf75e1b33345361201d36d04e98ac1ed8909151a017ed384700836104c74" dependencies = [ "proc-macro2", "quote", @@ -1194,12 +571,6 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" -[[package]] -name = "ppv-lite86" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" - [[package]] name = "proc-macro-crate" version = "1.1.3" @@ -1210,111 +581,24 @@ dependencies = [ "toml", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" - [[package]] name = "proc-macro2" -version = "1.0.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" -dependencies = [ - "unicode-xid", -] - -[[package]] -name = "prost" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355f634b43cdd80724ee7848f95770e7e70eefa6dcf14fea676216573b8fd603" -dependencies = [ - "bytes", - "heck", - "itertools", - "log 0.4.16", - "multimap", - "petgraph", - "prost", - "prost-types", - "tempfile", - "which", -] - -[[package]] -name = "prost-derive" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "600d2f334aa05acb02a755e217ef1ab6dea4d51b58b7846588b747edec04efba" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "prost-types" -version = "0.8.0" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b" +checksum = "dd96a1e8ed2596c337f8eae5f24924ec83f5ad5ab21ea8e455d3566c69fbcaf7" dependencies = [ - "bytes", - "prost", + "unicode-ident", ] [[package]] name = "quote" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804" dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" -dependencies = [ - "getrandom", -] - [[package]] name = "rdkafka" version = "0.28.0" @@ -1323,7 +607,7 @@ checksum = "1de127f294f2dba488ed46760b129d5ecbeabbd337ccbf3739cb29d50db2161c" dependencies = [ "futures", "libc", - "log 0.4.16", + "log", "rdkafka-sys", "serde", "serde_derive", @@ -1356,9 +640,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.5" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" dependencies = [ "regex-syntax", ] @@ -1374,9 +658,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.6.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] name = "remove_dir_all" @@ -1388,143 +672,44 @@ dependencies = [ ] [[package]] -name = "reqwest" -version = "0.11.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb" -dependencies = [ - "base64", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-tls", - "ipnet", - "js-sys", - "lazy_static", - "log 0.4.16", - "mime 0.3.16", - "native-tls", - "percent-encoding 2.1.0", - "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", - "tokio", - "tokio-native-tls", - "url 2.2.2", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg", -] - -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin", - "untrusted", - "web-sys", - "winapi", -] - -[[package]] -name = "rocksdb" -version = "0.18.0" +name = "rstest" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "620f4129485ff1a7128d184bc687470c21c7951b64779ebc9cfdad3dcd920290" +checksum = "e9c9dc66cc29792b663ffb5269be669f1613664e69ad56441fdb895c2347b930" dependencies = [ - "libc", - "librocksdb-sys", + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", ] [[package]] -name = "rstest" -version = "0.12.0" +name = "rstest_macros" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d912f35156a3f99a66ee3e11ac2e0b3f34ac85a07e05263d05a7e2c8810d616f" +checksum = "5015e68a0685a95ade3eee617ff7101ab6a3fc689203101ca16ebc16f2b89c66" dependencies = [ "cfg-if", "proc-macro2", "quote", - "rustc_version 0.4.0", + "rustc_version", "syn", ] -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] - [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.7", -] - -[[package]] -name = "rustls" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" -dependencies = [ - "base64", - "log 0.4.16", - "ring", - "sct", - "webpki", -] - -[[package]] -name = "rustls-native-certs" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" -dependencies = [ - "openssl-probe", - "rustls", - "schannel", - "security-framework", + "semver", ] [[package]] name = "ryu" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" - -[[package]] -name = "schannel" -version = "0.1.19" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" -dependencies = [ - "lazy_static", - "winapi", -] +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" [[package]] name = "scopeguard" @@ -1532,80 +717,26 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" -[[package]] -name = "sct" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - -[[package]] -name = "security-framework" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - [[package]] name = "semver" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" - -[[package]] -name = "semver-parser" -version = "0.7.0" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +checksum = "a2333e6df6d6598f2b1974829f853c2b4c5f4a6e503c10af918081aa6f8564e1" [[package]] name = "serde" -version = "1.0.136" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +checksum = "1578c6245786b9d168c5447eeacfb96856573ca56c9d68fdcf394be134882a47" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.136" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +checksum = "023e9b1467aef8a10fb88f25611870ada9800ef7e22afce356bb0d2387b6f27c" dependencies = [ "proc-macro2", "quote", @@ -1614,42 +745,15 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" dependencies = [ - "form_urlencoded", "itoa", "ryu", "serde", ] -[[package]] -name = "sha1" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" -dependencies = [ - "sha1_smol", -] - -[[package]] -name = "sha1_smol" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" - [[package]] name = "sharded-slab" version = "0.1.4" @@ -1659,12 +763,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "shlex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" - [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -1674,12 +772,6 @@ dependencies = [ "libc", ] -[[package]] -name = "simpl" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a30f10c911c0355f80f1c2faa8096efc4a58cdf8590b954d5b395efa071c711" - [[package]] name = "slab" version = "0.4.6" @@ -1698,115 +790,35 @@ dependencies = [ "fs2", "fxhash", "libc", - "log 0.4.16", + "log", "parking_lot 0.11.2", ] [[package]] -name = "smallvec" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" - -[[package]] -name = "smpl_jwt" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4370044f8b20f944e05c35d77edd3518e6f21fc4de77e593919f287c6a3f428a" -dependencies = [ - "base64", - "log 0.4.16", - "openssl", - "serde", - "serde_derive", - "serde_json", - "simpl", - "time 0.2.27", -] - -[[package]] -name = "socket2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - -[[package]] -name = "standback" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" -dependencies = [ - "version_check", -] - -[[package]] -name = "stdweb" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" -dependencies = [ - "discard", - "rustc_version 0.2.3", - "stdweb-derive", - "stdweb-internal-macros", - "stdweb-internal-runtime", - "wasm-bindgen", -] - -[[package]] -name = "stdweb-derive" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "serde_derive", - "syn", -] - -[[package]] -name = "stdweb-internal-macros" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" -dependencies = [ - "base-x", - "proc-macro2", - "quote", - "serde", - "serde_derive", - "serde_json", - "sha1", - "syn", -] - -[[package]] -name = "stdweb-internal-runtime" -version = "0.1.5" +name = "smallvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" + +[[package]] +name = "socket2" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +dependencies = [ + "libc", + "winapi", +] [[package]] name = "syn" -version = "1.0.92" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff7c592601f11445996a06f8ad0c27f094a58857c2f89e97974ab9235b92c52" +checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] @@ -1863,75 +875,11 @@ dependencies = [ "winapi", ] -[[package]] -name = "time" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" -dependencies = [ - "const_fn", - "libc", - "standback", - "stdweb", - "time-macros", - "version_check", - "winapi", -] - -[[package]] -name = "time" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" -dependencies = [ - "libc", - "num_threads", - "serde", -] - -[[package]] -name = "time-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" -dependencies = [ - "proc-macro-hack", - "time-macros-impl", -] - -[[package]] -name = "time-macros-impl" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" -dependencies = [ - "proc-macro-hack", - "proc-macro2", - "quote", - "standback", - "syn", -] - -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" - [[package]] name = "tokio" -version = "1.18.0" +version = "1.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f48b6d60512a392e34dbf7fd456249fd2de3c83669ab642e021903f4015185b" +checksum = "c51a52ed6686dd62c320f9b89299e9dfb46f730c7a48e635c19f21d116cb1439" dependencies = [ "bytes", "libc", @@ -1939,7 +887,7 @@ dependencies = [ "mio", "num_cpus", "once_cell", - "parking_lot 0.12.0", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2", @@ -1947,86 +895,40 @@ dependencies = [ "winapi", ] -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-macros" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" dependencies = [ "proc-macro2", "quote", "syn", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" -dependencies = [ - "rustls", - "tokio", - "webpki", -] - [[package]] name = "tokio-stream" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", - "tokio-util 0.6.9", -] - -[[package]] -name = "tokio-util" -version = "0.6.9" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" +checksum = "df54d54117d6fdc4e4fea40fe1e4e566b3505700e148a6827e59b34b0d2600d9" dependencies = [ - "bytes", "futures-core", - "futures-sink", - "log 0.4.16", "pin-project-lite", "tokio", + "tokio-util", ] [[package]] name = "tokio-util" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" +checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", - "tracing", ] [[package]] @@ -2038,90 +940,13 @@ dependencies = [ "serde", ] -[[package]] -name = "tonic" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "796c5e1cd49905e65dd8e700d4cb1dffcbfdb4fc9d017de08c1a537afd83627c" -dependencies = [ - "async-stream", - "async-trait", - "base64", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding 2.1.0", - "pin-project", - "prost", - "prost-derive", - "tokio", - "tokio-rustls", - "tokio-stream", - "tokio-util 0.6.9", - "tower", - "tower-layer", - "tower-service", - "tracing", - "tracing-futures", -] - -[[package]] -name = "tonic-build" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b52d07035516c2b74337d2ac7746075e7dcae7643816c1b12c5ff8a7484c08" -dependencies = [ - "proc-macro2", - "prost-build", - "quote", - "syn", -] - -[[package]] -name = "tower" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e" -dependencies = [ - "futures-core", - "futures-util", - "indexmap", - "pin-project", - "pin-project-lite", - "rand", - "slab", - "tokio", - "tokio-util 0.7.1", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" - -[[package]] -name = "tower-service" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" - [[package]] name = "tracing" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" +checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" dependencies = [ "cfg-if", - "log 0.4.16", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2129,9 +954,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" dependencies = [ "proc-macro2", "quote", @@ -2140,24 +965,14 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.26" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" +checksum = "7b7358be39f2f274f322d2aaed611acc57f382e8eb1e5b48cb9ae30933495ce7" dependencies = [ - "lazy_static", + "once_cell", "valuable", ] -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - [[package]] name = "tracing-log" version = "0.1.3" @@ -2165,19 +980,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" dependencies = [ "lazy_static", - "log 0.4.16", + "log", "tracing-core", ] [[package]] name = "tracing-subscriber" -version = "0.3.11" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bc28f93baff38037f64e6f43d34cfa1605f27a49c34e8a04c5e78b0babf2596" +checksum = "3a713421342a5a666b7577783721d3117f1b69a393df803ee17bb73b1e122a59" dependencies = [ "ansi_term", - "lazy_static", "matchers", + "once_cell", "regex", "sharded-slab", "smallvec", @@ -2188,66 +1003,10 @@ dependencies = [ ] [[package]] -name = "try-lock" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" - -[[package]] -name = "unicode-bidi" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" - -[[package]] -name = "unicode-normalization" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-segmentation" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" - -[[package]] -name = "unicode-xid" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" - -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - -[[package]] -name = "url" -version = "1.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" -dependencies = [ - "idna 0.1.5", - "matches", - "percent-encoding 1.0.1", -] - -[[package]] -name = "url" -version = "2.2.2" +name = "unicode-ident" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" -dependencies = [ - "form_urlencoded", - "idna 0.2.3", - "matches", - "percent-encoding 2.1.0", -] +checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c" [[package]] name = "valuable" @@ -2261,22 +1020,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "want" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" -dependencies = [ - "log 0.4.16", - "try-lock", -] - [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" @@ -2289,103 +1032,6 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" -[[package]] -name = "wasm-bindgen" -version = "0.2.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" -dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" -dependencies = [ - "bumpalo", - "lazy_static", - "log 0.4.16", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" - -[[package]] -name = "web-sys" -version = "0.3.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki" -version = "0.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "which" -version = "4.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" -dependencies = [ - "either", - "lazy_static", - "libc", -] - [[package]] name = "winapi" version = "0.3.9" @@ -2410,9 +1056,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -2423,74 +1069,30 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_i686_gnu" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_x86_64_gnu" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_msvc" -version = "0.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" - -[[package]] -name = "winreg" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" -dependencies = [ - "winapi", -] - -[[package]] -name = "yup-oauth2" -version = "6.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfccb8835215946ebac32b89db2a4c7fbec7f289ea0e3ea53cfe2552983e257b" -dependencies = [ - "anyhow", - "async-trait", - "base64", - "futures", - "http", - "hyper", - "hyper-rustls", - "itertools", - "log 0.4.16", - "percent-encoding 2.1.0", - "rustls", - "seahash", - "serde", - "serde_json", - "time 0.3.9", - "tokio", - "url 2.2.2", -] - -[[package]] -name = "zstd-sys" -version = "1.6.3+zstd.1.5.2" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc49afa5c8d634e75761feda8c592051e7eeb4683ba827211eb0d731d3402ea8" -dependencies = [ - "cc", - "libc", -] +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" diff --git a/Cargo.toml b/Cargo.toml index 75cd736..859b410 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,14 +33,9 @@ serde_json = "1.0.79" anyhow = "1.0.56" pin-project = "*" sled = "0.34.7" -rocksdb = "0.18.0" -bigtable_rs = "0.1.5" dyn-clone = "1.0.5" chrono = "0.4.19" async-trait = "0.1.53" -google-bigtableadmin2 = "3.0.0" -# bigtable depends on `yup-oauth2` ^6, but on hyper-rustls==0.22, but `yup-oauth2` 6.6.0 pulls hyper-rustls==0.23 :( -yup-oauth2 = "=6.5.0" thiserror = "1.0.31" async-channel = { version = "*", optional = true } From f33670f5d11419a0f41e9bebbbdd2f9f5953b288 Mon Sep 17 00:00:00 2001 From: RazHemo Date: Thu, 7 Jul 2022 18:07:34 +0300 Subject: [PATCH 13/21] fix naming --- src/encoders.rs | 10 +++++----- src/input.rs | 20 ++++++++++---------- src/test_utils.rs | 26 +++++++++++++------------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/encoders.rs b/src/encoders.rs index 1e1bade..bd7c942 100644 --- a/src/encoders.rs +++ b/src/encoders.rs @@ -4,7 +4,7 @@ use std::marker::PhantomData; pub trait Decoder { type In; - fn encode(&self, data: Option<&[u8]>) -> Self::In; + fn decode(&self, data: Option<&[u8]>) -> Self::In; } impl Decoder for F @@ -13,13 +13,13 @@ where { type In = T; - fn encode(&self, data: Option<&[u8]>) -> Self::In { + fn decode(&self, data: Option<&[u8]>) -> Self::In { (self)(data) } } pub trait Encoder { - fn decode(&self, data: &T) -> Vec + fn encode(&self, data: &T) -> Vec where T: Serialize; } @@ -62,7 +62,7 @@ where { type In = T; - fn encode(&self, data: Option<&[u8]>) -> Self::In { + fn decode(&self, data: Option<&[u8]>) -> Self::In { serde_json::from_slice(data.expect("empty message")).unwrap() } } @@ -80,7 +80,7 @@ impl Default for JsonEncoder { } } impl Encoder for JsonEncoder { - fn decode(&self, data: &T) -> Vec + fn encode(&self, data: &T) -> Vec where T: Serialize, { diff --git a/src/input.rs b/src/input.rs index bebb12a..027cc6b 100644 --- a/src/input.rs +++ b/src/input.rs @@ -19,24 +19,24 @@ pub trait Handler<'a, T, S, R, TStore, VArgs> { } #[derive(Clone)] -pub struct Input +pub struct Input where - E: Decoder, + D: Decoder, { topic: String, - encoder: E, + decoder: D, callback: F, _marker: PhantomData<(S, VArgs)>, } -impl Input +impl Input where - E: Decoder, + D: Decoder, { - pub fn new(topic: &str, encoder: E, callback: F) -> Box { + pub fn new(topic: &str, decoder: D, callback: F) -> Box { Box::new(Input { topic: topic.to_string(), - encoder, + decoder, callback, _marker: PhantomData, }) @@ -44,10 +44,10 @@ where } #[async_trait::async_trait] -impl GenericInput for Input +impl GenericInput for Input where for<'a> F: Handler<'a, T, S, R, TStore, VArgs> + Send + Sync + Copy, - E: Decoder + Sync + Clone + Send + 'static, + D: Decoder + Sync + Clone + Send + 'static, R: Sync + Send + Clone + DeserializeOwned + 'static + std::fmt::Debug, T: Clone + Send + Sync + 'static, TStore: Send + Sync, @@ -59,7 +59,7 @@ where } async fn handle(&self, state: &mut S, ctx: &mut Context, data: Option<&[u8]>, headers: HashMap) -> Option { - let msg = self.encoder.encode(data); + let msg = self.decoder.decode(data); self.callback.call(state, ctx, msg, headers).await } } diff --git a/src/test_utils.rs b/src/test_utils.rs index a61d2e2..3d1a136 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -209,7 +209,7 @@ where F: Encoder, { topic: String, - decoder: F, + encoder: F, tx: async_channel::Sender, sem: Arc, _marker: PhantomData, @@ -222,13 +222,13 @@ where { fn new( topic: String, - decoder: F, + encoder: F, tx: async_channel::Sender, sem: Arc, ) -> Self { Self { topic, - decoder, + encoder, tx, sem, _marker: PhantomData, @@ -240,7 +240,7 @@ where key: String, msg: &T, ) -> Result<(), async_channel::SendError> { - let data = self.decoder.decode(msg); + let data = self.encoder.encode(msg); let msg = OwnedMessage::new( Some(data), Some(key.as_bytes().to_vec()), @@ -257,29 +257,29 @@ where } } -pub struct Receiver +pub struct Receiver where - E: Decoder, + D: Decoder, { - encoder: E, + decoder: D, rx: async_channel::Receiver, } -impl Receiver +impl Receiver where - E: Decoder, + D: Decoder, { - fn new(encoder: E, rx: async_channel::Receiver) -> Self { - Self { encoder, rx } + fn new(decoder: D, rx: async_channel::Receiver) -> Self { + Self { decoder, rx } } pub async fn recv(&mut self) -> Result { let data = self.rx.recv().await; - data.map(|d| self.encoder.encode(d.payload())) + data.map(|d| self.decoder.decode(d.payload())) } pub fn try_recv(&mut self) -> Result { let data = self.rx.try_recv(); - data.map(|d| self.encoder.encode(d.payload())) + data.map(|d| self.decoder.decode(d.payload())) } } From 4df36e70cca4a9a8c6e882293e66468d0aa81f4f Mon Sep 17 00:00:00 2001 From: RazHemo Date: Sun, 17 Jul 2022 18:02:21 +0300 Subject: [PATCH 14/21] add protobuf decoder --- Cargo.lock | 39 +++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/encoders.rs | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index ada783f..0afd1c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -172,6 +172,12 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "140206b78fb2bc3edbcfc9b5ccbd0b30699cfe8d348b8b31b330e47df5291a5a" +[[package]] +name = "either" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be" + [[package]] name = "event-listener" version = "2.5.2" @@ -319,6 +325,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "itertools" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.2" @@ -337,6 +352,7 @@ dependencies = [ "dyn-clone", "futures", "pin-project", + "prost", "rdkafka", "rstest", "serde", @@ -590,6 +606,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71adf41db68aa0daaefc69bb30bcd68ded9b9abaad5d1fbb6304c4fb390e083e" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "quote" version = "1.0.20" diff --git a/Cargo.toml b/Cargo.toml index 859b410..f88f902 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ tracing-subscriber = { version = "0.3.11", features = [ async-stream = "0.3.3" serde = { version = "1", features = ["derive"] } serde_json = "1.0.79" +prost = "0.10.4" anyhow = "1.0.56" pin-project = "*" sled = "0.34.7" diff --git a/src/encoders.rs b/src/encoders.rs index bd7c942..170c7c6 100644 --- a/src/encoders.rs +++ b/src/encoders.rs @@ -87,3 +87,35 @@ impl Encoder for JsonEncoder { serde_json::to_vec(&data).unwrap() } } + + +// Protobuf +#[derive(Clone)] +pub struct ProtobufDecoder { + _marker: PhantomData, +} + +impl ProtobufDecoder { + pub fn new() -> Self { + Self { + _marker: PhantomData, + } + } +} + +impl Default for ProtobufDecoder { + fn default() -> Self { + Self::new() + } +} + +impl Decoder for ProtobufDecoder +where + T: prost::Message + Default, +{ + type In = T; + + fn decode(&self, data: Option<&[u8]>) -> Self::In { + T::decode(data.expect("empty message")).unwrap() + } +} \ No newline at end of file From 2d8bdbcf8fe687ce821031c0a6991f44af63526e Mon Sep 17 00:00:00 2001 From: RazHemo Date: Tue, 16 Aug 2022 22:41:56 +0300 Subject: [PATCH 15/21] propagate panics in partition consumers --- src/processor.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/processor.rs b/src/processor.rs index d8093be..02e54ab 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -4,7 +4,7 @@ use std::{ sync::Arc, }; -use futures::{future::join_all, stream::select_all, StreamExt}; +use futures::{future::join_all, stream::select_all, StreamExt, try_join}; use rdkafka::{producer::FutureRecord, Message, message::{OwnedHeaders, Headers}}; use tokio::sync::{Barrier, Mutex}; @@ -72,6 +72,7 @@ where let num_partitions = self.helper.ensure_copartitioned().expect("Not copartitioned"); let partitions_barrier = Arc::new(Barrier::new(num_partitions + 1)); + let mut tokio_tasks = Vec::new(); for partition in 0..num_partitions as i32 { let inputs = self.inputs.clone(); let output_topics_set = output_topics_set.clone(); @@ -80,7 +81,7 @@ where let gen = self.state_store_gen.clone(); let gen2 = self.extra_state_gen.clone(); let state_namespace = self.state_namespace.clone(); - tokio::spawn(async move { + tokio_tasks.push(tokio::spawn(async move { let state_store = gen(); let mut extra_state = gen2(); let stream_gens: Vec<_> = inputs.keys().into_iter().map(|topic| { @@ -143,11 +144,18 @@ where helper_clone.store_offset(&msg_topic, *lock).unwrap(); }); } - }); + })); } partitions_barrier.wait().await; - self.helper.start().await + try_join!( + async { + futures::future::try_join_all(tokio_tasks).await.map_err( + |e| format!("Kast processor {} panicked: {:?}", self.state_namespace, e) + ) + }, + self.helper.start() + ).map(|_| ()) } pub async fn join(&self) -> Result<(), String> { From d7d06e3d96b452ca949d64732f9ef388b0d85262 Mon Sep 17 00:00:00 2001 From: RazHemo Date: Tue, 16 Aug 2022 23:20:27 +0300 Subject: [PATCH 16/21] fix bug --- src/processor.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/processor.rs b/src/processor.rs index 02e54ab..7e02807 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -4,9 +4,9 @@ use std::{ sync::Arc, }; -use futures::{future::join_all, stream::select_all, StreamExt, try_join}; +use futures::{future::join_all, stream::select_all, StreamExt}; use rdkafka::{producer::FutureRecord, Message, message::{OwnedHeaders, Headers}}; -use tokio::sync::{Barrier, Mutex}; +use tokio::{sync::{Barrier, Mutex}, select}; use crate::{ context::Context, @@ -148,14 +148,12 @@ where } partitions_barrier.wait().await; - try_join!( - async { - futures::future::try_join_all(tokio_tasks).await.map_err( - |e| format!("Kast processor {} panicked: {:?}", self.state_namespace, e) - ) - }, - self.helper.start() - ).map(|_| ()) + select!( + f = futures::future::try_join_all(tokio_tasks) => f.map_err( + |e| format!("Kast processor {} panicked: {:?}", self.state_namespace, e) + ).map(|_| ()), + f = self.helper.start() => f + ) } pub async fn join(&self) -> Result<(), String> { From afd6195900bb02f753e91cd0b69a565282b8486a Mon Sep 17 00:00:00 2001 From: RazHemo Date: Wed, 17 Aug 2022 11:49:45 +0300 Subject: [PATCH 17/21] try to debug --- src/processor.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/processor.rs b/src/processor.rs index 7e02807..07f4a63 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -147,6 +147,8 @@ where })); } + return Err("testing".to_string()); + partitions_barrier.wait().await; select!( f = futures::future::try_join_all(tokio_tasks) => f.map_err( From 2b4335cda4872e5f45053d949d5d4db8265664af Mon Sep 17 00:00:00 2001 From: RazHemo Date: Wed, 17 Aug 2022 12:26:15 +0300 Subject: [PATCH 18/21] revert debugging --- src/processor.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/processor.rs b/src/processor.rs index 07f4a63..7e02807 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -147,8 +147,6 @@ where })); } - return Err("testing".to_string()); - partitions_barrier.wait().await; select!( f = futures::future::try_join_all(tokio_tasks) => f.map_err( From 8da174678b724f0bba26f498dd716f54fbcf6a77 Mon Sep 17 00:00:00 2001 From: RazHemo Date: Wed, 17 Aug 2022 13:46:47 +0300 Subject: [PATCH 19/21] abort all consumers when one panics --- src/processor.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/processor.rs b/src/processor.rs index 7e02807..dd8b351 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -148,12 +148,18 @@ where } partitions_barrier.wait().await; - select!( - f = futures::future::try_join_all(tokio_tasks) => f.map_err( + let result = select!( + f = futures::future::try_join_all(tokio_tasks.iter_mut()) => f.map_err( |e| format!("Kast processor {} panicked: {:?}", self.state_namespace, e) ).map(|_| ()), f = self.helper.start() => f - ) + ); + + for t in tokio_tasks { + t.abort(); + } + + result } pub async fn join(&self) -> Result<(), String> { From bb085c3f344da65a9e66f791a7307defbdf18fe4 Mon Sep 17 00:00:00 2001 From: RazHemo Date: Wed, 17 Aug 2022 14:13:22 +0300 Subject: [PATCH 20/21] fix aborting for tests --- src/processor.rs | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/processor.rs b/src/processor.rs index dd8b351..536e3cd 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -16,6 +16,17 @@ use crate::{ state_store::{StateStore, StateStoreError}, }; +fn format_join_error(e: tokio::task::JoinError) -> String { + let p = e.into_panic(); + match p.downcast::() { + Ok(v) => *v, + Err(p) => match p.downcast::<&str>() { + Ok(v) => v.to_string(), + Err(_) => "Unknown panic type".to_string(), + }, + } +} + pub struct Processor { state_namespace: String, helper: H, @@ -147,18 +158,21 @@ where })); } + partitions_barrier.wait().await; let result = select!( - f = futures::future::try_join_all(tokio_tasks.iter_mut()) => f.map_err( - |e| format!("Kast processor {} panicked: {:?}", self.state_namespace, e) - ).map(|_| ()), + f = futures::future::try_join_all(tokio_tasks.iter_mut()) => { + for t in tokio_tasks { + t.abort(); + } + + f.map_err( + |e| format!("Kast processor {} panicked: {:?}", self.state_namespace, format_join_error(e)) + ).map(|_| ()) + }, f = self.helper.start() => f ); - for t in tokio_tasks { - t.abort(); - } - result } From 72552cd291c92f48a014b2380e35ae51d5807001 Mon Sep 17 00:00:00 2001 From: RazHemo Date: Wed, 17 Aug 2022 15:25:22 +0300 Subject: [PATCH 21/21] added debug logs --- src/processor.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/processor.rs b/src/processor.rs index 536e3cd..e368e8c 100644 --- a/src/processor.rs +++ b/src/processor.rs @@ -112,7 +112,8 @@ where // TODO: Key serializer? let key = std::str::from_utf8(msg.key().unwrap()).unwrap(); - + tracing::debug!("Received kafka message on {}:{}", state_namespace, key); + let state = match state_store.lock().await.get(&state_namespace, key).await { Ok(state) => Some(state), Err(StateStoreError::MissingKey(_)) => None, @@ -143,6 +144,7 @@ where let mut helper_clone = partition_handler.clone(); let offset_lock_for_partitioned_topic = topic_partition_offset_locks.entry(msg_topic.clone()).or_insert_with(|| Arc::new(Mutex::new(msg_offset))).clone(); + tracing::debug!("Handled kafka message on {}:{}", state_namespace, key); tokio::spawn(async move { join_all(sends).await;