From d17e1b12ee07883991bd4a2112392c34c63acfed Mon Sep 17 00:00:00 2001 From: Stephen Carman Date: Fri, 7 Nov 2025 10:46:12 -0500 Subject: [PATCH 1/3] Upgrade dependencies, fix a few lints --- examples/basic_example_proto/Cargo.toml | 11 +++-------- sdk/Cargo.toml | 16 ++++++++-------- sdk/build.rs | 3 +-- sdk/src/default_token_factory.rs | 17 +++++++---------- sdk/src/headers_provider.rs | 2 +- sdk/src/landing_zone.rs | 4 ++-- sdk/src/stream_configuration.rs | 2 +- tests/Cargo.toml | 14 +++++++------- tests/build.rs | 8 +++----- tools/generate_files/Cargo.toml | 9 ++------- tools/generate_files/src/generate.rs | 2 +- 11 files changed, 36 insertions(+), 52 deletions(-) diff --git a/examples/basic_example_proto/Cargo.toml b/examples/basic_example_proto/Cargo.toml index 96c9246..629550a 100644 --- a/examples/basic_example_proto/Cargo.toml +++ b/examples/basic_example_proto/Cargo.toml @@ -6,12 +6,7 @@ publish = false [dependencies] databricks-zerobus-ingest-sdk = { path = "../../sdk" } -tonic-build = "0.13.1" -prost = "0.13.3" -prost-build = "0.12" -prost-reflect = "0.14.2" -tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] } -tokio-stream = "0.1.16" -tonic = { version = "0.12.3", features = ["tls"] } -tracing = "0.1.41" +prost = "0.14.1" +prost-reflect = "0.16.2" +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } chrono = "0.4" diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml index 3e63b88..ff7f6db 100644 --- a/sdk/Cargo.toml +++ b/sdk/Cargo.toml @@ -14,20 +14,20 @@ documentation = "https://docs.rs/databricks-zerobus-ingest-sdk" [dependencies] async-trait = "0.1" -prost = "0.13.3" -prost-types = "0.13.3" +prost = "0.14.1" +prost-types = "0.14.1" reqwest = { version = "0.12", features = ["json", "rustls-tls"], default-features = false } serde_json = "1.0" -thiserror = "1.0" -tokio = { version = "1.42.0", features = ["macros", "rt-multi-thread", "fs"] } +thiserror = "2.0.17" +tokio = { version = "1", features = ["macros", "rt-multi-thread", "fs"] } tokio-retry = "0.3" -tokio-stream = "0.1.16" -tonic = { version = "0.12.3", features = ["tls", "tls-native-roots", "transport"] } +tokio-stream = "0.1" +tonic = { version = "0.14.2", features = ["tls-aws-lc", "transport", "tls-native-roots"] } +tonic-prost = "0.14.2" tracing = "0.1.41" [build-dependencies] -tonic-build = "0.12.3" -protoc-bin-vendored = "3.0.0" +tonic-prost-build = "0.14.2" [features] testing = [] diff --git a/sdk/build.rs b/sdk/build.rs index 54001c7..11c5407 100644 --- a/sdk/build.rs +++ b/sdk/build.rs @@ -1,5 +1,4 @@ fn main() { - std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path().unwrap()); - tonic_build::compile_protos("zerobus_service.proto") + tonic_prost_build::compile_protos("zerobus_service.proto") .unwrap_or_else(|e| panic!("Failed to compile protos {:?}", e)); } diff --git a/sdk/src/default_token_factory.rs b/sdk/src/default_token_factory.rs index 9480636..5044ae9 100644 --- a/sdk/src/default_token_factory.rs +++ b/sdk/src/default_token_factory.rs @@ -49,13 +49,13 @@ impl DefaultTokenFactory { "type": "unity_catalog_privileges", "privileges": ["USE SCHEMA"], "object_type": "SCHEMA", - "object_full_path": format!("{}.{}", catalog, schema) + "object_full_path": format!("{catalog}.{schema}") }, { "type": "unity_catalog_privileges", "privileges": ["SELECT", "MODIFY"], "object_type": "TABLE", - "object_full_path": format!("{}.{}.{}", catalog, schema, table) + "object_full_path": format!("{catalog}.{schema}.{table}") } ]); @@ -66,16 +66,13 @@ impl DefaultTokenFactory { ("scope", "all-apis".to_string()), ( "resource", - format!( - "api://databricks/workspaces/{}/zerobusDirectWriteApi", - workspace_id - ) - .to_string(), + format!("api://databricks/workspaces/{workspace_id}/zerobusDirectWriteApi") + .to_string(), ), ("authorization_details", authorization_details.to_string()), ]; - let token_endpoint = format!("{}/oidc/v1/token", uc_endpoint); + let token_endpoint = format!("{uc_endpoint}/oidc/v1/token"); let resp = client .post(&token_endpoint) .basic_auth(databricks_client_id, Some(databricks_client_secret)) @@ -83,7 +80,7 @@ impl DefaultTokenFactory { .send() .await .map_err(|e| { - ZerobusError::InvalidUCTokenError(format!("Request failed with error: {}", e)) + ZerobusError::InvalidUCTokenError(format!("Request failed with error: {e}")) })?; if !resp.status().is_success() { @@ -100,7 +97,7 @@ impl DefaultTokenFactory { } let body: serde_json::Value = resp.json().await.map_err(|e| { - ZerobusError::InvalidUCTokenError(format!("Parse failed with error: {}", e)) + ZerobusError::InvalidUCTokenError(format!("Parse failed with error: {e}")) })?; let token = body["access_token"] diff --git a/sdk/src/headers_provider.rs b/sdk/src/headers_provider.rs index 02d7166..354bd2c 100644 --- a/sdk/src/headers_provider.rs +++ b/sdk/src/headers_provider.rs @@ -84,7 +84,7 @@ impl HeadersProvider for OAuthHeadersProvider { ) .await?; let mut headers = HashMap::new(); - headers.insert("authorization", format!("Bearer {}", token)); + headers.insert("authorization", format!("Bearer {token}")); headers.insert("x-databricks-zerobus-table-name", self.table_name.clone()); Ok(headers) } diff --git a/sdk/src/landing_zone.rs b/sdk/src/landing_zone.rs index b4f54a3..a2e5205 100644 --- a/sdk/src/landing_zone.rs +++ b/sdk/src/landing_zone.rs @@ -255,7 +255,7 @@ mod tests { assert!(all_items.contains(&"item1".to_string())); assert!(all_items.contains(&"item2".to_string())); - assert!(lz.len() == 0); + assert_eq!(lz.len(), 0); } #[tokio::test] @@ -280,7 +280,7 @@ mod tests { _ = tokio::time::sleep(Duration::from_millis(50)) => { // This is expected, the task is still blocked. } - }; + } // Remove one item to free up space. let _observed = lz.observe().await; diff --git a/sdk/src/stream_configuration.rs b/sdk/src/stream_configuration.rs index b87474d..f7ce623 100644 --- a/sdk/src/stream_configuration.rs +++ b/sdk/src/stream_configuration.rs @@ -23,7 +23,7 @@ pub struct StreamConfigurationOptions { /// Maximum number of records that can be sending or pending acknowledgement at any given time. /// /// This limit controls memory usage and backpressure. When this limit is reached, - /// `ingest_record()` calls will block until acknowledgments free up space. + /// `ingest_record()` calls will block until acknowledgements free up space. /// /// Default: 1,000,000 pub max_inflight_records: usize, diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 83ece33..f90e61f 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -10,16 +10,16 @@ path = "src/rust_tests.rs" [dependencies] async-trait = "0.1" -prost = "0.13.3" -prost-reflect = "0.14" -prost-types = "0.13.3" -tokio = { version = "1.42.0", features = ["macros", "rt-multi-thread", "sync", "time"] } +prost = "0.14.1" +prost-reflect = "0.16.2" +prost-types = "0.14.1" +tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time"] } tokio-stream = { version = "0.1.16", features = ["net"] } -tonic = { version = "0.12.3", features = ["transport"] } +tonic = { version = "0.14.2", features = ["transport"] } +tonic-prost = "0.14.2" tracing = "0.1.41" tracing-subscriber = { version = "0.3", features = ["env-filter"] } databricks-zerobus-ingest-sdk = { path = "../sdk", features = ["testing"] } [build-dependencies] -tonic-build = "0.12.3" -protoc-bin-vendored = "3.0.0" \ No newline at end of file +tonic-prost-build = "0.14.2" \ No newline at end of file diff --git a/tests/build.rs b/tests/build.rs index 7f2aa60..7637efa 100644 --- a/tests/build.rs +++ b/tests/build.rs @@ -1,8 +1,6 @@ -fn main() -> Result<(), Box> { - tonic_build::configure() +fn main() -> Result<(), std::io::Error> { + tonic_prost_build::configure() .build_server(true) .build_client(false) - .compile_protos(&["../sdk/zerobus_service.proto"], &["../sdk"])?; - - Ok(()) + .compile_protos(&["../sdk/zerobus_service.proto"], &["../sdk"]) } diff --git a/tools/generate_files/Cargo.toml b/tools/generate_files/Cargo.toml index 89ac2b0..6896bc0 100644 --- a/tools/generate_files/Cargo.toml +++ b/tools/generate_files/Cargo.toml @@ -14,10 +14,5 @@ serde = { version = "1", features = ["derive"] } serde_json = "1.0" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } urlencoding = "2" -tonic-build = "0.12.3" -tempfile = "3.21.0" -prost = "0.12" -tonic = "0.12" - -[build-dependencies] -protoc-bin-vendored = "3" \ No newline at end of file +tonic-prost-build = "0.14.2" +tempfile = "3.21.0" \ No newline at end of file diff --git a/tools/generate_files/src/generate.rs b/tools/generate_files/src/generate.rs index 87c04cf..9f31a4f 100644 --- a/tools/generate_files/src/generate.rs +++ b/tools/generate_files/src/generate.rs @@ -434,7 +434,7 @@ pub fn generate_rust_and_descriptor( .context("bad filename")?; let desc_file = output_dir.join(format!("{}.descriptor", file_name)); - tonic_build::configure() + tonic_prost_build::configure() .out_dir(output_dir) .file_descriptor_set_path(&desc_file) .compile_protos( From d2b95ff38f5944b72d85cd321a3ea861dfe74032 Mon Sep 17 00:00:00 2001 From: Stephen Carman Date: Fri, 7 Nov 2025 11:59:18 -0500 Subject: [PATCH 2/3] Fix protoc not existing in lint --- .github/workflows/push.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 276a6f4..a8b3831 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -40,5 +40,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: components: clippy + - name: Install protoc + uses: arduino/setup-protoc@v3 - name: Lint code run: make lint \ No newline at end of file From 123f71a3e16246c20c2b3daf6236b4245a4fa1c1 Mon Sep 17 00:00:00 2001 From: Stephen Carman Date: Fri, 7 Nov 2025 12:07:47 -0500 Subject: [PATCH 3/3] Revert vendored protoc --- .github/workflows/push.yml | 2 -- sdk/Cargo.toml | 1 + sdk/build.rs | 1 + tests/Cargo.toml | 3 ++- tests/build.rs | 1 + tools/generate_files/Cargo.toml | 5 ++++- 6 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index a8b3831..276a6f4 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -40,7 +40,5 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: components: clippy - - name: Install protoc - uses: arduino/setup-protoc@v3 - name: Lint code run: make lint \ No newline at end of file diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml index ff7f6db..c96d903 100644 --- a/sdk/Cargo.toml +++ b/sdk/Cargo.toml @@ -28,6 +28,7 @@ tracing = "0.1.41" [build-dependencies] tonic-prost-build = "0.14.2" +protoc-bin-vendored = "3" [features] testing = [] diff --git a/sdk/build.rs b/sdk/build.rs index 11c5407..9588468 100644 --- a/sdk/build.rs +++ b/sdk/build.rs @@ -1,4 +1,5 @@ fn main() { + std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path().unwrap()); tonic_prost_build::compile_protos("zerobus_service.proto") .unwrap_or_else(|e| panic!("Failed to compile protos {:?}", e)); } diff --git a/tests/Cargo.toml b/tests/Cargo.toml index f90e61f..42c9839 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -22,4 +22,5 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } databricks-zerobus-ingest-sdk = { path = "../sdk", features = ["testing"] } [build-dependencies] -tonic-prost-build = "0.14.2" \ No newline at end of file +tonic-prost-build = "0.14.2" +protoc-bin-vendored = "3" \ No newline at end of file diff --git a/tests/build.rs b/tests/build.rs index 7637efa..04d2713 100644 --- a/tests/build.rs +++ b/tests/build.rs @@ -1,4 +1,5 @@ fn main() -> Result<(), std::io::Error> { + std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path().unwrap()); tonic_prost_build::configure() .build_server(true) .build_client(false) diff --git a/tools/generate_files/Cargo.toml b/tools/generate_files/Cargo.toml index 6896bc0..78c0391 100644 --- a/tools/generate_files/Cargo.toml +++ b/tools/generate_files/Cargo.toml @@ -15,4 +15,7 @@ serde_json = "1.0" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } urlencoding = "2" tonic-prost-build = "0.14.2" -tempfile = "3.21.0" \ No newline at end of file +tempfile = "3.21.0" + +[build-dependencies] +protoc-bin-vendored = "3" \ No newline at end of file