diff --git a/Cargo.lock b/Cargo.lock index 1b6281bae44b8..00646c46b391c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5517,9 +5517,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.48" +version = "0.10.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518915b97df115dd36109bfa429a48b8f737bd05508cf9588977b599648926d2" +checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" dependencies = [ "bitflags", "cfg-if", @@ -5558,11 +5558,10 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.83" +version = "0.9.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "666416d899cf077260dac8698d60a60b435a46d57e82acb1be3d0dad87284e5b" +checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" dependencies = [ - "autocfg", "cc", "libc", "openssl-src", diff --git a/src/adapter/src/catalog.rs b/src/adapter/src/catalog.rs index 5de5c3c83f2c1..eaa48f3af6d4e 100644 --- a/src/adapter/src/catalog.rs +++ b/src/adapter/src/catalog.rs @@ -7343,6 +7343,7 @@ impl Catalog { } } }, + finalize_shards: self.system_config().storage_client_finalize_shards(), } } diff --git a/src/ccsr/Cargo.toml b/src/ccsr/Cargo.toml index 7e0e5cca24623..4a017ea5f8b61 100644 --- a/src/ccsr/Cargo.toml +++ b/src/ccsr/Cargo.toml @@ -9,7 +9,7 @@ publish = false [dependencies] anyhow = "1.0.66" native-tls = "0.2.11" -openssl = { version = "0.10.48", features = ["vendored"] } +openssl = { version = "0.10.55", features = ["vendored"] } reqwest = { version = "0.11.13", features = ["blocking", "json", "native-tls-vendored"] } serde = { version = "1.0.152", features = ["derive"] } serde_json = "1.0.89" diff --git a/src/environmentd/Cargo.toml b/src/environmentd/Cargo.toml index 6062f70a633ee..13f8710be5f01 100644 --- a/src/environmentd/Cargo.toml +++ b/src/environmentd/Cargo.toml @@ -58,7 +58,7 @@ mz-stash = { path = "../stash" } mz-storage-client = { path = "../storage-client" } nix = "0.26.1" num_cpus = "1.14.0" -openssl = { version = "0.10.48", features = ["vendored"] } +openssl = { version = "0.10.55", features = ["vendored"] } openssl-sys = { version = "0.9.80", features = ["vendored"] } opentelemetry = { version = "0.19.0", features = ["rt-tokio", "trace"] } prometheus = { version = "0.13.3", default-features = false } diff --git a/src/ore/Cargo.toml b/src/ore/Cargo.toml index bbe2f6baf8800..b59129fc88ecc 100644 --- a/src/ore/Cargo.toml +++ b/src/ore/Cargo.toml @@ -25,7 +25,7 @@ futures = { version = "0.3.25", optional = true } mz-test-macro = { path = "../test-macro", default-features = false } once_cell = "1.16.0" # The vendored feature is transitively depended upon by tokio-openssl. -openssl = { version = "0.10.48", features = ["vendored"], optional = true } +openssl = { version = "0.10.55", features = ["vendored"], optional = true } paste = "1.0.11" pin-project = "1.0.12" prometheus = { version = "0.13.3", default-features = false, optional = true } diff --git a/src/persist/Cargo.toml b/src/persist/Cargo.toml index bea76f947e9a5..83bfd4b34c9c8 100644 --- a/src/persist/Cargo.toml +++ b/src/persist/Cargo.toml @@ -38,7 +38,7 @@ mz-aws-s3-util = { path = "../aws-s3-util" } mz-ore = { path = "../ore", default-features = false, features = ["metrics", "async", "bytes_"] } mz-persist-types = { path = "../persist-types" } mz-proto = { path = "../proto" } -openssl = { version = "0.10.48", features = ["vendored"] } +openssl = { version = "0.10.55", features = ["vendored"] } openssl-sys = { version = "0.9.80", features = ["vendored"] } postgres-openssl = { version = "0.5.0" } prometheus = { version = "0.13.3", default-features = false } diff --git a/src/pgwire/Cargo.toml b/src/pgwire/Cargo.toml index 64ed2bdc4ad1c..24bb9a9bd6949 100644 --- a/src/pgwire/Cargo.toml +++ b/src/pgwire/Cargo.toml @@ -22,7 +22,7 @@ mz-pgcopy = { path = "../pgcopy" } mz-pgrepr = { path = "../pgrepr" } mz-repr = { path = "../repr" } mz-sql = { path = "../sql" } -openssl = { version = "0.10.48", features = ["vendored"] } +openssl = { version = "0.10.55", features = ["vendored"] } postgres = { version = "0.19.5" } tokio = "1.24.2" tokio-openssl = "0.6.3" diff --git a/src/postgres-util/Cargo.toml b/src/postgres-util/Cargo.toml index 258fdc5fda68c..72eaa389fccaf 100644 --- a/src/postgres-util/Cargo.toml +++ b/src/postgres-util/Cargo.toml @@ -13,7 +13,7 @@ mz-ore = { path = "../ore", features = ["async"], optional = true } mz-proto = { path = "../proto", optional = true } mz-repr = { path = "../repr", optional = true } mz-ssh-util = { path = "../ssh-util", optional = true } -openssl = { version = "0.10.48", features = ["vendored"] } +openssl = { version = "0.10.55", features = ["vendored"] } openssh = { version = "0.9.8", default-features = false, features = ["native-mux"], optional = true } postgres_array = { version = "0.11.0", optional = true } postgres-openssl = { version = "0.5.0" } diff --git a/src/sql/src/session/vars.rs b/src/sql/src/session/vars.rs index da662d7c8c210..0090d744c5c4f 100644 --- a/src/sql/src/session/vars.rs +++ b/src/sql/src/session/vars.rs @@ -959,6 +959,15 @@ const KEEP_N_SOURCE_STATUS_HISTORY_ENTRIES: ServerVar = ServerVar { internal: true }; +/// Whether compute rendering should use Materialize's custom linear join implementation rather +/// than the one from Differential Dataflow. +const STORAGE_CLIENT_FINALIZE_SHARDS: ServerVar = ServerVar { + name: UncasedStr::new("storage_client_finalize_shards"), + value: &true, + description: "Whether to all the storage client to finalize shards (Materialize).", + internal: true, +}; + // Macro to simplify creating feature flags, i.e. boolean flags that we use to toggle the // availability of features. // @@ -2203,6 +2212,11 @@ impl SystemVars { pub fn enable_mz_join_core(&self) -> bool { *self.expect_value(&ENABLE_MZ_JOIN_CORE) } + + /// Returns the `pg_replication_tcp_user_timeout` configuration parameter. + pub fn storage_client_finalize_shards(&self) -> bool { + *self.expect_value(&STORAGE_CLIENT_FINALIZE_SHARDS) + } } /// A `Var` represents a configuration parameter of an arbitrary type. diff --git a/src/ssh-util/Cargo.toml b/src/ssh-util/Cargo.toml index 20f526e5f054c..9def706d82ed5 100644 --- a/src/ssh-util/Cargo.toml +++ b/src/ssh-util/Cargo.toml @@ -11,7 +11,7 @@ anyhow = { version = "1.0.66" } mz-ore = { path = "../ore" } openssh = { version = "0.9.8", default-features = false, features = ["native-mux"] } openssh-mux-client = "0.15.5" -openssl = { version = "0.10.48", features = ["vendored"] } +openssl = { version = "0.10.55", features = ["vendored"] } rand = "0.8.5" scopeguard = "1.1.0" serde = { version = "1.0.152", features = ["derive"] } diff --git a/src/storage-client/src/controller.rs b/src/storage-client/src/controller.rs index 5e541410e4de5..8473fe323c7bd 100644 --- a/src/storage-client/src/controller.rs +++ b/src/storage-client/src/controller.rs @@ -2295,7 +2295,9 @@ where .await .expect("stash operation must succeed"); - self.finalize_shards().await; + if self.state.config.finalize_shards { + self.finalize_shards().await; + } } Some(StorageResponse::StatisticsUpdates(source_stats, sink_stats)) => { // Note we only hold the locks while moving some plain-old-data around here. diff --git a/src/storage-client/src/types/parameters.proto b/src/storage-client/src/types/parameters.proto index 1aa52c9202213..72fa4fcb3d47d 100644 --- a/src/storage-client/src/types/parameters.proto +++ b/src/storage-client/src/types/parameters.proto @@ -21,6 +21,7 @@ message ProtoStorageParameters { ProtoPgReplicationTimeouts pg_replication_timeouts = 3; uint64 keep_n_source_status_history_entries = 4; mz_rocksdb.config.ProtoRocksDbTuningParameters upsert_rocksdb_tuning_config = 5; + bool finalize_shards = 6; } diff --git a/src/storage-client/src/types/parameters.rs b/src/storage-client/src/types/parameters.rs index 52db6ef50b72f..c2d5244994f52 100644 --- a/src/storage-client/src/types/parameters.rs +++ b/src/storage-client/src/types/parameters.rs @@ -31,6 +31,10 @@ pub struct StorageParameters { pub keep_n_source_status_history_entries: usize, /// A set of parameters used to tune RocksDB when used with `UPSERT` sources. pub upsert_rocksdb_tuning_config: mz_rocksdb::RocksDBTuningParameters, + /// Whether or not to allow shard finalization to occur. Note that this will + /// only disable the actual finalization of shards, not registering them for + /// finalization. + pub finalize_shards: bool, } impl StorageParameters { @@ -42,12 +46,14 @@ impl StorageParameters { pg_replication_timeouts, keep_n_source_status_history_entries, upsert_rocksdb_tuning_config, + finalize_shards, }: StorageParameters, ) { self.persist.update(persist); self.pg_replication_timeouts = pg_replication_timeouts; self.keep_n_source_status_history_entries = keep_n_source_status_history_entries; self.upsert_rocksdb_tuning_config = upsert_rocksdb_tuning_config; + self.finalize_shards = finalize_shards } } @@ -60,6 +66,7 @@ impl RustType for StorageParameters { self.keep_n_source_status_history_entries, ), upsert_rocksdb_tuning_config: Some(self.upsert_rocksdb_tuning_config.into_proto()), + finalize_shards: self.finalize_shards, } } @@ -77,6 +84,7 @@ impl RustType for StorageParameters { upsert_rocksdb_tuning_config: proto .upsert_rocksdb_tuning_config .into_rust_if_some("ProtoStorageParameters::upsert_rocksdb_tuning_config")?, + finalize_shards: proto.finalize_shards, }) } } diff --git a/src/workspace-hack/Cargo.toml b/src/workspace-hack/Cargo.toml index 01dfbe610ebc4..30bd0eca7b072 100644 --- a/src/workspace-hack/Cargo.toml +++ b/src/workspace-hack/Cargo.toml @@ -66,7 +66,7 @@ nom = { version = "7.1.2" } num-bigint = { version = "0.4.3" } num-integer = { version = "0.1.44", features = ["i128"] } num-traits = { version = "0.2.15", features = ["i128"] } -openssl = { version = "0.10.48", features = ["vendored"] } +openssl = { version = "0.10.55", features = ["vendored"] } openssl-sys = { version = "0.9.83", default-features = false, features = ["vendored"] } ordered-float = { version = "3.4.0", features = ["serde"] } parking_lot = { version = "0.12.1", features = ["send_guard"] } @@ -166,7 +166,7 @@ nom = { version = "7.1.2" } num-bigint = { version = "0.4.3" } num-integer = { version = "0.1.44", features = ["i128"] } num-traits = { version = "0.2.15", features = ["i128"] } -openssl = { version = "0.10.48", features = ["vendored"] } +openssl = { version = "0.10.55", features = ["vendored"] } openssl-sys = { version = "0.9.83", default-features = false, features = ["vendored"] } ordered-float = { version = "3.4.0", features = ["serde"] } parking_lot = { version = "0.12.1", features = ["send_guard"] }