From bb07e8bff3cee678080c5ebc41aa3b5105aa6b11 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Mon, 13 Oct 2025 17:24:15 +0800 Subject: [PATCH 1/6] Add s3_url --- src/redis_service.cpp | 64 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/src/redis_service.cpp b/src/redis_service.cpp index 435b1c9b..3fd780e6 100644 --- a/src/redis_service.cpp +++ b/src/redis_service.cpp @@ -297,6 +297,14 @@ DEFINE_string(txlog_rocksdb_cloud_endpoint_url, DEFINE_string(txlog_rocksdb_cloud_sst_file_cache_size, "1GB", "Local sst cache size for txlog"); +DEFINE_string(txlog_rocksdb_cloud_s3_url, + "", + "TxLog RocksDB cloud S3 URL. Format: s3://{bucket}/{path} or " + "http(s)://{host}:{port}/{bucket}/{path}. " + "Examples: s3://my-bucket/my-path, " + "http://localhost:9000/my-bucket/my-path. " + "This option takes precedence over legacy configuration options " + "if both are provided"); #endif #ifdef WITH_CLOUD_AZ_INFO DEFINE_string(txlog_rocksdb_cloud_prefer_zone, @@ -1697,6 +1705,16 @@ bool RedisServiceImpl::InitTxLogService( ? FLAGS_aws_secret_key : config_reader.GetString("store", "aws_secret_key", ""); #endif /* LOG_STATE_TYPE_RKDB_S3 */ + + // Get the S3 URL configuration (new style) + txlog_rocksdb_cloud_config.s3_url_ = + !CheckCommandLineFlagIsDefault("txlog_rocksdb_cloud_s3_url") + ? FLAGS_txlog_rocksdb_cloud_s3_url + : config_reader.GetString("local", + "txlog_rocksdb_cloud_s3_url", + FLAGS_txlog_rocksdb_cloud_s3_url); + + // Get legacy configuration txlog_rocksdb_cloud_config.endpoint_url_ = !CheckCommandLineFlagIsDefault("txlog_rocksdb_cloud_endpoint_url") ? FLAGS_txlog_rocksdb_cloud_endpoint_url @@ -1783,6 +1801,50 @@ bool RedisServiceImpl::InitTxLogService( txlog_rocksdb_cloud_config.log_purger_starting_second_ = log_purger_tm.tm_sec; } + + // Check if s3_url was explicitly set (prefer URL-based config) + // URL-based config takes precedence over legacy config if both are present + bool has_s3_url_explicit = + !CheckCommandLineFlagIsDefault("txlog_rocksdb_cloud_s3_url") || + !config_reader.GetString("local", "txlog_rocksdb_cloud_s3_url", "") + .empty(); + + // If using S3 URL configuration, parse it and populate the fields + // This overrides any legacy configuration settings + if (has_s3_url_explicit) + { + txlog::S3UrlComponents url_components = + txlog::ParseS3Url(txlog_rocksdb_cloud_config.s3_url_); + if (!url_components.is_valid) + { + LOG(ERROR) << "Invalid txlog_rocksdb_cloud_s3_url: " + << url_components.error_message + << ". URL format: s3://{bucket}/{path} or " + "http(s)://{host}:{port}/{bucket}/{path}. " + << "Examples: s3://my-bucket/my-path, " + << "http://localhost:9000/my-bucket/my-path"; + return false; + } + + // Populate config fields from parsed URL (overriding legacy configs) + txlog_rocksdb_cloud_config.bucket_name_ = url_components.bucket_name; + txlog_rocksdb_cloud_config.bucket_prefix_ = + ""; // No prefix in URL-based config + txlog_rocksdb_cloud_config.object_path_ = url_components.object_path; + txlog_rocksdb_cloud_config.endpoint_url_ = url_components.endpoint_url; + + LOG(INFO) << "Using TxLog S3 URL configuration (overrides legacy " + "config if present): " + << txlog_rocksdb_cloud_config.s3_url_ + << " (bucket: " << txlog_rocksdb_cloud_config.bucket_name_ + << ", object_path: " + << txlog_rocksdb_cloud_config.object_path_ << ", endpoint: " + << (txlog_rocksdb_cloud_config.endpoint_url_.empty() + ? "default" + : txlog_rocksdb_cloud_config.endpoint_url_) + << ")"; + } + if (FLAGS_bootstrap) { log_server_ = std::make_unique<::txlog::LogServer>( @@ -2341,7 +2403,7 @@ void RedisServiceImpl::RedisClusterSlots(std::vector &info) } } } // end-if - } // end-for + } // end-for if (info.size() > 1) { From 9296c346e31a1b7d854b382c99a573ce76def82a Mon Sep 17 00:00:00 2001 From: githubzilla Date: Mon, 13 Oct 2025 17:24:25 +0800 Subject: [PATCH 2/6] Update store_handler --- store_handler | 2 +- tx_service | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/store_handler b/store_handler index a3fdbbc8..b4b2f8bd 160000 --- a/store_handler +++ b/store_handler @@ -1 +1 @@ -Subproject commit a3fdbbc866c98b4e70a02aac31a17f408d45d15a +Subproject commit b4b2f8bd8ed460673322ad209f51791563f41dc3 diff --git a/tx_service b/tx_service index b823488d..a3cd3a01 160000 --- a/tx_service +++ b/tx_service @@ -1 +1 @@ -Subproject commit b823488d3c6e01e1bb2e11a4469e2823ba0be9a6 +Subproject commit a3cd3a013b07a9d9be2944aac5b549e7dd334c4e From 844c70676d2995589c70d96a5df5e678df4f1bca Mon Sep 17 00:00:00 2001 From: githubzilla Date: Mon, 13 Oct 2025 18:32:27 +0800 Subject: [PATCH 3/6] Remove parse s3_url logic --- src/redis_service.cpp | 43 ------------------------------------------- 1 file changed, 43 deletions(-) diff --git a/src/redis_service.cpp b/src/redis_service.cpp index 3fd780e6..248b4862 100644 --- a/src/redis_service.cpp +++ b/src/redis_service.cpp @@ -1802,49 +1802,6 @@ bool RedisServiceImpl::InitTxLogService( log_purger_tm.tm_sec; } - // Check if s3_url was explicitly set (prefer URL-based config) - // URL-based config takes precedence over legacy config if both are present - bool has_s3_url_explicit = - !CheckCommandLineFlagIsDefault("txlog_rocksdb_cloud_s3_url") || - !config_reader.GetString("local", "txlog_rocksdb_cloud_s3_url", "") - .empty(); - - // If using S3 URL configuration, parse it and populate the fields - // This overrides any legacy configuration settings - if (has_s3_url_explicit) - { - txlog::S3UrlComponents url_components = - txlog::ParseS3Url(txlog_rocksdb_cloud_config.s3_url_); - if (!url_components.is_valid) - { - LOG(ERROR) << "Invalid txlog_rocksdb_cloud_s3_url: " - << url_components.error_message - << ". URL format: s3://{bucket}/{path} or " - "http(s)://{host}:{port}/{bucket}/{path}. " - << "Examples: s3://my-bucket/my-path, " - << "http://localhost:9000/my-bucket/my-path"; - return false; - } - - // Populate config fields from parsed URL (overriding legacy configs) - txlog_rocksdb_cloud_config.bucket_name_ = url_components.bucket_name; - txlog_rocksdb_cloud_config.bucket_prefix_ = - ""; // No prefix in URL-based config - txlog_rocksdb_cloud_config.object_path_ = url_components.object_path; - txlog_rocksdb_cloud_config.endpoint_url_ = url_components.endpoint_url; - - LOG(INFO) << "Using TxLog S3 URL configuration (overrides legacy " - "config if present): " - << txlog_rocksdb_cloud_config.s3_url_ - << " (bucket: " << txlog_rocksdb_cloud_config.bucket_name_ - << ", object_path: " - << txlog_rocksdb_cloud_config.object_path_ << ", endpoint: " - << (txlog_rocksdb_cloud_config.endpoint_url_.empty() - ? "default" - : txlog_rocksdb_cloud_config.endpoint_url_) - << ")"; - } - if (FLAGS_bootstrap) { log_server_ = std::make_unique<::txlog::LogServer>( From b57d30f3bf538802e9f86cf4fe629e2b16d76f11 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Thu, 23 Oct 2025 14:51:48 +0800 Subject: [PATCH 4/6] Fix clang-format --- src/redis_service.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis_service.cpp b/src/redis_service.cpp index 248b4862..e4bcad36 100644 --- a/src/redis_service.cpp +++ b/src/redis_service.cpp @@ -2360,7 +2360,7 @@ void RedisServiceImpl::RedisClusterSlots(std::vector &info) } } } // end-if - } // end-for + } // end-for if (info.size() > 1) { From d214eecd38c3e9cf12de1b15cde908a21eacaf89 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Thu, 23 Oct 2025 15:01:00 +0800 Subject: [PATCH 5/6] Update store_handler --- store_handler | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store_handler b/store_handler index b4b2f8bd..cefcd85a 160000 --- a/store_handler +++ b/store_handler @@ -1 +1 @@ -Subproject commit b4b2f8bd8ed460673322ad209f51791563f41dc3 +Subproject commit cefcd85a4ba9cf80eb8f6ca0805e72770ecb9083 From b612408b2d6e632f3d7ac38bdf7815144b5eeac8 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Thu, 23 Oct 2025 15:25:16 +0800 Subject: [PATCH 6/6] Update store_handler --- store_handler | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store_handler b/store_handler index cefcd85a..70ef6795 160000 --- a/store_handler +++ b/store_handler @@ -1 +1 @@ -Subproject commit cefcd85a4ba9cf80eb8f6ca0805e72770ecb9083 +Subproject commit 70ef6795c0fa7b73b0d83afd949765cbd4f734f1