From e51bee58ecf8f684abfb0f1f9ac1806fb47b5efd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 Apr 2022 22:32:45 +0200 Subject: [PATCH 001/274] Remove useless "install" from CMake (step 1) --- CMakeLists.txt | 1 - base/glibc-compatibility/CMakeLists.txt | 6 ---- base/harmful/CMakeLists.txt | 1 - tests/CMakeLists.txt | 26 --------------- tests/integration/CMakeLists.txt | 24 -------------- utils/CMakeLists.txt | 5 --- utils/config-processor/CMakeLists.txt | 2 -- utils/config-processor/config-processor.cpp | 35 --------------------- utils/report/CMakeLists.txt | 1 - 9 files changed, 101 deletions(-) delete mode 100644 tests/CMakeLists.txt delete mode 100644 tests/integration/CMakeLists.txt delete mode 100644 utils/config-processor/CMakeLists.txt delete mode 100644 utils/config-processor/config-processor.cpp delete mode 100644 utils/report/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index e8b6e9217d27..bffdd8106864 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -577,7 +577,6 @@ include (cmake/print_flags.cmake) add_subdirectory (base) add_subdirectory (src) add_subdirectory (programs) -add_subdirectory (tests) add_subdirectory (utils) include (cmake/sanitize_target_link_libraries.cmake) diff --git a/base/glibc-compatibility/CMakeLists.txt b/base/glibc-compatibility/CMakeLists.txt index ef7ec6d7fc03..37423bb68a60 100644 --- a/base/glibc-compatibility/CMakeLists.txt +++ b/base/glibc-compatibility/CMakeLists.txt @@ -43,12 +43,6 @@ if (GLIBC_COMPATIBILITY) target_link_libraries(global-libs INTERFACE glibc-compatibility ${MEMCPY_LIBRARY}) - install( - TARGETS glibc-compatibility ${MEMCPY_LIBRARY} - EXPORT global - ARCHIVE DESTINATION lib - ) - message (STATUS "Some symbols from glibc will be replaced for compatibility") elseif (CLICKHOUSE_OFFICIAL_BUILD) diff --git a/base/harmful/CMakeLists.txt b/base/harmful/CMakeLists.txt index 399f6ecc6253..c19661875be4 100644 --- a/base/harmful/CMakeLists.txt +++ b/base/harmful/CMakeLists.txt @@ -1,2 +1 @@ add_library(harmful harmful.c) -install(TARGETS harmful EXPORT global ARCHIVE DESTINATION lib) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt deleted file mode 100644 index 22c89aaafa7d..000000000000 --- a/tests/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -enable_testing() - -# Run tests with "ninja check" or "make check" -if (TARGET check) - message (STATUS "Target check already exists") -else () - include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake) -endif () - -option (ENABLE_CLICKHOUSE_TEST "Install clickhouse-test script and relevant tests scenarios" OFF) - -if (ENABLE_CLICKHOUSE_TEST) - install (PROGRAMS clickhouse-test DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - install ( - DIRECTORY queries performance config - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse-test - USE_SOURCE_PERMISSIONS - COMPONENT clickhouse - PATTERN "CMakeLists.txt" EXCLUDE - PATTERN ".gitignore" EXCLUDE - ) -endif () - -if (ENABLE_TEST_INTEGRATION) - add_subdirectory (integration) -endif () diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt deleted file mode 100644 index 68c695f57a0d..000000000000 --- a/tests/integration/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -if(CLICKHOUSE_SPLIT_BINARY) - set (TEST_USE_BINARIES CLICKHOUSE_TESTS_SERVER_BIN_PATH=${ClickHouse_BINARY_DIR}/programs/clickhouse-server CLICKHOUSE_TESTS_CLIENT_BIN_PATH=${ClickHouse_BINARY_DIR}/programs/clickhouse-client) -else() - set (TEST_USE_BINARIES CLICKHOUSE_TESTS_SERVER_BIN_PATH=${ClickHouse_BINARY_DIR}/programs/clickhouse CLICKHOUSE_TESTS_CLIENT_BIN_PATH=${ClickHouse_BINARY_DIR}/programs/clickhouse) -endif() - -find_program(DOCKER_CMD docker) -find_program(DOCKER_COMPOSE_CMD docker-compose) -find_program(PYTEST_CMD pytest) -find_program(SUDO_CMD sudo) - -# will mount only one binary to docker container - build with .so cant work -if(USE_STATIC_LIBRARIES AND DOCKER_CMD) - if(INTEGRATION_USE_RUNNER AND SUDO_CMD) - add_test(NAME integration-runner WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMAND ${SUDO_CMD} ${CMAKE_CURRENT_SOURCE_DIR}/runner --binary ${ClickHouse_BINARY_DIR}/programs/clickhouse --configs-dir ${ClickHouse_SOURCE_DIR}/programs/server/) - message(STATUS "Using tests in docker with runner SUDO=${SUDO_CMD}; DOCKER=${DOCKER_CMD};") - endif() - if(NOT INTEGRATION_USE_RUNNER AND DOCKER_COMPOSE_CMD AND PYTEST_CMD) - # To run one test with debug: - # cmake . -DPYTEST_OPT="-ss;test_cluster_copier" - add_test(NAME integration-pytest WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMAND env ${TEST_USE_BINARIES} "CLICKHOUSE_TESTS_BASE_CONFIG_DIR=${ClickHouse_SOURCE_DIR}/programs/server/" "CLICKHOUSE_TESTS_CONFIG_DIR=${ClickHouse_SOURCE_DIR}/tests/config/" ${PYTEST_STARTER} ${PYTEST_CMD} ${PYTEST_OPT}) - message(STATUS "Using tests in docker DOCKER=${DOCKER_CMD}; DOCKER_COMPOSE=${DOCKER_COMPOSE_CMD}; PYTEST=${PYTEST_STARTER} ${PYTEST_CMD} ${PYTEST_OPT}") - endif() -endif() diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 51300472ed11..d4f22d8065d6 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -9,11 +9,6 @@ else() endif() include(../cmake/limit_jobs.cmake) -# Utils used in package -add_subdirectory (config-processor) -add_subdirectory (report) - -# Not used in package if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS) add_subdirectory (compressor) add_subdirectory (iotest) diff --git a/utils/config-processor/CMakeLists.txt b/utils/config-processor/CMakeLists.txt deleted file mode 100644 index 76c10b5f2fd1..000000000000 --- a/utils/config-processor/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_executable (config-processor config-processor.cpp) -target_link_libraries(config-processor PRIVATE clickhouse_common_config_no_zookeeper_log) diff --git a/utils/config-processor/config-processor.cpp b/utils/config-processor/config-processor.cpp deleted file mode 100644 index 242a6782b3b1..000000000000 --- a/utils/config-processor/config-processor.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include -#include - -int main(int argc, char ** argv) -{ - try - { - if (argc != 2) - { - std::cerr << "usage: " << argv[0] << " path" << std::endl; - return 3; - } - - DB::ConfigProcessor processor(argv[1], false, true); - DB::XMLDocumentPtr document = processor.processConfig(); - Poco::XML::DOMWriter().writeNode(std::cout, document); - } - catch (Poco::Exception & e) - { - std::cerr << "Exception: " << e.displayText() << std::endl; - return 1; - } - catch (std::exception & e) - { - std::cerr << "std::exception: " << e.what() << std::endl; - return 3; - } - catch (...) - { - std::cerr << "Some exception" << std::endl; - return 2; - } - - return 0; -} diff --git a/utils/report/CMakeLists.txt b/utils/report/CMakeLists.txt deleted file mode 100644 index e39dd155b157..000000000000 --- a/utils/report/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -install (PROGRAMS clickhouse-report DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) From a72541d93f1cf46aa0fb577bcb59e5f2845c7a81 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 29 Sep 2023 17:39:38 +0200 Subject: [PATCH 002/274] Improvement for big reads --- src/Common/ProfileEvents.cpp | 1 + src/Core/Settings.h | 1 + .../IO/CachedOnDiskReadBufferFromFile.cpp | 34 ++++++++++++------- src/Disks/IO/CachedOnDiskReadBufferFromFile.h | 6 ++-- src/IO/ReadSettings.h | 1 + src/Interpreters/Cache/FileCache.cpp | 14 +++++++- src/Interpreters/Cache/FileCache.h | 4 +-- src/Interpreters/Cache/FileCache_fwd.h | 2 +- src/Interpreters/Cache/FileSegment.cpp | 5 ++- src/Interpreters/Cache/FileSegment.h | 5 ++- src/Interpreters/Context.cpp | 1 + 11 files changed, 50 insertions(+), 24 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 0df0bc89b387..033335bd5ecf 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -415,6 +415,7 @@ The server successfully detected this situation and will download merged part fr M(FilesystemCacheEvictMicroseconds, "Filesystem cache eviction time") \ M(FilesystemCacheGetOrSetMicroseconds, "Filesystem cache getOrSet() time") \ M(FilesystemCacheGetMicroseconds, "Filesystem cache get() time") \ + M(FilesystemCacheUnusedHoldFileSegments, "Filesystem cache file segments count, which were hold, but not used (because of seek or LIMIT n, etc)") \ M(FileSegmentWaitMicroseconds, "Wait on DOWNLOADING state") \ M(FileSegmentCompleteMicroseconds, "Duration of FileSegment::complete() in filesystem cache") \ M(FileSegmentLockMicroseconds, "Lock file segment time") \ diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 1fada4ae569b..416d8f481b14 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -717,6 +717,7 @@ class IColumn; M(Bool, skip_download_if_exceeds_query_cache, true, "Skip download from remote filesystem if exceeds query cache size", 0) \ M(UInt64, filesystem_cache_max_download_size, (128UL * 1024 * 1024 * 1024), "Max remote filesystem cache size that can be downloaded by a single query", 0) \ M(Bool, throw_on_error_from_cache_on_write_operations, false, "Ignore error from cache when caching on write operations (INSERT, merges)", 0) \ + M(UInt64, filesystem_cache_getorset_batch_size, 100, "A batch size for holding file segments for a single read range", 0) \ \ M(Bool, load_marks_asynchronously, false, "Load MergeTree marks asynchronously", 0) \ M(Bool, enable_filesystem_read_prefetches_log, false, "Log to system.filesystem prefetch_log during query. Should be used only for testing or debugging, not recommended to be turned on by default", 0) \ diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 1cfdd96b271b..3c16d3d9ae2f 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -114,30 +114,40 @@ void CachedOnDiskReadBufferFromFile::appendFilesystemCacheLog( cache_log->add(std::move(elem)); } -void CachedOnDiskReadBufferFromFile::initialize(size_t offset, size_t size) +bool CachedOnDiskReadBufferFromFile::nextFileSegmentsBatch() { - if (initialized) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Caching buffer already initialized"); - - implementation_buffer.reset(); + size_t size = getRemainingSizeToRead(); + if (!size) + return false; if (settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache) { - file_segments = cache->get(cache_key, offset, size); + file_segments = cache->get(cache_key, file_offset_of_buffer_end, size, settings.filesystem_cache_getorset_batch_size); } else { CreateFileSegmentSettings create_settings(FileSegmentKind::Regular); - file_segments = cache->getOrSet(cache_key, offset, size, file_size.value(), create_settings); + file_segments = cache->getOrSet(cache_key, file_offset_of_buffer_end, size, file_size.value(), settings.filesystem_cache_getorset_batch_size, create_settings); } + return !file_segments->empty(); +} + +void CachedOnDiskReadBufferFromFile::initialize() +{ + if (initialized) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Caching buffer already initialized"); + + implementation_buffer.reset(); /** * Segments in returned list are ordered in ascending order and represent a full contiguous * interval (no holes). Each segment in returned list has state: DOWNLOADED, DOWNLOADING or EMPTY. */ - if (file_segments->empty()) + if (!nextFileSegmentsBatch()) throw Exception(ErrorCodes::LOGICAL_ERROR, "List of file segments cannot be empty"); + chassert(!file_segments->empty()); + LOG_TEST( log, "Having {} file segments to read: {}, current offset: {}", @@ -512,7 +522,7 @@ bool CachedOnDiskReadBufferFromFile::completeFileSegmentAndGetNext() cache_file_reader.reset(); file_segments->popFront(); - if (file_segments->empty()) + if (file_segments->empty() && !nextFileSegmentsBatch()) return false; current_file_segment = &file_segments->front(); @@ -788,9 +798,9 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep() return false; if (!initialized) - initialize(file_offset_of_buffer_end, getTotalSizeToRead()); + initialize(); - if (file_segments->empty()) + if (file_segments->empty() && !nextFileSegmentsBatch()) return false; const size_t original_buffer_size = internal_buffer.size(); @@ -1159,7 +1169,7 @@ off_t CachedOnDiskReadBufferFromFile::seek(off_t offset, int whence) return new_pos; } -size_t CachedOnDiskReadBufferFromFile::getTotalSizeToRead() +size_t CachedOnDiskReadBufferFromFile::getRemainingSizeToRead() { /// Last position should be guaranteed to be set, as at least we always know file size. if (!read_until_position) diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h index 0b9b01b8a942..f1eea66d41d5 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h @@ -63,7 +63,7 @@ class CachedOnDiskReadBufferFromFile : public ReadBufferFromFileBase private: using ImplementationBufferPtr = std::shared_ptr; - void initialize(size_t offset, size_t size); + void initialize(); /** * Return a list of file segments ordered in ascending order. This list represents @@ -85,7 +85,7 @@ class CachedOnDiskReadBufferFromFile : public ReadBufferFromFileBase bool nextImplStep(); - size_t getTotalSizeToRead(); + size_t getRemainingSizeToRead(); bool completeFileSegmentAndGetNext(); @@ -95,6 +95,8 @@ class CachedOnDiskReadBufferFromFile : public ReadBufferFromFileBase static bool canStartFromCache(size_t current_offset, const FileSegment & file_segment); + bool nextFileSegmentsBatch(); + Poco::Logger * log; FileCache::Key cache_key; String source_file_path; diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index 87f249823b2c..197ae563d258 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -100,6 +100,7 @@ struct ReadSettings bool enable_filesystem_cache_log = false; /// Don't populate cache when the read is not part of query execution (e.g. background thread). bool avoid_readthrough_cache_outside_query_context = true; + size_t filesystem_cache_getorset_batch_size = 100; size_t filesystem_cache_max_download_size = (128UL * 1024 * 1024 * 1024); bool skip_download_if_exceeds_query_cache = true; diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 3ed2c9c2dd60..bef1f3086df5 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -400,6 +400,7 @@ FileCache::getOrSet( size_t offset, size_t size, size_t file_size, + size_t file_segments_limit, const CreateFileSegmentSettings & settings) { ProfileEventTimeIncrement watch(ProfileEvents::FilesystemCacheGetOrSetMicroseconds); @@ -432,11 +433,17 @@ FileCache::getOrSet( while (!file_segments.empty() && file_segments.back()->range().left >= offset + size) file_segments.pop_back(); + if (file_segments_limit) + { + while (file_segments.size() > file_segments_limit) + file_segments.pop_back(); + } + chassert(!file_segments.empty()); return std::make_unique(std::move(file_segments)); } -FileSegmentsHolderPtr FileCache::get(const Key & key, size_t offset, size_t size) +FileSegmentsHolderPtr FileCache::get(const Key & key, size_t offset, size_t size, size_t file_segments_limit) { ProfileEventTimeIncrement watch(ProfileEvents::FilesystemCacheGetMicroseconds); @@ -454,6 +461,11 @@ FileSegmentsHolderPtr FileCache::get(const Key & key, size_t offset, size_t size fillHolesWithEmptyFileSegments( *locked_key, file_segments, range, /* fill_with_detached */true, CreateFileSegmentSettings{}); + if (file_segments_limit) + { + while (file_segments.size() > file_segments_limit) + file_segments.pop_back(); + } return std::make_unique(std::move(file_segments)); } } diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index b5c2fa28f4b1..1a1a25cd9c19 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -85,7 +85,7 @@ class FileCache : private boost::noncopyable * it is guaranteed that these file segments are not removed from cache. */ FileSegmentsHolderPtr - getOrSet(const Key & key, size_t offset, size_t size, size_t file_size, const CreateFileSegmentSettings & settings); + getOrSet(const Key & key, size_t offset, size_t size, size_t file_size, size_t file_segments_limit, const CreateFileSegmentSettings & settings); /** * Segments in returned list are ordered in ascending order and represent a full contiguous @@ -96,7 +96,7 @@ class FileCache : private boost::noncopyable * with the destruction of the holder, while in getOrSet() EMPTY file segments can eventually change * it's state (and become DOWNLOADED). */ - FileSegmentsHolderPtr get(const Key & key, size_t offset, size_t size); + FileSegmentsHolderPtr get(const Key & key, size_t offset, size_t size, size_t file_segments_limit); FileSegmentsHolderPtr set(const Key & key, size_t offset, size_t size, const CreateFileSegmentSettings & settings); diff --git a/src/Interpreters/Cache/FileCache_fwd.h b/src/Interpreters/Cache/FileCache_fwd.h index 3e7150ad253b..1f61617668e5 100644 --- a/src/Interpreters/Cache/FileCache_fwd.h +++ b/src/Interpreters/Cache/FileCache_fwd.h @@ -4,7 +4,7 @@ namespace DB { -static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE = 32 * 1024 * 1024; /// 32Mi +static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE = 16 * 1024 * 1024; /// 16Mi static constexpr int FILECACHE_DEFAULT_FILE_SEGMENT_ALIGNMENT = 4 * 1024 * 1024; /// 4Mi static constexpr int FILECACHE_DEFAULT_BACKGROUND_DOWNLOAD_THREADS = 2; static constexpr int FILECACHE_DEFAULT_LOAD_METADATA_THREADS = 1; diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index bb3216cb20e1..a351df33b4b2 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -23,6 +23,7 @@ namespace ProfileEvents extern const Event FileSegmentWriteMicroseconds; extern const Event FileSegmentUseMicroseconds; extern const Event FileSegmentHolderCompleteMicroseconds; + extern const Event FilesystemCacheUnusedHoldFileSegments; } namespace DB @@ -916,11 +917,9 @@ FileSegments::iterator FileSegmentsHolder::completeAndPopFrontImpl() FileSegmentsHolder::~FileSegmentsHolder() { + ProfileEvents::increment(ProfileEvents::FilesystemCacheUnusedHoldFileSegments, file_segments.size()); ProfileEventTimeIncrement watch(ProfileEvents::FileSegmentHolderCompleteMicroseconds); - if (!complete_on_dtor) - return; - for (auto file_segment_it = file_segments.begin(); file_segment_it != file_segments.end();) file_segment_it = completeAndPopFrontImpl(); } diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 8948b67fe2a0..7c145664fe03 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -320,8 +320,8 @@ struct FileSegmentsHolder : private boost::noncopyable { FileSegmentsHolder() = default; - explicit FileSegmentsHolder(FileSegments && file_segments_, bool complete_on_dtor_ = true) - : file_segments(std::move(file_segments_)), complete_on_dtor(complete_on_dtor_) {} + explicit FileSegmentsHolder(FileSegments && file_segments_) + : file_segments(std::move(file_segments_)) {} ~FileSegmentsHolder(); @@ -351,7 +351,6 @@ struct FileSegmentsHolder : private boost::noncopyable private: FileSegments file_segments{}; - const bool complete_on_dtor = true; FileSegments::iterator completeAndPopFrontImpl(); }; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 58d60c640e7d..807215412c2a 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -4591,6 +4591,7 @@ ReadSettings Context::getReadSettings() const res.enable_filesystem_cache = settings.enable_filesystem_cache; res.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache; res.enable_filesystem_cache_log = settings.enable_filesystem_cache_log; + res.filesystem_cache_getorset_batch_size = settings.filesystem_cache_getorset_batch_size; res.filesystem_cache_max_download_size = settings.filesystem_cache_max_download_size; res.skip_download_if_exceeds_query_cache = settings.skip_download_if_exceeds_query_cache; From c7a3c74cde8d6dd68b23fb318827d0eaadf8f292 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 7 Oct 2023 15:20:31 +0200 Subject: [PATCH 003/274] Better --- src/Core/Settings.h | 2 +- src/Interpreters/Cache/FileCache.cpp | 66 +++++++++++++++++++++----- src/Interpreters/Cache/FileCache.h | 3 +- src/Interpreters/Cache/FileCache_fwd.h | 2 +- 4 files changed, 57 insertions(+), 16 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b9aa678a3c5b..485b96b23415 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -719,7 +719,7 @@ class IColumn; M(Bool, skip_download_if_exceeds_query_cache, true, "Skip download from remote filesystem if exceeds query cache size", 0) \ M(UInt64, filesystem_cache_max_download_size, (128UL * 1024 * 1024 * 1024), "Max remote filesystem cache size that can be downloaded by a single query", 0) \ M(Bool, throw_on_error_from_cache_on_write_operations, false, "Ignore error from cache when caching on write operations (INSERT, merges)", 0) \ - M(UInt64, filesystem_cache_getorset_batch_size, 100, "A batch size for holding file segments for a single read range", 0) \ + M(UInt64, filesystem_cache_getorset_batch_size, 20, "A batch size for holding file segments for a single read range", 0) \ \ M(Bool, load_marks_asynchronously, false, "Load MergeTree marks asynchronously", 0) \ M(Bool, enable_filesystem_read_prefetches_log, false, "Log to system.filesystem prefetch_log during query. Should be used only for testing or debugging, not recommended to be turned on by default", 0) \ diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index bef1f3086df5..576aab31adc2 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -147,7 +147,7 @@ CacheGuard::Lock FileCache::lockCache() const return cache_guard.lock(); } -FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment::Range & range) const +FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment::Range & range, size_t file_segments_limit) const { /// Given range = [left, right] and non-overlapping ordered set of file segments, /// find list [segment1, ..., segmentN] of segments which intersect with given range. @@ -166,6 +166,9 @@ FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment: FileSegments result; auto add_to_result = [&](const FileSegmentMetadata & file_segment_metadata) { + if (file_segments_limit && result.size() == file_segments_limit) + return false; + FileSegmentPtr file_segment; if (!file_segment_metadata.evicting()) { @@ -181,6 +184,7 @@ FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment: } result.push_back(file_segment); + return true; }; auto segment_it = file_segments.lower_bound(range.left); @@ -197,7 +201,8 @@ FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment: if (file_segment_metadata.file_segment->range().right < range.left) return {}; - add_to_result(file_segment_metadata); + if (!add_to_result(file_segment_metadata)) + return result; } else /// segment_it <-- segmment{k} { @@ -213,7 +218,8 @@ FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment: /// [___________ /// ^ /// range.left - add_to_result(prev_file_segment_metadata); + if (!add_to_result(prev_file_segment_metadata)) + return result; } } @@ -229,7 +235,8 @@ FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment: if (range.right < file_segment_metadata.file_segment->range().left) break; - add_to_result(file_segment_metadata); + if (!add_to_result(file_segment_metadata)) + return result; ++segment_it; } } @@ -273,6 +280,7 @@ void FileCache::fillHolesWithEmptyFileSegments( LockedKey & locked_key, FileSegments & file_segments, const FileSegment::Range & range, + size_t file_segments_limit, bool fill_with_detached_file_segments, const CreateFileSegmentSettings & settings) { @@ -338,6 +346,9 @@ void FileCache::fillHolesWithEmptyFileSegments( ++it; } + if (file_segments.size() >= file_segments_limit) + return; + if (current_pos <= range.right) { /// ________] -- requested range @@ -374,7 +385,7 @@ FileSegmentsHolderPtr FileCache::set( auto locked_key = metadata.lockKeyMetadata(key, CacheMetadata::KeyNotFoundPolicy::CREATE_EMPTY); FileSegment::Range range(offset, offset + size - 1); - auto file_segments = getImpl(*locked_key, range); + auto file_segments = getImpl(*locked_key, range, /* file_segments_limit */0); if (!file_segments.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Having intersection with already existing cache"); @@ -416,20 +427,47 @@ FileCache::getOrSet( auto locked_key = metadata.lockKeyMetadata(key, CacheMetadata::KeyNotFoundPolicy::CREATE_EMPTY); /// Get all segments which intersect with the given range. - auto file_segments = getImpl(*locked_key, range); + auto file_segments = getImpl(*locked_key, range, file_segments_limit); + + bool limit_reached = false; if (file_segments.empty()) { file_segments = splitRangeIntoFileSegments(*locked_key, range.left, range.size(), FileSegment::State::EMPTY, settings); + + while (!file_segments.empty() && file_segments.front()->range().right < offset) + file_segments.pop_front(); } else { + limit_reached = file_segments_limit && file_segments.size() >= file_segments_limit; + + /// A while loop for the case if we set a limit to n, but all these n file segments are removed + /// as they turned out redundant because of the alignment of offset to aligned_offset. + while (true) + { + size_t last_offset = file_segments.back()->range().right; + + while (!file_segments.empty() && file_segments.front()->range().right < offset) + file_segments.pop_front(); + + if (!file_segments.empty()) + break; + + if (!limit_reached) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty list of file segments"); + + range.left = std::min(offset, last_offset + 1); + file_segments = getImpl(*locked_key, range, file_segments_limit); + } + + range.left = std::min(offset, file_segments.front()->range().left); + if (limit_reached) + range.right = file_segments.back()->range().right; + fillHolesWithEmptyFileSegments( - *locked_key, file_segments, range, /* fill_with_detached */false, settings); + *locked_key, file_segments, range, file_segments_limit, /* fill_with_detached */false, settings); } - while (!file_segments.empty() && file_segments.front()->range().right < offset) - file_segments.pop_front(); - while (!file_segments.empty() && file_segments.back()->range().left >= offset + size) file_segments.pop_back(); @@ -439,7 +477,9 @@ FileCache::getOrSet( file_segments.pop_back(); } - chassert(!file_segments.empty()); + if (file_segments.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty list of file segments for offset {}, size {} (file size: {})", offset, size, file_size); + return std::make_unique(std::move(file_segments)); } @@ -455,11 +495,11 @@ FileSegmentsHolderPtr FileCache::get(const Key & key, size_t offset, size_t size FileSegment::Range range(offset, offset + size - 1); /// Get all segments which intersect with the given range. - auto file_segments = getImpl(*locked_key, range); + auto file_segments = getImpl(*locked_key, range, file_segments_limit); if (!file_segments.empty()) { fillHolesWithEmptyFileSegments( - *locked_key, file_segments, range, /* fill_with_detached */true, CreateFileSegmentSettings{}); + *locked_key, file_segments, range, file_segments_limit, /* fill_with_detached */true, CreateFileSegmentSettings{}); if (file_segments_limit) { diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index 1a1a25cd9c19..14f27a69a681 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -215,7 +215,7 @@ class FileCache : private boost::noncopyable void loadMetadataImpl(); void loadMetadataForKeys(const std::filesystem::path & keys_dir); - FileSegments getImpl(const LockedKey & locked_key, const FileSegment::Range & range) const; + FileSegments getImpl(const LockedKey & locked_key, const FileSegment::Range & range, size_t file_segments_limit) const; FileSegments splitRangeIntoFileSegments( LockedKey & locked_key, @@ -228,6 +228,7 @@ class FileCache : private boost::noncopyable LockedKey & locked_key, FileSegments & file_segments, const FileSegment::Range & range, + size_t file_segments_limit, bool fill_with_detached_file_segments, const CreateFileSegmentSettings & settings); diff --git a/src/Interpreters/Cache/FileCache_fwd.h b/src/Interpreters/Cache/FileCache_fwd.h index 1f61617668e5..3e7150ad253b 100644 --- a/src/Interpreters/Cache/FileCache_fwd.h +++ b/src/Interpreters/Cache/FileCache_fwd.h @@ -4,7 +4,7 @@ namespace DB { -static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE = 16 * 1024 * 1024; /// 16Mi +static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE = 32 * 1024 * 1024; /// 32Mi static constexpr int FILECACHE_DEFAULT_FILE_SEGMENT_ALIGNMENT = 4 * 1024 * 1024; /// 4Mi static constexpr int FILECACHE_DEFAULT_BACKGROUND_DOWNLOAD_THREADS = 2; static constexpr int FILECACHE_DEFAULT_LOAD_METADATA_THREADS = 1; From d7c5caef927d4e0ca2091f483e824a1fbddc3909 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 7 Oct 2023 15:23:32 +0200 Subject: [PATCH 004/274] Better --- src/Interpreters/Cache/FileCache.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 576aab31adc2..39cef20829e3 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -456,7 +456,8 @@ FileCache::getOrSet( if (!limit_reached) throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty list of file segments"); - range.left = std::min(offset, last_offset + 1); + range.left = last_offset + 1; + chassert(offset >= range.left); file_segments = getImpl(*locked_key, range, file_segments_limit); } From 09072097ec33231cd0df95dc91cfd1317f662da2 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 16 Oct 2023 14:32:47 +0200 Subject: [PATCH 005/274] Better --- src/Interpreters/Cache/FileCache.cpp | 80 ++++++++++---------- src/Interpreters/Cache/FileCacheSettings.cpp | 3 + src/Interpreters/Cache/FileSegment.h | 2 + 3 files changed, 46 insertions(+), 39 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index bbb5fd71b8f5..6d507413bab8 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -237,6 +237,7 @@ FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment: if (!add_to_result(file_segment_metadata)) return result; + ++segment_it; } } @@ -418,66 +419,67 @@ FileCache::getOrSet( assertInitialized(); + const auto end_offset = offset + size - 1; const auto aligned_offset = roundDownToMultiple(offset, boundary_alignment); - const auto aligned_end = std::min(roundUpToMultiple(offset + size, boundary_alignment), file_size); - const auto aligned_size = aligned_end - aligned_offset; - - FileSegment::Range range(aligned_offset, aligned_offset + aligned_size - 1); + const auto aligned_end_offset = std::min(roundUpToMultiple(offset + size, boundary_alignment), file_size) - 1; + chassert(aligned_offset <= offset); auto locked_key = metadata.lockKeyMetadata(key, CacheMetadata::KeyNotFoundPolicy::CREATE_EMPTY); /// Get all segments which intersect with the given range. + FileSegment::Range range(offset, end_offset); auto file_segments = getImpl(*locked_key, range, file_segments_limit); - bool limit_reached = false; - if (file_segments.empty()) + if (aligned_offset < offset && (file_segments.empty() || offset < file_segments.front()->range().left)) { - file_segments = splitRangeIntoFileSegments(*locked_key, range.left, range.size(), FileSegment::State::EMPTY, settings); + auto prefix_range = FileSegment::Range(aligned_offset, file_segments.empty() ? offset - 1 : file_segments.front()->range().left - 1); + auto prefix_file_segments = getImpl(*locked_key, prefix_range, /* file_segments_limit */0); - while (!file_segments.empty() && file_segments.front()->range().right < offset) - file_segments.pop_front(); - } - else - { - limit_reached = file_segments_limit && file_segments.size() >= file_segments_limit; + while (!prefix_file_segments.empty() && prefix_file_segments.front()->range().right < offset) + prefix_file_segments.pop_front(); - /// A while loop for the case if we set a limit to n, but all these n file segments are removed - /// as they turned out redundant because of the alignment of offset to aligned_offset. - while (true) + if (!prefix_file_segments.empty()) { - size_t last_offset = file_segments.back()->range().right; - - while (!file_segments.empty() && file_segments.front()->range().right < offset) - file_segments.pop_front(); - - if (!file_segments.empty()) - break; + file_segments.splice(file_segments.begin(), prefix_file_segments); + range.left = file_segments.front()->range().left; + } + } - if (!limit_reached) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty list of file segments"); + if (end_offset < aligned_end_offset && (file_segments.empty() || file_segments.back()->range().right < end_offset)) + { + auto suffix_range = FileSegment::Range(end_offset, aligned_end_offset); + /// Get only 1 file segment. + auto suffix_file_segments = getImpl(*locked_key, suffix_range, /* file_segments_limit */1); - range.left = last_offset + 1; - chassert(offset >= range.left); - file_segments = getImpl(*locked_key, range, file_segments_limit); - } + if (!suffix_file_segments.empty()) + range.right = suffix_file_segments.front()->range().left - 1; + } - range.left = std::min(offset, file_segments.front()->range().left); - if (limit_reached) - range.right = file_segments.back()->range().right; + if (file_segments.empty()) + { + file_segments = splitRangeIntoFileSegments(*locked_key, range.left, range.size(), FileSegment::State::EMPTY, settings); + } + else + { + chassert(file_segments.front()->range().right >= offset); + chassert(file_segments.back()->range().left <= end_offset); fillHolesWithEmptyFileSegments( *locked_key, file_segments, range, file_segments_limit, /* fill_with_detached */false, settings); - } - while (!file_segments.empty() && file_segments.back()->range().left >= offset + size) - file_segments.pop_back(); + if (!file_segments.front()->range().contains(offset)) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected {} to include {} " + "(end offset: {}, aligned offset: {}, aligned end offset: {})", + file_segments.front()->range().toString(), offset, end_offset, aligned_offset, aligned_end_offset); + } - if (file_segments_limit) - { - while (file_segments.size() > file_segments_limit) - file_segments.pop_back(); + chassert(file_segments_limit ? file_segments.back()->range().left <= end_offset : file_segments.back()->range().contains(end_offset)); } + while (file_segments_limit && file_segments.size() > file_segments_limit) + file_segments.pop_back(); + if (file_segments.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty list of file segments for offset {}, size {} (file size: {})", offset, size, file_size); diff --git a/src/Interpreters/Cache/FileCacheSettings.cpp b/src/Interpreters/Cache/FileCacheSettings.cpp index 6f2f8c4b778f..de21555c0503 100644 --- a/src/Interpreters/Cache/FileCacheSettings.cpp +++ b/src/Interpreters/Cache/FileCacheSettings.cpp @@ -47,6 +47,9 @@ void FileCacheSettings::loadFromConfig(const Poco::Util::AbstractConfiguration & if (config.has(config_prefix + ".boundary_alignment")) boundary_alignment = parseWithSizeSuffix(config.getString(config_prefix + ".boundary_alignment")); + if (boundary_alignment > max_file_segment_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Setting `boundary_alignment` cannot exceed `max_file_segment_size`"); + if (config.has(config_prefix + ".background_download_threads")) background_download_threads = config.getUInt(config_prefix + ".background_download_threads"); diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 03d534f906dc..04c82d71b5ec 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -136,6 +136,8 @@ friend class FileCache; /// Because of reserved_size in tryReserve(). size_t size() const { return right - left + 1; } String toString() const { return fmt::format("[{}, {}]", std::to_string(left), std::to_string(right)); } + + bool contains(size_t offset) const { return left <= offset && offset <= right; } }; static String getCallerId(); From 7515853ad4b3e910f20df99038d706ef77ab2819 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 17 Oct 2023 11:43:51 +0200 Subject: [PATCH 006/274] Fix build --- src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp | 2 +- src/Interpreters/Cache/FileCache.cpp | 4 ++-- src/Interpreters/Cache/FileCache.h | 9 +++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 3c16d3d9ae2f..27d0b6706a66 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -127,7 +127,7 @@ bool CachedOnDiskReadBufferFromFile::nextFileSegmentsBatch() else { CreateFileSegmentSettings create_settings(FileSegmentKind::Regular); - file_segments = cache->getOrSet(cache_key, file_offset_of_buffer_end, size, file_size.value(), settings.filesystem_cache_getorset_batch_size, create_settings); + file_segments = cache->getOrSet(cache_key, file_offset_of_buffer_end, size, file_size.value(), create_settings, settings.filesystem_cache_getorset_batch_size); } return !file_segments->empty(); } diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 6d507413bab8..82a724523e7a 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -412,8 +412,8 @@ FileCache::getOrSet( size_t offset, size_t size, size_t file_size, - size_t file_segments_limit, - const CreateFileSegmentSettings & settings) + const CreateFileSegmentSettings & settings, + size_t file_segments_limit) { ProfileEventTimeIncrement watch(ProfileEvents::FilesystemCacheGetOrSetMicroseconds); diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index 2bf7b9281d5e..d85f50cf34c2 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -84,8 +84,13 @@ class FileCache : private boost::noncopyable * As long as pointers to returned file segments are held * it is guaranteed that these file segments are not removed from cache. */ - FileSegmentsHolderPtr - getOrSet(const Key & key, size_t offset, size_t size, size_t file_size, size_t file_segments_limit, const CreateFileSegmentSettings & settings); + FileSegmentsHolderPtr getOrSet( + const Key & key, + size_t offset, + size_t size, + size_t file_size, + const CreateFileSegmentSettings & settings, + size_t file_segments_limit = 0); /** * Segments in returned list are ordered in ascending order and represent a full contiguous From d837aa675f5ec56434aa7f58332fc4b922b1b9ba Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 17 Oct 2023 13:14:28 +0200 Subject: [PATCH 007/274] Fix --- src/Interpreters/Cache/FileCache.cpp | 29 +++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 82a724523e7a..ba4998bfa00e 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -435,13 +435,26 @@ FileCache::getOrSet( auto prefix_range = FileSegment::Range(aligned_offset, file_segments.empty() ? offset - 1 : file_segments.front()->range().left - 1); auto prefix_file_segments = getImpl(*locked_key, prefix_range, /* file_segments_limit */0); - while (!prefix_file_segments.empty() && prefix_file_segments.front()->range().right < offset) - prefix_file_segments.pop_front(); - - if (!prefix_file_segments.empty()) + if (prefix_file_segments.empty()) + { + range.left = aligned_offset; + } + else { - file_segments.splice(file_segments.begin(), prefix_file_segments); - range.left = file_segments.front()->range().left; + size_t last_right_offset = prefix_file_segments.back()->range().right; + + while (!prefix_file_segments.empty() && prefix_file_segments.front()->range().right < offset) + prefix_file_segments.pop_front(); + + if (prefix_file_segments.empty()) + { + range.left = last_right_offset + 1; + } + else + { + file_segments.splice(file_segments.begin(), prefix_file_segments); + range.left = file_segments.front()->range().left; + } } } @@ -451,7 +464,9 @@ FileCache::getOrSet( /// Get only 1 file segment. auto suffix_file_segments = getImpl(*locked_key, suffix_range, /* file_segments_limit */1); - if (!suffix_file_segments.empty()) + if (suffix_file_segments.empty()) + range.right = aligned_end_offset; + else range.right = suffix_file_segments.front()->range().left - 1; } From 5d8b1cea910b3beb531e1e6122d2596f7197eae8 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 17 Oct 2023 19:19:19 +0200 Subject: [PATCH 008/274] Fix --- src/Interpreters/Cache/FileCache.cpp | 2 +- .../0_stateless/02503_cache_on_write_with_small_segment_size.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index ba4998bfa00e..3349dcc8d3fb 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -347,7 +347,7 @@ void FileCache::fillHolesWithEmptyFileSegments( ++it; } - if (file_segments.size() >= file_segments_limit) + if (file_segments_limit && file_segments.size() >= file_segments_limit) return; if (current_pos <= range.right) diff --git a/tests/queries/0_stateless/02503_cache_on_write_with_small_segment_size.sh b/tests/queries/0_stateless/02503_cache_on_write_with_small_segment_size.sh index 63f912c6bffd..4f3fd0e54f64 100755 --- a/tests/queries/0_stateless/02503_cache_on_write_with_small_segment_size.sh +++ b/tests/queries/0_stateless/02503_cache_on_write_with_small_segment_size.sh @@ -22,6 +22,7 @@ SETTINGS min_bytes_for_wide_part = 0, type = cache, max_size = '128Mi', max_file_segment_size = '10Ki', + boundary_alignment = '5Ki', path = '${CLICKHOUSE_TEST_UNIQUE_NAME}', cache_on_write_operations = 1, enable_filesystem_query_cache_limit = 1, From 22bab4bcc3b13741f19e0b8dd4afa23b4f1cca44 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 18 Oct 2023 11:26:44 +0200 Subject: [PATCH 009/274] Fix configs --- tests/integration/test_filesystem_cache/test.py | 2 +- .../configs/config.d/storage_configuration.xml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_filesystem_cache/test.py b/tests/integration/test_filesystem_cache/test.py index be7b12946a7a..3a6a1ef76eb9 100644 --- a/tests/integration/test_filesystem_cache/test.py +++ b/tests/integration/test_filesystem_cache/test.py @@ -46,7 +46,7 @@ def test_parallel_cache_loading_on_startup(cluster, node_name): path = 'paralel_loading_test', disk = 'hdd_blob', max_file_segment_size = '1Ki', - boundary_alignemt = '1Ki', + boundary_alignment = '1Ki', max_size = '1Gi', max_elements = 10000000, load_metadata_threads = 30); diff --git a/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml b/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml index b527c74e8de4..3064003e6c84 100644 --- a/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml +++ b/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml @@ -12,6 +12,7 @@ /tiny_local_cache/ 10M 1M + 1M 1 0 From 7aa57516c199f45548aea308c4ce2ee1d814e73e Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 18 Oct 2023 11:31:10 +0200 Subject: [PATCH 010/274] Update tests config --- tests/config/users.d/s3_cache.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/config/users.d/s3_cache.xml b/tests/config/users.d/s3_cache.xml index 69b24ecbbc4a..4740f37a90c4 100644 --- a/tests/config/users.d/s3_cache.xml +++ b/tests/config/users.d/s3_cache.xml @@ -3,6 +3,7 @@ 1 1 + 10 From c792d952716f333b198bf014d99fc1dceb6a062b Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 18 Oct 2023 14:41:06 +0200 Subject: [PATCH 011/274] Update config --- tests/config/config.d/s3_storage_policy_by_default.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/config/config.d/s3_storage_policy_by_default.xml b/tests/config/config.d/s3_storage_policy_by_default.xml index dd93a317a779..e161c2ee01a5 100644 --- a/tests/config/config.d/s3_storage_policy_by_default.xml +++ b/tests/config/config.d/s3_storage_policy_by_default.xml @@ -12,6 +12,7 @@ 1Gi cached_s3/ s3 + 10 From 89272e0925c91ed659b51741c58ddc364e149792 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 19 Oct 2023 11:23:56 +0200 Subject: [PATCH 012/274] Fix upgrade check, randomize more settings --- docker/test/upgrade/run.sh | 2 ++ tests/clickhouse-test | 3 +++ tests/config/config.d/s3_storage_policy_by_default.xml | 1 - tests/config/install.sh | 1 + tests/config/users.d/s3_cache_new.xml | 7 +++++++ 5 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/config/users.d/s3_cache_new.xml diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh index c69d90b9af07..3580f8e50215 100644 --- a/docker/test/upgrade/run.sh +++ b/docker/test/upgrade/run.sh @@ -78,6 +78,7 @@ remove_keeper_config "create_if_not_exists" "[01]" rm /etc/clickhouse-server/config.d/merge_tree.xml rm /etc/clickhouse-server/config.d/enable_wait_for_shutdown_replicated_tables.xml rm /etc/clickhouse-server/users.d/nonconst_timezone.xml +rm /etc/clickhouse-server/users.d/s3_cache_new.xml start stop @@ -114,6 +115,7 @@ sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_defau rm /etc/clickhouse-server/config.d/merge_tree.xml rm /etc/clickhouse-server/config.d/enable_wait_for_shutdown_replicated_tables.xml rm /etc/clickhouse-server/users.d/nonconst_timezone.xml +rm /etc/clickhouse-server/users.d/s3_cache_new.xml start diff --git a/tests/clickhouse-test b/tests/clickhouse-test index cab7d7e79ff9..c0c2d482703b 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -577,6 +577,9 @@ class SettingsRandomizer: ), "remote_filesystem_read_method": lambda: random.choice(["read", "threadpool"]), "local_filesystem_read_prefetch": lambda: random.randint(0, 1), + "filesystem_cache_getorset_batch_size": lambda: random.randint(0, 3, 10, 50), + "read_from_filesystem_cache_if_exists_otherwise_bypass_cache": lambda: random.randint(0, 1), + "throw_on_error_from_cache_on_write_operations": lambda: random.randint(0, 1), "remote_filesystem_read_prefetch": lambda: random.randint(0, 1), "allow_prefetched_read_pool_for_remote_filesystem": lambda: random.randint( 0, 1 diff --git a/tests/config/config.d/s3_storage_policy_by_default.xml b/tests/config/config.d/s3_storage_policy_by_default.xml index e161c2ee01a5..dd93a317a779 100644 --- a/tests/config/config.d/s3_storage_policy_by_default.xml +++ b/tests/config/config.d/s3_storage_policy_by_default.xml @@ -12,7 +12,6 @@ 1Gi cached_s3/ s3 - 10 diff --git a/tests/config/install.sh b/tests/config/install.sh index 9e3b235515d9..d76949fadc78 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -151,6 +151,7 @@ if [[ -n "$EXPORT_S3_STORAGE_POLICIES" ]]; then ln -sf $SRC_PATH/config.d/storage_conf.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/users.d/s3_cache.xml $DEST_SERVER_PATH/users.d/ + ln -sf $SRC_PATH/users.d/s3_cache_new.xml $DEST_SERVER_PATH/users.d/ fi if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then diff --git a/tests/config/users.d/s3_cache_new.xml b/tests/config/users.d/s3_cache_new.xml new file mode 100644 index 000000000000..638b72679600 --- /dev/null +++ b/tests/config/users.d/s3_cache_new.xml @@ -0,0 +1,7 @@ + + + + 10 + + + From b13adbbeab1d5d0c91562f187ad5f4d651316311 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 20 Oct 2023 11:48:27 +0200 Subject: [PATCH 013/274] Fix style check --- tests/clickhouse-test | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index c0c2d482703b..36ac409a4cb5 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -577,8 +577,10 @@ class SettingsRandomizer: ), "remote_filesystem_read_method": lambda: random.choice(["read", "threadpool"]), "local_filesystem_read_prefetch": lambda: random.randint(0, 1), - "filesystem_cache_getorset_batch_size": lambda: random.randint(0, 3, 10, 50), - "read_from_filesystem_cache_if_exists_otherwise_bypass_cache": lambda: random.randint(0, 1), + "filesystem_cache_getorset_batch_size": lambda: random.choice([0, 3, 10, 50]), + "read_from_filesystem_cache_if_exists_otherwise_bypass_cache": lambda: random.randint( + 0, 1 + ), "throw_on_error_from_cache_on_write_operations": lambda: random.randint(0, 1), "remote_filesystem_read_prefetch": lambda: random.randint(0, 1), "allow_prefetched_read_pool_for_remote_filesystem": lambda: random.randint( From 2c055480d622d0ccc05b65c9c0252b36b66f7eca Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 24 Oct 2023 14:52:47 +0000 Subject: [PATCH 014/274] Remove unnecessary flag --- src/Processors/Sources/RemoteSource.cpp | 6 ++---- src/Processors/Sources/RemoteSource.h | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Processors/Sources/RemoteSource.cpp b/src/Processors/Sources/RemoteSource.cpp index 74ab36490686..6ca5e6117135 100644 --- a/src/Processors/Sources/RemoteSource.cpp +++ b/src/Processors/Sources/RemoteSource.cpp @@ -42,7 +42,7 @@ void RemoteSource::setStorageLimits(const std::shared_ptr RemoteSource::tryGenerate() { /// onCancel() will do the cancel if the query was sent. - if (was_query_canceled) + if (isCancelled()) return {}; if (!was_query_sent) @@ -169,7 +169,6 @@ std::optional RemoteSource::tryGenerate() void RemoteSource::onCancel() { - was_query_canceled = true; query_executor->cancel(); } @@ -177,7 +176,6 @@ void RemoteSource::onUpdatePorts() { if (getPort().isFinished()) { - was_query_canceled = true; query_executor->finish(); } } diff --git a/src/Processors/Sources/RemoteSource.h b/src/Processors/Sources/RemoteSource.h index da39b5d00464..dbfa01563316 100644 --- a/src/Processors/Sources/RemoteSource.h +++ b/src/Processors/Sources/RemoteSource.h @@ -39,7 +39,6 @@ class RemoteSource final : public ISource void onCancel() override; private: - std::atomic was_query_canceled = false; bool was_query_sent = false; bool add_aggregation_info = false; RemoteQueryExecutorPtr query_executor; From efbcac4e600430262e4999c2a799e92dc6e5c4c8 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Wed, 25 Oct 2023 10:21:06 +0200 Subject: [PATCH 015/274] Bug fix explain ast with parameterized view --- src/Interpreters/executeQuery.cpp | 6 ++++++ .../02903_parameterized_view_explain_ast.reference | 12 ++++++++++++ .../02903_parameterized_view_explain_ast.sql | 3 +++ 3 files changed, 21 insertions(+) create mode 100644 tests/queries/0_stateless/02903_parameterized_view_explain_ast.reference create mode 100644 tests/queries/0_stateless/02903_parameterized_view_explain_ast.sql diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index decda4c62f92..557f80e8d70a 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -730,6 +730,12 @@ static std::tuple executeQueryImpl( bool is_create_parameterized_view = false; if (const auto * create_query = ast->as()) is_create_parameterized_view = create_query->isParameterizedView(); + else if (const auto * explain_query = ast->as()) + { + assert(explain_query->children.size() => 1); + if (const auto * create_of_explain_query = explain_query->children[0]->as()) + is_create_parameterized_view = create_of_explain_query->isParameterizedView(); + } /// Replace ASTQueryParameter with ASTLiteral for prepared statements. /// Even if we don't have parameters in query_context, check that AST doesn't have unknown parameters diff --git a/tests/queries/0_stateless/02903_parameterized_view_explain_ast.reference b/tests/queries/0_stateless/02903_parameterized_view_explain_ast.reference new file mode 100644 index 000000000000..6ee8d0c3d230 --- /dev/null +++ b/tests/queries/0_stateless/02903_parameterized_view_explain_ast.reference @@ -0,0 +1,12 @@ +CreateQuery numbers_pv (children 2) + Identifier numbers_pv + SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 3) + ExpressionList (children 1) + Asterisk + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + TableIdentifier numbers + QueryParameter amount:UInt8 diff --git a/tests/queries/0_stateless/02903_parameterized_view_explain_ast.sql b/tests/queries/0_stateless/02903_parameterized_view_explain_ast.sql new file mode 100644 index 000000000000..6af6dab2f4eb --- /dev/null +++ b/tests/queries/0_stateless/02903_parameterized_view_explain_ast.sql @@ -0,0 +1,3 @@ +EXPLAIN AST +CREATE VIEW numbers_pv AS +SELECT * FROM numbers LIMIT {amount:UInt8}; \ No newline at end of file From d6c63e07ce97932bdfcf7baa5ab76bd9c1fee85b Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Wed, 25 Oct 2023 10:23:22 +0200 Subject: [PATCH 016/274] Fixed expression --- src/Interpreters/executeQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 557f80e8d70a..27fbce5311c3 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -732,7 +732,7 @@ static std::tuple executeQueryImpl( is_create_parameterized_view = create_query->isParameterizedView(); else if (const auto * explain_query = ast->as()) { - assert(explain_query->children.size() => 1); + assert(explain_query->children.size() >= 1); if (const auto * create_of_explain_query = explain_query->children[0]->as()) is_create_parameterized_view = create_of_explain_query->isParameterizedView(); } From 940d099e84d92eaaacaa96682c5a94b26f7a782c Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Fri, 27 Oct 2023 16:50:34 -0700 Subject: [PATCH 017/274] Set correct max_block_size value in docs --- docs/en/operations/settings/settings.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index ccf290c8e20c..60eda45ab22f 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -731,11 +731,13 @@ Default value: LZ4. ## max_block_size {#setting-max_block_size} -In ClickHouse, data is processed by blocks (sets of column parts). The internal processing cycles for a single block are efficient enough, but there are noticeable expenditures on each block. The `max_block_size` setting is a recommendation for what size of the block (in a count of rows) to load from tables. The block size shouldn’t be too small, so that the expenditures on each block are still noticeable, but not too large so that the query with LIMIT that is completed after the first block is processed quickly. The goal is to avoid consuming too much memory when extracting a large number of columns in multiple threads and to preserve at least some cache locality. +In ClickHouse, data is processed by blocks, which are sets of column parts. The internal processing cycles for a single block are efficient but there are noticeable costs when processing each block. -Default value: 65,536. +The `max_block_size` setting indicates the recommended maximum number of rows to include in a single block when loading data from tables. Blocks the size of `max_block_size` are not always loaded from the table: if ClickHouse determines that less data needs to be retrieved, a smaller block is processed. -Blocks the size of `max_block_size` are not always loaded from the table. If it is obvious that less data needs to be retrieved, a smaller block is processed. +The block size should not be too small to avoid noticeable costs when processing each block. It should also not be too large to ensure that queries with a LIMIT clause execute quickly after processing the first block. When setting `max_block_size`, the goal should be to avoid consuming too much memory when extracting a large number of columns in multiple threads and to preserve at least some cache locality. + +Default value: `65,409` ## preferred_block_size_bytes {#preferred-block-size-bytes} From f8e209ebd26f278ed582adf0aab8f786be8bb591 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 30 Oct 2023 13:45:18 +0300 Subject: [PATCH 018/274] WindowTransform decrease amount of virtual function calls --- src/Processors/Transforms/WindowTransform.cpp | 19 ++++++++++++------- src/Processors/Transforms/WindowTransform.h | 3 +++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp index 9565a073f481..df6246510bd5 100644 --- a/src/Processors/Transforms/WindowTransform.cpp +++ b/src/Processors/Transforms/WindowTransform.cpp @@ -257,6 +257,7 @@ WindowTransform::WindowTransform(const Block & input_header_, window_description.frame = *custom_default_frame; } + workspace.is_aggregate_function_state = workspace.aggregate_function->isState(); workspace.aggregate_function_state.reset( aggregate_function->sizeOfData(), aggregate_function->alignOfData()); @@ -957,10 +958,7 @@ void WindowTransform::updateAggregationState() auto * columns = ws.argument_columns.data(); // Removing arena.get() from the loop makes it faster somehow... auto * arena_ptr = arena.get(); - for (auto row = first_row; row < past_the_end_row; ++row) - { - a->add(buf, columns, row, arena_ptr); - } + a->addBatchSinglePlaceFromInterval(first_row, past_the_end_row, buf, columns, arena_ptr); } } } @@ -987,9 +985,16 @@ void WindowTransform::writeOutCurrentRow() // FIXME does it also allocate the result on the arena? // We'll have to pass it out with blocks then... - /// We should use insertMergeResultInto to insert result into ColumnAggregateFunction - /// correctly if result contains AggregateFunction's states - a->insertMergeResultInto(buf, *result_column, arena.get()); + if (ws.is_aggregate_function_state) + { + /// We should use insertMergeResultInto to insert result into ColumnAggregateFunction + /// correctly if result contains AggregateFunction's states + a->insertMergeResultInto(buf, *result_column, arena.get()); + } + else + { + a->insertResultInto(buf, *result_column, arena.get()); + } } } } diff --git a/src/Processors/Transforms/WindowTransform.h b/src/Processors/Transforms/WindowTransform.h index de3e82d15ee6..347c2516230f 100644 --- a/src/Processors/Transforms/WindowTransform.h +++ b/src/Processors/Transforms/WindowTransform.h @@ -26,6 +26,9 @@ struct WindowFunctionWorkspace { AggregateFunctionPtr aggregate_function; + // Cached value of aggregate function isState virtual method + bool is_aggregate_function_state = false; + // This field is set for pure window functions. When set, we ignore the // window_function.aggregate_function, and work through this interface // instead. From ce36a6475dcff34f9d4f0510c6608bdf261478e8 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 31 Oct 2023 10:12:31 +0100 Subject: [PATCH 019/274] Fixed cland tidy build --- src/Interpreters/executeQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 27fbce5311c3..9655e8b28554 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -732,7 +732,7 @@ static std::tuple executeQueryImpl( is_create_parameterized_view = create_query->isParameterizedView(); else if (const auto * explain_query = ast->as()) { - assert(explain_query->children.size() >= 1); + assert(!explain_query->children.empty()); if (const auto * create_of_explain_query = explain_query->children[0]->as()) is_create_parameterized_view = create_of_explain_query->isParameterizedView(); } From 8a1ab02b96e234a9cdc1018b692381e8e4abf9c2 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 31 Oct 2023 12:59:16 +0100 Subject: [PATCH 020/274] Update s3_cache.xml --- tests/config/users.d/s3_cache.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/config/users.d/s3_cache.xml b/tests/config/users.d/s3_cache.xml index 4740f37a90c4..69b24ecbbc4a 100644 --- a/tests/config/users.d/s3_cache.xml +++ b/tests/config/users.d/s3_cache.xml @@ -3,7 +3,6 @@ 1 1 - 10 From 52a3d37ebe6008fc4301d369f6309587b32e648b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 27 Oct 2023 16:59:14 +0200 Subject: [PATCH 021/274] Try reducing number of different images --- .../integration/test_backward_compatibility/test.py | 2 +- .../test_aggregate_fixed_key.py | 2 +- .../test_convert_ordinary.py | 2 +- .../test_cte_distributed.py | 2 +- .../test_insert_profile_events.py | 2 +- .../test_memory_bound_aggregation.py | 4 ++-- .../test_normalized_count_comparison.py | 2 +- .../test_select_aggregate_alias_column.py | 2 +- .../test_vertical_merges_from_compact_parts.py | 2 +- .../test_default_compression_codec/test.py | 2 +- tests/integration/test_disk_over_web_server/test.py | 2 +- .../test_distributed_backward_compatability/test.py | 2 +- .../test.py | 2 +- .../test_distributed_inter_server_secret/test.py | 2 +- .../test_groupBitmapAnd_on_distributed/test.py | 2 +- tests/integration/test_old_versions/test.py | 2 +- tests/integration/test_polymorphic_parts/test.py | 2 +- .../test_replicated_merge_tree_compatibility/test.py | 4 ++-- tests/integration/test_replicating_constants/test.py | 2 +- tests/integration/test_ttl_replicated/test.py | 6 +++--- tests/integration/test_version_update/test.py | 12 ++++++------ .../test_version_update_after_mutation/test.py | 6 +++--- 22 files changed, 33 insertions(+), 33 deletions(-) diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py index 6f21b184a953..847483f2b9bf 100644 --- a/tests/integration/test_backward_compatibility/test.py +++ b/tests/integration/test_backward_compatibility/test.py @@ -7,7 +7,7 @@ "node1", with_zookeeper=True, image="yandex/clickhouse-server", - tag="19.17.8.54", + tag="19.16.9.37", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py index cf258987cbfc..94bc1d3bfc96 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py @@ -7,7 +7,7 @@ "node1", with_zookeeper=True, image="yandex/clickhouse-server", - tag="21.3", + tag="20.8.11.17", with_installed_binary=True, allow_analyzer=False, ) diff --git a/tests/integration/test_backward_compatibility/test_convert_ordinary.py b/tests/integration/test_backward_compatibility/test_convert_ordinary.py index 36facdd59b17..034a68e0f309 100644 --- a/tests/integration/test_backward_compatibility/test_convert_ordinary.py +++ b/tests/integration/test_backward_compatibility/test_convert_ordinary.py @@ -5,7 +5,7 @@ node = cluster.add_instance( "node", image="yandex/clickhouse-server", - tag="19.17.8.54", + tag="19.16.9.37", stay_alive=True, with_zookeeper=True, with_installed_binary=True, diff --git a/tests/integration/test_backward_compatibility/test_cte_distributed.py b/tests/integration/test_backward_compatibility/test_cte_distributed.py index c68468aad758..d47ae3aa255b 100644 --- a/tests/integration/test_backward_compatibility/test_cte_distributed.py +++ b/tests/integration/test_backward_compatibility/test_cte_distributed.py @@ -8,7 +8,7 @@ "node2", with_zookeeper=False, image="yandex/clickhouse-server", - tag="21.7.3.14", + tag="21.6", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_backward_compatibility/test_insert_profile_events.py b/tests/integration/test_backward_compatibility/test_insert_profile_events.py index 8564c6b59526..d38bece78551 100644 --- a/tests/integration/test_backward_compatibility/test_insert_profile_events.py +++ b/tests/integration/test_backward_compatibility/test_insert_profile_events.py @@ -11,7 +11,7 @@ old_node = cluster.add_instance( "old_node", image="clickhouse/clickhouse-server", - tag="22.5.1.2079", + tag="22.6", with_installed_binary=True, allow_analyzer=False, ) diff --git a/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py index 96b41c813848..5261a279a4fe 100644 --- a/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py @@ -7,7 +7,7 @@ "node1", with_zookeeper=False, image="yandex/clickhouse-server", - tag="21.1", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, allow_analyzer=False, @@ -16,7 +16,7 @@ "node2", with_zookeeper=False, image="yandex/clickhouse-server", - tag="21.1", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py b/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py index 3cd708d50293..cf7a25e8dc1b 100644 --- a/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py +++ b/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py @@ -8,7 +8,7 @@ "node2", with_zookeeper=False, image="yandex/clickhouse-server", - tag="21.7.2.7", + tag="21.6", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py index 7e10b6ab4306..ec1d7fedac55 100644 --- a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py +++ b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py @@ -8,7 +8,7 @@ "node2", with_zookeeper=False, image="yandex/clickhouse-server", - tag="21.7.2.7", + tag="21.6", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py index 9c9d1a4d3121..e0a9b5ebad61 100644 --- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py +++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py @@ -7,7 +7,7 @@ node_old = cluster.add_instance( "node1", image="clickhouse/clickhouse-server", - tag="22.8", + tag="22.6", stay_alive=True, with_installed_binary=True, with_zookeeper=True, diff --git a/tests/integration/test_default_compression_codec/test.py b/tests/integration/test_default_compression_codec/test.py index 82d5eb04d2aa..db116ff42f3b 100644 --- a/tests/integration/test_default_compression_codec/test.py +++ b/tests/integration/test_default_compression_codec/test.py @@ -29,7 +29,7 @@ "node3", main_configs=["configs/default_compression.xml", "configs/wide_parts_only.xml"], image="yandex/clickhouse-server", - tag="20.3.16", + tag="19.16.9.37", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_disk_over_web_server/test.py b/tests/integration/test_disk_over_web_server/test.py index 7695d2354251..a71fdeff302b 100644 --- a/tests/integration/test_disk_over_web_server/test.py +++ b/tests/integration/test_disk_over_web_server/test.py @@ -38,7 +38,7 @@ def cluster(): stay_alive=True, with_installed_binary=True, image="clickhouse/clickhouse-server", - tag="22.8.14.53", + tag="22.6", allow_analyzer=False, ) diff --git a/tests/integration/test_distributed_backward_compatability/test.py b/tests/integration/test_distributed_backward_compatability/test.py index c48a7ad1fa19..319a4c08e60b 100644 --- a/tests/integration/test_distributed_backward_compatability/test.py +++ b/tests/integration/test_distributed_backward_compatability/test.py @@ -8,7 +8,7 @@ "node1", main_configs=["configs/remote_servers.xml"], image="yandex/clickhouse-server", - tag="20.8.9.6", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_distributed_insert_backward_compatibility/test.py b/tests/integration/test_distributed_insert_backward_compatibility/test.py index 1e566d5e2da0..7cfea61ffffb 100644 --- a/tests/integration/test_distributed_insert_backward_compatibility/test.py +++ b/tests/integration/test_distributed_insert_backward_compatibility/test.py @@ -11,7 +11,7 @@ "node2", main_configs=["configs/remote_servers.xml"], image="yandex/clickhouse-server", - tag="21.11.9.1", + tag="21.6", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py index 1aeaddcf3c5a..62beeee80e14 100644 --- a/tests/integration/test_distributed_inter_server_secret/test.py +++ b/tests/integration/test_distributed_inter_server_secret/test.py @@ -31,7 +31,7 @@ def make_instance(name, cfg, *args, **kwargs): "configs/remote_servers_backward.xml", image="clickhouse/clickhouse-server", # version without DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 - tag="23.2.3", + tag="22.6", with_installed_binary=True, allow_analyzer=False, ) diff --git a/tests/integration/test_groupBitmapAnd_on_distributed/test.py b/tests/integration/test_groupBitmapAnd_on_distributed/test.py index 8cf7e0fb2c19..5d3dda8ecf27 100644 --- a/tests/integration/test_groupBitmapAnd_on_distributed/test.py +++ b/tests/integration/test_groupBitmapAnd_on_distributed/test.py @@ -26,7 +26,7 @@ "node4", main_configs=["configs/clusters.xml"], image="yandex/clickhouse-server", - tag="21.5", + tag="21.6", with_zookeeper=True, allow_analyzer=False, ) diff --git a/tests/integration/test_old_versions/test.py b/tests/integration/test_old_versions/test.py index aff07c531141..b59bfcc4f6b5 100644 --- a/tests/integration/test_old_versions/test.py +++ b/tests/integration/test_old_versions/test.py @@ -55,7 +55,7 @@ node19_16 = cluster.add_instance( "node19_16", image="yandex/clickhouse-server", - tag="19.16.2.2", + tag="19.16.9.37", with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"], allow_analyzer=False, diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index debb509de90e..ba9b5ec6cac7 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -360,7 +360,7 @@ def test_different_part_types_on_replicas(start_cluster, table, part_type): user_configs=["configs_old/users.d/not_optimize_count.xml"], with_zookeeper=True, image="yandex/clickhouse-server", - tag="19.17.8.54", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_replicated_merge_tree_compatibility/test.py b/tests/integration/test_replicated_merge_tree_compatibility/test.py index c30a0d86c98a..32a44aa65b98 100644 --- a/tests/integration/test_replicated_merge_tree_compatibility/test.py +++ b/tests/integration/test_replicated_merge_tree_compatibility/test.py @@ -6,7 +6,7 @@ "node1", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.12.4.5", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, allow_analyzer=False, @@ -15,7 +15,7 @@ "node2", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.12.4.5", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, allow_analyzer=False, diff --git a/tests/integration/test_replicating_constants/test.py b/tests/integration/test_replicating_constants/test.py index 00781e473c70..9669e890cd37 100644 --- a/tests/integration/test_replicating_constants/test.py +++ b/tests/integration/test_replicating_constants/test.py @@ -9,7 +9,7 @@ "node2", with_zookeeper=True, image="yandex/clickhouse-server", - tag="19.1.14", + tag="19.16.9.37", with_installed_binary=True, allow_analyzer=False, ) diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index 117ebe37dd2b..29ce2b3dc8dc 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -17,7 +17,7 @@ "node4", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.12.4.5", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, main_configs=[ @@ -30,7 +30,7 @@ "node5", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.12.4.5", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, main_configs=[ @@ -42,7 +42,7 @@ "node6", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.12.4.5", + tag="20.8.11.17", stay_alive=True, with_installed_binary=True, main_configs=[ diff --git a/tests/integration/test_version_update/test.py b/tests/integration/test_version_update/test.py index b8fa3e7ebb40..a752960bc76f 100644 --- a/tests/integration/test_version_update/test.py +++ b/tests/integration/test_version_update/test.py @@ -12,18 +12,18 @@ "node2", with_zookeeper=True, image="yandex/clickhouse-server", - tag="21.2", + tag="20.8.11.17", with_installed_binary=True, stay_alive=True, allow_analyzer=False, ) -# Use differents nodes because if there is node.restart_from_latest_version(), then in later tests +# Use different nodes because if there is node.restart_from_latest_version(), then in later tests # it will be with latest version, but shouldn't, order of tests in CI is shuffled. node3 = cluster.add_instance( "node3", image="yandex/clickhouse-server", - tag="21.5", + tag="21.6", with_installed_binary=True, stay_alive=True, allow_analyzer=False, @@ -31,7 +31,7 @@ node4 = cluster.add_instance( "node4", image="yandex/clickhouse-server", - tag="21.5", + tag="21.6", with_installed_binary=True, stay_alive=True, allow_analyzer=False, @@ -39,7 +39,7 @@ node5 = cluster.add_instance( "node5", image="yandex/clickhouse-server", - tag="21.5", + tag="21.6", with_installed_binary=True, stay_alive=True, allow_analyzer=False, @@ -47,7 +47,7 @@ node6 = cluster.add_instance( "node6", image="yandex/clickhouse-server", - tag="21.5", + tag="21.6", with_installed_binary=True, stay_alive=True, allow_analyzer=False, diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index f3ae190ee461..9fb396b1c149 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -10,7 +10,7 @@ "node1", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.4.9.110", + tag="20.8.11.17", with_installed_binary=True, stay_alive=True, main_configs=[ @@ -22,7 +22,7 @@ "node2", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.4.9.110", + tag="20.8.11.17", with_installed_binary=True, stay_alive=True, main_configs=[ @@ -34,7 +34,7 @@ "node3", with_zookeeper=True, image="yandex/clickhouse-server", - tag="20.4.9.110", + tag="20.8.11.17", with_installed_binary=True, stay_alive=True, main_configs=[ From 957671bf744cd173676f5be0f8ca14d0f03118df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 31 Oct 2023 18:06:28 +0100 Subject: [PATCH 022/274] Adapt to work with releases without DROP SYNC --- tests/integration/test_ttl_replicated/test.py | 222 ++++++++---------- 1 file changed, 102 insertions(+), 120 deletions(-) diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index 29ce2b3dc8dc..119a211ae451 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -66,47 +66,41 @@ def started_cluster(): cluster.shutdown() -def drop_table(nodes, table_name): - for node in nodes: - node.query("DROP TABLE IF EXISTS {} SYNC".format(table_name)) - - # Column TTL works only with wide parts, because it's very expensive to apply it for compact parts def test_ttl_columns(started_cluster): - drop_table([node1, node2], "test_ttl") + table_name = f"test_ttl_{node1.name}_{node2.name}" for node in [node1, node2]: node.query( """ - CREATE TABLE test_ttl(date DateTime, id UInt32, a Int32 TTL date + INTERVAL 1 DAY, b Int32 TTL date + INTERVAL 1 MONTH) + CREATE TABLE {table_name}(date DateTime, id UInt32, a Int32 TTL date + INTERVAL 1 DAY, b Int32 TTL date + INTERVAL 1 MONTH) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_columns', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) SETTINGS merge_with_ttl_timeout=0, min_bytes_for_wide_part=0, max_merge_selecting_sleep_ms=6000; """.format( - replica=node.name + table_name=table_name, replica=node.name ) ) node1.query( - "INSERT INTO test_ttl VALUES (toDateTime('2000-10-10 00:00:00'), 1, 1, 3)" + f"INSERT INTO {table_name} VALUES (toDateTime('2000-10-10 00:00:00'), 1, 1, 3)" ) node1.query( - "INSERT INTO test_ttl VALUES (toDateTime('2000-10-11 10:00:00'), 2, 2, 4)" + f"INSERT INTO {table_name} VALUES (toDateTime('2000-10-11 10:00:00'), 2, 2, 4)" ) time.sleep(1) # sleep to allow use ttl merge selector for second time - node1.query("OPTIMIZE TABLE test_ttl FINAL") + node1.query(f"OPTIMIZE TABLE {table_name} FINAL") expected = "1\t0\t0\n2\t0\t0\n" - assert TSV(node1.query("SELECT id, a, b FROM test_ttl ORDER BY id")) == TSV( + assert TSV(node1.query(f"SELECT id, a, b FROM {table_name} ORDER BY id")) == TSV( expected ) - assert TSV(node2.query("SELECT id, a, b FROM test_ttl ORDER BY id")) == TSV( + assert TSV(node2.query(f"SELECT id, a, b FROM {table_name} ORDER BY id")) == TSV( expected ) def test_merge_with_ttl_timeout(started_cluster): - table = "test_merge_with_ttl_timeout" - drop_table([node1, node2], table) + table = f"test_merge_with_ttl_timeout_{node1.name}_{node2.name}" for node in [node1, node2]: node.query( """ @@ -157,11 +151,11 @@ def test_merge_with_ttl_timeout(started_cluster): def test_ttl_many_columns(started_cluster): - drop_table([node1, node2], "test_ttl_2") + table = f"test_ttl_2{node1.name}_{node2.name}" for node in [node1, node2]: node.query( """ - CREATE TABLE test_ttl_2(date DateTime, id UInt32, + CREATE TABLE {table}(date DateTime, id UInt32, a Int32 TTL date, _idx Int32 TTL date, _offset Int32 TTL date, @@ -169,44 +163,40 @@ def test_ttl_many_columns(started_cluster): ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_2', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) SETTINGS merge_with_ttl_timeout=0, max_merge_selecting_sleep_ms=6000; """.format( - replica=node.name + table=table, replica=node.name ) ) - node1.query("SYSTEM STOP TTL MERGES test_ttl_2") - node2.query("SYSTEM STOP TTL MERGES test_ttl_2") + node1.query(f"SYSTEM STOP TTL MERGES {table}") + node2.query(f"SYSTEM STOP TTL MERGES {table}") node1.query( - "INSERT INTO test_ttl_2 VALUES (toDateTime('2000-10-10 00:00:00'), 1, 2, 3, 4, 5)" + f"INSERT INTO {table} VALUES (toDateTime('2000-10-10 00:00:00'), 1, 2, 3, 4, 5)" ) node1.query( - "INSERT INTO test_ttl_2 VALUES (toDateTime('2100-10-10 10:00:00'), 6, 7, 8, 9, 10)" + f"INSERT INTO {table} VALUES (toDateTime('2100-10-10 10:00:00'), 6, 7, 8, 9, 10)" ) - node2.query("SYSTEM SYNC REPLICA test_ttl_2", timeout=5) + node2.query(f"SYSTEM SYNC REPLICA {table}", timeout=5) # Check that part will appear in result of merge - node1.query("SYSTEM STOP FETCHES test_ttl_2") - node2.query("SYSTEM STOP FETCHES test_ttl_2") + node1.query(f"SYSTEM STOP FETCHES {table}") + node2.query(f"SYSTEM STOP FETCHES {table}") - node1.query("SYSTEM START TTL MERGES test_ttl_2") - node2.query("SYSTEM START TTL MERGES test_ttl_2") + node1.query(f"SYSTEM START TTL MERGES {table}") + node2.query(f"SYSTEM START TTL MERGES {table}") time.sleep(1) # sleep to allow use ttl merge selector for second time - node1.query("OPTIMIZE TABLE test_ttl_2 FINAL", timeout=5) + node1.query(f"OPTIMIZE TABLE {table} FINAL", timeout=5) - node2.query("SYSTEM SYNC REPLICA test_ttl_2", timeout=5) + node2.query(f"SYSTEM SYNC REPLICA {table}", timeout=5) expected = "1\t0\t0\t0\t0\n6\t7\t8\t9\t10\n" assert TSV( - node1.query( - "SELECT id, a, _idx, _offset, _partition FROM test_ttl_2 ORDER BY id" - ) + node1.query(f"SELECT id, a, _idx, _offset, _partition FROM {table} ORDER BY id") ) == TSV(expected) assert TSV( - node2.query( - "SELECT id, a, _idx, _offset, _partition FROM test_ttl_2 ORDER BY id" - ) + node2.query(f"SELECT id, a, _idx, _offset, _partition FROM {table} ORDER BY id") ) == TSV(expected) @@ -218,107 +208,107 @@ def test_ttl_many_columns(started_cluster): ], ) def test_ttl_table(started_cluster, delete_suffix): - drop_table([node1, node2], "test_ttl") + table = f"test_ttl_table_{delete_suffix}_{node1.name}_{node2.name}" for node in [node1, node2]: node.query( """ - CREATE TABLE test_ttl(date DateTime, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl', '{replica}') + CREATE TABLE {table}(date DateTime, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 1 DAY {delete_suffix} SETTINGS merge_with_ttl_timeout=0, max_merge_selecting_sleep_ms=6000; """.format( - replica=node.name, delete_suffix=delete_suffix + table=table, replica=node.name, delete_suffix=delete_suffix ) ) - node1.query("INSERT INTO test_ttl VALUES (toDateTime('2000-10-10 00:00:00'), 1)") - node1.query("INSERT INTO test_ttl VALUES (toDateTime('2000-10-11 10:00:00'), 2)") + node1.query(f"INSERT INTO {table} VALUES (toDateTime('2000-10-10 00:00:00'), 1)") + node1.query(f"INSERT INTO {table} VALUES (toDateTime('2000-10-11 10:00:00'), 2)") time.sleep(1) # sleep to allow use ttl merge selector for second time - node1.query("OPTIMIZE TABLE test_ttl FINAL") + node1.query(f"OPTIMIZE TABLE {table} FINAL") - assert TSV(node1.query("SELECT * FROM test_ttl")) == TSV("") - assert TSV(node2.query("SELECT * FROM test_ttl")) == TSV("") + assert TSV(node1.query(f"SELECT * FROM {table}")) == TSV("") + assert TSV(node2.query(f"SELECT * FROM {table}")) == TSV("") def test_modify_ttl(started_cluster): - drop_table([node1, node2], "test_ttl") + table = f"test_modify_ttl_{node1.name}_{node2.name}" for node in [node1, node2]: node.query( """ - CREATE TABLE test_ttl(d DateTime, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_modify', '{replica}') + CREATE TABLE {table}(d DateTime, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}', '{replica}') ORDER BY id """.format( - replica=node.name + table=table, replica=node.name ) ) node1.query( - "INSERT INTO test_ttl VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)" + f"INSERT INTO {table} VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)" ) - node2.query("SYSTEM SYNC REPLICA test_ttl", timeout=20) + node2.query(f"SYSTEM SYNC REPLICA {table}", timeout=20) node1.query( - "ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 4 HOUR SETTINGS replication_alter_partitions_sync = 2" + f"ALTER TABLE {table} MODIFY TTL d + INTERVAL 4 HOUR SETTINGS replication_alter_partitions_sync = 2" ) - assert node2.query("SELECT id FROM test_ttl") == "2\n3\n" + assert node2.query(f"SELECT id FROM {table}") == "2\n3\n" node2.query( - "ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 2 HOUR SETTINGS replication_alter_partitions_sync = 2" + f"ALTER TABLE {table} MODIFY TTL d + INTERVAL 2 HOUR SETTINGS replication_alter_partitions_sync = 2" ) - assert node1.query("SELECT id FROM test_ttl") == "3\n" + assert node1.query(f"SELECT id FROM {table}") == "3\n" node1.query( - "ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 30 MINUTE SETTINGS replication_alter_partitions_sync = 2" + f"ALTER TABLE {table} MODIFY TTL d + INTERVAL 30 MINUTE SETTINGS replication_alter_partitions_sync = 2" ) - assert node2.query("SELECT id FROM test_ttl") == "" + assert node2.query(f"SELECT id FROM {table}") == "" def test_modify_column_ttl(started_cluster): - drop_table([node1, node2], "test_ttl") + table = f"test_modify_column_ttl_{node1.name}_{node2.name}" for node in [node1, node2]: node.query( """ - CREATE TABLE test_ttl(d DateTime, id UInt32 DEFAULT 42) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_column', '{replica}') + CREATE TABLE {table}(d DateTime, id UInt32 DEFAULT 42) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}', '{replica}') ORDER BY d """.format( - replica=node.name + table=table, replica=node.name ) ) node1.query( - "INSERT INTO test_ttl VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)" + f"INSERT INTO {table} VALUES (now() - INTERVAL 5 HOUR, 1), (now() - INTERVAL 3 HOUR, 2), (now() - INTERVAL 1 HOUR, 3)" ) - node2.query("SYSTEM SYNC REPLICA test_ttl", timeout=20) + node2.query(f"SYSTEM SYNC REPLICA {table}", timeout=20) node1.query( - "ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 4 HOUR SETTINGS replication_alter_partitions_sync = 2" + f"ALTER TABLE {table} MODIFY COLUMN id UInt32 TTL d + INTERVAL 4 HOUR SETTINGS replication_alter_partitions_sync = 2" ) - assert node2.query("SELECT id FROM test_ttl") == "42\n2\n3\n" + assert node2.query(f"SELECT id FROM {table}") == "42\n2\n3\n" node1.query( - "ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 2 HOUR SETTINGS replication_alter_partitions_sync = 2" + f"ALTER TABLE {table} MODIFY COLUMN id UInt32 TTL d + INTERVAL 2 HOUR SETTINGS replication_alter_partitions_sync = 2" ) - assert node1.query("SELECT id FROM test_ttl") == "42\n42\n3\n" + assert node1.query(f"SELECT id FROM {table}") == "42\n42\n3\n" node1.query( - "ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 30 MINUTE SETTINGS replication_alter_partitions_sync = 2" + f"ALTER TABLE {table} MODIFY COLUMN id UInt32 TTL d + INTERVAL 30 MINUTE SETTINGS replication_alter_partitions_sync = 2" ) - assert node2.query("SELECT id FROM test_ttl") == "42\n42\n42\n" + assert node2.query(f"SELECT id FROM {table}") == "42\n42\n42\n" def test_ttl_double_delete_rule_returns_error(started_cluster): - drop_table([node1, node2], "test_ttl") + table = "test_ttl_double_delete_rule_returns_error" try: node1.query( """ - CREATE TABLE test_ttl(date DateTime, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_double_delete', '{replica}') + CREATE TABLE {table}(date DateTime, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 1 DAY, date + INTERVAL 2 DAY SETTINGS merge_with_ttl_timeout=0, max_merge_selecting_sleep_ms=6000 """.format( - replica=node1.name + table=table, replica=node1.name ) ) assert False @@ -364,7 +354,6 @@ def test_ttl_alter_delete(started_cluster, name, engine): for a table that has TTL delete expression defined but no explicit storage policy assigned. """ - drop_table([node1], name) node1.query( """ @@ -426,7 +415,6 @@ def test_ttl_alter_delete(started_cluster, name, engine): def test_ttl_empty_parts(started_cluster): - drop_table([node1, node2], "test_ttl_empty_parts") for node in [node1, node2]: node.query( """ @@ -519,65 +507,59 @@ def test_ttl_empty_parts(started_cluster): [(node1, node2, 0), (node3, node4, 1), (node5, node6, 2)], ) def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): - drop_table([node_left, node_right], "test_ttl_delete") - drop_table([node_left, node_right], "test_ttl_group_by") - drop_table([node_left, node_right], "test_ttl_where") - + table = f"test_ttl_compatibility_{node_left.name}_{node_right.name}_{num_run}" for node in [node_left, node_right]: node.query( """ - CREATE TABLE test_ttl_delete(date DateTime, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_delete_{suff}', '{replica}') + CREATE TABLE {table}_delete(date DateTime, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}_delete', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 3 SECOND - SETTINGS max_number_of_merges_with_ttl_in_pool=100, max_replicated_merges_with_ttl_in_queue=100 """.format( - suff=num_run, replica=node.name + table=table, replica=node.name ) ) node.query( """ - CREATE TABLE test_ttl_group_by(date DateTime, id UInt32, val UInt64) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_group_by_{suff}', '{replica}') + CREATE TABLE {table}_group_by(date DateTime, id UInt32, val UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}_group_by', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 3 SECOND GROUP BY id SET val = sum(val) - SETTINGS max_number_of_merges_with_ttl_in_pool=100, max_replicated_merges_with_ttl_in_queue=100 """.format( - suff=num_run, replica=node.name + table=table, replica=node.name ) ) node.query( """ - CREATE TABLE test_ttl_where(date DateTime, id UInt32) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_where_{suff}', '{replica}') + CREATE TABLE {table}_where(date DateTime, id UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table}_where', '{replica}') ORDER BY id PARTITION BY toDayOfMonth(date) TTL date + INTERVAL 3 SECOND DELETE WHERE id % 2 = 1 - SETTINGS max_number_of_merges_with_ttl_in_pool=100, max_replicated_merges_with_ttl_in_queue=100 """.format( - suff=num_run, replica=node.name + table=table, replica=node.name ) ) - node_left.query("INSERT INTO test_ttl_delete VALUES (now(), 1)") + node_left.query(f"INSERT INTO {table}_delete VALUES (now(), 1)") node_left.query( - "INSERT INTO test_ttl_delete VALUES (toDateTime('2100-10-11 10:00:00'), 2)" + f"INSERT INTO {table}_delete VALUES (toDateTime('2100-10-11 10:00:00'), 2)" ) - node_right.query("INSERT INTO test_ttl_delete VALUES (now(), 3)") + node_right.query(f"INSERT INTO {table}_delete VALUES (now(), 3)") node_right.query( - "INSERT INTO test_ttl_delete VALUES (toDateTime('2100-10-11 10:00:00'), 4)" + f"INSERT INTO {table}_delete VALUES (toDateTime('2100-10-11 10:00:00'), 4)" ) - node_left.query("INSERT INTO test_ttl_group_by VALUES (now(), 0, 1)") - node_left.query("INSERT INTO test_ttl_group_by VALUES (now(), 0, 2)") - node_right.query("INSERT INTO test_ttl_group_by VALUES (now(), 0, 3)") - node_right.query("INSERT INTO test_ttl_group_by VALUES (now(), 0, 4)") + node_left.query(f"INSERT INTO {table}_group_by VALUES (now(), 0, 1)") + node_left.query(f"INSERT INTO {table}_group_by VALUES (now(), 0, 2)") + node_right.query(f"INSERT INTO {table}_group_by VALUES (now(), 0, 3)") + node_right.query(f"INSERT INTO {table}_group_by VALUES (now(), 0, 4)") - node_left.query("INSERT INTO test_ttl_where VALUES (now(), 1)") - node_left.query("INSERT INTO test_ttl_where VALUES (now(), 2)") - node_right.query("INSERT INTO test_ttl_where VALUES (now(), 3)") - node_right.query("INSERT INTO test_ttl_where VALUES (now(), 4)") + node_left.query(f"INSERT INTO {table}_where VALUES (now(), 1)") + node_left.query(f"INSERT INTO {table}_where VALUES (now(), 2)") + node_right.query(f"INSERT INTO {table}_where VALUES (now(), 3)") + node_right.query(f"INSERT INTO {table}_where VALUES (now(), 4)") if node_left.with_installed_binary: node_left.restart_with_latest_version() @@ -588,13 +570,13 @@ def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): time.sleep(5) # Wait for TTL # after restart table can be in readonly mode - exec_query_with_retry(node_right, "OPTIMIZE TABLE test_ttl_delete FINAL") - node_right.query("OPTIMIZE TABLE test_ttl_group_by FINAL") - node_right.query("OPTIMIZE TABLE test_ttl_where FINAL") + exec_query_with_retry(node_right, f"OPTIMIZE TABLE {table}_delete FINAL") + node_right.query(f"OPTIMIZE TABLE {table}_group_by FINAL") + node_right.query(f"OPTIMIZE TABLE {table}_where FINAL") - exec_query_with_retry(node_left, "OPTIMIZE TABLE test_ttl_delete FINAL") - node_left.query("OPTIMIZE TABLE test_ttl_group_by FINAL", timeout=20) - node_left.query("OPTIMIZE TABLE test_ttl_where FINAL", timeout=20) + exec_query_with_retry(node_left, f"OPTIMIZE TABLE {table}_delete FINAL") + node_left.query(f"OPTIMIZE TABLE {table}_group_by FINAL", timeout=20) + node_left.query(f"OPTIMIZE TABLE {table}_where FINAL", timeout=20) # After OPTIMIZE TABLE, it is not guaranteed that everything is merged. # Possible scenario (for test_ttl_group_by): @@ -605,19 +587,19 @@ def test_ttl_compatibility(started_cluster, node_left, node_right, num_run): # 4. OPTIMIZE FINAL does nothing, cause there is an entry for 0_3 # # So, let's also sync replicas for node_right (for now). - exec_query_with_retry(node_right, "SYSTEM SYNC REPLICA test_ttl_delete") - node_right.query("SYSTEM SYNC REPLICA test_ttl_group_by", timeout=20) - node_right.query("SYSTEM SYNC REPLICA test_ttl_where", timeout=20) + exec_query_with_retry(node_right, f"SYSTEM SYNC REPLICA {table}_delete") + node_right.query(f"SYSTEM SYNC REPLICA {table}_group_by", timeout=20) + node_right.query(f"SYSTEM SYNC REPLICA {table}_where", timeout=20) - exec_query_with_retry(node_left, "SYSTEM SYNC REPLICA test_ttl_delete") - node_left.query("SYSTEM SYNC REPLICA test_ttl_group_by", timeout=20) - node_left.query("SYSTEM SYNC REPLICA test_ttl_where", timeout=20) + exec_query_with_retry(node_left, f"SYSTEM SYNC REPLICA {table}_delete") + node_left.query(f"SYSTEM SYNC REPLICA {table}_group_by", timeout=20) + node_left.query(f"SYSTEM SYNC REPLICA {table}_where", timeout=20) - assert node_left.query("SELECT id FROM test_ttl_delete ORDER BY id") == "2\n4\n" - assert node_right.query("SELECT id FROM test_ttl_delete ORDER BY id") == "2\n4\n" + assert node_left.query(f"SELECT id FROM {table}_delete ORDER BY id") == "2\n4\n" + assert node_right.query(f"SELECT id FROM {table}_delete ORDER BY id") == "2\n4\n" - assert node_left.query("SELECT val FROM test_ttl_group_by ORDER BY id") == "10\n" - assert node_right.query("SELECT val FROM test_ttl_group_by ORDER BY id") == "10\n" + assert node_left.query(f"SELECT val FROM {table}_group_by ORDER BY id") == "10\n" + assert node_right.query(f"SELECT val FROM {table}_group_by ORDER BY id") == "10\n" - assert node_left.query("SELECT id FROM test_ttl_where ORDER BY id") == "2\n4\n" - assert node_right.query("SELECT id FROM test_ttl_where ORDER BY id") == "2\n4\n" + assert node_left.query(f"SELECT id FROM {table}_where ORDER BY id") == "2\n4\n" + assert node_right.query(f"SELECT id FROM {table}_where ORDER BY id") == "2\n4\n" From f2f84fe6b7f49ae3bad1ed6f8c19c608f73d53d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 31 Oct 2023 18:24:33 +0100 Subject: [PATCH 023/274] Adapt version changes --- .../test_vertical_merges_from_compact_parts.py | 2 +- tests/integration/test_default_compression_codec/test.py | 2 +- .../integration/test_version_update_after_mutation/test.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py index e0a9b5ebad61..9c9d1a4d3121 100644 --- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py +++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py @@ -7,7 +7,7 @@ node_old = cluster.add_instance( "node1", image="clickhouse/clickhouse-server", - tag="22.6", + tag="22.8", stay_alive=True, with_installed_binary=True, with_zookeeper=True, diff --git a/tests/integration/test_default_compression_codec/test.py b/tests/integration/test_default_compression_codec/test.py index db116ff42f3b..ffe22c62325e 100644 --- a/tests/integration/test_default_compression_codec/test.py +++ b/tests/integration/test_default_compression_codec/test.py @@ -27,7 +27,7 @@ ) node3 = cluster.add_instance( "node3", - main_configs=["configs/default_compression.xml", "configs/wide_parts_only.xml"], + main_configs=["configs/default_compression.xml"], image="yandex/clickhouse-server", tag="19.16.9.37", stay_alive=True, diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index 9fb396b1c149..4e84b4c10ca7 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -72,8 +72,8 @@ def test_mutate_and_upgrade(start_cluster): node1.query("DETACH TABLE mt") # stop being leader node1.query("SYSTEM FLUSH LOGS") node2.query("SYSTEM FLUSH LOGS") - node1.restart_with_latest_version(signal=9, fix_metadata=True) - node2.restart_with_latest_version(signal=9, fix_metadata=True) + node1.restart_with_latest_version(signal=9, fix_metadata=False) + node2.restart_with_latest_version(signal=9, fix_metadata=False) # After hard restart table can be in readonly mode exec_query_with_retry( @@ -129,7 +129,7 @@ def test_upgrade_while_mutation(start_cluster): # (We could be in process of creating some system table, which will leave empty directory on restart, # so when we start moving system tables from ordinary to atomic db, it will complain about some undeleted files) node3.query("SYSTEM FLUSH LOGS") - node3.restart_with_latest_version(signal=9, fix_metadata=True) + node3.restart_with_latest_version(signal=9, fix_metadata=False) # checks for readonly exec_query_with_retry(node3, "OPTIMIZE TABLE mt1", sleep_time=5, retry_count=60) From 77507b843b66c4994ea46401d12b67f447154a39 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 1 Nov 2023 19:09:43 +0100 Subject: [PATCH 024/274] Fix build --- src/Common/ProfileEvents.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index d0589f7d2818..31a62d0ff5c0 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -415,7 +415,6 @@ The server successfully detected this situation and will download merged part fr M(FilesystemCacheEvictMicroseconds, "Filesystem cache eviction time") \ M(FilesystemCacheGetOrSetMicroseconds, "Filesystem cache getOrSet() time") \ M(FilesystemCacheGetMicroseconds, "Filesystem cache get() time") \ - M(FilesystemCacheUnusedHoldFileSegments, "Filesystem cache file segments count, which were hold, but not used (because of seek or LIMIT n, etc)") \ M(FileSegmentWaitMicroseconds, "Wait on DOWNLOADING state") \ M(FileSegmentCompleteMicroseconds, "Duration of FileSegment::complete() in filesystem cache") \ M(FileSegmentLockMicroseconds, "Lock file segment time") \ From 83689c2a04b60288cbeda25d2c57762180273c29 Mon Sep 17 00:00:00 2001 From: flynn Date: Sat, 4 Nov 2023 14:35:39 +0000 Subject: [PATCH 025/274] Support create and materialized index in the same alter query --- src/Interpreters/InterpreterAlterQuery.cpp | 36 +++++++++++-------- ..._add_index_and_materialize_index.reference | 0 .../02911_add_index_and_materialize_index.sql | 16 +++++++++ 3 files changed, 38 insertions(+), 14 deletions(-) create mode 100644 tests/queries/0_stateless/02911_add_index_and_materialize_index.reference create mode 100644 tests/queries/0_stateless/02911_add_index_and_materialize_index.sql diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index f851607000c0..c9a1bd17a469 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -128,10 +128,6 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) } else if (auto mut_command = MutationCommand::parse(command_ast)) { - if (mut_command->type == MutationCommand::MATERIALIZE_TTL && !metadata_snapshot->hasAnyTTL()) - throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot MATERIALIZE TTL as there is no TTL set for table {}", - table->getStorageID().getNameForLogs()); - if (mut_command->type == MutationCommand::UPDATE || mut_command->type == MutationCommand::DELETE) { /// TODO: add a check for result query size. @@ -162,8 +158,30 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) "to execute ALTERs of different types (replicated and non replicated) in single query"); } + if (!alter_commands.empty()) + { + auto alter_lock = table->lockForAlter(getContext()->getSettingsRef().lock_acquire_timeout); + StorageInMemoryMetadata metadata = table->getInMemoryMetadata(); + alter_commands.validate(table, getContext()); + alter_commands.prepare(metadata); + table->checkAlterIsPossible(alter_commands, getContext()); + table->alter(alter_commands, getContext(), alter_lock); + } + + /// Get newest metadata_snapshot after execute ALTER command, in order to + /// support like materialize index in the same ALTER query that creates it. + metadata_snapshot = table->getInMemoryMetadataPtr(); + if (mutation_commands.hasNonEmptyMutationCommands()) { + for (const auto & command : mutation_commands) + { + /// Check it after alter finished, so we can add TTL and materialize TTL in the same ALTER query. + if (command.type == MutationCommand::MATERIALIZE_TTL && !metadata_snapshot->hasAnyTTL()) + throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot MATERIALIZE TTL as there is no TTL set for table {}", + table->getStorageID().getNameForLogs()); + + } table->checkMutationIsPossible(mutation_commands, getContext()->getSettingsRef()); MutationsInterpreter::Settings settings(false); MutationsInterpreter(table, metadata_snapshot, mutation_commands, getContext(), settings).validate(); @@ -178,16 +196,6 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) res.pipeline = QueryPipeline(std::move(partition_commands_pipe)); } - if (!alter_commands.empty()) - { - auto alter_lock = table->lockForAlter(getContext()->getSettingsRef().lock_acquire_timeout); - StorageInMemoryMetadata metadata = table->getInMemoryMetadata(); - alter_commands.validate(table, getContext()); - alter_commands.prepare(metadata); - table->checkAlterIsPossible(alter_commands, getContext()); - table->alter(alter_commands, getContext(), alter_lock); - } - return res; } diff --git a/tests/queries/0_stateless/02911_add_index_and_materialize_index.reference b/tests/queries/0_stateless/02911_add_index_and_materialize_index.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/02911_add_index_and_materialize_index.sql b/tests/queries/0_stateless/02911_add_index_and_materialize_index.sql new file mode 100644 index 000000000000..57b144a3a8d3 --- /dev/null +++ b/tests/queries/0_stateless/02911_add_index_and_materialize_index.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS index_test; + +CREATE TABLE index_test +( + x UInt32, + y UInt32, + z UInt32 +) ENGINE = MergeTree order by x; + +ALTER TABLE index_test + ADD INDEX i_x mortonDecode(2, z).1 TYPE minmax GRANULARITY 1, + ADD INDEX i_y mortonDecode(2, z).2 TYPE minmax GRANULARITY 1, + MATERIALIZE INDEX i_x, + MATERIALIZE INDEX i_y; + +drop table index_test; From d8b44dadd5c28a16e627c4c815996af75c37036e Mon Sep 17 00:00:00 2001 From: flynn Date: Sat, 4 Nov 2023 16:16:55 +0000 Subject: [PATCH 026/274] update test --- .../0_stateless/02911_add_index_and_materialize_index.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02911_add_index_and_materialize_index.sql b/tests/queries/0_stateless/02911_add_index_and_materialize_index.sql index 57b144a3a8d3..f8785ec9a38e 100644 --- a/tests/queries/0_stateless/02911_add_index_and_materialize_index.sql +++ b/tests/queries/0_stateless/02911_add_index_and_materialize_index.sql @@ -1,3 +1,5 @@ +-- Tags: no-replicated-database + DROP TABLE IF EXISTS index_test; CREATE TABLE index_test From cc5179078c3c01bc797732edfabf852befcfaf2f Mon Sep 17 00:00:00 2001 From: flynn Date: Sun, 5 Nov 2023 02:29:15 +0000 Subject: [PATCH 027/274] remove unused code Fix --- src/Interpreters/InterpreterAlterQuery.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index c9a1bd17a469..54b4334eda9d 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -105,7 +105,6 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) if (table->isStaticStorage()) throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is read-only"); auto table_lock = table->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); - auto metadata_snapshot = table->getInMemoryMetadataPtr(); /// Add default database to table identifiers that we can encounter in e.g. default expressions, mutation expression, etc. AddDefaultDatabaseVisitor visitor(getContext(), table_id.getDatabaseName()); @@ -170,7 +169,7 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) /// Get newest metadata_snapshot after execute ALTER command, in order to /// support like materialize index in the same ALTER query that creates it. - metadata_snapshot = table->getInMemoryMetadataPtr(); + auto metadata_snapshot = table->getInMemoryMetadataPtr(); if (mutation_commands.hasNonEmptyMutationCommands()) { From e4400ec24c1a7949638f80642f8510978dc6bbea Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 7 Nov 2023 13:33:02 -0300 Subject: [PATCH 028/274] add transition from reading key to reading quoted key when double quotes are found --- src/Functions/keyvaluepair/impl/StateHandlerImpl.h | 5 +++++ ..._extract_key_value_pairs_multiple_input.reference | 12 ++++++++++++ .../02499_extract_key_value_pairs_multiple_input.sql | 12 ++++++++++++ 3 files changed, 29 insertions(+) diff --git a/src/Functions/keyvaluepair/impl/StateHandlerImpl.h b/src/Functions/keyvaluepair/impl/StateHandlerImpl.h index 7fc3ba548333..b4fd91ec3c99 100644 --- a/src/Functions/keyvaluepair/impl/StateHandlerImpl.h +++ b/src/Functions/keyvaluepair/impl/StateHandlerImpl.h @@ -106,6 +106,11 @@ class StateHandlerImpl : public StateHandler { return {next_pos, State::WAITING_KEY}; } + else if (isQuotingCharacter(*p)) + { + // +1 to skip quoting character + return {next_pos, State::READING_QUOTED_KEY}; + } pos = next_pos; } diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference index d0cf9ff680bd..f646583bbd3b 100644 --- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference +++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.reference @@ -345,6 +345,18 @@ WITH SELECT x; {'argument1':'1','argument2':'2','char':'=','char2':'=','formula':'1+2=3','result':'3','string':'foo=bar'} +-- https://github.com/ClickHouse/ClickHouse/issues/56357 +WITH + extractKeyValuePairs('{"a":"1", "b":"2"}') as s_map, + CAST( + arrayMap( + (x) -> (x, s_map[x]), arraySort(mapKeys(s_map)) + ), + 'Map(String,String)' + ) AS x +SELECT + x; +{'a':'1','b':'2'} -- check str_to_map alias (it is case-insensitive) WITH sTr_tO_mAp('name:neymar, age:31 team:psg,nationality:brazil') AS s_map, diff --git a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql index 804ff4ce8803..9277ba6d7ec1 100644 --- a/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql +++ b/tests/queries/0_stateless/02499_extract_key_value_pairs_multiple_input.sql @@ -481,6 +481,18 @@ WITH SELECT x; +-- https://github.com/ClickHouse/ClickHouse/issues/56357 +WITH + extractKeyValuePairs('{"a":"1", "b":"2"}') as s_map, + CAST( + arrayMap( + (x) -> (x, s_map[x]), arraySort(mapKeys(s_map)) + ), + 'Map(String,String)' + ) AS x +SELECT + x; + -- check str_to_map alias (it is case-insensitive) WITH sTr_tO_mAp('name:neymar, age:31 team:psg,nationality:brazil') AS s_map, From 28ca29fda24350a954dc0747a822161e74992a44 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 7 Nov 2023 14:31:34 -0300 Subject: [PATCH 029/274] remove stale comment --- src/Functions/keyvaluepair/impl/StateHandlerImpl.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/keyvaluepair/impl/StateHandlerImpl.h b/src/Functions/keyvaluepair/impl/StateHandlerImpl.h index b4fd91ec3c99..687d8d95d42a 100644 --- a/src/Functions/keyvaluepair/impl/StateHandlerImpl.h +++ b/src/Functions/keyvaluepair/impl/StateHandlerImpl.h @@ -108,7 +108,6 @@ class StateHandlerImpl : public StateHandler } else if (isQuotingCharacter(*p)) { - // +1 to skip quoting character return {next_pos, State::READING_QUOTED_KEY}; } From f619f73f284039c4e3bfebeda2228dd5a799d6e3 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 29 Aug 2023 11:53:32 +0000 Subject: [PATCH 030/274] Fix incorrect header in grace hash join and filter pushdown --- src/QueryPipeline/QueryPipelineBuilder.cpp | 12 +++++------- .../02861_filter_pushdown_const_bug.reference | 2 ++ .../0_stateless/02861_filter_pushdown_const_bug.sql | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/QueryPipeline/QueryPipelineBuilder.cpp b/src/QueryPipeline/QueryPipelineBuilder.cpp index f9726339872d..f13d1c56d7f8 100644 --- a/src/QueryPipeline/QueryPipelineBuilder.cpp +++ b/src/QueryPipeline/QueryPipelineBuilder.cpp @@ -483,8 +483,6 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesRightLe Block left_header = left->getHeader(); - Block joined_header = JoiningTransform::transformHeader(left_header, join); - for (size_t i = 0; i < num_streams; ++i) { auto joining = std::make_shared( @@ -496,9 +494,9 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesRightLe { // Process delayed joined blocks when all JoiningTransform are finished. auto delayed = std::make_shared( - joined_header, - [left_header, joined_header, max_block_size, join]() - { return join->getNonJoinedBlocks(left_header, joined_header, max_block_size); }); + output_header, + [left_header, output_header, max_block_size, join]() + { return join->getNonJoinedBlocks(left_header, output_header, max_block_size); }); if (delayed->getInputs().size() != 1 || delayed->getOutputs().size() != 1) throw Exception(ErrorCodes::LOGICAL_ERROR, "DelayedJoinedBlocksWorkerTransform should have one input and one output"); @@ -533,7 +531,7 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesRightLe for (size_t i = 1; i < joined_output_ports.size(); i += 2) delayed_ports_numbers.push_back(i); - auto delayed_processor = std::make_shared(joined_header, 2 * num_streams, delayed_ports_numbers); + auto delayed_processor = std::make_shared(output_header, 2 * num_streams, delayed_ports_numbers); if (collected_processors) collected_processors->emplace_back(delayed_processor); left->pipe.processors->emplace_back(delayed_processor); @@ -545,7 +543,7 @@ std::unique_ptr QueryPipelineBuilder::joinPipelinesRightLe left->pipe.output_ports.clear(); for (OutputPort & port : delayed_processor->getOutputs()) left->pipe.output_ports.push_back(&port); - left->pipe.header = joined_header; + left->pipe.header = output_header; left->resize(num_streams); } diff --git a/tests/queries/0_stateless/02861_filter_pushdown_const_bug.reference b/tests/queries/0_stateless/02861_filter_pushdown_const_bug.reference index 428ba88bff04..df8198bc8568 100644 --- a/tests/queries/0_stateless/02861_filter_pushdown_const_bug.reference +++ b/tests/queries/0_stateless/02861_filter_pushdown_const_bug.reference @@ -6,3 +6,5 @@ 1 1 1 1 +1 1 +1 1 diff --git a/tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql b/tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql index a5ddf830d480..a299e50984f9 100644 --- a/tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql +++ b/tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql @@ -15,4 +15,8 @@ SELECT key FROM ( SELECT key FROM t1 ) AS t1 JOIN ( SELECT key FROM t1 ) AS t2 O SELECT key FROM ( SELECT 1 AS key ) AS t1 JOIN ( SELECT 1 AS key ) AS t2 ON t1.key = t2.key WHERE key; SELECT * FROM ( SELECT 1 AS key GROUP BY NULL ) AS t1 INNER JOIN (SELECT 1 AS key) AS t2 ON t1.key = t2.key WHERE t1.key ORDER BY key; +SET join_algorithm = 'grace_hash'; + +SELECT * FROM (SELECT key AS a FROM t1 ) t1 INNER JOIN (SELECT key AS c FROM t1 ) t2 ON c = a WHERE a; + DROP TABLE IF EXISTS t1; From 86ba6ad1e85c5c6651ef4d6d2f83567e50ab6d69 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 8 Nov 2023 10:22:44 +0000 Subject: [PATCH 031/274] Local backup and restore --- .../BackupCoordinationKeeperMapTables.cpp | 23 + .../BackupCoordinationKeeperMapTables.h | 23 + src/Backups/BackupCoordinationLocal.cpp | 12 + src/Backups/BackupCoordinationLocal.h | 14 + src/Backups/BackupCoordinationRemote.cpp | 13 + src/Backups/BackupCoordinationRemote.h | 13 + src/Backups/BackupsWorker.cpp | 11 +- src/Backups/IBackupCoordination.h | 6 + src/Backups/IRestoreCoordination.h | 4 + src/Backups/RestoreCoordinationLocal.cpp | 6 + src/Backups/RestoreCoordinationLocal.h | 5 + src/Backups/RestoreCoordinationRemote.cpp | 5 + src/Backups/RestoreCoordinationRemote.h | 4 + src/Backups/WithRetries.cpp | 20 + src/Backups/WithRetries.h | 5 + src/Core/Settings.h | 1 + src/Storages/StorageKeeperMap.cpp | 420 +++++++++++++++--- src/Storages/StorageKeeperMap.h | 22 +- 18 files changed, 527 insertions(+), 80 deletions(-) create mode 100644 src/Backups/BackupCoordinationKeeperMapTables.cpp create mode 100644 src/Backups/BackupCoordinationKeeperMapTables.h diff --git a/src/Backups/BackupCoordinationKeeperMapTables.cpp b/src/Backups/BackupCoordinationKeeperMapTables.cpp new file mode 100644 index 000000000000..50561560dd55 --- /dev/null +++ b/src/Backups/BackupCoordinationKeeperMapTables.cpp @@ -0,0 +1,23 @@ +#include + +namespace DB +{ + +void BackupCoordinationKeeperMapTables::addTable(const std::string & table_zookeeper_root_path, const std::string & table_id, const std::string & data_path_in_backup) +{ + if (auto it = tables_with_info.find(table_zookeeper_root_path); it != tables_with_info.end()) + { + if (table_id > it->second.table_id) + it->second = KeeperMapTableInfo{table_id, data_path_in_backup}; + return; + } + + tables_with_info.emplace(table_zookeeper_root_path, KeeperMapTableInfo{table_id, data_path_in_backup}); +} + +std::string BackupCoordinationKeeperMapTables::getDataPath(const std::string & table_zookeeper_root_path) const +{ + return tables_with_info.at(table_zookeeper_root_path).data_path_in_backup; +} + +} diff --git a/src/Backups/BackupCoordinationKeeperMapTables.h b/src/Backups/BackupCoordinationKeeperMapTables.h new file mode 100644 index 000000000000..28894bb9c6ef --- /dev/null +++ b/src/Backups/BackupCoordinationKeeperMapTables.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include + +namespace DB +{ + +struct BackupCoordinationKeeperMapTables +{ + void addTable(const std::string & table_zookeeper_root_path, const std::string & table_id, const std::string & data_path_in_backup); + std::string getDataPath(const std::string & table_zookeeper_root_path) const; +private: + struct KeeperMapTableInfo + { + std::string table_id; + std::string data_path_in_backup; + }; + + std::unordered_map tables_with_info; +}; + +} diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp index 27e0f173cf32..fb91bae2303b 100644 --- a/src/Backups/BackupCoordinationLocal.cpp +++ b/src/Backups/BackupCoordinationLocal.cpp @@ -97,6 +97,18 @@ Strings BackupCoordinationLocal::getReplicatedSQLObjectsDirs(const String & load return replicated_sql_objects.getDirectories(loader_zk_path, object_type, ""); } +void BackupCoordinationLocal::addKeeperMapTable(const String & table_zookeeper_root_path, const String & table_id, const String & data_path_in_backup) +{ + std::lock_guard lock(keeper_map_tables_mutex); + keeper_map_tables.addTable(table_zookeeper_root_path, table_id, data_path_in_backup); +} + +String BackupCoordinationLocal::getKeeperMapDataPath(const String & table_zookeeper_root_path) const +{ + std::lock_guard lock(keeper_map_tables_mutex); + return keeper_map_tables.getDataPath(table_zookeeper_root_path); +} + void BackupCoordinationLocal::addFileInfos(BackupFileInfos && file_infos_) { diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h index 60fcc0147206..1fecf30c51cf 100644 --- a/src/Backups/BackupCoordinationLocal.h +++ b/src/Backups/BackupCoordinationLocal.h @@ -6,6 +6,8 @@ #include #include #include +#include "Backups/BackupCoordinationKeeperMapTables.h" +#include #include #include @@ -44,6 +46,9 @@ class BackupCoordinationLocal : public IBackupCoordination void addReplicatedSQLObjectsDir(const String & loader_zk_path, UserDefinedSQLObjectType object_type, const String & dir_path) override; Strings getReplicatedSQLObjectsDirs(const String & loader_zk_path, UserDefinedSQLObjectType object_type) const override; + void addKeeperMapTable(const String & table_zookeeper_root_path, const String & table_id, const String & data_path_in_backup) override; + String getKeeperMapDataPath(const String & table_zookeeper_root_path) const override; + void addFileInfos(BackupFileInfos && file_infos) override; BackupFileInfos getFileInfos() const override; BackupFileInfos getFileInfosForAllHosts() const override; @@ -58,13 +63,22 @@ class BackupCoordinationLocal : public IBackupCoordination BackupCoordinationReplicatedAccess TSA_GUARDED_BY(replicated_access_mutex) replicated_access; BackupCoordinationReplicatedSQLObjects TSA_GUARDED_BY(replicated_sql_objects_mutex) replicated_sql_objects; BackupCoordinationFileInfos TSA_GUARDED_BY(file_infos_mutex) file_infos; + BackupCoordinationKeeperMapTables keeper_map_tables TSA_GUARDED_BY(keeper_map_tables_mutex); std::unordered_set TSA_GUARDED_BY(writing_files_mutex) writing_files; + + struct KeeperMapTableInfo + { + String table_id; + String data_path_in_backup; + }; + mutable std::mutex replicated_tables_mutex; mutable std::mutex replicated_access_mutex; mutable std::mutex replicated_sql_objects_mutex; mutable std::mutex file_infos_mutex; mutable std::mutex writing_files_mutex; + mutable std::mutex keeper_map_tables_mutex; }; } diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp index e5fcbf267817..72fc25090896 100644 --- a/src/Backups/BackupCoordinationRemote.cpp +++ b/src/Backups/BackupCoordinationRemote.cpp @@ -666,6 +666,19 @@ void BackupCoordinationRemote::prepareReplicatedSQLObjects() const replicated_sql_objects->addDirectory(std::move(directory)); } +void BackupCoordinationRemote::addKeeperMapTable(const String & table_zookeeper_root_path, const String & table_id, const String & data_path_in_backup) +{ + std::lock_guard lock(keeper_map_tables_mutex); + keeper_map_tables.addTable(table_zookeeper_root_path, table_id, data_path_in_backup); +} + +String BackupCoordinationRemote::getKeeperMapDataPath(const String & table_zookeeper_root_path) const +{ + std::lock_guard lock(keeper_map_tables_mutex); + return keeper_map_tables.getDataPath(table_zookeeper_root_path); +} + + void BackupCoordinationRemote::addFileInfos(BackupFileInfos && file_infos_) { { diff --git a/src/Backups/BackupCoordinationRemote.h b/src/Backups/BackupCoordinationRemote.h index c1c7a40fc442..28c24c574a65 100644 --- a/src/Backups/BackupCoordinationRemote.h +++ b/src/Backups/BackupCoordinationRemote.h @@ -7,6 +7,7 @@ #include #include #include +#include "Backups/BackupCoordinationKeeperMapTables.h" namespace DB @@ -63,6 +64,9 @@ class BackupCoordinationRemote : public IBackupCoordination void addReplicatedSQLObjectsDir(const String & loader_zk_path, UserDefinedSQLObjectType object_type, const String & dir_path) override; Strings getReplicatedSQLObjectsDirs(const String & loader_zk_path, UserDefinedSQLObjectType object_type) const override; + void addKeeperMapTable(const String & table_zookeeper_root_path, const String & table_id, const String & data_path_in_backup) override; + String getKeeperMapDataPath(const String & table_zookeeper_root_path) const override; + void addFileInfos(BackupFileInfos && file_infos) override; BackupFileInfos getFileInfos() const override; BackupFileInfos getFileInfosForAllHosts() const override; @@ -108,12 +112,21 @@ class BackupCoordinationRemote : public IBackupCoordination mutable std::optional TSA_GUARDED_BY(file_infos_mutex) file_infos; std::unordered_set TSA_GUARDED_BY(writing_files_mutex) writing_files; + struct KeeperMapTableInfo + { + String table_id; + String data_path_in_backup; + }; + + mutable BackupCoordinationKeeperMapTables keeper_map_tables TSA_GUARDED_BY(keeper_map_tables_mutex); + mutable std::mutex zookeeper_mutex; mutable std::mutex replicated_tables_mutex; mutable std::mutex replicated_access_mutex; mutable std::mutex replicated_sql_objects_mutex; mutable std::mutex file_infos_mutex; mutable std::mutex writing_files_mutex; + mutable std::mutex keeper_map_tables_mutex; }; } diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index da814dcbc086..b19135c5cbad 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -58,16 +58,7 @@ namespace auto get_zookeeper = [global_context = context->getGlobalContext()] { return global_context->getZooKeeper(); }; - BackupCoordinationRemote::BackupKeeperSettings keeper_settings - { - .keeper_max_retries = context->getSettingsRef().backup_restore_keeper_max_retries, - .keeper_retry_initial_backoff_ms = context->getSettingsRef().backup_restore_keeper_retry_initial_backoff_ms, - .keeper_retry_max_backoff_ms = context->getSettingsRef().backup_restore_keeper_retry_max_backoff_ms, - .batch_size_for_keeper_multiread = context->getSettingsRef().backup_restore_batch_size_for_keeper_multiread, - .keeper_fault_injection_probability = context->getSettingsRef().backup_restore_keeper_fault_injection_probability, - .keeper_fault_injection_seed = context->getSettingsRef().backup_restore_keeper_fault_injection_seed, - .keeper_value_max_size = context->getSettingsRef().backup_restore_keeper_value_max_size, - }; + BackupCoordinationRemote::BackupKeeperSettings keeper_settings = WithRetries::KeeperSettings::fromContext(context); auto all_hosts = BackupSettings::Util::filterHostIDs( backup_settings.cluster_host_ids, backup_settings.shard_num, backup_settings.replica_num); diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h index 75d9202374b0..f80b5dee8837 100644 --- a/src/Backups/IBackupCoordination.h +++ b/src/Backups/IBackupCoordination.h @@ -56,6 +56,12 @@ class IBackupCoordination /// Returns all mutations of a replicated table which are not finished for some data parts added by addReplicatedPartNames(). virtual std::vector getReplicatedMutations(const String & table_shared_id, const String & replica_name) const = 0; + /// Adds information about KeeperMap tables + virtual void addKeeperMapTable(const String & table_zookeeper_root_path, const String & table_id, const String & data_path_in_backup) = 0; + + /// KeeperMap tables use shared storage without local data so only one table should backup the data + virtual String getKeeperMapDataPath(const String & table_zookeeper_root_path) const = 0; + /// Adds a data path in backup for a replicated table. /// Multiple replicas of the replicated table call this function and then all the added paths can be returned by call of the function /// getReplicatedDataPaths(). diff --git a/src/Backups/IRestoreCoordination.h b/src/Backups/IRestoreCoordination.h index fd6f014c3263..489292cb88f2 100644 --- a/src/Backups/IRestoreCoordination.h +++ b/src/Backups/IRestoreCoordination.h @@ -41,6 +41,10 @@ class IRestoreCoordination /// The function returns false if user-defined function at a specified zk path are being already restored by another replica. virtual bool acquireReplicatedSQLObjects(const String & loader_zk_path, UserDefinedSQLObjectType object_type) = 0; + /// Sets that this table is going to restore data into Keeper for all KeeperMap tables defined on root_zk_path. + /// The function returns false if data for this specific root path is already being restored by another table. + virtual bool acquireInsertingDataForKeeperMap(const String & root_zk_path) = 0; + /// Generates a new UUID for a table. The same UUID must be used for a replicated table on each replica, /// (because otherwise the macro "{uuid}" in the ZooKeeper path will not work correctly). virtual void generateUUIDForTable(ASTCreateQuery & create_query) = 0; diff --git a/src/Backups/RestoreCoordinationLocal.cpp b/src/Backups/RestoreCoordinationLocal.cpp index 1bd2f8e2ed13..d32625c2c514 100644 --- a/src/Backups/RestoreCoordinationLocal.cpp +++ b/src/Backups/RestoreCoordinationLocal.cpp @@ -52,6 +52,12 @@ bool RestoreCoordinationLocal::acquireReplicatedSQLObjects(const String &, UserD return true; } +bool RestoreCoordinationLocal::acquireInsertingDataForKeeperMap(const String & root_zk_path) +{ + std::lock_guard lock{mutex}; + return acquired_data_in_keeper_map_tables.emplace(root_zk_path).second; +} + void RestoreCoordinationLocal::generateUUIDForTable(ASTCreateQuery & create_query) { String query_str = serializeAST(create_query); diff --git a/src/Backups/RestoreCoordinationLocal.h b/src/Backups/RestoreCoordinationLocal.h index 339b754fca59..93fbdb79d9f5 100644 --- a/src/Backups/RestoreCoordinationLocal.h +++ b/src/Backups/RestoreCoordinationLocal.h @@ -40,6 +40,10 @@ class RestoreCoordinationLocal : public IRestoreCoordination /// The function returns false if user-defined function at a specified zk path are being already restored by another replica. bool acquireReplicatedSQLObjects(const String & loader_zk_path, UserDefinedSQLObjectType object_type) override; + /// Sets that this table is going to restore data into Keeper for all KeeperMap tables defined on root_zk_path. + /// The function returns false if data for this specific root path is already being restored by another table. + bool acquireInsertingDataForKeeperMap(const String & root_zk_path) override; + /// Generates a new UUID for a table. The same UUID must be used for a replicated table on each replica, /// (because otherwise the macro "{uuid}" in the ZooKeeper path will not work correctly). void generateUUIDForTable(ASTCreateQuery & create_query) override; @@ -52,6 +56,7 @@ class RestoreCoordinationLocal : public IRestoreCoordination std::set> acquired_tables_in_replicated_databases; std::unordered_set acquired_data_in_replicated_tables; std::unordered_map create_query_uuids; + std::unordered_set acquired_data_in_keeper_map_tables; mutable std::mutex mutex; }; diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp index c71466ad8f4d..7e059b8d9cce 100644 --- a/src/Backups/RestoreCoordinationRemote.cpp +++ b/src/Backups/RestoreCoordinationRemote.cpp @@ -234,6 +234,11 @@ bool RestoreCoordinationRemote::acquireReplicatedSQLObjects(const String & loade return result; } +bool RestoreCoordinationRemote::acquireInsertingDataForKeeperMap(const String & /*root_zk_path*/) +{ + return true; +} + void RestoreCoordinationRemote::generateUUIDForTable(ASTCreateQuery & create_query) { String query_str = serializeAST(create_query); diff --git a/src/Backups/RestoreCoordinationRemote.h b/src/Backups/RestoreCoordinationRemote.h index 22d0c0ed6dfe..7d3ae4ceec91 100644 --- a/src/Backups/RestoreCoordinationRemote.h +++ b/src/Backups/RestoreCoordinationRemote.h @@ -46,6 +46,10 @@ class RestoreCoordinationRemote : public IRestoreCoordination /// The function returns false if user-defined function at a specified zk path are being already restored by another replica. bool acquireReplicatedSQLObjects(const String & loader_zk_path, UserDefinedSQLObjectType object_type) override; + /// Sets that this table is going to restore data into Keeper for all KeeperMap tables defined on root_zk_path. + /// The function returns false if data for this specific root path is already being restored by another table. + bool acquireInsertingDataForKeeperMap(const String & root_zk_path) override; + /// Generates a new UUID for a table. The same UUID must be used for a replicated table on each replica, /// (because otherwise the macro "{uuid}" in the ZooKeeper path will not work correctly). void generateUUIDForTable(ASTCreateQuery & create_query) override; diff --git a/src/Backups/WithRetries.cpp b/src/Backups/WithRetries.cpp index 0893c65d8fd1..d1612a7da4fa 100644 --- a/src/Backups/WithRetries.cpp +++ b/src/Backups/WithRetries.cpp @@ -5,6 +5,21 @@ namespace DB { +WithRetries::KeeperSettings WithRetries::KeeperSettings::fromContext(ContextPtr context) +{ + return + { + .keeper_max_retries = context->getSettingsRef().backup_restore_keeper_max_retries, + .keeper_retry_initial_backoff_ms = context->getSettingsRef().backup_restore_keeper_retry_initial_backoff_ms, + .keeper_retry_max_backoff_ms = context->getSettingsRef().backup_restore_keeper_retry_max_backoff_ms, + .batch_size_for_keeper_multiread = context->getSettingsRef().backup_restore_batch_size_for_keeper_multiread, + .keeper_fault_injection_probability = context->getSettingsRef().backup_restore_keeper_fault_injection_probability, + .keeper_fault_injection_seed = context->getSettingsRef().backup_restore_keeper_fault_injection_seed, + .keeper_value_max_size = context->getSettingsRef().backup_restore_keeper_value_max_size, + .batch_size_for_keeper_multi = context->getSettingsRef().backup_restore_batch_size_for_keeper_multi, + }; +} + WithRetries::WithRetries(Poco::Logger * log_, zkutil::GetZooKeeper get_zookeeper_, const KeeperSettings & settings_, RenewerCallback callback_) : log(log_) , get_zookeeper(get_zookeeper_) @@ -42,6 +57,11 @@ void WithRetries::renewZooKeeper(FaultyKeeper my_faulty_zookeeper) const } } +const WithRetries::KeeperSettings & WithRetries::getKeeperSettings() const +{ + return settings; +} + WithRetries::FaultyKeeper WithRetries::getFaultyZooKeeper() const { /// We need to create new instance of ZooKeeperWithFaultInjection each time a copy a pointer to ZooKeeper client there diff --git a/src/Backups/WithRetries.h b/src/Backups/WithRetries.h index 3955682be944..8f4a730e6a1b 100644 --- a/src/Backups/WithRetries.h +++ b/src/Backups/WithRetries.h @@ -26,6 +26,9 @@ class WithRetries Float64 keeper_fault_injection_probability{0}; UInt64 keeper_fault_injection_seed{42}; UInt64 keeper_value_max_size{1048576}; + UInt64 batch_size_for_keeper_multi{1000}; + + static KeeperSettings fromContext(ContextPtr context); }; /// For simplicity a separate ZooKeeperRetriesInfo and a faulty [Zoo]Keeper client @@ -53,6 +56,8 @@ class WithRetries /// Used to re-establish new connection inside a retry loop. void renewZooKeeper(FaultyKeeper my_faulty_zookeeper) const; + + const KeeperSettings & getKeeperSettings() const; private: /// This will provide a special wrapper which is useful for testing FaultyKeeper getFaultyZooKeeper() const; diff --git a/src/Core/Settings.h b/src/Core/Settings.h index aa5c8569be6a..bab9005a22c4 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -465,6 +465,7 @@ class IColumn; M(UInt64, backup_restore_keeper_fault_injection_seed, 0, "0 - random seed, otherwise the setting value", 0) \ M(UInt64, backup_restore_keeper_value_max_size, 1048576, "Maximum size of data of a [Zoo]Keeper's node during backup", 0) \ M(UInt64, backup_restore_batch_size_for_keeper_multiread, 10000, "Maximum size of batch for multiread request to [Zoo]Keeper during backup or restore", 0) \ + M(UInt64, backup_restore_batch_size_for_keeper_multi, 1000, "Maximum size of batch for multi request to [Zoo]Keeper during backup or restore", 0) \ M(UInt64, max_backup_bandwidth, 0, "The maximum read speed in bytes per second for particular backup on server. Zero means unlimited.", 0) \ \ M(Bool, log_profile_events, true, "Log query performance statistics into the query_log, query_thread_log and query_views_log.", 0) \ diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index f98728c012ea..33a97af53f1d 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -13,6 +14,9 @@ #include #include +#include +#include + #include #include #include @@ -38,6 +42,16 @@ #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include #include @@ -54,6 +68,7 @@ namespace ErrorCodes extern const int KEEPER_EXCEPTION; extern const int LOGICAL_ERROR; extern const int LIMIT_EXCEEDED; + extern const int CANNOT_RESTORE_TABLE; } namespace @@ -296,13 +311,13 @@ StorageKeeperMap::StorageKeeperMap( const StorageInMemoryMetadata & metadata, bool attach, std::string_view primary_key_, - const std::string & root_path_, + const std::string & zk_root_path_, UInt64 keys_limit_) : IStorage(table_id) , WithContext(context_->getGlobalContext()) - , root_path(zkutil::extractZooKeeperPath(root_path_, false)) + , zk_root_path(zkutil::extractZooKeeperPath(zk_root_path_, false)) , primary_key(primary_key_) - , zookeeper_name(zkutil::extractZooKeeperName(root_path_)) + , zookeeper_name(zkutil::extractZooKeeperName(zk_root_path_)) , keys_limit(keys_limit_) , log(&Poco::Logger::get(fmt::format("StorageKeeperMap ({})", table_id.getNameForLogs()))) { @@ -320,10 +335,10 @@ StorageKeeperMap::StorageKeeperMap( << "primary key: " << formattedAST(metadata.getPrimaryKey().expression_list_ast) << "\n"; metadata_string = out.str(); - if (root_path.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "root_path should not be empty"); - if (!root_path.starts_with('/')) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "root_path should start with '/'"); + if (zk_root_path.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "zk_root_path should not be empty"); + if (!zk_root_path.starts_with('/')) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "zk_root_path should start with '/'"); auto config_keys_limit = context_->getConfigRef().getUInt64("keeper_map_keys_limit", 0); if (config_keys_limit != 0 && (keys_limit == 0 || keys_limit > config_keys_limit)) @@ -341,20 +356,20 @@ StorageKeeperMap::StorageKeeperMap( LOG_INFO(log, "Keys limit will be set to {}", keys_limit); } - auto root_path_fs = fs::path(path_prefix) / std::string_view{root_path}.substr(1); - root_path = root_path_fs.generic_string(); + auto zk_root_path_fs = fs::path(path_prefix) / std::string_view{zk_root_path}.substr(1); + zk_root_path = zk_root_path_fs.generic_string(); - data_path = root_path_fs / "data"; + zk_data_path = zk_root_path_fs / "data"; - auto metadata_path_fs = root_path_fs / "metadata"; - metadata_path = metadata_path_fs; - tables_path = metadata_path_fs / "tables"; + auto metadata_path_fs = zk_root_path_fs / "metadata"; + zk_metadata_path = metadata_path_fs; + zk_tables_path = metadata_path_fs / "tables"; auto table_unique_id = toString(table_id.uuid) + toString(ServerUUID::get()); - table_path = fs::path(tables_path) / table_unique_id; + zk_table_path = fs::path(zk_tables_path) / table_unique_id; - dropped_path = metadata_path_fs / "dropped"; - dropped_lock_path = fs::path(dropped_path) / "lock"; + zk_dropped_path = metadata_path_fs / "dropped"; + zk_dropped_lock_path = fs::path(zk_dropped_path) / "lock"; if (attach) { @@ -364,17 +379,17 @@ StorageKeeperMap::StorageKeeperMap( auto client = getClient(); - if (root_path != "/" && !client->exists(root_path)) + if (zk_root_path != "/" && !client->exists(zk_root_path)) { - LOG_TRACE(log, "Creating root path {}", root_path); - client->createAncestors(root_path); - client->createIfNotExists(root_path, ""); + LOG_TRACE(log, "Creating root path {}", zk_root_path); + client->createAncestors(zk_root_path); + client->createIfNotExists(zk_root_path, ""); } for (size_t i = 0; i < 1000; ++i) { std::string stored_metadata_string; - auto exists = client->tryGet(metadata_path, stored_metadata_string); + auto exists = client->tryGet(zk_metadata_path, stored_metadata_string); if (exists) { @@ -384,10 +399,10 @@ StorageKeeperMap::StorageKeeperMap( throw Exception( ErrorCodes::BAD_ARGUMENTS, "Path {} is already used but the stored table definition doesn't match. Stored metadata: {}", - root_path, + zk_root_path, stored_metadata_string); - auto code = client->tryCreate(table_path, "", zkutil::CreateMode::Persistent); + auto code = client->tryCreate(zk_table_path, "", zkutil::CreateMode::Persistent); // tables_path was removed with drop if (code == Coordination::Error::ZNONODE) @@ -397,16 +412,16 @@ StorageKeeperMap::StorageKeeperMap( } else if (code != Coordination::Error::ZOK) { - throw zkutil::KeeperException(code, "Failed to create table on path {} because a table with same UUID already exists", root_path); + throw zkutil::KeeperException(code, "Failed to create table on path {} because a table with same UUID already exists", zk_root_path); } return; } - if (client->exists(dropped_path)) + if (client->exists(zk_dropped_path)) { LOG_INFO(log, "Removing leftover nodes"); - auto code = client->tryCreate(dropped_lock_path, "", zkutil::CreateMode::Ephemeral); + auto code = client->tryCreate(zk_dropped_lock_path, "", zkutil::CreateMode::Ephemeral); if (code == Coordination::Error::ZNONODE) { @@ -419,11 +434,11 @@ StorageKeeperMap::StorageKeeperMap( } else if (code != Coordination::Error::ZOK) { - throw Coordination::Exception::fromPath(code, dropped_lock_path); + throw Coordination::Exception::fromPath(code, zk_dropped_lock_path); } else { - auto metadata_drop_lock = zkutil::EphemeralNodeHolder::existing(dropped_lock_path, *client); + auto metadata_drop_lock = zkutil::EphemeralNodeHolder::existing(zk_dropped_lock_path, *client); if (!dropTable(client, metadata_drop_lock)) continue; } @@ -431,17 +446,17 @@ StorageKeeperMap::StorageKeeperMap( Coordination::Requests create_requests { - zkutil::makeCreateRequest(metadata_path, metadata_string, zkutil::CreateMode::Persistent), - zkutil::makeCreateRequest(data_path, metadata_string, zkutil::CreateMode::Persistent), - zkutil::makeCreateRequest(tables_path, "", zkutil::CreateMode::Persistent), - zkutil::makeCreateRequest(table_path, "", zkutil::CreateMode::Persistent), + zkutil::makeCreateRequest(zk_metadata_path, metadata_string, zkutil::CreateMode::Persistent), + zkutil::makeCreateRequest(zk_data_path, metadata_string, zkutil::CreateMode::Persistent), + zkutil::makeCreateRequest(zk_tables_path, "", zkutil::CreateMode::Persistent), + zkutil::makeCreateRequest(zk_table_path, "", zkutil::CreateMode::Persistent), }; Coordination::Responses create_responses; auto code = client->tryMulti(create_requests, create_responses); if (code == Coordination::Error::ZNODEEXISTS) { - LOG_INFO(log, "It looks like a table on path {} was created by another server at the same moment, will retry", root_path); + LOG_INFO(log, "It looks like a table on path {} was created by another server at the same moment, will retry", zk_root_path); continue; } else if (code != Coordination::Error::ZOK) @@ -456,7 +471,7 @@ StorageKeeperMap::StorageKeeperMap( throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot create metadata for table, because it is removed concurrently or because " - "of wrong root_path ({})", root_path); + "of wrong zk_root_path ({})", zk_root_path); } @@ -519,7 +534,7 @@ Pipe StorageKeeperMap::read( auto client = getClient(); if (all_scan) - return process_keys(std::make_shared>(client->getChildren(data_path))); + return process_keys(std::make_shared>(client->getChildren(zk_data_path))); return process_keys(std::move(filtered_keys)); } @@ -534,19 +549,19 @@ void StorageKeeperMap::truncate(const ASTPtr &, const StorageMetadataPtr &, Cont { checkTable(); auto client = getClient(); - client->tryRemoveChildrenRecursive(data_path, true); + client->tryRemoveChildrenRecursive(zk_data_path, true); } bool StorageKeeperMap::dropTable(zkutil::ZooKeeperPtr zookeeper, const zkutil::EphemeralNodeHolder::Ptr & metadata_drop_lock) { - zookeeper->removeChildrenRecursive(data_path); + zookeeper->removeChildrenRecursive(zk_data_path); bool completely_removed = false; Coordination::Requests ops; ops.emplace_back(zkutil::makeRemoveRequest(metadata_drop_lock->getPath(), -1)); - ops.emplace_back(zkutil::makeRemoveRequest(dropped_path, -1)); - ops.emplace_back(zkutil::makeRemoveRequest(data_path, -1)); - ops.emplace_back(zkutil::makeRemoveRequest(metadata_path, -1)); + ops.emplace_back(zkutil::makeRemoveRequest(zk_dropped_path, -1)); + ops.emplace_back(zkutil::makeRemoveRequest(zk_data_path, -1)); + ops.emplace_back(zkutil::makeRemoveRequest(zk_metadata_path, -1)); Coordination::Responses responses; auto code = zookeeper->tryMulti(ops, responses); @@ -557,7 +572,7 @@ bool StorageKeeperMap::dropTable(zkutil::ZooKeeperPtr zookeeper, const zkutil::E { metadata_drop_lock->setAlreadyRemoved(); completely_removed = true; - LOG_INFO(log, "Metadata ({}) and data ({}) was successfully removed from ZooKeeper", metadata_path, data_path); + LOG_INFO(log, "Metadata ({}) and data ({}) was successfully removed from ZooKeeper", zk_metadata_path, zk_data_path); break; } case ZNONODE: @@ -578,25 +593,25 @@ void StorageKeeperMap::drop() auto client = getClient(); // we allow ZNONODE in case we got hardware error on previous drop - if (auto code = client->tryRemove(table_path); code == Coordination::Error::ZNOTEMPTY) + if (auto code = client->tryRemove(zk_table_path); code == Coordination::Error::ZNOTEMPTY) { throw zkutil::KeeperException( - code, "{} contains children which shouldn't happen. Please DETACH the table if you want to delete it", table_path); + code, "{} contains children which shouldn't happen. Please DETACH the table if you want to delete it", zk_table_path); } std::vector children; // if the tables_path is not found, some other table removed it // if there are children, some other tables are still using this path as storage - if (auto code = client->tryGetChildren(tables_path, children); + if (auto code = client->tryGetChildren(zk_tables_path, children); code != Coordination::Error::ZOK || !children.empty()) return; Coordination::Requests ops; Coordination::Responses responses; - ops.emplace_back(zkutil::makeRemoveRequest(tables_path, -1)); - ops.emplace_back(zkutil::makeCreateRequest(dropped_path, "", zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest(dropped_lock_path, "", zkutil::CreateMode::Ephemeral)); + ops.emplace_back(zkutil::makeRemoveRequest(zk_tables_path, -1)); + ops.emplace_back(zkutil::makeCreateRequest(zk_dropped_path, "", zkutil::CreateMode::Persistent)); + ops.emplace_back(zkutil::makeCreateRequest(zk_dropped_lock_path, "", zkutil::CreateMode::Ephemeral)); auto code = client->tryMulti(ops, responses); @@ -613,7 +628,7 @@ void StorageKeeperMap::drop() else if (code != Coordination::Error::ZOK) zkutil::KeeperMultiException::check(code, ops, responses); - auto metadata_drop_lock = zkutil::EphemeralNodeHolder::existing(dropped_lock_path, *client); + auto metadata_drop_lock = zkutil::EphemeralNodeHolder::existing(zk_dropped_lock_path, *client); dropTable(client, metadata_drop_lock); } @@ -623,6 +638,285 @@ NamesAndTypesList StorageKeeperMap::getVirtuals() const {std::string{version_column_name}, std::make_shared()}}; } +namespace +{ + +constexpr std::string_view backup_data_filename = "data.bin"; +constexpr std::string_view backup_data_location_filename = "data_location.bin"; + +class KeeperMapBackup : public IBackupEntriesLazyBatch, boost::noncopyable +{ +public: + KeeperMapBackup( + const std::string & data_zookeeper_path_, + const std::string & data_path_in_backup, + const DiskPtr & temp_disk_, + UInt64 max_compress_block_size_, + std::shared_ptr with_retries_) + : data_zookeeper_path(data_zookeeper_path_) + , temp_disk(temp_disk_) + , max_compress_block_size(max_compress_block_size_) + , with_retries(std::move(with_retries_)) + { + file_path = fs::path(data_path_in_backup) / backup_data_filename; + } + +private: + size_t getSize() const override + { + return 1; + } + + const String & getName(size_t i) const override + { + chassert(i == 0); + return file_path; + } + + BackupEntries generate() override + { + temp_dir_owner.emplace(temp_disk); + fs::path temp_dir = temp_dir_owner->getRelativePath(); + temp_disk->createDirectories(temp_dir); + + auto data_file_path = temp_dir / fs::path{file_path}.filename(); + auto data_out_compressed = temp_disk->writeFile(data_file_path); + auto data_out = std::make_unique(*data_out_compressed, CompressionCodecFactory::instance().getDefaultCodec(), max_compress_block_size); + std::vector data_children; + { + auto holder = with_retries->createRetriesControlHolder("getKeeperMapDataKeys"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper]() + { + with_retries->renewZooKeeper(zk); + data_children = zk->getChildren(data_zookeeper_path); + }); + } + LOG_INFO(&Poco::Logger::get("BACKUPER"), "Got {} children", data_children.size()); + + const auto write_rows = [&](std::span keys) + { + std::vector keys_full_path; + keys_full_path.reserve(data_children.size()); + + for (const auto & key : data_children) + keys_full_path.push_back(data_zookeeper_path / key); + + zkutil::ZooKeeper::MultiGetResponse data; + auto holder = with_retries->createRetriesControlHolder("getKeeperMapDataKeys"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper] + { + with_retries->renewZooKeeper(zk); + data = zk->get(keys_full_path); + }); + + for (size_t i = 0; i < keys.size(); ++i) + { + auto & child_data = data[i]; + if (child_data.error != Coordination::Error::ZOK) + continue; + + writeStringBinary(keys[i], *data_out); + writeStringBinary(child_data.data, *data_out); + } + }; + + auto max_multiread_size = with_retries->getKeeperSettings().batch_size_for_keeper_multiread; + + auto keys_it = data_children.begin(); + while (keys_it != data_children.end()) + { + auto step = std::min(static_cast(std::distance(keys_it, data_children.end())), max_multiread_size); + write_rows(std::span{keys_it, keys_it + step}); + keys_it = keys_it + step; + } + + data_out->finalize(); + data_out.reset(); + data_out_compressed->finalize(); + data_out_compressed.reset(); + + return {{file_path, std::make_shared(temp_disk, data_file_path)}}; + } + + fs::path data_zookeeper_path; + DiskPtr temp_disk; + std::optional temp_dir_owner; + UInt64 max_compress_block_size; + String file_path; + std::shared_ptr with_retries; +}; +} + +void StorageKeeperMap::backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & /*partitions*/) +{ + auto table_id = toString(getStorageID().uuid); + + std::cout << "Backing up for path " << zk_root_path << " table id " << table_id << std::endl; + auto coordination = backup_entries_collector.getBackupCoordination(); + coordination->addKeeperMapTable(zk_root_path, table_id, data_path_in_backup); + + /// This task will be executed after all tables have registered their root zk path and the coordination is ready to + /// assign each path to a single table only. + auto post_collecting_task = [my_table_id = std::move(table_id), coordination, &backup_entries_collector, my_data_path_in_backup = data_path_in_backup, this] + { + auto path_with_data = coordination->getKeeperMapDataPath(zk_root_path); + if (path_with_data == my_data_path_in_backup) + { + std::cout << "Will be backing up data for path " << zk_root_path << " table id " << my_table_id << std::endl; + + auto temp_disk = backup_entries_collector.getContext()->getGlobalTemporaryVolume()->getDisk(0); + auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef().max_compress_block_size; + + auto with_retries = std::make_shared + ( + &Poco::Logger::get(fmt::format("StorageKeeperMapBackup ({})", getStorageID().getNameForLogs())), + [&] { return getClient(); }, + WithRetries::KeeperSettings::fromContext(backup_entries_collector.getContext()), + [](WithRetries::FaultyKeeper &) {} + ); + + backup_entries_collector.addBackupEntries( + std::make_shared(this->zk_data_path, path_with_data, temp_disk, max_compress_block_size, std::move(with_retries)) + ->getBackupEntries()); + return; + } + + std::cout << "Not backing up data for path " << zk_root_path << " table id " << my_table_id << " writing only path with data " << path_with_data << std::endl; + auto file_path = fs::path(my_data_path_in_backup) / backup_data_location_filename; + backup_entries_collector.addBackupEntries({{file_path, std::make_shared(path_with_data)}}); + }; + + backup_entries_collector.addPostTask(post_collecting_task); +} + +void StorageKeeperMap::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & /*partitions*/) +{ + auto backup = restorer.getBackup(); + if (!backup->hasFiles(data_path_in_backup)) + return; + + if (!restorer.getRestoreCoordination()->acquireInsertingDataForKeeperMap(zk_root_path)) + { + /// Other table is already restoring the data for this Keeper path. + /// Tables defined on the same path share data + return; + } + + auto with_retries = std::make_shared + ( + &Poco::Logger::get(fmt::format("StorageKeeperMapRestore ({})", getStorageID().getNameForLogs())), + [&] { return getClient(); }, + WithRetries::KeeperSettings::fromContext(restorer.getContext()), + [](WithRetries::FaultyKeeper &) {} + ); + + bool allow_non_empty_tables = restorer.isNonEmptyTableAllowed(); + if (!allow_non_empty_tables) + { + Coordination::Stat data_stats; + + auto holder = with_retries->createRetriesControlHolder("checkKeeperMapData"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper]() + { + with_retries->renewZooKeeper(zk); + zk->get(zk_data_path, &data_stats); + }); + + if (data_stats.numChildren != 0) + RestorerFromBackup::throwTableIsNotEmpty(getStorageID()); + } + + /// TODO: Should we backup and verify the table structure? + + //auto temp_disk = restorer.getContext()->getGlobalTemporaryVolume()->getDisk(0); + /// only 1 table should restore data for a single path + restorer.addDataRestoreTask( + [storage = std::static_pointer_cast(shared_from_this()), backup, data_path_in_backup, with_retries, allow_non_empty_tables] + { storage->restoreDataImpl(backup, data_path_in_backup, with_retries, allow_non_empty_tables); }); +} + +void StorageKeeperMap::restoreDataImpl(const BackupPtr & backup, const String & data_path_in_backup, std::shared_ptr with_retries, bool allow_non_empty_tables) +{ + auto table_id = toString(getStorageID().uuid); + + std::cout << "Restoring into " << zk_root_path << " table id " << table_id << std::endl; + + fs::path data_path_in_backup_fs = data_path_in_backup; + + String data_file = data_path_in_backup_fs / backup_data_filename; + + if (!backup->fileExists(data_file)) + { + String data_location_file = data_path_in_backup_fs / "data_location.bin"; + if (!backup->fileExists(data_location_file)) + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Files {} or {} in backup are required to restore table", data_file, data_location_file); + + auto in = backup->readFile(data_location_file); + readStringUntilEOF(data_file, *in); + + data_file = fs::path(data_file) / backup_data_filename; + + if (!backup->fileExists(data_file)) + throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File {} in backup is required to restore table", data_file); + } + + /// should we store locally in temp file? + auto in = backup->readFile(data_file); + CompressedReadBuffer compressed_in{*in}; + fs::path data_path_fs(zk_data_path); + + auto max_multi_size = with_retries->getKeeperSettings().batch_size_for_keeper_multi; + + Coordination::Requests create_requests; + const auto flush_create_requests = [&] + { + auto holder = with_retries->createRetriesControlHolder("addKeeperMapData"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper]() + { + with_retries->renewZooKeeper(zk); + zk->multi(create_requests); + }); + }; + + while (!in->eof()) + { + std::string key; + std::string value; + readStringBinary(key, compressed_in); + readStringBinary(value, compressed_in); + + /// if a table can be non empty we can have conflicting keys so we need to do single create for each row + if (allow_non_empty_tables) + { + auto holder = with_retries->createRetriesControlHolder("addKeeperMapData"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper]() + { + with_retries->renewZooKeeper(zk); + zk->tryCreate(data_path_fs / key, value, zkutil::CreateMode::Persistent); + }); + } + /// otherwise we can do multi requests + else + { + create_requests.push_back(zkutil::makeCreateRequest(data_path_fs / key, value, zkutil::CreateMode::Persistent)); + + if (create_requests.size() == max_multi_size) + { + flush_create_requests(); + create_requests.clear(); + } + } + } + + if (!create_requests.empty()) + flush_create_requests(); +} + zkutil::ZooKeeperPtr StorageKeeperMap::getClient() const { std::lock_guard lock{zookeeper_mutex}; @@ -634,7 +928,7 @@ zkutil::ZooKeeperPtr StorageKeeperMap::getClient() const else zookeeper_client = getContext()->getAuxiliaryZooKeeper(zookeeper_name); - zookeeper_client->sync(root_path); + zookeeper_client->sync(zk_root_path); } return zookeeper_client; @@ -642,12 +936,12 @@ zkutil::ZooKeeperPtr StorageKeeperMap::getClient() const const std::string & StorageKeeperMap::dataPath() const { - return data_path; + return zk_data_path; } std::string StorageKeeperMap::fullPathForKey(const std::string_view key) const { - return fs::path(data_path) / key; + return fs::path(zk_data_path) / key; } UInt64 StorageKeeperMap::keysLimit() const @@ -668,7 +962,7 @@ std::optional StorageKeeperMap::isTableValid() const auto client = getClient(); Coordination::Stat metadata_stat; - auto stored_metadata_string = client->get(metadata_path, &metadata_stat); + auto stored_metadata_string = client->get(zk_metadata_path, &metadata_stat); if (metadata_stat.numChildren == 0) { @@ -681,7 +975,7 @@ std::optional StorageKeeperMap::isTableValid() const LOG_ERROR( log, "Table definition does not match to the one stored in the path {}. Stored definition: {}", - root_path, + zk_root_path, stored_metadata_string); table_is_valid = false; return; @@ -689,9 +983,9 @@ std::optional StorageKeeperMap::isTableValid() const // validate all metadata and data nodes are present Coordination::Requests requests; - requests.push_back(zkutil::makeCheckRequest(table_path, -1)); - requests.push_back(zkutil::makeCheckRequest(data_path, -1)); - requests.push_back(zkutil::makeCheckRequest(dropped_path, -1)); + requests.push_back(zkutil::makeCheckRequest(zk_table_path, -1)); + requests.push_back(zkutil::makeCheckRequest(zk_data_path, -1)); + requests.push_back(zkutil::makeCheckRequest(zk_dropped_path, -1)); Coordination::Responses responses; client->tryMulti(requests, responses); @@ -699,19 +993,19 @@ std::optional StorageKeeperMap::isTableValid() const table_is_valid = false; if (responses[0]->error != Coordination::Error::ZOK) { - LOG_ERROR(log, "Table node ({}) is missing", table_path); + LOG_ERROR(log, "Table node ({}) is missing", zk_table_path); return; } if (responses[1]->error != Coordination::Error::ZOK) { - LOG_ERROR(log, "Data node ({}) is missing", data_path); + LOG_ERROR(log, "Data node ({}) is missing", zk_data_path); return; } if (responses[2]->error == Coordination::Error::ZOK) { - LOG_ERROR(log, "Tables with root node {} are being dropped", root_path); + LOG_ERROR(log, "Tables with root node {} are being dropped", zk_root_path); return; } @@ -962,11 +1256,11 @@ StoragePtr create(const StorageFactory::Arguments & args) throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Storage KeeperMap requires 1-3 arguments:\n" - "root_path: path in the Keeper where the values will be stored (required)\n" + "zk_root_path: path in the Keeper where the values will be stored (required)\n" "keys_limit: number of keys allowed to be stored, 0 is no limit (default: 0)"); - const auto root_path_node = evaluateConstantExpressionAsLiteral(engine_args[0], args.getLocalContext()); - auto root_path = checkAndGetLiteralArgument(root_path_node, "root_path"); + const auto zk_root_path_node = evaluateConstantExpressionAsLiteral(engine_args[0], args.getLocalContext()); + auto zk_root_path = checkAndGetLiteralArgument(zk_root_path_node, "zk_root_path"); UInt64 keys_limit = 0; if (engine_args.size() > 1) @@ -985,7 +1279,7 @@ StoragePtr create(const StorageFactory::Arguments & args) throw Exception(ErrorCodes::BAD_ARGUMENTS, "StorageKeeperMap requires one column in primary key"); return std::make_shared( - args.getContext(), args.table_id, metadata, args.query.attach, primary_key_names[0], root_path, keys_limit); + args.getContext(), args.table_id, metadata, args.query.attach, primary_key_names[0], zk_root_path, keys_limit); } } diff --git a/src/Storages/StorageKeeperMap.h b/src/Storages/StorageKeeperMap.h index ad7b719e972c..94b02ca0242b 100644 --- a/src/Storages/StorageKeeperMap.h +++ b/src/Storages/StorageKeeperMap.h @@ -10,6 +10,9 @@ #include #include +#include +#include + #include namespace DB @@ -72,6 +75,9 @@ class StorageKeeperMap final : public IStorage, public IKeyValueEntity, WithCont } bool supportsDelete() const override { return true; } + void backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional & partitions) override; + void restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional & partitions) override; + zkutil::ZooKeeperPtr getClient() const; const std::string & dataPath() const; std::string fullPathForKey(std::string_view key) const; @@ -114,18 +120,20 @@ class StorageKeeperMap final : public IStorage, public IKeyValueEntity, WithCont std::optional isTableValid() const; - std::string root_path; + void restoreDataImpl(const BackupPtr & backup, const String & data_path_in_backup, std::shared_ptr with_retries, bool allow_non_empty_tables); + + std::string zk_root_path; std::string primary_key; - std::string data_path; + std::string zk_data_path; - std::string metadata_path; + std::string zk_metadata_path; - std::string tables_path; - std::string table_path; + std::string zk_tables_path; + std::string zk_table_path; - std::string dropped_path; - std::string dropped_lock_path; + std::string zk_dropped_path; + std::string zk_dropped_lock_path; std::string zookeeper_name; From 18a5eeec38663bca5a894bb01f0cf84ad39e0b64 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 8 Nov 2023 13:14:09 +0000 Subject: [PATCH 032/274] Make on cluster backup/restore work --- .../BackupCoordinationKeeperMapTables.h | 4 +- src/Backups/BackupCoordinationLocal.h | 4 +- src/Backups/BackupCoordinationRemote.cpp | 65 ++++++++++++++++++- src/Backups/BackupCoordinationRemote.h | 5 +- src/Backups/RestoreCoordinationRemote.cpp | 39 ++++++++++- src/Storages/StorageKeeperMap.cpp | 8 +-- 6 files changed, 107 insertions(+), 18 deletions(-) diff --git a/src/Backups/BackupCoordinationKeeperMapTables.h b/src/Backups/BackupCoordinationKeeperMapTables.h index 28894bb9c6ef..a642903cfae7 100644 --- a/src/Backups/BackupCoordinationKeeperMapTables.h +++ b/src/Backups/BackupCoordinationKeeperMapTables.h @@ -10,13 +10,13 @@ struct BackupCoordinationKeeperMapTables { void addTable(const std::string & table_zookeeper_root_path, const std::string & table_id, const std::string & data_path_in_backup); std::string getDataPath(const std::string & table_zookeeper_root_path) const; -private: + struct KeeperMapTableInfo { std::string table_id; std::string data_path_in_backup; }; - +private: std::unordered_map tables_with_info; }; diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h index 1fecf30c51cf..6f8e750697c5 100644 --- a/src/Backups/BackupCoordinationLocal.h +++ b/src/Backups/BackupCoordinationLocal.h @@ -5,8 +5,8 @@ #include #include #include +#include #include -#include "Backups/BackupCoordinationKeeperMapTables.h" #include #include #include @@ -65,7 +65,7 @@ class BackupCoordinationLocal : public IBackupCoordination BackupCoordinationFileInfos TSA_GUARDED_BY(file_infos_mutex) file_infos; BackupCoordinationKeeperMapTables keeper_map_tables TSA_GUARDED_BY(keeper_map_tables_mutex); std::unordered_set TSA_GUARDED_BY(writing_files_mutex) writing_files; - + struct KeeperMapTableInfo { String table_id; diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp index 72fc25090896..309cbc8be6a1 100644 --- a/src/Backups/BackupCoordinationRemote.cpp +++ b/src/Backups/BackupCoordinationRemote.cpp @@ -230,6 +230,7 @@ void BackupCoordinationRemote::createRootNodes() ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/repl_data_paths", "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/repl_access", "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/repl_sql_objects", "", zkutil::CreateMode::Persistent)); + ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/keeper_map_tables", "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/file_infos", "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/writing_files", "", zkutil::CreateMode::Persistent)); zk->tryMulti(ops, responses); @@ -668,14 +669,72 @@ void BackupCoordinationRemote::prepareReplicatedSQLObjects() const void BackupCoordinationRemote::addKeeperMapTable(const String & table_zookeeper_root_path, const String & table_id, const String & data_path_in_backup) { - std::lock_guard lock(keeper_map_tables_mutex); - keeper_map_tables.addTable(table_zookeeper_root_path, table_id, data_path_in_backup); + { + std::lock_guard lock{keeper_map_tables_mutex}; + if (keeper_map_tables) + throw Exception(ErrorCodes::LOGICAL_ERROR, "addKeeperMapTable() must not be called after preparing"); + } + + auto holder = with_retries.createRetriesControlHolder("addKeeperMapTable"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper]() + { + with_retries.renewZooKeeper(zk); + String path = zookeeper_path + "/keeper_map_tables/" + escapeForFileName(table_id); + zk->create(path, fmt::format("{}\n{}", table_zookeeper_root_path, data_path_in_backup), zkutil::CreateMode::Persistent); + }); +} + +void BackupCoordinationRemote::prepareKeeperMapTables() const +{ + if (keeper_map_tables) + return; + + std::vector> keeper_map_table_infos; + auto holder = with_retries.createRetriesControlHolder("prepareKeeperMapTables"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper]() + { + keeper_map_table_infos.clear(); + + with_retries.renewZooKeeper(zk); + + fs::path tables_path = fs::path(zookeeper_path) / "keeper_map_tables"; + + auto tables = zk->getChildren(tables_path); + keeper_map_table_infos.reserve(tables.size()); + + for (auto & table : tables) + table = tables_path / table; + + auto tables_info = zk->get(tables); + for (size_t i = 0; i < tables_info.size(); ++i) + { + const auto & table_info = tables_info[i]; + + if (table_info.error != Coordination::Error::ZOK) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Path in Keeper {} is unexpectedly missing", tables[i]); + + std::vector data; + boost::split(data, table_info.data, [](char c) { return c == '\n'; }); + keeper_map_table_infos.emplace_back( + std::move(data[0]), + BackupCoordinationKeeperMapTables::KeeperMapTableInfo{ + .table_id = fs::path(tables[i]).filename(), .data_path_in_backup = std::move(data[1])}); + } + }); + + keeper_map_tables.emplace(); + for (const auto & [zk_root_path, table_info] : keeper_map_table_infos) + keeper_map_tables->addTable(zk_root_path, table_info.table_id, table_info.data_path_in_backup); + } String BackupCoordinationRemote::getKeeperMapDataPath(const String & table_zookeeper_root_path) const { std::lock_guard lock(keeper_map_tables_mutex); - return keeper_map_tables.getDataPath(table_zookeeper_root_path); + prepareKeeperMapTables(); + return keeper_map_tables->getDataPath(table_zookeeper_root_path); } diff --git a/src/Backups/BackupCoordinationRemote.h b/src/Backups/BackupCoordinationRemote.h index 28c24c574a65..a0a9224bf714 100644 --- a/src/Backups/BackupCoordinationRemote.h +++ b/src/Backups/BackupCoordinationRemote.h @@ -5,9 +5,9 @@ #include #include #include +#include #include #include -#include "Backups/BackupCoordinationKeeperMapTables.h" namespace DB @@ -89,6 +89,7 @@ class BackupCoordinationRemote : public IBackupCoordination void prepareReplicatedTables() const TSA_REQUIRES(replicated_tables_mutex); void prepareReplicatedAccess() const TSA_REQUIRES(replicated_access_mutex); void prepareReplicatedSQLObjects() const TSA_REQUIRES(replicated_sql_objects_mutex); + void prepareKeeperMapTables() const TSA_REQUIRES(keeper_map_tables_mutex); void prepareFileInfos() const TSA_REQUIRES(file_infos_mutex); const String root_zookeeper_path; @@ -110,6 +111,7 @@ class BackupCoordinationRemote : public IBackupCoordination mutable std::optional TSA_GUARDED_BY(replicated_access_mutex) replicated_access; mutable std::optional TSA_GUARDED_BY(replicated_sql_objects_mutex) replicated_sql_objects; mutable std::optional TSA_GUARDED_BY(file_infos_mutex) file_infos; + mutable std::optional keeper_map_tables TSA_GUARDED_BY(keeper_map_tables_mutex); std::unordered_set TSA_GUARDED_BY(writing_files_mutex) writing_files; struct KeeperMapTableInfo @@ -118,7 +120,6 @@ class BackupCoordinationRemote : public IBackupCoordination String data_path_in_backup; }; - mutable BackupCoordinationKeeperMapTables keeper_map_tables TSA_GUARDED_BY(keeper_map_tables_mutex); mutable std::mutex zookeeper_mutex; mutable std::mutex replicated_tables_mutex; diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp index 7e059b8d9cce..12a67d2a55dc 100644 --- a/src/Backups/RestoreCoordinationRemote.cpp +++ b/src/Backups/RestoreCoordinationRemote.cpp @@ -89,6 +89,7 @@ void RestoreCoordinationRemote::createRootNodes() ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/repl_tables_data_acquired", "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/repl_access_storages_acquired", "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/repl_sql_objects_acquired", "", zkutil::CreateMode::Persistent)); + ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/keeper_map_tables", "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/table_uuids", "", zkutil::CreateMode::Persistent)); zk->tryMulti(ops, responses); }); @@ -234,9 +235,43 @@ bool RestoreCoordinationRemote::acquireReplicatedSQLObjects(const String & loade return result; } -bool RestoreCoordinationRemote::acquireInsertingDataForKeeperMap(const String & /*root_zk_path*/) +bool RestoreCoordinationRemote::acquireInsertingDataForKeeperMap(const String & root_zk_path) { - return true; + bool result = false; + auto holder = with_retries.createRetriesControlHolder("acquireInsertingDataForKeeperMap"); + holder.retries_ctl.retryLoop( + [&, &zk = holder.faulty_zookeeper]() + { + with_retries.renewZooKeeper(zk); + + fs::path base_path = fs::path(zookeeper_path) / "keeper_map_tables" / root_zk_path; + zk->createAncestors(base_path); + std::string restore_lock_path = base_path / "restore_lock"; + result = zk->tryCreate(restore_lock_path, "restorelock", zkutil::CreateMode::Persistent) == Coordination::Error::ZOK; + + if (result) + return; + + /// there can be an edge case where a path contains `/restore_lock/ in the middle of it + /// to differentiate that case from lock we also set the data + for (size_t i = 0; i < 1000; ++i) + { + Coordination::Stat lock_stat; + auto data = zk->get(restore_lock_path, &lock_stat); + if (data == "restorelock") + return; + + if (auto set_result = zk->trySet(restore_lock_path, "restorelock", lock_stat.version); + set_result == Coordination::Error::ZOK) + { + result = true; + return; + } + else if (set_result == Coordination::Error::ZNONODE) + throw zkutil::KeeperException::fromPath(set_result, restore_lock_path); + } + }); + return result; } void RestoreCoordinationRemote::generateUUIDForTable(ASTCreateQuery & create_query) diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index 33a97af53f1d..c583a6930359 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -753,7 +753,6 @@ void StorageKeeperMap::backupData(BackupEntriesCollector & backup_entries_collec { auto table_id = toString(getStorageID().uuid); - std::cout << "Backing up for path " << zk_root_path << " table id " << table_id << std::endl; auto coordination = backup_entries_collector.getBackupCoordination(); coordination->addKeeperMapTable(zk_root_path, table_id, data_path_in_backup); @@ -764,8 +763,6 @@ void StorageKeeperMap::backupData(BackupEntriesCollector & backup_entries_collec auto path_with_data = coordination->getKeeperMapDataPath(zk_root_path); if (path_with_data == my_data_path_in_backup) { - std::cout << "Will be backing up data for path " << zk_root_path << " table id " << my_table_id << std::endl; - auto temp_disk = backup_entries_collector.getContext()->getGlobalTemporaryVolume()->getDisk(0); auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef().max_compress_block_size; @@ -783,7 +780,6 @@ void StorageKeeperMap::backupData(BackupEntriesCollector & backup_entries_collec return; } - std::cout << "Not backing up data for path " << zk_root_path << " table id " << my_table_id << " writing only path with data " << path_with_data << std::endl; auto file_path = fs::path(my_data_path_in_backup) / backup_data_location_filename; backup_entries_collector.addBackupEntries({{file_path, std::make_shared(path_with_data)}}); }; @@ -842,8 +838,6 @@ void StorageKeeperMap::restoreDataImpl(const BackupPtr & backup, const String & { auto table_id = toString(getStorageID().uuid); - std::cout << "Restoring into " << zk_root_path << " table id " << table_id << std::endl; - fs::path data_path_in_backup_fs = data_path_in_backup; String data_file = data_path_in_backup_fs / backup_data_filename; @@ -882,7 +876,7 @@ void StorageKeeperMap::restoreDataImpl(const BackupPtr & backup, const String & }); }; - while (!in->eof()) + while (!compressed_in.eof()) { std::string key; std::string value; From 4438c2f70aeadd6ec6ab221f2980210351b7d1c5 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 8 Nov 2023 14:36:39 +0000 Subject: [PATCH 033/274] Remove unnecassary log --- src/Storages/StorageKeeperMap.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index c583a6930359..74c1905cd61b 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -692,14 +692,13 @@ class KeeperMapBackup : public IBackupEntriesLazyBatch, boost::noncopyable data_children = zk->getChildren(data_zookeeper_path); }); } - LOG_INFO(&Poco::Logger::get("BACKUPER"), "Got {} children", data_children.size()); const auto write_rows = [&](std::span keys) { std::vector keys_full_path; - keys_full_path.reserve(data_children.size()); + keys_full_path.reserve(keys.size()); - for (const auto & key : data_children) + for (const auto & key : keys) keys_full_path.push_back(data_zookeeper_path / key); zkutil::ZooKeeper::MultiGetResponse data; From 93e22e85d1ce07fde446b68f5fcaa9d75cb2a090 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 8 Nov 2023 17:16:56 +0100 Subject: [PATCH 034/274] Better --- src/Interpreters/Cache/FileCache.cpp | 188 ++++++++++++++---- src/Interpreters/Cache/FileCache.h | 4 + src/Interpreters/Cache/FileSegment.cpp | 1 + .../tests/gtest_lru_file_cache.cpp | 2 +- 4 files changed, 154 insertions(+), 41 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 31e9008c69ad..1a09db7a3f0f 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -156,7 +156,7 @@ FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment: { auto file_segment = std::make_shared( locked_key.getKey(), range.left, range.size(), FileSegment::State::DETACHED); - return { file_segment }; + return {file_segment}; } if (locked_key.empty()) @@ -245,11 +245,34 @@ FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment: return result; } +std::vector FileCache::splitRange(size_t offset, size_t size) +{ + assert(size > 0); + std::vector ranges; + + size_t current_pos = offset; + size_t end_pos_non_included = offset + size; + size_t remaining_size = size; + + FileSegments file_segments; + while (current_pos < end_pos_non_included) + { + auto current_file_segment_size = std::min(remaining_size, max_file_segment_size); + ranges.emplace_back(current_pos, current_pos + current_file_segment_size - 1); + + remaining_size -= current_file_segment_size; + current_pos += current_file_segment_size; + } + + return ranges; +} + FileSegments FileCache::splitRangeIntoFileSegments( LockedKey & locked_key, size_t offset, size_t size, FileSegment::State state, + size_t file_segments_limit, const CreateFileSegmentSettings & settings) { assert(size > 0); @@ -261,7 +284,7 @@ FileSegments FileCache::splitRangeIntoFileSegments( size_t remaining_size = size; FileSegments file_segments; - while (current_pos < end_pos_non_included) + while (current_pos < end_pos_non_included && (!file_segments_limit || file_segments.size() < file_segments_limit)) { current_file_segment_size = std::min(remaining_size, max_file_segment_size); remaining_size -= current_file_segment_size; @@ -273,7 +296,7 @@ FileSegments FileCache::splitRangeIntoFileSegments( current_pos += current_file_segment_size; } - assert(file_segments.empty() || offset + size - 1 == file_segments.back()->range().right); + assert(file_segments.empty() || file_segments_limit > 0 || offset + size - 1 == file_segments.back()->range().right); return file_segments; } @@ -298,6 +321,7 @@ void FileCache::fillHolesWithEmptyFileSegments( assert(!file_segments.empty()); auto it = file_segments.begin(); + size_t added = 0; auto segment_range = (*it)->range(); size_t current_pos; @@ -310,11 +334,12 @@ void FileCache::fillHolesWithEmptyFileSegments( current_pos = segment_range.right + 1; ++it; + ++added; } else current_pos = range.left; - while (current_pos <= range.right && it != file_segments.end()) + while (current_pos <= range.right && it != file_segments.end() && (!file_segments_limit || added < file_segments_limit)) { segment_range = (*it)->range(); @@ -322,6 +347,7 @@ void FileCache::fillHolesWithEmptyFileSegments( { current_pos = segment_range.right + 1; ++it; + ++added; continue; } @@ -338,17 +364,38 @@ void FileCache::fillHolesWithEmptyFileSegments( } else { - auto split = splitRangeIntoFileSegments( - locked_key, current_pos, hole_size, FileSegment::State::EMPTY, settings); - file_segments.splice(it, std::move(split)); + auto ranges = splitRange(current_pos, hole_size); + FileSegments hole; + for (const auto & r : ranges) + { + auto metadata_it = addFileSegment(locked_key, r.left, r.size(), FileSegment::State::EMPTY, settings, nullptr); + hole.push_back(metadata_it->second->file_segment); + ++added; + + if (file_segments_limit && added == file_segments_limit) + { + file_segments.splice(it, std::move(hole)); + file_segments.erase(it, file_segments.end()); + return; + } + } + file_segments.splice(it, std::move(hole)); } current_pos = segment_range.right + 1; ++it; + ++added; } - if (file_segments_limit && file_segments.size() >= file_segments_limit) + if (file_segments_limit && added == file_segments_limit) + { + chassert(file_segments.size() >= file_segments_limit); + file_segments.erase(it, file_segments.end()); + chassert(file_segments.size() == file_segments_limit); return; + } + + chassert(!file_segments_limit || file_segments.size() < file_segments_limit); if (current_pos <= range.right) { @@ -368,9 +415,22 @@ void FileCache::fillHolesWithEmptyFileSegments( } else { - auto split = splitRangeIntoFileSegments( - locked_key, current_pos, hole_size, FileSegment::State::EMPTY, settings); - file_segments.splice(file_segments.end(), std::move(split)); + auto ranges = splitRange(current_pos, hole_size); + FileSegments hole; + for (const auto & r : ranges) + { + auto metadata_it = addFileSegment(locked_key, r.left, r.size(), FileSegment::State::EMPTY, settings, nullptr); + hole.push_back(metadata_it->second->file_segment); + ++added; + + if (file_segments_limit && added == file_segments_limit) + { + file_segments.splice(it, std::move(hole)); + file_segments.erase(it, file_segments.end()); + return; + } + } + file_segments.splice(it, std::move(hole)); } } } @@ -400,7 +460,7 @@ FileSegmentsHolderPtr FileCache::set( else { file_segments = splitRangeIntoFileSegments( - *locked_key, offset, size, FileSegment::State::EMPTY, settings); + *locked_key, offset, size, FileSegment::State::EMPTY, /* file_segments_limit */0, settings); } return std::make_unique(std::move(file_segments)); @@ -419,77 +479,125 @@ FileCache::getOrSet( assertInitialized(); - const auto end_offset = offset + size - 1; - const auto aligned_offset = roundDownToMultiple(offset, boundary_alignment); - const auto aligned_end_offset = std::min(roundUpToMultiple(offset + size, boundary_alignment), file_size) - 1; - chassert(aligned_offset <= offset); + FileSegment::Range range(offset, offset + size - 1); - auto locked_key = metadata.lockKeyMetadata(key, CacheMetadata::KeyNotFoundPolicy::CREATE_EMPTY); + const auto aligned_offset = roundDownToMultiple(range.left, boundary_alignment); + auto aligned_end_offset = std::min(roundUpToMultiple(offset + size, boundary_alignment), file_size) - 1; + chassert(aligned_offset <= range.left); + chassert(aligned_end_offset >= range.right); + + auto locked_key = metadata.lockKeyMetadata(key, CacheMetadata::KeyNotFoundPolicy::CREATE_EMPTY); /// Get all segments which intersect with the given range. - FileSegment::Range range(offset, end_offset); auto file_segments = getImpl(*locked_key, range, file_segments_limit); - if (aligned_offset < offset && (file_segments.empty() || offset < file_segments.front()->range().left)) + if (file_segments_limit) + { + chassert(file_segments.size() <= file_segments_limit); + if (file_segments.size() == file_segments_limit) + range.right = aligned_end_offset = file_segments.back()->range().right; + } + + /// Check case if we have uncovered prefix, e.g. + /// + /// [_______________] + /// ^ ^ + /// range.left range.right + /// [___] [__________] <-- current cache (example) + /// [ ] + /// ^----^ + /// uncovered prefix. + const bool has_uncovered_prefix = file_segments.empty() || range.left < file_segments.front()->range().left; + + if (aligned_offset < range.left && has_uncovered_prefix) { - auto prefix_range = FileSegment::Range(aligned_offset, file_segments.empty() ? offset - 1 : file_segments.front()->range().left - 1); + auto prefix_range = FileSegment::Range(aligned_offset, file_segments.empty() ? range.left - 1 : file_segments.front()->range().left - 1); auto prefix_file_segments = getImpl(*locked_key, prefix_range, /* file_segments_limit */0); if (prefix_file_segments.empty()) { + /// [____________________][_______________] + /// ^ ^ ^ + /// aligned_offset range.left range.right + /// [___] [__________] <-- current cache (example) range.left = aligned_offset; } else { - size_t last_right_offset = prefix_file_segments.back()->range().right; + /// [____________________][_______________] + /// ^ ^ ^ + /// aligned_offset range.left range.right + /// ____] [____] [___] [__________] <-- current cache (example) + /// ^ + /// prefix_file_segments.back().right - while (!prefix_file_segments.empty() && prefix_file_segments.front()->range().right < offset) - prefix_file_segments.pop_front(); + chassert(prefix_file_segments.back()->range().right < range.left); + chassert(prefix_file_segments.back()->range().right >= aligned_offset); - if (prefix_file_segments.empty()) - { - range.left = last_right_offset + 1; - } - else - { - file_segments.splice(file_segments.begin(), prefix_file_segments); - range.left = file_segments.front()->range().left; - } + range.left = prefix_file_segments.back()->range().right + 1; } } - if (end_offset < aligned_end_offset && (file_segments.empty() || file_segments.back()->range().right < end_offset)) + /// Check case if we have uncovered suffix. + /// + /// [___________________] + /// ^ ^ + /// range.left range.right + /// [___] [___] <-- current cache (example) + /// [___] + /// ^---^ + /// uncovered_suffix + const bool has_uncovered_suffix = file_segments.empty() || file_segments.back()->range().right < range.right; + + if (range.right < aligned_end_offset && has_uncovered_suffix) { - auto suffix_range = FileSegment::Range(end_offset, aligned_end_offset); - /// Get only 1 file segment. + auto suffix_range = FileSegment::Range(range.right, aligned_end_offset); + /// We need to get 1 file segment, so file_segments_limit = 1 here. auto suffix_file_segments = getImpl(*locked_key, suffix_range, /* file_segments_limit */1); if (suffix_file_segments.empty()) + { + /// [__________________][ ] + /// ^ ^ ^ + /// range.left range.right aligned_end_offset + /// [___] [___] <-- current cache (example) + range.right = aligned_end_offset; + } else + { + /// [__________________][ ] + /// ^ ^ ^ + /// range.left range.right aligned_end_offset + /// [___] [___] [_________] <-- current cache (example) + /// ^ + /// suffix_file_segments.front().left range.right = suffix_file_segments.front()->range().left - 1; + } } if (file_segments.empty()) { - file_segments = splitRangeIntoFileSegments(*locked_key, range.left, range.size(), FileSegment::State::EMPTY, settings); + file_segments = splitRangeIntoFileSegments(*locked_key, range.left, range.size(), FileSegment::State::EMPTY, file_segments_limit, settings); } else { - chassert(file_segments.front()->range().right >= offset); - chassert(file_segments.back()->range().left <= end_offset); + chassert(file_segments.front()->range().right >= range.left); + chassert(file_segments.back()->range().left <= range.right); fillHolesWithEmptyFileSegments( *locked_key, file_segments, range, file_segments_limit, /* fill_with_detached */false, settings); + chassert(!file_segments_limit || file_segments.size() <= file_segments_limit); + if (!file_segments.front()->range().contains(offset)) { throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected {} to include {} " "(end offset: {}, aligned offset: {}, aligned end offset: {})", - file_segments.front()->range().toString(), offset, end_offset, aligned_offset, aligned_end_offset); + file_segments.front()->range().toString(), offset, range.right, aligned_offset, aligned_end_offset); } - chassert(file_segments_limit ? file_segments.back()->range().left <= end_offset : file_segments.back()->range().contains(end_offset)); + chassert(file_segments_limit ? file_segments.back()->range().left <= range.right : file_segments.back()->range().contains(range.right)); } while (file_segments_limit && file_segments.size() > file_segments_limit) diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index f8fd9635cd54..523ff90e33e5 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -216,13 +216,17 @@ class FileCache : private boost::noncopyable void loadMetadataImpl(); void loadMetadataForKeys(const std::filesystem::path & keys_dir); + /// bool - if `file_segments_limit` reached or not. FileSegments getImpl(const LockedKey & locked_key, const FileSegment::Range & range, size_t file_segments_limit) const; + std::vector splitRange(size_t offset, size_t size); + FileSegments splitRangeIntoFileSegments( LockedKey & locked_key, size_t offset, size_t size, FileSegment::State state, + size_t file_segments_limit, const CreateFileSegmentSettings & create_settings); void fillHolesWithEmptyFileSegments( diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 362103f3e226..591342299974 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -926,6 +926,7 @@ void FileSegment::use() FileSegmentsHolder::FileSegmentsHolder(FileSegments && file_segments_) : file_segments(std::move(file_segments_)) +{ CurrentMetrics::add(CurrentMetrics::FilesystemCacheHoldFileSegments, file_segments.size()); ProfileEvents::increment(ProfileEvents::FilesystemCacheHoldFileSegments, file_segments.size()); } diff --git a/src/Interpreters/tests/gtest_lru_file_cache.cpp b/src/Interpreters/tests/gtest_lru_file_cache.cpp index ab2a128de348..e1db07958fc9 100644 --- a/src/Interpreters/tests/gtest_lru_file_cache.cpp +++ b/src/Interpreters/tests/gtest_lru_file_cache.cpp @@ -242,7 +242,7 @@ TEST_F(FileCacheTest, get) settings.max_elements = 5; settings.boundary_alignment = 1; - const size_t file_size = -1; // the value doesn't really matter because boundary_alignment == 1. + const size_t file_size = INT_MAX; // the value doesn't really matter because boundary_alignment == 1. { std::cerr << "Step 1\n"; From f9895ab37b2133a36296b67b8904d251ffdaf3e4 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 9 Nov 2023 15:56:57 +0000 Subject: [PATCH 035/274] Small fixes and add test --- src/Backups/BackupCoordinationRemote.cpp | 5 +- src/Backups/RestoreCoordinationRemote.cpp | 7 +- src/Common/ZooKeeper/ZooKeeper.cpp | 1 + src/Storages/StorageKeeperMap.cpp | 43 +++++-- src/Storages/StorageKeeperMap.h | 7 +- .../__init__.py | 0 .../configs/backups_disk.xml | 13 ++ .../configs/keeper_map_path_prefix.xml | 3 + .../configs/remote_servers.xml | 22 ++++ .../configs/zookeeper_retries.xml | 11 ++ .../test_backup_restore_keeper_map/test.py | 111 ++++++++++++++++++ .../02911_backup_restore_keeper_map.reference | 13 ++ .../02911_backup_restore_keeper_map.sh | 47 ++++++++ 13 files changed, 269 insertions(+), 14 deletions(-) create mode 100644 tests/integration/test_backup_restore_keeper_map/__init__.py create mode 100644 tests/integration/test_backup_restore_keeper_map/configs/backups_disk.xml create mode 100644 tests/integration/test_backup_restore_keeper_map/configs/keeper_map_path_prefix.xml create mode 100644 tests/integration/test_backup_restore_keeper_map/configs/remote_servers.xml create mode 100644 tests/integration/test_backup_restore_keeper_map/configs/zookeeper_retries.xml create mode 100644 tests/integration/test_backup_restore_keeper_map/test.py create mode 100644 tests/queries/0_stateless/02911_backup_restore_keeper_map.reference create mode 100755 tests/queries/0_stateless/02911_backup_restore_keeper_map.sh diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp index 309cbc8be6a1..064e0599f6e6 100644 --- a/src/Backups/BackupCoordinationRemote.cpp +++ b/src/Backups/BackupCoordinationRemote.cpp @@ -681,7 +681,10 @@ void BackupCoordinationRemote::addKeeperMapTable(const String & table_zookeeper_ { with_retries.renewZooKeeper(zk); String path = zookeeper_path + "/keeper_map_tables/" + escapeForFileName(table_id); - zk->create(path, fmt::format("{}\n{}", table_zookeeper_root_path, data_path_in_backup), zkutil::CreateMode::Persistent); + if (auto res + = zk->tryCreate(path, fmt::format("{}\n{}", table_zookeeper_root_path, data_path_in_backup), zkutil::CreateMode::Persistent); + res != Coordination::Error::ZOK && res != Coordination::Error::ZNODEEXISTS) + throw zkutil::KeeperException(res); }); } diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp index 12a67d2a55dc..1b814c2889e0 100644 --- a/src/Backups/RestoreCoordinationRemote.cpp +++ b/src/Backups/RestoreCoordinationRemote.cpp @@ -244,9 +244,10 @@ bool RestoreCoordinationRemote::acquireInsertingDataForKeeperMap(const String & { with_retries.renewZooKeeper(zk); - fs::path base_path = fs::path(zookeeper_path) / "keeper_map_tables" / root_zk_path; - zk->createAncestors(base_path); - std::string restore_lock_path = base_path / "restore_lock"; + /// we need to remove leading '/' from root_zk_path + auto normalized_root_zk_path = std::string_view{root_zk_path}.substr(1); + std::string restore_lock_path = fs::path(zookeeper_path) / "keeper_map_tables" / normalized_root_zk_path / "restore_lock"; + zk->createAncestors(restore_lock_path); result = zk->tryCreate(restore_lock_path, "restorelock", zkutil::CreateMode::Persistent) == Coordination::Error::ZOK; if (result) diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 436a4e14f14b..8a97362aa963 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -385,6 +385,7 @@ void ZooKeeper::createAncestors(const std::string & path) size_t last_pos = path.rfind('/'); if (last_pos == std::string::npos || last_pos == 0) return; + std::string current_node = path.substr(0, last_pos); while (true) diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index 74c1905cd61b..15ebc4d92d19 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -15,7 +15,7 @@ #include #include -#include +#include #include #include @@ -52,6 +52,8 @@ #include #include +#include + #include #include @@ -824,16 +826,24 @@ void StorageKeeperMap::restoreDataFromBackup(RestorerFromBackup & restorer, cons RestorerFromBackup::throwTableIsNotEmpty(getStorageID()); } - /// TODO: Should we backup and verify the table structure? + auto temp_disk = restorer.getContext()->getGlobalTemporaryVolume()->getDisk(0); - //auto temp_disk = restorer.getContext()->getGlobalTemporaryVolume()->getDisk(0); /// only 1 table should restore data for a single path restorer.addDataRestoreTask( - [storage = std::static_pointer_cast(shared_from_this()), backup, data_path_in_backup, with_retries, allow_non_empty_tables] - { storage->restoreDataImpl(backup, data_path_in_backup, with_retries, allow_non_empty_tables); }); + [storage = std::static_pointer_cast(shared_from_this()), + backup, + data_path_in_backup, + with_retries, + allow_non_empty_tables, + temp_disk] { storage->restoreDataImpl(backup, data_path_in_backup, with_retries, allow_non_empty_tables, temp_disk); }); } -void StorageKeeperMap::restoreDataImpl(const BackupPtr & backup, const String & data_path_in_backup, std::shared_ptr with_retries, bool allow_non_empty_tables) +void StorageKeeperMap::restoreDataImpl( + const BackupPtr & backup, + const String & data_path_in_backup, + std::shared_ptr with_retries, + bool allow_non_empty_tables, + const DiskPtr & temporary_disk) { auto table_id = toString(getStorageID().uuid); @@ -858,7 +868,17 @@ void StorageKeeperMap::restoreDataImpl(const BackupPtr & backup, const String & /// should we store locally in temp file? auto in = backup->readFile(data_file); - CompressedReadBuffer compressed_in{*in}; + std::optional temp_data_file; + if (!dynamic_cast(in.get())) + { + temp_data_file.emplace(temporary_disk); + auto out = std::make_unique(temp_data_file->getAbsolutePath()); + copyData(*in, *out); + out.reset(); + in = createReadBufferFromFileBase(temp_data_file->getAbsolutePath(), {}); + } + std::unique_ptr in_from_file{static_cast(in.release())}; + CompressedReadBufferFromFile compressed_in{std::move(in_from_file)}; fs::path data_path_fs(zk_data_path); auto max_multi_size = with_retries->getKeeperSettings().batch_size_for_keeper_multi; @@ -871,7 +891,10 @@ void StorageKeeperMap::restoreDataImpl(const BackupPtr & backup, const String & [&, &zk = holder.faulty_zookeeper]() { with_retries->renewZooKeeper(zk); - zk->multi(create_requests); + Coordination::Responses create_responses; + if (auto res = zk->tryMulti(create_requests, create_responses); + res != Coordination::Error::ZOK && res != Coordination::Error::ZNODEEXISTS) + throw zkutil::KeeperMultiException(res, create_requests, create_responses); }); }; @@ -890,7 +913,9 @@ void StorageKeeperMap::restoreDataImpl(const BackupPtr & backup, const String & [&, &zk = holder.faulty_zookeeper]() { with_retries->renewZooKeeper(zk); - zk->tryCreate(data_path_fs / key, value, zkutil::CreateMode::Persistent); + if (auto res = zk->tryCreate(data_path_fs / key, value, zkutil::CreateMode::Persistent); + res != Coordination::Error::ZOK && res != Coordination::Error::ZNODEEXISTS) + throw zkutil::KeeperException::fromPath(res, data_path_fs / key); }); } /// otherwise we can do multi requests diff --git a/src/Storages/StorageKeeperMap.h b/src/Storages/StorageKeeperMap.h index 94b02ca0242b..10eebdd0129b 100644 --- a/src/Storages/StorageKeeperMap.h +++ b/src/Storages/StorageKeeperMap.h @@ -120,7 +120,12 @@ class StorageKeeperMap final : public IStorage, public IKeyValueEntity, WithCont std::optional isTableValid() const; - void restoreDataImpl(const BackupPtr & backup, const String & data_path_in_backup, std::shared_ptr with_retries, bool allow_non_empty_tables); + void restoreDataImpl( + const BackupPtr & backup, + const String & data_path_in_backup, + std::shared_ptr with_retries, + bool allow_non_empty_tables, + const DiskPtr & temporary_disk); std::string zk_root_path; std::string primary_key; diff --git a/tests/integration/test_backup_restore_keeper_map/__init__.py b/tests/integration/test_backup_restore_keeper_map/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_backup_restore_keeper_map/configs/backups_disk.xml b/tests/integration/test_backup_restore_keeper_map/configs/backups_disk.xml new file mode 100644 index 000000000000..b99a51cd56d0 --- /dev/null +++ b/tests/integration/test_backup_restore_keeper_map/configs/backups_disk.xml @@ -0,0 +1,13 @@ + + + + + local + /backups/ + + + + + backups + + diff --git a/tests/integration/test_backup_restore_keeper_map/configs/keeper_map_path_prefix.xml b/tests/integration/test_backup_restore_keeper_map/configs/keeper_map_path_prefix.xml new file mode 100644 index 000000000000..91d7b9d3f8fd --- /dev/null +++ b/tests/integration/test_backup_restore_keeper_map/configs/keeper_map_path_prefix.xml @@ -0,0 +1,3 @@ + + /keeper_map_tables + diff --git a/tests/integration/test_backup_restore_keeper_map/configs/remote_servers.xml b/tests/integration/test_backup_restore_keeper_map/configs/remote_servers.xml new file mode 100644 index 000000000000..5cf07c69fd6b --- /dev/null +++ b/tests/integration/test_backup_restore_keeper_map/configs/remote_servers.xml @@ -0,0 +1,22 @@ + + + + + + node1 + 9000 + + + node2 + 9000 + + + + + node3 + 9000 + + + + + diff --git a/tests/integration/test_backup_restore_keeper_map/configs/zookeeper_retries.xml b/tests/integration/test_backup_restore_keeper_map/configs/zookeeper_retries.xml new file mode 100644 index 000000000000..1283f28a8cb4 --- /dev/null +++ b/tests/integration/test_backup_restore_keeper_map/configs/zookeeper_retries.xml @@ -0,0 +1,11 @@ + + + + 1000 + 1 + 1 + 42 + 0.002 + + + diff --git a/tests/integration/test_backup_restore_keeper_map/test.py b/tests/integration/test_backup_restore_keeper_map/test.py new file mode 100644 index 000000000000..95e8a8b30279 --- /dev/null +++ b/tests/integration/test_backup_restore_keeper_map/test.py @@ -0,0 +1,111 @@ +from time import sleep +import pytest +from helpers.cluster import ClickHouseCluster + + +cluster = ClickHouseCluster(__file__) + +main_configs = [ + "configs/remote_servers.xml", + "configs/backups_disk.xml", + "configs/keeper_map_path_prefix.xml", +] + +user_configs = [ + "configs/zookeeper_retries.xml", +] + +node1 = cluster.add_instance( + "node1", + main_configs=main_configs, + user_configs=user_configs, + external_dirs=["/backups/"], + macros={"replica": "node1", "shard": "shard1"}, + with_zookeeper=True, + stay_alive=True, +) + +node2 = cluster.add_instance( + "node2", + main_configs=main_configs, + user_configs=user_configs, + external_dirs=["/backups/"], + macros={"replica": "node2", "shard": "shard1"}, + with_zookeeper=True, + stay_alive=True, +) + + +node3 = cluster.add_instance( + "node3", + main_configs=main_configs, + user_configs=user_configs, + external_dirs=["/backups/"], + macros={"replica": "node3", "shard": "shard2"}, + with_zookeeper=True, + stay_alive=True, +) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +backup_id_counter = 0 + +def new_backup_name(base_name): + global backup_id_counter + backup_id_counter += 1 + return f"Disk('backups', '{base_name}{backup_id_counter}')" + +def test_on_cluster(): + node1.query_with_retry("CREATE DATABASE keeper_backup ON CLUSTER cluster") + node1.query_with_retry("CREATE TABLE keeper_backup.keeper1 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key") + node1.query_with_retry("CREATE TABLE keeper_backup.keeper2 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key") + node1.query_with_retry("CREATE TABLE keeper_backup.keeper3 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster2') PRIMARY KEY key") + node1.query_with_retry("INSERT INTO keeper_backup.keeper2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5") + node1.query_with_retry("INSERT INTO keeper_backup.keeper3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5") + + expected_result = ''.join(f'{i}\ttest{i}\n' for i in range(5)) + + def verify_data(): + for node in [node1, node2, node3]: + for i in range(1, 4): + result = node.query_with_retry(f'SELECT key, value FROM keeper_backup.keeper{i} ORDER BY key FORMAT TSV') + assert result == expected_result + + verify_data() + + backup_name = new_backup_name('test_on_cluster') + node1.query(f"BACKUP DATABASE keeper_backup ON CLUSTER cluster TO {backup_name} SETTINGS async = false;") + + node1.query("DROP DATABASE keeper_backup ON CLUSTER cluster SYNC;") + + def apply_for_all_nodes(f): + for node in [node1, node2, node3]: + f(node) + + def change_keeper_map_prefix(node): + node.replace_config( + "/etc/clickhouse-server/config.d/keeper_map_path_prefix.xml", """ + + /different_path/keeper_map + +""") + + apply_for_all_nodes(lambda node: node.stop_clickhouse()) + apply_for_all_nodes(change_keeper_map_prefix) + apply_for_all_nodes(lambda node: node.start_clickhouse()) + + node1.query(f"RESTORE DATABASE keeper_backup ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;") + + verify_data() + + node1.query("DROP TABLE keeper_backup.keeper3 ON CLUSTER cluster SYNC;") + node1.query(f"RESTORE TABLE keeper_backup.keeper3 ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;") + + verify_data() \ No newline at end of file diff --git a/tests/queries/0_stateless/02911_backup_restore_keeper_map.reference b/tests/queries/0_stateless/02911_backup_restore_keeper_map.reference new file mode 100644 index 000000000000..e58335de67c8 --- /dev/null +++ b/tests/queries/0_stateless/02911_backup_restore_keeper_map.reference @@ -0,0 +1,13 @@ +5000 +5000 +3000 +OK +OK +OK +5000 +5000 +3000 +OK +5000 +5000 +3000 diff --git a/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh b/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh new file mode 100755 index 000000000000..6c463beb2217 --- /dev/null +++ b/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -nm -q " + DROP DATABASE IF EXISTS 02911_keeper_map; + CREATE DATABASE 02911_keeper_map; + CREATE TABLE 02911_keeper_map.02911_backup_restore_keeper_map1 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911') PRIMARY KEY key; + CREATE TABLE 02911_keeper_map.02911_backup_restore_keeper_map2 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911') PRIMARY KEY key; + CREATE TABLE 02911_keeper_map.02911_backup_restore_keeper_map3 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911_different') PRIMARY KEY key; + + INSERT INTO 02911_keeper_map.02911_backup_restore_keeper_map2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5000; + INSERT INTO 02911_keeper_map.02911_backup_restore_keeper_map3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 3000; +" + +backup_path="$CLICKHOUSE_DATABASE/02911_keeper_map" +for i in $(seq 1 3); do + $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" +done + +$CLICKHOUSE_CLIENT -q "BACKUP DATABASE 02911_keeper_map TO Disk('backups', '$backup_path');" > /dev/null + +$CLICKHOUSE_CLIENT -q "DROP DATABASE 02911_keeper_map SYNC;" + +for i in $(seq 1 3); do + $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" 2>&1 | grep -Fq "UNKNOWN_DATABASE" && echo 'OK' || echo 'ERROR' +done + +$CLICKHOUSE_CLIENT -q "RESTORE DATABASE 02911_keeper_map FROM Disk('backups', '$backup_path');" > /dev/null + +for i in $(seq 1 3); do + $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" +done + +$CLICKHOUSE_CLIENT -q "DROP TABLE 02911_keeper_map.02911_backup_restore_keeper_map3 SYNC;" + +$CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map3;" 2>&1 | grep -Fq "UNKNOWN_TABLE" && echo 'OK' || echo 'ERROR' + +$CLICKHOUSE_CLIENT -q "RESTORE TABLE 02911_keeper_map.02911_backup_restore_keeper_map3 FROM Disk('backups', '$backup_path');" > /dev/null + +for i in $(seq 1 3); do + $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" +done + +$CLICKHOUSE_CLIENT -q "DROP DATABASE 02911_keeper_map SYNC;" \ No newline at end of file From ae09b16701624825da053a1b0501c20243e71fd3 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 9 Nov 2023 17:05:11 +0100 Subject: [PATCH 036/274] Debug logging --- src/Interpreters/Cache/FileCache.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 1a09db7a3f0f..ed8cc547fbc6 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -395,6 +395,14 @@ void FileCache::fillHolesWithEmptyFileSegments( return; } + if (file_segments_limit && file_segments.size() >= file_segments_limit) + { + std::string res; + for (const auto & f : file_segments) + res += " - " + f->range().toString(); + LOG_ERROR(log, "Limit: {}, file segments: {}, added: {}, range: {}, file_segments: {}", + file_segments_limit, file_segments.size(), added, range.toString(), res); + } chassert(!file_segments_limit || file_segments.size() < file_segments_limit); if (current_pos <= range.right) From 188a88fa3391cd2044fce96d00da12c22b654319 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 9 Nov 2023 16:15:14 +0000 Subject: [PATCH 037/274] Automatic style fix --- .../test_backup_restore_keeper_map/test.py | 57 +++++++++++++------ 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/tests/integration/test_backup_restore_keeper_map/test.py b/tests/integration/test_backup_restore_keeper_map/test.py index 95e8a8b30279..8343ad3177fa 100644 --- a/tests/integration/test_backup_restore_keeper_map/test.py +++ b/tests/integration/test_backup_restore_keeper_map/test.py @@ -55,33 +55,50 @@ def start_cluster(): finally: cluster.shutdown() + backup_id_counter = 0 + def new_backup_name(base_name): global backup_id_counter backup_id_counter += 1 return f"Disk('backups', '{base_name}{backup_id_counter}')" + def test_on_cluster(): node1.query_with_retry("CREATE DATABASE keeper_backup ON CLUSTER cluster") - node1.query_with_retry("CREATE TABLE keeper_backup.keeper1 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key") - node1.query_with_retry("CREATE TABLE keeper_backup.keeper2 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key") - node1.query_with_retry("CREATE TABLE keeper_backup.keeper3 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster2') PRIMARY KEY key") - node1.query_with_retry("INSERT INTO keeper_backup.keeper2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5") - node1.query_with_retry("INSERT INTO keeper_backup.keeper3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5") - - expected_result = ''.join(f'{i}\ttest{i}\n' for i in range(5)) + node1.query_with_retry( + "CREATE TABLE keeper_backup.keeper1 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key" + ) + node1.query_with_retry( + "CREATE TABLE keeper_backup.keeper2 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key" + ) + node1.query_with_retry( + "CREATE TABLE keeper_backup.keeper3 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster2') PRIMARY KEY key" + ) + node1.query_with_retry( + "INSERT INTO keeper_backup.keeper2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5" + ) + node1.query_with_retry( + "INSERT INTO keeper_backup.keeper3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5" + ) + + expected_result = "".join(f"{i}\ttest{i}\n" for i in range(5)) def verify_data(): for node in [node1, node2, node3]: for i in range(1, 4): - result = node.query_with_retry(f'SELECT key, value FROM keeper_backup.keeper{i} ORDER BY key FORMAT TSV') + result = node.query_with_retry( + f"SELECT key, value FROM keeper_backup.keeper{i} ORDER BY key FORMAT TSV" + ) assert result == expected_result verify_data() - backup_name = new_backup_name('test_on_cluster') - node1.query(f"BACKUP DATABASE keeper_backup ON CLUSTER cluster TO {backup_name} SETTINGS async = false;") + backup_name = new_backup_name("test_on_cluster") + node1.query( + f"BACKUP DATABASE keeper_backup ON CLUSTER cluster TO {backup_name} SETTINGS async = false;" + ) node1.query("DROP DATABASE keeper_backup ON CLUSTER cluster SYNC;") @@ -91,21 +108,27 @@ def apply_for_all_nodes(f): def change_keeper_map_prefix(node): node.replace_config( - "/etc/clickhouse-server/config.d/keeper_map_path_prefix.xml", """ + "/etc/clickhouse-server/config.d/keeper_map_path_prefix.xml", + """ /different_path/keeper_map -""") +""", + ) apply_for_all_nodes(lambda node: node.stop_clickhouse()) apply_for_all_nodes(change_keeper_map_prefix) apply_for_all_nodes(lambda node: node.start_clickhouse()) - node1.query(f"RESTORE DATABASE keeper_backup ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;") - + node1.query( + f"RESTORE DATABASE keeper_backup ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;" + ) + verify_data() - + node1.query("DROP TABLE keeper_backup.keeper3 ON CLUSTER cluster SYNC;") - node1.query(f"RESTORE TABLE keeper_backup.keeper3 ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;") + node1.query( + f"RESTORE TABLE keeper_backup.keeper3 ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;" + ) - verify_data() \ No newline at end of file + verify_data() From 124af73f1d51dd52b26ac6cb697c34a548dff29e Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Thu, 9 Nov 2023 17:54:45 +0100 Subject: [PATCH 038/274] Add support of arbitrary types to concat --- src/Functions/concat.cpp | 35 ++++++++++++++------------- src/Functions/concatWithSeparator.cpp | 2 +- src/Functions/formatString.cpp | 2 +- src/Functions/formatString.h | 2 +- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 9eb222d8c093..350cbee58a34 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -56,18 +57,6 @@ class ConcatImpl : public IFunction getName(), arguments.size()); - for (const auto arg_idx : collections::range(0, arguments.size())) - { - const auto * arg = arguments[arg_idx].get(); - if (!isStringOrFixedString(arg)) - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of argument {} of function {}", - arg->getName(), - arg_idx + 1, - getName()); - } - return std::make_shared(); } @@ -76,7 +65,7 @@ class ConcatImpl : public IFunction /// Format function is not proven to be faster for two arguments. /// Actually there is overhead of 2 to 5 extra instructions for each string for checking empty strings in FormatImpl. /// Though, benchmarks are really close, for most examples we saw executeBinary is slightly faster (0-3%). - /// For 3 and more arguments FormatImpl is much faster (up to 50-60%). + /// For 3 and more arguments FormatStringImpl is much faster (up to 50-60%). if (arguments.size() == 2) return executeBinary(arguments, input_rows_count); else @@ -107,6 +96,7 @@ class ConcatImpl : public IFunction else { /// Fallback: use generic implementation for not very important cases. + /// Concat of arbitrary types also goes here. return executeFormatImpl(arguments, input_rows_count); } @@ -145,8 +135,18 @@ class ConcatImpl : public IFunction constant_strings[i] = const_col->getValue(); } else - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", - column->getName(), getName()); + { + // An arbitrary type argument: converting it to a StringColumn as if `toString` was called + ColumnsWithTypeAndName args; + args.emplace_back(column, arguments[i].type, "tmp"); + const ColumnPtr converted_col_ptr = ConvertImplGenericToString::execute( + args, std::make_shared(), column->size()); + const ColumnString * converted_col_str = assert_cast(converted_col_ptr.get()); + // Same as the normal `ColumnString` branch + has_column_string = true; + data[i] = &converted_col_str->getChars(); + offsets[i] = &converted_col_str->getOffsets(); + } } String pattern; @@ -155,7 +155,7 @@ class ConcatImpl : public IFunction for (size_t i = 0; i < num_arguments; ++i) pattern += "{}"; - FormatImpl::formatExecute( + FormatStringImpl::formatExecute( has_column_string, has_column_fixed_string, std::move(pattern), @@ -185,7 +185,8 @@ using FunctionConcat = ConcatImpl; using FunctionConcatAssumeInjective = ConcatImpl; -/// Also works with arrays. +/// Works with arrays via `arrayConcat`, maps via `mapConcat`, and tuples via `tupleConcat`. +/// Additionally, allows concatenation of arbitrary types that can be cast to string using the corresponding default serialization. class ConcatOverloadResolver : public IFunctionOverloadResolver { public: diff --git a/src/Functions/concatWithSeparator.cpp b/src/Functions/concatWithSeparator.cpp index bfd1bc392db1..f0a7afbbaa7a 100644 --- a/src/Functions/concatWithSeparator.cpp +++ b/src/Functions/concatWithSeparator.cpp @@ -122,7 +122,7 @@ class ConcatWithSeparatorImpl : public IFunction for (size_t i = 0; i < num_args; ++i) pattern += "{}"; - FormatImpl::formatExecute( + FormatStringImpl::formatExecute( has_column_string, has_column_fixed_string, std::move(pattern), diff --git a/src/Functions/formatString.cpp b/src/Functions/formatString.cpp index ee6e26b775ad..8e0b3a238cb2 100644 --- a/src/Functions/formatString.cpp +++ b/src/Functions/formatString.cpp @@ -110,7 +110,7 @@ class FormatFunction : public IFunction column->getName(), getName()); } - FormatImpl::formatExecute( + FormatStringImpl::formatExecute( has_column_string, has_column_fixed_string, std::move(pattern), diff --git a/src/Functions/formatString.h b/src/Functions/formatString.h index 44fbdac93786..30149e9a5b0e 100644 --- a/src/Functions/formatString.h +++ b/src/Functions/formatString.h @@ -18,7 +18,7 @@ namespace DB { -struct FormatImpl +struct FormatStringImpl { static constexpr size_t right_padding = 15; From 5fea9f9dc6fd33baf332affc458d2bbbd82f4d0f Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 10 Nov 2023 09:25:57 +0000 Subject: [PATCH 039/274] Small fixes --- src/Common/ZooKeeper/ZooKeeper.h | 19 +++++++++ .../ZooKeeper/ZooKeeperWithFaultInjection.h | 5 +++ src/Storages/StorageKeeperMap.cpp | 5 ++- .../02911_backup_restore_keeper_map.sh | 41 ++++++++++--------- 4 files changed, 48 insertions(+), 22 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index c41d1d8dbab7..785842b94bda 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -135,6 +135,16 @@ struct MultiReadResponses responses); } + /// If Keeper/ZooKeeper doesn't support MultiRead feature we will dispatch + /// asynchronously all the read requests separately + /// Sometimes it's important to process all requests instantly + /// e.g. we want to trigger exceptions while we are in the ZK client retry loop + void waitForResponses() + { + if (auto * responses_with_futures = std::get_if(&responses)) + responses_with_futures->waitForResponses(); + } + private: using RegularResponses = std::vector; using FutureResponses = std::vector>; @@ -158,6 +168,15 @@ struct MultiReadResponses return *cached_responses[index]; } + void waitForResponses() + { + for (size_t i = 0; i < size(); ++i) + { + if (!cached_responses[i].has_value()) + cached_responses[i] = future_responses[i].get(); + } + } + size_t size() const { return future_responses.size(); } }; diff --git a/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h b/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h index 4887e896e9b9..be4642c2988a 100644 --- a/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h +++ b/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.h @@ -242,6 +242,11 @@ class ZooKeeperWithFaultInjection return access("get", !paths.empty() ? paths.front() : "", [&]() { return keeper->get(paths); }); } + zkutil::ZooKeeper::MultiTryGetResponse tryGet(const std::vector & paths) + { + return access("tryGet", !paths.empty() ? paths.front() : "", [&]() { return keeper->tryGet(paths); }); + } + bool exists(const std::string & path, Coordination::Stat * stat = nullptr, const zkutil::EventPtr & watch = nullptr) { return access("exists", path, [&]() { return keeper->exists(path, stat, watch); }); diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index 15ebc4d92d19..3032973c4114 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -703,13 +703,14 @@ class KeeperMapBackup : public IBackupEntriesLazyBatch, boost::noncopyable for (const auto & key : keys) keys_full_path.push_back(data_zookeeper_path / key); - zkutil::ZooKeeper::MultiGetResponse data; + zkutil::ZooKeeper::MultiTryGetResponse data; auto holder = with_retries->createRetriesControlHolder("getKeeperMapDataKeys"); holder.retries_ctl.retryLoop( [&, &zk = holder.faulty_zookeeper] { with_retries->renewZooKeeper(zk); - data = zk->get(keys_full_path); + data = zk->tryGet(keys_full_path); + data.waitForResponses(); }); for (size_t i = 0; i < keys.size(); ++i) diff --git a/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh b/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh index 6c463beb2217..ae7c22f68205 100755 --- a/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh +++ b/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh @@ -4,44 +4,45 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh +database_name="$CLICKHOUSE_DATABASE"_02911_keeper_map $CLICKHOUSE_CLIENT -nm -q " - DROP DATABASE IF EXISTS 02911_keeper_map; - CREATE DATABASE 02911_keeper_map; - CREATE TABLE 02911_keeper_map.02911_backup_restore_keeper_map1 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911') PRIMARY KEY key; - CREATE TABLE 02911_keeper_map.02911_backup_restore_keeper_map2 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911') PRIMARY KEY key; - CREATE TABLE 02911_keeper_map.02911_backup_restore_keeper_map3 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911_different') PRIMARY KEY key; - - INSERT INTO 02911_keeper_map.02911_backup_restore_keeper_map2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5000; - INSERT INTO 02911_keeper_map.02911_backup_restore_keeper_map3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 3000; + DROP DATABASE IF EXISTS $database_name; + CREATE DATABASE $database_name; + CREATE TABLE $database_name.02911_backup_restore_keeper_map1 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911') PRIMARY KEY key; + CREATE TABLE $database_name.02911_backup_restore_keeper_map2 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911') PRIMARY KEY key; + CREATE TABLE $database_name.02911_backup_restore_keeper_map3 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911_different') PRIMARY KEY key; + + INSERT INTO $database_name.02911_backup_restore_keeper_map2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5000; + INSERT INTO $database_name.02911_backup_restore_keeper_map3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 3000; " -backup_path="$CLICKHOUSE_DATABASE/02911_keeper_map" +backup_path="$database_name" for i in $(seq 1 3); do - $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" + $CLICKHOUSE_CLIENT -q "SELECT count() FROM $database_name.02911_backup_restore_keeper_map$i;" done -$CLICKHOUSE_CLIENT -q "BACKUP DATABASE 02911_keeper_map TO Disk('backups', '$backup_path');" > /dev/null +$CLICKHOUSE_CLIENT -q "BACKUP DATABASE $database_name TO Disk('backups', '$backup_path');" > /dev/null -$CLICKHOUSE_CLIENT -q "DROP DATABASE 02911_keeper_map SYNC;" +$CLICKHOUSE_CLIENT -q "DROP DATABASE $database_name SYNC;" for i in $(seq 1 3); do - $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" 2>&1 | grep -Fq "UNKNOWN_DATABASE" && echo 'OK' || echo 'ERROR' + $CLICKHOUSE_CLIENT -q "SELECT count() FROM $database_name.02911_backup_restore_keeper_map$i;" 2>&1 | grep -Fq "UNKNOWN_DATABASE" && echo 'OK' || echo 'ERROR' done -$CLICKHOUSE_CLIENT -q "RESTORE DATABASE 02911_keeper_map FROM Disk('backups', '$backup_path');" > /dev/null +$CLICKHOUSE_CLIENT -q "RESTORE DATABASE $database_name FROM Disk('backups', '$backup_path');" > /dev/null for i in $(seq 1 3); do - $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" + $CLICKHOUSE_CLIENT -q "SELECT count() FROM $database_name.02911_backup_restore_keeper_map$i;" done -$CLICKHOUSE_CLIENT -q "DROP TABLE 02911_keeper_map.02911_backup_restore_keeper_map3 SYNC;" +$CLICKHOUSE_CLIENT -q "DROP TABLE $database_name.02911_backup_restore_keeper_map3 SYNC;" -$CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map3;" 2>&1 | grep -Fq "UNKNOWN_TABLE" && echo 'OK' || echo 'ERROR' +$CLICKHOUSE_CLIENT -q "SELECT count() FROM $database_name.02911_backup_restore_keeper_map3;" 2>&1 | grep -Fq "UNKNOWN_TABLE" && echo 'OK' || echo 'ERROR' -$CLICKHOUSE_CLIENT -q "RESTORE TABLE 02911_keeper_map.02911_backup_restore_keeper_map3 FROM Disk('backups', '$backup_path');" > /dev/null +$CLICKHOUSE_CLIENT -q "RESTORE TABLE $database_name.02911_backup_restore_keeper_map3 FROM Disk('backups', '$backup_path');" > /dev/null for i in $(seq 1 3); do - $CLICKHOUSE_CLIENT -q "SELECT count() FROM 02911_keeper_map.02911_backup_restore_keeper_map$i;" + $CLICKHOUSE_CLIENT -q "SELECT count() FROM $database_name.02911_backup_restore_keeper_map$i;" done -$CLICKHOUSE_CLIENT -q "DROP DATABASE 02911_keeper_map SYNC;" \ No newline at end of file +$CLICKHOUSE_CLIENT -q "DROP DATABASE $database_name SYNC;" \ No newline at end of file From 214ac112a75703470c6dc607299f02d58a14f093 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 10 Nov 2023 12:48:03 +0100 Subject: [PATCH 040/274] Looks fixed --- src/Interpreters/Cache/FileCache.cpp | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index ed8cc547fbc6..914292c003d6 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -361,6 +361,7 @@ void FileCache::fillHolesWithEmptyFileSegments( locked_key.getKey(), current_pos, hole_size, FileSegment::State::DETACHED, settings); file_segments.insert(it, file_segment); + ++added; } else { @@ -395,14 +396,6 @@ void FileCache::fillHolesWithEmptyFileSegments( return; } - if (file_segments_limit && file_segments.size() >= file_segments_limit) - { - std::string res; - for (const auto & f : file_segments) - res += " - " + f->range().toString(); - LOG_ERROR(log, "Limit: {}, file segments: {}, added: {}, range: {}, file_segments: {}", - file_segments_limit, file_segments.size(), added, range.toString(), res); - } chassert(!file_segments_limit || file_segments.size() < file_segments_limit); if (current_pos <= range.right) From a7fb6a30f8ac8751ce63972e22d6e829d3ecdb5d Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 10 Nov 2023 13:29:01 +0100 Subject: [PATCH 041/274] Better --- src/Interpreters/Cache/FileCache.cpp | 77 +++++++++++++------------- src/Interpreters/Cache/FileCache.h | 16 +++++- src/Interpreters/Cache/FileSegment.cpp | 1 - 3 files changed, 53 insertions(+), 41 deletions(-) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 914292c003d6..e998d2a36397 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -156,7 +156,7 @@ FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment: { auto file_segment = std::make_shared( locked_key.getKey(), range.left, range.size(), FileSegment::State::DETACHED); - return {file_segment}; + return { file_segment }; } if (locked_key.empty()) @@ -296,7 +296,6 @@ FileSegments FileCache::splitRangeIntoFileSegments( current_pos += current_file_segment_size; } - assert(file_segments.empty() || file_segments_limit > 0 || offset + size - 1 == file_segments.back()->range().right); return file_segments; } @@ -321,7 +320,7 @@ void FileCache::fillHolesWithEmptyFileSegments( assert(!file_segments.empty()); auto it = file_segments.begin(); - size_t added = 0; + size_t processed_count = 0; auto segment_range = (*it)->range(); size_t current_pos; @@ -334,12 +333,17 @@ void FileCache::fillHolesWithEmptyFileSegments( current_pos = segment_range.right + 1; ++it; - ++added; + ++processed_count; } else current_pos = range.left; - while (current_pos <= range.right && it != file_segments.end() && (!file_segments_limit || added < file_segments_limit)) + auto is_limit_reached = [&]() -> bool + { + return file_segments_limit && processed_count >= file_segments_limit; + }; + + while (current_pos <= range.right && it != file_segments.end() && !is_limit_reached()) { segment_range = (*it)->range(); @@ -347,7 +351,7 @@ void FileCache::fillHolesWithEmptyFileSegments( { current_pos = segment_range.right + 1; ++it; - ++added; + ++processed_count; continue; } @@ -361,7 +365,7 @@ void FileCache::fillHolesWithEmptyFileSegments( locked_key.getKey(), current_pos, hole_size, FileSegment::State::DETACHED, settings); file_segments.insert(it, file_segment); - ++added; + ++processed_count; } else { @@ -371,28 +375,32 @@ void FileCache::fillHolesWithEmptyFileSegments( { auto metadata_it = addFileSegment(locked_key, r.left, r.size(), FileSegment::State::EMPTY, settings, nullptr); hole.push_back(metadata_it->second->file_segment); - ++added; + ++processed_count; - if (file_segments_limit && added == file_segments_limit) - { - file_segments.splice(it, std::move(hole)); - file_segments.erase(it, file_segments.end()); - return; - } + if (is_limit_reached()) + break; } file_segments.splice(it, std::move(hole)); } + if (is_limit_reached()) + break; + current_pos = segment_range.right + 1; ++it; - ++added; + ++processed_count; } - if (file_segments_limit && added == file_segments_limit) + auto erase_unprocessed = [&]() { chassert(file_segments.size() >= file_segments_limit); file_segments.erase(it, file_segments.end()); chassert(file_segments.size() == file_segments_limit); + }; + + if (is_limit_reached()) + { + erase_unprocessed(); return; } @@ -422,16 +430,15 @@ void FileCache::fillHolesWithEmptyFileSegments( { auto metadata_it = addFileSegment(locked_key, r.left, r.size(), FileSegment::State::EMPTY, settings, nullptr); hole.push_back(metadata_it->second->file_segment); - ++added; + ++processed_count; - if (file_segments_limit && added == file_segments_limit) - { - file_segments.splice(it, std::move(hole)); - file_segments.erase(it, file_segments.end()); - return; - } + if (is_limit_reached()) + break; } file_segments.splice(it, std::move(hole)); + + if (is_limit_reached()) + erase_unprocessed(); } } } @@ -589,23 +596,16 @@ FileCache::getOrSet( fillHolesWithEmptyFileSegments( *locked_key, file_segments, range, file_segments_limit, /* fill_with_detached */false, settings); - chassert(!file_segments_limit || file_segments.size() <= file_segments_limit); - if (!file_segments.front()->range().contains(offset)) { throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected {} to include {} " "(end offset: {}, aligned offset: {}, aligned end offset: {})", file_segments.front()->range().toString(), offset, range.right, aligned_offset, aligned_end_offset); } - - chassert(file_segments_limit ? file_segments.back()->range().left <= range.right : file_segments.back()->range().contains(range.right)); } - while (file_segments_limit && file_segments.size() > file_segments_limit) - file_segments.pop_back(); - - if (file_segments.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty list of file segments for offset {}, size {} (file size: {})", offset, size, file_size); + chassert(file_segments_limit ? file_segments.back()->range().left <= range.right : file_segments.back()->range().contains(range.right)); + chassert(!file_segments_limit || file_segments.size() <= file_segments_limit); return std::make_unique(std::move(file_segments)); } @@ -625,14 +625,17 @@ FileSegmentsHolderPtr FileCache::get(const Key & key, size_t offset, size_t size auto file_segments = getImpl(*locked_key, range, file_segments_limit); if (!file_segments.empty()) { - fillHolesWithEmptyFileSegments( - *locked_key, file_segments, range, file_segments_limit, /* fill_with_detached */true, CreateFileSegmentSettings{}); - if (file_segments_limit) { - while (file_segments.size() > file_segments_limit) - file_segments.pop_back(); + chassert(file_segments.size() <= file_segments_limit); + if (file_segments.size() == file_segments_limit) + range.right = file_segments.back()->range().right; } + + fillHolesWithEmptyFileSegments( + *locked_key, file_segments, range, file_segments_limit, /* fill_with_detached */true, CreateFileSegmentSettings{}); + + chassert(!file_segments_limit || file_segments.size() <= file_segments_limit); return std::make_unique(std::move(file_segments)); } } diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index 523ff90e33e5..7485c7d2b906 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -209,18 +209,28 @@ class FileCache : private boost::noncopyable std::unique_ptr cleanup_thread; void assertInitialized() const; - void assertCacheCorrectness(); void loadMetadata(); void loadMetadataImpl(); void loadMetadataForKeys(const std::filesystem::path & keys_dir); - /// bool - if `file_segments_limit` reached or not. - FileSegments getImpl(const LockedKey & locked_key, const FileSegment::Range & range, size_t file_segments_limit) const; + /// Get all file segments from cache which intersect with `range`. + /// If `file_segments_limit` > 0, return no more than first file_segments_limit + /// file segments. + FileSegments getImpl( + const LockedKey & locked_key, + const FileSegment::Range & range, + size_t file_segments_limit) const; + /// Split range into subranges by max_file_segment_size, + /// each subrange size must be less or equal to max_file_segment_size. std::vector splitRange(size_t offset, size_t size); + /// Split range into subranges by max_file_segment_size (same as in splitRange()) + /// and create a new file segment for each subrange. + /// If `file_segments_limit` > 0, create no more than first file_segments_limit + /// file segments. FileSegments splitRangeIntoFileSegments( LockedKey & locked_key, size_t offset, diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 591342299974..794dd663beef 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -933,7 +933,6 @@ FileSegmentsHolder::FileSegmentsHolder(FileSegments && file_segments_) FileSegmentsHolder::~FileSegmentsHolder() { - ProfileEvents::increment(ProfileEvents::FilesystemCacheUnusedHoldFileSegments, file_segments.size()); ProfileEventTimeIncrement watch(ProfileEvents::FileSegmentHolderCompleteMicroseconds); ProfileEvents::increment(ProfileEvents::FilesystemCacheUnusedHoldFileSegments, file_segments.size()); From 356ae52e9b122545ea9a26a18ccce0e5311d4df5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 11 Nov 2023 08:46:10 +0100 Subject: [PATCH 042/274] Enable access and named collections control by default --- .../config/users.d/perf-comparison-tweaks-users.xml | 5 ----- programs/server/embedded.xml | 2 ++ programs/server/users.xml | 5 ++++- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml b/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml index cb591f1a184b..e780a99ecde7 100644 --- a/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml +++ b/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml @@ -34,9 +34,4 @@ 0 - - - 1 - - diff --git a/programs/server/embedded.xml b/programs/server/embedded.xml index c2336e0d5826..9311749a1734 100644 --- a/programs/server/embedded.xml +++ b/programs/server/embedded.xml @@ -23,7 +23,9 @@ default default + 1 + 1 diff --git a/programs/server/users.xml b/programs/server/users.xml index fbb5a2c228f7..57bc6309a54c 100644 --- a/programs/server/users.xml +++ b/programs/server/users.xml @@ -85,7 +85,10 @@ default - + 1 + + + 1 0 + 0 20000 @@ -33,6 +34,7 @@ true 1 + 0 1 20000 diff --git a/tests/integration/test_storage_s3/configs/defaultS3.xml b/tests/integration/test_storage_s3/configs/defaultS3.xml index 37454ef6781f..7dac6d9fbb57 100644 --- a/tests/integration/test_storage_s3/configs/defaultS3.xml +++ b/tests/integration/test_storage_s3/configs/defaultS3.xml @@ -1,9 +1,4 @@ - - - 5 - - http://resolver:8080 diff --git a/tests/integration/test_storage_s3/configs/s3_retry.xml b/tests/integration/test_storage_s3/configs/s3_retry.xml index 727e23273cf3..581fc44c8d4f 100644 --- a/tests/integration/test_storage_s3/configs/s3_retry.xml +++ b/tests/integration/test_storage_s3/configs/s3_retry.xml @@ -1,7 +1,7 @@ - 5 + 10 From 45de9beab4231a806ea247adef0a1fc5180748ba Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 8 Nov 2023 13:19:28 +0100 Subject: [PATCH 053/274] set new timeout for session from connection pool --- base/poco/Net/src/HTTPSession.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index d2663baaf9fe..d30f5590280b 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -94,8 +94,22 @@ void HTTPSession::setTimeout(const Poco::Timespan& timeout) void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco::Timespan& sendTimeout, const Poco::Timespan& receiveTimeout) { _connectionTimeout = connectionTimeout; - _sendTimeout = sendTimeout; - _receiveTimeout = receiveTimeout; + + if (_sendTimeout != sendTimeout) + { + _sendTimeout = sendTimeout; + + if (connected()) + _socket.setSendTimeout(_sendTimeout); + } + + if (_receiveTimeout != receiveTimeout) + { + _receiveTimeout = receiveTimeout; + + if (connected()) + _socket.setReceiveTimeout(_receiveTimeout); + } } From be01a5cd3e07eeba990a8dcc3e69e62f3492d05e Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 8 Nov 2023 17:32:06 +0100 Subject: [PATCH 054/274] turn off agressive timeouts for heavy requests --- src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp | 6 +++++- src/Disks/ObjectStorages/S3/S3ObjectStorage.h | 1 + src/IO/S3/Client.cpp | 9 ++++++--- src/IO/S3/Client.h | 6 ++---- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index b36185249af7..aa4bcd7fbad8 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -537,7 +537,11 @@ std::unique_ptr S3ObjectStorage::cloneObjectStorage( } S3ObjectStorage::Clients::Clients(std::shared_ptr client_, const S3ObjectStorageSettings & settings) - : client(std::move(client_)), client_with_long_timeout(client->clone(std::nullopt, settings.request_settings.long_request_timeout_ms)) {} + : client(std::move(client_)) + , client_with_long_timeout(client->clone( + /*override_aggressive_timeouts*/ false, + settings.request_settings.long_request_timeout_ms)) +{} ObjectStorageKey S3ObjectStorage::generateObjectKeyForPath(const std::string &) const { diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index b1b3fb22366f..37e491e21dc8 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -184,6 +184,7 @@ class S3ObjectStorage : public IObjectStorage std::string bucket; String object_key_prefix; + MultiVersion clients; MultiVersion s3_settings; S3Capabilities s3_capabilities; diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 4250342c49fb..12a0cb8f93ce 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -119,14 +119,17 @@ std::unique_ptr Client::create( } std::unique_ptr Client::clone( - std::optional> override_retry_strategy, + std::optional override_aggressive_timeouts, std::optional override_request_timeout_ms) const { PocoHTTPClientConfiguration new_configuration = client_configuration; - if (override_retry_strategy.has_value()) - new_configuration.retryStrategy = *override_retry_strategy; + if (override_request_timeout_ms.has_value()) new_configuration.requestTimeoutMs = *override_request_timeout_ms; + + if (override_aggressive_timeouts.has_value()) + new_configuration.s3_aggressive_timeouts = *override_aggressive_timeouts; + return std::unique_ptr(new Client(*this, new_configuration)); } diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index 48310bc21af9..81ab3854d3d2 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -119,13 +119,11 @@ class Client : private Aws::S3::S3Client bool use_virtual_addressing); /// Create a client with adjusted settings: - /// * override_retry_strategy can be used to disable retries to avoid nested retries when we have - /// a retry loop outside of S3 client. Specifically, for read and write buffers. Currently not - /// actually used. /// * override_request_timeout_ms is used to increase timeout for CompleteMultipartUploadRequest /// because it often sits idle for 10 seconds: https://github.com/ClickHouse/ClickHouse/pull/42321 + /// * s3_aggressive_timeouts is used to turn off s3_aggressive_timeouts feature for CompleteMultipartUploadRequest std::unique_ptr clone( - std::optional> override_retry_strategy = std::nullopt, + std::optional override_aggressive_timeouts = std::nullopt, std::optional override_request_timeout_ms = std::nullopt) const; Client & operator=(const Client &) = delete; From 27fb25d056c420bca141ce2ecd83868d15fd07ef Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 9 Nov 2023 13:10:52 +0100 Subject: [PATCH 055/274] alter the naming, fix client_with_long_timeout in s3 storage --- src/Core/Settings.h | 2 +- src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp | 2 +- src/Disks/ObjectStorages/S3/diskSettings.cpp | 4 ++-- src/IO/S3/Client.cpp | 8 ++++---- src/IO/S3/Client.h | 4 ++-- src/IO/S3/PocoHTTPClient.cpp | 8 ++++---- src/IO/S3/PocoHTTPClient.h | 6 +++--- src/Storages/StorageS3.cpp | 2 +- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b1459b6f3289..3f80c83ff5fb 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -94,7 +94,7 @@ class IColumn; M(UInt64, s3_max_put_rps, 0, "Limit on S3 PUT request per second rate before throttling. Zero means unlimited.", 0) \ M(UInt64, s3_max_put_burst, 0, "Max number of requests that can be issued simultaneously before hitting request per second limit. By default (0) equals to `s3_max_put_rps`", 0) \ M(UInt64, s3_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ - M(Bool, s3_aggressive_timeouts, true, "When aggressive timeouts are enabled first two attempts are made with low receive and send timeout", 0) \ + M(Bool, s3_use_adaptive_timeouts, true, "When aggressive timeouts are enabled first two attempts are made with low receive and send timeout", 0) \ M(UInt64, azure_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ M(Bool, s3_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables.", 0) \ M(Bool, azure_truncate_on_insert, false, "Enables or disables truncate before insert in azure engine tables.", 0) \ diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index aa4bcd7fbad8..8a46bfd59d1f 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -539,7 +539,7 @@ std::unique_ptr S3ObjectStorage::cloneObjectStorage( S3ObjectStorage::Clients::Clients(std::shared_ptr client_, const S3ObjectStorageSettings & settings) : client(std::move(client_)) , client_with_long_timeout(client->clone( - /*override_aggressive_timeouts*/ false, + /*override_use_adaptive_timeouts*/ false, settings.request_settings.long_request_timeout_ms)) {} diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 43618c647762..573fa744ce65 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -67,8 +67,8 @@ std::unique_ptr getClient( config_prefix + ".http_keep_alive_timeout_ms", DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT * 1000); client_configuration.http_connection_pool_size = config.getUInt(config_prefix + ".http_connection_pool_size", 1000); client_configuration.wait_on_pool_size_limit = false; - client_configuration.s3_aggressive_timeouts = config.getUInt( - config_prefix + ".aggressive_timeouts", client_configuration.s3_aggressive_timeouts); + client_configuration.s3_use_adaptive_timeouts = config.getUInt( + config_prefix + ".use_adaptive_timeouts", client_configuration.s3_use_adaptive_timeouts); /* * Override proxy configuration for backwards compatibility with old configuration format. diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 12a0cb8f93ce..90806852c1e4 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -119,7 +119,7 @@ std::unique_ptr Client::create( } std::unique_ptr Client::clone( - std::optional override_aggressive_timeouts, + std::optional override_use_adaptive_timeouts, std::optional override_request_timeout_ms) const { PocoHTTPClientConfiguration new_configuration = client_configuration; @@ -127,8 +127,8 @@ std::unique_ptr Client::clone( if (override_request_timeout_ms.has_value()) new_configuration.requestTimeoutMs = *override_request_timeout_ms; - if (override_aggressive_timeouts.has_value()) - new_configuration.s3_aggressive_timeouts = *override_aggressive_timeouts; + if (override_use_adaptive_timeouts.has_value()) + new_configuration.s3_use_adaptive_timeouts = *override_use_adaptive_timeouts; return std::unique_ptr(new Client(*this, new_configuration)); } @@ -908,7 +908,7 @@ PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT s3_retry_attempts, enable_s3_requests_logging, for_disk_s3, - context->getGlobalContext()->getSettingsRef().s3_aggressive_timeouts, + context->getGlobalContext()->getSettingsRef().s3_use_adaptive_timeouts, get_request_throttler, put_request_throttler, error_report); diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index 81ab3854d3d2..be7235eb9f16 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -121,9 +121,9 @@ class Client : private Aws::S3::S3Client /// Create a client with adjusted settings: /// * override_request_timeout_ms is used to increase timeout for CompleteMultipartUploadRequest /// because it often sits idle for 10 seconds: https://github.com/ClickHouse/ClickHouse/pull/42321 - /// * s3_aggressive_timeouts is used to turn off s3_aggressive_timeouts feature for CompleteMultipartUploadRequest + /// * s3_use_adaptive_timeouts is used to turn off s3_use_adaptive_timeouts feature for CompleteMultipartUploadRequest std::unique_ptr clone( - std::optional override_aggressive_timeouts = std::nullopt, + std::optional override_use_adaptive_timeouts = std::nullopt, std::optional override_request_timeout_ms = std::nullopt) const; Client & operator=(const Client &) = delete; diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 08ba04ee8759..f783a8868775 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -100,7 +100,7 @@ PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( unsigned int s3_retry_attempts_, bool enable_s3_requests_logging_, bool for_disk_s3_, - bool s3_aggressive_timeouts_, + bool s3_use_adaptive_timeouts_, const ThrottlerPtr & get_request_throttler_, const ThrottlerPtr & put_request_throttler_, std::function error_report_) @@ -113,7 +113,7 @@ PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( , for_disk_s3(for_disk_s3_) , get_request_throttler(get_request_throttler_) , put_request_throttler(put_request_throttler_) - , s3_aggressive_timeouts(s3_aggressive_timeouts_) + , s3_use_adaptive_timeouts(s3_use_adaptive_timeouts_) , error_report(error_report_) { } @@ -160,7 +160,7 @@ PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & client_config Poco::Timespan(client_configuration.http_keep_alive_timeout_ms * 1000))) /// flag indicating whether keep-alive is enabled is set to each session upon creation , remote_host_filter(client_configuration.remote_host_filter) , s3_max_redirects(client_configuration.s3_max_redirects) - , s3_aggressive_timeouts(client_configuration.s3_aggressive_timeouts) + , s3_use_adaptive_timeouts(client_configuration.s3_use_adaptive_timeouts) , enable_s3_requests_logging(client_configuration.enable_s3_requests_logging) , for_disk_s3(client_configuration.for_disk_s3) , get_request_throttler(client_configuration.get_request_throttler) @@ -295,7 +295,7 @@ UInt32 extractAttempt(const Aws::String & request_info) ConnectionTimeouts PocoHTTPClient::getTimeouts(Aws::Http::HttpRequest & request) const { - if (!s3_aggressive_timeouts) + if (!s3_use_adaptive_timeouts) return timeouts; const auto & request_info = request.GetHeaderValue(Aws::Http::SDK_REQUEST_HEADER); diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 6eeff4315697..9ba5f4ffe640 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -55,7 +55,7 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration size_t http_connection_pool_size = 0; /// See PoolBase::BehaviourOnLimit bool wait_on_pool_size_limit = true; - bool s3_aggressive_timeouts = false; + bool s3_use_adaptive_timeouts = false; std::function error_report; @@ -70,7 +70,7 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration unsigned int s3_retry_attempts, bool enable_s3_requests_logging_, bool for_disk_s3_, - bool s3_aggressive_timeouts_, + bool s3_use_adaptive_timeouts_, const ThrottlerPtr & get_request_throttler_, const ThrottlerPtr & put_request_throttler_, std::function error_report_ @@ -182,7 +182,7 @@ class PocoHTTPClient : public Aws::Http::HttpClient ConnectionTimeouts timeouts; const RemoteHostFilter & remote_host_filter; unsigned int s3_max_redirects; - bool s3_aggressive_timeouts = false; + bool s3_use_adaptive_timeouts = false; bool enable_s3_requests_logging; bool for_disk_s3; diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 63ed84680c98..231efb87e872 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -1330,7 +1330,7 @@ void StorageS3::Configuration::connect(ContextPtr context) auth_settings.no_sign_request.value_or(context->getConfigRef().getBool("s3.no_sign_request", false)), }); - client_with_long_timeout = client->clone(std::nullopt, request_settings.long_request_timeout_ms); + client_with_long_timeout = client->clone(/*override_use_adaptive_timeouts*/ false, request_settings.long_request_timeout_ms); } void StorageS3::processNamedCollectionResult(StorageS3::Configuration & configuration, const NamedCollection & collection) From bb0b6afe14319799028fbd8483b3bc4042e6f951 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 9 Nov 2023 13:12:38 +0100 Subject: [PATCH 056/274] reduce cuncurrent request number to the minio in test_storage_s3 --- tests/integration/test_storage_s3/configs/s3_retry.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_storage_s3/configs/s3_retry.xml b/tests/integration/test_storage_s3/configs/s3_retry.xml index 581fc44c8d4f..b7a7bbc8a9bd 100644 --- a/tests/integration/test_storage_s3/configs/s3_retry.xml +++ b/tests/integration/test_storage_s3/configs/s3_retry.xml @@ -2,6 +2,7 @@ 10 + 5 From 76d11687a76ede0a0fd080fd76ff04da501e7af6 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 9 Nov 2023 13:12:56 +0100 Subject: [PATCH 057/274] adjuct docs --- docs/en/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 306529c4b96f..34ed85c773a7 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -4821,7 +4821,7 @@ When set to `false` the metadata files are written with the previous format vers Default value: `false`. -## s3_aggressive_timeouts {#s3_aggressive_timeouts} +## s3_use_adaptive_timeouts {#s3_use_adaptive_timeouts} When set to `true` than for all s3 requests first two attempts are made with low send and receive timeouts. When set to `false` than all attempts are made with identical timeouts. From 8d36fd6e54cc66ab826d80fb6c4ec867fad4b731 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 9 Nov 2023 23:54:31 +0100 Subject: [PATCH 058/274] get rid off of client_with_long_timeout_ptr --- src/Backups/BackupIO_S3.cpp | 5 +- src/Coordination/KeeperSnapshotManagerS3.cpp | 1 - src/Core/Settings.h | 2 +- .../ObjectStorages/S3/S3ObjectStorage.cpp | 62 ++++++------------- src/Disks/ObjectStorages/S3/S3ObjectStorage.h | 14 +---- src/Disks/ObjectStorages/S3/diskSettings.cpp | 2 +- src/IO/ConnectionTimeouts.cpp | 53 ++++++++++++---- src/IO/ConnectionTimeouts.h | 3 +- src/IO/S3/Client.cpp | 14 +---- src/IO/S3/Client.h | 8 +-- src/IO/S3/PocoHTTPClient.cpp | 2 +- src/IO/S3/copyS3File.cpp | 26 +++----- src/IO/S3/copyS3File.h | 7 --- src/IO/S3/tests/gtest_aws_s3_client.cpp | 1 - src/IO/WriteBufferFromS3.cpp | 4 +- src/IO/WriteBufferFromS3.h | 3 - src/IO/tests/gtest_writebuffer_s3.cpp | 1 - src/Storages/StorageS3.cpp | 3 - src/Storages/StorageS3.h | 1 - src/Storages/StorageS3Settings.h | 3 +- 20 files changed, 83 insertions(+), 132 deletions(-) diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 0b7006659887..4f83158d07d8 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -169,7 +169,6 @@ void BackupReaderS3::copyFileToDisk(const String & path_in_backup, size_t file_s blob_path.size(), mode); copyS3File( - client, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, @@ -231,7 +230,6 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src { LOG_TRACE(log, "Copying file {} from disk {} to S3", src_path, src_disk->getName()); copyS3File( - client, client, /* src_bucket */ blob_path[1], /* src_key= */ blob_path[0], @@ -253,7 +251,7 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src void BackupWriterS3::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) { - copyDataToS3File(create_read_buffer, start_pos, length, client, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, s3_settings.request_settings, {}, + copyDataToS3File(create_read_buffer, start_pos, length, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, s3_settings.request_settings, {}, threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterS3")); } @@ -283,7 +281,6 @@ std::unique_ptr BackupWriterS3::writeFile(const String & file_name) { return std::make_unique( client, - client, // already has long timeout s3_uri.bucket, fs::path(s3_uri.key) / file_name, DBMS_DEFAULT_BUFFER_SIZE, diff --git a/src/Coordination/KeeperSnapshotManagerS3.cpp b/src/Coordination/KeeperSnapshotManagerS3.cpp index 302e05c84184..bedde0d7b39c 100644 --- a/src/Coordination/KeeperSnapshotManagerS3.cpp +++ b/src/Coordination/KeeperSnapshotManagerS3.cpp @@ -148,7 +148,6 @@ void KeeperSnapshotManagerS3::uploadSnapshotImpl(const SnapshotFileInfo & snapsh const auto create_writer = [&](const auto & key) { return WriteBufferFromS3( - s3_client->client, s3_client->client, s3_client->uri.bucket, key, diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 3f80c83ff5fb..34547aded9c3 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -105,7 +105,7 @@ class IColumn; M(Bool, s3_allow_parallel_part_upload, true, "Use multiple threads for s3 multipart upload. It may lead to slightly higher memory usage", 0) \ M(Bool, s3_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \ M(UInt64, s3_retry_attempts, 100, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries", 0) \ - M(UInt64, s3_request_timeout_ms, 3000, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \ + M(UInt64, s3_request_timeout_ms, 30000, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \ M(UInt64, s3_http_connection_pool_size, 1000, "How many reusable open connections to keep per S3 endpoint. Only applies to the S3 table engine and table function, not to S3 disks (for disks, use disk config instead). Global setting, can only be set in config, overriding it per session or per query has no effect.", 0) \ M(Bool, enable_s3_requests_logging, false, "Enable very explicit logging of S3 requests. Makes sense for debug only.", 0) \ M(String, s3queue_default_zookeeper_path, "/clickhouse/s3queue/", "Default zookeeper path prefix for S3Queue engine", 0) \ diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 8a46bfd59d1f..75dd405f6aaf 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -153,7 +153,7 @@ class S3IteratorAsync final : public IObjectStorageIteratorAsync bool S3ObjectStorage::exists(const StoredObject & object) const { auto settings_ptr = s3_settings.get(); - return S3::objectExists(*clients.get()->client, bucket, object.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + return S3::objectExists(*client.get(), bucket, object.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); } std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT @@ -172,7 +172,7 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT (const std::string & path, size_t read_until_position) -> std::unique_ptr { return std::make_unique( - clients.get()->client, + client.get(), bucket, path, version_id, @@ -222,7 +222,7 @@ std::unique_ptr S3ObjectStorage::readObject( /// NOLINT { auto settings_ptr = s3_settings.get(); return std::make_unique( - clients.get()->client, + client.get(), bucket, object.remote_path, version_id, @@ -247,10 +247,8 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN if (write_settings.s3_allow_parallel_part_upload) scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "VFSWrite"); - auto clients_ = clients.get(); return std::make_unique( - clients_->client, - clients_->client_with_long_timeout, + client.get(), bucket, object.remote_path, buf_size, @@ -264,15 +262,12 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN ObjectStorageIteratorPtr S3ObjectStorage::iterate(const std::string & path_prefix) const { auto settings_ptr = s3_settings.get(); - auto client_ptr = clients.get()->client; - - return std::make_shared(bucket, path_prefix, client_ptr, settings_ptr->list_object_keys_size); + return std::make_shared(bucket, path_prefix, client.get(), settings_ptr->list_object_keys_size); } void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const { auto settings_ptr = s3_settings.get(); - auto client_ptr = clients.get()->client; S3::ListObjectsV2Request request; request.SetBucket(bucket); @@ -287,7 +282,7 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet { ProfileEvents::increment(ProfileEvents::S3ListObjects); ProfileEvents::increment(ProfileEvents::DiskS3ListObjects); - outcome = client_ptr->ListObjectsV2(request); + outcome = client.get()->ListObjectsV2(request); throwIfError(outcome); auto result = outcome.GetResult(); @@ -318,14 +313,12 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet void S3ObjectStorage::removeObjectImpl(const StoredObject & object, bool if_exists) { - auto client_ptr = clients.get()->client; - ProfileEvents::increment(ProfileEvents::S3DeleteObjects); ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); S3::DeleteObjectRequest request; request.SetBucket(bucket); request.SetKey(object.remote_path); - auto outcome = client_ptr->DeleteObject(request); + auto outcome = client.get()->DeleteObject(request); throwIfUnexpectedError(outcome, if_exists); @@ -344,7 +337,6 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e } else { - auto client_ptr = clients.get()->client; auto settings_ptr = s3_settings.get(); size_t chunk_size_limit = settings_ptr->objects_chunk_size_to_delete; @@ -373,7 +365,7 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e S3::DeleteObjectsRequest request; request.SetBucket(bucket); request.SetDelete(delkeys); - auto outcome = client_ptr->DeleteObjects(request); + auto outcome = client.get()->DeleteObjects(request); throwIfUnexpectedError(outcome, if_exists); @@ -405,7 +397,7 @@ void S3ObjectStorage::removeObjectsIfExist(const StoredObjects & objects) std::optional S3ObjectStorage::tryGetObjectMetadata(const std::string & path) const { auto settings_ptr = s3_settings.get(); - auto object_info = S3::getObjectInfo(*clients.get()->client, bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true, /* throw_on_error= */ false); + auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true, /* throw_on_error= */ false); if (object_info.size == 0 && object_info.last_modification_time == 0 && object_info.metadata.empty()) return {}; @@ -421,7 +413,7 @@ std::optional S3ObjectStorage::tryGetObjectMetadata(const std::s ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path) const { auto settings_ptr = s3_settings.get(); - auto object_info = S3::getObjectInfo(*clients.get()->client, bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true); + auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true); ObjectMetadata result; result.size_bytes = object_info.size; @@ -442,12 +434,12 @@ void S3ObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT /// Shortcut for S3 if (auto * dest_s3 = dynamic_cast(&object_storage_to); dest_s3 != nullptr) { - auto clients_ = clients.get(); + auto client_ = client.get(); auto settings_ptr = s3_settings.get(); - auto size = S3::getObjectSize(*clients_->client, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + auto size = S3::getObjectSize(*client_, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); auto scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "S3ObjStor_copy"); - copyS3File(clients_->client, - clients_->client_with_long_timeout, + copyS3File( + client.get(), bucket, object_from.remote_path, 0, @@ -471,12 +463,11 @@ void S3ObjectStorage::copyObject( // NOLINT const WriteSettings &, std::optional object_to_attributes) { - auto clients_ = clients.get(); + auto client_ = client.get(); auto settings_ptr = s3_settings.get(); - auto size = S3::getObjectSize(*clients_->client, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + auto size = S3::getObjectSize(*client_, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); auto scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "S3ObjStor_copy"); - copyS3File(clients_->client, - clients_->client_with_long_timeout, + copyS3File(client_, bucket, object_from.remote_path, 0, @@ -497,31 +488,25 @@ void S3ObjectStorage::setNewSettings(std::unique_ptr && void S3ObjectStorage::shutdown() { - auto clients_ptr = clients.get(); /// This call stops any next retry attempts for ongoing S3 requests. /// If S3 request is failed and the method below is executed S3 client immediately returns the last failed S3 request outcome. /// If S3 is healthy nothing wrong will be happened and S3 requests will be processed in a regular way without errors. /// This should significantly speed up shutdown process if S3 is unhealthy. - const_cast(*clients_ptr->client).DisableRequestProcessing(); - const_cast(*clients_ptr->client_with_long_timeout).DisableRequestProcessing(); + const_cast(*client.get()).DisableRequestProcessing(); } void S3ObjectStorage::startup() { - auto clients_ptr = clients.get(); - /// Need to be enabled if it was disabled during shutdown() call. - const_cast(*clients_ptr->client).EnableRequestProcessing(); - const_cast(*clients_ptr->client_with_long_timeout).EnableRequestProcessing(); + const_cast(*client.get()).EnableRequestProcessing(); } void S3ObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) { auto new_s3_settings = getSettings(config, config_prefix, context); auto new_client = getClient(config, config_prefix, context, *new_s3_settings); - auto new_clients = std::make_unique(std::move(new_client), *new_s3_settings); s3_settings.set(std::move(new_s3_settings)); - clients.set(std::move(new_clients)); + client.set(std::move(new_client)); } std::unique_ptr S3ObjectStorage::cloneObjectStorage( @@ -536,13 +521,6 @@ std::unique_ptr S3ObjectStorage::cloneObjectStorage( endpoint, object_key_prefix); } -S3ObjectStorage::Clients::Clients(std::shared_ptr client_, const S3ObjectStorageSettings & settings) - : client(std::move(client_)) - , client_with_long_timeout(client->clone( - /*override_use_adaptive_timeouts*/ false, - settings.request_settings.long_request_timeout_ms)) -{} - ObjectStorageKey S3ObjectStorage::generateObjectKeyForPath(const std::string &) const { /// Path to store the new S3 object. diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index 37e491e21dc8..7d14482311f0 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -39,16 +39,6 @@ struct S3ObjectStorageSettings class S3ObjectStorage : public IObjectStorage { -public: - struct Clients - { - std::shared_ptr client; - std::shared_ptr client_with_long_timeout; - - Clients() = default; - Clients(std::shared_ptr client, const S3ObjectStorageSettings & settings); - }; - private: friend class S3PlainObjectStorage; @@ -63,7 +53,7 @@ class S3ObjectStorage : public IObjectStorage String object_key_prefix_) : bucket(std::move(bucket_)) , object_key_prefix(std::move(object_key_prefix_)) - , clients(std::make_unique(std::move(client_), *s3_settings_)) + , client(std::move(client_)) , s3_settings(std::move(s3_settings_)) , s3_capabilities(s3_capabilities_) , version_id(std::move(version_id_)) @@ -185,7 +175,7 @@ class S3ObjectStorage : public IObjectStorage String object_key_prefix; - MultiVersion clients; + MultiVersion client; MultiVersion s3_settings; S3Capabilities s3_capabilities; diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 573fa744ce65..b0384daab2d2 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -60,7 +60,7 @@ std::unique_ptr getClient( uri.uri.getScheme()); client_configuration.connectTimeoutMs = config.getUInt(config_prefix + ".connect_timeout_ms", 1000); - client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", 3000); + client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", 30000); client_configuration.maxConnections = config.getUInt(config_prefix + ".max_connections", 100); client_configuration.endpointOverride = uri.endpoint; client_configuration.http_keep_alive_timeout_ms = config.getUInt( diff --git a/src/IO/ConnectionTimeouts.cpp b/src/IO/ConnectionTimeouts.cpp index a9eebb1a7552..90406dcf409c 100644 --- a/src/IO/ConnectionTimeouts.cpp +++ b/src/IO/ConnectionTimeouts.cpp @@ -133,22 +133,51 @@ ConnectionTimeouts ConnectionTimeouts::getHTTPTimeouts(const Settings & settings settings.http_receive_timeout); } -ConnectionTimeouts ConnectionTimeouts::aggressiveTimeouts(UInt32 attempt) const +ConnectionTimeouts ConnectionTimeouts::getAdaptiveTimeouts(Aws::Http::HttpMethod method, UInt32 attempt) const { + constexpr size_t first_method_index = size_t(Aws::Http::HttpMethod::HTTP_GET); + constexpr size_t last_method_index = size_t(Aws::Http::HttpMethod::HTTP_PATCH); + constexpr size_t methods_count = last_method_index - first_method_index + 1; + + /// HTTP_POST is used for CompleteMultipartUpload requests. + /// These requests need longer timeout, especially when minio is used + /// The same assumption are made for HTTP_DELETE, HTTP_PATCH + /// That requests are more heavy that HTTP_GET, HTTP_HEAD, HTTP_PUT + + static const UInt32 first_attempt_send_receive_timeouts_ms[methods_count][2] = { + /*HTTP_GET*/ {200, 200}, + /*HTTP_POST*/ {200, 30000}, + /*HTTP_DELETE*/ {200, 1000}, + /*HTTP_PUT*/ {200, 200}, + /*HTTP_HEAD*/ {200, 200}, + /*HTTP_PATCH*/ {200, 1000}, + }; + + static const UInt32 second_attempt_send_receive_timeouts_ms[methods_count][2] = { + /*HTTP_GET*/ {1000, 1000}, + /*HTTP_POST*/ {1000, 30000}, + /*HTTP_DELETE*/ {1000, 10000}, + /*HTTP_PUT*/ {1000, 1000}, + /*HTTP_HEAD*/ {1000, 1000}, + /*HTTP_PATCH*/ {1000, 10000}, + }; + + static_assert(methods_count == 6); + static_assert(sizeof(first_attempt_send_receive_timeouts_ms) == sizeof(second_attempt_send_receive_timeouts_ms)); + static_assert(sizeof(first_attempt_send_receive_timeouts_ms) == methods_count * sizeof(UInt32) * 2); + auto aggressive = *this; + if (attempt > 2) + return aggressive; + + auto timeout_map = first_attempt_send_receive_timeouts_ms; if (attempt == 2) - { - auto one_second = Poco::Timespan(1, 0); - aggressive.send_timeout = saturate(one_second, send_timeout); - aggressive.receive_timeout = saturate(one_second, receive_timeout); - } - else if (attempt == 1) - { - auto two_hundred_ms = Poco::Timespan(0, 200 * 1000); - aggressive.send_timeout = saturate(two_hundred_ms, send_timeout); - aggressive.receive_timeout = saturate(two_hundred_ms, receive_timeout); - } + timeout_map = second_attempt_send_receive_timeouts_ms; + + const size_t method_index = size_t(method) - first_method_index; + aggressive.send_timeout = saturate(Poco::Timespan(timeout_map[method_index][0]), send_timeout); + aggressive.receive_timeout = saturate(Poco::Timespan(timeout_map[method_index][1]), receive_timeout); return aggressive; } diff --git a/src/IO/ConnectionTimeouts.h b/src/IO/ConnectionTimeouts.h index 17ee1907d898..0ef133c8378d 100644 --- a/src/IO/ConnectionTimeouts.h +++ b/src/IO/ConnectionTimeouts.h @@ -4,6 +4,7 @@ #include #include +#include namespace DB { @@ -68,7 +69,7 @@ struct ConnectionTimeouts static ConnectionTimeouts getTCPTimeoutsWithFailover(const Settings & settings); static ConnectionTimeouts getHTTPTimeouts(const Settings & settings, Poco::Timespan http_keep_alive_timeout); - ConnectionTimeouts aggressiveTimeouts(UInt32 attempt) const; + ConnectionTimeouts getAdaptiveTimeouts(Aws::Http::HttpMethod method, UInt32 attempt) const; }; } diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 90806852c1e4..4630e68fbb6c 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -118,19 +118,9 @@ std::unique_ptr Client::create( new Client(max_redirects_, std::move(sse_kms_config_), credentials_provider, client_configuration, sign_payloads, use_virtual_addressing)); } -std::unique_ptr Client::clone( - std::optional override_use_adaptive_timeouts, - std::optional override_request_timeout_ms) const +std::unique_ptr Client::clone() const { - PocoHTTPClientConfiguration new_configuration = client_configuration; - - if (override_request_timeout_ms.has_value()) - new_configuration.requestTimeoutMs = *override_request_timeout_ms; - - if (override_use_adaptive_timeouts.has_value()) - new_configuration.s3_use_adaptive_timeouts = *override_use_adaptive_timeouts; - - return std::unique_ptr(new Client(*this, new_configuration)); + return std::unique_ptr(new Client(*this, client_configuration)); } namespace diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index be7235eb9f16..5ad57a9d8272 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -118,13 +118,7 @@ class Client : private Aws::S3::S3Client Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy sign_payloads, bool use_virtual_addressing); - /// Create a client with adjusted settings: - /// * override_request_timeout_ms is used to increase timeout for CompleteMultipartUploadRequest - /// because it often sits idle for 10 seconds: https://github.com/ClickHouse/ClickHouse/pull/42321 - /// * s3_use_adaptive_timeouts is used to turn off s3_use_adaptive_timeouts feature for CompleteMultipartUploadRequest - std::unique_ptr clone( - std::optional override_use_adaptive_timeouts = std::nullopt, - std::optional override_request_timeout_ms = std::nullopt) const; + std::unique_ptr clone() const; Client & operator=(const Client &) = delete; diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index f783a8868775..b26c36f8029f 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -300,7 +300,7 @@ ConnectionTimeouts PocoHTTPClient::getTimeouts(Aws::Http::HttpRequest & request) const auto & request_info = request.GetHeaderValue(Aws::Http::SDK_REQUEST_HEADER); auto attempt = extractAttempt(request_info); - return timeouts.aggressiveTimeouts(attempt); + return timeouts.getAdaptiveTimeouts(request.GetMethod(), attempt); } void PocoHTTPClient::makeRequestInternal( diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index a16a1a415058..30da1c580c13 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -53,7 +53,6 @@ namespace public: UploadHelper( const std::shared_ptr & client_ptr_, - const std::shared_ptr & client_with_long_timeout_ptr_, const String & dest_bucket_, const String & dest_key_, const S3Settings::RequestSettings & request_settings_, @@ -62,7 +61,6 @@ namespace bool for_disk_s3_, const Poco::Logger * log_) : client_ptr(client_ptr_) - , client_with_long_timeout_ptr(client_with_long_timeout_ptr_) , dest_bucket(dest_bucket_) , dest_key(dest_key_) , request_settings(request_settings_) @@ -78,7 +76,6 @@ namespace protected: std::shared_ptr client_ptr; - std::shared_ptr client_with_long_timeout_ptr; const String & dest_bucket; const String & dest_key; const S3Settings::RequestSettings & request_settings; @@ -179,7 +176,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); - auto outcome = client_with_long_timeout_ptr->CompleteMultipartUpload(request); + auto outcome = client_ptr->CompleteMultipartUpload(request); if (outcome.IsSuccess()) { @@ -433,14 +430,13 @@ namespace size_t offset_, size_t size_, const std::shared_ptr & client_ptr_, - const std::shared_ptr & client_with_long_timeout_ptr_, const String & dest_bucket_, const String & dest_key_, const S3Settings::RequestSettings & request_settings_, const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, bool for_disk_s3_) - : UploadHelper(client_ptr_, client_with_long_timeout_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyDataToS3File")) + : UploadHelper(client_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyDataToS3File")) , create_read_buffer(create_read_buffer_) , offset(offset_) , size(size_) @@ -602,7 +598,6 @@ namespace public: CopyFileHelper( const std::shared_ptr & client_ptr_, - const std::shared_ptr & client_with_long_timeout_ptr_, const String & src_bucket_, const String & src_key_, size_t src_offset_, @@ -614,7 +609,7 @@ namespace const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, bool for_disk_s3_) - : UploadHelper(client_ptr_, client_with_long_timeout_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyS3File")) + : UploadHelper(client_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyS3File")) , src_bucket(src_bucket_) , src_key(src_key_) , offset(src_offset_) @@ -677,7 +672,7 @@ namespace /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 request.SetContentType("binary/octet-stream"); - client_with_long_timeout_ptr->setKMSHeaders(request); + client_ptr->setKMSHeaders(request); } void processCopyRequest(const S3::CopyObjectRequest & request) @@ -689,7 +684,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3CopyObject); - auto outcome = client_with_long_timeout_ptr->CopyObject(request); + auto outcome = client_ptr->CopyObject(request); if (outcome.IsSuccess()) { LOG_TRACE( @@ -714,7 +709,6 @@ namespace offset, size, client_ptr, - client_with_long_timeout_ptr, dest_bucket, dest_key, request_settings, @@ -788,7 +782,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3UploadPartCopy); - auto outcome = client_with_long_timeout_ptr->UploadPartCopy(req); + auto outcome = client_ptr->UploadPartCopy(req); if (!outcome.IsSuccess()) { abortMultipartUpload(); @@ -806,7 +800,6 @@ void copyDataToS3File( size_t offset, size_t size, const std::shared_ptr & dest_s3_client, - const std::shared_ptr & dest_s3_client_with_long_timeout, const String & dest_bucket, const String & dest_key, const S3Settings::RequestSettings & settings, @@ -814,14 +807,13 @@ void copyDataToS3File( ThreadPoolCallbackRunner schedule, bool for_disk_s3) { - CopyDataToFileHelper helper{create_read_buffer, offset, size, dest_s3_client, dest_s3_client_with_long_timeout, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3}; + CopyDataToFileHelper helper{create_read_buffer, offset, size, dest_s3_client, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3}; helper.performCopy(); } void copyS3File( const std::shared_ptr & s3_client, - const std::shared_ptr & s3_client_with_long_timeout, const String & src_bucket, const String & src_key, size_t src_offset, @@ -836,7 +828,7 @@ void copyS3File( { if (settings.allow_native_copy) { - CopyFileHelper helper{s3_client, s3_client_with_long_timeout, src_bucket, src_key, src_offset, src_size, dest_bucket, dest_key, settings, read_settings, object_metadata, schedule, for_disk_s3}; + CopyFileHelper helper{s3_client, src_bucket, src_key, src_offset, src_size, dest_bucket, dest_key, settings, read_settings, object_metadata, schedule, for_disk_s3}; helper.performCopy(); } else @@ -845,7 +837,7 @@ void copyS3File( { return std::make_unique(s3_client, src_bucket, src_key, "", settings, read_settings); }; - copyDataToS3File(create_read_buffer, src_offset, src_size, s3_client, s3_client_with_long_timeout, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3); + copyDataToS3File(create_read_buffer, src_offset, src_size, s3_client, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3); } } diff --git a/src/IO/S3/copyS3File.h b/src/IO/S3/copyS3File.h index 1bcbfd7735e4..33e22fdfba22 100644 --- a/src/IO/S3/copyS3File.h +++ b/src/IO/S3/copyS3File.h @@ -27,15 +27,9 @@ using CreateReadBuffer = std::function()>; /// because it is a known issue, it is fallbacks to read-write copy /// (copyDataToS3File()). /// -/// s3_client_with_long_timeout (may be equal to s3_client) is used for native copy and -/// CompleteMultipartUpload requests. These requests need longer timeout because S3 servers often -/// block on them for multiple seconds without sending or receiving data from us (maybe the servers -/// are copying data internally, or maybe throttling, idk). -/// /// read_settings - is used for throttling in case of native copy is not possible void copyS3File( const std::shared_ptr & s3_client, - const std::shared_ptr & s3_client_with_long_timeout, const String & src_bucket, const String & src_key, size_t src_offset, @@ -58,7 +52,6 @@ void copyDataToS3File( size_t offset, size_t size, const std::shared_ptr & dest_s3_client, - const std::shared_ptr & dest_s3_client_with_long_timeout, const String & dest_bucket, const String & dest_key, const S3Settings::RequestSettings & settings, diff --git a/src/IO/S3/tests/gtest_aws_s3_client.cpp b/src/IO/S3/tests/gtest_aws_s3_client.cpp index c42f14e9a53e..d4b9a017398d 100644 --- a/src/IO/S3/tests/gtest_aws_s3_client.cpp +++ b/src/IO/S3/tests/gtest_aws_s3_client.cpp @@ -91,7 +91,6 @@ void doWriteRequest(std::shared_ptr client, const DB::S3:: DB::S3Settings::RequestSettings request_settings; request_settings.max_unexpected_write_error_retries = max_unexpected_write_error_retries; DB::WriteBufferFromS3 write_buffer( - client, client, uri.bucket, uri.key, diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index e1b9c17efe97..62d0c80f1f28 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -77,7 +77,6 @@ struct WriteBufferFromS3::PartData WriteBufferFromS3::WriteBufferFromS3( std::shared_ptr client_ptr_, - std::shared_ptr client_with_long_timeout_ptr_, const String & bucket_, const String & key_, size_t buf_size_, @@ -92,7 +91,6 @@ WriteBufferFromS3::WriteBufferFromS3( , upload_settings(request_settings.getUploadSettings()) , write_settings(write_settings_) , client_ptr(std::move(client_ptr_)) - , client_with_long_timeout_ptr(std::move(client_with_long_timeout_ptr_)) , object_metadata(std::move(object_metadata_)) , buffer_allocation_policy(ChooseBufferPolicy(upload_settings)) , task_tracker( @@ -566,7 +564,7 @@ void WriteBufferFromS3::completeMultipartUpload() ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Stopwatch watch; - auto outcome = client_with_long_timeout_ptr->CompleteMultipartUpload(req); + auto outcome = client_ptr->CompleteMultipartUpload(req); watch.stop(); ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Microseconds, watch.elapsedMicroseconds()); diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 95148c497791..590342cc997f 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -30,8 +30,6 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase public: WriteBufferFromS3( std::shared_ptr client_ptr_, - /// for CompleteMultipartUploadRequest, because it blocks on recv() for a few seconds on big uploads - std::shared_ptr client_with_long_timeout_ptr_, const String & bucket_, const String & key_, size_t buf_size_, @@ -90,7 +88,6 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase const S3Settings::RequestSettings::PartUploadSettings & upload_settings; const WriteSettings write_settings; const std::shared_ptr client_ptr; - const std::shared_ptr client_with_long_timeout_ptr; const std::optional> object_metadata; Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3"); LogSeriesLimiterPtr limitedLog = std::make_shared(log, 1, 5); diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp index 21bdd9a6f26d..c82f97f8b201 100644 --- a/src/IO/tests/gtest_writebuffer_s3.cpp +++ b/src/IO/tests/gtest_writebuffer_s3.cpp @@ -549,7 +549,6 @@ class WBS3Test : public ::testing::Test getAsyncPolicy().setAutoExecute(false); return std::make_unique( - client, client, bucket, file_name, diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 231efb87e872..b0cd40a2e057 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -824,7 +824,6 @@ class StorageS3Sink : public SinkToStorage write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique( configuration_.client, - configuration_.client_with_long_timeout, bucket, key, DBMS_DEFAULT_BUFFER_SIZE, @@ -1329,8 +1328,6 @@ void StorageS3::Configuration::connect(ContextPtr context) context->getConfigRef().getUInt64("s3.expiration_window_seconds", S3::DEFAULT_EXPIRATION_WINDOW_SECONDS)), auth_settings.no_sign_request.value_or(context->getConfigRef().getBool("s3.no_sign_request", false)), }); - - client_with_long_timeout = client->clone(/*override_use_adaptive_timeouts*/ false, request_settings.long_request_timeout_ms); } void StorageS3::processNamedCollectionResult(StorageS3::Configuration & configuration, const NamedCollection & collection) diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 3330ac6c210e..3f35c578e19b 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -311,7 +311,6 @@ class StorageS3 : public IStorage HTTPHeaderEntries headers_from_ast; std::shared_ptr client; - std::shared_ptr client_with_long_timeout; std::vector keys; }; diff --git a/src/Storages/StorageS3Settings.h b/src/Storages/StorageS3Settings.h index e3d577ca0b36..728972c948c9 100644 --- a/src/Storages/StorageS3Settings.h +++ b/src/Storages/StorageS3Settings.h @@ -69,8 +69,7 @@ struct S3Settings ThrottlerPtr get_request_throttler; ThrottlerPtr put_request_throttler; size_t retry_attempts = 10; - size_t request_timeout_ms = 3000; - size_t long_request_timeout_ms = 30000; // TODO: Take this from config like request_timeout_ms + size_t request_timeout_ms = 30000; bool allow_native_copy = true; bool throw_on_zero_files_match = false; From 3075bd97450d42f00320b18c1b177fd700a19bec Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Fri, 10 Nov 2023 15:15:24 +0100 Subject: [PATCH 059/274] track clickhouse high level retries --- base/poco/Net/src/HTTPSession.cpp | 4 +- src/IO/ConnectionTimeouts.cpp | 97 +++++++++++++------ src/IO/ConnectionTimeouts.h | 3 +- src/IO/HTTPCommon.cpp | 12 +-- src/IO/HTTPCommon.h | 2 + src/IO/ReadBufferFromS3.cpp | 24 ++--- src/IO/ReadBufferFromS3.h | 4 +- src/IO/S3/PocoHTTPClient.cpp | 87 +++++++++-------- src/IO/S3/PocoHTTPClient.h | 2 +- src/IO/S3/tests/gtest_aws_s3_client.cpp | 2 + .../configs/inf_s3_retries.xml | 2 +- .../configs/s3_retries.xml | 2 +- .../configs/storage_conf.xml | 1 + .../test_checking_s3_blobs_paranoid/test.py | 16 +-- .../test_storage_s3/configs/s3_retry.xml | 1 + .../s3_mocks/unstable_server.py | 17 +++- tests/integration/test_storage_s3/test.py | 9 ++ 17 files changed, 178 insertions(+), 107 deletions(-) diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index d30f5590280b..9ebbd7d04cd7 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -95,7 +95,7 @@ void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco { _connectionTimeout = connectionTimeout; - if (_sendTimeout != sendTimeout) + if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) { _sendTimeout = sendTimeout; @@ -103,7 +103,7 @@ void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco _socket.setSendTimeout(_sendTimeout); } - if (_receiveTimeout != receiveTimeout) + if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) { _receiveTimeout = receiveTimeout; diff --git a/src/IO/ConnectionTimeouts.cpp b/src/IO/ConnectionTimeouts.cpp index 90406dcf409c..970afc75ec3e 100644 --- a/src/IO/ConnectionTimeouts.cpp +++ b/src/IO/ConnectionTimeouts.cpp @@ -133,51 +133,84 @@ ConnectionTimeouts ConnectionTimeouts::getHTTPTimeouts(const Settings & settings settings.http_receive_timeout); } -ConnectionTimeouts ConnectionTimeouts::getAdaptiveTimeouts(Aws::Http::HttpMethod method, UInt32 attempt) const +class SendReceiveTimeoutsForFirstAttempt { - constexpr size_t first_method_index = size_t(Aws::Http::HttpMethod::HTTP_GET); - constexpr size_t last_method_index = size_t(Aws::Http::HttpMethod::HTTP_PATCH); - constexpr size_t methods_count = last_method_index - first_method_index + 1; +private: + static constexpr size_t known_methods_count = 6; + using KnownMethodsArray = std::array; + static const KnownMethodsArray known_methods; - /// HTTP_POST is used for CompleteMultipartUpload requests. - /// These requests need longer timeout, especially when minio is used + /// HTTP_POST is used for CompleteMultipartUpload requests. Its latency could be high. + /// These requests need longer timeout, especially when minio is used. /// The same assumption are made for HTTP_DELETE, HTTP_PATCH /// That requests are more heavy that HTTP_GET, HTTP_HEAD, HTTP_PUT - static const UInt32 first_attempt_send_receive_timeouts_ms[methods_count][2] = { - /*HTTP_GET*/ {200, 200}, - /*HTTP_POST*/ {200, 30000}, - /*HTTP_DELETE*/ {200, 1000}, - /*HTTP_PUT*/ {200, 200}, - /*HTTP_HEAD*/ {200, 200}, - /*HTTP_PATCH*/ {200, 1000}, + static constexpr Poco::Timestamp::TimeDiff first_byte_ms[known_methods_count][2] = + { + /* GET */ {200, 200}, + /* POST */ {200, 200}, + /* DELETE */ {200, 200}, + /* PUT */ {200, 200}, + /* HEAD */ {200, 200}, + /* PATCH */ {200, 200}, }; - static const UInt32 second_attempt_send_receive_timeouts_ms[methods_count][2] = { - /*HTTP_GET*/ {1000, 1000}, - /*HTTP_POST*/ {1000, 30000}, - /*HTTP_DELETE*/ {1000, 10000}, - /*HTTP_PUT*/ {1000, 1000}, - /*HTTP_HEAD*/ {1000, 1000}, - /*HTTP_PATCH*/ {1000, 10000}, + static constexpr Poco::Timestamp::TimeDiff rest_bytes_ms[known_methods_count][2] = + { + /* GET */ {500, 500}, + /* POST */ {1000, 30000}, + /* DELETE */ {1000, 10000}, + /* PUT */ {1000, 3000}, + /* HEAD */ {500, 500}, + /* PATCH */ {1000, 10000}, }; - static_assert(methods_count == 6); - static_assert(sizeof(first_attempt_send_receive_timeouts_ms) == sizeof(second_attempt_send_receive_timeouts_ms)); - static_assert(sizeof(first_attempt_send_receive_timeouts_ms) == methods_count * sizeof(UInt32) * 2); + static_assert(sizeof(first_byte_ms) == sizeof(rest_bytes_ms)); + static_assert(sizeof(first_byte_ms) == known_methods_count * sizeof(Poco::Timestamp::TimeDiff) * 2); + + static size_t getMethodIndex(const String & method) + { + KnownMethodsArray::const_iterator it = std::find(known_methods.begin(), known_methods.end(), method); + chassert(it != known_methods.end()); + if (it == known_methods.end()) + return 0; + return std::distance(known_methods.begin(), it); + } + +public: + static std::pair getSendReceiveTimeout(const String & method, bool first_byte) + { + auto idx = getMethodIndex(method); + + if (first_byte) + return std::make_pair( + Poco::Timespan(first_byte_ms[idx][0] * 1000), + Poco::Timespan(first_byte_ms[idx][1] * 1000) + ); + + return std::make_pair( + Poco::Timespan(rest_bytes_ms[idx][0] * 1000), + Poco::Timespan(rest_bytes_ms[idx][1] * 1000) + ); + } +}; + +const SendReceiveTimeoutsForFirstAttempt::KnownMethodsArray SendReceiveTimeoutsForFirstAttempt::known_methods = +{ + "GET", "POST", "DELETE", "PUT", "HEAD", "PATCH" +}; - auto aggressive = *this; - if (attempt > 2) - return aggressive; +ConnectionTimeouts ConnectionTimeouts::getAdaptiveTimeouts(const String & method, bool first_attempt, bool first_byte) const +{ + if (!first_attempt) + return *this; - auto timeout_map = first_attempt_send_receive_timeouts_ms; - if (attempt == 2) - timeout_map = second_attempt_send_receive_timeouts_ms; + auto [send, recv] = SendReceiveTimeoutsForFirstAttempt::getSendReceiveTimeout(method, first_byte); - const size_t method_index = size_t(method) - first_method_index; - aggressive.send_timeout = saturate(Poco::Timespan(timeout_map[method_index][0]), send_timeout); - aggressive.receive_timeout = saturate(Poco::Timespan(timeout_map[method_index][1]), receive_timeout); + auto aggressive = *this; + aggressive.send_timeout = saturate(send, send_timeout); + aggressive.receive_timeout = saturate(recv, receive_timeout); return aggressive; } diff --git a/src/IO/ConnectionTimeouts.h b/src/IO/ConnectionTimeouts.h index 0ef133c8378d..aabebdb836d5 100644 --- a/src/IO/ConnectionTimeouts.h +++ b/src/IO/ConnectionTimeouts.h @@ -4,7 +4,6 @@ #include #include -#include namespace DB { @@ -69,7 +68,7 @@ struct ConnectionTimeouts static ConnectionTimeouts getTCPTimeoutsWithFailover(const Settings & settings); static ConnectionTimeouts getHTTPTimeouts(const Settings & settings, Poco::Timespan http_keep_alive_timeout); - ConnectionTimeouts getAdaptiveTimeouts(Aws::Http::HttpMethod method, UInt32 attempt) const; + ConnectionTimeouts getAdaptiveTimeouts(const String & method, bool first_attempt, bool first_byte) const; }; } diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 65ffa51a466a..cce394c67c98 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -50,12 +50,6 @@ namespace ErrorCodes namespace { - void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts) - { - session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout); - session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout); - } - Poco::Net::HTTPClientSession::ProxyConfig proxyConfigurationToPocoProxyConfig(const ProxyConfiguration & proxy_configuration) { Poco::Net::HTTPClientSession::ProxyConfig poco_proxy_config; @@ -359,6 +353,12 @@ namespace }; } +void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts) +{ + session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout); + session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout); +} + void setResponseDefaultHeaders(HTTPServerResponse & response, size_t keep_alive_timeout) { if (!response.getKeepAlive()) diff --git a/src/IO/HTTPCommon.h b/src/IO/HTTPCommon.h index de62b5d5c165..c9968fc6915e 100644 --- a/src/IO/HTTPCommon.h +++ b/src/IO/HTTPCommon.h @@ -113,4 +113,6 @@ std::istream * receiveResponse( void assertResponseIsOk( const Poco::Net::HTTPRequest & request, Poco::Net::HTTPResponse & response, std::istream & istr, bool allow_redirects = false); + +void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts); } diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index f19978ccb471..c9c9319c44ce 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -167,9 +167,9 @@ bool ReadBufferFromS3::nextImpl() } size_t sleep_time_with_backoff_milliseconds = 100; - for (size_t attempt = 0; !next_result; ++attempt) + for (size_t attempt = 1; !next_result; ++attempt) { - bool last_attempt = attempt + 1 >= request_settings.max_single_read_retries; + bool last_attempt = attempt >= request_settings.max_single_read_retries; ProfileEventTimeIncrement watch(ProfileEvents::ReadBufferFromS3Microseconds); @@ -177,7 +177,7 @@ bool ReadBufferFromS3::nextImpl() { if (!impl) { - impl = initialize(); + impl = initialize(attempt); if (use_external_buffer) { @@ -232,9 +232,9 @@ size_t ReadBufferFromS3::readBigAt(char * to, size_t n, size_t range_begin, cons { size_t initial_n = n; size_t sleep_time_with_backoff_milliseconds = 100; - for (size_t attempt = 0; n > 0; ++attempt) + for (size_t attempt = 1; n > 0; ++attempt) { - bool last_attempt = attempt + 1 >= request_settings.max_single_read_retries; + bool last_attempt = attempt >= request_settings.max_single_read_retries; size_t bytes_copied = 0; ProfileEventTimeIncrement watch(ProfileEvents::ReadBufferFromS3Microseconds); @@ -266,7 +266,7 @@ size_t ReadBufferFromS3::readBigAt(char * to, size_t n, size_t range_begin, cons try { - result = sendRequest(range_begin, range_begin + n - 1); + result = sendRequest(attempt, range_begin, range_begin + n - 1); std::istream & istr = result->GetBody(); copyFromIStreamWithProgressCallback(istr, to, n, progress_callback, &bytes_copied); @@ -304,8 +304,8 @@ bool ReadBufferFromS3::processException(Poco::Exception & e, size_t read_offset, LOG_DEBUG( log, "Caught exception while reading S3 object. Bucket: {}, Key: {}, Version: {}, Offset: {}, " - "Attempt: {}, Message: {}", - bucket, key, version_id.empty() ? "Latest" : version_id, read_offset, attempt, e.message()); + "Attempt: {}/{}, Message: {}", + bucket, key, version_id.empty() ? "Latest" : version_id, read_offset, attempt, request_settings.max_single_read_retries, e.message()); if (auto * s3_exception = dynamic_cast(&e)) @@ -463,7 +463,7 @@ ReadBufferFromS3::~ReadBufferFromS3() } } -std::unique_ptr ReadBufferFromS3::initialize() +std::unique_ptr ReadBufferFromS3::initialize(size_t attempt) { resetSessionIfNeeded(readAllRangeSuccessfully(), read_result); read_all_range_successfully = false; @@ -475,13 +475,13 @@ std::unique_ptr ReadBufferFromS3::initialize() if (read_until_position && offset >= read_until_position) throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read beyond right offset ({} > {})", offset, read_until_position - 1); - read_result = sendRequest(offset, read_until_position ? std::make_optional(read_until_position - 1) : std::nullopt); + read_result = sendRequest(attempt, offset, read_until_position ? std::make_optional(read_until_position - 1) : std::nullopt); size_t buffer_size = use_external_buffer ? 0 : read_settings.remote_fs_buffer_size; return std::make_unique(read_result->GetBody(), buffer_size); } -Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t range_begin, std::optional range_end_incl) const +Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t attempt, size_t range_begin, std::optional range_end_incl) const { S3::GetObjectRequest req; req.SetBucket(bucket); @@ -489,6 +489,8 @@ Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t range_begin if (!version_id.empty()) req.SetVersionId(version_id); + req.SetAdditionalCustomHeaderValue("clickhouse-request", fmt::format("attempt={}", attempt)); + if (range_end_incl) { req.SetRange(fmt::format("bytes={}-{}", range_begin, *range_end_incl)); diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h index 0835e52a5b2e..101e25f8b436 100644 --- a/src/IO/ReadBufferFromS3.h +++ b/src/IO/ReadBufferFromS3.h @@ -79,7 +79,7 @@ class ReadBufferFromS3 : public ReadBufferFromFileBase bool supportsReadAt() override { return true; } private: - std::unique_ptr initialize(); + std::unique_ptr initialize(size_t attempt); /// If true, if we destroy impl now, no work was wasted. Just for metrics. bool atEndOfRequestedRangeGuess(); @@ -88,7 +88,7 @@ class ReadBufferFromS3 : public ReadBufferFromFileBase /// Returns true if the error looks retriable. bool processException(Poco::Exception & e, size_t read_offset, size_t attempt) const; - Aws::S3::Model::GetObjectResult sendRequest(size_t range_begin, std::optional range_end_incl) const; + Aws::S3::Model::GetObjectResult sendRequest(size_t attempt, size_t range_begin, std::optional range_end_incl) const; bool readAllRangeSuccessfully() const; diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index b26c36f8029f..904e23241454 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -272,35 +272,36 @@ void PocoHTTPClient::addMetric(const Aws::Http::HttpRequest & request, S3MetricT ProfileEvents::increment(disk_s3_events_map[static_cast(type)][static_cast(kind)], amount); } -UInt32 extractAttempt(const Aws::String & request_info) +String extractAttemptFromInfo(const Aws::String & request_info) { static auto key = Aws::String("attempt="); auto key_begin = request_info.find(key, 0); if (key_begin == Aws::String::npos) - return 1; + return "1"; auto val_begin = key_begin + key.size(); auto val_end = request_info.find(';', val_begin); if (val_end == Aws::String::npos) val_end = request_info.size(); - Aws::String value = request_info.substr(val_begin, val_end-val_begin); + return request_info.substr(val_begin, val_end-val_begin); +} - UInt32 attempt = 1; - ReadBufferFromString buf(value); - readIntText(attempt, buf); - return attempt; +String getOrEmpty(const Aws::Http::HeaderValueCollection & map, const String & key) +{ + auto it = map.find(key); + if (it == map.end()) + return {}; + return it->second; } -ConnectionTimeouts PocoHTTPClient::getTimeouts(Aws::Http::HttpRequest & request) const +ConnectionTimeouts PocoHTTPClient::getTimeouts(const String & method, bool first_attempt, bool first_byte) const { if (!s3_use_adaptive_timeouts) return timeouts; - const auto & request_info = request.GetHeaderValue(Aws::Http::SDK_REQUEST_HEADER); - auto attempt = extractAttempt(request_info); - return timeouts.getAdaptiveTimeouts(request.GetMethod(), attempt); + return timeouts.getAdaptiveTimeouts(method, first_attempt, first_byte); } void PocoHTTPClient::makeRequestInternal( @@ -317,6 +318,25 @@ void PocoHTTPClient::makeRequestInternal( makeRequestInternalImpl(request, request_configuration, response, readLimiter, writeLimiter); } +String getMethod(const Aws::Http::HttpRequest & request) +{ + switch (request.GetMethod()) + { + case Aws::Http::HttpMethod::HTTP_GET: + return Poco::Net::HTTPRequest::HTTP_GET; + case Aws::Http::HttpMethod::HTTP_POST: + return Poco::Net::HTTPRequest::HTTP_POST; + case Aws::Http::HttpMethod::HTTP_DELETE: + return Poco::Net::HTTPRequest::HTTP_DELETE; + case Aws::Http::HttpMethod::HTTP_PUT: + return Poco::Net::HTTPRequest::HTTP_PUT; + case Aws::Http::HttpMethod::HTTP_HEAD: + return Poco::Net::HTTPRequest::HTTP_HEAD; + case Aws::Http::HttpMethod::HTTP_PATCH: + return Poco::Net::HTTPRequest::HTTP_PATCH; + } +} + template void PocoHTTPClient::makeRequestInternalImpl( Aws::Http::HttpRequest & request, @@ -330,9 +350,14 @@ void PocoHTTPClient::makeRequestInternalImpl( Poco::Logger * log = &Poco::Logger::get("AWSClient"); auto uri = request.GetUri().GetURIString(); + auto method = getMethod(request); + + auto sdk_attempt = extractAttemptFromInfo(getOrEmpty(request.GetHeaders(), Aws::Http::SDK_REQUEST_HEADER)); + auto ch_attempt = extractAttemptFromInfo(getOrEmpty(request.GetHeaders(), "clickhouse-request")); + bool first_attempt = ch_attempt == "1" && sdk_attempt == "1"; if (enable_s3_requests_logging) - LOG_TEST(log, "Make request to: {}", uri); + LOG_TEST(log, "Make request to: {}, aws sdk attempt: {}, clickhouse attempt: {}", uri, sdk_attempt, ch_attempt); switch (request.GetMethod()) { @@ -383,17 +408,17 @@ void PocoHTTPClient::makeRequestInternalImpl( /// This can lead to request signature difference on S3 side. if constexpr (pooled) session = makePooledHTTPSession( - target_uri, getTimeouts(request), http_connection_pool_size, wait_on_pool_size_limit, proxy_configuration); + target_uri, getTimeouts(method, first_attempt), http_connection_pool_size, wait_on_pool_size_limit, proxy_configuration); else - session = makeHTTPSession(target_uri, getTimeouts(request), proxy_configuration); + session = makeHTTPSession(target_uri, getTimeouts(method, first_attempt), proxy_configuration); } else { if constexpr (pooled) session = makePooledHTTPSession( - target_uri, getTimeouts(request), http_connection_pool_size, wait_on_pool_size_limit); + target_uri, getTimeouts(method, first_attempt), http_connection_pool_size, wait_on_pool_size_limit); else - session = makeHTTPSession(target_uri, getTimeouts(request)); + session = makeHTTPSession(target_uri, getTimeouts(method, first_attempt)); } /// In case of error this address will be written to logs @@ -427,28 +452,7 @@ void PocoHTTPClient::makeRequestInternalImpl( path_and_query = "/"; poco_request.setURI(path_and_query); - - switch (request.GetMethod()) - { - case Aws::Http::HttpMethod::HTTP_GET: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_GET); - break; - case Aws::Http::HttpMethod::HTTP_POST: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_POST); - break; - case Aws::Http::HttpMethod::HTTP_DELETE: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_DELETE); - break; - case Aws::Http::HttpMethod::HTTP_PUT: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_PUT); - break; - case Aws::Http::HttpMethod::HTTP_HEAD: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_HEAD); - break; - case Aws::Http::HttpMethod::HTTP_PATCH: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_PATCH); - break; - } + poco_request.setMethod(method); /// Headers coming from SDK are lower-cased. for (const auto & [header_name, header_value] : request.GetHeaders()) @@ -473,6 +477,7 @@ void PocoHTTPClient::makeRequestInternalImpl( request.GetContentBody()->clear(); request.GetContentBody()->seekg(0); + setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); auto size = Poco::StreamCopier::copyStream(*request.GetContentBody(), request_body_stream); if (enable_s3_requests_logging) LOG_TEST(log, "Written {} bytes to request body", size); @@ -482,6 +487,8 @@ void PocoHTTPClient::makeRequestInternalImpl( LOG_TEST(log, "Receiving response..."); auto & response_body_stream = session->receiveResponse(poco_response); + setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); + watch.stop(); addMetric(request, S3MetricType::Microseconds, watch.elapsedMicroseconds()); @@ -533,6 +540,7 @@ void PocoHTTPClient::makeRequestInternalImpl( /// Request is successful but for some special requests we can have actual error message in body if (status_code >= SUCCESS_RESPONSE_MIN && status_code <= SUCCESS_RESPONSE_MAX && checkRequestCanReturn2xxAndErrorInBody(request)) { + /// reading the full response std::string response_string((std::istreambuf_iterator(response_body_stream)), std::istreambuf_iterator()); @@ -547,7 +555,6 @@ void PocoHTTPClient::makeRequestInternalImpl( addMetric(request, S3MetricType::Errors); if (error_report) error_report(proxy_configuration); - } /// Set response from string @@ -566,6 +573,8 @@ void PocoHTTPClient::makeRequestInternalImpl( if (status_code >= 500 && error_report) error_report(proxy_configuration); } + + /// expose stream, after that client reads data from that stream without built-in retries response->SetResponseBody(response_body_stream, session); } diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 9ba5f4ffe640..14c4fec5dd7f 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -171,7 +171,7 @@ class PocoHTTPClient : public Aws::Http::HttpClient Aws::Utils::RateLimits::RateLimiterInterface * readLimiter, Aws::Utils::RateLimits::RateLimiterInterface * writeLimiter) const; - ConnectionTimeouts getTimeouts(Aws::Http::HttpRequest & request) const; + ConnectionTimeouts getTimeouts(const String & method, bool first_attempt, bool first_byte = true) const; protected: static S3MetricKind getMetricKind(const Aws::Http::HttpRequest & request); diff --git a/src/IO/S3/tests/gtest_aws_s3_client.cpp b/src/IO/S3/tests/gtest_aws_s3_client.cpp index d4b9a017398d..bff9ca6fa7b3 100644 --- a/src/IO/S3/tests/gtest_aws_s3_client.cpp +++ b/src/IO/S3/tests/gtest_aws_s3_client.cpp @@ -170,6 +170,7 @@ TEST(IOTestAwsS3Client, AppendExtraSSECHeadersRead) "authorization: ... SignedHeaders=" "amz-sdk-invocation-id;" "amz-sdk-request;" + "clickhouse-request;" "content-type;" "host;" "x-amz-api-version;" @@ -215,6 +216,7 @@ TEST(IOTestAwsS3Client, AppendExtraSSEKMSHeadersRead) "authorization: ... SignedHeaders=" "amz-sdk-invocation-id;" "amz-sdk-request;" + "clickhouse-request;" "content-type;" "host;" "x-amz-api-version;" diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml index 5f0860ac1200..4210c13b727c 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml @@ -4,7 +4,7 @@ 1000000 - 1 + 1 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml index f215a89f613d..95a313ea4f27 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml @@ -4,7 +4,7 @@ 5 - 0 + 0 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml index 264c411b59b3..7b1f503ed553 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml @@ -18,6 +18,7 @@ http://resolver:8083/root/data/ minio minio123 + 1 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py index 441a5a541e8a..b000ccabcf47 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/test.py +++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py @@ -556,7 +556,7 @@ def test_query_is_canceled_with_inf_retries(cluster, broken_s3): @pytest.mark.parametrize("node_name", ["node", "node_with_inf_s3_retries"]) -def test_aggressive_timeouts(cluster, broken_s3, node_name): +def test_adaptive_timeouts(cluster, broken_s3, node_name): node = cluster.instances[node_name] broken_s3.setup_fake_puts(part_length=1) @@ -565,12 +565,12 @@ def test_aggressive_timeouts(cluster, broken_s3, node_name): count=1000000, ) - insert_query_id = f"TEST_AGGRESSIVE_TIMEOUTS_{node_name}" + insert_query_id = f"TEST_ADAPTIVE_TIMEOUTS_{node_name}" node.query( f""" INSERT INTO TABLE FUNCTION s3( - 'http://resolver:8083/root/data/aggressive_timeouts', + 'http://resolver:8083/root/data/adaptive_timeouts', 'minio', 'minio123', 'CSV', auto, 'none' ) @@ -593,20 +593,20 @@ def test_aggressive_timeouts(cluster, broken_s3, node_name): assert put_objects == 1 - s3_aggressive_timeouts_state = node.query( + s3_use_adaptive_timeouts = node.query( f""" SELECT value FROM system.settings WHERE - name='s3_aggressive_timeouts' + name='s3_use_adaptive_timeouts' """ ).strip() if node_name == "node_with_inf_s3_retries": # first 2 attempts failed - assert s3_aggressive_timeouts_state == "1" - assert s3_errors == 2 + assert s3_use_adaptive_timeouts == "1" + assert s3_errors == 1 else: - assert s3_aggressive_timeouts_state == "0" + assert s3_use_adaptive_timeouts == "0" assert s3_errors == 0 diff --git a/tests/integration/test_storage_s3/configs/s3_retry.xml b/tests/integration/test_storage_s3/configs/s3_retry.xml index b7a7bbc8a9bd..3171da051d0c 100644 --- a/tests/integration/test_storage_s3/configs/s3_retry.xml +++ b/tests/integration/test_storage_s3/configs/s3_retry.xml @@ -1,6 +1,7 @@ + 1 10 5 diff --git a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py index 103dd30340ce..5ef781bdc9e2 100644 --- a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py +++ b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py @@ -4,6 +4,7 @@ import socket import struct import sys +import time def gen_n_digit_number(n): @@ -39,14 +40,14 @@ def add_number(): # Generating some "random" data and append a line which contains sum of numbers in column 4. lines = ( - b"".join((gen_line() for _ in range(500000))) + b"".join([gen_line() for _ in range(500000)]) + f"0,0,0,{-sum_in_4_column}\n".encode() ) class RequestHandler(http.server.BaseHTTPRequestHandler): def do_HEAD(self): - if self.path == "/root/test.csv": + if self.path == "/root/test.csv" or self.path == "/root/slow_send_test.csv": self.from_bytes = 0 self.end_bytes = len(lines) self.size = self.end_bytes @@ -101,6 +102,18 @@ def do_GET(self): print("Dropping connection") break + if self.path == "/root/slow_send_test.csv": + self.send_block_size = 81920 + + for c, i in enumerate( + range(self.from_bytes, self.end_bytes, self.send_block_size) + ): + self.wfile.write( + lines[i : min(i + self.send_block_size, self.end_bytes)] + ) + self.wfile.flush() + time.sleep(1) + elif self.path == "/": self.wfile.write(b"OK") diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 01ade1acc4d5..8c79ad024456 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -818,6 +818,15 @@ def test_storage_s3_get_unstable(started_cluster): assert result.splitlines() == ["500001,500000,0"] +def test_storage_s3_get_slow(started_cluster): + bucket = started_cluster.minio_bucket + instance = started_cluster.instances["dummy"] + table_format = "column1 Int64, column2 Int64, column3 Int64, column4 Int64" + get_query = f"SELECT count(), sum(column3), sum(column4) FROM s3('http://resolver:8081/{started_cluster.minio_bucket}/slow_send_test.csv', 'CSV', '{table_format}') FORMAT CSV" + result = run_query(instance, get_query) + assert result.splitlines() == ["500001,500000,0"] + + def test_storage_s3_put_uncompressed(started_cluster): bucket = started_cluster.minio_bucket instance = started_cluster.instances["dummy"] From 2f9ac9b49cc8b2f8fbd7e824138855f73403ce38 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 14 Nov 2023 14:33:34 +0000 Subject: [PATCH 060/274] Address comments --- src/Backups/BackupCoordinationFileInfos.cpp | 81 +++++++++++++++++-- src/Backups/BackupFileInfo.h | 4 + src/Backups/BackupIO.h | 2 + src/Backups/BackupIO_Default.cpp | 2 + src/Backups/BackupIO_Disk.cpp | 9 +++ src/Backups/BackupIO_Disk.h | 2 + src/Backups/BackupIO_File.cpp | 10 +++ src/Backups/BackupIO_File.h | 2 + src/Backups/BackupIO_S3.cpp | 18 +++++ src/Backups/BackupIO_S3.h | 2 + src/Backups/BackupImpl.cpp | 27 ++----- .../test_backup_restore_keeper_map/test.py | 28 ++++--- 12 files changed, 149 insertions(+), 38 deletions(-) diff --git a/src/Backups/BackupCoordinationFileInfos.cpp b/src/Backups/BackupCoordinationFileInfos.cpp index ac2cbc337aa9..b17b755b966c 100644 --- a/src/Backups/BackupCoordinationFileInfos.cpp +++ b/src/Backups/BackupCoordinationFileInfos.cpp @@ -80,24 +80,76 @@ void BackupCoordinationFileInfos::prepare() const if (plain_backup) { + std::vector unresolved_references; + std::unordered_map file_name_to_info; + + const auto try_resolve_reference = [&](BackupFileInfo & reference) + { + auto it = file_name_to_info.find(reference.reference_target); + + if (it == file_name_to_info.end()) + return false; + + auto & target_info = it->second; + target_info->reference_sources.push_back(reference.file_name); + reference.size = target_info->size; + total_size_of_files += reference.size; + reference.checksum = target_info->checksum; + return true; + }; + /// For plain backup all file infos are stored as is, without checking for duplicates or skipping empty files. for (size_t i = 0; i != file_infos_for_all_hosts.size(); ++i) { auto & info = *(file_infos_for_all_hosts[i]); - - if (!info.reference_target.empty()) - continue; - info.data_file_name = info.file_name; info.data_file_index = i; info.base_size = 0; /// Base backup must not be used while creating a plain backup. info.base_checksum = 0; - total_size_of_files += info.size; + + if (info.reference_target.empty()) + { + file_name_to_info.emplace(info.file_name, &info); + total_size_of_files += info.size; + } + else if (!try_resolve_reference(info)) + { + unresolved_references.push_back(&info); + } + } + + for (auto * reference : unresolved_references) + { + if (!try_resolve_reference(*reference)) + throw DB::Exception( + ErrorCodes::LOGICAL_ERROR, + "Couldn't resolve reference {} with target {}", + reference->file_name, + reference->reference_target); } + num_files = file_infos_for_all_hosts.size(); } else { + std::vector unresolved_references; + std::unordered_map file_name_to_info; + + const auto try_resolve_reference = [&](BackupFileInfo & reference) + { + auto it = file_name_to_info.find(reference.reference_target); + + if (it == file_name_to_info.end()) + return false; + + auto & target_info = it->second; + reference.size = target_info->size; + reference.checksum = target_info->checksum; + reference.data_file_name = target_info->data_file_name; + reference.data_file_index = target_info->data_file_index; + return true; + }; + /// For non-plain backups files with the same size and checksum are stored only once, /// in order to find those files we'll use this map. std::map data_file_index_by_checksum; @@ -107,7 +159,12 @@ void BackupCoordinationFileInfos::prepare() const auto & info = *(file_infos_for_all_hosts[i]); if (!info.reference_target.empty()) + { + if (!try_resolve_reference(info)) + unresolved_references.push_back(&info); + continue; + } if (info.size == info.base_size) { @@ -134,7 +191,21 @@ void BackupCoordinationFileInfos::prepare() const info.data_file_name = file_infos_for_all_hosts[it->second]->data_file_name; } } + + file_name_to_info.emplace(info.file_name, &info); + } + + for (auto * reference : unresolved_references) + { + if (!try_resolve_reference(*reference)) + throw DB::Exception( + ErrorCodes::LOGICAL_ERROR, + "Couldn't resolve reference {} with target {}", + reference->file_name, + reference->reference_target); } + + num_files = file_infos_for_all_hosts.size(); } prepared = true; diff --git a/src/Backups/BackupFileInfo.h b/src/Backups/BackupFileInfo.h index 1d5607fd4183..42bda3aa6ed8 100644 --- a/src/Backups/BackupFileInfo.h +++ b/src/Backups/BackupFileInfo.h @@ -42,6 +42,10 @@ struct BackupFileInfo /// Set if this file is just a reference to another file String reference_target; + /// List of files that are referencing this file + /// Used for plain backup which needs to resolve all references + Strings reference_sources; + struct LessByFileName { bool operator()(const BackupFileInfo & lhs, const BackupFileInfo & rhs) const { return (lhs.file_name < rhs.file_name); } diff --git a/src/Backups/BackupIO.h b/src/Backups/BackupIO.h index e4a82a604e83..91d57e5ab0ac 100644 --- a/src/Backups/BackupIO.h +++ b/src/Backups/BackupIO.h @@ -61,6 +61,8 @@ class IBackupWriter virtual void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, bool copy_encrypted, UInt64 start_pos, UInt64 length) = 0; + virtual void copyFile(const String & destination, const String & source, size_t size) = 0; + virtual void removeFile(const String & file_name) = 0; virtual void removeFiles(const Strings & file_names) = 0; diff --git a/src/Backups/BackupIO_Default.cpp b/src/Backups/BackupIO_Default.cpp index 5ac522695ce2..95f2c66b6b95 100644 --- a/src/Backups/BackupIO_Default.cpp +++ b/src/Backups/BackupIO_Default.cpp @@ -91,4 +91,6 @@ void BackupWriterDefault::copyFileFromDisk(const String & path_in_backup, DiskPt copyDataToFile(path_in_backup, create_read_buffer, start_pos, length); } + + } diff --git a/src/Backups/BackupIO_Disk.cpp b/src/Backups/BackupIO_Disk.cpp index 1e260ad22d96..91e8b97bc20a 100644 --- a/src/Backups/BackupIO_Disk.cpp +++ b/src/Backups/BackupIO_Disk.cpp @@ -128,4 +128,13 @@ void BackupWriterDisk::copyFileFromDisk(const String & path_in_backup, DiskPtr s BackupWriterDefault::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length); } +void BackupWriterDisk::copyFile(const String & destination, const String & source, size_t /*size*/) +{ + LOG_TRACE(log, "Copying file inside backup from {} to {} ", source, destination); + auto dest_file_path = root_path / destination; + auto src_file_path = root_path / source; + disk->createDirectories(dest_file_path.parent_path()); + disk->copyFile(src_file_path, *disk, dest_file_path, read_settings, write_settings); +} + } diff --git a/src/Backups/BackupIO_Disk.h b/src/Backups/BackupIO_Disk.h index 70d31eacc1a2..575ec3f57073 100644 --- a/src/Backups/BackupIO_Disk.h +++ b/src/Backups/BackupIO_Disk.h @@ -44,6 +44,8 @@ class BackupWriterDisk : public BackupWriterDefault void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, bool copy_encrypted, UInt64 start_pos, UInt64 length) override; + void copyFile(const String & destination, const String & source, size_t size) override; + void removeFile(const String & file_name) override; void removeFiles(const Strings & file_names) override; diff --git a/src/Backups/BackupIO_File.cpp b/src/Backups/BackupIO_File.cpp index 2bedb5470fb8..5384637a9693 100644 --- a/src/Backups/BackupIO_File.cpp +++ b/src/Backups/BackupIO_File.cpp @@ -152,4 +152,14 @@ void BackupWriterFile::copyFileFromDisk(const String & path_in_backup, DiskPtr s BackupWriterDefault::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length); } +void BackupWriterFile::copyFile(const String & destination, const String & source, size_t /*size*/) +{ + LOG_TRACE(log, "Copying file inside backup from {} to {} ", source, destination); + + auto abs_source_path = root_path / source; + auto abs_dest_path = root_path / destination; + fs::create_directories(abs_dest_path.parent_path()); + fs::copy(abs_source_path, abs_dest_path, fs::copy_options::overwrite_existing); +} + } diff --git a/src/Backups/BackupIO_File.h b/src/Backups/BackupIO_File.h index 6bb4b11e134d..ebe9a0f02cb5 100644 --- a/src/Backups/BackupIO_File.h +++ b/src/Backups/BackupIO_File.h @@ -38,6 +38,8 @@ class BackupWriterFile : public BackupWriterDefault void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, bool copy_encrypted, UInt64 start_pos, UInt64 length) override; + void copyFile(const String & destination, const String & source, size_t size) override; + void removeFile(const String & file_name) override; void removeFiles(const Strings & file_names) override; diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 8bb2f895e385..9688d7f07308 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -249,6 +249,24 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src BackupWriterDefault::copyFileFromDisk(path_in_backup, src_disk, src_path, copy_encrypted, start_pos, length); } +void BackupWriterS3::copyFile(const String & destination, const String & source, size_t size) +{ + LOG_TRACE(log, "Copying file inside backup from {} to {} ", source, destination); + copyS3File( + client, + client, + /* src_bucket */ s3_uri.bucket, + /* src_key= */ fs::path(s3_uri.key) / source, + 0, + size, + s3_uri.bucket, + fs::path(s3_uri.key) / destination, + s3_settings.request_settings, + read_settings, + {}, + threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterS3")); +} + void BackupWriterS3::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) { copyDataToS3File(create_read_buffer, start_pos, length, client, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, s3_settings.request_settings, {}, diff --git a/src/Backups/BackupIO_S3.h b/src/Backups/BackupIO_S3.h index 4abcbedf89fb..c00ce747ff5e 100644 --- a/src/Backups/BackupIO_S3.h +++ b/src/Backups/BackupIO_S3.h @@ -49,6 +49,8 @@ class BackupWriterS3 : public BackupWriterDefault void copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path, bool copy_encrypted, UInt64 start_pos, UInt64 length) override; + void copyFile(const String & destination, const String & source, size_t size) override; + void removeFile(const String & file_name) override; void removeFiles(const Strings & file_names) override; diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index bf1853828df6..56c30fab5c2b 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -362,10 +362,10 @@ void BackupImpl::writeBackupMetadata() *out << ""; *out << "" << xml << info.file_name << ""; + *out << "" << info.size << ""; if (info.size) { - *out << "" << info.size << ""; *out << "" << hexChecksum(info.checksum) << ""; if (info.base_size) { @@ -381,10 +381,6 @@ void BackupImpl::writeBackupMetadata() if (info.encrypted_by_disk) *out << "true"; } - else if (!info.reference_target.empty()) - *out << "" << xml << info.reference_target << ""; - else - *out << "" << info.size << ""; total_size += info.size; bool has_entry = !deduplicate_files || (info.size && (info.size != info.base_size) && (info.data_file_name.empty() || (info.data_file_name == info.file_name))); @@ -465,13 +461,6 @@ void BackupImpl::readBackupMetadata() BackupFileInfo info; info.file_name = getString(file_config, "name"); - info.reference_target = getString(file_config, "reference_target", ""); - if (!info.reference_target.empty()) - { - reference_files.emplace_back(std::move(info.file_name), std::move(info.reference_target)); - continue; - } - info.size = getUInt64(file_config, "size"); if (info.size) { @@ -521,14 +510,6 @@ void BackupImpl::readBackupMetadata() } } - for (auto & [source_file, target_file] : reference_files) - { - auto it = file_names.find(target_file); - if (it == file_names.end()) - throw Exception(ErrorCodes::BACKUP_ENTRY_NOT_FOUND, "Backup entry {} referenced by {} not found", target_file, source_file); - file_names.emplace(std::move(source_file), it->second); - } - uncompressed_size = size_of_entries + str.size(); compressed_size = uncompressed_size; if (!use_archive) @@ -954,6 +935,12 @@ void BackupImpl::writeFile(const BackupFileInfo & info, BackupEntryPtr entry) writer->copyDataToFile(info.data_file_name, create_read_buffer, info.base_size, info.size - info.base_size); } + if (!deduplicate_files) + { + for (const auto & reference : info.reference_sources) + writer->copyFile(reference, info.data_file_name, info.size - info.base_size); + } + { std::lock_guard lock{mutex}; ++num_entries; diff --git a/tests/integration/test_backup_restore_keeper_map/test.py b/tests/integration/test_backup_restore_keeper_map/test.py index 8343ad3177fa..c401f482c3ff 100644 --- a/tests/integration/test_backup_restore_keeper_map/test.py +++ b/tests/integration/test_backup_restore_keeper_map/test.py @@ -65,22 +65,24 @@ def new_backup_name(base_name): return f"Disk('backups', '{base_name}{backup_id_counter}')" -def test_on_cluster(): - node1.query_with_retry("CREATE DATABASE keeper_backup ON CLUSTER cluster") +@pytest.mark.parametrize("deduplicate_files", [0, 1]) +def test_on_cluster(deduplicate_files): + database_name = f"keeper_backup{deduplicate_files}" + node1.query_with_retry(f"CREATE DATABASE {database_name} ON CLUSTER cluster") node1.query_with_retry( - "CREATE TABLE keeper_backup.keeper1 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key" + f"CREATE TABLE {database_name}.keeper1 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/{database_name}/test_on_cluster1') PRIMARY KEY key" ) node1.query_with_retry( - "CREATE TABLE keeper_backup.keeper2 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster1') PRIMARY KEY key" + f"CREATE TABLE {database_name}.keeper2 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/{database_name}/test_on_cluster1') PRIMARY KEY key" ) node1.query_with_retry( - "CREATE TABLE keeper_backup.keeper3 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/test_on_cluster2') PRIMARY KEY key" + f"CREATE TABLE {database_name}.keeper3 ON CLUSTER cluster (key UInt64, value String) Engine=KeeperMap('/{database_name}/test_on_cluster2') PRIMARY KEY key" ) node1.query_with_retry( - "INSERT INTO keeper_backup.keeper2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5" + f"INSERT INTO {database_name}.keeper2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5" ) node1.query_with_retry( - "INSERT INTO keeper_backup.keeper3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5" + f"INSERT INTO {database_name}.keeper3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5" ) expected_result = "".join(f"{i}\ttest{i}\n" for i in range(5)) @@ -89,7 +91,7 @@ def verify_data(): for node in [node1, node2, node3]: for i in range(1, 4): result = node.query_with_retry( - f"SELECT key, value FROM keeper_backup.keeper{i} ORDER BY key FORMAT TSV" + f"SELECT key, value FROM {database_name}.keeper{i} ORDER BY key FORMAT TSV" ) assert result == expected_result @@ -97,10 +99,10 @@ def verify_data(): backup_name = new_backup_name("test_on_cluster") node1.query( - f"BACKUP DATABASE keeper_backup ON CLUSTER cluster TO {backup_name} SETTINGS async = false;" + f"BACKUP DATABASE {database_name} ON CLUSTER cluster TO {backup_name} SETTINGS async = false, deduplicate_files = {deduplicate_files};" ) - node1.query("DROP DATABASE keeper_backup ON CLUSTER cluster SYNC;") + node1.query(f"DROP DATABASE {database_name} ON CLUSTER cluster SYNC;") def apply_for_all_nodes(f): for node in [node1, node2, node3]: @@ -121,14 +123,14 @@ def change_keeper_map_prefix(node): apply_for_all_nodes(lambda node: node.start_clickhouse()) node1.query( - f"RESTORE DATABASE keeper_backup ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;" + f"RESTORE DATABASE {database_name} ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;" ) verify_data() - node1.query("DROP TABLE keeper_backup.keeper3 ON CLUSTER cluster SYNC;") + node1.query(f"DROP TABLE {database_name}.keeper3 ON CLUSTER cluster SYNC;") node1.query( - f"RESTORE TABLE keeper_backup.keeper3 ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;" + f"RESTORE TABLE {database_name}.keeper3 ON CLUSTER cluster FROM {backup_name} SETTINGS async = false;" ) verify_data() From 9df2775f08d5ab377ba3aa2dd05010329b67ef20 Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Tue, 14 Nov 2023 02:28:09 +0000 Subject: [PATCH 061/274] reduce timeout and setTimeout earlier. Signed-off-by: Jianfei Hu --- src/Coordination/KeeperContext.cpp | 7 ++++--- src/IO/S3/Credentials.cpp | 10 +++++----- tests/integration/helpers/keeper_config2.xml | 1 + .../integration/test_keeper_availability_zone/test.py | 6 ++++-- .../integration/test_keeper_four_word_command/test.py | 8 ++++---- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/Coordination/KeeperContext.cpp b/src/Coordination/KeeperContext.cpp index 9745a53d1ab1..c3cb166abeef 100644 --- a/src/Coordination/KeeperContext.cpp +++ b/src/Coordination/KeeperContext.cpp @@ -39,8 +39,9 @@ void KeeperContext::initialize(const Poco::Util::AbstractConfiguration & config, if (config.hasProperty("keeper_server.availability_zone")) { - auto keeper_az = config.getString("keeper_server.availability_zone.value"); - if (config.getBool("keeper_server.availability_zone.enable_auto_detection_on_cloud", false)) + auto keeper_az = config.getString("keeper_server.availability_zone.value", ""); + const auto auto_detect_for_cloud = config.getBool("keeper_server.availability_zone.enable_auto_detection_on_cloud", false); + if (keeper_az.empty() && auto_detect_for_cloud) { try { @@ -54,7 +55,7 @@ void KeeperContext::initialize(const Poco::Util::AbstractConfiguration & config, if (!keeper_az.empty()) { system_nodes_with_data[keeper_availability_zone_path] = keeper_az; - LOG_INFO(&Poco::Logger::get("KeeperContext"), "Initialize the KeeperContext with availability zone: '{}'.'. ", keeper_az); + LOG_INFO(&Poco::Logger::get("KeeperContext"), "Initialize the KeeperContext with availability zone: '{}'", keeper_az); } } diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index 4ba145725891..7d6ed0944861 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -65,7 +65,7 @@ bool areCredentialsEmptyOrExpired(const Aws::Auth::AWSCredentials & credentials, } const char SSO_CREDENTIALS_PROVIDER_LOG_TAG[] = "SSOCredentialsProvider"; -const int AVAILABILITY_ZONE_REQUEST_TIMEOUT_SECONDS = 5; +const int AVAILABILITY_ZONE_REQUEST_TIMEOUT_SECONDS = 3; } @@ -241,11 +241,11 @@ String AWSEC2MetadataClient::getAvailabilityZoneOrException() { Poco::URI uri(getAWSMetadataEndpoint() + EC2_AVAILABILITY_ZONE_RESOURCE); Poco::Net::HTTPClientSession session(uri.getHost(), uri.getPort()); + session.setTimeout(Poco::Timespan(AVAILABILITY_ZONE_REQUEST_TIMEOUT_SECONDS, 0)); Poco::Net::HTTPResponse response; Poco::Net::HTTPRequest request(Poco::Net::HTTPRequest::HTTP_GET, uri.getPath()); session.sendRequest(request); - session.setTimeout(Poco::Timespan(AVAILABILITY_ZONE_REQUEST_TIMEOUT_SECONDS, 0)); std::istream & rs = session.receiveResponse(response); if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) @@ -287,17 +287,17 @@ String getRunningAvailabilityZoneImpl() auto aws_az = AWSEC2MetadataClient::getAvailabilityZoneOrException(); return aws_az; } - catch (const DB::Exception & aws_ex) + catch (const std::exception & aws_ex) { try { auto gcp_zone = getGCPAvailabilityZoneOrException(); return gcp_zone; } - catch (const DB::Exception & gcp_ex) + catch (const std::exception & gcp_ex) { throw DB::Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Failed to find the availability zone, tried AWS and GCP. AWS Error: {}\nGCP Error: {}", aws_ex.displayText(), gcp_ex.displayText()); + "Failed to find the availability zone, tried AWS and GCP. AWS Error: {}\nGCP Error: {}", aws_ex.what(), gcp_ex.what()); } } } diff --git a/tests/integration/helpers/keeper_config2.xml b/tests/integration/helpers/keeper_config2.xml index 0c58aaceb1cd..2afff2f5e59e 100644 --- a/tests/integration/helpers/keeper_config2.xml +++ b/tests/integration/helpers/keeper_config2.xml @@ -14,6 +14,7 @@ 2 az-zoo2 + 1 diff --git a/tests/integration/test_keeper_availability_zone/test.py b/tests/integration/test_keeper_availability_zone/test.py index b78e776f3c63..a2003f8539ef 100644 --- a/tests/integration/test_keeper_availability_zone/test.py +++ b/tests/integration/test_keeper_availability_zone/test.py @@ -27,10 +27,12 @@ def test_get_availability_zone(): with KeeperClient.from_cluster(cluster, "zoo1") as client1: assert client1.get("/keeper/availability_zone") == "az-zoo1" + # Keeper2 set enable_auto_detection_on_cloud to true, but is ignored and az-zoo2 is used. with KeeperClient.from_cluster(cluster, "zoo2") as client2: assert client2.get("/keeper/availability_zone") == "az-zoo2" - + assert "availability_zone" in client2.ls("/keeper") + # keeper3 is not configured with availability_zone value. with KeeperClient.from_cluster(cluster, "zoo3") as client3: with pytest.raises(Exception): - client3.get("/keeper/availability_zone") \ No newline at end of file + client3.get("/keeper/availability_zone") diff --git a/tests/integration/test_keeper_four_word_command/test.py b/tests/integration/test_keeper_four_word_command/test.py index 25c4bc553275..71501133ae76 100644 --- a/tests/integration/test_keeper_four_word_command/test.py +++ b/tests/integration/test_keeper_four_word_command/test.py @@ -183,8 +183,8 @@ def test_cmd_mntr(started_cluster): # contains: # 10 nodes created by test # 3 nodes created by clickhouse "/clickhouse/task_queue/ddl" - # 1 root node, 4 keeper system nodes - assert int(result["zk_znode_count"]) == 15 + # 1 root node, 3 keeper system nodes + assert int(result["zk_znode_count"]) == 14 assert int(result["zk_watch_count"]) == 2 assert int(result["zk_ephemerals_count"]) == 2 assert int(result["zk_approximate_data_size"]) > 0 @@ -333,7 +333,7 @@ def test_cmd_srvr(started_cluster): assert int(result["Connections"]) == 1 assert int(result["Zxid"], 16) > 10 assert result["Mode"] == "leader" - assert result["Node count"] == "15" + assert result["Node count"] == "14" finally: destroy_zk_client(zk) @@ -373,7 +373,7 @@ def test_cmd_stat(started_cluster): assert int(result["Connections"]) == 1 assert int(result["Zxid"], 16) >= 10 assert result["Mode"] == "leader" - assert result["Node count"] == "15" + assert result["Node count"] == "14" # filter connection statistics cons = [n for n in data.split("\n") if "=" in n] From 64c2a696666d594783c1996c0910166cacba000f Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Tue, 14 Nov 2023 20:28:37 +0100 Subject: [PATCH 062/274] check performance --- base/poco/Net/src/HTTPSession.cpp | 32 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index 9ebbd7d04cd7..97decded282f 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -94,22 +94,24 @@ void HTTPSession::setTimeout(const Poco::Timespan& timeout) void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco::Timespan& sendTimeout, const Poco::Timespan& receiveTimeout) { _connectionTimeout = connectionTimeout; + _sendTimeout = sendTimeout; + _receiveTimeout = receiveTimeout; - if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) - { - _sendTimeout = sendTimeout; - - if (connected()) - _socket.setSendTimeout(_sendTimeout); - } - - if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) - { - _receiveTimeout = receiveTimeout; - - if (connected()) - _socket.setReceiveTimeout(_receiveTimeout); - } +// if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) +// { +// _sendTimeout = sendTimeout; +// +// if (connected()) +// _socket.setSendTimeout(_sendTimeout); +// } +// +// if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) +// { +// _receiveTimeout = receiveTimeout; +// +// if (connected()) +// _socket.setReceiveTimeout(_receiveTimeout); +// } } From cd909ffc48cc4fb6fb7bc23843659c9559c5921a Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 14 Nov 2023 20:13:09 +0000 Subject: [PATCH 063/274] Test RabbitMQ with secure connection --- .../compose/docker_compose_rabbitmq.yml | 6 ++- .../integration/runner/misc/rabbitmq.conf | 8 --- .../runner/misc/rabbitmq/ca-cert.pem | 32 ++++++++++++ .../runner/misc/rabbitmq/generate_certs.sh | 10 ++++ .../runner/misc/rabbitmq/rabbitmq.conf | 15 ++++++ .../runner/misc/rabbitmq/server-cert.pem | 33 ++++++++++++ .../runner/misc/rabbitmq/server-key.pem | 52 +++++++++++++++++++ .../integration/test_storage_rabbitmq/test.py | 28 +++++++--- 8 files changed, 168 insertions(+), 16 deletions(-) delete mode 100644 docker/test/integration/runner/misc/rabbitmq.conf create mode 100644 docker/test/integration/runner/misc/rabbitmq/ca-cert.pem create mode 100755 docker/test/integration/runner/misc/rabbitmq/generate_certs.sh create mode 100644 docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf create mode 100644 docker/test/integration/runner/misc/rabbitmq/server-cert.pem create mode 100644 docker/test/integration/runner/misc/rabbitmq/server-key.pem diff --git a/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml b/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml index 2db9fb589d2a..61b21e0e3d94 100644 --- a/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml +++ b/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml @@ -6,9 +6,13 @@ services: hostname: rabbitmq1 expose: - ${RABBITMQ_PORT:-5672} + - ${RABBITMQ_SECURE_PORT:-5671} volumes: - type: ${RABBITMQ_LOGS_FS:-tmpfs} source: ${RABBITMQ_LOGS:-} target: /rabbitmq_logs/ - "${RABBITMQ_COOKIE_FILE}:/var/lib/rabbitmq/.erlang.cookie" - - /misc/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf \ No newline at end of file + - /misc/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf + - /misc/rabbitmq/ca-cert.pem:/etc/rabbitmq/ca-cert.pem + - /misc/rabbitmq/server-cert.pem:/etc/rabbitmq/server-cert.pem + - /misc/rabbitmq/server-key.pem:/etc/rabbitmq/server-key.pem diff --git a/docker/test/integration/runner/misc/rabbitmq.conf b/docker/test/integration/runner/misc/rabbitmq.conf deleted file mode 100644 index 3527c83880b8..000000000000 --- a/docker/test/integration/runner/misc/rabbitmq.conf +++ /dev/null @@ -1,8 +0,0 @@ -loopback_users.guest = false -listeners.tcp.default = 5672 -default_pass = clickhouse -default_user = root -management.tcp.port = 15672 - -log.file = /rabbitmq_logs/rabbit.log -log.file.level = debug diff --git a/docker/test/integration/runner/misc/rabbitmq/ca-cert.pem b/docker/test/integration/runner/misc/rabbitmq/ca-cert.pem new file mode 100644 index 000000000000..4a7b88f79368 --- /dev/null +++ b/docker/test/integration/runner/misc/rabbitmq/ca-cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFhTCCA22gAwIBAgIUWhfjFfbwannH3KIqITDtgcvSItMwDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjMxMTE0 +MTgyODI2WhcNMzMxMTExMTgyODI2WjBSMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQsw +CQYDVQQDDAJjYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAJfJegdC +gavNGYzSdva+5QMxGvqyLwZzjophMeyEzlW/Di4KFGPho+fVlVMB/EwaTRoBRLEu +SQusQwoFg71mGvUTOpgHzlsUz4vcVVFOiL4bJdzCWQKzdC8M8rUFoks9FMboVeSx +jhAnKAm/NpCLpm9VYnRjEq2KEbJp7VkPAHgZEXR7VABwCFvmDcztrfcWfmXxm6IH +o+AkF/nqdphLu7Q1yDQiF8Q8TuszuhqgQ7/1PrRcaSADrF15jJjQb05sILpGCT3e +lxJYId5RF0+fgTIqy03bAKB53+8V8cAkowI4rvPTmcFXhcG3rkDO6lyZixHhlpKi +PmXEzHh0kfsRjzkNBP0CKqPnu3D2iymROiPAH2cteaYe6jdD2HIjuVLk/TjX1ZFy +DlZCrJIwj0l8A2xAfLq8Gw5RSr0a9k5TiMD5nZtfd12Vd0K82vO32vmcjO2Igddc +VWccDDwUY/ZWV3uznkusOBrB8wba3ZsXA5hjJzs0KlTvQKPjX0y4lFMmZGbelwjt +pR5dRNLi5XTdMPzV0mAnvJhDTFEmME19Bh6AEsjuAz3gHUdwNTbSxUS3mF/hTL9k +v2wh5udUAOwqD1uEzqPJyG4JCJQozIDOEEZVixWqQ60b9wUHN8meqO4y9fxTdmHW +Vo5BAF1xEJhJJb0QY/O6GahPtWqb/Mr1rtPJAgMBAAGjUzBRMB0GA1UdDgQWBBSw +fQcOabXwX/v9F1hd2cmuIug56jAfBgNVHSMEGDAWgBSwfQcOabXwX/v9F1hd2cmu +Iug56jAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQAms8y6RVxl +mKSUbsU8JscYwOzcRUQJWETeIr4rtZvMHH+3vkdBU0yKxGpEm7U8J3+5oVTYPhbs +11ZAL+DvIZ6gT6pjDvECyVox1OkjNogz843fTMbNqjuuehjSKXwpMTy5/kmT2aLj +//nBi5UX1xo3RQ9vtmBwzZ3VFK99DFXraDOPS/yk43WV2uqdWsXCNvyEyCHmM1IB +9FQe2EFcO6s4/N+TarhIZ8Udhj5bl8d4eDd1yEckmTD4aHJBgMII2uEwrAxR5CT1 +tCqUKutvNrkXI5PIULvmy+Lwm7PJAC7grPtUHK6anSugpljd7bFj18fHH9APiC45 +Ou4OOK1BUZogCEo7rD36UlanxQO0GEzgDCVEoEdoe0WRdc6T9b4fM8vpQqwBdf9t +nkPB8oLCKerqqYwCiMuWm4BcRmExA7ypIkUCcluGO9/kTmdps3NqOvET9oLTjXuA +z5TPmaK5a3poKLoxBfv6WfRTgisOnMNTsjL1R8+xuhEn5hSlE2r3wAi8Cys9Z9PV +LhTj0SRTXILd2NW3lO8QfO0pGdjgk90GqkyUY9YjuiMVPvdUAFQsHm+0GEZEXjOD +Bw7tLSJQ4IKhfactg/Puxd15ahcWAxeelyED+w/zVGdHYblqbvfdtiGj370KVhoj +DL5HkdPa0IhTPqMBnmoVQ4C/WzKofXBjQQ== +-----END CERTIFICATE----- diff --git a/docker/test/integration/runner/misc/rabbitmq/generate_certs.sh b/docker/test/integration/runner/misc/rabbitmq/generate_certs.sh new file mode 100755 index 000000000000..442d2fe004f8 --- /dev/null +++ b/docker/test/integration/runner/misc/rabbitmq/generate_certs.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# 1. Generate CA's private key and self-signed certificate +openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout ca-key.pem -out ca-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=ca" + +# 2. Generate server's private key and certificate signing request (CSR) +openssl req -newkey rsa:4096 -nodes -batch -keyout server-key.pem -out server-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=server" + +# 3. Use CA's private key to sign server's CSR and get back the signed certificate +openssl x509 -req -days 3650 -in server-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -extfile server-ext.cnf -out server-cert.pem diff --git a/docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf b/docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf new file mode 100644 index 000000000000..307871ba5899 --- /dev/null +++ b/docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf @@ -0,0 +1,15 @@ +loopback_users.guest = false +listeners.tcp.default = 5672 +default_pass = clickhouse +default_user = root +management.tcp.port = 15672 + +log.file = /rabbitmq_logs/rabbit.log +log.file.level = debug + +listeners.ssl.default = 5671 +ssl_options.verify = verify_none +ssl_options.fail_if_no_peer_cert = false +ssl_options.cacertfile = /etc/rabbitmq/ca_cert.pem +ssl_options.certfile = /etc/rabbitmq/server_cert.pem +ssl_options.keyfile = /etc/rabbitmq/server_key.pem diff --git a/docker/test/integration/runner/misc/rabbitmq/server-cert.pem b/docker/test/integration/runner/misc/rabbitmq/server-cert.pem new file mode 100644 index 000000000000..338de91aa0fb --- /dev/null +++ b/docker/test/integration/runner/misc/rabbitmq/server-cert.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFpTCCA42gAwIBAgIUJvQslezZO09XgFGQCxOM6orIsWowDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjMxMTE0 +MTgyODI5WhcNMzMxMTExMTgyODI5WjBWMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8w +DQYDVQQDDAZzZXJ2ZXIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCe +o/K71WdKpVpdDvhaZy6wBVhFlu7j7DhfTSYvcPpAJfExmzO8JK3vh5/yGyAO1t79 +gAjqyXLMCZKw7ajM2rez9YnGYqaFi70BlTcU2KQ8LbFEYRc3cYNDmmWIKBpwpSri +We5SQrRLnDXqAn6T8FG5ejQ/t+1IUMrtZENB4lp8fBmEOJb5yr1TE++6EhiDBQho +cLDWWWP8b55kyZhqP/VgmId4lvboGMRKxbiRJ6/SPr/i/pteBD8jTYfbJr6ceXov +/p5yxIp61z5ry1anU7W3B8jTl/gj7SqtFdSnRajZ0DGJJAUKpiiJSCSlp5YB5Ub2 +eBBMHmdA5R1MuiU9TOA35nUW5wkhEOJXnBR/WCsYioVmn/+5dm6JPYiwp/TefYnr +x9iLbb/Tyx7MnXzeyvKg781SwmnvS6Blhtr0zhAW9szZz8cVHPBqFs6PzGs/5mwE +C+tM3Zp85aHd28nIT4NQLHdMDwVmGwmPdy4uavtYWMDhsuIyEU8hCZymiHhPnuHU +VbmfZ8GOTIzUgQAvZb0fL1Xow2Tf6XuARnvuU9weRttg9jSOqPuUENRsFXv0mU8M +EpQjrxry88Wfz7bBEjN5JHC16PB/Nu7zTGJ4/slThbxNv0bIONzvTBPbXrKnxw7Z +d9WhGJI+LQxRqLTynQe6yzDwIuW9LRdBNTp7CtQRwQIDAQABo28wbTArBgNVHREE +JDAigiBpbnRlZ3JhdGlvbi10ZXN0cy5jbGlja2hvdXNlLmNvbTAdBgNVHQ4EFgQU +54GvBUYWvMADpTz/zglwMlaJuskwHwYDVR0jBBgwFoAUsH0HDmm18F/7/RdYXdnJ +riLoOeowDQYJKoZIhvcNAQELBQADggIBADfNH6O6ay+xg0XmV6sR0n4j6PwL9Cnc +VjuCmHQbpFXfMvgCdfHvbtT0Y/pG7IoeKmrrm0JPvKa2E9Ht0j6ZnowQ2m9mJk8U +5Fd/PbC1I4KgVCw6HRSOcwqANJxOGe7RyN9PTZZ8fxzmzIR3FiQ2bXfr+LaotZOK +aVS8F8xCOzoMvL9LFls2YpEn20p/1EATIf2MFX3j9vKfcJVOyDJV4i5BMImStFLM +g3sdC96de/59yxt9khM0PNucU1ldNFs/kZVEcNSwGOAIgQEPwULJtDY+ZSWeROpX +EpWndN6zQsv1pdNvLtXsDXfi4YoH9QVaA/k4aFFJ08CjSZfMYmwyPOGsf/wqT65i +ADID2yb1A/FIIe/fM+d2gXHBVFBDmydJ1JCdCoYrEJgfWj1LO/0jLi34ZZ17Hu7F +D33fLARF9nlLzlUiWjcQlOjNoCM48AgG/3wHk4eiSfc/3PIJDuDGDa0NdtDeKKhH +XkP2ll4cMUH6EQ9KO1jHPmf5RokX4QJgH+ofO4U5XQFwc3lOyJzEQnED+wame7do +R7TE4F/OXhxLqA6DFkzXe89/kSCoAF9bjzmUn/ilrg8NXKKgprgHg4DJHgvCQVVC +34ab7Xj7msUm4D9vI+GAeUbUqnqCaWxDF6vCMT0Qq7iSVDxa/SV8TX8Vp2Zh+PSh +4m23Did+KjLq +-----END CERTIFICATE----- diff --git a/docker/test/integration/runner/misc/rabbitmq/server-key.pem b/docker/test/integration/runner/misc/rabbitmq/server-key.pem new file mode 100644 index 000000000000..92e93e8fba5c --- /dev/null +++ b/docker/test/integration/runner/misc/rabbitmq/server-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCeo/K71WdKpVpd +DvhaZy6wBVhFlu7j7DhfTSYvcPpAJfExmzO8JK3vh5/yGyAO1t79gAjqyXLMCZKw +7ajM2rez9YnGYqaFi70BlTcU2KQ8LbFEYRc3cYNDmmWIKBpwpSriWe5SQrRLnDXq +An6T8FG5ejQ/t+1IUMrtZENB4lp8fBmEOJb5yr1TE++6EhiDBQhocLDWWWP8b55k +yZhqP/VgmId4lvboGMRKxbiRJ6/SPr/i/pteBD8jTYfbJr6ceXov/p5yxIp61z5r +y1anU7W3B8jTl/gj7SqtFdSnRajZ0DGJJAUKpiiJSCSlp5YB5Ub2eBBMHmdA5R1M +uiU9TOA35nUW5wkhEOJXnBR/WCsYioVmn/+5dm6JPYiwp/TefYnrx9iLbb/Tyx7M +nXzeyvKg781SwmnvS6Blhtr0zhAW9szZz8cVHPBqFs6PzGs/5mwEC+tM3Zp85aHd +28nIT4NQLHdMDwVmGwmPdy4uavtYWMDhsuIyEU8hCZymiHhPnuHUVbmfZ8GOTIzU +gQAvZb0fL1Xow2Tf6XuARnvuU9weRttg9jSOqPuUENRsFXv0mU8MEpQjrxry88Wf +z7bBEjN5JHC16PB/Nu7zTGJ4/slThbxNv0bIONzvTBPbXrKnxw7Zd9WhGJI+LQxR +qLTynQe6yzDwIuW9LRdBNTp7CtQRwQIDAQABAoICAA0lev0T3z5xW36wueYL/PN7 +TehebKeYsMc9BngR/bsJKea5fN0PkRZzf865brusFMifLp3+WbQM6wocd8uaKHUS +WPuGu1P/04bpDap9lYajJriK7ziaAI2+osFYyXAiT954I2bPvk8xv8oHsOOjm7Iq +LWBGZrSCdX6cu3IfRu5f/mFVqzVCFtRmp4wc6ckZxquZAx6QQ9fsjAzAJBBSAoyh +t0BICmgLfWDQ582no0tiBdbS0J9G7NCJIUQI/uzKqFSH3iuWm/84DSUzsZemOT3U +uFDInDil885qK7g87pQ2S5SY1o4eXOebgeX0cFrx3CKaqocUUewv0HDGUEW3NDFs +KhUvlJZIFgk6bMend16U6kfRCUsjLA22Rfxzanl53cGVywCeIMirnLYuEu0TsxyK +CblBvyhcpjrGi7FQskzR+J9LpZPnmtn6TAb7JCAALRVHcAGKhGeh613SjPUfkWb0 +KpDps08x8MWGEAALuHbOK0nMLFm+PuMt7+krqCeJET+XM44GT+6ZstrDv0RufxUN ++pkLW7AsVZoXcFvaOWjuyBvX/f6UHCSfueo0mB3H80WoftDIfdhM+AI7/oBTYCBx +Z8BtW+g7Eq3pOUg/Um7S7Z2bybBWE14kpi95gRf3upEYPqHJUpJPdu20lk24iAt9 +LCXF4AjZBIdAuyJrYOJBAoIBAQDd/Bm14WvmBOablGLn6hmohi6M75D+/eQanlg9 +eJhXJUVd8FzOTjKi70EHWvkqswenNDbe/WGtImqG+9G+N/ol2qhi5xVSQ2XQmcVQ +U+k15Bzm9xKM0OqsStFvRgP1Cy6Ms3/jxr5JEEwUepmjvWTDGTlhTQASA/D7Uh2q +5HpPiHEVm4g5eTAYWeAbI6cGwVS0L4y6xkFGde37Kh2P8ZodWB+d3fglVu4Ok9Nf +wE2f8MK2ewQ0SbF/Nj2WjlVomvOvOJG/2CDLuiH/vc4YUvLAm8pNwvsmgtSh1Okt +E/HfXegrlPPEgw6owqoQFt+aGUITgEhiwEVAcYS0pXzzkQX5AoIBAQC28wJ8ueKr +fINpJM2pSc7WRDFduP5yGsRreSLBXLKMbvOlIVb3PaWp11Cg3+X5O90bPXYJ9mBI +WGR0g14/VD8edxs2D5TUZcP4/vKXGHaWRY9Z4A3jVpjzAxAaviNDHJ08tLXEMXZQ +lbA7dX8z6lpoQfwnPzjBwB01mVegwXPeIwIIfT/FmAiGzvSnAMXBGSGWRRdzof0M +/vPFbgllcQmM4AnEGcErCgFRpwcssO87T2jnvf6QVE5JCcnUcGIli1ThxCU9TRZM +5s6R7Nvk3/UjwcpRcqMtnGpTT2QXSnRwvWUfM+bKTwaxz4PjqKpgIc11kwJAjlxk +4CxYf1mDGLwJAoIBAGFJRTNS8ejDKRXyOE6PaGNVOz2FGLTILJoF34JBQfKfYQFE +gEfiOYry9Dr3AdBW2fnLhmi//3jTZoB2CHwnKDhC1h1STSPaadq8KZ+ExuZZbNlE +WxrfzJlpyNPNiZpxJht/54K57Vc0D0PCX2dFb82ZVm5wQqGinJBocpwcugX1NCpW +GaOmmw9xBCigvWjWffriA/kvPhhVQtEaqg4Vwoctwd18FG645Gf7HV4Pd3WrHIrA +6xzHV0T7To6XHpNTpYybbDT50ZW3o4LjellqsPz8yfK+izdbizjJiM+6t/w+uauw +Ag2Tqm8HsWSPwbtVaoIFbLPqs+8EUTaieFp+qnECggEAVuaTdd9uFfrtCNKchh8z +CoAV2uj2pAim6E3//k0j2qURQozVnFdCC6zk9aWkvYB8BGZrXUwUbAjgnp+P8xD3 +cmctG77G+STls66WWMMcAUFFWHGe5y/JMxVvXuSWJ1i+L4m/FVRRWPHhZjznkSdu +jjtZpOLY+N9igIU4JHn/qbKDUrj7w8X1tuMzPuiVBqYDWDe1bg2x/6xS6qLb/71z +xeDdgrKhGOqFud1XARmCaW/M6tdKxg/lp7fokOpZFHBcf2kGL1ogj6LK2HHj+ZGQ +Bc4VZh7H9/BmaPA7IP0S1kKAeBPVOp/TFD737Pm/BC7KQ2DzHusAZEI/jkHfqO/k +0QKCAQEAuiYLn9iLgk4uQO9oaSBGWKrJsR2L2dqI7IWU0X9xJlsQrJKcEeWg4LXt +djLsz0HrxZV/c+Pnh79hmFlBoEmH+hz32D/xd+/qrwwAcMkHAwMbznJu0IIuW2O9 +Uzma++7SvVmr9H0DkUwXFP3jn1A2n3uuI4czqtQ8N7GiH0UAWR5CsIP7azHvZTSj +s4Fzf8rTE6pNqVgQXjrVbI9H/h0uPP4alJbhnPba9mgB1cGmfBEnPkKgYNqSZse+ +95G2TlcK74sKBUSdBKqYBZ4ZUeTXV974Nva9guE9vzDQt1Cj6k0HWISVPUshPzIh +qrdHdxcM6yhA0Z0Gu6zj+Zsy4lU8gA== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 983e52ca294a..837263bd70f1 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -102,18 +102,32 @@ def rabbitmq_setup_teardown(): # Tests -def test_rabbitmq_select(rabbitmq_cluster): +@pytest.mark.parametrize( + "secure", + [ + pytest.param(0), + pytest.param(1), + ], +) +def test_rabbitmq_select(rabbitmq_cluster, secure): + port = 5672 + if secure: + port = 5671 + instance.query( """ CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ - SETTINGS rabbitmq_host_port = '{}:5672', + SETTINGS rabbitmq_host_port = '{}:{}', rabbitmq_exchange_name = 'select', rabbitmq_commit_on_select = 1, rabbitmq_format = 'JSONEachRow', - rabbitmq_row_delimiter = '\\n'; + rabbitmq_row_delimiter = '\\n', + rabbitmq_secure = {}; """.format( - rabbitmq_cluster.rabbitmq_host + rabbitmq_cluster.rabbitmq_host, + port, + secure ) ) @@ -3442,18 +3456,18 @@ def test_rabbitmq_handle_error_mode_stream(rabbitmq_cluster): rabbitmq_row_delimiter = '\\n', rabbitmq_handle_error_mode = 'stream'; - + CREATE TABLE test.errors (error Nullable(String), broken_message Nullable(String)) ENGINE = MergeTree() ORDER BY tuple(); CREATE MATERIALIZED VIEW test.errors_view TO test.errors AS SELECT _error as error, _raw_message as broken_message FROM test.rabbit where not isNull(_error); - + CREATE TABLE test.data (key UInt64, value UInt64) ENGINE = MergeTree() ORDER BY key; - + CREATE MATERIALIZED VIEW test.view TO test.data AS SELECT key, value FROM test.rabbit; """.format( From 369aaef92fb280c4b7ae4e5e04d0da2930718e3b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 14 Nov 2023 20:26:32 +0000 Subject: [PATCH 064/274] Automatic style fix --- tests/integration/test_storage_rabbitmq/test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 837263bd70f1..adb7f59769a1 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -125,9 +125,7 @@ def test_rabbitmq_select(rabbitmq_cluster, secure): rabbitmq_row_delimiter = '\\n', rabbitmq_secure = {}; """.format( - rabbitmq_cluster.rabbitmq_host, - port, - secure + rabbitmq_cluster.rabbitmq_host, port, secure ) ) From 07452b613a6b147c53530d6325fc9038ce58f675 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 14 Nov 2023 20:35:54 +0000 Subject: [PATCH 065/274] Fix certificate's file names --- docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf | 6 +++--- docker/test/integration/runner/misc/rabbitmq/server-ext.cnf | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 docker/test/integration/runner/misc/rabbitmq/server-ext.cnf diff --git a/docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf b/docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf index 307871ba5899..258a282907a0 100644 --- a/docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf +++ b/docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf @@ -10,6 +10,6 @@ log.file.level = debug listeners.ssl.default = 5671 ssl_options.verify = verify_none ssl_options.fail_if_no_peer_cert = false -ssl_options.cacertfile = /etc/rabbitmq/ca_cert.pem -ssl_options.certfile = /etc/rabbitmq/server_cert.pem -ssl_options.keyfile = /etc/rabbitmq/server_key.pem +ssl_options.cacertfile = /etc/rabbitmq/ca-cert.pem +ssl_options.certfile = /etc/rabbitmq/server-cert.pem +ssl_options.keyfile = /etc/rabbitmq/server-key.pem diff --git a/docker/test/integration/runner/misc/rabbitmq/server-ext.cnf b/docker/test/integration/runner/misc/rabbitmq/server-ext.cnf new file mode 100644 index 000000000000..49859873222d --- /dev/null +++ b/docker/test/integration/runner/misc/rabbitmq/server-ext.cnf @@ -0,0 +1 @@ +subjectAltName=DNS:integration-tests.clickhouse.com From 9d965368a2a337ef6cc2566462670e26fc9e2799 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 15 Nov 2023 08:36:24 +0000 Subject: [PATCH 066/274] Fix build --- src/Backups/RestoreCoordinationRemote.cpp | 2 +- src/Common/escapeForFileName.cpp | 5 ----- src/Common/escapeForFileName.h | 1 - 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp index b54231afcf7c..60a83c580f0c 100644 --- a/src/Backups/RestoreCoordinationRemote.cpp +++ b/src/Backups/RestoreCoordinationRemote.cpp @@ -245,7 +245,7 @@ bool RestoreCoordinationRemote::acquireInsertingDataForKeeperMap(const String & with_retries.renewZooKeeper(zk); /// we need to remove leading '/' from root_zk_path - auto normalized_root_zk_path = std::string_view{root_zk_path}.substr(1); + auto normalized_root_zk_path = root_zk_path.substr(1); std::string restore_lock_path = fs::path(zookeeper_path) / "keeper_map_tables" / escapeForFileName(normalized_root_zk_path); zk->createAncestors(restore_lock_path); auto code = zk->tryCreate(restore_lock_path, table_unique_id, zkutil::CreateMode::Persistent); diff --git a/src/Common/escapeForFileName.cpp b/src/Common/escapeForFileName.cpp index 790d46a93ecf..a1f9bff28d09 100644 --- a/src/Common/escapeForFileName.cpp +++ b/src/Common/escapeForFileName.cpp @@ -6,11 +6,6 @@ namespace DB { std::string escapeForFileName(const std::string & s) -{ - return escapeForFileName(std::string_view{s}); -} - -std::string escapeForFileName(std::string_view s) { std::string res; const char * pos = s.data(); diff --git a/src/Common/escapeForFileName.h b/src/Common/escapeForFileName.h index 279275f55d54..9ae296508044 100644 --- a/src/Common/escapeForFileName.h +++ b/src/Common/escapeForFileName.h @@ -11,7 +11,6 @@ namespace DB */ std::string escapeForFileName(const std::string & s); -std::string escapeForFileName(std::string_view s); std::string unescapeForFileName(const std::string & s); } From 178d23b9511d953e84a2b0b7923699cfcb841cc6 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Wed, 15 Nov 2023 13:53:52 +0100 Subject: [PATCH 067/274] Concat with arbitrary types + tests --- src/Functions/concat.cpp | 64 ++++++++++--------- src/Functions/concatWithSeparator.cpp | 19 +++--- src/Functions/formatString.h | 13 ++-- .../0_stateless/00727_concat.reference | 51 ++++++++++++++- tests/queries/0_stateless/00727_concat.sql | 59 ++++++++++++++++- ...75_show_columns_called_from_clickhouse.sql | 2 +- 6 files changed, 157 insertions(+), 51 deletions(-) diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 350cbee58a34..37311e6c09bc 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -1,16 +1,14 @@ #include #include -#include #include #include +#include #include #include -#include #include #include #include #include -#include #include "formatString.h" @@ -18,9 +16,9 @@ namespace DB { namespace ErrorCodes { - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_COLUMN; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int ILLEGAL_COLUMN; } using namespace GatherUtils; @@ -33,7 +31,7 @@ class ConcatImpl : public IFunction { public: static constexpr auto name = Name::name; - explicit ConcatImpl(ContextPtr context_) : context(context_) {} + explicit ConcatImpl(ContextPtr context_) : context(context_) { } static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } String getName() const override { return name; } @@ -68,8 +66,7 @@ class ConcatImpl : public IFunction /// For 3 and more arguments FormatStringImpl is much faster (up to 50-60%). if (arguments.size() == 2) return executeBinary(arguments, input_rows_count); - else - return executeFormatImpl(arguments, input_rows_count); + return executeFormatImpl(arguments, input_rows_count); } private: @@ -113,6 +110,7 @@ class ConcatImpl : public IFunction std::vector offsets(num_arguments); std::vector fixed_string_sizes(num_arguments); std::vector> constant_strings(num_arguments); + std::vector converted_col_ptrs(num_arguments); bool has_column_string = false; bool has_column_fixed_string = false; for (size_t i = 0; i < num_arguments; ++i) @@ -136,16 +134,27 @@ class ConcatImpl : public IFunction } else { - // An arbitrary type argument: converting it to a StringColumn as if `toString` was called - ColumnsWithTypeAndName args; - args.emplace_back(column, arguments[i].type, "tmp"); - const ColumnPtr converted_col_ptr = ConvertImplGenericToString::execute( - args, std::make_shared(), column->size()); - const ColumnString * converted_col_str = assert_cast(converted_col_ptr.get()); + // An arbitrary type argument: converting it to a StringColumn first + const auto serialization = arguments[i].type->getDefaultSerialization(); + ColumnString::MutablePtr converted_col_str = ColumnString::create(); + static FormatSettings format_settings; + + ColumnStringHelpers::WriteHelper write_helper(*converted_col_str, column->size()); + auto & write_buffer = write_helper.getWriteBuffer(); + for (size_t j = 0; j < column->size(); ++j) + { + serialization->serializeText(*column, j, write_buffer, format_settings); + write_helper.rowWritten(); + } + write_helper.finalize(); + // Same as the normal `ColumnString` branch has_column_string = true; data[i] = &converted_col_str->getChars(); offsets[i] = &converted_col_str->getOffsets(); + + // keep the refcounted-pointer around (to be able to use data/offsets later) + converted_col_ptrs[i] = std::move(converted_col_str); } } @@ -193,7 +202,7 @@ class ConcatOverloadResolver : public IFunctionOverloadResolver static constexpr auto name = "concat"; static FunctionOverloadResolverPtr create(ContextPtr context) { return std::make_unique(context); } - explicit ConcatOverloadResolver(ContextPtr context_) : context(context_) {} + explicit ConcatOverloadResolver(ContextPtr context_) : context(context_) { } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 0; } @@ -202,28 +211,25 @@ class ConcatOverloadResolver : public IFunctionOverloadResolver FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type) const override { if (isArray(arguments.at(0).type)) - { return FunctionFactory::instance().getImpl("arrayConcat", context)->build(arguments); - } - else if (isMap(arguments.at(0).type)) - { + if (isMap(arguments.at(0).type)) return FunctionFactory::instance().getImpl("mapConcat", context)->build(arguments); - } - else if (isTuple(arguments.at(0).type)) - { + if (isTuple(arguments.at(0).type)) return FunctionFactory::instance().getImpl("tupleConcat", context)->build(arguments); - } - else - return std::make_unique( - FunctionConcat::create(context), collections::map(arguments, [](const auto & elem) { return elem.type; }), return_type); + return std::make_unique( + FunctionConcat::create(context), + collections::map(arguments, [](const auto & elem) { return elem.type; }), + return_type); } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (arguments.size() < 2) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Number of arguments for function {} doesn't match: passed {}, should be at least 2.", - getName(), arguments.size()); + getName(), + arguments.size()); /// We always return Strings from concat, even if arguments were fixed strings. return std::make_shared(); diff --git a/src/Functions/concatWithSeparator.cpp b/src/Functions/concatWithSeparator.cpp index f0a7afbbaa7a..f295d86943f9 100644 --- a/src/Functions/concatWithSeparator.cpp +++ b/src/Functions/concatWithSeparator.cpp @@ -1,5 +1,5 @@ -#include #include +#include #include #include #include @@ -14,9 +14,9 @@ namespace DB { namespace ErrorCodes { - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_COLUMN; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int ILLEGAL_COLUMN; } namespace @@ -26,7 +26,7 @@ class ConcatWithSeparatorImpl : public IFunction { public: static constexpr auto name = Name::name; - explicit ConcatWithSeparatorImpl(ContextPtr context_) : context(context_) {} + explicit ConcatWithSeparatorImpl(ContextPtr context_) : context(context_) { } static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } @@ -113,8 +113,7 @@ class ConcatWithSeparatorImpl : public IFunction else if (const ColumnConst * const_col = checkAndGetColumnConstStringOrFixedString(column.get())) constant_strings[2 * i] = const_col->getValue(); else - throw Exception(ErrorCodes::ILLEGAL_COLUMN, - "Illegal column {} of argument of function {}", column->getName(), getName()); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", column->getName(), getName()); } String pattern; @@ -156,14 +155,14 @@ using FunctionConcatWithSeparatorAssumeInjective = ConcatWithSeparatorImpl(FunctionDocumentation{ - .description=R"( + .description = R"( Returns the concatenation strings separated by string separator. Syntax: concatWithSeparator(sep, expr1, expr2, expr3...) )", .examples{{"concatWithSeparator", "SELECT concatWithSeparator('a', '1', '2', '3')", ""}}, .categories{"String"}}); factory.registerFunction(FunctionDocumentation{ - .description=R"( + .description = R"( Same as concatWithSeparator, the difference is that you need to ensure that concatWithSeparator(sep, expr1, expr2, expr3...) → result is injective, it will be used for optimization of GROUP BY. The function is named “injective” if it always returns different result for different values of arguments. In other words: different arguments never yield identical result. @@ -171,7 +170,7 @@ The function is named “injective” if it always returns different result for .examples{{"concatWithSeparatorAssumeInjective", "SELECT concatWithSeparatorAssumeInjective('a', '1', '2', '3')", ""}}, .categories{"String"}}); - /// Compatibility with Spark: + /// Compatibility with Spark and MySQL: factory.registerAlias("concat_ws", "concatWithSeparator", FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/formatString.h b/src/Functions/formatString.h index 30149e9a5b0e..4bdb672caf41 100644 --- a/src/Functions/formatString.h +++ b/src/Functions/formatString.h @@ -1,18 +1,13 @@ #pragma once -#include -#include -#include -#include -#include -#include - - #include #include #include -#include #include +#include +#include +#include +#include namespace DB diff --git a/tests/queries/0_stateless/00727_concat.reference b/tests/queries/0_stateless/00727_concat.reference index af5626b4a114..4785f67bdd98 100644 --- a/tests/queries/0_stateless/00727_concat.reference +++ b/tests/queries/0_stateless/00727_concat.reference @@ -1 +1,50 @@ -Hello, world! +-- Const string + non-const arbitrary type +With 42 +With 43 +With 44 +With 45 +With 46 +With 47 +With 48 +With 49 +With 50 +With 51 +With 52 +With 53 +With 42.42 +With 43.43 +With 44 +With true +With false +With foo +With bar +With foo +With bar +With foo +With bar +With foo +With bar +With fae310ca-d52a-4923-9e9b-02bf67f4b009 +With 2023-11-14 +With 2123-11-14 +With 2023-11-14 05:50:12 +With 2023-11-14 05:50:12.123 +With hallo +With [\'foo\',\'bar\'] +With {"foo":"bar"} +With (42,\'foo\') +With {42:\'foo\'} +With 122.233.64.201 +With 2001:1:130f:2:3:9c0:876a:130b +With (42,43) +With [(0,0),(10,0),(10,10),(0,10)] +With [[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]] +With [[[(0,0),(10,0),(10,10),(0,10)]],[[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]]] +\N +-- Miscellaneous tests +Non-const strings +Three arguments test +3 arguments test with int type +Testing the alias +\N +\N diff --git a/tests/queries/0_stateless/00727_concat.sql b/tests/queries/0_stateless/00727_concat.sql index 800ebd5ec539..3119bd76c0ce 100644 --- a/tests/queries/0_stateless/00727_concat.sql +++ b/tests/queries/0_stateless/00727_concat.sql @@ -1 +1,58 @@ -SELECT CONCAT('Hello', ', ', 'world!'); +-- Tags: no-fasttest +-- no-fasttest: json type needs rapidjson library, geo types need s2 geometry + +-- not tested here: (Simple)AggregateFunction, Nested + +SET allow_experimental_object_type = 1; + +SELECT '-- Const string + non-const arbitrary type'; +SELECT concat('With ', materialize(42 :: Int8)); +SELECT concat('With ', materialize(43 :: Int16)); +SELECT concat('With ', materialize(44 :: Int32)); +SELECT concat('With ', materialize(45 :: Int64)); +SELECT concat('With ', materialize(46 :: Int128)); +SELECT concat('With ', materialize(47 :: Int256)); +SELECT concat('With ', materialize(48 :: UInt8)); +SELECT concat('With ', materialize(49 :: UInt16)); +SELECT concat('With ', materialize(50 :: UInt32)); +SELECT concat('With ', materialize(51 :: UInt64)); +SELECT concat('With ', materialize(52 :: UInt128)); +SELECT concat('With ', materialize(53 :: UInt256)); +SELECT concat('With ', materialize(42.42 :: Float32)); +SELECT concat('With ', materialize(43.43 :: Float64)); +SELECT concat('With ', materialize(44.44 :: Decimal(2))); +SELECT concat('With ', materialize(true :: Bool)); +SELECT concat('With ', materialize(false :: Bool)); +SELECT concat('With ', materialize('foo' :: String)); +SELECT concat('With ', materialize('bar' :: FixedString(3))); +SELECT concat('With ', materialize('foo' :: Nullable(String))); +SELECT concat('With ', materialize('bar' :: Nullable(FixedString(3)))); +SELECT concat('With ', materialize('foo' :: LowCardinality(String))); +SELECT concat('With ', materialize('bar' :: LowCardinality(FixedString(3)))); +SELECT concat('With ', materialize('foo' :: LowCardinality(Nullable(String)))); +SELECT concat('With ', materialize('bar' :: LowCardinality(Nullable(FixedString(3))))); +SELECT concat('With ', materialize('fae310ca-d52a-4923-9e9b-02bf67f4b009' :: UUID)); +SELECT concat('With ', materialize('2023-11-14' :: Date)); +SELECT concat('With ', materialize('2123-11-14' :: Date32)); +SELECT concat('With ', materialize('2023-11-14 05:50:12' :: DateTime)); +SELECT concat('With ', materialize('2023-11-14 05:50:12.123' :: DateTime64(3))); +SELECT concat('With ', materialize('hallo' :: Enum('hallo' = 1))); +SELECT concat('With ', materialize(['foo', 'bar'] :: Array(String))); +SELECT concat('With ', materialize('{"foo": "bar"}' :: JSON)); +SELECT concat('With ', materialize((42, 'foo') :: Tuple(Int32, String))); +SELECT concat('With ', materialize(map(42, 'foo') :: Map(Int32, String))); +SELECT concat('With ', materialize('122.233.64.201' :: IPv4)); +SELECT concat('With ', materialize('2001:0001:130F:0002:0003:09C0:876A:130B' :: IPv6)); +SELECT concat('With ', materialize((42, 43) :: Point)); +SELECT concat('With ', materialize([(0,0),(10,0),(10,10),(0,10)] :: Ring)); +SELECT concat('With ', materialize([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]] :: Polygon)); +SELECT concat('With ', materialize([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]] :: MultiPolygon)); +SELECT concat('With ', materialize(NULL :: Nullable(UInt64))); + +SELECT '-- Miscellaneous tests'; +SELECT concat(materialize('Non-const'), materialize(' strings')); +SELECT concat('Three ', 'arguments', ' test'); +SELECT concat(materialize(3 :: Int64), ' arguments test', ' with int type'); +SELECT CONCAT('Testing the ', 'alias'); +SELECT concat(materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); +SELECT concat(42, materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); diff --git a/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql b/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql index 89073bd2943d..752367517af0 100644 --- a/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql +++ b/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql @@ -5,7 +5,7 @@ -- Tests the output of SHOW COLUMNS when called through the ClickHouse protocol. -- ----------------------------------------------------------------------------------- --- Please keep this test in-sync with 02775_show_columns_called_through_mysql.sql +-- Please keep this test in-sync with 02775_show_columns_called_from_mysql.expect -- ----------------------------------------------------------------------------------- DROP TABLE IF EXISTS tab; From 0aaea6e51ddbd13ad5becafd010e07dd652ab3c7 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Wed, 15 Nov 2023 14:42:38 +0100 Subject: [PATCH 068/274] Fix ColumnConst serialization issues, more tests --- src/Functions/concat.cpp | 9 +++---- .../0_stateless/00727_concat.reference | 17 ++++++++++--- tests/queries/0_stateless/00727_concat.sql | 24 +++++++++++++++---- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 37311e6c09bc..346f96e4f032 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -135,6 +135,7 @@ class ConcatImpl : public IFunction else { // An arbitrary type argument: converting it to a StringColumn first + const auto full_column = column->convertToFullIfNeeded(); const auto serialization = arguments[i].type->getDefaultSerialization(); ColumnString::MutablePtr converted_col_str = ColumnString::create(); static FormatSettings format_settings; @@ -143,7 +144,7 @@ class ConcatImpl : public IFunction auto & write_buffer = write_helper.getWriteBuffer(); for (size_t j = 0; j < column->size(); ++j) { - serialization->serializeText(*column, j, write_buffer, format_settings); + serialization->serializeText(*full_column, j, write_buffer, format_settings); write_helper.rowWritten(); } write_helper.finalize(); @@ -210,11 +211,11 @@ class ConcatOverloadResolver : public IFunctionOverloadResolver FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type) const override { - if (isArray(arguments.at(0).type)) + if (std::ranges::all_of(arguments, [](const auto & elem) { return isArray(elem.type); })) return FunctionFactory::instance().getImpl("arrayConcat", context)->build(arguments); - if (isMap(arguments.at(0).type)) + if (std::ranges::all_of(arguments, [](const auto & elem) { return isMap(elem.type); })) return FunctionFactory::instance().getImpl("mapConcat", context)->build(arguments); - if (isTuple(arguments.at(0).type)) + if (std::ranges::all_of(arguments, [](const auto & elem) { return isTuple(elem.type); })) return FunctionFactory::instance().getImpl("tupleConcat", context)->build(arguments); return std::make_unique( FunctionConcat::create(context), diff --git a/tests/queries/0_stateless/00727_concat.reference b/tests/queries/0_stateless/00727_concat.reference index 4785f67bdd98..9b6a8b3857b6 100644 --- a/tests/queries/0_stateless/00727_concat.reference +++ b/tests/queries/0_stateless/00727_concat.reference @@ -24,6 +24,7 @@ With foo With bar With foo With bar +With 42 With fae310ca-d52a-4923-9e9b-02bf67f4b009 With 2023-11-14 With 2123-11-14 @@ -40,11 +41,21 @@ With (42,43) With [(0,0),(10,0),(10,10),(0,10)] With [[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]] With [[[(0,0),(10,0),(10,10),(0,10)]],[[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]]] +-- NULL arguments +\N +\N +\N +\N \N --- Miscellaneous tests +\N +\N +-- Various arguments tests Non-const strings +Two arguments test Three arguments test 3 arguments test with int type +42144 +42144255 +42144 +42144255 Testing the alias -\N -\N diff --git a/tests/queries/0_stateless/00727_concat.sql b/tests/queries/0_stateless/00727_concat.sql index 3119bd76c0ce..ba76ff538843 100644 --- a/tests/queries/0_stateless/00727_concat.sql +++ b/tests/queries/0_stateless/00727_concat.sql @@ -4,6 +4,7 @@ -- not tested here: (Simple)AggregateFunction, Nested SET allow_experimental_object_type = 1; +SET allow_suspicious_low_cardinality_types=1; SELECT '-- Const string + non-const arbitrary type'; SELECT concat('With ', materialize(42 :: Int8)); @@ -31,6 +32,7 @@ SELECT concat('With ', materialize('foo' :: LowCardinality(String))); SELECT concat('With ', materialize('bar' :: LowCardinality(FixedString(3)))); SELECT concat('With ', materialize('foo' :: LowCardinality(Nullable(String)))); SELECT concat('With ', materialize('bar' :: LowCardinality(Nullable(FixedString(3))))); +SELECT concat('With ', materialize(42 :: LowCardinality(Nullable(UInt32)))); SELECT concat('With ', materialize('fae310ca-d52a-4923-9e9b-02bf67f4b009' :: UUID)); SELECT concat('With ', materialize('2023-11-14' :: Date)); SELECT concat('With ', materialize('2123-11-14' :: Date32)); @@ -47,12 +49,26 @@ SELECT concat('With ', materialize((42, 43) :: Point)); SELECT concat('With ', materialize([(0,0),(10,0),(10,10),(0,10)] :: Ring)); SELECT concat('With ', materialize([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]] :: Polygon)); SELECT concat('With ', materialize([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]] :: MultiPolygon)); -SELECT concat('With ', materialize(NULL :: Nullable(UInt64))); -SELECT '-- Miscellaneous tests'; +SELECT '-- NULL arguments'; +SELECT concat(NULL, NULL); +SELECT concat(NULL, materialize(NULL :: Nullable(UInt64))); +SELECT concat(materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); + +SELECT concat(42, materialize(NULL :: Nullable(UInt64))); +SELECT concat('42', materialize(NULL :: Nullable(UInt64))); + +SELECT concat(42, materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); +SELECT concat('42', materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); + +SELECT '-- Various arguments tests'; SELECT concat(materialize('Non-const'), materialize(' strings')); +SELECT concat('Two arguments ', 'test'); SELECT concat('Three ', 'arguments', ' test'); SELECT concat(materialize(3 :: Int64), ' arguments test', ' with int type'); +SELECT concat(materialize(42 :: Int32), materialize(144 :: UInt64)); +SELECT concat(materialize(42 :: Int32), materialize(144 :: UInt64), materialize(255 :: UInt32)); +SELECT concat(42, 144); +SELECT concat(42, 144, 255); + SELECT CONCAT('Testing the ', 'alias'); -SELECT concat(materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); -SELECT concat(42, materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); From c7cd4fa972893ad6dead0a6f3fe9038c2c61ebad Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Wed, 15 Nov 2023 14:57:34 +0100 Subject: [PATCH 069/274] Update concat docs --- .../sql-reference/functions/string-functions.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 4df987b5e2a5..dc324e8e3319 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -439,7 +439,7 @@ concat(s1, s2, ...) **Arguments** -Values of type String or FixedString. +Values of arbitrary types. If an argument is not a String or FixedString, it is converted to the String type using the default serialization. **Returned values** @@ -461,6 +461,20 @@ Result: └─────────────────────────────┠``` +**Example** + +```sql +SELECT concat(42, 144); +``` + +Result: + +```result +┌─concat(42, 144)─┠+│ 42144 │ +└─────────────────┠+``` + ## concatAssumeInjective Like [concat](#concat) but assumes that `concat(s1, s2, ...) → sn` is injective. Can be used for optimization of GROUP BY. From 1ea74cee3a5b664687fc8c059da0fed144fd9ea9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 15 Nov 2023 15:04:07 +0100 Subject: [PATCH 070/274] Early disconnect if there is authentication failure with interserver secret --- src/Server/TCPHandler.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index f929d0f5ff90..e7c400920773 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -588,6 +587,10 @@ void TCPHandler::runImpl() } catch (const Exception & e) { + /// Authentication failure with interserver secret. + if (e.code() == ErrorCodes::AUTHENTICATION_FAILED) + throw; + state.io.onException(); exception.reset(e.clone()); @@ -1717,7 +1720,18 @@ void TCPHandler::receiveQuery() { client_info.interface = ClientInfo::Interface::TCP_INTERSERVER; #if USE_SSL - String cluster_secret = server.context()->getCluster(cluster)->getSecret(); + + String cluster_secret; + try + { + cluster_secret = server.context()->getCluster(cluster)->getSecret(); + } + catch (const Exception & e) + { + auto exception = Exception::createRuntime(ErrorCodes::AUTHENTICATION_FAILED, e.message()); + session->onAuthenticationFailure(/* user_name= */ std::nullopt, socket().peerAddress(), exception); + throw exception; /// NOLINT + } if (salt.empty() || cluster_secret.empty()) { From 20cfe91ff967733bdbc34244759eb26d379c8869 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Wed, 15 Nov 2023 15:21:19 +0100 Subject: [PATCH 071/274] Remove unused error codes --- src/Functions/concat.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 346f96e4f032..081096b745e4 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -16,9 +16,7 @@ namespace DB { namespace ErrorCodes { -extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -extern const int ILLEGAL_COLUMN; } using namespace GatherUtils; From a7543e3c7c36eee293c169c408a7ec2607fd86ad Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 15 Nov 2023 16:05:30 +0100 Subject: [PATCH 072/274] Fix test --- .../0_stateless/01555_system_distribution_queue_mask.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql index 3a90765226a0..7ade1d24c596 100644 --- a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql +++ b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql @@ -17,7 +17,7 @@ system stop distributed sends dist_01555; insert into dist_01555 values (1)(2); -- since test_cluster_with_incorrect_pw contains incorrect password ignore error -system flush distributed dist_01555; -- { serverError 516 } +system flush distributed dist_01555; -- { clientError ATTEMPT_TO_READ_AFTER_EOF } select length(splitByChar('*', data_path)), replaceRegexpOne(data_path, '^.*/([^/]*)/' , '\\1'), extract(last_exception, 'AUTHENTICATION_FAILED'), dateDiff('s', last_exception_time, now()) < 3600 from system.distribution_queue where database = currentDatabase() and table = 'dist_01555' format CSV; drop table dist_01555; @@ -30,7 +30,7 @@ create table dist_01555 (key Int) Engine=Distributed(test_cluster_with_incorrect insert into dist_01555 values (1)(2); -- since test_cluster_with_incorrect_pw contains incorrect password ignore error -system flush distributed dist_01555; -- { serverError 516 } +system flush distributed dist_01555; -- { clientError ATTEMPT_TO_READ_AFTER_EOF } select length(splitByChar('*', data_path)), replaceRegexpOne(data_path, '^.*/([^/]*)/' , '\\1'), extract(last_exception, 'AUTHENTICATION_FAILED'), dateDiff('s', last_exception_time, now()) < 3600 from system.distribution_queue where database = currentDatabase() and table = 'dist_01555' format CSV; drop table dist_01555; From f21dd37d1838c2041a4dcd7671d616a3dca817a8 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Nov 2023 15:42:00 +0000 Subject: [PATCH 073/274] Some fixups --- .../en/sql-reference/functions/string-functions.md | 12 +++++++++--- src/Columns/ColumnStringHelpers.h | 2 +- src/Functions/concat.cpp | 14 ++++++-------- src/Functions/formatString.h | 10 +++++----- .../02775_show_columns_called_from_clickhouse.sql | 2 +- .../02775_show_columns_called_from_mysql.expect | 2 +- 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index dc324e8e3319..4b6e03563013 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -429,7 +429,7 @@ SELECT format('{} {}', 'Hello', 'World') ## concat -Concatenates the strings listed in the arguments without separator. +Concatenates the given arguments. **Syntax** @@ -439,7 +439,9 @@ concat(s1, s2, ...) **Arguments** -Values of arbitrary types. If an argument is not a String or FixedString, it is converted to the String type using the default serialization. +At least two values of arbitrary type. + +Arguments which are not of types [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md) are converted to strings using their default serialization. As this decreases performance, it is not recommended to use non-String/FixedString arguments. **Returned values** @@ -449,6 +451,8 @@ If any of arguments is `NULL`, the function returns `NULL`. **Example** +Query: + ``` sql SELECT concat('Hello, ', 'World!'); ``` @@ -461,7 +465,7 @@ Result: └─────────────────────────────┠``` -**Example** +Query: ```sql SELECT concat(42, 144); @@ -540,6 +544,8 @@ Concatenates the given strings with a given separator. concatWithSeparator(sep, expr1, expr2, expr3...) ``` +Alias: `concat_ws` + **Arguments** - sep — separator. Const [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md). diff --git a/src/Columns/ColumnStringHelpers.h b/src/Columns/ColumnStringHelpers.h index 851486e490ad..97b52506ae0e 100644 --- a/src/Columns/ColumnStringHelpers.h +++ b/src/Columns/ColumnStringHelpers.h @@ -62,7 +62,7 @@ class WriteHelper return buffer; } - inline void rowWritten() + void rowWritten() { if constexpr (std::is_same_v) { diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 081096b745e4..9aa6de5d2190 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -91,7 +91,6 @@ class ConcatImpl : public IFunction else { /// Fallback: use generic implementation for not very important cases. - /// Concat of arbitrary types also goes here. return executeFormatImpl(arguments, input_rows_count); } @@ -108,7 +107,7 @@ class ConcatImpl : public IFunction std::vector offsets(num_arguments); std::vector fixed_string_sizes(num_arguments); std::vector> constant_strings(num_arguments); - std::vector converted_col_ptrs(num_arguments); + std::vector converted_col_ptrs(num_arguments); bool has_column_string = false; bool has_column_fixed_string = false; for (size_t i = 0; i < num_arguments; ++i) @@ -132,14 +131,13 @@ class ConcatImpl : public IFunction } else { - // An arbitrary type argument: converting it to a StringColumn first + /// A non-String/non-FixedString-type argument: use the default serialization to convert it to String const auto full_column = column->convertToFullIfNeeded(); const auto serialization = arguments[i].type->getDefaultSerialization(); - ColumnString::MutablePtr converted_col_str = ColumnString::create(); - static FormatSettings format_settings; - + auto converted_col_str = ColumnString::create(); ColumnStringHelpers::WriteHelper write_helper(*converted_col_str, column->size()); auto & write_buffer = write_helper.getWriteBuffer(); + FormatSettings format_settings; for (size_t j = 0; j < column->size(); ++j) { serialization->serializeText(*full_column, j, write_buffer, format_settings); @@ -147,12 +145,12 @@ class ConcatImpl : public IFunction } write_helper.finalize(); - // Same as the normal `ColumnString` branch + /// Same as the normal `ColumnString` branch has_column_string = true; data[i] = &converted_col_str->getChars(); offsets[i] = &converted_col_str->getOffsets(); - // keep the refcounted-pointer around (to be able to use data/offsets later) + /// Keep the refcounted-pointer alive converted_col_ptrs[i] = std::move(converted_col_str); } } diff --git a/src/Functions/formatString.h b/src/Functions/formatString.h index 4bdb672caf41..315e5c06227e 100644 --- a/src/Functions/formatString.h +++ b/src/Functions/formatString.h @@ -1,14 +1,14 @@ #pragma once -#include -#include -#include -#include #include -#include #include #include +#include +#include +#include +#include +#include namespace DB { diff --git a/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql b/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql index 752367517af0..3bbcbb1a535c 100644 --- a/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql +++ b/tests/queries/0_stateless/02775_show_columns_called_from_clickhouse.sql @@ -5,7 +5,7 @@ -- Tests the output of SHOW COLUMNS when called through the ClickHouse protocol. -- ----------------------------------------------------------------------------------- --- Please keep this test in-sync with 02775_show_columns_called_from_mysql.expect +-- Please keep this test in-sync with 02775_show_columns_called_from_clickhouse.expect -- ----------------------------------------------------------------------------------- DROP TABLE IF EXISTS tab; diff --git a/tests/queries/0_stateless/02775_show_columns_called_from_mysql.expect b/tests/queries/0_stateless/02775_show_columns_called_from_mysql.expect index bef5bd10ff31..8ba5774820e3 100755 --- a/tests/queries/0_stateless/02775_show_columns_called_from_mysql.expect +++ b/tests/queries/0_stateless/02775_show_columns_called_from_mysql.expect @@ -6,7 +6,7 @@ # Tests the output of SHOW COLUMNS when called through the MySQL protocol. # ----------------------------------------------------------------------------------- -# Please keep this test in-sync with 02775_show_columns_called_through_clickhouse.sql +# Please keep this test in-sync with 02775_show_columns_called_from_clickhouse.sql # ----------------------------------------------------------------------------------- set basedir [file dirname $argv0] From fbaa6a4e54d8bc1435d78cf031525093cb945684 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 9 Nov 2023 16:39:41 +0000 Subject: [PATCH 074/274] Optimize equality with is null check in JOIN ON section --- .../Passes/LogicalExpressionOptimizerPass.cpp | 187 ++++++++++++++++++ .../Passes/LogicalExpressionOptimizerPass.h | 11 ++ ...11_join_on_nullsafe_optimization.reference | 25 +++ .../02911_join_on_nullsafe_optimization.sql | 27 +++ 4 files changed, 250 insertions(+) create mode 100644 tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference create mode 100644 tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index 46056aeaf6f6..372c760a20a8 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -5,11 +5,17 @@ #include #include #include +#include #include namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithContext { public: @@ -21,6 +27,15 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont void enterImpl(QueryTreeNodePtr & node) { + if (auto * join_node = node->as()) + { + join_stack.push_back(join_node); + return; + } + + if (!join_stack.empty() && join_stack.back()->getJoinExpression().get() == node.get()) + is_inside_on_section = true; + auto * function_node = node->as(); if (!function_node) @@ -29,6 +44,10 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont if (function_node->getFunctionName() == "or") { tryReplaceOrEqualsChainWithIn(node); + + /// Operator <=> is not supported outside of JOIN ON section + if (is_inside_on_section) + tryOptimizeIsNotDistinctOrIsNull(node); return; } @@ -38,6 +57,20 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont return; } } + + void leaveImpl(QueryTreeNodePtr & node) + { + if (!join_stack.empty() && join_stack.back()->getJoinExpression().get() == node.get()) + is_inside_on_section = false; + + if (auto * join_node = node->as()) + { + assert(join_stack.back() == join_node); + join_stack.pop_back(); + return; + } + } + private: void tryReplaceAndEqualsChainsWithConstant(QueryTreeNodePtr & node) { @@ -231,6 +264,160 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont function_node.getArguments().getNodes() = std::move(or_operands); function_node.resolveAsFunction(or_function_resolver); } + + void tryOptimizeIsNotDistinctOrIsNull(QueryTreeNodePtr & node) + { + auto & function_node = node->as(); + assert(function_node.getFunctionName() == "or"); + + QueryTreeNodes or_operands; + + /// Indices of `equals` or `isNotDistinctFrom` functions in the vector above + std::vector equals_functions_indices; + + /** Map from `isNull` argument to indices of operands that contains that `isNull` functions + * `a = b OR (a IS NULL AND b IS NULL) OR (a IS NULL AND c IS NULL)` + * will be mapped to + * { + * a => [(a IS NULL AND b IS NULL), (a IS NULL AND c IS NULL)] + * b => [(a IS NULL AND b IS NULL)] + * c => [(a IS NULL AND c IS NULL)] + * } + * Then for each a <=> b we can find all operands that contains both a IS NULL and b IS NULL + */ + QueryTreeNodePtrWithHashMap> is_null_argument_to_indices; + + for (const auto & argument : function_node.getArguments()) + { + or_operands.push_back(argument); + + auto * argument_function = argument->as(); + if (!argument_function) + continue; + + const auto & func_name = argument_function->getFunctionName(); + if (func_name == "equals" || func_name == "isNotDistinctFrom") + equals_functions_indices.push_back(or_operands.size() - 1); + + if (func_name == "and") + { + for (const auto & and_argument : argument_function->getArguments().getNodes()) + { + auto * and_argument_function = and_argument->as(); + if (and_argument_function && and_argument_function->getFunctionName() == "isNull") + { + const auto & is_null_argument = and_argument_function->getArguments().getNodes()[0]; + is_null_argument_to_indices[is_null_argument].push_back(or_operands.size() - 1); + } + } + } + } + + /// OR operands that are changed to and needs to be re-resolved + std::unordered_set arguments_to_reresolve; + + for (size_t equals_function_idx : equals_functions_indices) + { + auto * equals_function = or_operands[equals_function_idx]->as(); + + /// For a <=> b we are looking for expressions containing both `a IS NULL` and `b IS NULL` combined with AND + const auto & argument_nodes = equals_function->getArguments().getNodes(); + const auto & lhs_is_null_parents = is_null_argument_to_indices[argument_nodes[0]]; + const auto & rhs_is_null_parents = is_null_argument_to_indices[argument_nodes[1]]; + std::unordered_set operands_to_optimize; + std::set_intersection(lhs_is_null_parents.begin(), lhs_is_null_parents.end(), + rhs_is_null_parents.begin(), rhs_is_null_parents.end(), + std::inserter(operands_to_optimize, operands_to_optimize.begin())); + + /// If we have `a = b OR (a IS NULL AND b IS NULL)` we can optimize it to `a <=> b` + if (!operands_to_optimize.empty() && equals_function->getFunctionName() == "equals") + arguments_to_reresolve.insert(equals_function_idx); + + for (size_t to_optimize_idx : operands_to_optimize) + { + /// We are looking for operand `a IS NULL AND b IS NULL AND ...` + auto * operand_to_optimize = or_operands[to_optimize_idx]->as(); + + /// Remove `a IS NULL` and `b IS NULL` arguments from AND + QueryTreeNodes new_arguments; + for (const auto & and_argument : operand_to_optimize->getArguments().getNodes()) + { + bool to_eliminate = false; + + const auto * and_argument_function = and_argument->as(); + if (and_argument_function && and_argument_function->getFunctionName() == "isNull") + { + const auto & is_null_argument = and_argument_function->getArguments().getNodes()[0]; + to_eliminate = (is_null_argument->isEqual(*argument_nodes[0]) || is_null_argument->isEqual(*argument_nodes[1])); + } + + if (to_eliminate) + arguments_to_reresolve.insert(to_optimize_idx); + else + new_arguments.emplace_back(and_argument); + } + /// If less than two arguments left, we will remove or replace the whole AND below + operand_to_optimize->getArguments().getNodes() = std::move(new_arguments); + } + } + + + if (arguments_to_reresolve.empty()) + /// Nothing have been changed + return; + + auto and_function_resolver = FunctionFactory::instance().get("and", getContext()); + auto strict_equals_function_resolver = FunctionFactory::instance().get("isNotDistinctFrom", getContext()); + QueryTreeNodes new_or_operands; + for (size_t i = 0; i < or_operands.size(); ++i) + { + if (arguments_to_reresolve.contains(i)) + { + auto * function = or_operands[i]->as(); + if (function->getFunctionName() == "equals") + { + /// Because we removed checks for IS NULL, we should replace `a = b` with `a <=> b` + function->resolveAsFunction(strict_equals_function_resolver); + new_or_operands.emplace_back(std::move(or_operands[i])); + } + else if (function->getFunctionName() == "and") + { + const auto & and_arguments = function->getArguments().getNodes(); + if (and_arguments.size() > 1) + { + function->resolveAsFunction(and_function_resolver); + new_or_operands.emplace_back(std::move(or_operands[i])); + } + else if (and_arguments.size() == 1) + { + /// Replace AND with a single argument with the argument itself + new_or_operands.emplace_back(std::move(and_arguments[0])); + } + } + else + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected function name: '{}'", function->getFunctionName()); + } + else + { + new_or_operands.emplace_back(std::move(or_operands[i])); + } + } + + if (new_or_operands.size() == 1) + { + node = std::move(new_or_operands[0]); + return; + } + + /// Rebuild OR function + auto or_function_resolver = FunctionFactory::instance().get("or", getContext()); + function_node.getArguments().getNodes() = std::move(new_or_operands); + function_node.resolveAsFunction(or_function_resolver); + } + +private: + bool is_inside_on_section = false; + std::deque join_stack; }; void LogicalExpressionOptimizerPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h index 05c10ddc6852..80062f38eacd 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h @@ -67,6 +67,17 @@ namespace DB * FROM TABLE * WHERE a = 1 AND b = 'test'; * ------------------------------- + * + * 5. Remove unnecessary IS NULL checks in JOIN ON clause + * - equality check with explicit IS NULL check replaced with <=> operator + * ------------------------------- + * SELECT * FROM t1 JOIN t2 ON a = b OR (a IS NULL AND b IS NULL) + * SELECT * FROM t1 JOIN t2 ON a <=> b OR (a IS NULL AND b IS NULL) + * + * will be transformed into + * + * SELECT * FROM t1 JOIN t2 ON a <=> b + * ------------------------------- */ class LogicalExpressionOptimizerPass final : public IQueryTreePass diff --git a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference new file mode 100644 index 000000000000..976c1503b028 --- /dev/null +++ b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference @@ -0,0 +1,25 @@ +-- { echoOn } +SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR (t1.x IS NULL AND t2.x IS NULL)) ORDER BY t1.x NULLS LAST; +2 2 2 2 +3 3 3 33 +\N \N \N \N +SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR t1.x IS NULL AND t1.y <=> t2.y AND t2.x IS NULL) ORDER BY t1.x NULLS LAST; +1 42 4 42 +2 2 2 2 +3 3 3 33 +\N \N \N \N +SELECT * FROM t1 JOIN t2 ON (t1.x = t2.x OR t1.x IS NULL AND t2.x IS NULL) AND t1.y <=> t2.y ORDER BY t1.x NULLS LAST; +2 2 2 2 +\N \N \N \N +SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR t1.y <=> t2.y OR (t1.x IS NULL AND t1.y IS NULL AND t2.x IS NULL AND t2.y IS NULL)) ORDER BY t1.x NULLS LAST; +1 42 4 42 +2 2 2 2 +3 3 3 33 +\N \N \N \N +SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR (t1.x IS NULL AND t2.x IS NULL)) AND (t1.y == t2.y OR (t1.y IS NULL AND t2.y IS NULL)) AND COALESCE(t1.x, 0) != 2 ORDER BY t1.x NULLS LAST; +\N \N \N \N +SELECT x = y OR (x IS NULL AND y IS NULL) FROM t1 ORDER BY x NULLS LAST; +0 +1 +1 +1 diff --git a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql new file mode 100644 index 000000000000..6a98a7bb57bb --- /dev/null +++ b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql @@ -0,0 +1,27 @@ +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; + +CREATE TABLE t1 (x Nullable(Int64), y Nullable(UInt64)) ENGINE = TinyLog; +CREATE TABLE t2 (x Nullable(Int64), y Nullable(UInt64)) ENGINE = TinyLog; + +INSERT INTO t1 VALUES (1,42), (2,2), (3,3), (NULL,NULL); +INSERT INTO t2 VALUES (NULL,NULL), (2,2), (3,33), (4,42); + +SET allow_experimental_analyzer = 1; + +-- { echoOn } +SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR (t1.x IS NULL AND t2.x IS NULL)) ORDER BY t1.x NULLS LAST; + +SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR t1.x IS NULL AND t1.y <=> t2.y AND t2.x IS NULL) ORDER BY t1.x NULLS LAST; + +SELECT * FROM t1 JOIN t2 ON (t1.x = t2.x OR t1.x IS NULL AND t2.x IS NULL) AND t1.y <=> t2.y ORDER BY t1.x NULLS LAST; + +SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR t1.y <=> t2.y OR (t1.x IS NULL AND t1.y IS NULL AND t2.x IS NULL AND t2.y IS NULL)) ORDER BY t1.x NULLS LAST; + +SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR (t1.x IS NULL AND t2.x IS NULL)) AND (t1.y == t2.y OR (t1.y IS NULL AND t2.y IS NULL)) AND COALESCE(t1.x, 0) != 2 ORDER BY t1.x NULLS LAST; + +SELECT x = y OR (x IS NULL AND y IS NULL) FROM t1 ORDER BY x NULLS LAST; +-- { echoOff } + +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; From 05163be79c3aa7b106c93f084103be6af403ae80 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 10 Nov 2023 15:50:21 +0000 Subject: [PATCH 075/274] fix clang tidy --- src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index 372c760a20a8..9602ef8a7438 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -391,7 +391,7 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont else if (and_arguments.size() == 1) { /// Replace AND with a single argument with the argument itself - new_or_operands.emplace_back(std::move(and_arguments[0])); + new_or_operands.emplace_back(and_arguments[0]); } } else @@ -415,7 +415,6 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont function_node.resolveAsFunction(or_function_resolver); } -private: bool is_inside_on_section = false; std::deque join_stack; }; From e2b25aab0c4032c0b36f2124d02f9b4bfd36d242 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Nov 2023 15:53:38 +0000 Subject: [PATCH 076/274] Fixups, pt. II --- src/Functions/FunctionsConversion.h | 4 ++-- src/Functions/concat.cpp | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index e3ec7ebd3201..d7c2c70884b6 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -1247,9 +1247,9 @@ struct ConvertImplGenericToString FormatSettings format_settings; auto serialization = type.getDefaultSerialization(); - for (size_t i = 0; i < size; ++i) + for (size_t row = 0; row < size; ++row) { - serialization->serializeText(col_from, i, write_buffer, format_settings); + serialization->serializeText(col_from, row, write_buffer, format_settings); write_helper.rowWritten(); } diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index 9aa6de5d2190..f426f662868e 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -132,15 +132,15 @@ class ConcatImpl : public IFunction else { /// A non-String/non-FixedString-type argument: use the default serialization to convert it to String - const auto full_column = column->convertToFullIfNeeded(); - const auto serialization = arguments[i].type->getDefaultSerialization(); + auto full_column = column->convertToFullIfNeeded(); + auto serialization = arguments[i].type->getDefaultSerialization(); auto converted_col_str = ColumnString::create(); ColumnStringHelpers::WriteHelper write_helper(*converted_col_str, column->size()); auto & write_buffer = write_helper.getWriteBuffer(); FormatSettings format_settings; - for (size_t j = 0; j < column->size(); ++j) + for (size_t row = 0; row < column->size(); ++row) { - serialization->serializeText(*full_column, j, write_buffer, format_settings); + serialization->serializeText(*full_column, row, write_buffer, format_settings); write_helper.rowWritten(); } write_helper.finalize(); @@ -150,7 +150,7 @@ class ConcatImpl : public IFunction data[i] = &converted_col_str->getChars(); offsets[i] = &converted_col_str->getOffsets(); - /// Keep the refcounted-pointer alive + /// Keep the pointer alive converted_col_ptrs[i] = std::move(converted_col_str); } } From 6e3e6383ba0ad5b317c8189ac5a654ee5bb9057b Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 15 Nov 2023 19:00:27 +0100 Subject: [PATCH 077/274] perf check 2 --- base/poco/Net/src/HTTPServerSession.cpp | 1 - base/poco/Net/src/HTTPSession.cpp | 32 +++++++++----------- src/Disks/ObjectStorages/S3/diskSettings.cpp | 2 +- src/IO/S3/PocoHTTPClient.cpp | 1 - 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/base/poco/Net/src/HTTPServerSession.cpp b/base/poco/Net/src/HTTPServerSession.cpp index f6d3c4e5b923..d4f2b24879e4 100644 --- a/base/poco/Net/src/HTTPServerSession.cpp +++ b/base/poco/Net/src/HTTPServerSession.cpp @@ -26,7 +26,6 @@ HTTPServerSession::HTTPServerSession(const StreamSocket& socket, HTTPServerParam _maxKeepAliveRequests(pParams->getMaxKeepAliveRequests()) { setTimeout(pParams->getTimeout()); - this->socket().setReceiveTimeout(pParams->getTimeout()); } diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index 97decded282f..9ebbd7d04cd7 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -94,24 +94,22 @@ void HTTPSession::setTimeout(const Poco::Timespan& timeout) void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco::Timespan& sendTimeout, const Poco::Timespan& receiveTimeout) { _connectionTimeout = connectionTimeout; - _sendTimeout = sendTimeout; - _receiveTimeout = receiveTimeout; -// if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) -// { -// _sendTimeout = sendTimeout; -// -// if (connected()) -// _socket.setSendTimeout(_sendTimeout); -// } -// -// if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) -// { -// _receiveTimeout = receiveTimeout; -// -// if (connected()) -// _socket.setReceiveTimeout(_receiveTimeout); -// } + if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) + { + _sendTimeout = sendTimeout; + + if (connected()) + _socket.setSendTimeout(_sendTimeout); + } + + if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) + { + _receiveTimeout = receiveTimeout; + + if (connected()) + _socket.setReceiveTimeout(_receiveTimeout); + } } diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index b0384daab2d2..0232a6eb0706 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -67,7 +67,7 @@ std::unique_ptr getClient( config_prefix + ".http_keep_alive_timeout_ms", DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT * 1000); client_configuration.http_connection_pool_size = config.getUInt(config_prefix + ".http_connection_pool_size", 1000); client_configuration.wait_on_pool_size_limit = false; - client_configuration.s3_use_adaptive_timeouts = config.getUInt( + client_configuration.s3_use_adaptive_timeouts = config.getBool( config_prefix + ".use_adaptive_timeouts", client_configuration.s3_use_adaptive_timeouts); /* diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 904e23241454..f681362e607b 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include From f3f839205197a73cf3c4b40dd8d67077839701ba Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 15 Nov 2023 17:37:00 +0100 Subject: [PATCH 078/274] upgrade replication protocol --- src/Storages/MergeTree/DataPartsExchange.cpp | 83 ++++++++++++++----- src/Storages/MergeTree/MergeTreeSettings.h | 5 +- .../MergeTree/ReplicatedMergeTreeSink.cpp | 8 ++ tests/clickhouse-test | 3 + ...plication_protocol_wait_for_part.reference | 1 + ...916_replication_protocol_wait_for_part.sql | 23 +++++ 6 files changed, 101 insertions(+), 22 deletions(-) create mode 100644 tests/queries/0_stateless/02916_replication_protocol_wait_for_part.reference create mode 100644 tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 4545b2b98ae2..7fd6f59ed697 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -65,8 +65,7 @@ constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_UUID = 5; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_ZERO_COPY = 6; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION = 7; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION = 8; -// Reserved for ALTER PRIMARY KEY -// constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PRIMARY_KEY = 9; +constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE = 9; std::string getEndpointId(const std::string & node_id) { @@ -122,7 +121,7 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write MergeTreePartInfo::fromPartName(part_name, data.format_version); /// We pretend to work as older server version, to be sure that client will correctly process our version - response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION))}); + response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE))}); LOG_TRACE(log, "Sending part {}", part_name); @@ -140,6 +139,29 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write { part = findPart(part_name); + /// Ephemeral zero-copy lock may be lost for PreActive parts + /// do not expose PreActive parts + if (client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE) + { + bool part_is_ready = part->getState() != MergeTreeDataPartState::PreActive; + writeBinary(part_is_ready, out); + + if (!part_is_ready) + { + LOG_TRACE(log, "Part {} is in PreActive state, reply to the client that part is not ready yet", part_name); + return; + } + } + else + { + bool zero_copy_enabled = data.getSettings()->allow_remote_fs_zero_copy_replication; + if (part->getState() == MergeTreeDataPartState::PreActive && zero_copy_enabled) + { + /// report error, client will try again later, error message would be printed + throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in table", part_name); + } + } + CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedSend}; if (part->getDataPartStorage().isStoredOnRemoteDisk()) @@ -357,12 +379,8 @@ MergeTreeData::DataPartPtr Service::findPart(const String & name) /// determine the local state of the part, so queries for the parts in these states are completely normal. MergeTreeData::DataPartPtr part; - /// Ephemeral zero-copy lock may be lost for PreActive parts - bool zero_copy_enabled = data.getSettings()->allow_remote_fs_zero_copy_replication; - if (zero_copy_enabled) - part = data.getPartIfExists(name, {MergeTreeDataPartState::Active, MergeTreeDataPartState::Outdated}); - else - part = data.getPartIfExists(name, {MergeTreeDataPartState::PreActive, MergeTreeDataPartState::Active, MergeTreeDataPartState::Outdated}); + part = data.getPartIfExists(name, {MergeTreeDataPartState::PreActive, MergeTreeDataPartState::Active, MergeTreeDataPartState::Outdated}); + if (part) return part; @@ -424,7 +442,7 @@ std::pair Fetcher::fetchSelected { {"endpoint", endpoint_id}, {"part", part_name}, - {"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION)}, + {"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE)}, {"compress", "false"} }); @@ -482,17 +500,42 @@ std::pair Fetcher::fetchSelected creds.setPassword(password); } - std::unique_ptr in = std::make_unique( - uri, - Poco::Net::HTTPRequest::HTTP_POST, - nullptr, - timeouts, - creds, - DBMS_DEFAULT_BUFFER_SIZE, - 0, /* no redirects */ - static_cast(data_settings->replicated_max_parallel_fetches_for_host)); + std::unique_ptr in; + int server_protocol_version = 0; + bool part_is_ready = true; + + static const UInt32 part_not_ready_attempts = 5; + static const UInt32 wait_sleep_time_ms = 100; + + for (UInt32 attempt = 1; attempt <= part_not_ready_attempts; ++attempt) + { + in = std::make_unique( + uri, + Poco::Net::HTTPRequest::HTTP_POST, + nullptr, + timeouts, + creds, + DBMS_DEFAULT_BUFFER_SIZE, + 0, /* no redirects */ + static_cast(data_settings->replicated_max_parallel_fetches_for_host)); + + server_protocol_version = parse(in->getResponseCookie("server_protocol_version", "0")); + + if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE) + readBinary(part_is_ready, *in); + + if (part_is_ready) + break; + + sleepForMilliseconds(wait_sleep_time_ms); + + if (blocker.isCancelled()) + throw Exception(ErrorCodes::ABORTED, "Fetching of part was cancelled"); + } - int server_protocol_version = parse(in->getResponseCookie("server_protocol_version", "0")); + if (!part_is_ready) + throw Exception(ErrorCodes::ABORTED, "Part {} is still not ready in host {} after {} attempts, try another host", + part_name, host, part_not_ready_attempts); String remote_fs_metadata = parse(in->getResponseCookie("remote_fs_metadata", "")); diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 53876e773768..15c54ee37911 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -83,7 +83,8 @@ struct Settings; M(UInt64, max_delay_to_insert, 1, "Max delay of inserting data into MergeTree table in seconds, if there are a lot of unmerged parts in single partition.", 0) \ M(UInt64, min_delay_to_insert_ms, 10, "Min delay of inserting data into MergeTree table in milliseconds, if there are a lot of unmerged parts in single partition.", 0) \ M(UInt64, max_parts_in_total, 100000, "If more than this number active parts in all partitions in total, throw 'Too many parts ...' exception.", 0) \ - M(Bool, async_insert, false, "If true, data from INSERT query is stored in queue and later flushed to table in background.", 0) \ + M(Bool, async_insert, false, "If true, data from INSERT query is stored in queue and later flushed to table in background.", 0) \ + M(Milliseconds, sleep_before_commit_local_part_in_replicated_table_ms, 0, "For testing. Do not change it.", 0) \ \ /* Part removal settings. */ \ M(UInt64, simultaneous_parts_removal_limit, 0, "Maximum number of parts to remove during one CleanupThread iteration (0 means unlimited).", 0) \ @@ -121,7 +122,7 @@ struct Settings; M(UInt64, max_replicated_sends_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for replicated sends. Zero means unlimited.", 0) \ M(Milliseconds, wait_for_unique_parts_send_before_shutdown_ms, 0, "Before shutdown table will wait for required amount time for unique parts (exist only on current replica) to be fetched by other replicas (0 means disabled).", 0) \ M(Float, fault_probability_before_part_commit, 0, "For testing. Do not change it.", 0) \ - M(Float, fault_probability_after_part_commit, 0, "For testing. Do not change it.", 0) \ + M(Float, fault_probability_after_part_commit, 0, "For testing. Do not change it.", 0) \ \ /** Check delay of replicas settings. */ \ M(UInt64, min_relative_delay_to_measure, 120, "Calculate relative replica delay only if absolute delay is not less that this value.", 0) \ diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 7de5d46c66bd..37f808824b53 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -924,6 +924,14 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: Coordination::Error multi_code = zookeeper->tryMultiNoThrow(ops, responses); /// 1 RTT if (multi_code == Coordination::Error::ZOK) { + auto sleep_before_commit_local_part_in_replicated_table_ms = storage.getSettings()->sleep_before_commit_local_part_in_replicated_table_ms; + if (sleep_before_commit_local_part_in_replicated_table_ms.totalMilliseconds()) + { + LOG_INFO(log, "committing part {}, triggered sleep_before_commit_local_part_in_replicated_table_ms {}", + part->name, sleep_before_commit_local_part_in_replicated_table_ms.totalMilliseconds()); + sleepForMilliseconds(sleep_before_commit_local_part_in_replicated_table_ms.totalMilliseconds()); + } + part->new_part_was_committed_to_zookeeper_after_rename_on_disk = true; transaction.commit(); storage.merge_selecting_task->schedule(); diff --git a/tests/clickhouse-test b/tests/clickhouse-test index cab7d7e79ff9..048f848ff272 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -673,6 +673,9 @@ class MergeTreeSettingsRandomizer: "primary_key_compress_block_size": lambda: random.randint(8000, 100000), "replace_long_file_name_to_hash": lambda: random.randint(0, 1), "max_file_name_length": threshold_generator(0.3, 0.3, 0, 128), + "sleep_before_commit_local_part_in_replicated_table_ms": threshold_generator( + 0.3, 0.3, 0, 250 + ), } @staticmethod diff --git a/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.reference b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.reference new file mode 100644 index 000000000000..0cfbf08886fc --- /dev/null +++ b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.reference @@ -0,0 +1 @@ +2 diff --git a/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql new file mode 100644 index 000000000000..ed9cfd00b450 --- /dev/null +++ b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql @@ -0,0 +1,23 @@ +-- Tags: no-replicated-database, no-fasttest +-- Tag no-replicated-database: different number of replicas + +create table tableIn (n int) + engine=ReplicatedMergeTree('/test/02916/{database}/table', '1') + order by tuple() + settings + storage_policy='s3_cache', + allow_remote_fs_zero_copy_replication=1, + sleep_before_commit_local_part_in_replicated_table_ms=50000; +create table tableOut (n int) + engine=ReplicatedMergeTree('/test/02916/{database}/table', '2') + order by tuple() + settings + storage_policy='s3_cache', + allow_remote_fs_zero_copy_replication=1; + +SET send_logs_level = 'error'; + +insert into tableIn values(1); +insert into tableIn values(2); +system sync replica tableOut; +select count() from tableOut; From d862dfdf9c753e17896f1c3a9d5cb01e71a5cee3 Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Wed, 15 Nov 2023 18:38:23 +0000 Subject: [PATCH 079/274] fix comments Signed-off-by: Jianfei Hu --- src/IO/S3/Credentials.cpp | 53 +++++++++++---------------------------- src/IO/S3/Credentials.h | 11 ++++---- 2 files changed, 21 insertions(+), 43 deletions(-) diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index 7d6ed0944861..bc3366341141 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -1,9 +1,4 @@ -#include -#include #include -#include -#include -#include "Common/Exception.h" #if USE_AWS_S3 @@ -21,22 +16,24 @@ # include # include - +# include # include # include +# include +# include # include # include -#include -#include - - -#include -#include -#include -#include -#include +# include +# include +# include +# include +# include +# include +# include +# include +# include namespace DB @@ -65,7 +62,7 @@ bool areCredentialsEmptyOrExpired(const Aws::Auth::AWSCredentials & credentials, } const char SSO_CREDENTIALS_PROVIDER_LOG_TAG[] = "SSOCredentialsProvider"; -const int AVAILABILITY_ZONE_REQUEST_TIMEOUT_SECONDS = 3; +constexpr int AVAILABILITY_ZONE_REQUEST_TIMEOUT_SECONDS = 3; } @@ -275,11 +272,11 @@ String getGCPAvailabilityZoneOrException() boost::split(zone_info, response_data, boost::is_any_of("/")); /// We expect GCP returns a string as "projects/123456789/zones/us-central1a". if (zone_info.size() != 4) - throw DB::Exception(ErrorCodes::GCP_ERROR, "Invalid format of GCP zone information, expect projects//zones/, got {}", response_data); + throw DB::Exception(ErrorCodes::GCP_ERROR, "Invalid format of GCP zone information, expect projects//zones/"); return zone_info[3]; } -String getRunningAvailabilityZoneImpl() +String getRunningAvailabilityZone() { LOG_INFO(&Poco::Logger::get("Application"), "Trying to detect the availability zone."); try @@ -302,26 +299,6 @@ String getRunningAvailabilityZoneImpl() } } -std::variant getRunningAvailabilityZoneImplOrException() -{ - try - { - return getRunningAvailabilityZoneImpl(); - } - catch (...) - { - return std::current_exception(); - } -} - -String getRunningAvailabilityZone() -{ - static auto az_or_exception = getRunningAvailabilityZoneImplOrException(); - if (const auto * az = std::get_if(&az_or_exception)) - return *az; - else - std::rethrow_exception(std::get(az_or_exception)); -} AWSEC2InstanceProfileConfigLoader::AWSEC2InstanceProfileConfigLoader(const std::shared_ptr & client_, bool use_secure_pull_) : client(client_) diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index a978679348f6..b1666e13757a 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -1,12 +1,13 @@ #pragma once -#include -#include -#include #include "config.h" #if USE_AWS_S3 +# include +# include +# include + # include # include # include @@ -22,7 +23,7 @@ namespace DB::S3 inline static constexpr uint64_t DEFAULT_EXPIRATION_WINDOW_SECONDS = 120; /// In GCP metadata service can be accessed via DNS regardless of IPv4 or IPv6. -static constexpr char GCP_METADATA_SERVICE_ENDPOINT[] = "http://metadata.google.internal"; +static inline constexpr char GCP_METADATA_SERVICE_ENDPOINT[] = "http://metadata.google.internal"; /// getRunningAvailabilityZone returns the availability zone of the underlying compute resources where the current process runs. String getRunningAvailabilityZone(); @@ -59,7 +60,7 @@ class AWSEC2MetadataClient : public Aws::Internal::AWSHttpResourceClient virtual Aws::String getCurrentRegion() const; - friend String getRunningAvailabilityZoneImpl(); + friend String getRunningAvailabilityZone(); private: std::pair getEC2MetadataToken(const std::string & user_agent_string) const; From d0398e3c1d1f55281e65633a4947c807d3c0c022 Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Wed, 15 Nov 2023 18:47:28 +0000 Subject: [PATCH 080/274] remove variant header Signed-off-by: Jianfei Hu --- src/IO/S3/Credentials.cpp | 1 - src/IO/S3/Credentials.h | 1 - 2 files changed, 2 deletions(-) diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index bc3366341141..9ab214655934 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -21,7 +21,6 @@ # include # include -# include # include # include diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index b1666e13757a..1f35443adf4d 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -6,7 +6,6 @@ # include # include -# include # include # include From ea92dbb1c74c700b4df4172999d6ca504ff593bf Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Wed, 15 Nov 2023 19:18:38 +0000 Subject: [PATCH 081/274] fix build for non USE_S3 case Signed-off-by: Jianfei Hu --- src/IO/S3/Credentials.cpp | 6 ++++-- src/IO/S3/Credentials.h | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index 9ab214655934..e25f45517234 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -784,15 +784,17 @@ S3CredentialsProviderChain::S3CredentialsProviderChain( #else +# include + namespace DB { namespace S3 { -String getRunningAvailabilityZone() +std::string getRunningAvailabilityZone() { - throw Poco::Exception("Does not support availability zone detection for non-cloud environment"); + throw std::runtime_error("Does not support availability zone detection for non-cloud environment"); } } diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index 1f35443adf4d..d8d103a847a3 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -25,7 +25,7 @@ inline static constexpr uint64_t DEFAULT_EXPIRATION_WINDOW_SECONDS = 120; static inline constexpr char GCP_METADATA_SERVICE_ENDPOINT[] = "http://metadata.google.internal"; /// getRunningAvailabilityZone returns the availability zone of the underlying compute resources where the current process runs. -String getRunningAvailabilityZone(); +std::string getRunningAvailabilityZone(); class AWSEC2MetadataClient : public Aws::Internal::AWSHttpResourceClient { @@ -189,12 +189,14 @@ class S3CredentialsProviderChain : public Aws::Auth::AWSCredentialsProviderChain #else +# include + namespace DB { namespace S3 { -String getRunningAvailabilityZone(); +std::string getRunningAvailabilityZone(); } } From 3bbb329dd0f091b49a6dd771820110cde0e3a052 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 16 Nov 2023 00:13:05 +0100 Subject: [PATCH 082/274] Fix tests --- src/Server/TCPHandler.cpp | 15 +++++++++++++-- tests/queries/0_stateless/01119_session_log.sql | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index e7c400920773..884fc45f763c 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -587,8 +587,19 @@ void TCPHandler::runImpl() } catch (const Exception & e) { - /// Authentication failure with interserver secret. - if (e.code() == ErrorCodes::AUTHENTICATION_FAILED) + /// Authentication failure with interserver secret + /// - early exit without trying to send the exception to the client. + /// Because the server should not try to skip (parse, decompress) the remaining packets sent by the client, + /// as it will lead to additional work and unneeded exposure to unauthenticated connections. + + /// Note that the exception AUTHENTICATION_FAILED can be here in two cases: + /// 1. The authentication in receiveHello is skipped with "interserver secret", + /// postponed to receiving the query, and then failed. + /// 2. Receiving exception from a query using a table function to authenticate with another server. + /// In this case, the user is already authenticated with this server, + /// is_interserver_mode is false, and we can send the exception to the client normally. + + if (is_interserver_mode && e.code() == ErrorCodes::AUTHENTICATION_FAILED) throw; state.io.onException(); diff --git a/tests/queries/0_stateless/01119_session_log.sql b/tests/queries/0_stateless/01119_session_log.sql index 55f6228797a4..8f6967b89ec6 100644 --- a/tests/queries/0_stateless/01119_session_log.sql +++ b/tests/queries/0_stateless/01119_session_log.sql @@ -4,7 +4,7 @@ select * from remote('127.0.0.2', system, one, 'default', ''); select * from remote('127.0.0.2', system, one, 'default', 'wrong password'); -- { serverError AUTHENTICATION_FAILED } select * from remote('127.0.0.2', system, one, 'nonexistsnt_user_1119', ''); -- { serverError AUTHENTICATION_FAILED } set receive_timeout=1; -select * from remote('127.0.0.2', system, one, ' INTERSERVER SECRET ', ''); -- { serverError NO_REMOTE_SHARD_AVAILABLE } +select * from remote('127.0.0.2', system, one, ' INTERSERVER SECRET ', ''); -- { serverError AUTHENTICATION_FAILED } set receive_timeout=300; select * from remote('127.0.0.2', system, one, ' ', ''); -- { serverError AUTHENTICATION_FAILED } From 09bec3c754698f2c7e3cf37a9b02018c7adddc33 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 16 Nov 2023 00:16:02 +0100 Subject: [PATCH 083/274] Fix integration test --- .../test_distributed_inter_server_secret/test.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py index 4276fcffbf90..6e3f1e6e4166 100644 --- a/tests/integration/test_distributed_inter_server_secret/test.py +++ b/tests/integration/test_distributed_inter_server_secret/test.py @@ -304,26 +304,20 @@ def test_secure_insert_buffer_async(): def test_secure_disagree(): - with pytest.raises( - QueryRuntimeException, match=".*Interserver authentication failed.*" - ): + with pytest.raises(QueryRuntimeException): n1.query("SELECT * FROM dist_secure_disagree") def test_secure_disagree_insert(): n1.query("TRUNCATE TABLE data") n1.query("INSERT INTO dist_secure_disagree SELECT * FROM numbers(2)") - with pytest.raises( - QueryRuntimeException, match=".*Interserver authentication failed.*" - ): + with pytest.raises(QueryRuntimeException): n1.query( "SYSTEM FLUSH DISTRIBUTED ON CLUSTER secure_disagree dist_secure_disagree" ) - # check the the connection will be re-established + # check that the connection will be re-established # IOW that we will not get "Unknown BlockInfo field" - with pytest.raises( - QueryRuntimeException, match=".*Interserver authentication failed.*" - ): + with pytest.raises(QueryRuntimeException): assert int(n1.query("SELECT count() FROM dist_secure_disagree")) == 0 From db666bf2bc063eac1668258f890a6766dc3c4430 Mon Sep 17 00:00:00 2001 From: johnnymatthews <9611008+johnnymatthews@users.noreply.github.com> Date: Wed, 15 Nov 2023 21:47:35 -0400 Subject: [PATCH 084/274] Disables RU intro section. --- docs/ru/introduction/_category_.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/ru/introduction/_category_.yml b/docs/ru/introduction/_category_.yml index 539f7ab97edc..b3e58207c121 100644 --- a/docs/ru/introduction/_category_.yml +++ b/docs/ru/introduction/_category_.yml @@ -2,6 +2,3 @@ position: 1 label: 'Введение' collapsible: true collapsed: true -link: - type: generated-index - title: Введение From f505181b0d8c485ee02d3e62fc057c22d5d188cc Mon Sep 17 00:00:00 2001 From: johnnymatthews <9611008+johnnymatthews@users.noreply.github.com> Date: Wed, 15 Nov 2023 22:08:10 -0400 Subject: [PATCH 085/274] Adds basics index page to RU introduction. --- docs/ru/introduction/index.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docs/ru/introduction/index.md diff --git a/docs/ru/introduction/index.md b/docs/ru/introduction/index.md new file mode 100644 index 000000000000..74a6e4dd1351 --- /dev/null +++ b/docs/ru/introduction/index.md @@ -0,0 +1,13 @@ +--- +slug: /ru/introduction/ +sidebar_label: "Введение" +sidebar_position: 8 +--- + +# Введение + +Đ’ этом разделе ŃодержитŃŃŹ информация Đľ том, как начать Ń€Đ°Đ±ĐľŃ‚Ń Ń ClickHouse. + +- [Отличительные возможноŃти ClickHouse](./distinctive-features.md) +- [ПроизводительноŃŃ‚ŃŚ](./performance.md) +- [ĐŃтория ClickHouse](./history.md) From ef17d972ab64c90ff72a7c6c2beba61b522d6fcf Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Thu, 16 Nov 2023 05:18:50 +0000 Subject: [PATCH 086/274] Fix SET query formatting --- src/Parsers/ASTSetQuery.cpp | 48 ++++++++++++++++++- .../02916_set_formatting.reference | 11 +++++ .../0_stateless/02916_set_formatting.sql | 13 +++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02916_set_formatting.reference create mode 100644 tests/queries/0_stateless/02916_set_formatting.sql diff --git a/src/Parsers/ASTSetQuery.cpp b/src/Parsers/ASTSetQuery.cpp index e2c60e8369d5..78161b865ee5 100644 --- a/src/Parsers/ASTSetQuery.cpp +++ b/src/Parsers/ASTSetQuery.cpp @@ -4,11 +4,57 @@ #include #include #include +#include namespace DB { +class FieldVisitorToSetting : public StaticVisitor +{ +public: + template + String operator() (const T & x) const + { + FieldVisitorToString visitor; + return visitor(x); + } + + String operator() (const Map & x) const + { + WriteBufferFromOwnString wb; + + wb << '{'; + + auto it = x.begin(); + while (it != x.end()) + { + if (it != x.begin()) + wb << ", "; + wb << applyVisitor(*this, *it); + ++it; + } + wb << '}'; + + return wb.str(); + } + + String operator() (const Tuple & x) const + { + WriteBufferFromOwnString wb; + + for (auto it = x.begin(); it != x.end(); ++it) + { + if (it != x.begin()) + wb << ":"; + wb << applyVisitor(*this, *it); + } + + return wb.str(); + } +}; + + void ASTSetQuery::updateTreeHashImpl(SipHash & hash_state, bool /*ignore_aliases*/) const { for (const auto & change : changes) @@ -38,7 +84,7 @@ void ASTSetQuery::formatImpl(const FormatSettings & format, FormatState &, Forma if (!format.show_secrets && change.value.tryGet(custom) && custom.isSecret()) format.ostr << " = " << custom.toString(false); else - format.ostr << " = " << applyVisitor(FieldVisitorToString(), change.value); + format.ostr << " = " << applyVisitor(FieldVisitorToSetting(), change.value); } for (const auto & setting_name : default_settings) diff --git a/tests/queries/0_stateless/02916_set_formatting.reference b/tests/queries/0_stateless/02916_set_formatting.reference new file mode 100644 index 000000000000..34ff52365f95 --- /dev/null +++ b/tests/queries/0_stateless/02916_set_formatting.reference @@ -0,0 +1,11 @@ +SET additional_table_filters = {\'kjsnckjn\':\'ksanmn\', \'dkm\':\'dd\'} +SELECT v FROM t1 SETTINGS additional_table_filters = {\'default.t1\':\'s\'} +Row 1: +────── +statement: CREATE VIEW default.v1 +( + `v` UInt64 +) AS +SELECT v +FROM default.t1 +SETTINGS additional_table_filters = {'default.t1':'s != \'s1%\''} diff --git a/tests/queries/0_stateless/02916_set_formatting.sql b/tests/queries/0_stateless/02916_set_formatting.sql new file mode 100644 index 000000000000..10b875293f1d --- /dev/null +++ b/tests/queries/0_stateless/02916_set_formatting.sql @@ -0,0 +1,13 @@ +SELECT formatQuerySingleLine('set additional_table_filters = {\'kjsnckjn\': \'ksanmn\', \'dkm\': \'dd\'}'); +SELECT formatQuerySingleLine('SELECT v FROM t1 SETTINGS additional_table_filters = {\'default.t1\': \'s\'}'); + +DROP TABLE IF EXISTS t1; +DROP VIEW IF EXISTS v1; + +CREATE TABLE t1 (v UInt64, s String) ENGINE=MergeTree() ORDER BY v; +CREATE VIEW v1 (v UInt64) AS SELECT v FROM t1 SETTINGS additional_table_filters = {'default.t1': 's != \'s1%\''}; + +SHOW CREATE TABLE v1 FORMAT Vertical; + +DROP VIEW v1; +DROP TABLE t1; From 1d7eecaeece1bb2cdadf8c446bd8631592ecfb08 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Thu, 16 Nov 2023 06:08:12 +0000 Subject: [PATCH 087/274] Fix failure to start due to table dependency in joinGet --- src/Databases/DDLLoadingDependencyVisitor.cpp | 16 +++++------ .../02916_joinget_dependency.reference | 1 + .../0_stateless/02916_joinget_dependency.sh | 27 +++++++++++++++++++ 3 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/02916_joinget_dependency.reference create mode 100755 tests/queries/0_stateless/02916_joinget_dependency.sh diff --git a/src/Databases/DDLLoadingDependencyVisitor.cpp b/src/Databases/DDLLoadingDependencyVisitor.cpp index fc362dd8578d..77a40f674fda 100644 --- a/src/Databases/DDLLoadingDependencyVisitor.cpp +++ b/src/Databases/DDLLoadingDependencyVisitor.cpp @@ -144,22 +144,22 @@ void DDLLoadingDependencyVisitor::extractTableNameFromArgument(const ASTFunction const auto * arg = function.arguments->as()->children[arg_idx].get(); - if (const auto * dict_function = arg->as()) + if (const auto * function_arg = arg->as()) { - if (!functionIsDictGet(dict_function->name)) + if (!functionIsJoinGet(function_arg->name) && !functionIsDictGet(function_arg->name)) return; - /// Get the dictionary name from `dict*` function. - const auto * literal_arg = dict_function->arguments->as()->children[0].get(); - const auto * dictionary_name = literal_arg->as(); + /// Get the dictionary name from `dict*` function or the table name from 'joinGet' function. + const auto * literal_arg = function_arg->arguments->as()->children[0].get(); + const auto * name = literal_arg->as(); - if (!dictionary_name) + if (!name) return; - if (dictionary_name->value.getType() != Field::Types::String) + if (name->value.getType() != Field::Types::String) return; - auto maybe_qualified_name = QualifiedTableName::tryParseFromString(dictionary_name->value.get()); + auto maybe_qualified_name = QualifiedTableName::tryParseFromString(name->value.get()); if (!maybe_qualified_name) return; diff --git a/tests/queries/0_stateless/02916_joinget_dependency.reference b/tests/queries/0_stateless/02916_joinget_dependency.reference new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/tests/queries/0_stateless/02916_joinget_dependency.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02916_joinget_dependency.sh b/tests/queries/0_stateless/02916_joinget_dependency.sh new file mode 100755 index 000000000000..6477ae8c9670 --- /dev/null +++ b/tests/queries/0_stateless/02916_joinget_dependency.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# We test the dependency on the DROP + +$CLICKHOUSE_CLIENT -nm -q " + DROP TABLE IF EXISTS Sub_distributed; + DROP TABLE IF EXISTS Sub; + DROP TABLE IF EXISTS Mapping; + + CREATE TABLE Mapping (Id UInt64, RegionId UInt64) ENGINE = Join(ANY,LEFT,Id); + INSERT INTO Mapping VALUES (1,1); + CREATE TABLE Sub (Id UInt64, PropertyId UInt64) ENGINE = MergeTree() PRIMARY KEY (Id) ORDER BY (Id); + CREATE TABLE Sub_distributed (Id UInt64, PropertyId UInt64)ENGINE = Distributed('test_shard_localhost', $CLICKHOUSE_DATABASE, Sub, joinGet('$CLICKHOUSE_DATABASE.Mapping','RegionId',PropertyId));" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE Mapping; +" 2>&1 | grep -cm1 "HAVE_DEPENDENT_OBJECTS" + +$CLICKHOUSE_CLIENT -nm -q " + DROP TABLE Sub_distributed; + DROP TABLE Sub; + DROP TABLE Mapping; +" \ No newline at end of file From 7623153d3841d58f282915e24f9d032ee6d84e94 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Thu, 16 Nov 2023 07:01:49 +0000 Subject: [PATCH 088/274] Fix flattening existing Nested columns during ADD COLUMN --- src/Storages/AlterCommands.cpp | 21 +++++++++++++++---- .../02916_addcolumn_nested.reference | 3 +++ .../0_stateless/02916_addcolumn_nested.sql | 17 +++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/02916_addcolumn_nested.reference create mode 100644 tests/queries/0_stateless/02916_addcolumn_nested.sql diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 98bfa3b3f57a..7eeaa2d45941 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -395,11 +395,24 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) column.ttl = ttl; - metadata.columns.add(column, after_column, first); - - /// Slow, because each time a list is copied if (context->getSettingsRef().flatten_nested) - metadata.columns.flattenNested(); + { + StorageInMemoryMetadata temporary_metadata; + temporary_metadata.columns.add(column, /*after_column*/ "", /*first*/ true); + temporary_metadata.columns.flattenNested(); + + const auto transformed_columns = temporary_metadata.columns.getAll(); + + for (auto it = transformed_columns.rbegin(); it != transformed_columns.rend(); it++) + { + const auto & transformed_column = temporary_metadata.columns.get(it->name); + metadata.columns.add(transformed_column, after_column, first); + } + } + else + { + metadata.columns.add(column, after_column, first); + } } else if (type == DROP_COLUMN) { diff --git a/tests/queries/0_stateless/02916_addcolumn_nested.reference b/tests/queries/0_stateless/02916_addcolumn_nested.reference new file mode 100644 index 000000000000..869d4336c62f --- /dev/null +++ b/tests/queries/0_stateless/02916_addcolumn_nested.reference @@ -0,0 +1,3 @@ +CREATE TABLE default.nested_table\n(\n `id` UInt64,\n `first` Nested(a Int8, b String)\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.nested_table\n(\n `id` UInt64,\n `second.c` Array(Int8),\n `second.d` Array(String),\n `first` Nested(a Int8, b String)\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.nested_table\n(\n `third` Nested(e Int8, f String),\n `id` UInt64,\n `second.c` Array(Int8),\n `second.d` Array(String),\n `first` Nested(a Int8, b String)\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02916_addcolumn_nested.sql b/tests/queries/0_stateless/02916_addcolumn_nested.sql new file mode 100644 index 000000000000..b23854824b5d --- /dev/null +++ b/tests/queries/0_stateless/02916_addcolumn_nested.sql @@ -0,0 +1,17 @@ +SET flatten_nested = 0; + +DROP TABLE IF EXISTS nested_table; +CREATE TABLE nested_table (id UInt64, first Nested(a Int8, b String)) ENGINE = MergeTree() ORDER BY id; +SHOW CREATE nested_table; + +SET flatten_nested = 1; + +ALTER TABLE nested_table ADD COLUMN second Nested(c Int8, d String) AFTER id; +SHOW CREATE nested_table; + +SET flatten_nested = 0; + +ALTER TABLE nested_table ADD COLUMN third Nested(e Int8, f String) FIRST; +SHOW CREATE nested_table; + +DROP TABLE nested_table; From cffc6611e000faa19cac35dccfc093d77ae7e315 Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Thu, 16 Nov 2023 07:05:41 +0000 Subject: [PATCH 089/274] Empty commit. From 69f214cdbcc85808d77893112f7c560285747a09 Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Thu, 16 Nov 2023 08:04:57 +0000 Subject: [PATCH 090/274] fix comments. Signed-off-by: Jianfei Hu --- src/IO/S3/Credentials.cpp | 32 +++++++++++++++++++------------- src/IO/S3/Credentials.h | 1 - 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index e25f45517234..73763853713c 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -1,4 +1,15 @@ #include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNSUPPORTED_METHOD; +} + +} #if USE_AWS_S3 @@ -16,11 +27,9 @@ # include # include -# include # include # include -# include # include # include @@ -42,7 +51,6 @@ namespace ErrorCodes { extern const int AWS_ERROR; extern const int GCP_ERROR; - extern const int UNSUPPORTED_METHOD; } namespace S3 @@ -280,20 +288,20 @@ String getRunningAvailabilityZone() LOG_INFO(&Poco::Logger::get("Application"), "Trying to detect the availability zone."); try { - auto aws_az = AWSEC2MetadataClient::getAvailabilityZoneOrException(); - return aws_az; + return AWSEC2MetadataClient::getAvailabilityZoneOrException(); } - catch (const std::exception & aws_ex) + catch (...) { + auto aws_ex_msg = getExceptionMessage(std::current_exception(), false); try { - auto gcp_zone = getGCPAvailabilityZoneOrException(); - return gcp_zone; + return getGCPAvailabilityZoneOrException(); } - catch (const std::exception & gcp_ex) + catch (...) { + auto gcp_ex_msg = getExceptionMessage(std::current_exception(), false); throw DB::Exception(ErrorCodes::UNSUPPORTED_METHOD, - "Failed to find the availability zone, tried AWS and GCP. AWS Error: {}\nGCP Error: {}", aws_ex.what(), gcp_ex.what()); + "Failed to find the availability zone, tried AWS and GCP. AWS Error: {}\nGCP Error: {}", aws_ex_msg, gcp_ex_msg); } } } @@ -784,8 +792,6 @@ S3CredentialsProviderChain::S3CredentialsProviderChain( #else -# include - namespace DB { @@ -794,7 +800,7 @@ namespace S3 std::string getRunningAvailabilityZone() { - throw std::runtime_error("Does not support availability zone detection for non-cloud environment"); + throw DB::Exception(ErrorCodes::UNSUPPORTED_METHOD, "Does not support availability zone detection for non-cloud environment"); } } diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index d8d103a847a3..ad73de23486c 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -4,7 +4,6 @@ #if USE_AWS_S3 -# include # include # include From 3b226a10feac18ae16836f5e627d48b413c12cc1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 16 Nov 2023 09:50:34 +0100 Subject: [PATCH 091/274] Fix test --- .../0_stateless/01555_system_distribution_queue_mask.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql index 7ade1d24c596..3a90765226a0 100644 --- a/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql +++ b/tests/queries/0_stateless/01555_system_distribution_queue_mask.sql @@ -17,7 +17,7 @@ system stop distributed sends dist_01555; insert into dist_01555 values (1)(2); -- since test_cluster_with_incorrect_pw contains incorrect password ignore error -system flush distributed dist_01555; -- { clientError ATTEMPT_TO_READ_AFTER_EOF } +system flush distributed dist_01555; -- { serverError 516 } select length(splitByChar('*', data_path)), replaceRegexpOne(data_path, '^.*/([^/]*)/' , '\\1'), extract(last_exception, 'AUTHENTICATION_FAILED'), dateDiff('s', last_exception_time, now()) < 3600 from system.distribution_queue where database = currentDatabase() and table = 'dist_01555' format CSV; drop table dist_01555; @@ -30,7 +30,7 @@ create table dist_01555 (key Int) Engine=Distributed(test_cluster_with_incorrect insert into dist_01555 values (1)(2); -- since test_cluster_with_incorrect_pw contains incorrect password ignore error -system flush distributed dist_01555; -- { clientError ATTEMPT_TO_READ_AFTER_EOF } +system flush distributed dist_01555; -- { serverError 516 } select length(splitByChar('*', data_path)), replaceRegexpOne(data_path, '^.*/([^/]*)/' , '\\1'), extract(last_exception, 'AUTHENTICATION_FAILED'), dateDiff('s', last_exception_time, now()) < 3600 from system.distribution_queue where database = currentDatabase() and table = 'dist_01555' format CSV; drop table dist_01555; From 61948e31714f1aa3fa8143a569a72403c5c70408 Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Thu, 16 Nov 2023 11:12:45 +0100 Subject: [PATCH 092/274] Update src/Core/Settings.h Co-authored-by: Nikita Taranov --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 34547aded9c3..76016bc70fb0 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -94,7 +94,7 @@ class IColumn; M(UInt64, s3_max_put_rps, 0, "Limit on S3 PUT request per second rate before throttling. Zero means unlimited.", 0) \ M(UInt64, s3_max_put_burst, 0, "Max number of requests that can be issued simultaneously before hitting request per second limit. By default (0) equals to `s3_max_put_rps`", 0) \ M(UInt64, s3_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ - M(Bool, s3_use_adaptive_timeouts, true, "When aggressive timeouts are enabled first two attempts are made with low receive and send timeout", 0) \ + M(Bool, s3_use_adaptive_timeouts, true, "When adaptive timeouts are enabled first two attempts are made with low receive and send timeout", 0) \ M(UInt64, azure_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ M(Bool, s3_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables.", 0) \ M(Bool, azure_truncate_on_insert, false, "Enables or disables truncate before insert in azure engine tables.", 0) \ From bdeb04f7d3b403a425dd4138fdf4c5fd4c72cb96 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 15 Nov 2023 16:00:54 +0000 Subject: [PATCH 093/274] Exctract JOIN ON visitor from LogicalExpressionOptimizerVisitor --- .../Passes/LogicalExpressionOptimizerPass.cpp | 355 +++++++++--------- 1 file changed, 181 insertions(+), 174 deletions(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index 9602ef8a7438..50410c8c8294 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -16,6 +16,181 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +/// Visitor that optimizes logical expressions _only_ in JOIN ON section +class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithContext +{ +public: + using Base = InDepthQueryTreeVisitorWithContext; + + explicit JoinOnLogicalExpressionOptimizerVisitor(ContextPtr context) + : Base(std::move(context)) + {} + + void enterImpl(QueryTreeNodePtr & node) + { + auto * function_node = node->as(); + + if (!function_node) + return; + + if (function_node->getFunctionName() == "or") + { + tryOptimizeIsNotDistinctOrIsNull(node); + return; + } + } + +private: + void tryOptimizeIsNotDistinctOrIsNull(QueryTreeNodePtr & node) + { + auto & function_node = node->as(); + assert(function_node.getFunctionName() == "or"); + + QueryTreeNodes or_operands; + + /// Indices of `equals` or `isNotDistinctFrom` functions in the vector above + std::vector equals_functions_indices; + + /** Map from `isNull` argument to indices of operands that contains that `isNull` functions + * `a = b OR (a IS NULL AND b IS NULL) OR (a IS NULL AND c IS NULL)` + * will be mapped to + * { + * a => [(a IS NULL AND b IS NULL), (a IS NULL AND c IS NULL)] + * b => [(a IS NULL AND b IS NULL)] + * c => [(a IS NULL AND c IS NULL)] + * } + * Then for each a <=> b we can find all operands that contains both a IS NULL and b IS NULL + */ + QueryTreeNodePtrWithHashMap> is_null_argument_to_indices; + + for (const auto & argument : function_node.getArguments()) + { + or_operands.push_back(argument); + + auto * argument_function = argument->as(); + if (!argument_function) + continue; + + const auto & func_name = argument_function->getFunctionName(); + if (func_name == "equals" || func_name == "isNotDistinctFrom") + equals_functions_indices.push_back(or_operands.size() - 1); + + if (func_name == "and") + { + for (const auto & and_argument : argument_function->getArguments().getNodes()) + { + auto * and_argument_function = and_argument->as(); + if (and_argument_function && and_argument_function->getFunctionName() == "isNull") + { + const auto & is_null_argument = and_argument_function->getArguments().getNodes()[0]; + is_null_argument_to_indices[is_null_argument].push_back(or_operands.size() - 1); + } + } + } + } + + /// OR operands that are changed to and needs to be re-resolved + std::unordered_set arguments_to_reresolve; + + for (size_t equals_function_idx : equals_functions_indices) + { + auto * equals_function = or_operands[equals_function_idx]->as(); + + /// For a <=> b we are looking for expressions containing both `a IS NULL` and `b IS NULL` combined with AND + const auto & argument_nodes = equals_function->getArguments().getNodes(); + const auto & lhs_is_null_parents = is_null_argument_to_indices[argument_nodes[0]]; + const auto & rhs_is_null_parents = is_null_argument_to_indices[argument_nodes[1]]; + std::unordered_set operands_to_optimize; + std::set_intersection(lhs_is_null_parents.begin(), lhs_is_null_parents.end(), + rhs_is_null_parents.begin(), rhs_is_null_parents.end(), + std::inserter(operands_to_optimize, operands_to_optimize.begin())); + + /// If we have `a = b OR (a IS NULL AND b IS NULL)` we can optimize it to `a <=> b` + if (!operands_to_optimize.empty() && equals_function->getFunctionName() == "equals") + arguments_to_reresolve.insert(equals_function_idx); + + for (size_t to_optimize_idx : operands_to_optimize) + { + /// We are looking for operand `a IS NULL AND b IS NULL AND ...` + auto * operand_to_optimize = or_operands[to_optimize_idx]->as(); + + /// Remove `a IS NULL` and `b IS NULL` arguments from AND + QueryTreeNodes new_arguments; + for (const auto & and_argument : operand_to_optimize->getArguments().getNodes()) + { + bool to_eliminate = false; + + const auto * and_argument_function = and_argument->as(); + if (and_argument_function && and_argument_function->getFunctionName() == "isNull") + { + const auto & is_null_argument = and_argument_function->getArguments().getNodes()[0]; + to_eliminate = (is_null_argument->isEqual(*argument_nodes[0]) || is_null_argument->isEqual(*argument_nodes[1])); + } + + if (to_eliminate) + arguments_to_reresolve.insert(to_optimize_idx); + else + new_arguments.emplace_back(and_argument); + } + /// If less than two arguments left, we will remove or replace the whole AND below + operand_to_optimize->getArguments().getNodes() = std::move(new_arguments); + } + } + + if (arguments_to_reresolve.empty()) + /// Nothing have been changed + return; + + auto and_function_resolver = FunctionFactory::instance().get("and", getContext()); + auto strict_equals_function_resolver = FunctionFactory::instance().get("isNotDistinctFrom", getContext()); + QueryTreeNodes new_or_operands; + for (size_t i = 0; i < or_operands.size(); ++i) + { + if (arguments_to_reresolve.contains(i)) + { + auto * function = or_operands[i]->as(); + if (function->getFunctionName() == "equals") + { + /// Because we removed checks for IS NULL, we should replace `a = b` with `a <=> b` + function->resolveAsFunction(strict_equals_function_resolver); + new_or_operands.emplace_back(std::move(or_operands[i])); + } + else if (function->getFunctionName() == "and") + { + const auto & and_arguments = function->getArguments().getNodes(); + if (and_arguments.size() > 1) + { + function->resolveAsFunction(and_function_resolver); + new_or_operands.emplace_back(std::move(or_operands[i])); + } + else if (and_arguments.size() == 1) + { + /// Replace AND with a single argument with the argument itself + new_or_operands.emplace_back(and_arguments[0]); + } + } + else + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected function name: '{}'", function->getFunctionName()); + } + else + { + new_or_operands.emplace_back(std::move(or_operands[i])); + } + } + + if (new_or_operands.size() == 1) + { + node = std::move(new_or_operands[0]); + return; + } + + /// Rebuild OR function + auto or_function_resolver = FunctionFactory::instance().get("or", getContext()); + function_node.getArguments().getNodes() = std::move(new_or_operands); + function_node.resolveAsFunction(or_function_resolver); + } +}; + class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithContext { public: @@ -29,13 +204,15 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont { if (auto * join_node = node->as()) { - join_stack.push_back(join_node); + /// Operator <=> is not supported outside of JOIN ON section + if (join_node->hasJoinExpression()) + { + JoinOnLogicalExpressionOptimizerVisitor join_on_visitor(getContext()); + join_on_visitor.visit(join_node->getJoinExpression()); + } return; } - if (!join_stack.empty() && join_stack.back()->getJoinExpression().get() == node.get()) - is_inside_on_section = true; - auto * function_node = node->as(); if (!function_node) @@ -44,10 +221,6 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont if (function_node->getFunctionName() == "or") { tryReplaceOrEqualsChainWithIn(node); - - /// Operator <=> is not supported outside of JOIN ON section - if (is_inside_on_section) - tryOptimizeIsNotDistinctOrIsNull(node); return; } @@ -58,19 +231,6 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont } } - void leaveImpl(QueryTreeNodePtr & node) - { - if (!join_stack.empty() && join_stack.back()->getJoinExpression().get() == node.get()) - is_inside_on_section = false; - - if (auto * join_node = node->as()) - { - assert(join_stack.back() == join_node); - join_stack.pop_back(); - return; - } - } - private: void tryReplaceAndEqualsChainsWithConstant(QueryTreeNodePtr & node) { @@ -264,159 +424,6 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont function_node.getArguments().getNodes() = std::move(or_operands); function_node.resolveAsFunction(or_function_resolver); } - - void tryOptimizeIsNotDistinctOrIsNull(QueryTreeNodePtr & node) - { - auto & function_node = node->as(); - assert(function_node.getFunctionName() == "or"); - - QueryTreeNodes or_operands; - - /// Indices of `equals` or `isNotDistinctFrom` functions in the vector above - std::vector equals_functions_indices; - - /** Map from `isNull` argument to indices of operands that contains that `isNull` functions - * `a = b OR (a IS NULL AND b IS NULL) OR (a IS NULL AND c IS NULL)` - * will be mapped to - * { - * a => [(a IS NULL AND b IS NULL), (a IS NULL AND c IS NULL)] - * b => [(a IS NULL AND b IS NULL)] - * c => [(a IS NULL AND c IS NULL)] - * } - * Then for each a <=> b we can find all operands that contains both a IS NULL and b IS NULL - */ - QueryTreeNodePtrWithHashMap> is_null_argument_to_indices; - - for (const auto & argument : function_node.getArguments()) - { - or_operands.push_back(argument); - - auto * argument_function = argument->as(); - if (!argument_function) - continue; - - const auto & func_name = argument_function->getFunctionName(); - if (func_name == "equals" || func_name == "isNotDistinctFrom") - equals_functions_indices.push_back(or_operands.size() - 1); - - if (func_name == "and") - { - for (const auto & and_argument : argument_function->getArguments().getNodes()) - { - auto * and_argument_function = and_argument->as(); - if (and_argument_function && and_argument_function->getFunctionName() == "isNull") - { - const auto & is_null_argument = and_argument_function->getArguments().getNodes()[0]; - is_null_argument_to_indices[is_null_argument].push_back(or_operands.size() - 1); - } - } - } - } - - /// OR operands that are changed to and needs to be re-resolved - std::unordered_set arguments_to_reresolve; - - for (size_t equals_function_idx : equals_functions_indices) - { - auto * equals_function = or_operands[equals_function_idx]->as(); - - /// For a <=> b we are looking for expressions containing both `a IS NULL` and `b IS NULL` combined with AND - const auto & argument_nodes = equals_function->getArguments().getNodes(); - const auto & lhs_is_null_parents = is_null_argument_to_indices[argument_nodes[0]]; - const auto & rhs_is_null_parents = is_null_argument_to_indices[argument_nodes[1]]; - std::unordered_set operands_to_optimize; - std::set_intersection(lhs_is_null_parents.begin(), lhs_is_null_parents.end(), - rhs_is_null_parents.begin(), rhs_is_null_parents.end(), - std::inserter(operands_to_optimize, operands_to_optimize.begin())); - - /// If we have `a = b OR (a IS NULL AND b IS NULL)` we can optimize it to `a <=> b` - if (!operands_to_optimize.empty() && equals_function->getFunctionName() == "equals") - arguments_to_reresolve.insert(equals_function_idx); - - for (size_t to_optimize_idx : operands_to_optimize) - { - /// We are looking for operand `a IS NULL AND b IS NULL AND ...` - auto * operand_to_optimize = or_operands[to_optimize_idx]->as(); - - /// Remove `a IS NULL` and `b IS NULL` arguments from AND - QueryTreeNodes new_arguments; - for (const auto & and_argument : operand_to_optimize->getArguments().getNodes()) - { - bool to_eliminate = false; - - const auto * and_argument_function = and_argument->as(); - if (and_argument_function && and_argument_function->getFunctionName() == "isNull") - { - const auto & is_null_argument = and_argument_function->getArguments().getNodes()[0]; - to_eliminate = (is_null_argument->isEqual(*argument_nodes[0]) || is_null_argument->isEqual(*argument_nodes[1])); - } - - if (to_eliminate) - arguments_to_reresolve.insert(to_optimize_idx); - else - new_arguments.emplace_back(and_argument); - } - /// If less than two arguments left, we will remove or replace the whole AND below - operand_to_optimize->getArguments().getNodes() = std::move(new_arguments); - } - } - - - if (arguments_to_reresolve.empty()) - /// Nothing have been changed - return; - - auto and_function_resolver = FunctionFactory::instance().get("and", getContext()); - auto strict_equals_function_resolver = FunctionFactory::instance().get("isNotDistinctFrom", getContext()); - QueryTreeNodes new_or_operands; - for (size_t i = 0; i < or_operands.size(); ++i) - { - if (arguments_to_reresolve.contains(i)) - { - auto * function = or_operands[i]->as(); - if (function->getFunctionName() == "equals") - { - /// Because we removed checks for IS NULL, we should replace `a = b` with `a <=> b` - function->resolveAsFunction(strict_equals_function_resolver); - new_or_operands.emplace_back(std::move(or_operands[i])); - } - else if (function->getFunctionName() == "and") - { - const auto & and_arguments = function->getArguments().getNodes(); - if (and_arguments.size() > 1) - { - function->resolveAsFunction(and_function_resolver); - new_or_operands.emplace_back(std::move(or_operands[i])); - } - else if (and_arguments.size() == 1) - { - /// Replace AND with a single argument with the argument itself - new_or_operands.emplace_back(and_arguments[0]); - } - } - else - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected function name: '{}'", function->getFunctionName()); - } - else - { - new_or_operands.emplace_back(std::move(or_operands[i])); - } - } - - if (new_or_operands.size() == 1) - { - node = std::move(new_or_operands[0]); - return; - } - - /// Rebuild OR function - auto or_function_resolver = FunctionFactory::instance().get("or", getContext()); - function_node.getArguments().getNodes() = std::move(new_or_operands); - function_node.resolveAsFunction(or_function_resolver); - } - - bool is_inside_on_section = false; - std::deque join_stack; }; void LogicalExpressionOptimizerPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) From 6ad0e9066a65c30aa9abb758dd8f05b99a31bb68 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 15 Nov 2023 17:02:30 +0000 Subject: [PATCH 094/274] Rerun resolve in JoinOnLogicalExpressionOptimizerVisitor --- .../Passes/LogicalExpressionOptimizerPass.cpp | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index 50410c8c8294..e667b6030206 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB { @@ -35,13 +36,27 @@ class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWi if (function_node->getFunctionName() == "or") { - tryOptimizeIsNotDistinctOrIsNull(node); + bool is_argument_type_changed = tryOptimizeIsNotDistinctOrIsNull(node, getContext()); + if (is_argument_type_changed) + need_rerun_resolve = true; return; } } + void leaveImpl(QueryTreeNodePtr & node) + { + if (!need_rerun_resolve) + return; + + if (auto * function_node = node->as()) + rerunFunctionResolve(function_node, getContext()); + } + private: - void tryOptimizeIsNotDistinctOrIsNull(QueryTreeNodePtr & node) + bool need_rerun_resolve = false; + + /// Returns true if type of some operand is changed and parent function needs to be re-resolved + static bool tryOptimizeIsNotDistinctOrIsNull(QueryTreeNodePtr & node, const ContextPtr & context) { auto & function_node = node->as(); assert(function_node.getFunctionName() == "or"); @@ -139,10 +154,12 @@ class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWi if (arguments_to_reresolve.empty()) /// Nothing have been changed - return; + return false; - auto and_function_resolver = FunctionFactory::instance().get("and", getContext()); - auto strict_equals_function_resolver = FunctionFactory::instance().get("isNotDistinctFrom", getContext()); + auto and_function_resolver = FunctionFactory::instance().get("and", context); + auto strict_equals_function_resolver = FunctionFactory::instance().get("isNotDistinctFrom", context); + + bool need_reresolve = false; QueryTreeNodes new_or_operands; for (size_t i = 0; i < or_operands.size(); ++i) { @@ -151,7 +168,8 @@ class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWi auto * function = or_operands[i]->as(); if (function->getFunctionName() == "equals") { - /// Because we removed checks for IS NULL, we should replace `a = b` with `a <=> b` + /// We should replace `a = b` with `a <=> b` because we removed checks for IS NULL + need_reresolve = need_reresolve || function->getResultType()->isNullable(); function->resolveAsFunction(strict_equals_function_resolver); new_or_operands.emplace_back(std::move(or_operands[i])); } @@ -181,13 +199,14 @@ class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWi if (new_or_operands.size() == 1) { node = std::move(new_or_operands[0]); - return; + return need_reresolve; } /// Rebuild OR function - auto or_function_resolver = FunctionFactory::instance().get("or", getContext()); + auto or_function_resolver = FunctionFactory::instance().get("or", context); function_node.getArguments().getNodes() = std::move(new_or_operands); function_node.resolveAsFunction(or_function_resolver); + return need_reresolve; } }; From 649d734409fab1eb602026cd1e39b601a0b1673c Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Nov 2023 10:13:07 +0000 Subject: [PATCH 095/274] Bump gRPC to v1.56.3 --- contrib/grpc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/grpc b/contrib/grpc index a08fe1a34075..bc110c3dc91b 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit a08fe1a34075c93bb2d606dd608b9a3953288b81 +Subproject commit bc110c3dc91b77d1e54957871df54fd39f2a49d1 From a49db81b9f10aacee9528a02bea0f7b57dc532a7 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Nov 2023 10:15:11 +0000 Subject: [PATCH 096/274] Bump gRPC to v1.57.1 --- contrib/grpc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/grpc b/contrib/grpc index bc110c3dc91b..fd802577cc06 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit bc110c3dc91b77d1e54957871df54fd39f2a49d1 +Subproject commit fd802577cc06226428c99297d5be3a24f5e3ab96 From 1ba408eb0b2f37e2978376920920c7863a0325b8 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Nov 2023 10:38:03 +0000 Subject: [PATCH 097/274] Bump gRPC to v1.58.2 --- contrib/grpc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/grpc b/contrib/grpc index fd802577cc06..2e45a02f2b24 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit fd802577cc06226428c99297d5be3a24f5e3ab96 +Subproject commit 2e45a02f2b24e3cc455d1793a469e1dbba894f94 From a250c2bb08eda91f50eebd5bffd5086ef3251b9f Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Nov 2023 10:42:41 +0000 Subject: [PATCH 098/274] Bump gRPC to v1.59.2 --- contrib/grpc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/grpc b/contrib/grpc index 2e45a02f2b24..740e3dfd9730 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit 2e45a02f2b24e3cc455d1793a469e1dbba894f94 +Subproject commit 740e3dfd97301a52ad8165b65285bcc149d9e817 From ac7fd357e4f75d9b16023450d8fe5df6f3e68cfa Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 15 Nov 2023 10:47:10 +0000 Subject: [PATCH 099/274] Bump protobuf to v23.2 --- contrib/google-protobuf | 2 +- contrib/google-protobuf-cmake/CMakeLists.txt | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/contrib/google-protobuf b/contrib/google-protobuf index 089b89c8d414..5b1791519907 160000 --- a/contrib/google-protobuf +++ b/contrib/google-protobuf @@ -1 +1 @@ -Subproject commit 089b89c8d4140f0d49fe4222b047a8ea814bc752 +Subproject commit 5b1791519907360781cfe3bebe1c79e5b1b0bcba diff --git a/contrib/google-protobuf-cmake/CMakeLists.txt b/contrib/google-protobuf-cmake/CMakeLists.txt index f6955a3d8ce4..3b53ac822da7 100644 --- a/contrib/google-protobuf-cmake/CMakeLists.txt +++ b/contrib/google-protobuf-cmake/CMakeLists.txt @@ -174,6 +174,8 @@ set(libprotobuf_files ${protobuf_source_dir}/src/google/protobuf/message.cc ${protobuf_source_dir}/src/google/protobuf/message_lite.cc ${protobuf_source_dir}/src/google/protobuf/parse_context.cc + ${protobuf_source_dir}/src/google/protobuf/port.cc + ${protobuf_source_dir}/src/google/protobuf/reflection_mode.cc ${protobuf_source_dir}/src/google/protobuf/reflection_ops.cc ${protobuf_source_dir}/src/google/protobuf/repeated_field.cc ${protobuf_source_dir}/src/google/protobuf/repeated_ptr_field.cc @@ -213,6 +215,7 @@ set(libprotoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/enum.cc ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/extension.cc ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field_generators/cord_field.cc ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field_generators/enum_field.cc ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field_generators/map_field.cc ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/field_generators/message_field.cc @@ -299,6 +302,13 @@ set(libprotoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/python/pyi_generator.cc ${protobuf_source_dir}/src/google/protobuf/compiler/retention.cc ${protobuf_source_dir}/src/google/protobuf/compiler/ruby/ruby_generator.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/rust/accessors/accessors.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/rust/accessors/singular_bytes.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/rust/accessors/singular_scalar.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/rust/context.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/rust/generator.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/rust/message.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/rust/naming.cc ${protobuf_source_dir}/src/google/protobuf/compiler/subprocess.cc ${protobuf_source_dir}/src/google/protobuf/compiler/zip_writer.cc ) From ebd42187ad12ce2be24833820f59bb3d81def382 Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Thu, 16 Nov 2023 12:29:15 +0100 Subject: [PATCH 100/274] Update tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml Co-authored-by: Nikita Taranov --- .../configs/config.d/storage_conf.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml index f51b854de752..98c6f551be6e 100644 --- a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml @@ -34,7 +34,7 @@ true 1 - 0 + 0 1 20000 From 7d37c0e07073b2a1909e80c4aea45fdc4a35be75 Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Thu, 16 Nov 2023 12:29:21 +0100 Subject: [PATCH 101/274] Update tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml Co-authored-by: Nikita Taranov --- .../configs/config.d/storage_conf.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml index 98c6f551be6e..6303e9273fc3 100644 --- a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml @@ -11,7 +11,7 @@ true 0 - 0 + 0 20000 From 4a1e207e7a5b02da2b2f6ea46edecd9fe6a9185c Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 16 Nov 2023 12:31:00 +0100 Subject: [PATCH 102/274] review notes --- base/poco/Net/src/HTTPSession.cpp | 34 +++++++++++++++++++++---------- src/IO/S3/PocoHTTPClient.cpp | 20 ++++++++++++++---- src/IO/S3/PocoHTTPClient.h | 6 +++--- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index 9ebbd7d04cd7..d303a4c654b7 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -93,22 +93,34 @@ void HTTPSession::setTimeout(const Poco::Timespan& timeout) void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco::Timespan& sendTimeout, const Poco::Timespan& receiveTimeout) { - _connectionTimeout = connectionTimeout; - - if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) + try { - _sendTimeout = sendTimeout; + _connectionTimeout = connectionTimeout; - if (connected()) - _socket.setSendTimeout(_sendTimeout); - } + if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) { + _sendTimeout = sendTimeout; + + if (connected()) + _socket.setSendTimeout(_sendTimeout); + } - if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) + if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) { + _receiveTimeout = receiveTimeout; + + if (connected()) + _socket.setReceiveTimeout(_receiveTimeout); + } + } + catch (NetException &) { - _receiveTimeout = receiveTimeout; - if (connected()) - _socket.setReceiveTimeout(_receiveTimeout); +#ifndef NDEBUG + // mute exceptions in release + // just in case when changing settings on socket is not allowed + // however it should be OK for timeouts +#else + throw; +#endif } } diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index f681362e607b..4a1b6def1331 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -407,17 +407,29 @@ void PocoHTTPClient::makeRequestInternalImpl( /// This can lead to request signature difference on S3 side. if constexpr (pooled) session = makePooledHTTPSession( - target_uri, getTimeouts(method, first_attempt), http_connection_pool_size, wait_on_pool_size_limit, proxy_configuration); + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true), + http_connection_pool_size, + wait_on_pool_size_limit, + proxy_configuration); else - session = makeHTTPSession(target_uri, getTimeouts(method, first_attempt), proxy_configuration); + session = makeHTTPSession( + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true), + proxy_configuration); } else { if constexpr (pooled) session = makePooledHTTPSession( - target_uri, getTimeouts(method, first_attempt), http_connection_pool_size, wait_on_pool_size_limit); + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true), + http_connection_pool_size, + wait_on_pool_size_limit); else - session = makeHTTPSession(target_uri, getTimeouts(method, first_attempt)); + session = makeHTTPSession( + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true)); } /// In case of error this address will be written to logs diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 14c4fec5dd7f..5178d75e7b6d 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -55,7 +55,7 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration size_t http_connection_pool_size = 0; /// See PoolBase::BehaviourOnLimit bool wait_on_pool_size_limit = true; - bool s3_use_adaptive_timeouts = false; + bool s3_use_adaptive_timeouts = true; std::function error_report; @@ -171,7 +171,7 @@ class PocoHTTPClient : public Aws::Http::HttpClient Aws::Utils::RateLimits::RateLimiterInterface * readLimiter, Aws::Utils::RateLimits::RateLimiterInterface * writeLimiter) const; - ConnectionTimeouts getTimeouts(const String & method, bool first_attempt, bool first_byte = true) const; + ConnectionTimeouts getTimeouts(const String & method, bool first_attempt, bool first_byte) const; protected: static S3MetricKind getMetricKind(const Aws::Http::HttpRequest & request); @@ -182,7 +182,7 @@ class PocoHTTPClient : public Aws::Http::HttpClient ConnectionTimeouts timeouts; const RemoteHostFilter & remote_host_filter; unsigned int s3_max_redirects; - bool s3_use_adaptive_timeouts = false; + bool s3_use_adaptive_timeouts = true; bool enable_s3_requests_logging; bool for_disk_s3; From e53e723be81d9ba76595afa7bbe67c1bf8764c4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 16 Nov 2023 12:32:42 +0100 Subject: [PATCH 103/274] Apply same improvement to initializeAggregation --- src/Functions/initializeAggregation.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Functions/initializeAggregation.cpp b/src/Functions/initializeAggregation.cpp index 83df28808a11..eeeb03aeb309 100644 --- a/src/Functions/initializeAggregation.cpp +++ b/src/Functions/initializeAggregation.cpp @@ -141,10 +141,19 @@ ColumnPtr FunctionInitializeAggregation::executeImpl(const ColumnsWithTypeAndNam that->addBatch(0, input_rows_count, places.data(), 0, aggregate_arguments, arena.get()); } - for (size_t i = 0; i < input_rows_count; ++i) + if (agg_func.isState()) + { /// We should use insertMergeResultInto to insert result into ColumnAggregateFunction /// correctly if result contains AggregateFunction's states - agg_func.insertMergeResultInto(places[i], res_col, arena.get()); + for (size_t i = 0; i < input_rows_count; ++i) + agg_func.insertMergeResultInto(places[i], res_col, arena.get()); + } + else + { + for (size_t i = 0; i < input_rows_count; ++i) + agg_func.insertResultInto(places[i], res_col, arena.get()); + } + return result_holder; } From a0934253deff176da4c2eb21844cf31f3f7a3c61 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 16 Nov 2023 11:39:36 +0000 Subject: [PATCH 104/274] Bump protobuf to v24.4 --- contrib/google-protobuf | 2 +- contrib/google-protobuf-cmake/CMakeLists.txt | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/contrib/google-protobuf b/contrib/google-protobuf index 5b1791519907..0862007f6ca1 160000 --- a/contrib/google-protobuf +++ b/contrib/google-protobuf @@ -1 +1 @@ -Subproject commit 5b1791519907360781cfe3bebe1c79e5b1b0bcba +Subproject commit 0862007f6ca1f5723c58f10f0ca34f3f25a63b2e diff --git a/contrib/google-protobuf-cmake/CMakeLists.txt b/contrib/google-protobuf-cmake/CMakeLists.txt index 3b53ac822da7..1ed4133270b3 100644 --- a/contrib/google-protobuf-cmake/CMakeLists.txt +++ b/contrib/google-protobuf-cmake/CMakeLists.txt @@ -82,7 +82,6 @@ set(libprotobuf_lite_files ${protobuf_source_dir}/src/google/protobuf/any_lite.cc ${protobuf_source_dir}/src/google/protobuf/arena.cc ${protobuf_source_dir}/src/google/protobuf/arena_align.cc - ${protobuf_source_dir}/src/google/protobuf/arena_config.cc ${protobuf_source_dir}/src/google/protobuf/arenastring.cc ${protobuf_source_dir}/src/google/protobuf/arenaz_sampler.cc ${protobuf_source_dir}/src/google/protobuf/extension_set.cc @@ -131,17 +130,18 @@ set(libprotobuf_files ${protobuf_source_dir}/src/google/protobuf/any_lite.cc ${protobuf_source_dir}/src/google/protobuf/arena.cc ${protobuf_source_dir}/src/google/protobuf/arena_align.cc - ${protobuf_source_dir}/src/google/protobuf/arena_config.cc ${protobuf_source_dir}/src/google/protobuf/arenastring.cc ${protobuf_source_dir}/src/google/protobuf/arenaz_sampler.cc ${protobuf_source_dir}/src/google/protobuf/compiler/importer.cc ${protobuf_source_dir}/src/google/protobuf/compiler/parser.cc + ${protobuf_source_dir}/src/google/protobuf/cpp_features.pb.cc ${protobuf_source_dir}/src/google/protobuf/descriptor.cc ${protobuf_source_dir}/src/google/protobuf/descriptor.pb.cc ${protobuf_source_dir}/src/google/protobuf/descriptor_database.cc ${protobuf_source_dir}/src/google/protobuf/dynamic_message.cc ${protobuf_source_dir}/src/google/protobuf/extension_set.cc ${protobuf_source_dir}/src/google/protobuf/extension_set_heavy.cc + ${protobuf_source_dir}/src/google/protobuf/feature_resolver.cc ${protobuf_source_dir}/src/google/protobuf/generated_enum_util.cc ${protobuf_source_dir}/src/google/protobuf/generated_message_bases.cc ${protobuf_source_dir}/src/google/protobuf/generated_message_reflection.cc @@ -175,6 +175,7 @@ set(libprotobuf_files ${protobuf_source_dir}/src/google/protobuf/message_lite.cc ${protobuf_source_dir}/src/google/protobuf/parse_context.cc ${protobuf_source_dir}/src/google/protobuf/port.cc + ${protobuf_source_dir}/src/google/protobuf/raw_ptr.cc ${protobuf_source_dir}/src/google/protobuf/reflection_mode.cc ${protobuf_source_dir}/src/google/protobuf/reflection_ops.cc ${protobuf_source_dir}/src/google/protobuf/repeated_field.cc @@ -210,6 +211,7 @@ add_library(protobuf::libprotobuf ALIAS _libprotobuf) set(libprotoc_files + ${protobuf_source_dir}/src/google/protobuf/compiler/allowlists/editions.cc ${protobuf_source_dir}/src/google/protobuf/compiler/code_generator.cc ${protobuf_source_dir}/src/google/protobuf/compiler/command_line_interface.cc ${protobuf_source_dir}/src/google/protobuf/compiler/cpp/enum.cc From a0840d36afbddad03c06c54fca17632917bd4d7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 16 Nov 2023 12:48:27 +0100 Subject: [PATCH 105/274] Apply the same to arrayReduce --- src/Functions/array/arrayReduce.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Functions/array/arrayReduce.cpp b/src/Functions/array/arrayReduce.cpp index ea087f4f9a87..46777ceb05cc 100644 --- a/src/Functions/array/arrayReduce.cpp +++ b/src/Functions/array/arrayReduce.cpp @@ -182,10 +182,19 @@ ColumnPtr FunctionArrayReduce::executeImpl(const ColumnsWithTypeAndName & argume that->addBatchArray(0, input_rows_count, places.data(), 0, aggregate_arguments, offsets->data(), arena.get()); } - for (size_t i = 0; i < input_rows_count; ++i) - /// We should use insertMergeResultInto to insert result into ColumnAggregateFunction - /// correctly if result contains AggregateFunction's states - agg_func.insertMergeResultInto(places[i], res_col, arena.get()); + if (agg_func.isState()) + { + for (size_t i = 0; i < input_rows_count; ++i) + /// We should use insertMergeResultInto to insert result into ColumnAggregateFunction + /// correctly if result contains AggregateFunction's states + agg_func.insertMergeResultInto(places[i], res_col, arena.get()); + } + else + { + for (size_t i = 0; i < input_rows_count; ++i) + agg_func.insertResultInto(places[i], res_col, arena.get()); + } + return result_holder; } From e8c14562ab512d5704f2ec05f8e053346c6737fd Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 16 Nov 2023 11:50:25 +0000 Subject: [PATCH 106/274] Bump absl to HEAD --- contrib/abseil-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/abseil-cpp b/contrib/abseil-cpp index 5655528c4183..3bd86026c93d 160000 --- a/contrib/abseil-cpp +++ b/contrib/abseil-cpp @@ -1 +1 @@ -Subproject commit 5655528c41830f733160de4fb0b99073841bae9e +Subproject commit 3bd86026c93da5a40006fd53403dff9d5f5e30e3 From 4d16c096a1c7464573c5101fd9068b9d451492dc Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 16 Nov 2023 12:09:13 +0000 Subject: [PATCH 107/274] Use ports from cluster --- tests/integration/helpers/cluster.py | 2 ++ tests/integration/test_storage_rabbitmq/test.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 729b30ba9340..cbc511628f0d 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -583,6 +583,7 @@ def __init__( self.rabbitmq_host = "rabbitmq1" self.rabbitmq_ip = None self.rabbitmq_port = 5672 + self.rabbitmq_secure_port = 5671 self.rabbitmq_dir = p.abspath(p.join(self.instances_dir, "rabbitmq")) self.rabbitmq_cookie_file = os.path.join(self.rabbitmq_dir, "erlang.cookie") self.rabbitmq_logs_dir = os.path.join(self.rabbitmq_dir, "logs") @@ -1316,6 +1317,7 @@ def setup_rabbitmq_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_rabbitmq = True env_variables["RABBITMQ_HOST"] = self.rabbitmq_host env_variables["RABBITMQ_PORT"] = str(self.rabbitmq_port) + env_variables["RABBITMQ_SECURE_PORT"] = str(self.rabbitmq_secure_port) env_variables["RABBITMQ_LOGS"] = self.rabbitmq_logs_dir env_variables["RABBITMQ_LOGS_FS"] = "bind" env_variables["RABBITMQ_COOKIE_FILE"] = self.rabbitmq_cookie_file diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index adb7f59769a1..021cdf54af94 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -110,9 +110,9 @@ def rabbitmq_setup_teardown(): ], ) def test_rabbitmq_select(rabbitmq_cluster, secure): - port = 5672 + port = cluster.rabbitmq_port if secure: - port = 5671 + port = cluster.rabbitmq_secure_port instance.query( """ From bdf038191ac85bb9f38524a96480bbf6704d3a24 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 16 Nov 2023 13:05:37 +0000 Subject: [PATCH 108/274] better test_keeper_broken_logs --- tests/integration/test_keeper_broken_logs/test.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_keeper_broken_logs/test.py b/tests/integration/test_keeper_broken_logs/test.py index e283d946174c..49b8d985ee84 100644 --- a/tests/integration/test_keeper_broken_logs/test.py +++ b/tests/integration/test_keeper_broken_logs/test.py @@ -1,13 +1,7 @@ import pytest from helpers.cluster import ClickHouseCluster import helpers.keeper_utils as keeper_utils -import random -import string -import os import time -from multiprocessing.dummy import Pool -from helpers.network import PartitionManager -from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( @@ -82,6 +76,13 @@ def verify_nodes(zk_conn): node1_conn.close() node1.stop_clickhouse() + + # wait until cluster stabilizes with a new leader + while not keeper_utils.is_leader( + started_cluster, node2 + ) and not keeper_utils.is_leader(started_cluster, node3): + time.sleep(1) + node1.exec_in_container( [ "truncate", From bb68321fc153be034fb0e2234b59bf6319cdd281 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 16 Nov 2023 09:02:11 +0000 Subject: [PATCH 109/274] More stable test_keeper_reconfig_replace_leader --- tests/integration/helpers/keeper_utils.py | 101 +++++++++++------- .../test.py | 7 ++ 2 files changed, 68 insertions(+), 40 deletions(-) diff --git a/tests/integration/helpers/keeper_utils.py b/tests/integration/helpers/keeper_utils.py index 83d0f2969b75..1ca17e923e45 100644 --- a/tests/integration/helpers/keeper_utils.py +++ b/tests/integration/helpers/keeper_utils.py @@ -37,39 +37,59 @@ class KeeperException(Exception): class KeeperClient(object): SEPARATOR = b"\a\a\a\a\n" - def __init__(self, bin_path: str, host: str, port: int): + def __init__(self, bin_path: str, host: str, port: int, connection_tries=30): self.bin_path = bin_path self.host = host self.port = port - self.proc = subprocess.Popen( - [ - bin_path, - "keeper-client", - "--host", - host, - "--port", - str(port), - "--log-level", - "error", - "--tests-mode", - "--no-confirmation", - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - self.poller = select.epoll() - self.poller.register(self.proc.stdout) - self.poller.register(self.proc.stderr) - - self._fd_nums = { - self.proc.stdout.fileno(): self.proc.stdout, - self.proc.stderr.fileno(): self.proc.stderr, - } - - self.stopped = False + retry_count = 0 + + while True: + try: + self.proc = subprocess.Popen( + [ + bin_path, + "keeper-client", + "--host", + host, + "--port", + str(port), + "--log-level", + "error", + "--tests-mode", + "--no-confirmation", + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + self.poller = select.epoll() + self.poller.register(self.proc.stdout) + self.poller.register(self.proc.stderr) + + self._fd_nums = { + self.proc.stdout.fileno(): self.proc.stdout, + self.proc.stderr.fileno(): self.proc.stderr, + } + + self.stopped = False + + self.get("/keeper", 60.0) + break + except Exception as e: + retry_count += 1 + if ( + "All connection tries failed while connecting to ZooKeeper" + in str(e) + and retry_count < connection_tries + ): + print( + f"Got exception while connecting to Keeper: {e}\nWill reconnect, reconnect count = {retry_count}" + ) + time.sleep(1) + else: + raise def execute_query(self, query: str, timeout: float = 60.0) -> str: output = io.BytesIO() @@ -94,7 +114,7 @@ def execute_query(self, query: str, timeout: float = 60.0) -> str: output.write(chunk) elif file == self.proc.stderr: - assert self.proc.stdout.readline() == self.SEPARATOR + self.proc.stdout.readline() raise KeeperException(self.proc.stderr.readline().strip().decode()) else: @@ -221,13 +241,12 @@ def send_4lw_cmd(cluster, node, cmd="ruok", port=9181): def wait_until_connected(cluster, node, port=9181, timeout=30.0): - elapsed = 0.0 + start = time.time() while send_4lw_cmd(cluster, node, "mntr", port) == NOT_SERVING_REQUESTS_ERROR_MSG: time.sleep(0.1) - elapsed += 0.1 - if elapsed >= timeout: + if time.time() - start > timeout: raise Exception( f"{timeout}s timeout while waiting for {node.name} to start serving requests" ) @@ -280,14 +299,16 @@ def wait_configs_equal(left_config: str, right_zk: KeeperClient, timeout: float Check whether get /keeper/config result in left_config is equal to get /keeper/config on right_zk ZK connection. """ - elapsed: float = 0.0 - while sorted(left_config.split("\n")) != sorted( - get_config_str(right_zk).split("\n") - ): + start = time.time() + left_config = sorted(left_config.split("\n")) + while True: + right_config = sorted(get_config_str(right_zk).split("\n")) + if left_config == right_config: + return + time.sleep(1) - elapsed += 1 - if elapsed >= timeout: + if time.time() - start > timeout: raise Exception( f"timeout while checking nodes configs to get equal. " - f"Left: {left_config}, right: {get_config_str(right_zk)}" + f"Left: {left_config}, right: {right_config}" ) diff --git a/tests/integration/test_keeper_reconfig_replace_leader/test.py b/tests/integration/test_keeper_reconfig_replace_leader/test.py index 4cdd48fcf7cb..8e621eef279c 100644 --- a/tests/integration/test_keeper_reconfig_replace_leader/test.py +++ b/tests/integration/test_keeper_reconfig_replace_leader/test.py @@ -3,6 +3,7 @@ import pytest from helpers.cluster import ClickHouseCluster, ClickHouseInstance from os.path import join, dirname, realpath +import time import helpers.keeper_utils as ku import typing as tp @@ -83,6 +84,12 @@ def test_reconfig_replace_leader(started_cluster): assert "node3" in config assert "node4" not in config + # wait until cluster stabilizes with a new leader + while not ku.is_leader(started_cluster, node2) and not ku.is_leader( + started_cluster, node3 + ): + time.sleep(1) + # additional 20s wait before removing leader ku.wait_configs_equal(config, zk2, timeout=50) From 85d363fb285b0b278e8cfc413c98a481d90c3476 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Thu, 16 Nov 2023 14:58:52 +0100 Subject: [PATCH 110/274] Update tests --- .../0_stateless/00727_concat.reference | 6 +++++ tests/queries/0_stateless/00727_concat.sql | 23 ++++++++++++++----- .../0_stateless/02233_interpolate_1.sql | 2 +- .../02389_analyzer_nested_lambda.reference | 2 +- .../02389_analyzer_nested_lambda.sql | 2 +- .../02521_analyzer_array_join_crash.reference | 2 +- .../02521_analyzer_array_join_crash.sql | 2 +- 7 files changed, 28 insertions(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/00727_concat.reference b/tests/queries/0_stateless/00727_concat.reference index 9b6a8b3857b6..7c48ba97c2b1 100644 --- a/tests/queries/0_stateless/00727_concat.reference +++ b/tests/queries/0_stateless/00727_concat.reference @@ -25,6 +25,7 @@ With bar With foo With bar With 42 +With 42 With fae310ca-d52a-4923-9e9b-02bf67f4b009 With 2023-11-14 With 2123-11-14 @@ -41,6 +42,11 @@ With (42,43) With [(0,0),(10,0),(10,10),(0,10)] With [[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]] With [[[(0,0),(10,0),(10,10),(0,10)]],[[(20,20),(50,20),(50,50),(20,50)],[(30,30),(50,50),(50,30)]]] +-- SimpleAggregateFunction +With 42 +With 4 +-- Nested +With [(\'foo\',\'qaz\'),(\'bar\',\'qux\')] -- NULL arguments \N \N diff --git a/tests/queries/0_stateless/00727_concat.sql b/tests/queries/0_stateless/00727_concat.sql index ba76ff538843..7d901514aea6 100644 --- a/tests/queries/0_stateless/00727_concat.sql +++ b/tests/queries/0_stateless/00727_concat.sql @@ -1,8 +1,6 @@ -- Tags: no-fasttest -- no-fasttest: json type needs rapidjson library, geo types need s2 geometry --- not tested here: (Simple)AggregateFunction, Nested - SET allow_experimental_object_type = 1; SET allow_suspicious_low_cardinality_types=1; @@ -33,11 +31,12 @@ SELECT concat('With ', materialize('bar' :: LowCardinality(FixedString(3)))); SELECT concat('With ', materialize('foo' :: LowCardinality(Nullable(String)))); SELECT concat('With ', materialize('bar' :: LowCardinality(Nullable(FixedString(3))))); SELECT concat('With ', materialize(42 :: LowCardinality(Nullable(UInt32)))); +SELECT concat('With ', materialize(42 :: LowCardinality(UInt32))); SELECT concat('With ', materialize('fae310ca-d52a-4923-9e9b-02bf67f4b009' :: UUID)); SELECT concat('With ', materialize('2023-11-14' :: Date)); SELECT concat('With ', materialize('2123-11-14' :: Date32)); -SELECT concat('With ', materialize('2023-11-14 05:50:12' :: DateTime)); -SELECT concat('With ', materialize('2023-11-14 05:50:12.123' :: DateTime64(3))); +SELECT concat('With ', materialize('2023-11-14 05:50:12' :: DateTime('Europe/Amsterdam'))); +SELECT concat('With ', materialize('2023-11-14 05:50:12.123' :: DateTime64(3, 'Europe/Amsterdam'))); SELECT concat('With ', materialize('hallo' :: Enum('hallo' = 1))); SELECT concat('With ', materialize(['foo', 'bar'] :: Array(String))); SELECT concat('With ', materialize('{"foo": "bar"}' :: JSON)); @@ -50,14 +49,23 @@ SELECT concat('With ', materialize([(0,0),(10,0),(10,10),(0,10)] :: Ring)); SELECT concat('With ', materialize([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]] :: Polygon)); SELECT concat('With ', materialize([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]] :: MultiPolygon)); +SELECT '-- SimpleAggregateFunction'; +CREATE OR REPLACE TABLE concat_saf_test(x SimpleAggregateFunction(max, Int32)) ENGINE=MergeTree ORDER BY tuple(); +INSERT INTO concat_saf_test VALUES (42); +INSERT INTO concat_saf_test SELECT max(number) FROM numbers(5); +SELECT concat('With ', x) FROM concat_saf_test ORDER BY x DESC; + +SELECT '-- Nested'; +CREATE OR REPLACE TABLE concat_nested_test(kv Nested(k String, v String)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO concat_nested_test VALUES (['foo', 'bar'], ['qaz', 'qux']); +SELECT concat('With ', kv) FROM concat_nested_test; + SELECT '-- NULL arguments'; SELECT concat(NULL, NULL); SELECT concat(NULL, materialize(NULL :: Nullable(UInt64))); SELECT concat(materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); - SELECT concat(42, materialize(NULL :: Nullable(UInt64))); SELECT concat('42', materialize(NULL :: Nullable(UInt64))); - SELECT concat(42, materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); SELECT concat('42', materialize(NULL :: Nullable(UInt64)), materialize(NULL :: Nullable(UInt64))); @@ -72,3 +80,6 @@ SELECT concat(42, 144); SELECT concat(42, 144, 255); SELECT CONCAT('Testing the ', 'alias'); + +SELECT concat(); -- { serverError 42 } +SELECT concat(1); -- { serverError 42 } diff --git a/tests/queries/0_stateless/02233_interpolate_1.sql b/tests/queries/0_stateless/02233_interpolate_1.sql index 3d416b27f452..d589a18421be 100644 --- a/tests/queries/0_stateless/02233_interpolate_1.sql +++ b/tests/queries/0_stateless/02233_interpolate_1.sql @@ -26,7 +26,7 @@ SELECT n, source, inter FROM ( # Test INTERPOLATE with incompatible expression - should produce error SELECT n, source, inter FROM ( SELECT toFloat32(number % 10) AS n, 'original' AS source, number as inter FROM numbers(10) WHERE number % 3 = 1 -) ORDER BY n WITH FILL FROM 0 TO 11.51 STEP 0.5 INTERPOLATE (inter AS inter||'inter'); -- { serverError 44 } +) ORDER BY n WITH FILL FROM 0 TO 11.51 STEP 0.5 INTERPOLATE (inter AS reverse(inter)); -- { serverError 44 } # Test INTERPOLATE with column from WITH FILL expression - should produce error SELECT n, source, inter FROM ( diff --git a/tests/queries/0_stateless/02389_analyzer_nested_lambda.reference b/tests/queries/0_stateless/02389_analyzer_nested_lambda.reference index 935c53358c05..68eb282a6a1b 100644 --- a/tests/queries/0_stateless/02389_analyzer_nested_lambda.reference +++ b/tests/queries/0_stateless/02389_analyzer_nested_lambda.reference @@ -117,5 +117,5 @@ SELECT arrayMap(x -> concat(concat(concat(concat(concat(toString(id), '___\0____ FROM test_table WHERE concat(concat(concat(toString(id), '___\0_______\0____'), toString(id)), concat(toString(id), NULL), toString(id)); SELECT '--'; -- -SELECT arrayMap(x -> concat(toString(id), arrayMap(x -> toString(1), [NULL])), [NULL]) FROM test_table; -- { serverError 44 }; +SELECT arrayMap(x -> splitByChar(toString(id), arrayMap(x -> toString(1), [NULL])), [NULL]) FROM test_table; -- { serverError 44 }; DROP TABLE test_table; diff --git a/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql b/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql index 8f8b5537da90..48e84246d1c9 100644 --- a/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql +++ b/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql @@ -122,7 +122,7 @@ FROM test_table WHERE concat(concat(concat(toString(id), '___\0_______\0____'), SELECT '--'; -SELECT arrayMap(x -> concat(toString(id), arrayMap(x -> toString(1), [NULL])), [NULL]) FROM test_table; -- { serverError 44 }; +SELECT arrayMap(x -> splitByChar(toString(id), arrayMap(x -> toString(1), [NULL])), [NULL]) FROM test_table; -- { serverError 44 }; DROP TABLE test_table; diff --git a/tests/queries/0_stateless/02521_analyzer_array_join_crash.reference b/tests/queries/0_stateless/02521_analyzer_array_join_crash.reference index 59da8ccad1a7..5e7728e05908 100644 --- a/tests/queries/0_stateless/02521_analyzer_array_join_crash.reference +++ b/tests/queries/0_stateless/02521_analyzer_array_join_crash.reference @@ -8,4 +8,4 @@ SELECT id, value_element, value FROM test_table ARRAY JOIN [[1,2,3]] AS value_el 0 [1,2,3] 3 SELECT value_element, value FROM test_table ARRAY JOIN [1048577] AS value_element, arrayMap(x -> value_element, ['']) AS value; 1048577 [1048577] -SELECT arrayFilter(x -> notEmpty(concat(x)), [NULL, NULL]) FROM system.one ARRAY JOIN [1048577] AS elem, arrayMap(x -> concat(x, elem, ''), ['']) AS unused; -- { serverError 44 } +SELECT arrayFilter(x -> notEmpty(concat(x)), [NULL, NULL]) FROM system.one ARRAY JOIN [1048577] AS elem, arrayMap(x -> splitByChar(x, elem), ['']) AS unused; -- { serverError 44 } diff --git a/tests/queries/0_stateless/02521_analyzer_array_join_crash.sql b/tests/queries/0_stateless/02521_analyzer_array_join_crash.sql index c7641a3bee04..53606e01ab7b 100644 --- a/tests/queries/0_stateless/02521_analyzer_array_join_crash.sql +++ b/tests/queries/0_stateless/02521_analyzer_array_join_crash.sql @@ -17,7 +17,7 @@ SELECT id, value_element, value FROM test_table ARRAY JOIN [[1,2,3]] AS value_el SELECT value_element, value FROM test_table ARRAY JOIN [1048577] AS value_element, arrayMap(x -> value_element, ['']) AS value; -SELECT arrayFilter(x -> notEmpty(concat(x)), [NULL, NULL]) FROM system.one ARRAY JOIN [1048577] AS elem, arrayMap(x -> concat(x, elem, ''), ['']) AS unused; -- { serverError 44 } +SELECT arrayFilter(x -> notEmpty(concat(x)), [NULL, NULL]) FROM system.one ARRAY JOIN [1048577] AS elem, arrayMap(x -> splitByChar(x, elem), ['']) AS unused; -- { serverError 44 } -- { echoOff } From ddca2c2187d42b39fb139460cd32eb7e71adbce1 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 16 Nov 2023 14:29:53 +0100 Subject: [PATCH 111/274] server side waiting --- src/Storages/MergeTree/DataPartsExchange.cpp | 113 ++++++++---------- src/Storages/MergeTree/MergeTreeSettings.h | 4 +- tests/clickhouse-test | 2 +- ...916_replication_protocol_wait_for_part.sql | 7 +- 4 files changed, 57 insertions(+), 69 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 7fd6f59ed697..c39263a0b73c 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -65,7 +65,6 @@ constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_UUID = 5; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_ZERO_COPY = 6; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION = 7; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION = 8; -constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE = 9; std::string getEndpointId(const std::string & node_id) { @@ -121,7 +120,7 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write MergeTreePartInfo::fromPartName(part_name, data.format_version); /// We pretend to work as older server version, to be sure that client will correctly process our version - response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE))}); + response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION))}); LOG_TRACE(log, "Sending part {}", part_name); @@ -139,29 +138,6 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write { part = findPart(part_name); - /// Ephemeral zero-copy lock may be lost for PreActive parts - /// do not expose PreActive parts - if (client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE) - { - bool part_is_ready = part->getState() != MergeTreeDataPartState::PreActive; - writeBinary(part_is_ready, out); - - if (!part_is_ready) - { - LOG_TRACE(log, "Part {} is in PreActive state, reply to the client that part is not ready yet", part_name); - return; - } - } - else - { - bool zero_copy_enabled = data.getSettings()->allow_remote_fs_zero_copy_replication; - if (part->getState() == MergeTreeDataPartState::PreActive && zero_copy_enabled) - { - /// report error, client will try again later, error message would be printed - throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in table", part_name); - } - } - CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedSend}; if (part->getDataPartStorage().isStoredOnRemoteDisk()) @@ -373,6 +349,25 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( return data_checksums; } +bool wait_loop(UInt32 wait_timeout_ms, std::function pred) +{ + static const UInt32 loop_delay_ms = 5; + + /// this is sleep-based wait, it has to be short + chassert(wait_timeout_ms < 2000); + + if (pred()) + return true; + + Stopwatch timer; + while (!pred() && timer.elapsedMilliseconds() < wait_timeout_ms) + { + sleepForMilliseconds(loop_delay_ms); + } + + return pred(); +} + MergeTreeData::DataPartPtr Service::findPart(const String & name) { /// It is important to include Outdated parts here because remote replicas cannot reliably @@ -381,10 +376,26 @@ MergeTreeData::DataPartPtr Service::findPart(const String & name) part = data.getPartIfExists(name, {MergeTreeDataPartState::PreActive, MergeTreeDataPartState::Active, MergeTreeDataPartState::Outdated}); - if (part) + if (!part) + throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in table", name); + + bool zero_copy_enabled = data.getSettings()->allow_remote_fs_zero_copy_replication; + if (!zero_copy_enabled) return part; - throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in table", name); + /// Ephemeral zero-copy lock may be lost for PreActive parts + /// do not expose PreActive parts for zero-copy + + static const UInt32 wait_timeout_ms = 1000; + bool pred_result = wait_loop(wait_timeout_ms, [&] () { return part->getState() != MergeTreeDataPartState::PreActive; }); + + if (!pred_result) + throw Exception( + ErrorCodes::ABORTED, + "Part {} is in PreActive state for {} ms. Another host has to be asked.", + name, wait_timeout_ms); + + return part; } Fetcher::Fetcher(StorageReplicatedMergeTree & data_) @@ -442,7 +453,7 @@ std::pair Fetcher::fetchSelected { {"endpoint", endpoint_id}, {"part", part_name}, - {"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE)}, + {"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_METADATA_VERSION)}, {"compress", "false"} }); @@ -500,43 +511,17 @@ std::pair Fetcher::fetchSelected creds.setPassword(password); } - std::unique_ptr in; - int server_protocol_version = 0; - bool part_is_ready = true; - - static const UInt32 part_not_ready_attempts = 5; - static const UInt32 wait_sleep_time_ms = 100; - - for (UInt32 attempt = 1; attempt <= part_not_ready_attempts; ++attempt) - { - in = std::make_unique( - uri, - Poco::Net::HTTPRequest::HTTP_POST, - nullptr, - timeouts, - creds, - DBMS_DEFAULT_BUFFER_SIZE, - 0, /* no redirects */ - static_cast(data_settings->replicated_max_parallel_fetches_for_host)); - - server_protocol_version = parse(in->getResponseCookie("server_protocol_version", "0")); - - if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_WAITING_PREACTIVE) - readBinary(part_is_ready, *in); - - if (part_is_ready) - break; - - sleepForMilliseconds(wait_sleep_time_ms); - - if (blocker.isCancelled()) - throw Exception(ErrorCodes::ABORTED, "Fetching of part was cancelled"); - } - - if (!part_is_ready) - throw Exception(ErrorCodes::ABORTED, "Part {} is still not ready in host {} after {} attempts, try another host", - part_name, host, part_not_ready_attempts); + std::unique_ptr in = std::make_unique( + uri, + Poco::Net::HTTPRequest::HTTP_POST, + nullptr, + timeouts, + creds, + DBMS_DEFAULT_BUFFER_SIZE, + 0, /* no redirects */ + static_cast(data_settings->replicated_max_parallel_fetches_for_host)); + int server_protocol_version = parse(in->getResponseCookie("server_protocol_version", "0")); String remote_fs_metadata = parse(in->getResponseCookie("remote_fs_metadata", "")); DiskPtr preffered_disk = disk; diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 15c54ee37911..41476bab5b19 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -83,7 +83,7 @@ struct Settings; M(UInt64, max_delay_to_insert, 1, "Max delay of inserting data into MergeTree table in seconds, if there are a lot of unmerged parts in single partition.", 0) \ M(UInt64, min_delay_to_insert_ms, 10, "Min delay of inserting data into MergeTree table in milliseconds, if there are a lot of unmerged parts in single partition.", 0) \ M(UInt64, max_parts_in_total, 100000, "If more than this number active parts in all partitions in total, throw 'Too many parts ...' exception.", 0) \ - M(Bool, async_insert, false, "If true, data from INSERT query is stored in queue and later flushed to table in background.", 0) \ + M(Bool, async_insert, false, "If true, data from INSERT query is stored in queue and later flushed to table in background.", 0) \ M(Milliseconds, sleep_before_commit_local_part_in_replicated_table_ms, 0, "For testing. Do not change it.", 0) \ \ /* Part removal settings. */ \ @@ -122,7 +122,7 @@ struct Settings; M(UInt64, max_replicated_sends_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for replicated sends. Zero means unlimited.", 0) \ M(Milliseconds, wait_for_unique_parts_send_before_shutdown_ms, 0, "Before shutdown table will wait for required amount time for unique parts (exist only on current replica) to be fetched by other replicas (0 means disabled).", 0) \ M(Float, fault_probability_before_part_commit, 0, "For testing. Do not change it.", 0) \ - M(Float, fault_probability_after_part_commit, 0, "For testing. Do not change it.", 0) \ + M(Float, fault_probability_after_part_commit, 0, "For testing. Do not change it.", 0) \ \ /** Check delay of replicas settings. */ \ M(UInt64, min_relative_delay_to_measure, 120, "Calculate relative replica delay only if absolute delay is not less that this value.", 0) \ diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 048f848ff272..053bd040bce2 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -674,7 +674,7 @@ class MergeTreeSettingsRandomizer: "replace_long_file_name_to_hash": lambda: random.randint(0, 1), "max_file_name_length": threshold_generator(0.3, 0.3, 0, 128), "sleep_before_commit_local_part_in_replicated_table_ms": threshold_generator( - 0.3, 0.3, 0, 250 + 0.7, 0.7, 0, 100 ), } diff --git a/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql index ed9cfd00b450..97ef33f96e8a 100644 --- a/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql +++ b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql @@ -7,7 +7,7 @@ create table tableIn (n int) settings storage_policy='s3_cache', allow_remote_fs_zero_copy_replication=1, - sleep_before_commit_local_part_in_replicated_table_ms=50000; + sleep_before_commit_local_part_in_replicated_table_ms=5000; create table tableOut (n int) engine=ReplicatedMergeTree('/test/02916/{database}/table', '2') order by tuple() @@ -15,9 +15,12 @@ create table tableOut (n int) storage_policy='s3_cache', allow_remote_fs_zero_copy_replication=1; -SET send_logs_level = 'error'; +SET send_logs_level='error'; insert into tableIn values(1); insert into tableIn values(2); system sync replica tableOut; select count() from tableOut; + +drop table tableIn +drop table tableOut From b13cd9792b599d3686a73f9114fadda7d0e090f0 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 16 Nov 2023 14:46:57 +0000 Subject: [PATCH 112/274] Fix cross build --- contrib/google-protobuf-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/google-protobuf-cmake/CMakeLists.txt b/contrib/google-protobuf-cmake/CMakeLists.txt index 1ed4133270b3..89bdbb89eca1 100644 --- a/contrib/google-protobuf-cmake/CMakeLists.txt +++ b/contrib/google-protobuf-cmake/CMakeLists.txt @@ -369,7 +369,7 @@ else () "-Dprotobuf_BUILD_PROTOC_BINARIES=1" "-DABSL_ROOT_DIR=${abseil_source_dir}" "-DABSL_ENABLE_INSTALL=0" - "${protobuf_source_dir}/cmake" + "${protobuf_source_dir}" WORKING_DIRECTORY "${PROTOC_BUILD_DIR}" COMMAND_ECHO STDOUT) From cc64397e9282091d073386a5fd912689257b1445 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 16 Nov 2023 19:21:58 +0300 Subject: [PATCH 113/274] Planner support transactions --- src/Planner/Planner.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 891663162619..12e8d7953477 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -116,7 +116,7 @@ namespace void checkStoragesSupportTransactions(const PlannerContextPtr & planner_context) { const auto & query_context = planner_context->getQueryContext(); - if (query_context->getSettingsRef().throw_on_unsupported_query_inside_transaction) + if (!query_context->getSettingsRef().throw_on_unsupported_query_inside_transaction) return; if (!query_context->getCurrentTransaction()) @@ -130,13 +130,11 @@ void checkStoragesSupportTransactions(const PlannerContextPtr & planner_context) else if (auto * table_function_node = table_expression->as()) storage = table_function_node->getStorage(); - if (storage->supportsTransactions()) - continue; - - throw Exception(ErrorCodes::NOT_IMPLEMENTED, - "Storage {} (table {}) does not support transactions", - storage->getName(), - storage->getStorageID().getNameForLogs()); + if (storage && !storage->supportsTransactions()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "Storage {} (table {}) does not support transactions", + storage->getName(), + storage->getStorageID().getNameForLogs()); } } @@ -1333,9 +1331,9 @@ void Planner::buildPlanForQueryNode() query_node.getHaving() = {}; } - checkStoragesSupportTransactions(planner_context); collectSets(query_tree, *planner_context); collectTableExpressionData(query_tree, planner_context); + checkStoragesSupportTransactions(planner_context); if (!select_query_options.only_analyze) collectFiltersForAnalysis(query_tree, planner_context); From 00569baf4b9703891918be979f192d425dc81a2e Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 16 Nov 2023 19:23:55 +0300 Subject: [PATCH 114/274] Updated analyzer failed tests --- tests/analyzer_tech_debt.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/analyzer_tech_debt.txt b/tests/analyzer_tech_debt.txt index e155ee72ebb0..d969b9e6fadd 100644 --- a/tests/analyzer_tech_debt.txt +++ b/tests/analyzer_tech_debt.txt @@ -6,7 +6,6 @@ 01064_incremental_streaming_from_2_src_with_feedback 01083_expressions_in_engine_arguments 01155_rename_move_materialized_view -01173_transaction_control_queries 01214_test_storage_merge_aliases_with_where 01244_optimize_distributed_group_by_sharding_key 01268_mv_scalars @@ -30,7 +29,6 @@ 02139_MV_with_scalar_subquery 02174_cte_scalar_cache_mv 02302_s3_file_pruning -02345_implicit_transaction 02352_grouby_shadows_arg 02354_annoy 02428_parameterized_view From 49c58e76099b6a3771182c9bd093159054d707a0 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 16 Nov 2023 16:24:09 +0000 Subject: [PATCH 115/274] Disable RabbitMQ secure connection test in intergatiion test with TSAN --- tests/integration/test_storage_rabbitmq/test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 021cdf54af94..4f4d4dec02f4 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -110,6 +110,9 @@ def rabbitmq_setup_teardown(): ], ) def test_rabbitmq_select(rabbitmq_cluster, secure): + if secure and instance.is_built_with_memory_sanitizer(): + pytest.skip("Data races: see https://github.com/ClickHouse/ClickHouse/issues/56866") + port = cluster.rabbitmq_port if secure: port = cluster.rabbitmq_secure_port From d03a1aab7bea331738d915d23b7353ccbbac9cf4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 16 Nov 2023 16:39:57 +0000 Subject: [PATCH 116/274] Automatic style fix --- tests/integration/test_storage_rabbitmq/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 4f4d4dec02f4..cb34f7203d63 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -111,7 +111,9 @@ def rabbitmq_setup_teardown(): ) def test_rabbitmq_select(rabbitmq_cluster, secure): if secure and instance.is_built_with_memory_sanitizer(): - pytest.skip("Data races: see https://github.com/ClickHouse/ClickHouse/issues/56866") + pytest.skip( + "Data races: see https://github.com/ClickHouse/ClickHouse/issues/56866" + ) port = cluster.rabbitmq_port if secure: From 472cfdc86d73b0a8135f2f135c4fa6118fcfa287 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 16 Nov 2023 17:47:51 +0100 Subject: [PATCH 117/274] Review fix --- src/Core/Settings.h | 2 +- src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp | 5 +++-- src/IO/ReadSettings.h | 2 +- src/Interpreters/Context.cpp | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index ee503322e2a6..3a65496adc23 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -723,7 +723,7 @@ class IColumn; M(Bool, skip_download_if_exceeds_query_cache, true, "Skip download from remote filesystem if exceeds query cache size", 0) \ M(UInt64, filesystem_cache_max_download_size, (128UL * 1024 * 1024 * 1024), "Max remote filesystem cache size that can be downloaded by a single query", 0) \ M(Bool, throw_on_error_from_cache_on_write_operations, false, "Ignore error from cache when caching on write operations (INSERT, merges)", 0) \ - M(UInt64, filesystem_cache_getorset_batch_size, 20, "A batch size for holding file segments for a single read range", 0) \ + M(UInt64, filesystem_cache_segments_batch_size, 20, "Limit on size of a single batch of file segments that a read buffer can request from cache. Too low value will lead to excessive requests to cache, too large may slow down eviction from cache", 0) \ \ M(Bool, load_marks_asynchronously, false, "Load MergeTree marks asynchronously", 0) \ M(Bool, enable_filesystem_read_prefetches_log, false, "Log to system.filesystem prefetch_log during query. Should be used only for testing or debugging, not recommended to be turned on by default", 0) \ diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 27d0b6706a66..06ee9eb4e850 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -116,18 +116,19 @@ void CachedOnDiskReadBufferFromFile::appendFilesystemCacheLog( bool CachedOnDiskReadBufferFromFile::nextFileSegmentsBatch() { + chassert(!file_segments || file_segments->empty()); size_t size = getRemainingSizeToRead(); if (!size) return false; if (settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache) { - file_segments = cache->get(cache_key, file_offset_of_buffer_end, size, settings.filesystem_cache_getorset_batch_size); + file_segments = cache->get(cache_key, file_offset_of_buffer_end, size, settings.filesystem_cache_segments_batch_size); } else { CreateFileSegmentSettings create_settings(FileSegmentKind::Regular); - file_segments = cache->getOrSet(cache_key, file_offset_of_buffer_end, size, file_size.value(), create_settings, settings.filesystem_cache_getorset_batch_size); + file_segments = cache->getOrSet(cache_key, file_offset_of_buffer_end, size, file_size.value(), create_settings, settings.filesystem_cache_segments_batch_size); } return !file_segments->empty(); } diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index 197ae563d258..4c8a6cb020ac 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -100,7 +100,7 @@ struct ReadSettings bool enable_filesystem_cache_log = false; /// Don't populate cache when the read is not part of query execution (e.g. background thread). bool avoid_readthrough_cache_outside_query_context = true; - size_t filesystem_cache_getorset_batch_size = 100; + size_t filesystem_cache_segments_batch_size = 20; size_t filesystem_cache_max_download_size = (128UL * 1024 * 1024 * 1024); bool skip_download_if_exceeds_query_cache = true; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index c2b1db2fe180..ec56412b74be 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -4761,7 +4761,7 @@ ReadSettings Context::getReadSettings() const res.enable_filesystem_cache = settings.enable_filesystem_cache; res.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache; res.enable_filesystem_cache_log = settings.enable_filesystem_cache_log; - res.filesystem_cache_getorset_batch_size = settings.filesystem_cache_getorset_batch_size; + res.filesystem_cache_segments_batch_size = settings.filesystem_cache_segments_batch_size; res.filesystem_cache_max_download_size = settings.filesystem_cache_max_download_size; res.skip_download_if_exceeds_query_cache = settings.skip_download_if_exceeds_query_cache; From 2ec96f9e9e737b2805f0408c3639c19760b445af Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 16 Nov 2023 17:56:06 +0100 Subject: [PATCH 118/274] Update 01052_window_view_proc_tumble_to_now.sh --- .../queries/0_stateless/01052_window_view_proc_tumble_to_now.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh b/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh index e75b7d9570bc..4325ebeed247 100755 --- a/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh +++ b/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh @@ -16,7 +16,7 @@ DROP TABLE IF EXISTS wv; CREATE TABLE dst(count UInt64) Engine=MergeTree ORDER BY tuple(); CREATE TABLE mt(a Int32) ENGINE=MergeTree ORDER BY tuple(); -CREATE WINDOW VIEW wv TO dst AS SELECT count(a) AS count FROM mt GROUP BY tumble(now('US/Samoa'), INTERVAL '5' SECOND, 'US/Samoa') AS wid; +CREATE WINDOW VIEW wv TO dst AS SELECT count(a) AS count FROM mt GROUP BY tumble(now('US/Samoa'), INTERVAL '10' SECOND, 'US/Samoa') AS wid; INSERT INTO mt VALUES (1); EOF From 409f781c0907f4862ddf57d80c6e1adc1ad8b77e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 16 Nov 2023 18:10:26 +0100 Subject: [PATCH 119/274] Fix test --- tests/queries/0_stateless/01119_session_log.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01119_session_log.sql b/tests/queries/0_stateless/01119_session_log.sql index 8f6967b89ec6..55f6228797a4 100644 --- a/tests/queries/0_stateless/01119_session_log.sql +++ b/tests/queries/0_stateless/01119_session_log.sql @@ -4,7 +4,7 @@ select * from remote('127.0.0.2', system, one, 'default', ''); select * from remote('127.0.0.2', system, one, 'default', 'wrong password'); -- { serverError AUTHENTICATION_FAILED } select * from remote('127.0.0.2', system, one, 'nonexistsnt_user_1119', ''); -- { serverError AUTHENTICATION_FAILED } set receive_timeout=1; -select * from remote('127.0.0.2', system, one, ' INTERSERVER SECRET ', ''); -- { serverError AUTHENTICATION_FAILED } +select * from remote('127.0.0.2', system, one, ' INTERSERVER SECRET ', ''); -- { serverError NO_REMOTE_SHARD_AVAILABLE } set receive_timeout=300; select * from remote('127.0.0.2', system, one, ' ', ''); -- { serverError AUTHENTICATION_FAILED } From d24757bbbf6b4050429d5e2d4f1e7ecfd9fed93d Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 16 Nov 2023 18:11:40 +0100 Subject: [PATCH 120/274] Add assertions --- src/Interpreters/Cache/FileCache.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 5de24977db5f..0591038fc1d7 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -801,6 +801,11 @@ void FileCache::removePathIfExists(const String & path) void FileCache::removeAllReleasable() { assertInitialized(); + +#ifdef ABORT_ON_LOGICAL_ERROR + assertCacheCorrectness(); +#endif + metadata.removeAllKeys(/* if_releasable */true); if (stash) From fdf5cfdec09d8816c1fe067fc548c6b43ba37497 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 16 Nov 2023 18:54:09 +0100 Subject: [PATCH 121/274] Update FileCacheSettings.cpp --- src/Interpreters/Cache/FileCacheSettings.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/Cache/FileCacheSettings.cpp b/src/Interpreters/Cache/FileCacheSettings.cpp index ad414530c2a4..e333d9a3cd85 100644 --- a/src/Interpreters/Cache/FileCacheSettings.cpp +++ b/src/Interpreters/Cache/FileCacheSettings.cpp @@ -58,7 +58,7 @@ void FileCacheSettings::loadImpl(FuncHas has, FuncGetUInt get_uint, FuncGetStrin if (has("load_metadata_threads")) load_metadata_threads = get_uint("load_metadata_threads"); - + if (boundary_alignment > max_file_segment_size) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Setting `boundary_alignment` cannot exceed `max_file_segment_size`"); } From d83cf03c30f5cde7d7442c6f01da04f8fb7c2a3a Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 16 Nov 2023 19:21:27 +0100 Subject: [PATCH 122/274] no randomization sleep_before_commit_local_part_in_replicated_table_ms --- tests/clickhouse-test | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 053bd040bce2..cab7d7e79ff9 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -673,9 +673,6 @@ class MergeTreeSettingsRandomizer: "primary_key_compress_block_size": lambda: random.randint(8000, 100000), "replace_long_file_name_to_hash": lambda: random.randint(0, 1), "max_file_name_length": threshold_generator(0.3, 0.3, 0, 128), - "sleep_before_commit_local_part_in_replicated_table_ms": threshold_generator( - 0.7, 0.7, 0, 100 - ), } @staticmethod From 2a1467b8da19bfb91edcb1c4f986697f7c2f13e3 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 16 Nov 2023 19:33:17 +0100 Subject: [PATCH 123/274] Update test_storage_s3_queue/test.py --- tests/integration/test_storage_s3_queue/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 9f41cfd176d9..ec27b7326344 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -734,7 +734,7 @@ def test_multiple_tables_streaming_sync_distributed(started_cluster, mode): create_mv(instance, table_name, dst_table_name) total_values = generate_random_files( - started_cluster, files_path, files_to_generate, row_num=1 + started_cluster, files_path, files_to_generate, row_num=50 ) def get_count(node, table_name): From e533abef75f9eb304ceea09a74afb37672728420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 16 Nov 2023 20:54:52 +0100 Subject: [PATCH 124/274] Resolve 01572_kill_window_function flakiness --- .../0_stateless/01572_kill_window_function.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/01572_kill_window_function.sh b/tests/queries/0_stateless/01572_kill_window_function.sh index 7103b7f72106..de6de3510a04 100755 --- a/tests/queries/0_stateless/01572_kill_window_function.sh +++ b/tests/queries/0_stateless/01572_kill_window_function.sh @@ -6,21 +6,20 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -o pipefail +function wait_for_query_to_start() +{ + while [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT count() FROM system.processes WHERE query_id = '$1'") == 0 ]]; do sleep 0.1; done +} + # Run a test query that takes very long to run. query_id="01572_kill_window_function-$CLICKHOUSE_DATABASE" -$CLICKHOUSE_CLIENT --query_id="$query_id" --query "SELECT count(1048575) OVER (PARTITION BY intDiv(NULL, number) ORDER BY number DESC NULLS FIRST ROWS BETWEEN CURRENT ROW AND 1048575 FOLLOWING) FROM numbers(255, 1048575)" >/dev/null 2>&1 & +$CLICKHOUSE_CLIENT --query_id="$query_id" --query "SELECT sum(number) OVER (PARTITION BY number % 10 ORDER BY number DESC NULLS FIRST ROWS BETWEEN CURRENT ROW AND 99999 FOLLOWING) FROM numbers(0, 10000000) format Null;" >/dev/null 2>&1 & client_pid=$! echo Started -# Use one query to both kill the test query and verify that it has started, -# because if we try to kill it before it starts, the test will fail. -while [ -z "$($CLICKHOUSE_CLIENT --query "kill query where query_id = '$query_id' and current_database = currentDatabase()")" ] -do - # If we don't yet see the query in the process list, the client should still - # be running. The query is very long. - kill -0 -- $client_pid - sleep 1 -done +wait_for_query_to_start $query_id + +$CLICKHOUSE_CLIENT --query "kill query where query_id = '$query_id' and current_database = currentDatabase() format Null" echo Sent kill request # Wait for the client to terminate. From 1abcb28624ee4d204e8e1723ad58e87d1d80be56 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 16 Nov 2023 23:32:17 +0100 Subject: [PATCH 125/274] Remove ctest --- CMakeLists.txt | 1 - cmake/add_check.cmake | 19 ------------------- src/CMakeLists.txt | 4 ---- tests/CMakeLists.txt | 26 -------------------------- 4 files changed, 50 deletions(-) delete mode 100644 cmake/add_check.cmake delete mode 100644 tests/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 9f05b2b78cee..4fe7a1e05e74 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -561,7 +561,6 @@ option(CHECK_LARGE_OBJECT_SIZES "Check that there are no large object files afte add_subdirectory (base) add_subdirectory (src) add_subdirectory (programs) -add_subdirectory (tests) add_subdirectory (utils) if (FUZZER) diff --git a/cmake/add_check.cmake b/cmake/add_check.cmake deleted file mode 100644 index ba30ee8676fa..000000000000 --- a/cmake/add_check.cmake +++ /dev/null @@ -1,19 +0,0 @@ -# Adding test output on failure -enable_testing () - -if (NOT TARGET check) - if (CMAKE_CONFIGURATION_TYPES) - add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND} - --force-new-ctest-process --output-on-failure --build-config "$" - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) - else () - add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND} - --force-new-ctest-process --output-on-failure - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) - endif () -endif () - -macro (add_check target) - add_test (NAME test_${target} COMMAND ${target} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - add_dependencies (check ${target}) -endmacro (add_check) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5254743e154c..3733295e9b48 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -600,8 +600,6 @@ if (TARGET ch_rust::skim) dbms_target_link_libraries(PUBLIC ch_rust::skim) endif() -include ("${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake") - if (ENABLE_TESTS) macro (grep_gtest_sources BASE_DIR DST_VAR) # Cold match files that are not in tests/ directories @@ -645,6 +643,4 @@ if (ENABLE_TESTS) if (TARGET ch_contrib::parquet) target_link_libraries(unit_tests_dbms PRIVATE ch_contrib::parquet) endif() - - add_check(unit_tests_dbms) endif () diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt deleted file mode 100644 index 22c89aaafa7d..000000000000 --- a/tests/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -enable_testing() - -# Run tests with "ninja check" or "make check" -if (TARGET check) - message (STATUS "Target check already exists") -else () - include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake) -endif () - -option (ENABLE_CLICKHOUSE_TEST "Install clickhouse-test script and relevant tests scenarios" OFF) - -if (ENABLE_CLICKHOUSE_TEST) - install (PROGRAMS clickhouse-test DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - install ( - DIRECTORY queries performance config - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse-test - USE_SOURCE_PERMISSIONS - COMPONENT clickhouse - PATTERN "CMakeLists.txt" EXCLUDE - PATTERN ".gitignore" EXCLUDE - ) -endif () - -if (ENABLE_TEST_INTEGRATION) - add_subdirectory (integration) -endif () From c72136b123685f6e4704b9d39122f637ee95ce0c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 08:07:32 +0100 Subject: [PATCH 126/274] Simpler CMake --- contrib/arrow-cmake/CMakeLists.txt | 16 ++++++++-------- contrib/azure-cmake/CMakeLists.txt | 3 +-- contrib/thrift-cmake/CMakeLists.txt | 6 ------ 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 71133451889f..935fc8863309 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -77,16 +77,16 @@ set(FLATBUFFERS_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/flatbuffers") set(FLATBUFFERS_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/flatbuffers") set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_SRC_DIR}/include") -# set flatbuffers CMake options -set(FLATBUFFERS_BUILD_FLATLIB ON CACHE BOOL "Enable the build of the flatbuffers library") -set(FLATBUFFERS_BUILD_SHAREDLIB OFF CACHE BOOL "Disable the build of the flatbuffers shared library") -set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "Skip flatbuffers tests") +set(FLATBUFFERS_SRCS + ${FLATBUFFERS_SRC_DIR}/src/idl_parser.cpp + ${FLATBUFFERS_SRC_DIR}/src/idl_gen_text.cpp + ${FLATBUFFERS_SRC_DIR}/src/reflection.cpp + ${FLATBUFFERS_SRC_DIR}/src/util.cpp) -add_subdirectory(${FLATBUFFERS_SRC_DIR} "${FLATBUFFERS_BINARY_DIR}") +add_library(_flatbuffers STATIC ${FLATBUFFERS_SRCS}) +target_include_directories(_flatbuffers PUBLIC ${FLATBUFFERS_INCLUDE_DIR}) +target_compile_definitions(_flatbuffers PRIVATE -DFLATBUFFERS_LOCALE_INDEPENDENT=0) -add_library(_flatbuffers INTERFACE) -target_link_libraries(_flatbuffers INTERFACE flatbuffers) -target_include_directories(_flatbuffers INTERFACE ${FLATBUFFERS_INCLUDE_DIR}) # === hdfs # NOTE: cannot use ch_contrib::hdfs since it's INCLUDE_DIRECTORIES does not includes trailing "hdfs/" diff --git a/contrib/azure-cmake/CMakeLists.txt b/contrib/azure-cmake/CMakeLists.txt index 7aba81259d3a..bb44c993e79a 100644 --- a/contrib/azure-cmake/CMakeLists.txt +++ b/contrib/azure-cmake/CMakeLists.txt @@ -48,9 +48,8 @@ set(AZURE_SDK_INCLUDES "${AZURE_SDK_LIBRARY_DIR}/storage/azure-storage-blobs/inc/" ) -include("${AZURE_DIR}/cmake-modules/AzureTransportAdapters.cmake") - add_library(_azure_sdk ${AZURE_SDK_UNIFIED_SRC}) +target_compile_definitions(_azure_sdk PRIVATE BUILD_CURL_HTTP_TRANSPORT_ADAPTER) # Originally, on Windows azure-core is built with bcrypt and crypt32 by default if (TARGET OpenSSL::SSL) diff --git a/contrib/thrift-cmake/CMakeLists.txt b/contrib/thrift-cmake/CMakeLists.txt index d6aa6b9e5f22..89a444cfb83c 100644 --- a/contrib/thrift-cmake/CMakeLists.txt +++ b/contrib/thrift-cmake/CMakeLists.txt @@ -47,8 +47,6 @@ set(thriftcpp_threads_SOURCES "${LIBRARY_DIR}/src/thrift/concurrency/Mutex.cpp" ) -include("${ClickHouse_SOURCE_DIR}/contrib/thrift/build/cmake/ConfigureChecks.cmake") # makes config.h - set (HAVE_ARPA_INET_H 1) set (HAVE_FCNTL_H 1) set (HAVE_GETOPT_H 1) @@ -81,10 +79,6 @@ if (OS_LINUX AND NOT USE_MUSL) set (STRERROR_R_CHAR_P 1) endif () -#set(PACKAGE ${PACKAGE_NAME}) -#set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") -#set(VERSION ${thrift_VERSION}) - # generate a config.h file configure_file("${CMAKE_CURRENT_SOURCE_DIR}/build/cmake/config.h.in" "${CMAKE_CURRENT_BINARY_DIR}/thrift/config.h") From 5496e2d6ac483beeccca03a2ed92b586e33f28dd Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Fri, 17 Nov 2023 10:24:47 +0300 Subject: [PATCH 127/274] test for #56790 --- .../00059_shard_global_in_mergetree.reference | 8 +++++++ .../00059_shard_global_in_mergetree.sql | 24 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/queries/0_stateless/00059_shard_global_in_mergetree.reference create mode 100644 tests/queries/0_stateless/00059_shard_global_in_mergetree.sql diff --git a/tests/queries/0_stateless/00059_shard_global_in_mergetree.reference b/tests/queries/0_stateless/00059_shard_global_in_mergetree.reference new file mode 100644 index 000000000000..829419dc759a --- /dev/null +++ b/tests/queries/0_stateless/00059_shard_global_in_mergetree.reference @@ -0,0 +1,8 @@ +20 +20 +20 +20 +20 +20 +20 +20 diff --git a/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql b/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql new file mode 100644 index 000000000000..b85560d2bea4 --- /dev/null +++ b/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql @@ -0,0 +1,24 @@ +-- Tags: shard + +-- test for #56790 + +CREATE TABLE test_local (x Int64) +ENGINE = MergeTree order by x as select * from numbers(10); + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local); + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local); + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where 'XXX' global in (select 'XXX'); + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * global in (select * from test_local); + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * in (select * from test_local); + +set prefer_localhost_replica=0; + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where 'XXX' global in (select 'XXX'); + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * global in (select * from test_local); + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * in (select * from test_local); From c4f46c7ce575fbfcb52c69343534b9539b3f719d Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Fri, 17 Nov 2023 10:27:02 +0300 Subject: [PATCH 128/274] test for #56790 --- .../0_stateless/00059_shard_global_in_mergetree.reference | 1 - tests/queries/0_stateless/00059_shard_global_in_mergetree.sql | 2 -- 2 files changed, 3 deletions(-) diff --git a/tests/queries/0_stateless/00059_shard_global_in_mergetree.reference b/tests/queries/0_stateless/00059_shard_global_in_mergetree.reference index 829419dc759a..208e649c0560 100644 --- a/tests/queries/0_stateless/00059_shard_global_in_mergetree.reference +++ b/tests/queries/0_stateless/00059_shard_global_in_mergetree.reference @@ -5,4 +5,3 @@ 20 20 20 -20 diff --git a/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql b/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql index b85560d2bea4..cbd4245a4867 100644 --- a/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql +++ b/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql @@ -7,8 +7,6 @@ ENGINE = MergeTree order by x as select * from numbers(10); select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local); -select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local); - select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where 'XXX' global in (select 'XXX'); select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * global in (select * from test_local); From 224b282d947daf5275e9a38d1c62e8887eb44868 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 08:27:19 +0100 Subject: [PATCH 129/274] Remove garbage --- contrib/cassandra-cmake/CMakeLists.txt | 4 ---- contrib/qpl-cmake/CMakeLists.txt | 3 +-- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/contrib/cassandra-cmake/CMakeLists.txt b/contrib/cassandra-cmake/CMakeLists.txt index 32611e0e151b..9e729c436d5e 100644 --- a/contrib/cassandra-cmake/CMakeLists.txt +++ b/contrib/cassandra-cmake/CMakeLists.txt @@ -83,10 +83,6 @@ set(HAVE_MEMCPY 1) set(HAVE_LONG_LONG 1) set(HAVE_UINT16_T 1) -configure_file("${CASS_SRC_DIR}/third_party/sparsehash/config.h.cmake" "${CMAKE_CURRENT_BINARY_DIR}/sparsehash/internal/sparseconfig.h") - - - # Determine random availability if (OS_LINUX) #set (HAVE_GETRANDOM 1) - not on every Linux kernel diff --git a/contrib/qpl-cmake/CMakeLists.txt b/contrib/qpl-cmake/CMakeLists.txt index 4e6c66fe7317..19501209b266 100644 --- a/contrib/qpl-cmake/CMakeLists.txt +++ b/contrib/qpl-cmake/CMakeLists.txt @@ -16,8 +16,7 @@ function(GetLibraryVersion _content _outputVar) SET(${_outputVar} ${CMAKE_MATCH_1} PARENT_SCOPE) endfunction() -FILE(READ "${QPL_PROJECT_DIR}/CMakeLists.txt" HEADER_CONTENT) -GetLibraryVersion("${HEADER_CONTENT}" QPL_VERSION) +set (QPL_VERSION 1.2.0) message(STATUS "Intel QPL version: ${QPL_VERSION}") From df3c066591758813c99422bab11f0811e1d487d3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 08:42:39 +0100 Subject: [PATCH 130/274] Remove more trash --- contrib/arrow-cmake/CMakeLists.txt | 1 - contrib/cassandra-cmake/CMakeLists.txt | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 935fc8863309..96d1f4adda72 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -127,7 +127,6 @@ set(ORC_SRCS "${ORC_SOURCE_SRC_DIR}/BpackingDefault.hh" "${ORC_SOURCE_SRC_DIR}/ByteRLE.cc" "${ORC_SOURCE_SRC_DIR}/ByteRLE.hh" - "${ORC_SOURCE_SRC_DIR}/CMakeLists.txt" "${ORC_SOURCE_SRC_DIR}/ColumnPrinter.cc" "${ORC_SOURCE_SRC_DIR}/ColumnReader.cc" "${ORC_SOURCE_SRC_DIR}/ColumnReader.hh" diff --git a/contrib/cassandra-cmake/CMakeLists.txt b/contrib/cassandra-cmake/CMakeLists.txt index 9e729c436d5e..0082364c130d 100644 --- a/contrib/cassandra-cmake/CMakeLists.txt +++ b/contrib/cassandra-cmake/CMakeLists.txt @@ -68,8 +68,7 @@ list(APPEND INCLUDE_DIRS ${CASS_SRC_DIR}/third_party/hdr_histogram ${CASS_SRC_DIR}/third_party/http-parser ${CASS_SRC_DIR}/third_party/mt19937_64 - ${CASS_SRC_DIR}/third_party/rapidjson/rapidjson - ${CASS_SRC_DIR}/third_party/sparsehash/src) + ${CASS_SRC_DIR}/third_party/rapidjson/rapidjson) list(APPEND INCLUDE_DIRS ${CASS_INCLUDE_DIR} ${CASS_SRC_DIR}) @@ -112,17 +111,17 @@ configure_file( ${CASS_ROOT_DIR}/driver_config.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/driver_config.hpp) - add_library(_cassandra ${SOURCES} $ $ $) -target_link_libraries(_cassandra ch_contrib::zlib ch_contrib::minizip) +target_link_libraries(_cassandra ch_contrib::zlib ch_contrib::minizip ch_contrib::sparsehash) target_include_directories(_cassandra PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${INCLUDE_DIRS}) target_include_directories(_cassandra SYSTEM BEFORE PUBLIC ${CASS_INCLUDE_DIR}) target_compile_definitions(_cassandra PRIVATE CASS_BUILDING) +target_compile_definitions(_cassandra PRIVATE -DSPARSEHASH_HASH=std::hash -Dsparsehash=google) target_link_libraries(_cassandra ch_contrib::uv) From f73b3e10ac0c4094cbc907cad2533ea73ae916b6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 08:46:45 +0100 Subject: [PATCH 131/274] Ensure no new dependencies --- docker/packager/binary/build.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 6b6374d08c94..ad31397c8d91 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -22,6 +22,7 @@ if [ "$EXTRACT_TOOLCHAIN_DARWIN" = "1" ]; then fi fi + # Uncomment to debug ccache. Don't put ccache log in /output right away, or it # will be confusingly packed into the "performance" package. # export CCACHE_LOGFILE=/build/ccache.log @@ -32,6 +33,16 @@ mkdir -p /build/build_docker cd /build/build_docker rm -f CMakeCache.txt + +# We don't want to depend on any third-party CMake files. +# To check it, find and delete them. + +grep -o -P '"contrib/[^"]+"' ../.gitmodules | + grep -v -P 'llvm-project|abseil-cpp|qpl|grpc|corrosion' | + xargs -I@ find @ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | + xargs rm + + if [ -n "$MAKE_DEB" ]; then rm -rf /build/packages/root # NOTE: this is for backward compatibility with previous releases, From c7d8465897e6d51a71d9c28bec51ba676fb70fa3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 09:12:49 +0100 Subject: [PATCH 132/274] Ensure no new dependencies --- docker/packager/binary/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index ad31397c8d91..d469b359d1a9 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -39,7 +39,7 @@ rm -f CMakeCache.txt grep -o -P '"contrib/[^"]+"' ../.gitmodules | grep -v -P 'llvm-project|abseil-cpp|qpl|grpc|corrosion' | - xargs -I@ find @ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | + xargs -I@ find ../@ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | xargs rm From f456ac97fe7c411f897a432e80055e7ed3599ad2 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Fri, 17 Nov 2023 04:19:22 -0400 Subject: [PATCH 133/274] fix currentdatabase issue --- .../00059_shard_global_in_mergetree.sql | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql b/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql index cbd4245a4867..62eec6f324b8 100644 --- a/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql +++ b/tests/queries/0_stateless/00059_shard_global_in_mergetree.sql @@ -2,21 +2,24 @@ -- test for #56790 -CREATE TABLE test_local (x Int64) -ENGINE = MergeTree order by x as select * from numbers(10); +DROP TABLE IF EXISTS test_local; + +CREATE TABLE test_local (x Int64) ENGINE = MergeTree order by x as select * from numbers(10); select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local); select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where 'XXX' global in (select 'XXX'); -select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * global in (select * from test_local); +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * global in (select * from numbers(10)); -select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * in (select * from test_local); +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * in (select * from numbers(10)); set prefer_localhost_replica=0; select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where 'XXX' global in (select 'XXX'); -select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * global in (select * from test_local); +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * global in (select * from numbers(10)); + +select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * in (select * from numbers(10)); -select count() from remote('127.0.0.1,127.0.0.2', currentDatabase(), test_local) where * in (select * from test_local); +DROP TABLE test_local; From 864dd32b05c2eb40baf545279b15cc7aacc5937a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 09:48:42 +0100 Subject: [PATCH 134/274] Remove garbage --- .../client_scripts/allin1_ssb.sh | 530 ------------------ .../client_scripts/client_stressing_test.py | 278 --------- .../client_scripts/queries_ssb.sql | 10 - .../client_scripts/run_ssb.sh | 6 - .../database_dir/deflate/config_deflate.xml | 49 -- .../deflate_s2/config_deflate_s2.xml | 49 -- .../database_dir/lz4/config_lz4.xml | 49 -- .../database_dir/lz4_s2/config_lz4_s2.xml | 49 -- .../database_dir/zstd/config_zstd.xml | 49 -- .../database_dir/zstd_s2/config_zstd_s2.xml | 49 -- 10 files changed, 1118 deletions(-) delete mode 100644 contrib/qpl-cmake/benchmark_sample/client_scripts/allin1_ssb.sh delete mode 100644 contrib/qpl-cmake/benchmark_sample/client_scripts/client_stressing_test.py delete mode 100644 contrib/qpl-cmake/benchmark_sample/client_scripts/queries_ssb.sql delete mode 100644 contrib/qpl-cmake/benchmark_sample/client_scripts/run_ssb.sh delete mode 100644 contrib/qpl-cmake/benchmark_sample/database_dir/deflate/config_deflate.xml delete mode 100644 contrib/qpl-cmake/benchmark_sample/database_dir/deflate_s2/config_deflate_s2.xml delete mode 100644 contrib/qpl-cmake/benchmark_sample/database_dir/lz4/config_lz4.xml delete mode 100644 contrib/qpl-cmake/benchmark_sample/database_dir/lz4_s2/config_lz4_s2.xml delete mode 100644 contrib/qpl-cmake/benchmark_sample/database_dir/zstd/config_zstd.xml delete mode 100644 contrib/qpl-cmake/benchmark_sample/database_dir/zstd_s2/config_zstd_s2.xml diff --git a/contrib/qpl-cmake/benchmark_sample/client_scripts/allin1_ssb.sh b/contrib/qpl-cmake/benchmark_sample/client_scripts/allin1_ssb.sh deleted file mode 100644 index 31017b565b69..000000000000 --- a/contrib/qpl-cmake/benchmark_sample/client_scripts/allin1_ssb.sh +++ /dev/null @@ -1,530 +0,0 @@ -#!/bin/bash -ckhost="localhost" -ckport=("9000" "9001" "9002" "9003") -WORKING_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." -OUTPUT_DIR="${WORKING_DIR}/output" -LOG_DIR="${OUTPUT_DIR}/log" -RAWDATA_DIR="${WORKING_DIR}/rawdata_dir" -database_dir="${WORKING_DIR}/database_dir" -CLIENT_SCRIPTS_DIR="${WORKING_DIR}/client_scripts" -LOG_PACK_FILE="$(date +%Y-%m-%d-%H-%M-%S)" -QUERY_FILE="queries_ssb.sql" -SERVER_BIND_CMD[0]="numactl -m 0 -N 0" -SERVER_BIND_CMD[1]="numactl -m 0 -N 0" -SERVER_BIND_CMD[2]="numactl -m 1 -N 1" -SERVER_BIND_CMD[3]="numactl -m 1 -N 1" -CLIENT_BIND_CMD="" -SSB_GEN_FACTOR=20 -TABLE_NAME="lineorder_flat" -TALBE_ROWS="119994608" -CODEC_CONFIG="lz4 deflate zstd" - -# define instance number -inst_num=$1 -if [ ! -n "$1" ]; then - echo "Please clarify instance number from 1,2,3 or 4" - exit 1 -else - echo "Benchmarking with instance number:$1" -fi - -if [ ! -d "$OUTPUT_DIR" ]; then -mkdir $OUTPUT_DIR -fi -if [ ! -d "$LOG_DIR" ]; then -mkdir $LOG_DIR -fi -if [ ! -d "$RAWDATA_DIR" ]; then -mkdir $RAWDATA_DIR -fi - -# define different directories -dir_server=("" "_s2" "_s3" "_s4") -ckreadSql=" - CREATE TABLE customer - ( - C_CUSTKEY UInt32, - C_NAME String, - C_ADDRESS String, - C_CITY LowCardinality(String), - C_NATION LowCardinality(String), - C_REGION LowCardinality(String), - C_PHONE String, - C_MKTSEGMENT LowCardinality(String) - ) - ENGINE = MergeTree ORDER BY (C_CUSTKEY); - - CREATE TABLE lineorder - ( - LO_ORDERKEY UInt32, - LO_LINENUMBER UInt8, - LO_CUSTKEY UInt32, - LO_PARTKEY UInt32, - LO_SUPPKEY UInt32, - LO_ORDERDATE Date, - LO_ORDERPRIORITY LowCardinality(String), - LO_SHIPPRIORITY UInt8, - LO_QUANTITY UInt8, - LO_EXTENDEDPRICE UInt32, - LO_ORDTOTALPRICE UInt32, - LO_DISCOUNT UInt8, - LO_REVENUE UInt32, - LO_SUPPLYCOST UInt32, - LO_TAX UInt8, - LO_COMMITDATE Date, - LO_SHIPMODE LowCardinality(String) - ) - ENGINE = MergeTree PARTITION BY toYear(LO_ORDERDATE) ORDER BY (LO_ORDERDATE, LO_ORDERKEY); - - CREATE TABLE part - ( - P_PARTKEY UInt32, - P_NAME String, - P_MFGR LowCardinality(String), - P_CATEGORY LowCardinality(String), - P_BRAND LowCardinality(String), - P_COLOR LowCardinality(String), - P_TYPE LowCardinality(String), - P_SIZE UInt8, - P_CONTAINER LowCardinality(String) - ) - ENGINE = MergeTree ORDER BY P_PARTKEY; - - CREATE TABLE supplier - ( - S_SUPPKEY UInt32, - S_NAME String, - S_ADDRESS String, - S_CITY LowCardinality(String), - S_NATION LowCardinality(String), - S_REGION LowCardinality(String), - S_PHONE String - ) - ENGINE = MergeTree ORDER BY S_SUPPKEY; -" -supplier_table=" - CREATE TABLE supplier - ( - S_SUPPKEY UInt32, - S_NAME String, - S_ADDRESS String, - S_CITY LowCardinality(String), - S_NATION LowCardinality(String), - S_REGION LowCardinality(String), - S_PHONE String - ) - ENGINE = MergeTree ORDER BY S_SUPPKEY; -" -part_table=" - CREATE TABLE part - ( - P_PARTKEY UInt32, - P_NAME String, - P_MFGR LowCardinality(String), - P_CATEGORY LowCardinality(String), - P_BRAND LowCardinality(String), - P_COLOR LowCardinality(String), - P_TYPE LowCardinality(String), - P_SIZE UInt8, - P_CONTAINER LowCardinality(String) - ) - ENGINE = MergeTree ORDER BY P_PARTKEY; -" -lineorder_table=" - CREATE TABLE lineorder - ( - LO_ORDERKEY UInt32, - LO_LINENUMBER UInt8, - LO_CUSTKEY UInt32, - LO_PARTKEY UInt32, - LO_SUPPKEY UInt32, - LO_ORDERDATE Date, - LO_ORDERPRIORITY LowCardinality(String), - LO_SHIPPRIORITY UInt8, - LO_QUANTITY UInt8, - LO_EXTENDEDPRICE UInt32, - LO_ORDTOTALPRICE UInt32, - LO_DISCOUNT UInt8, - LO_REVENUE UInt32, - LO_SUPPLYCOST UInt32, - LO_TAX UInt8, - LO_COMMITDATE Date, - LO_SHIPMODE LowCardinality(String) - ) - ENGINE = MergeTree PARTITION BY toYear(LO_ORDERDATE) ORDER BY (LO_ORDERDATE, LO_ORDERKEY); -" -customer_table=" - CREATE TABLE customer - ( - C_CUSTKEY UInt32, - C_NAME String, - C_ADDRESS String, - C_CITY LowCardinality(String), - C_NATION LowCardinality(String), - C_REGION LowCardinality(String), - C_PHONE String, - C_MKTSEGMENT LowCardinality(String) - ) - ENGINE = MergeTree ORDER BY (C_CUSTKEY); -" - -lineorder_flat_table=" - SET max_memory_usage = 20000000000; - CREATE TABLE lineorder_flat - ENGINE = MergeTree - PARTITION BY toYear(LO_ORDERDATE) - ORDER BY (LO_ORDERDATE, LO_ORDERKEY) AS - SELECT - l.LO_ORDERKEY AS LO_ORDERKEY, - l.LO_LINENUMBER AS LO_LINENUMBER, - l.LO_CUSTKEY AS LO_CUSTKEY, - l.LO_PARTKEY AS LO_PARTKEY, - l.LO_SUPPKEY AS LO_SUPPKEY, - l.LO_ORDERDATE AS LO_ORDERDATE, - l.LO_ORDERPRIORITY AS LO_ORDERPRIORITY, - l.LO_SHIPPRIORITY AS LO_SHIPPRIORITY, - l.LO_QUANTITY AS LO_QUANTITY, - l.LO_EXTENDEDPRICE AS LO_EXTENDEDPRICE, - l.LO_ORDTOTALPRICE AS LO_ORDTOTALPRICE, - l.LO_DISCOUNT AS LO_DISCOUNT, - l.LO_REVENUE AS LO_REVENUE, - l.LO_SUPPLYCOST AS LO_SUPPLYCOST, - l.LO_TAX AS LO_TAX, - l.LO_COMMITDATE AS LO_COMMITDATE, - l.LO_SHIPMODE AS LO_SHIPMODE, - c.C_NAME AS C_NAME, - c.C_ADDRESS AS C_ADDRESS, - c.C_CITY AS C_CITY, - c.C_NATION AS C_NATION, - c.C_REGION AS C_REGION, - c.C_PHONE AS C_PHONE, - c.C_MKTSEGMENT AS C_MKTSEGMENT, - s.S_NAME AS S_NAME, - s.S_ADDRESS AS S_ADDRESS, - s.S_CITY AS S_CITY, - s.S_NATION AS S_NATION, - s.S_REGION AS S_REGION, - s.S_PHONE AS S_PHONE, - p.P_NAME AS P_NAME, - p.P_MFGR AS P_MFGR, - p.P_CATEGORY AS P_CATEGORY, - p.P_BRAND AS P_BRAND, - p.P_COLOR AS P_COLOR, - p.P_TYPE AS P_TYPE, - p.P_SIZE AS P_SIZE, - p.P_CONTAINER AS P_CONTAINER - FROM lineorder AS l - INNER JOIN customer AS c ON c.C_CUSTKEY = l.LO_CUSTKEY - INNER JOIN supplier AS s ON s.S_SUPPKEY = l.LO_SUPPKEY - INNER JOIN part AS p ON p.P_PARTKEY = l.LO_PARTKEY; - show settings ilike 'max_memory_usage'; -" - -function insert_data(){ - echo "insert_data:$1" - create_table_prefix="clickhouse client --host ${ckhost} --port $2 --multiquery -q" - insert_data_prefix="clickhouse client --query " - case $1 in - all) - clickhouse client --host ${ckhost} --port $2 --multiquery -q"$ckreadSql" && { - ${insert_data_prefix} "INSERT INTO customer FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/customer.tbl --port=$2 - ${insert_data_prefix} "INSERT INTO part FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/part.tbl --port=$2 - ${insert_data_prefix} "INSERT INTO supplier FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/supplier.tbl --port=$2 - ${insert_data_prefix} "INSERT INTO lineorder FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/lineorder.tbl --port=$2 - } - ${create_table_prefix}"${lineorder_flat_table}" - ;; - customer) - echo ${create_table_prefix}\"${customer_table}\" - ${create_table_prefix}"${customer_table}" && { - echo "${insert_data_prefix} \"INSERT INTO $1 FORMAT CSV\" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2" - ${insert_data_prefix} "INSERT INTO $1 FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2 - } - ;; - part) - echo ${create_table_prefix}\"${part_table}\" - ${create_table_prefix}"${part_table}" && { - echo "${insert_data_prefix} \"INSERT INTO $1 FORMAT CSV\" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2" - ${insert_data_prefix} "INSERT INTO $1 FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2 - } - ;; - supplier) - echo ${create_table_prefix}"${supplier_table}" - ${create_table_prefix}"${supplier_table}" && { - echo "${insert_data_prefix} \"INSERT INTO $1 FORMAT CSV\" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2" - ${insert_data_prefix} "INSERT INTO $1 FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2 - } - ;; - lineorder) - echo ${create_table_prefix}"${lineorder_table}" - ${create_table_prefix}"${lineorder_table}" && { - echo "${insert_data_prefix} \"INSERT INTO $1 FORMAT CSV\" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2" - ${insert_data_prefix} "INSERT INTO $1 FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2 - } - ;; - lineorder_flat) - echo ${create_table_prefix}"${lineorder_flat_table}" - ${create_table_prefix}"${lineorder_flat_table}" - return 0 - ;; - *) - exit 0 - ;; - - esac -} - -function check_sql(){ - select_sql="select * from "$1" limit 1" - clickhouse client --host ${ckhost} --port $2 --multiquery -q"${select_sql}" -} - -function check_table(){ - checknum=0 - source_tables="customer part supplier lineorder lineorder_flat" - test_tables=${1:-${source_tables}} - echo "Checking table data required in server..." - for i in $(seq 0 $[inst_num-1]) - do - for j in `echo ${test_tables}` - do - check_sql $j ${ckport[i]} &> /dev/null || { - let checknum+=1 && insert_data "$j" ${ckport[i]} - } - done - done - - for i in $(seq 0 $[inst_num-1]) - do - echo "clickhouse client --host ${ckhost} --port ${ckport[i]} -m -q\"select count() from ${TABLE_NAME};\"" - var=$(clickhouse client --host ${ckhost} --port ${ckport[i]} -m -q"select count() from ${TABLE_NAME};") - if [ $var -eq $TALBE_ROWS ];then - echo "Instance_${i} Table data integrity check OK -> Rows:$var" - else - echo "Instance_${i} Table data integrity check Failed -> Rows:$var" - exit 1 - fi - done - if [ $checknum -gt 0 ];then - echo "Need sleep 10s after first table data insertion...$checknum" - sleep 10 - fi -} - -function check_instance(){ -instance_alive=0 -for i in {1..10} -do - sleep 1 - netstat -nltp | grep ${1} > /dev/null - if [ $? -ne 1 ];then - instance_alive=1 - break - fi - -done - -if [ $instance_alive -eq 0 ];then - echo "check_instance -> clickhouse server instance faild to launch due to 10s timeout!" - exit 1 -else - echo "check_instance -> clickhouse server instance launch successfully!" -fi -} - -function start_clickhouse_for_insertion(){ - echo "start_clickhouse_for_insertion" - for i in $(seq 0 $[inst_num-1]) - do - echo "cd ${database_dir}/$1${dir_server[i]}" - echo "${SERVER_BIND_CMD[i]} clickhouse server -C config_${1}${dir_server[i]}.xml >&${LOG_DIR}/${1}_${i}_server_log& > /dev/null" - - cd ${database_dir}/$1${dir_server[i]} - ${SERVER_BIND_CMD[i]} clickhouse server -C config_${1}${dir_server[i]}.xml >&${LOG_DIR}/${1}_${i}_server_log& > /dev/null - check_instance ${ckport[i]} - done -} - -function start_clickhouse_for_stressing(){ - echo "start_clickhouse_for_stressing" - for i in $(seq 0 $[inst_num-1]) - do - echo "cd ${database_dir}/$1${dir_server[i]}" - echo "${SERVER_BIND_CMD[i]} clickhouse server -C config_${1}${dir_server[i]}.xml >&/dev/null&" - - cd ${database_dir}/$1${dir_server[i]} - ${SERVER_BIND_CMD[i]} clickhouse server -C config_${1}${dir_server[i]}.xml >&/dev/null& - check_instance ${ckport[i]} - done -} -yum -y install git make gcc sudo net-tools &> /dev/null -pip3 install clickhouse_driver numpy &> /dev/null -test -d ${RAWDATA_DIR}/ssb-dbgen || git clone https://github.com/vadimtk/ssb-dbgen.git ${RAWDATA_DIR}/ssb-dbgen && cd ${RAWDATA_DIR}/ssb-dbgen - -if [ ! -f ${RAWDATA_DIR}/ssb-dbgen/dbgen ];then - make && { - test -f ${RAWDATA_DIR}/ssb-dbgen/customer.tbl || echo y |./dbgen -s ${SSB_GEN_FACTOR} -T c - test -f ${RAWDATA_DIR}/ssb-dbgen/part.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T p - test -f ${RAWDATA_DIR}/ssb-dbgen/supplier.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T s - test -f ${RAWDATA_DIR}/ssb-dbgen/date.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T d - test -f ${RAWDATA_DIR}/ssb-dbgen/lineorder.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T l - } -else - test -f ${RAWDATA_DIR}/ssb-dbgen/customer.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T c - test -f ${RAWDATA_DIR}/ssb-dbgen/part.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T p - test -f ${RAWDATA_DIR}/ssb-dbgen/supplier.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T s - test -f ${RAWDATA_DIR}/ssb-dbgen/date.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T d - test -f ${RAWDATA_DIR}/ssb-dbgen/lineorder.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T l - -fi - -filenum=`find ${RAWDATA_DIR}/ssb-dbgen/ -name "*.tbl" | wc -l` - -if [ $filenum -ne 5 ];then - echo "generate ssb data file *.tbl faild" - exit 1 -fi - -function kill_instance(){ -instance_alive=1 -for i in {1..2} -do - pkill clickhouse && sleep 5 - instance_alive=0 - for i in $(seq 0 $[inst_num-1]) - do - netstat -nltp | grep ${ckport[i]} > /dev/null - if [ $? -ne 1 ];then - instance_alive=1 - break; - fi - done - if [ $instance_alive -eq 0 ];then - break; - fi -done -if [ $instance_alive -eq 0 ];then - echo "kill_instance OK!" -else - echo "kill_instance Failed -> clickhouse server instance still alive due to 10s timeout" - exit 1 -fi -} - -function run_test(){ -is_xml=0 -for i in $(seq 0 $[inst_num-1]) -do - if [ -f ${database_dir}/${1}${dir_server[i]}/config_${1}${dir_server[i]}.xml ]; then - is_xml=$[is_xml+1] - fi -done -if [ $is_xml -eq $inst_num ];then - echo "Benchmark with $inst_num instance" - start_clickhouse_for_insertion ${1} - - for i in $(seq 0 $[inst_num-1]) - do - clickhouse client --host ${ckhost} --port ${ckport[i]} -m -q"show databases;" >/dev/null - done - - if [ $? -eq 0 ];then - check_table - fi - kill_instance - - if [ $1 == "deflate" ];then - test -f ${LOG_DIR}/${1}_server_log && deflatemsg=`cat ${LOG_DIR}/${1}_server_log | grep DeflateJobHWPool` - if [ -n "$deflatemsg" ];then - echo ------------------------------------------------------ - echo $deflatemsg - echo ------------------------------------------------------ - fi - fi - echo "Check table data required in server_${1} -> Done! " - - start_clickhouse_for_stressing ${1} - for i in $(seq 0 $[inst_num-1]) - do - clickhouse client --host ${ckhost} --port ${ckport[i]} -m -q"show databases;" >/dev/null - done - if [ $? -eq 0 ];then - test -d ${CLIENT_SCRIPTS_DIR} && cd ${CLIENT_SCRIPTS_DIR} - echo "Client stressing... " - echo "${CLIENT_BIND_CMD} python3 client_stressing_test.py ${QUERY_FILE} $inst_num &> ${LOG_DIR}/${1}.log" - ${CLIENT_BIND_CMD} python3 client_stressing_test.py ${QUERY_FILE} $inst_num &> ${LOG_DIR}/${1}.log - echo "Completed client stressing, checking log... " - finish_log=`grep "Finished" ${LOG_DIR}/${1}.log | wc -l` - if [ $finish_log -eq 1 ] ;then - kill_instance - test -f ${LOG_DIR}/${1}.log && echo "${1}.log ===> ${LOG_DIR}/${1}.log" - else - kill_instance - echo "No find 'Finished' in client log -> Performance test may fail" - exit 1 - - fi - - else - echo "${1} clickhouse server start fail" - exit 1 - fi -else - echo "clickhouse server start fail -> Please check xml files required in ${database_dir} for each instance" - exit 1 - -fi -} -function clear_log(){ - if [ -d "$LOG_DIR" ]; then - cd ${LOG_DIR} && rm -rf * - fi -} - -function gather_log_for_codec(){ - cd ${OUTPUT_DIR} && mkdir -p ${LOG_PACK_FILE}/${1} - cp -rf ${LOG_DIR} ${OUTPUT_DIR}/${LOG_PACK_FILE}/${1} -} - -function pack_log(){ - if [ -e "${OUTPUT_DIR}/run.log" ]; then - cp ${OUTPUT_DIR}/run.log ${OUTPUT_DIR}/${LOG_PACK_FILE}/ - fi - echo "Please check all log information in ${OUTPUT_DIR}/${LOG_PACK_FILE}" -} - -function setup_check(){ - - iax_dev_num=`accel-config list | grep iax | wc -l` - if [ $iax_dev_num -eq 0 ] ;then - iax_dev_num=`accel-config list | grep iax | wc -l` - if [ $iax_dev_num -eq 0 ] ;then - echo "No IAA devices available -> Please check IAA hardware setup manually!" - exit 1 - else - echo "IAA enabled devices number:$iax_dev_num" - fi - else - echo "IAA enabled devices number:$iax_dev_num" - fi - libaccel_version=`accel-config -v` - clickhouser_version=`clickhouse server --version` - kernel_dxd_log=`dmesg | grep dxd` - echo "libaccel_version:$libaccel_version" - echo "clickhouser_version:$clickhouser_version" - echo -e "idxd section in kernel log:\n$kernel_dxd_log" -} - -setup_check -export CLICKHOUSE_WATCHDOG_ENABLE=0 -for i in ${CODEC_CONFIG[@]} -do - clear_log - codec=${i} - echo "run test------------$codec" - run_test $codec - gather_log_for_codec $codec -done - -pack_log -echo "Done." \ No newline at end of file diff --git a/contrib/qpl-cmake/benchmark_sample/client_scripts/client_stressing_test.py b/contrib/qpl-cmake/benchmark_sample/client_scripts/client_stressing_test.py deleted file mode 100644 index f12381a198c3..000000000000 --- a/contrib/qpl-cmake/benchmark_sample/client_scripts/client_stressing_test.py +++ /dev/null @@ -1,278 +0,0 @@ -from operator import eq -import os -import random -import time -import sys -from clickhouse_driver import Client -import numpy as np -import subprocess -import multiprocessing -from multiprocessing import Manager - -warmup_runs = 10 -calculated_runs = 10 -seconds = 30 -max_instances_number = 8 -retest_number = 3 -retest_tolerance = 10 - - -def checkInt(str): - try: - int(str) - return True - except ValueError: - return False - - -def setup_client(index): - if index < 4: - port_idx = index - else: - port_idx = index + 4 - client = Client( - host="localhost", - database="default", - user="default", - password="", - port="900%d" % port_idx, - ) - union_mode_query = "SET union_default_mode='DISTINCT'" - client.execute(union_mode_query) - return client - - -def warm_client(clientN, clientL, query, loop): - for c_idx in range(clientN): - for _ in range(loop): - clientL[c_idx].execute(query) - - -def read_queries(queries_list): - queries = list() - queries_id = list() - with open(queries_list, "r") as f: - for line in f: - line = line.rstrip() - line = line.split("$") - queries_id.append(line[0]) - queries.append(line[1]) - return queries_id, queries - - -def run_task(client, cname, query, loop, query_latency): - start_time = time.time() - for i in range(loop): - client.execute(query) - query_latency.append(client.last_query.elapsed) - - end_time = time.time() - p95 = np.percentile(query_latency, 95) - print( - "CLIENT: {0} end. -> P95: %f, qps: %f".format(cname) - % (p95, loop / (end_time - start_time)) - ) - - -def run_multi_clients(clientN, clientList, query, loop): - client_pids = {} - start_time = time.time() - manager = multiprocessing.Manager() - query_latency_list0 = manager.list() - query_latency_list1 = manager.list() - query_latency_list2 = manager.list() - query_latency_list3 = manager.list() - query_latency_list4 = manager.list() - query_latency_list5 = manager.list() - query_latency_list6 = manager.list() - query_latency_list7 = manager.list() - - for c_idx in range(clientN): - client_name = "Role_%d" % c_idx - if c_idx == 0: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list0), - ) - elif c_idx == 1: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list1), - ) - elif c_idx == 2: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list2), - ) - elif c_idx == 3: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list3), - ) - elif c_idx == 4: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list4), - ) - elif c_idx == 5: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list5), - ) - elif c_idx == 6: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list6), - ) - elif c_idx == 7: - client_pids[c_idx] = multiprocessing.Process( - target=run_task, - args=(clientList[c_idx], client_name, query, loop, query_latency_list7), - ) - else: - print("ERROR: CLIENT number dismatch!!") - exit() - print("CLIENT: %s start" % client_name) - client_pids[c_idx].start() - - for c_idx in range(clientN): - client_pids[c_idx].join() - end_time = time.time() - totalT = end_time - start_time - - query_latencyTotal = list() - for item in query_latency_list0: - query_latencyTotal.append(item) - for item in query_latency_list1: - query_latencyTotal.append(item) - for item in query_latency_list2: - query_latencyTotal.append(item) - for item in query_latency_list3: - query_latencyTotal.append(item) - for item in query_latency_list4: - query_latencyTotal.append(item) - for item in query_latency_list5: - query_latencyTotal.append(item) - for item in query_latency_list6: - query_latencyTotal.append(item) - for item in query_latency_list7: - query_latencyTotal.append(item) - - totalP95 = np.percentile(query_latencyTotal, 95) * 1000 - return totalT, totalP95 - - -def run_task_caculated(client, cname, query, loop): - query_latency = list() - start_time = time.time() - for i in range(loop): - client.execute(query) - query_latency.append(client.last_query.elapsed) - end_time = time.time() - p95 = np.percentile(query_latency, 95) - - -def run_multi_clients_caculated(clientN, clientList, query, loop): - client_pids = {} - start_time = time.time() - for c_idx in range(clientN): - client_name = "Role_%d" % c_idx - client_pids[c_idx] = multiprocessing.Process( - target=run_task_caculated, - args=(clientList[c_idx], client_name, query, loop), - ) - client_pids[c_idx].start() - for c_idx in range(clientN): - client_pids[c_idx].join() - end_time = time.time() - totalT = end_time - start_time - return totalT - - -if __name__ == "__main__": - client_number = 1 - queries = list() - queries_id = list() - - if len(sys.argv) != 3: - print( - "usage: python3 client_stressing_test.py [queries_file_path] [client_number]" - ) - sys.exit() - else: - queries_list = sys.argv[1] - client_number = int(sys.argv[2]) - print( - "queries_file_path: %s, client_number: %d" % (queries_list, client_number) - ) - if not os.path.isfile(queries_list) or not os.access(queries_list, os.R_OK): - print("please check the right path for queries file") - sys.exit() - if ( - not checkInt(sys.argv[2]) - or int(sys.argv[2]) > max_instances_number - or int(sys.argv[2]) < 1 - ): - print("client_number should be in [1~%d]" % max_instances_number) - sys.exit() - - client_list = {} - queries_id, queries = read_queries(queries_list) - - for c_idx in range(client_number): - client_list[c_idx] = setup_client(c_idx) - # clear cache - os.system("sync; echo 3 > /proc/sys/vm/drop_caches") - - print("###Polit Run Begin") - for i in queries: - warm_client(client_number, client_list, i, 1) - print("###Polit Run End -> Start stressing....") - - query_index = 0 - for q in queries: - print( - "\n###START -> Index: %d, ID: %s, Query: %s" - % (query_index, queries_id[query_index], q) - ) - warm_client(client_number, client_list, q, warmup_runs) - print("###Warm Done!") - for j in range(0, retest_number): - totalT = run_multi_clients_caculated( - client_number, client_list, q, calculated_runs - ) - curr_loop = int(seconds * calculated_runs / totalT) + 1 - print( - "###Calculation Done! -> loopN: %d, expected seconds:%d" - % (curr_loop, seconds) - ) - - print("###Stress Running! -> %d iterations......" % curr_loop) - - totalT, totalP95 = run_multi_clients( - client_number, client_list, q, curr_loop - ) - - if totalT > (seconds - retest_tolerance) and totalT < ( - seconds + retest_tolerance - ): - break - else: - print( - "###totalT:%d is far way from expected seconds:%d. Run again ->j:%d!" - % (totalT, seconds, j) - ) - - print( - "###Completed! -> ID: %s, clientN: %d, totalT: %.2f s, latencyAVG: %.2f ms, P95: %.2f ms, QPS_Final: %.2f" - % ( - queries_id[query_index], - client_number, - totalT, - totalT * 1000 / (curr_loop * client_number), - totalP95, - ((curr_loop * client_number) / totalT), - ) - ) - query_index += 1 - print("###Finished!") diff --git a/contrib/qpl-cmake/benchmark_sample/client_scripts/queries_ssb.sql b/contrib/qpl-cmake/benchmark_sample/client_scripts/queries_ssb.sql deleted file mode 100644 index abf2df6503a5..000000000000 --- a/contrib/qpl-cmake/benchmark_sample/client_scripts/queries_ssb.sql +++ /dev/null @@ -1,10 +0,0 @@ -Q1.1$SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE toYear(LO_ORDERDATE) = 1993 AND LO_DISCOUNT BETWEEN 1 AND 3 AND LO_QUANTITY < 25; -Q2.1$SELECT sum(LO_REVENUE),toYear(LO_ORDERDATE) AS year,P_BRAND FROM lineorder_flat WHERE P_CATEGORY = 'MFGR#12' AND S_REGION = 'AMERICA' GROUP BY year,P_BRAND ORDER BY year,P_BRAND; -Q2.2$SELECT sum(LO_REVENUE),toYear(LO_ORDERDATE) AS year,P_BRAND FROM lineorder_flat WHERE P_BRAND >= 'MFGR#2221' AND P_BRAND <= 'MFGR#2228' AND S_REGION = 'ASIA' GROUP BY year,P_BRAND ORDER BY year,P_BRAND; -Q2.3$SELECT sum(LO_REVENUE),toYear(LO_ORDERDATE) AS year,P_BRAND FROM lineorder_flat WHERE P_BRAND = 'MFGR#2239' AND S_REGION = 'EUROPE' GROUP BY year,P_BRAND ORDER BY year,P_BRAND; -Q3.1$SELECT C_NATION,S_NATION,toYear(LO_ORDERDATE) AS year,sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE C_REGION = 'ASIA' AND S_REGION = 'ASIA' AND year >= 1992 AND year <= 1997 GROUP BY C_NATION,S_NATION,year ORDER BY year ASC,revenue DESC; -Q3.2$SELECT C_CITY,S_CITY,toYear(LO_ORDERDATE) AS year,sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE C_NATION = 'UNITED STATES' AND S_NATION = 'UNITED STATES' AND year >= 1992 AND year <= 1997 GROUP BY C_CITY,S_CITY,year ORDER BY year ASC,revenue DESC; -Q3.3$SELECT C_CITY,S_CITY,toYear(LO_ORDERDATE) AS year,sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE (C_CITY = 'UNITED KI1' OR C_CITY = 'UNITED KI5') AND (S_CITY = 'UNITED KI1' OR S_CITY = 'UNITED KI5') AND year >= 1992 AND year <= 1997 GROUP BY C_CITY,S_CITY,year ORDER BY year ASC,revenue DESC; -Q4.1$SELECT toYear(LO_ORDERDATE) AS year,C_NATION,sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') GROUP BY year,C_NATION ORDER BY year ASC,C_NATION ASC; -Q4.2$SELECT toYear(LO_ORDERDATE) AS year,S_NATION,P_CATEGORY,sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (year = 1997 OR year = 1998) AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') GROUP BY year,S_NATION,P_CATEGORY ORDER BY year ASC,S_NATION ASC,P_CATEGORY ASC; -Q4.3$SELECT toYear(LO_ORDERDATE) AS year,S_CITY,P_BRAND,sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE S_NATION = 'UNITED STATES' AND (year = 1997 OR year = 1998) AND P_CATEGORY = 'MFGR#14' GROUP BY year,S_CITY,P_BRAND ORDER BY year ASC,S_CITY ASC,P_BRAND ASC; diff --git a/contrib/qpl-cmake/benchmark_sample/client_scripts/run_ssb.sh b/contrib/qpl-cmake/benchmark_sample/client_scripts/run_ssb.sh deleted file mode 100644 index 6067b1058f25..000000000000 --- a/contrib/qpl-cmake/benchmark_sample/client_scripts/run_ssb.sh +++ /dev/null @@ -1,6 +0,0 @@ -WORKING_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." -if [ ! -d "${WORKING_DIR}/output" ]; then -mkdir ${WORKING_DIR}/output -fi -bash allin1_ssb.sh 2 > ${WORKING_DIR}/output/run.log -echo "Please check log in: ${WORKING_DIR}/output/run.log" \ No newline at end of file diff --git a/contrib/qpl-cmake/benchmark_sample/database_dir/deflate/config_deflate.xml b/contrib/qpl-cmake/benchmark_sample/database_dir/deflate/config_deflate.xml deleted file mode 100644 index ab77a9cdcbe4..000000000000 --- a/contrib/qpl-cmake/benchmark_sample/database_dir/deflate/config_deflate.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - trace - true - - - 8123 - 9000 - 9004 - - ./ - - 8589934592 - 5368709120 - true - - - - deflate_qpl - - - - - - - - - ::/0 - - - default - default - 1 - - - - - - - - - - - diff --git a/contrib/qpl-cmake/benchmark_sample/database_dir/deflate_s2/config_deflate_s2.xml b/contrib/qpl-cmake/benchmark_sample/database_dir/deflate_s2/config_deflate_s2.xml deleted file mode 100644 index b71456486f53..000000000000 --- a/contrib/qpl-cmake/benchmark_sample/database_dir/deflate_s2/config_deflate_s2.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - trace - true - - - 8124 - 9001 - 9005 - - ./ - - 8589934592 - 5368709120 - true - - - - deflate_qpl - - - - - - - - - ::/0 - - - default - default - 1 - - - - - - - - - - - diff --git a/contrib/qpl-cmake/benchmark_sample/database_dir/lz4/config_lz4.xml b/contrib/qpl-cmake/benchmark_sample/database_dir/lz4/config_lz4.xml deleted file mode 100644 index f4dc59b60aa9..000000000000 --- a/contrib/qpl-cmake/benchmark_sample/database_dir/lz4/config_lz4.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - trace - true - - - 8123 - 9000 - 9004 - - ./ - - 8589934592 - 5368709120 - true - - - - lz4 - - - - - - - - - ::/0 - - - default - default - 1 - - - - - - - - - - - diff --git a/contrib/qpl-cmake/benchmark_sample/database_dir/lz4_s2/config_lz4_s2.xml b/contrib/qpl-cmake/benchmark_sample/database_dir/lz4_s2/config_lz4_s2.xml deleted file mode 100644 index 357db8942d77..000000000000 --- a/contrib/qpl-cmake/benchmark_sample/database_dir/lz4_s2/config_lz4_s2.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - trace - true - - - 8124 - 9001 - 9005 - - ./ - - 8589934592 - 5368709120 - true - - - - lz4 - - - - - - - - - ::/0 - - - default - default - 1 - - - - - - - - - - - diff --git a/contrib/qpl-cmake/benchmark_sample/database_dir/zstd/config_zstd.xml b/contrib/qpl-cmake/benchmark_sample/database_dir/zstd/config_zstd.xml deleted file mode 100644 index 1c4c738edaf9..000000000000 --- a/contrib/qpl-cmake/benchmark_sample/database_dir/zstd/config_zstd.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - trace - true - - - 8123 - 9000 - 9004 - - ./ - - 8589934592 - 5368709120 - true - - - - zstd - - - - - - - - - ::/0 - - - default - default - 1 - - - - - - - - - - - diff --git a/contrib/qpl-cmake/benchmark_sample/database_dir/zstd_s2/config_zstd_s2.xml b/contrib/qpl-cmake/benchmark_sample/database_dir/zstd_s2/config_zstd_s2.xml deleted file mode 100644 index f3db01b77390..000000000000 --- a/contrib/qpl-cmake/benchmark_sample/database_dir/zstd_s2/config_zstd_s2.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - trace - true - - - 8124 - 9001 - 9005 - - ./ - - 8589934592 - 5368709120 - true - - - - zstd - - - - - - - - - ::/0 - - - default - default - 1 - - - - - - - - - - - From c65607484e51a5e9aa8f59612e7817b899bb88ee Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 10:03:10 +0100 Subject: [PATCH 135/274] Remove garbage --- contrib/qpl-cmake/CMakeLists.txt | 464 ++++++++++++++++++++++++++++--- docker/packager/binary/build.sh | 2 +- 2 files changed, 419 insertions(+), 47 deletions(-) diff --git a/contrib/qpl-cmake/CMakeLists.txt b/contrib/qpl-cmake/CMakeLists.txt index 19501209b266..7a84048e16b9 100644 --- a/contrib/qpl-cmake/CMakeLists.txt +++ b/contrib/qpl-cmake/CMakeLists.txt @@ -27,16 +27,422 @@ message(STATUS "Intel QPL version: ${QPL_VERSION}") # The qpl submodule comes with its own version of isal. It contains code which does not exist in upstream isal. It would be nice to link # only upstream isal (ch_contrib::isal) but at this point we can't. -include("${QPL_PROJECT_DIR}/cmake/CompileOptions.cmake") +# ========================================================================== +# Copyright (C) 2022 Intel Corporation +# +# SPDX-License-Identifier: MIT +# ========================================================================== + +set(QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS "-fno-exceptions;-fno-rtti") + +function(modify_standard_language_flag) + # Declaring function parameters + set(OPTIONS "") + set(ONE_VALUE_ARGS + LANGUAGE_NAME + FLAG_NAME + NEW_FLAG_VALUE) + set(MULTI_VALUE_ARGS "") + + # Parsing function parameters + cmake_parse_arguments(MODIFY + "${OPTIONS}" + "${ONE_VALUE_ARGS}" + "${MULTI_VALUE_ARGS}" + ${ARGN}) + + # Variables + set(FLAG_REGULAR_EXPRESSION "${MODIFY_FLAG_NAME}.*[ ]*") + set(NEW_VALUE "${MODIFY_FLAG_NAME}${MODIFY_NEW_FLAG_VALUE}") + + # Replacing specified flag with new value + string(REGEX REPLACE + ${FLAG_REGULAR_EXPRESSION} ${NEW_VALUE} + NEW_COMPILE_FLAGS + "${CMAKE_${MODIFY_LANGUAGE_NAME}_FLAGS}") + + # Returning the value + set(CMAKE_${MODIFY_LANGUAGE_NAME}_FLAGS ${NEW_COMPILE_FLAGS} PARENT_SCOPE) +endfunction() + +function(get_function_name_with_default_bit_width in_function_name bit_width out_function_name) + + if(in_function_name MATCHES ".*_i") + + string(REPLACE "_i" "" in_function_name ${in_function_name}) + + set(${out_function_name} "${in_function_name}_${bit_width}_i" PARENT_SCOPE) + + else() + + set(${out_function_name} "${in_function_name}_${bit_width}" PARENT_SCOPE) + + endif() + +endfunction() + +macro(get_list_of_supported_optimizations PLATFORMS_LIST) + list(APPEND PLATFORMS_LIST "") + list(APPEND PLATFORMS_LIST "px") + list(APPEND PLATFORMS_LIST "avx512") +endmacro(get_list_of_supported_optimizations) + +function(generate_unpack_kernel_arrays current_directory PLATFORMS_LIST) + list(APPEND UNPACK_POSTFIX_LIST "") + list(APPEND UNPACK_PRLE_POSTFIX_LIST "") + list(APPEND PACK_POSTFIX_LIST "") + list(APPEND PACK_INDEX_POSTFIX_LIST "") + list(APPEND SCAN_POSTFIX_LIST "") + list(APPEND DEFAULT_BIT_WIDTH_FUNCTIONS_LIST "") + list(APPEND DEFAULT_BIT_WIDTH_LIST "") + + #create list of functions that use only 8u 16u 32u postfixes + list(APPEND DEFAULT_BIT_WIDTH_FUNCTIONS_LIST "unpack_prle") + list(APPEND DEFAULT_BIT_WIDTH_FUNCTIONS_LIST "extract") + list(APPEND DEFAULT_BIT_WIDTH_FUNCTIONS_LIST "extract_i") + list(APPEND DEFAULT_BIT_WIDTH_FUNCTIONS_LIST "select") + list(APPEND DEFAULT_BIT_WIDTH_FUNCTIONS_LIST "select_i") + list(APPEND DEFAULT_BIT_WIDTH_FUNCTIONS_LIST "expand") + + #create default bit width list + list(APPEND DEFAULT_BIT_WIDTH_LIST "8u") + list(APPEND DEFAULT_BIT_WIDTH_LIST "16u") + list(APPEND DEFAULT_BIT_WIDTH_LIST "32u") + + #create scan kernel postfixes + list(APPEND SCAN_COMPARATOR_LIST "") + + list(APPEND SCAN_COMPARATOR_LIST "eq") + list(APPEND SCAN_COMPARATOR_LIST "ne") + list(APPEND SCAN_COMPARATOR_LIST "lt") + list(APPEND SCAN_COMPARATOR_LIST "le") + list(APPEND SCAN_COMPARATOR_LIST "gt") + list(APPEND SCAN_COMPARATOR_LIST "ge") + list(APPEND SCAN_COMPARATOR_LIST "range") + list(APPEND SCAN_COMPARATOR_LIST "not_range") + + foreach(SCAN_COMPARATOR IN LISTS SCAN_COMPARATOR_LIST) + list(APPEND SCAN_POSTFIX_LIST "_${SCAN_COMPARATOR}_8u") + list(APPEND SCAN_POSTFIX_LIST "_${SCAN_COMPARATOR}_16u8u") + list(APPEND SCAN_POSTFIX_LIST "_${SCAN_COMPARATOR}_32u8u") + endforeach() + + # create unpack kernel postfixes + foreach(input_width RANGE 1 32 1) + if(input_width LESS 8 OR input_width EQUAL 8) + list(APPEND UNPACK_POSTFIX_LIST "_${input_width}u8u") + + elseif(input_width LESS 16 OR input_width EQUAL 16) + list(APPEND UNPACK_POSTFIX_LIST "_${input_width}u16u") + + else() + list(APPEND UNPACK_POSTFIX_LIST "_${input_width}u32u") + endif() + endforeach() + + # create pack kernel postfixes + foreach(output_width RANGE 1 8 1) + list(APPEND PACK_POSTFIX_LIST "_8u${output_width}u") + endforeach() + + foreach(output_width RANGE 9 16 1) + list(APPEND PACK_POSTFIX_LIST "_16u${output_width}u") + endforeach() + + foreach(output_width RANGE 17 32 1) + list(APPEND PACK_POSTFIX_LIST "_32u${output_width}u") + endforeach() + + list(APPEND PACK_POSTFIX_LIST "_8u16u") + list(APPEND PACK_POSTFIX_LIST "_8u32u") + list(APPEND PACK_POSTFIX_LIST "_16u32u") + + # create pack index kernel postfixes + list(APPEND PACK_INDEX_POSTFIX_LIST "_nu") + list(APPEND PACK_INDEX_POSTFIX_LIST "_8u") + list(APPEND PACK_INDEX_POSTFIX_LIST "_8u16u") + list(APPEND PACK_INDEX_POSTFIX_LIST "_8u32u") + + # write to file + file(MAKE_DIRECTORY ${current_directory}/generated) + + foreach(PLATFORM_VALUE IN LISTS PLATFORMS_LIST) + set(directory "${current_directory}/generated") + set(PLATFORM_PREFIX "${PLATFORM_VALUE}_") + + # + # Write unpack table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}unpack.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}unpack.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}unpack.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}unpack.cpp "unpack_table_t ${PLATFORM_PREFIX}unpack_table = {\n") + + #write LE kernels + foreach(UNPACK_POSTFIX IN LISTS UNPACK_POSTFIX_LIST) + file(APPEND ${directory}/${PLATFORM_PREFIX}unpack.cpp "\t${PLATFORM_PREFIX}qplc_unpack${UNPACK_POSTFIX},\n") + endforeach() + + #write BE kernels + + #get last element of the list + set(LAST_ELEMENT "") + list(GET UNPACK_POSTFIX_LIST -1 LAST_ELEMENT) + + foreach(UNPACK_POSTFIX IN LISTS UNPACK_POSTFIX_LIST) + + if(UNPACK_POSTFIX STREQUAL LAST_ELEMENT) + file(APPEND ${directory}/${PLATFORM_PREFIX}unpack.cpp "\t${PLATFORM_PREFIX}qplc_unpack_be${UNPACK_POSTFIX}};\n") + else() + file(APPEND ${directory}/${PLATFORM_PREFIX}unpack.cpp "\t${PLATFORM_PREFIX}qplc_unpack_be${UNPACK_POSTFIX},\n") + endif() + endforeach() + + file(APPEND ${directory}/${PLATFORM_PREFIX}unpack.cpp "}\n") + + # + # Write pack table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}pack.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack.cpp "pack_table_t ${PLATFORM_PREFIX}pack_table = {\n") + + #write LE kernels + foreach(PACK_POSTFIX IN LISTS PACK_POSTFIX_LIST) + file(APPEND ${directory}/${PLATFORM_PREFIX}pack.cpp "\t${PLATFORM_PREFIX}qplc_pack${PACK_POSTFIX},\n") + endforeach() + + #write BE kernels + + #get last element of the list + set(LAST_ELEMENT "") + list(GET PACK_POSTFIX_LIST -1 LAST_ELEMENT) + + foreach(PACK_POSTFIX IN LISTS PACK_POSTFIX_LIST) + + if(PACK_POSTFIX STREQUAL LAST_ELEMENT) + file(APPEND ${directory}/${PLATFORM_PREFIX}pack.cpp "\t${PLATFORM_PREFIX}qplc_pack_be${PACK_POSTFIX}};\n") + else() + file(APPEND ${directory}/${PLATFORM_PREFIX}pack.cpp "\t${PLATFORM_PREFIX}qplc_pack_be${PACK_POSTFIX},\n") + endif() + endforeach() + + file(APPEND ${directory}/${PLATFORM_PREFIX}pack.cpp "}\n") + + # + # Write scan table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}scan.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}scan.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}scan.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}scan.cpp "scan_table_t ${PLATFORM_PREFIX}scan_table = {\n") + + #get last element of the list + set(LAST_ELEMENT "") + list(GET SCAN_POSTFIX_LIST -1 LAST_ELEMENT) + + foreach(SCAN_POSTFIX IN LISTS SCAN_POSTFIX_LIST) + + if(SCAN_POSTFIX STREQUAL LAST_ELEMENT) + file(APPEND ${directory}/${PLATFORM_PREFIX}scan.cpp "\t${PLATFORM_PREFIX}qplc_scan${SCAN_POSTFIX}};\n") + else() + file(APPEND ${directory}/${PLATFORM_PREFIX}scan.cpp "\t${PLATFORM_PREFIX}qplc_scan${SCAN_POSTFIX},\n") + endif() + endforeach() + + file(APPEND ${directory}/${PLATFORM_PREFIX}scan.cpp "}\n") + + # + # Write scan_i table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}scan_i.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}scan_i.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}scan_i.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}scan_i.cpp "scan_i_table_t ${PLATFORM_PREFIX}scan_i_table = {\n") + + #get last element of the list + set(LAST_ELEMENT "") + list(GET SCAN_POSTFIX_LIST -1 LAST_ELEMENT) + + foreach(SCAN_POSTFIX IN LISTS SCAN_POSTFIX_LIST) + + if(SCAN_POSTFIX STREQUAL LAST_ELEMENT) + file(APPEND ${directory}/${PLATFORM_PREFIX}scan_i.cpp "\t${PLATFORM_PREFIX}qplc_scan${SCAN_POSTFIX}_i};\n") + else() + file(APPEND ${directory}/${PLATFORM_PREFIX}scan_i.cpp "\t${PLATFORM_PREFIX}qplc_scan${SCAN_POSTFIX}_i,\n") + endif() + endforeach() + + file(APPEND ${directory}/${PLATFORM_PREFIX}scan_i.cpp "}\n") + + # + # Write pack_index table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}pack_index.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "pack_index_table_t ${PLATFORM_PREFIX}pack_index_table = {\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_bits_nu,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_index_8u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_index_8u16u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_index_8u32u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_bits_be_nu,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_index_8u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_index_be_8u16u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "\t${PLATFORM_PREFIX}qplc_pack_index_be_8u32u};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}pack_index.cpp "}\n") + + # + # Write default bit width functions + # + foreach(DEAULT_BIT_WIDTH_FUNCTION IN LISTS DEFAULT_BIT_WIDTH_FUNCTIONS_LIST) + file(WRITE ${directory}/${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}.cpp "${DEAULT_BIT_WIDTH_FUNCTION}_table_t ${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}_table = {\n") + + #get last element of the list + set(LAST_ELEMENT "") + list(GET DEFAULT_BIT_WIDTH_LIST -1 LAST_ELEMENT) + + foreach(BIT_WIDTH IN LISTS DEFAULT_BIT_WIDTH_LIST) + + set(FUNCTION_NAME "") + get_function_name_with_default_bit_width(${DEAULT_BIT_WIDTH_FUNCTION} ${BIT_WIDTH} FUNCTION_NAME) + + if(BIT_WIDTH STREQUAL LAST_ELEMENT) + file(APPEND ${directory}/${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}.cpp "\t${PLATFORM_PREFIX}qplc_${FUNCTION_NAME}};\n") + else() + file(APPEND ${directory}/${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}.cpp "\t${PLATFORM_PREFIX}qplc_${FUNCTION_NAME},\n") + endif() + endforeach() + + file(APPEND ${directory}/${PLATFORM_PREFIX}${DEAULT_BIT_WIDTH_FUNCTION}.cpp "}\n") + endforeach() + + # + # Write aggregates table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}aggregates.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "aggregates_table_t ${PLATFORM_PREFIX}aggregates_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "\t${PLATFORM_PREFIX}qplc_bit_aggregates_8u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "\t${PLATFORM_PREFIX}qplc_aggregates_8u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "\t${PLATFORM_PREFIX}qplc_aggregates_16u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "\t${PLATFORM_PREFIX}qplc_aggregates_32u};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}aggregates.cpp "}\n") + + # + # Write mem_copy functions table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "memory_copy_table_t ${PLATFORM_PREFIX}memory_copy_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "\t${PLATFORM_PREFIX}qplc_copy_8u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "\t${PLATFORM_PREFIX}qplc_copy_16u,\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "\t${PLATFORM_PREFIX}qplc_copy_32u};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}memory_copy.cpp "}\n") + + # + # Write mem_copy functions table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}zero.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}zero.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}zero.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}zero.cpp "zero_table_t ${PLATFORM_PREFIX}zero_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}zero.cpp "\t${PLATFORM_PREFIX}qplc_zero_8u};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}zero.cpp "}\n") + + # + # Write move functions table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}move.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}move.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}move.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}move.cpp "move_table_t ${PLATFORM_PREFIX}move_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}move.cpp "\t${PLATFORM_PREFIX}qplc_move_8u};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}move.cpp "}\n") + + # + # Write crc64 function table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}crc64.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}crc64.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}crc64.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}crc64.cpp "crc64_table_t ${PLATFORM_PREFIX}crc64_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}crc64.cpp "\t${PLATFORM_PREFIX}qplc_crc64};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}crc64.cpp "}\n") + + # + # Write xor_checksum function table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}xor_checksum.cpp "#include \"qplc_api.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}xor_checksum.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}xor_checksum.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}xor_checksum.cpp "xor_checksum_table_t ${PLATFORM_PREFIX}xor_checksum_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}xor_checksum.cpp "\t${PLATFORM_PREFIX}qplc_xor_checksum_8u};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}xor_checksum.cpp "}\n") + + # + # Write deflate functions table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}deflate.cpp "#include \"deflate_slow_icf.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "#include \"deflate_hash_table.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "#include \"deflate_histogram.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "deflate_table_t ${PLATFORM_PREFIX}deflate_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "\t reinterpret_cast(&${PLATFORM_PREFIX}slow_deflate_icf_body),\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "\t reinterpret_cast(&${PLATFORM_PREFIX}deflate_histogram_reset),\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "\t reinterpret_cast(&${PLATFORM_PREFIX}deflate_hash_table_reset)};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate.cpp "}\n") + + # + # Write deflate fix functions table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}deflate_fix.cpp "#include \"deflate_slow.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate_fix.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate_fix.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate_fix.cpp "deflate_fix_table_t ${PLATFORM_PREFIX}deflate_fix_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate_fix.cpp "\t reinterpret_cast(&${PLATFORM_PREFIX}slow_deflate_body)};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}deflate_fix.cpp "}\n") + + # + # Write setup_dictionary functions table + # + file(WRITE ${directory}/${PLATFORM_PREFIX}setup_dictionary.cpp "#include \"deflate_slow_utils.h\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}setup_dictionary.cpp "#include \"dispatcher/dispatcher.hpp\"\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}setup_dictionary.cpp "namespace qpl::core_sw::dispatcher\n{\n") + file(APPEND ${directory}/${PLATFORM_PREFIX}setup_dictionary.cpp "setup_dictionary_table_t ${PLATFORM_PREFIX}setup_dictionary_table = {\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}setup_dictionary.cpp "\t reinterpret_cast(&${PLATFORM_PREFIX}setup_dictionary)};\n") + + file(APPEND ${directory}/${PLATFORM_PREFIX}setup_dictionary.cpp "}\n") + + endforeach() +endfunction() -# check nasm compiler -include(CheckLanguage) -check_language(ASM_NASM) -if(NOT CMAKE_ASM_NASM_COMPILER) - message(FATAL_ERROR "Please install NASM from 'https://www.nasm.us/' because NASM compiler can not be found!") -endif() -# [SUBDIR]isal enable_language(ASM_NASM) set(ISAL_C_SRC ${QPL_SRC_DIR}/isal/igzip/adler32_base.c @@ -106,11 +512,6 @@ set_target_properties(isal PROPERTIES CXX_STANDARD 11 C_STANDARD 99) -target_compile_options(isal PRIVATE - "$<$:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}>" - "$<$:>" - "$<$:>") - # AS_FEATURE_LEVEL=10 means "Check SIMD capabilities of the target system at runtime and use up to AVX512 if available". # HAVE_KNOWS_AVX512 means rely on AVX512 being available on the target system. target_compile_options(isal_asm PRIVATE "-I${QPL_SRC_DIR}/isal/include/" @@ -163,15 +564,7 @@ foreach(PLATFORM_ID IN LISTS PLATFORMS_LIST) PUBLIC $ PRIVATE $) - set_target_properties(qplcore_${PLATFORM_ID} PROPERTIES - $<$:C_STANDARD 17>) - - target_compile_options(qplcore_${PLATFORM_ID} - PRIVATE ${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS} - PRIVATE "$<$:>" - PRIVATE "$<$:-O3;-D_FORTIFY_SOURCE=2>") - - # Set specific compiler options and/or definitions based on a platform + # Set specific compiler options and/or definitions based on a platform if (${PLATFORM_ID} MATCHES "avx512") target_compile_definitions(qplcore_${PLATFORM_ID} PRIVATE PLATFORM=2) target_compile_options(qplcore_${PLATFORM_ID} PRIVATE -march=skylake-avx512) @@ -220,10 +613,7 @@ set_target_properties(qplcore_sw_dispatcher PROPERTIES CXX_STANDARD 17) target_compile_definitions(qplcore_sw_dispatcher PUBLIC -DQPL_LIB) target_compile_options(qplcore_sw_dispatcher - PRIVATE $<$:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}; - ${QPL_LINUX_TOOLCHAIN_DYNAMIC_LIBRARY_FLAGS}; - $<$:-O3;-D_FORTIFY_SOURCE=2>> - PRIVATE $<$:${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}>) + PRIVATE ${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}) # [SUBDIR]core-iaa file(GLOB HW_PATH_SRC ${QPL_SRC_DIR}/core-iaa/sources/aecs/*.c @@ -248,14 +638,6 @@ target_include_directories(core_iaa PRIVATE $ # own_checkers.h PRIVATE $) -set_target_properties(core_iaa PROPERTIES - $<$:C_STANDARD 17> - CXX_STANDARD 17) - -target_compile_options(core_iaa - PRIVATE $<$:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}; - $<$:-O3;-D_FORTIFY_SOURCE=2>>) - target_compile_features(core_iaa PRIVATE c_std_11) target_compile_definitions(core_iaa PRIVATE QPL_BADARG_CHECK @@ -285,10 +667,7 @@ set_property(GLOBAL APPEND PROPERTY QPL_LIB_DEPS $) target_compile_options(middle_layer_lib - PRIVATE $<$:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}; - ${QPL_LINUX_TOOLCHAIN_DYNAMIC_LIBRARY_FLAGS}; - $<$:-O3;-D_FORTIFY_SOURCE=2>> - PRIVATE $<$:${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}>) + PRIVATE ${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}) target_compile_definitions(middle_layer_lib PUBLIC QPL_VERSION="${QPL_VERSION}" @@ -323,15 +702,8 @@ target_include_directories(_qpl PRIVATE $ PRIVATE $) -set_target_properties(_qpl PROPERTIES - $<$:C_STANDARD 17> - CXX_STANDARD 17) - target_compile_options(_qpl - PRIVATE $<$:${QPL_LINUX_TOOLCHAIN_REQUIRED_FLAGS}; - ${QPL_LINUX_TOOLCHAIN_DYNAMIC_LIBRARY_FLAGS}; - $<$:-O3;-D_FORTIFY_SOURCE=2>> - PRIVATE $<$:${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}>) + PRIVATE ${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}) target_compile_definitions(_qpl PRIVATE -DQPL_LIB diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index d469b359d1a9..42bfb48db708 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -38,7 +38,7 @@ rm -f CMakeCache.txt # To check it, find and delete them. grep -o -P '"contrib/[^"]+"' ../.gitmodules | - grep -v -P 'llvm-project|abseil-cpp|qpl|grpc|corrosion' | + grep -v -P 'llvm-project|abseil-cpp|grpc|corrosion' | xargs -I@ find ../@ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | xargs rm From 7dda3b2353b22639a0304219c01beed16407c6eb Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 17 Nov 2023 10:11:15 +0000 Subject: [PATCH 136/274] Review comments --- src/Backups/BackupCoordinationFileInfos.cpp | 65 ++++++++++++--------- src/Backups/BackupFileInfo.h | 7 ++- src/Backups/BackupImpl.cpp | 29 ++++++--- src/Backups/BackupImpl.h | 7 +-- 4 files changed, 65 insertions(+), 43 deletions(-) diff --git a/src/Backups/BackupCoordinationFileInfos.cpp b/src/Backups/BackupCoordinationFileInfos.cpp index b17b755b966c..3280064a5d70 100644 --- a/src/Backups/BackupCoordinationFileInfos.cpp +++ b/src/Backups/BackupCoordinationFileInfos.cpp @@ -50,6 +50,21 @@ BackupFileInfo BackupCoordinationFileInfos::getFileInfoByDataFileIndex(size_t da return *(file_infos_for_all_hosts[data_file_index]); } +namespace +{ + +/// copy all the file infos that are shared between reference target and source +void copyFileInfoToReference(const BackupFileInfo & target, BackupFileInfo & reference) +{ + reference.size = target.size; + reference.checksum = target.checksum; + reference.base_size = target.base_size; + reference.base_checksum = target.base_checksum; + reference.encrypted_by_disk = target.encrypted_by_disk; +} + +} + void BackupCoordinationFileInfos::prepare() const { if (prepared) @@ -78,11 +93,24 @@ void BackupCoordinationFileInfos::prepare() const num_files = 0; total_size_of_files = 0; - if (plain_backup) + std::vector unresolved_references; + std::unordered_map file_name_to_info; + + const auto handle_unresolved_references = [&](const auto & try_resolve_reference) { - std::vector unresolved_references; - std::unordered_map file_name_to_info; + for (auto * reference : unresolved_references) + { + if (!try_resolve_reference(*reference)) + throw DB::Exception( + ErrorCodes::LOGICAL_ERROR, + "Couldn't resolve reference {} with target {}", + reference->file_name, + reference->reference_target); + } + }; + if (plain_backup) + { const auto try_resolve_reference = [&](BackupFileInfo & reference) { auto it = file_name_to_info.find(reference.reference_target); @@ -91,10 +119,9 @@ void BackupCoordinationFileInfos::prepare() const return false; auto & target_info = it->second; - target_info->reference_sources.push_back(reference.file_name); - reference.size = target_info->size; + target_info->data_file_copies.push_back(reference.file_name); + copyFileInfoToReference(*target_info, reference); total_size_of_files += reference.size; - reference.checksum = target_info->checksum; return true; }; @@ -118,23 +145,12 @@ void BackupCoordinationFileInfos::prepare() const } } - for (auto * reference : unresolved_references) - { - if (!try_resolve_reference(*reference)) - throw DB::Exception( - ErrorCodes::LOGICAL_ERROR, - "Couldn't resolve reference {} with target {}", - reference->file_name, - reference->reference_target); - } + handle_unresolved_references(try_resolve_reference); num_files = file_infos_for_all_hosts.size(); } else { - std::vector unresolved_references; - std::unordered_map file_name_to_info; - const auto try_resolve_reference = [&](BackupFileInfo & reference) { auto it = file_name_to_info.find(reference.reference_target); @@ -143,8 +159,7 @@ void BackupCoordinationFileInfos::prepare() const return false; auto & target_info = it->second; - reference.size = target_info->size; - reference.checksum = target_info->checksum; + copyFileInfoToReference(*target_info, reference); reference.data_file_name = target_info->data_file_name; reference.data_file_index = target_info->data_file_index; return true; @@ -195,15 +210,7 @@ void BackupCoordinationFileInfos::prepare() const file_name_to_info.emplace(info.file_name, &info); } - for (auto * reference : unresolved_references) - { - if (!try_resolve_reference(*reference)) - throw DB::Exception( - ErrorCodes::LOGICAL_ERROR, - "Couldn't resolve reference {} with target {}", - reference->file_name, - reference->reference_target); - } + handle_unresolved_references(try_resolve_reference); num_files = file_infos_for_all_hosts.size(); } diff --git a/src/Backups/BackupFileInfo.h b/src/Backups/BackupFileInfo.h index 42bda3aa6ed8..009fee091e04 100644 --- a/src/Backups/BackupFileInfo.h +++ b/src/Backups/BackupFileInfo.h @@ -42,9 +42,10 @@ struct BackupFileInfo /// Set if this file is just a reference to another file String reference_target; - /// List of files that are referencing this file - /// Used for plain backup which needs to resolve all references - Strings reference_sources; + /// (While writing a backup) if this list is not empty then after writing + /// `data_file_name` it should be copied to this list of destinations too. + /// This is used for plain backups. + Strings data_file_copies; struct LessByFileName { diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index 56c30fab5c2b..61984d58889a 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -24,6 +24,8 @@ #include #include +#include + namespace ProfileEvents { @@ -452,7 +454,6 @@ void BackupImpl::readBackupMetadata() size_of_entries = 0; const auto * contents = config_root->getNodeByPath("contents"); - std::vector> reference_files; for (const Poco::XML::Node * child = contents->firstChild(); child; child = child->nextSibling()) { if (child->nodeName() == "file") @@ -913,15 +914,20 @@ void BackupImpl::writeFile(const BackupFileInfo & info, BackupEntryPtr entry) /// NOTE: `mutex` must be unlocked during copying otherwise writing will be in one thread maximum and hence slow. - if (use_archive) + const auto write_info_to_archive = [&](const auto & file_name) { - LOG_TRACE(log, "Writing backup for file {} from {}: data file #{}, adding to archive", info.data_file_name, src_file_desc, info.data_file_index); - auto out = archive_writer->writeFile(info.data_file_name); + auto out = archive_writer->writeFile(file_name); auto read_buffer = entry->getReadBuffer(writer->getReadSettings()); if (info.base_size != 0) read_buffer->seek(info.base_size, SEEK_SET); copyData(*read_buffer, *out); out->finalize(); + }; + + if (use_archive) + { + LOG_TRACE(log, "Writing backup for file {} from {}: data file #{}, adding to archive", info.data_file_name, src_file_desc, info.data_file_index); + write_info_to_archive(info.data_file_name); } else if (src_disk && from_immutable_file) { @@ -935,12 +941,21 @@ void BackupImpl::writeFile(const BackupFileInfo & info, BackupEntryPtr entry) writer->copyDataToFile(info.data_file_name, create_read_buffer, info.base_size, info.size - info.base_size); } - if (!deduplicate_files) + std::function copy_file_inside_backup; + if (use_archive) + { + copy_file_inside_backup = write_info_to_archive; + } + else { - for (const auto & reference : info.reference_sources) - writer->copyFile(reference, info.data_file_name, info.size - info.base_size); + copy_file_inside_backup = [&](const auto & data_file_copy) + { + writer->copyFile(data_file_copy, info.data_file_name, info.size - info.base_size); + }; } + std::ranges::for_each(info.data_file_copies, copy_file_inside_backup); + { std::lock_guard lock{mutex}; ++num_entries; diff --git a/src/Backups/BackupImpl.h b/src/Backups/BackupImpl.h index a4ab3d84d0c0..6070db79aa62 100644 --- a/src/Backups/BackupImpl.h +++ b/src/Backups/BackupImpl.h @@ -72,11 +72,14 @@ class BackupImpl : public IBackup Strings listFiles(const String & directory, bool recursive) const override; bool hasFiles(const String & directory) const override; bool fileExists(const String & file_name) const override; + bool fileExists(const SizeAndChecksum & size_and_checksum) const override; UInt64 getFileSize(const String & file_name) const override; UInt128 getFileChecksum(const String & file_name) const override; SizeAndChecksum getFileSizeAndChecksum(const String & file_name) const override; std::unique_ptr readFile(const String & file_name) const override; + std::unique_ptr readFile(const SizeAndChecksum & size_and_checksum) const override; size_t copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const override; + size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const override; void writeFile(const BackupFileInfo & info, BackupEntryPtr entry) override; void finalizeWriting() override; bool supportsWritingInMultipleThreads() const override { return !use_archive; } @@ -111,10 +114,6 @@ class BackupImpl : public IBackup std::unique_ptr readFileImpl(const SizeAndChecksum & size_and_checksum, bool read_encrypted) const; - bool fileExists(const SizeAndChecksum & size_and_checksum) const override; - std::unique_ptr readFile(const SizeAndChecksum & size_and_checksum) const override; - size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const override; - BackupInfo backup_info; const String backup_name_for_logging; const bool use_archive; From 9bcedf376436bb066db639a7ab7325cd71fb3b73 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 17 Nov 2023 10:27:19 +0000 Subject: [PATCH 137/274] Cleanup --- src/Backups/BackupIO_Default.cpp | 2 -- src/Common/ZooKeeper/ZooKeeper.cpp | 1 - src/Storages/StorageKeeperMap.cpp | 40 ++++++++++++++++-------------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/Backups/BackupIO_Default.cpp b/src/Backups/BackupIO_Default.cpp index 95f2c66b6b95..5ac522695ce2 100644 --- a/src/Backups/BackupIO_Default.cpp +++ b/src/Backups/BackupIO_Default.cpp @@ -91,6 +91,4 @@ void BackupWriterDefault::copyFileFromDisk(const String & path_in_backup, DiskPt copyDataToFile(path_in_backup, create_read_buffer, start_pos, length); } - - } diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 8a97362aa963..436a4e14f14b 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -385,7 +385,6 @@ void ZooKeeper::createAncestors(const std::string & path) size_t last_pos = path.rfind('/'); if (last_pos == std::string::npos || last_pos == 0) return; - std::string current_node = path.substr(0, last_pos); while (true) diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index 237b65c6a72a..e3c960529de7 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -763,28 +763,29 @@ void StorageKeeperMap::backupData(BackupEntriesCollector & backup_entries_collec auto post_collecting_task = [my_table_id = std::move(table_id), coordination, &backup_entries_collector, my_data_path_in_backup = data_path_in_backup, this] { auto path_with_data = coordination->getKeeperMapDataPath(zk_root_path); - if (path_with_data == my_data_path_in_backup) + if (path_with_data != my_data_path_in_backup) { - auto temp_disk = backup_entries_collector.getContext()->getGlobalTemporaryVolume()->getDisk(0); - auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef().max_compress_block_size; - - auto with_retries = std::make_shared - ( - &Poco::Logger::get(fmt::format("StorageKeeperMapBackup ({})", getStorageID().getNameForLogs())), - [&] { return getClient(); }, - WithRetries::KeeperSettings::fromContext(backup_entries_collector.getContext()), - [](WithRetries::FaultyKeeper &) {} - ); - - backup_entries_collector.addBackupEntries( - std::make_shared(this->zk_data_path, path_with_data, temp_disk, max_compress_block_size, std::move(with_retries)) - ->getBackupEntries()); + std::string source_path = fs::path(my_data_path_in_backup) / backup_data_filename; + std::string target_path = fs::path(path_with_data) / backup_data_filename; + backup_entries_collector.addBackupEntries({{source_path, std::make_shared(std::move(target_path))}}); return; } - std::string source_path = fs::path(my_data_path_in_backup) / backup_data_filename; - std::string target_path = fs::path(path_with_data) / backup_data_filename; - backup_entries_collector.addBackupEntries({{source_path, std::make_shared(std::move(target_path))}}); + auto temp_disk = backup_entries_collector.getContext()->getGlobalTemporaryVolume()->getDisk(0); + auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef().max_compress_block_size; + + auto with_retries = std::make_shared + ( + &Poco::Logger::get(fmt::format("StorageKeeperMapBackup ({})", getStorageID().getNameForLogs())), + [&] { return getClient(); }, + WithRetries::KeeperSettings::fromContext(backup_entries_collector.getContext()), + [](WithRetries::FaultyKeeper &) {} + ); + + backup_entries_collector.addBackupEntries( + std::make_shared( + this->zk_data_path, path_with_data, temp_disk, max_compress_block_size, std::move(with_retries)) + ->getBackupEntries()); }; backup_entries_collector.addPostTask(post_collecting_task); @@ -796,7 +797,8 @@ void StorageKeeperMap::restoreDataFromBackup(RestorerFromBackup & restorer, cons if (!backup->hasFiles(data_path_in_backup)) return; - auto table_id = toString(getStorageID().uuid); if (!restorer.getRestoreCoordination()->acquireInsertingDataForKeeperMap(zk_root_path, table_id)) + auto table_id = toString(getStorageID().uuid); + if (!restorer.getRestoreCoordination()->acquireInsertingDataForKeeperMap(zk_root_path, table_id)) { /// Other table is already restoring the data for this Keeper path. /// Tables defined on the same path share data From 4f441ec1319ac3f80a70f773f98ea99a80dfc407 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 11:32:27 +0100 Subject: [PATCH 138/274] Own CMake for Abseil --- contrib/abseil-cpp-cmake/CMakeLists.txt | 3437 ++++++++++++++++++++++- contrib/re2-cmake/CMakeLists.txt | 13 +- docker/packager/binary/build.sh | 2 +- 3 files changed, 3429 insertions(+), 23 deletions(-) diff --git a/contrib/abseil-cpp-cmake/CMakeLists.txt b/contrib/abseil-cpp-cmake/CMakeLists.txt index 2901daf32db6..e84b4d46c4a6 100644 --- a/contrib/abseil-cpp-cmake/CMakeLists.txt +++ b/contrib/abseil-cpp-cmake/CMakeLists.txt @@ -1,33 +1,3428 @@ set(ABSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp") +set(ABSL_COMMON_INCLUDE_DIRS "${ABSL_ROOT_DIR}") -set(ABSL_PROPAGATE_CXX_STD ON) -add_subdirectory("${ABSL_ROOT_DIR}" "${ClickHouse_BINARY_DIR}/contrib/abseil-cpp") +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -add_library(_abseil_swiss_tables INTERFACE) +function(absl_cc_library) + cmake_parse_arguments(ABSL_CC_LIB + "DISABLE_INSTALL;PUBLIC;TESTONLY" + "NAME" + "HDRS;SRCS;COPTS;DEFINES;LINKOPTS;DEPS" + ${ARGN} + ) + + set(_NAME "absl_${ABSL_CC_LIB_NAME}") + + # Check if this is a header-only library + set(ABSL_CC_SRCS "${ABSL_CC_LIB_SRCS}") + foreach(src_file IN LISTS ABSL_CC_SRCS) + if(${src_file} MATCHES ".*\\.(h|inc)") + list(REMOVE_ITEM ABSL_CC_SRCS "${src_file}") + endif() + endforeach() + + if(ABSL_CC_SRCS STREQUAL "") + set(ABSL_CC_LIB_IS_INTERFACE 1) + else() + set(ABSL_CC_LIB_IS_INTERFACE 0) + endif() + + if(NOT ABSL_CC_LIB_IS_INTERFACE) + add_library(${_NAME} "") + target_sources(${_NAME} PRIVATE ${ABSL_CC_LIB_SRCS} ${ABSL_CC_LIB_HDRS}) + target_link_libraries(${_NAME} + PUBLIC ${ABSL_CC_LIB_DEPS} + PRIVATE + ${ABSL_CC_LIB_LINKOPTS} + ${ABSL_DEFAULT_LINKOPTS} + ) + + target_include_directories(${_NAME} + PUBLIC "${ABSL_COMMON_INCLUDE_DIRS}") + target_compile_options(${_NAME} + PRIVATE ${ABSL_CC_LIB_COPTS}) + target_compile_definitions(${_NAME} PUBLIC ${ABSL_CC_LIB_DEFINES}) + + else() + # Generating header-only library + add_library(${_NAME} INTERFACE) + target_include_directories(${_NAME} + INTERFACE "${ABSL_COMMON_INCLUDE_DIRS}") + + target_link_libraries(${_NAME} + INTERFACE + ${ABSL_CC_LIB_DEPS} + ${ABSL_CC_LIB_LINKOPTS} + ${ABSL_DEFAULT_LINKOPTS} + ) + target_compile_definitions(${_NAME} INTERFACE ${ABSL_CC_LIB_DEFINES}) + + endif() + + add_library(absl::${ABSL_CC_LIB_NAME} ALIAS ${_NAME}) +endfunction() + + +set(DIR ${ABSL_ROOT_DIR}/absl/algorithm) -target_link_libraries(_abseil_swiss_tables INTERFACE - absl::flat_hash_map - absl::flat_hash_set +absl_cc_library( + NAME + algorithm + HDRS + "${DIR}/algorithm.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC ) -get_target_property(FLAT_HASH_MAP_INCLUDE_DIR absl::flat_hash_map INTERFACE_INCLUDE_DIRECTORIES) -target_include_directories (_abseil_swiss_tables SYSTEM BEFORE INTERFACE ${FLAT_HASH_MAP_INCLUDE_DIR}) +absl_cc_library( + NAME + algorithm_container + HDRS + "${DIR}/container.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::algorithm + absl::core_headers + absl::meta + PUBLIC +) -get_target_property(FLAT_HASH_SET_INCLUDE_DIR absl::flat_hash_set INTERFACE_INCLUDE_DIRECTORIES) -target_include_directories (_abseil_swiss_tables SYSTEM BEFORE INTERFACE ${FLAT_HASH_SET_INCLUDE_DIR}) +set(DIR ${ABSL_ROOT_DIR}/absl/base) -add_library(ch_contrib::abseil_swiss_tables ALIAS _abseil_swiss_tables) +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + atomic_hook + HDRS + "${DIR}/internal/atomic_hook.h" + DEPS + absl::config + absl::core_headers + COPTS + ${ABSL_DEFAULT_COPTS} +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + errno_saver + HDRS + "${DIR}/internal/errno_saver.h" + DEPS + absl::config + COPTS + ${ABSL_DEFAULT_COPTS} +) + +absl_cc_library( + NAME + log_severity + HDRS + "${DIR}/log_severity.h" + SRCS + "${DIR}/log_severity.cc" + DEPS + absl::config + absl::core_headers + COPTS + ${ABSL_DEFAULT_COPTS} +) + +absl_cc_library( + NAME + nullability + HDRS + "${DIR}/nullability.h" + SRCS + "${DIR}/internal/nullability_impl.h" + DEPS + absl::core_headers + absl::type_traits + COPTS + ${ABSL_DEFAULT_COPTS} +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + raw_logging_internal + HDRS + "${DIR}/internal/raw_logging.h" + SRCS + "${DIR}/internal/raw_logging.cc" + DEPS + absl::atomic_hook + absl::config + absl::core_headers + absl::errno_saver + absl::log_severity + COPTS + ${ABSL_DEFAULT_COPTS} +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + spinlock_wait + HDRS + "${DIR}/internal/spinlock_wait.h" + SRCS + "${DIR}/internal/spinlock_akaros.inc" + "${DIR}/internal/spinlock_linux.inc" + "${DIR}/internal/spinlock_posix.inc" + "${DIR}/internal/spinlock_wait.cc" + "${DIR}/internal/spinlock_win32.inc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::core_headers + absl::errno_saver +) + +absl_cc_library( + NAME + config + HDRS + "${DIR}/config.h" + "${DIR}/options.h" + "${DIR}/policy_checks.h" + COPTS + ${ABSL_DEFAULT_COPTS} + PUBLIC +) + +absl_cc_library( + NAME + dynamic_annotations + HDRS + "${DIR}/dynamic_annotations.h" + SRCS + "${DIR}/internal/dynamic_annotations.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +absl_cc_library( + NAME + core_headers + HDRS + "${DIR}/attributes.h" + "${DIR}/const_init.h" + "${DIR}/macros.h" + "${DIR}/optimization.h" + "${DIR}/port.h" + "${DIR}/thread_annotations.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + malloc_internal + HDRS + "${DIR}/internal/direct_mmap.h" + "${DIR}/internal/low_level_alloc.h" + SRCS + "${DIR}/internal/low_level_alloc.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::base_internal + absl::config + absl::core_headers + absl::dynamic_annotations + absl::raw_logging_internal + Threads::Threads +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + base_internal + HDRS + "${DIR}/internal/hide_ptr.h" + "${DIR}/internal/identity.h" + "${DIR}/internal/inline_variable.h" + "${DIR}/internal/invoke.h" + "${DIR}/internal/scheduling_mode.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::type_traits +) + +absl_cc_library( + NAME + base + HDRS + "${DIR}/call_once.h" + "${DIR}/casts.h" + "${DIR}/internal/cycleclock.h" + "${DIR}/internal/cycleclock_config.h" + "${DIR}/internal/low_level_scheduling.h" + "${DIR}/internal/per_thread_tls.h" + "${DIR}/internal/spinlock.h" + "${DIR}/internal/sysinfo.h" + "${DIR}/internal/thread_identity.h" + "${DIR}/internal/tsan_mutex_interface.h" + "${DIR}/internal/unscaledcycleclock.h" + "${DIR}/internal/unscaledcycleclock_config.h" + SRCS + "${DIR}/internal/cycleclock.cc" + "${DIR}/internal/spinlock.cc" + "${DIR}/internal/sysinfo.cc" + "${DIR}/internal/thread_identity.cc" + "${DIR}/internal/unscaledcycleclock.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::atomic_hook + absl::base_internal + absl::config + absl::core_headers + absl::dynamic_annotations + absl::log_severity + absl::raw_logging_internal + absl::spinlock_wait + absl::type_traits + Threads::Threads + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + throw_delegate + HDRS + "${DIR}/internal/throw_delegate.h" + SRCS + "${DIR}/internal/throw_delegate.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::raw_logging_internal +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + pretty_function + HDRS + "${DIR}/internal/pretty_function.h" + COPTS + ${ABSL_DEFAULT_COPTS} +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + endian + HDRS + "${DIR}/internal/endian.h" + "${DIR}/internal/unaligned_access.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::core_headers + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + scoped_set_env + SRCS + "${DIR}/internal/scoped_set_env.cc" + HDRS + "${DIR}/internal/scoped_set_env.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::raw_logging_internal +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + strerror + SRCS + "${DIR}/internal/strerror.cc" + HDRS + "${DIR}/internal/strerror.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::errno_saver +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + fast_type_id + HDRS + "${DIR}/internal/fast_type_id.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +absl_cc_library( + NAME + prefetch + HDRS + "${DIR}/prefetch.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers +) + +set(DIR ${ABSL_ROOT_DIR}/absl/cleanup) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cleanup_internal + HDRS + "${DIR}/internal/cleanup.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::core_headers + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + cleanup + HDRS + "${DIR}/cleanup.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::cleanup_internal + absl::config + absl::core_headers + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/container) + +absl_cc_library( + NAME + btree + HDRS + "${DIR}/btree_map.h" + "${DIR}/btree_set.h" + "${DIR}/internal/btree.h" + "${DIR}/internal/btree_container.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::container_common + absl::common_policy_traits + absl::compare + absl::compressed_tuple + absl::container_memory + absl::cord + absl::core_headers + absl::layout + absl::memory + absl::raw_logging_internal + absl::strings + absl::throw_delegate + absl::type_traits + absl::utility +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + compressed_tuple + HDRS + "${DIR}/internal/compressed_tuple.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + fixed_array + HDRS + "${DIR}/fixed_array.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::compressed_tuple + absl::algorithm + absl::config + absl::core_headers + absl::dynamic_annotations + absl::throw_delegate + absl::memory + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + inlined_vector_internal + HDRS + "${DIR}/internal/inlined_vector.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::compressed_tuple + absl::core_headers + absl::memory + absl::span + absl::type_traits + PUBLIC +) + +absl_cc_library( + NAME + inlined_vector + HDRS + "${DIR}/inlined_vector.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::algorithm + absl::core_headers + absl::inlined_vector_internal + absl::throw_delegate + absl::memory + absl::type_traits + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + counting_allocator + HDRS + "${DIR}/internal/counting_allocator.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config +) + +absl_cc_library( + NAME + flat_hash_map + HDRS + "${DIR}/flat_hash_map.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::container_memory + absl::core_headers + absl::hash_function_defaults + absl::raw_hash_map + absl::algorithm_container + absl::memory + PUBLIC +) + +absl_cc_library( + NAME + flat_hash_set + HDRS + "${DIR}/flat_hash_set.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::container_memory + absl::hash_function_defaults + absl::raw_hash_set + absl::algorithm_container + absl::core_headers + absl::memory + PUBLIC +) + +absl_cc_library( + NAME + node_hash_map + HDRS + "${DIR}/node_hash_map.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::container_memory + absl::core_headers + absl::hash_function_defaults + absl::node_slot_policy + absl::raw_hash_map + absl::algorithm_container + absl::memory + PUBLIC +) + +absl_cc_library( + NAME + node_hash_set + HDRS + "${DIR}/node_hash_set.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::hash_function_defaults + absl::node_slot_policy + absl::raw_hash_set + absl::algorithm_container + absl::memory + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + container_memory + HDRS + "${DIR}/internal/container_memory.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::memory + absl::type_traits + absl::utility + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + hash_function_defaults + HDRS + "${DIR}/internal/hash_function_defaults.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::cord + absl::hash + absl::strings + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + hash_policy_traits + HDRS + "${DIR}/internal/hash_policy_traits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::common_policy_traits + absl::meta + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + common_policy_traits + HDRS + "${DIR}/internal/common_policy_traits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::meta + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + hashtablez_sampler + HDRS + "${DIR}/internal/hashtablez_sampler.h" + SRCS + "${DIR}/internal/hashtablez_sampler.cc" + "${DIR}/internal/hashtablez_sampler_force_weak_definition.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::exponential_biased + absl::raw_logging_internal + absl::sample_recorder + absl::synchronization + absl::time +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + hashtable_debug + HDRS + "${DIR}/internal/hashtable_debug.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::hashtable_debug_hooks +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + hashtable_debug_hooks + HDRS + "${DIR}/internal/hashtable_debug_hooks.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + node_slot_policy + HDRS + "${DIR}/internal/node_slot_policy.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + raw_hash_map + HDRS + "${DIR}/internal/raw_hash_map.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::container_memory + absl::raw_hash_set + absl::throw_delegate + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + container_common + HDRS + "${DIR}/internal/common.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + raw_hash_set + HDRS + "${DIR}/internal/raw_hash_set.h" + SRCS + "${DIR}/internal/raw_hash_set.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bits + absl::compressed_tuple + absl::config + absl::container_common + absl::container_memory + absl::core_headers + absl::dynamic_annotations + absl::endian + absl::hash + absl::hash_policy_traits + absl::hashtable_debug_hooks + absl::hashtablez_sampler + absl::memory + absl::meta + absl::optional + absl::prefetch + absl::raw_logging_internal + absl::utility + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + layout + HDRS + "${DIR}/internal/layout.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::meta + absl::strings + absl::span + absl::utility + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/crc) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + crc_cpu_detect + HDRS + "${DIR}/internal/cpu_detect.h" + SRCS + "${DIR}/internal/cpu_detect.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + crc_internal + HDRS + "${DIR}/internal/crc.h" + "${DIR}/internal/crc32_x86_arm_combined_simd.h" + SRCS + "${DIR}/internal/crc.cc" + "${DIR}/internal/crc_internal.h" + "${DIR}/internal/crc_x86_arm_combined.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::crc_cpu_detect + absl::config + absl::core_headers + absl::endian + absl::prefetch + absl::raw_logging_internal + absl::memory + absl::bits +) + +absl_cc_library( + NAME + crc32c + HDRS + "${DIR}/crc32c.h" + "${DIR}/internal/crc32c.h" + "${DIR}/internal/crc_memcpy.h" + SRCS + "${DIR}/crc32c.cc" + "${DIR}/internal/crc32c_inline.h" + "${DIR}/internal/crc_memcpy_fallback.cc" + "${DIR}/internal/crc_memcpy_x86_arm_combined.cc" + "${DIR}/internal/crc_non_temporal_memcpy.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::crc_cpu_detect + absl::crc_internal + absl::non_temporal_memcpy + absl::config + absl::core_headers + absl::endian + absl::prefetch + absl::str_format + absl::strings +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + non_temporal_arm_intrinsics + HDRS + "${DIR}/internal/non_temporal_arm_intrinsics.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + non_temporal_memcpy + HDRS + "${DIR}/internal/non_temporal_memcpy.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::non_temporal_arm_intrinsics + absl::config + absl::core_headers +) + +absl_cc_library( + NAME + crc_cord_state + HDRS + "${DIR}/internal/crc_cord_state.h" + SRCS + "${DIR}/internal/crc_cord_state.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::crc32c + absl::config + absl::strings +) + +set(DIR ${ABSL_ROOT_DIR}/absl/debugging) + +absl_cc_library( + NAME + stacktrace + HDRS + "${DIR}/stacktrace.h" + "${DIR}/internal/stacktrace_aarch64-inl.inc" + "${DIR}/internal/stacktrace_arm-inl.inc" + "${DIR}/internal/stacktrace_config.h" + "${DIR}/internal/stacktrace_emscripten-inl.inc" + "${DIR}/internal/stacktrace_generic-inl.inc" + "${DIR}/internal/stacktrace_powerpc-inl.inc" + "${DIR}/internal/stacktrace_riscv-inl.inc" + "${DIR}/internal/stacktrace_unimplemented-inl.inc" + "${DIR}/internal/stacktrace_win32-inl.inc" + "${DIR}/internal/stacktrace_x86-inl.inc" + SRCS + "${DIR}/stacktrace.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::debugging_internal + absl::config + absl::core_headers + absl::dynamic_annotations + absl::raw_logging_internal + PUBLIC +) + +absl_cc_library( + NAME + symbolize + HDRS + "${DIR}/symbolize.h" + "${DIR}/internal/symbolize.h" + SRCS + "${DIR}/symbolize.cc" + "${DIR}/symbolize_darwin.inc" + "${DIR}/symbolize_elf.inc" + "${DIR}/symbolize_emscripten.inc" + "${DIR}/symbolize_unimplemented.inc" + "${DIR}/symbolize_win32.inc" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::debugging_internal + absl::demangle_internal + absl::base + absl::config + absl::core_headers + absl::dynamic_annotations + absl::malloc_internal + absl::raw_logging_internal + absl::strings + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + examine_stack + HDRS + "${DIR}/internal/examine_stack.h" + SRCS + "${DIR}/internal/examine_stack.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::stacktrace + absl::symbolize + absl::config + absl::core_headers + absl::raw_logging_internal +) + +absl_cc_library( + NAME + failure_signal_handler + HDRS + "${DIR}/failure_signal_handler.h" + SRCS + "${DIR}/failure_signal_handler.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::examine_stack + absl::stacktrace + absl::base + absl::config + absl::core_headers + absl::raw_logging_internal + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + debugging_internal + HDRS + "${DIR}/internal/address_is_readable.h" + "${DIR}/internal/elf_mem_image.h" + "${DIR}/internal/vdso_support.h" + SRCS + "${DIR}/internal/address_is_readable.cc" + "${DIR}/internal/elf_mem_image.cc" + "${DIR}/internal/vdso_support.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::config + absl::dynamic_annotations + absl::errno_saver + absl::raw_logging_internal +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + demangle_internal + HDRS + "${DIR}/internal/demangle.h" + SRCS + "${DIR}/internal/demangle.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::core_headers + PUBLIC +) + +absl_cc_library( + NAME + leak_check + HDRS + "${DIR}/leak_check.h" + SRCS + "${DIR}/leak_check.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + PUBLIC +) + +# component target +absl_cc_library( + NAME + debugging + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::stacktrace + absl::leak_check + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/flags) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_path_util + HDRS + "${DIR}/internal/path_util.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::strings + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_program_name + SRCS + "${DIR}/internal/program_name.cc" + HDRS + "${DIR}/internal/program_name.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::flags_path_util + absl::strings + absl::synchronization + PUBLIC +) + +absl_cc_library( + NAME + flags_config + SRCS + "${DIR}/usage_config.cc" + HDRS + "${DIR}/config.h" + "${DIR}/usage_config.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::flags_path_util + absl::flags_program_name + absl::core_headers + absl::strings + absl::synchronization +) + +absl_cc_library( + NAME + flags_marshalling + SRCS + "${DIR}/marshalling.cc" + HDRS + "${DIR}/marshalling.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log_severity + absl::int128 + absl::optional + absl::strings + absl::str_format +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_commandlineflag_internal + SRCS + "${DIR}/internal/commandlineflag.cc" + HDRS + "${DIR}/internal/commandlineflag.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::dynamic_annotations + absl::fast_type_id +) + +absl_cc_library( + NAME + flags_commandlineflag + SRCS + "${DIR}/commandlineflag.cc" + HDRS + "${DIR}/commandlineflag.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::fast_type_id + absl::flags_commandlineflag_internal + absl::optional + absl::strings +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_private_handle_accessor + SRCS + "${DIR}/internal/private_handle_accessor.cc" + HDRS + "${DIR}/internal/private_handle_accessor.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::flags_commandlineflag + absl::flags_commandlineflag_internal + absl::strings +) + +absl_cc_library( + NAME + flags_reflection + SRCS + "${DIR}/reflection.cc" + HDRS + "${DIR}/reflection.h" + "${DIR}/internal/registry.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::flags_commandlineflag + absl::flags_private_handle_accessor + absl::flags_config + absl::strings + absl::synchronization + absl::flat_hash_map +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_internal + SRCS + "${DIR}/internal/flag.cc" + HDRS + "${DIR}/internal/flag.h" + "${DIR}/internal/sequence_lock.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::base + absl::config + absl::flags_commandlineflag + absl::flags_commandlineflag_internal + absl::flags_config + absl::flags_marshalling + absl::synchronization + absl::meta + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + flags + SRCS + "${DIR}/flag.cc" + HDRS + "${DIR}/declare.h" + "${DIR}/flag.h" + "${DIR}/internal/flag_msvc.inc" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::flags_commandlineflag + absl::flags_config + absl::flags_internal + absl::flags_reflection + absl::base + absl::core_headers + absl::strings +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_usage_internal + SRCS + "${DIR}/internal/usage.cc" + HDRS + "${DIR}/internal/usage.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::flags_config + absl::flags + absl::flags_commandlineflag + absl::flags_internal + absl::flags_path_util + absl::flags_private_handle_accessor + absl::flags_program_name + absl::flags_reflection + absl::strings + absl::synchronization +) + +absl_cc_library( + NAME + flags_usage + SRCS + "${DIR}/usage.cc" + HDRS + "${DIR}/usage.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::flags_usage_internal + absl::raw_logging_internal + absl::strings + absl::synchronization +) + +absl_cc_library( + NAME + flags_parse + SRCS + "${DIR}/parse.cc" + HDRS + "${DIR}/internal/parse.h" + "${DIR}/parse.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::algorithm_container + absl::config + absl::core_headers + absl::flags_config + absl::flags + absl::flags_commandlineflag + absl::flags_commandlineflag_internal + absl::flags_internal + absl::flags_private_handle_accessor + absl::flags_program_name + absl::flags_reflection + absl::flags_usage + absl::strings + absl::synchronization +) + +set(DIR ${ABSL_ROOT_DIR}/absl/functional) + +absl_cc_library( + NAME + any_invocable + SRCS + "${DIR}/internal/any_invocable.h" + HDRS + "${DIR}/any_invocable.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::config + absl::core_headers + absl::type_traits + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + bind_front + SRCS + "${DIR}/internal/front_binder.h" + HDRS + "${DIR}/bind_front.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::compressed_tuple + PUBLIC +) + +absl_cc_library( + NAME + function_ref + SRCS + "${DIR}/internal/function_ref.h" + HDRS + "${DIR}/function_ref.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::core_headers + absl::any_invocable + absl::meta + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/hash) + +absl_cc_library( + NAME + hash + HDRS + "${DIR}/hash.h" + SRCS + "${DIR}/internal/hash.cc" + "${DIR}/internal/hash.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bits + absl::city + absl::config + absl::core_headers + absl::endian + absl::fixed_array + absl::function_ref + absl::meta + absl::int128 + absl::strings + absl::optional + absl::variant + absl::utility + absl::low_level_hash + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + city + HDRS + "${DIR}/internal/city.h" + SRCS + "${DIR}/internal/city.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::endian +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + low_level_hash + HDRS + "${DIR}/internal/low_level_hash.h" + SRCS + "${DIR}/internal/low_level_hash.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::endian + absl::int128 + absl::prefetch +) + +set(DIR ${ABSL_ROOT_DIR}/absl/log) + +# Internal targets +absl_cc_library( + NAME + log_internal_check_impl + SRCS + HDRS + "${DIR}/internal/check_impl.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::core_headers + absl::log_internal_check_op + absl::log_internal_conditions + absl::log_internal_message + absl::log_internal_strip +) + +absl_cc_library( + NAME + log_internal_check_op + SRCS + "${DIR}/internal/check_op.cc" + HDRS + "${DIR}/internal/check_op.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log_internal_nullguard + absl::log_internal_nullstream + absl::log_internal_strip + absl::strings +) + +absl_cc_library( + NAME + log_internal_conditions + SRCS + "${DIR}/internal/conditions.cc" + HDRS + "${DIR}/internal/conditions.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::base + absl::config + absl::core_headers + absl::log_internal_voidify +) + +absl_cc_library( + NAME + log_internal_config + SRCS + HDRS + "${DIR}/internal/config.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers +) + +absl_cc_library( + NAME + log_internal_flags + SRCS + HDRS + "${DIR}/internal/flags.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::flags +) + +absl_cc_library( + NAME + log_internal_format + SRCS + "${DIR}/internal/log_format.cc" + HDRS + "${DIR}/internal/log_format.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log_internal_append_truncated + absl::log_internal_config + absl::log_internal_globals + absl::log_severity + absl::strings + absl::str_format + absl::time + absl::span +) + +absl_cc_library( + NAME + log_internal_globals + SRCS + "${DIR}/internal/globals.cc" + HDRS + "${DIR}/internal/globals.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log_severity + absl::raw_logging_internal + absl::strings + absl::time +) + +absl_cc_library( + NAME + log_internal_log_impl + SRCS + HDRS + "${DIR}/internal/log_impl.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::log_internal_conditions + absl::log_internal_message + absl::log_internal_strip +) + +absl_cc_library( + NAME + log_internal_proto + SRCS + "${DIR}/internal/proto.cc" + HDRS + "${DIR}/internal/proto.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::base + absl::config + absl::core_headers + absl::strings + absl::span +) + +absl_cc_library( + NAME + log_internal_message + SRCS + "${DIR}/internal/log_message.cc" + HDRS + "${DIR}/internal/log_message.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::base + absl::config + absl::core_headers + absl::errno_saver + absl::inlined_vector + absl::examine_stack + absl::log_internal_append_truncated + absl::log_internal_format + absl::log_internal_globals + absl::log_internal_proto + absl::log_internal_log_sink_set + absl::log_internal_nullguard + absl::log_globals + absl::log_entry + absl::log_severity + absl::log_sink + absl::log_sink_registry + absl::memory + absl::raw_logging_internal + absl::strings + absl::strerror + absl::time + absl::span +) + +absl_cc_library( + NAME + log_internal_log_sink_set + SRCS + "${DIR}/internal/log_sink_set.cc" + HDRS + "${DIR}/internal/log_sink_set.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + $<$:-llog> + DEPS + absl::base + absl::cleanup + absl::config + absl::core_headers + absl::log_internal_config + absl::log_internal_globals + absl::log_globals + absl::log_entry + absl::log_severity + absl::log_sink + absl::raw_logging_internal + absl::synchronization + absl::span + absl::strings +) + +absl_cc_library( + NAME + log_internal_nullguard + SRCS + "${DIR}/internal/nullguard.cc" + HDRS + "${DIR}/internal/nullguard.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers +) + +absl_cc_library( + NAME + log_internal_nullstream + SRCS + HDRS + "${DIR}/internal/nullstream.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log_severity + absl::strings +) + +absl_cc_library( + NAME + log_internal_strip + SRCS + HDRS + "${DIR}/internal/strip.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::log_internal_message + absl::log_internal_nullstream + absl::log_severity +) + +absl_cc_library( + NAME + log_internal_voidify + SRCS + HDRS + "${DIR}/internal/voidify.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +absl_cc_library( + NAME + log_internal_append_truncated + SRCS + HDRS + "${DIR}/internal/append_truncated.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::strings + absl::span +) + +# Public targets +absl_cc_library( + NAME + absl_check + SRCS + HDRS + "${DIR}/absl_check.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::log_internal_check_impl + PUBLIC +) + +absl_cc_library( + NAME + absl_log + SRCS + HDRS + "${DIR}/absl_log.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::log_internal_log_impl + PUBLIC +) + +absl_cc_library( + NAME + check + SRCS + HDRS + "${DIR}/check.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::log_internal_check_impl + absl::core_headers + absl::log_internal_check_op + absl::log_internal_conditions + absl::log_internal_message + absl::log_internal_strip + PUBLIC +) + +absl_cc_library( + NAME + die_if_null + SRCS + "${DIR}/die_if_null.cc" + HDRS + "${DIR}/die_if_null.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log + absl::strings + PUBLIC +) + +absl_cc_library( + NAME + log_flags + SRCS + "${DIR}/flags.cc" + HDRS + "${DIR}/flags.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log_globals + absl::log_severity + absl::log_internal_config + absl::log_internal_flags + absl::flags + absl::flags_marshalling + absl::strings + PUBLIC +) + +absl_cc_library( + NAME + log_globals + SRCS + "${DIR}/globals.cc" + HDRS + "${DIR}/globals.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::atomic_hook + absl::config + absl::core_headers + absl::hash + absl::log_severity + absl::raw_logging_internal + absl::strings +) + +absl_cc_library( + NAME + log_initialize + SRCS + "${DIR}/initialize.cc" + HDRS + "${DIR}/initialize.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::log_globals + absl::log_internal_globals + absl::time + PUBLIC +) + +absl_cc_library( + NAME + log + SRCS + HDRS + "${DIR}/log.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::log_internal_log_impl + PUBLIC +) + +absl_cc_library( + NAME + log_entry + SRCS + "${DIR}/log_entry.cc" + HDRS + "${DIR}/log_entry.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log_internal_config + absl::log_severity + absl::span + absl::strings + absl::time + PUBLIC +) + +absl_cc_library( + NAME + log_sink + SRCS + "${DIR}/log_sink.cc" + HDRS + "${DIR}/log_sink.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::log_entry + PUBLIC +) + +absl_cc_library( + NAME + log_sink_registry + SRCS + HDRS + "${DIR}/log_sink_registry.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::log_sink + absl::log_internal_log_sink_set + PUBLIC +) + +absl_cc_library( + NAME + log_streamer + SRCS + HDRS + "${DIR}/log_streamer.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::absl_log + absl::log_severity + absl::optional + absl::strings + absl::strings_internal + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + log_internal_structured + HDRS + "${DIR}/internal/structured.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::log_internal_message + absl::strings +) + +absl_cc_library( + NAME + log_structured + HDRS + "${DIR}/structured.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::log_internal_structured + absl::strings + PUBLIC +) + +absl_cc_library( + NAME + log_internal_fnmatch + SRCS + "${DIR}/internal/fnmatch.cc" + HDRS + "${DIR}/internal/fnmatch.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::strings +) + +set(DIR ${ABSL_ROOT_DIR}/absl/memory) + +absl_cc_library( + NAME + memory + HDRS + "${DIR}/memory.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::meta + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/meta) + +absl_cc_library( + NAME + type_traits + HDRS + "${DIR}/type_traits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + PUBLIC +) + +# component target +absl_cc_library( + NAME + meta + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::type_traits + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/numeric) + +absl_cc_library( + NAME + bits + HDRS + "${DIR}/bits.h" + "${DIR}/internal/bits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + PUBLIC +) + +absl_cc_library( + NAME + int128 + HDRS + "${DIR}/int128.h" + SRCS + "${DIR}/int128.cc" + "${DIR}/int128_have_intrinsic.inc" + "${DIR}/int128_no_intrinsic.inc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::bits + PUBLIC +) + +# component target +absl_cc_library( + NAME + numeric + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::int128 + PUBLIC +) + +absl_cc_library( + NAME + numeric_representation + HDRS + "${DIR}/internal/representation.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +absl_cc_library( + NAME + sample_recorder + HDRS + "${DIR}/internal/sample_recorder.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::synchronization +) + +set(DIR ${ABSL_ROOT_DIR}/absl/profiling) + +absl_cc_library( + NAME + exponential_biased + SRCS + "${DIR}/internal/exponential_biased.cc" + HDRS + "${DIR}/internal/exponential_biased.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers +) + +absl_cc_library( + NAME + periodic_sampler + SRCS + "${DIR}/internal/periodic_sampler.cc" + HDRS + "${DIR}/internal/periodic_sampler.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::exponential_biased +) + +set(DIR ${ABSL_ROOT_DIR}/absl/random) + +absl_cc_library( + NAME + random_random + HDRS + "${DIR}/random.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_distributions + absl::random_internal_nonsecure_base + absl::random_internal_pcg_engine + absl::random_internal_pool_urbg + absl::random_internal_randen_engine + absl::random_seed_sequences +) + +absl_cc_library( + NAME + random_bit_gen_ref + HDRS + "${DIR}/bit_gen_ref.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::core_headers + absl::random_internal_distribution_caller + absl::random_internal_fast_uniform_bits + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_mock_helpers + HDRS + "${DIR}/internal/mock_helpers.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::fast_type_id + absl::optional +) + +absl_cc_library( + NAME + random_distributions + SRCS + "${DIR}/discrete_distribution.cc" + "${DIR}/gaussian_distribution.cc" + HDRS + "${DIR}/bernoulli_distribution.h" + "${DIR}/beta_distribution.h" + "${DIR}/discrete_distribution.h" + "${DIR}/distributions.h" + "${DIR}/exponential_distribution.h" + "${DIR}/gaussian_distribution.h" + "${DIR}/log_uniform_int_distribution.h" + "${DIR}/poisson_distribution.h" + "${DIR}/uniform_int_distribution.h" + "${DIR}/uniform_real_distribution.h" + "${DIR}/zipf_distribution.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::base_internal + absl::config + absl::core_headers + absl::random_internal_generate_real + absl::random_internal_distribution_caller + absl::random_internal_fast_uniform_bits + absl::random_internal_fastmath + absl::random_internal_iostream_state_saver + absl::random_internal_traits + absl::random_internal_uniform_helper + absl::random_internal_wide_multiply + absl::strings + absl::type_traits +) + +absl_cc_library( + NAME + random_seed_gen_exception + SRCS + "${DIR}/seed_gen_exception.cc" + HDRS + "${DIR}/seed_gen_exception.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +absl_cc_library( + NAME + random_seed_sequences + SRCS + "${DIR}/seed_sequences.cc" + HDRS + "${DIR}/seed_sequences.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::inlined_vector + absl::random_internal_pool_urbg + absl::random_internal_salted_seed_seq + absl::random_internal_seed_material + absl::random_seed_gen_exception + absl::span +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_traits + HDRS + "${DIR}/internal/traits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_distribution_caller + HDRS + "${DIR}/internal/distribution_caller.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::utility + absl::fast_type_id +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_fast_uniform_bits + HDRS + "${DIR}/internal/fast_uniform_bits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_seed_material + SRCS + "${DIR}/internal/seed_material.cc" + HDRS + "${DIR}/internal/seed_material.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::core_headers + absl::optional + absl::random_internal_fast_uniform_bits + absl::raw_logging_internal + absl::span + absl::strings +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_pool_urbg + SRCS + "${DIR}/internal/pool_urbg.cc" + HDRS + "${DIR}/internal/pool_urbg.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::base + absl::config + absl::core_headers + absl::endian + absl::random_internal_randen + absl::random_internal_seed_material + absl::random_internal_traits + absl::random_seed_gen_exception + absl::raw_logging_internal + absl::span +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_salted_seed_seq + HDRS + "${DIR}/internal/salted_seed_seq.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::inlined_vector + absl::optional + absl::span + absl::random_internal_seed_material + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_iostream_state_saver + HDRS + "${DIR}/internal/iostream_state_saver.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::int128 + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_generate_real + HDRS + "${DIR}/internal/generate_real.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::bits + absl::random_internal_fastmath + absl::random_internal_traits + absl::type_traits +) -set(ABSL_FORMAT_SRC - ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/arg.cc - ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/bind.cc - ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/extension.cc - ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/float_conversion.cc - ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/output.cc - ${ABSL_ROOT_DIR}/absl/strings/internal/str_format/parser.cc +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_wide_multiply + HDRS + "${DIR}/internal/wide_multiply.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::bits + absl::config + absl::int128 ) -add_library(_abseil_str_format ${ABSL_FORMAT_SRC}) -target_include_directories(_abseil_str_format PUBLIC ${ABSL_ROOT_DIR}) +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_fastmath + HDRS + "${DIR}/internal/fastmath.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::bits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_nonsecure_base + HDRS + "${DIR}/internal/nonsecure_base.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::core_headers + absl::inlined_vector + absl::random_internal_pool_urbg + absl::random_internal_salted_seed_seq + absl::random_internal_seed_material + absl::span + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_pcg_engine + HDRS + "${DIR}/internal/pcg_engine.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::int128 + absl::random_internal_fastmath + absl::random_internal_iostream_state_saver + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_randen_engine + HDRS + "${DIR}/internal/randen_engine.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::endian + absl::random_internal_iostream_state_saver + absl::random_internal_randen + absl::raw_logging_internal + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_platform + HDRS + "${DIR}/internal/randen_traits.h" + "${DIR}/internal/platform.h" + SRCS + "${DIR}/internal/randen_round_keys.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_randen + SRCS + "${DIR}/internal/randen.cc" + HDRS + "${DIR}/internal/randen.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_platform + absl::random_internal_randen_hwaes + absl::random_internal_randen_slow +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_randen_slow + SRCS + "${DIR}/internal/randen_slow.cc" + HDRS + "${DIR}/internal/randen_slow.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_platform + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_randen_hwaes + SRCS + "${DIR}/internal/randen_detect.cc" + HDRS + "${DIR}/internal/randen_detect.h" + "${DIR}/internal/randen_hwaes.h" + COPTS + ${ABSL_DEFAULT_COPTS} + ${ABSL_RANDOM_RANDEN_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_platform + absl::random_internal_randen_hwaes_impl + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_randen_hwaes_impl + SRCS + "${DIR}/internal/randen_hwaes.cc" + "${DIR}/internal/randen_hwaes.h" + COPTS + ${ABSL_DEFAULT_COPTS} + ${ABSL_RANDOM_RANDEN_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_platform + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_uniform_helper + HDRS + "${DIR}/internal/uniform_helper.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::random_internal_traits + absl::type_traits +) + +set(DIR ${ABSL_ROOT_DIR}/absl/status) + +absl_cc_library( + NAME + status + HDRS + "${DIR}/status.h" + SRCS + "${DIR}/internal/status_internal.h" + "${DIR}/status.cc" + "${DIR}/status_payload_printer.h" + "${DIR}/status_payload_printer.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEFINES + "$<$:_LINUX_SOURCE_COMPAT>" + DEPS + absl::atomic_hook + absl::config + absl::cord + absl::core_headers + absl::function_ref + absl::inlined_vector + absl::memory + absl::optional + absl::raw_logging_internal + absl::span + absl::stacktrace + absl::strerror + absl::str_format + absl::strings + absl::symbolize + PUBLIC +) + +absl_cc_library( + NAME + statusor + HDRS + "${DIR}/statusor.h" + SRCS + "${DIR}/statusor.cc" + "${DIR}/internal/statusor_internal.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::core_headers + absl::raw_logging_internal + absl::status + absl::strings + absl::type_traits + absl::utility + absl::variant + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/strings) + +absl_cc_library( + NAME + string_view + HDRS + "${DIR}/string_view.h" + SRCS + "${DIR}/string_view.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::core_headers + absl::throw_delegate + PUBLIC +) + +absl_cc_library( + NAME + strings + HDRS + "${DIR}/ascii.h" + "${DIR}/charconv.h" + "${DIR}/escaping.h" + "${DIR}/has_absl_stringify.h" + "${DIR}/internal/damerau_levenshtein_distance.h" + "${DIR}/internal/string_constant.h" + "${DIR}/internal/has_absl_stringify.h" + "${DIR}/match.h" + "${DIR}/numbers.h" + "${DIR}/str_cat.h" + "${DIR}/str_join.h" + "${DIR}/str_replace.h" + "${DIR}/str_split.h" + "${DIR}/strip.h" + "${DIR}/substitute.h" + SRCS + "${DIR}/ascii.cc" + "${DIR}/charconv.cc" + "${DIR}/escaping.cc" + "${DIR}/internal/charconv_bigint.cc" + "${DIR}/internal/charconv_bigint.h" + "${DIR}/internal/charconv_parse.cc" + "${DIR}/internal/charconv_parse.h" + "${DIR}/internal/damerau_levenshtein_distance.cc" + "${DIR}/internal/memutil.cc" + "${DIR}/internal/memutil.h" + "${DIR}/internal/stringify_sink.h" + "${DIR}/internal/stringify_sink.cc" + "${DIR}/internal/stl_type_traits.h" + "${DIR}/internal/str_join_internal.h" + "${DIR}/internal/str_split_internal.h" + "${DIR}/match.cc" + "${DIR}/numbers.cc" + "${DIR}/str_cat.cc" + "${DIR}/str_replace.cc" + "${DIR}/str_split.cc" + "${DIR}/substitute.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::string_view + absl::strings_internal + absl::base + absl::bits + absl::charset + absl::config + absl::core_headers + absl::endian + absl::int128 + absl::memory + absl::raw_logging_internal + absl::throw_delegate + absl::type_traits + PUBLIC +) + +absl_cc_library( + NAME + charset + HDRS + charset.h + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::string_view + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + strings_internal + HDRS + "${DIR}/internal/escaping.cc" + "${DIR}/internal/escaping.h" + "${DIR}/internal/ostringstream.h" + "${DIR}/internal/resize_uninitialized.h" + "${DIR}/internal/utf8.h" + SRCS + "${DIR}/internal/ostringstream.cc" + "${DIR}/internal/utf8.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::endian + absl::raw_logging_internal + absl::type_traits +) + +absl_cc_library( + NAME + str_format + HDRS + "${DIR}/str_format.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::str_format_internal + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + str_format_internal + HDRS + "${DIR}/internal/str_format/arg.h" + "${DIR}/internal/str_format/bind.h" + "${DIR}/internal/str_format/checker.h" + "${DIR}/internal/str_format/constexpr_parser.h" + "${DIR}/internal/str_format/extension.h" + "${DIR}/internal/str_format/float_conversion.h" + "${DIR}/internal/str_format/output.h" + "${DIR}/internal/str_format/parser.h" + SRCS + "${DIR}/internal/str_format/arg.cc" + "${DIR}/internal/str_format/bind.cc" + "${DIR}/internal/str_format/extension.cc" + "${DIR}/internal/str_format/float_conversion.cc" + "${DIR}/internal/str_format/output.cc" + "${DIR}/internal/str_format/parser.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bits + absl::strings + absl::config + absl::core_headers + absl::inlined_vector + absl::numeric_representation + absl::type_traits + absl::utility + absl::int128 + absl::span +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cord_internal + HDRS + "${DIR}/internal/cord_data_edge.h" + "${DIR}/internal/cord_internal.h" + "${DIR}/internal/cord_rep_btree.h" + "${DIR}/internal/cord_rep_btree_navigator.h" + "${DIR}/internal/cord_rep_btree_reader.h" + "${DIR}/internal/cord_rep_crc.h" + "${DIR}/internal/cord_rep_consume.h" + "${DIR}/internal/cord_rep_flat.h" + SRCS + "${DIR}/internal/cord_internal.cc" + "${DIR}/internal/cord_rep_btree.cc" + "${DIR}/internal/cord_rep_btree_navigator.cc" + "${DIR}/internal/cord_rep_btree_reader.cc" + "${DIR}/internal/cord_rep_crc.cc" + "${DIR}/internal/cord_rep_consume.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::compressed_tuple + absl::config + absl::container_memory + absl::core_headers + absl::crc_cord_state + absl::endian + absl::inlined_vector + absl::layout + absl::raw_logging_internal + absl::strings + absl::throw_delegate + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_update_tracker + HDRS + "${DIR}/internal/cordz_update_tracker.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_functions + HDRS + "${DIR}/internal/cordz_functions.h" + SRCS + "${DIR}/internal/cordz_functions.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::exponential_biased + absl::raw_logging_internal +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_statistics + HDRS + "${DIR}/internal/cordz_statistics.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::cordz_update_tracker + absl::synchronization +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_handle + HDRS + "${DIR}/internal/cordz_handle.h" + SRCS + "${DIR}/internal/cordz_handle.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::raw_logging_internal + absl::synchronization +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_info + HDRS + "${DIR}/internal/cordz_info.h" + SRCS + "${DIR}/internal/cordz_info.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::cord_internal + absl::cordz_functions + absl::cordz_handle + absl::cordz_statistics + absl::cordz_update_tracker + absl::core_headers + absl::inlined_vector + absl::span + absl::raw_logging_internal + absl::stacktrace + absl::synchronization + absl::time +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_sample_token + HDRS + "${DIR}/internal/cordz_sample_token.h" + SRCS + "${DIR}/internal/cordz_sample_token.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::cordz_handle + absl::cordz_info +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_update_scope + HDRS + "${DIR}/internal/cordz_update_scope.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::cord_internal + absl::cordz_info + absl::cordz_update_tracker + absl::core_headers +) + +absl_cc_library( + NAME + cord + HDRS + "${DIR}/cord.h" + "${DIR}/cord_buffer.h" + SRCS + "${DIR}/cord.cc" + "${DIR}/cord_analysis.cc" + "${DIR}/cord_analysis.h" + "${DIR}/cord_buffer.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::cord_internal + absl::cordz_functions + absl::cordz_info + absl::cordz_update_scope + absl::cordz_update_tracker + absl::core_headers + absl::crc32c + absl::crc_cord_state + absl::endian + absl::function_ref + absl::inlined_vector + absl::optional + absl::raw_logging_internal + absl::span + absl::strings + absl::type_traits + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/synchronization) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + graphcycles_internal + HDRS + "${DIR}/internal/graphcycles.h" + SRCS + "${DIR}/internal/graphcycles.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::base_internal + absl::config + absl::core_headers + absl::malloc_internal + absl::raw_logging_internal +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + kernel_timeout_internal + HDRS + "${DIR}/internal/kernel_timeout.h" + SRCS + "${DIR}/internal/kernel_timeout.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::core_headers + absl::raw_logging_internal + absl::time +) + +absl_cc_library( + NAME + synchronization + HDRS + "${DIR}/barrier.h" + "${DIR}/blocking_counter.h" + "${DIR}/internal/create_thread_identity.h" + "${DIR}/internal/futex.h" + "${DIR}/internal/futex_waiter.h" + "${DIR}/internal/per_thread_sem.h" + "${DIR}/internal/pthread_waiter.h" + "${DIR}/internal/sem_waiter.h" + "${DIR}/internal/stdcpp_waiter.h" + "${DIR}/internal/waiter.h" + "${DIR}/internal/waiter_base.h" + "${DIR}/internal/win32_waiter.h" + "${DIR}/mutex.h" + "${DIR}/notification.h" + SRCS + "${DIR}/barrier.cc" + "${DIR}/blocking_counter.cc" + "${DIR}/internal/create_thread_identity.cc" + "${DIR}/internal/futex_waiter.cc" + "${DIR}/internal/per_thread_sem.cc" + "${DIR}/internal/pthread_waiter.cc" + "${DIR}/internal/sem_waiter.cc" + "${DIR}/internal/stdcpp_waiter.cc" + "${DIR}/internal/waiter_base.cc" + "${DIR}/internal/win32_waiter.cc" + "${DIR}/notification.cc" + "${DIR}/mutex.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::graphcycles_internal + absl::kernel_timeout_internal + absl::atomic_hook + absl::base + absl::base_internal + absl::config + absl::core_headers + absl::dynamic_annotations + absl::malloc_internal + absl::raw_logging_internal + absl::stacktrace + absl::symbolize + absl::time + Threads::Threads + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/time) + +absl_cc_library( + NAME + time + HDRS + "${DIR}/civil_time.h" + "${DIR}/clock.h" + "${DIR}/time.h" + SRCS + "${DIR}/civil_time.cc" + "${DIR}/clock.cc" + "${DIR}/duration.cc" + "${DIR}/format.cc" + "${DIR}/internal/get_current_time_chrono.inc" + "${DIR}/internal/get_current_time_posix.inc" + "${DIR}/time.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::civil_time + absl::core_headers + absl::int128 + absl::raw_logging_internal + absl::strings + absl::time_zone + PUBLIC +) + +absl_cc_library( + NAME + civil_time + HDRS + "${DIR}/internal/cctz/include/cctz/civil_time.h" + "${DIR}/internal/cctz/include/cctz/civil_time_detail.h" + SRCS + "${DIR}/internal/cctz/src/civil_time_detail.cc" + COPTS + ${ABSL_DEFAULT_COPTS} +) + +absl_cc_library( + NAME + time_zone + HDRS + "${DIR}/internal/cctz/include/cctz/time_zone.h" + "${DIR}/internal/cctz/include/cctz/zone_info_source.h" + SRCS + "${DIR}/internal/cctz/src/time_zone_fixed.cc" + "${DIR}/internal/cctz/src/time_zone_fixed.h" + "${DIR}/internal/cctz/src/time_zone_format.cc" + "${DIR}/internal/cctz/src/time_zone_if.cc" + "${DIR}/internal/cctz/src/time_zone_if.h" + "${DIR}/internal/cctz/src/time_zone_impl.cc" + "${DIR}/internal/cctz/src/time_zone_impl.h" + "${DIR}/internal/cctz/src/time_zone_info.cc" + "${DIR}/internal/cctz/src/time_zone_info.h" + "${DIR}/internal/cctz/src/time_zone_libc.cc" + "${DIR}/internal/cctz/src/time_zone_libc.h" + "${DIR}/internal/cctz/src/time_zone_lookup.cc" + "${DIR}/internal/cctz/src/time_zone_posix.cc" + "${DIR}/internal/cctz/src/time_zone_posix.h" + "${DIR}/internal/cctz/src/tzfile.h" + "${DIR}/internal/cctz/src/zone_info_source.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + Threads::Threads + $<$:-Wl,-framework,CoreFoundation> +) + +set(DIR ${ABSL_ROOT_DIR}/absl/types) + +absl_cc_library( + NAME + any + HDRS + "${DIR}/any.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bad_any_cast + absl::config + absl::core_headers + absl::fast_type_id + absl::type_traits + absl::utility + PUBLIC +) -add_library(ch_contrib::abseil_str_format ALIAS _abseil_str_format) +absl_cc_library( + NAME + bad_any_cast + HDRS + "${DIR}/bad_any_cast.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bad_any_cast_impl + absl::config + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + bad_any_cast_impl + SRCS + "${DIR}/bad_any_cast.h" + "${DIR}/bad_any_cast.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::raw_logging_internal +) + +absl_cc_library( + NAME + span + HDRS + "${DIR}/span.h" + SRCS + "${DIR}/internal/span.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::algorithm + absl::core_headers + absl::throw_delegate + absl::type_traits + PUBLIC +) + +absl_cc_library( + NAME + optional + HDRS + "${DIR}/optional.h" + SRCS + "${DIR}/internal/optional.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bad_optional_access + absl::base_internal + absl::config + absl::core_headers + absl::memory + absl::type_traits + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + bad_optional_access + HDRS + "${DIR}/bad_optional_access.h" + SRCS + "${DIR}/bad_optional_access.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::raw_logging_internal + PUBLIC +) + +absl_cc_library( + NAME + bad_variant_access + HDRS + "${DIR}/bad_variant_access.h" + SRCS + "${DIR}/bad_variant_access.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::raw_logging_internal + PUBLIC +) + +absl_cc_library( + NAME + variant + HDRS + "${DIR}/variant.h" + SRCS + "${DIR}/internal/variant.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bad_variant_access + absl::base_internal + absl::config + absl::core_headers + absl::type_traits + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + compare + HDRS + "${DIR}/compare.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::type_traits + PUBLIC +) + +set(DIR ${ABSL_ROOT_DIR}/absl/utility) + +absl_cc_library( + NAME + utility + HDRS + "${DIR}/utility.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::config + absl::type_traits + PUBLIC +) + +absl_cc_library( + NAME + if_constexpr + HDRS + "${DIR}/internal/if_constexpr.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + + +add_library(_abseil_swiss_tables INTERFACE) +target_include_directories (_abseil_swiss_tables SYSTEM BEFORE INTERFACE ${ABSL_ROOT_DIR}) +add_library(ch_contrib::abseil_swiss_tables ALIAS _abseil_swiss_tables) diff --git a/contrib/re2-cmake/CMakeLists.txt b/contrib/re2-cmake/CMakeLists.txt index e72b5e1fca89..f773bc65a69f 100644 --- a/contrib/re2-cmake/CMakeLists.txt +++ b/contrib/re2-cmake/CMakeLists.txt @@ -27,6 +27,17 @@ set(RE2_SOURCES add_library(_re2 ${RE2_SOURCES}) target_include_directories(_re2 PUBLIC "${SRC_DIR}") -target_link_libraries(_re2 ch_contrib::abseil_str_format) +target_link_libraries(_re2 PRIVATE + absl::base + absl::core_headers + absl::fixed_array + absl::flat_hash_map + absl::flat_hash_set + absl::inlined_vector + absl::strings + absl::str_format + absl::synchronization + absl::optional + absl::span) add_library(ch_contrib::re2 ALIAS _re2) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 42bfb48db708..c764a4dd8c1f 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -38,7 +38,7 @@ rm -f CMakeCache.txt # To check it, find and delete them. grep -o -P '"contrib/[^"]+"' ../.gitmodules | - grep -v -P 'llvm-project|abseil-cpp|grpc|corrosion' | + grep -v -P 'llvm-project|grpc|corrosion' | xargs -I@ find ../@ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | xargs rm From 0548eefbb784de32ec7e409ca9150482f3a171a9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 11:35:48 +0100 Subject: [PATCH 139/274] Simpler CMake --- docker/packager/binary/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index c764a4dd8c1f..37440fe8202a 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -38,7 +38,7 @@ rm -f CMakeCache.txt # To check it, find and delete them. grep -o -P '"contrib/[^"]+"' ../.gitmodules | - grep -v -P 'llvm-project|grpc|corrosion' | + grep -v -P 'llvm-project|google-protobuf|grpc|corrosion' | xargs -I@ find ../@ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | xargs rm From 74a8f3191dc96f3ac46187b556b5d127b7ae6030 Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:38:56 +0100 Subject: [PATCH 140/274] Update HTTPSession.cpp --- base/poco/Net/src/HTTPSession.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index d303a4c654b7..8f951b3102cd 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -113,13 +113,12 @@ void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco } catch (NetException &) { - #ifndef NDEBUG + throw; +#else // mute exceptions in release // just in case when changing settings on socket is not allowed // however it should be OK for timeouts -#else - throw; #endif } } From d035e5f44fc8820a0aef96b30106ed9e78bb6408 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Thu, 16 Nov 2023 17:41:05 +0100 Subject: [PATCH 141/274] Improve diagnostics in test 02908_many_requests_to_system_replicas --- .../02908_many_requests_to_system_replicas.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh index 70dc5f4d8c41..c620fcf4beaf 100755 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh @@ -14,12 +14,13 @@ echo "Creating $NUM_TABLES tables" function init_table() { + set -e i=$1 - curl $CLICKHOUSE_URL --silent --fail --data "CREATE TABLE test_02908_r1_$i (a UInt64) ENGINE=ReplicatedMergeTree('/02908/{database}/test_$i', 'r1') ORDER BY tuple()" - curl $CLICKHOUSE_URL --silent --fail --data "CREATE TABLE test_02908_r2_$i (a UInt64) ENGINE=ReplicatedMergeTree('/02908/{database}/test_$i', 'r2') ORDER BY tuple()" - curl $CLICKHOUSE_URL --silent --fail --data "CREATE TABLE test_02908_r3_$i (a UInt64) ENGINE=ReplicatedMergeTree('/02908/{database}/test_$i', 'r3') ORDER BY tuple()" + curl $CLICKHOUSE_URL --silent --fail --show-error --data "CREATE TABLE test_02908_r1_$i (a UInt64) ENGINE=ReplicatedMergeTree('/02908/{database}/test_$i', 'r1') ORDER BY tuple()" 2>&1 + curl $CLICKHOUSE_URL --silent --fail --show-error --data "CREATE TABLE test_02908_r2_$i (a UInt64) ENGINE=ReplicatedMergeTree('/02908/{database}/test_$i', 'r2') ORDER BY tuple()" 2>&1 + curl $CLICKHOUSE_URL --silent --fail --show-error --data "CREATE TABLE test_02908_r3_$i (a UInt64) ENGINE=ReplicatedMergeTree('/02908/{database}/test_$i', 'r3') ORDER BY tuple()" 2>&1 - curl $CLICKHOUSE_URL --silent --fail --data "INSERT INTO test_02908_r1_$i SELECT rand64() FROM numbers(5);" + curl $CLICKHOUSE_URL --silent --fail --show-error --data "INSERT INTO test_02908_r1_$i SELECT rand64() FROM numbers(5);" 2>&1 } export init_table; @@ -36,13 +37,13 @@ echo "Making making $CONCURRENCY requests to system.replicas" for i in `seq 1 $CONCURRENCY`; do - curl $CLICKHOUSE_URL --silent --fail --data "SELECT * FROM system.replicas WHERE database=currentDatabase() FORMAT Null;" & + curl $CLICKHOUSE_URL --silent --fail --show-error --data "SELECT * FROM system.replicas WHERE database=currentDatabase() FORMAT Null;" 2>&1 || echo "query $i failed" & done echo "Query system.replicas while waiting for other concurrent requests to finish" # lost_part_count column is read from ZooKeeper -curl $CLICKHOUSE_URL --silent --fail --data "SELECT sum(lost_part_count) FROM system.replicas WHERE database=currentDatabase();"; +curl $CLICKHOUSE_URL --silent --fail --show-error --data "SELECT sum(lost_part_count) FROM system.replicas WHERE database=currentDatabase();" 2>&1; # is_leader column is filled without ZooKeeper -curl $CLICKHOUSE_URL --silent --fail --data "SELECT sum(is_leader) FROM system.replicas WHERE database=currentDatabase();"; +curl $CLICKHOUSE_URL --silent --fail --show-error --data "SELECT sum(is_leader) FROM system.replicas WHERE database=currentDatabase();" 2>&1; wait; From 6c3793acb0162454a6327cf1c4e9c097d924fc78 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Fri, 17 Nov 2023 12:03:00 +0100 Subject: [PATCH 142/274] Allow delegate disk to handle retries for createDirectories --- src/Disks/DiskEncrypted.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Disks/DiskEncrypted.h b/src/Disks/DiskEncrypted.h index 8b4461a8dee1..eddf08f4e91c 100644 --- a/src/Disks/DiskEncrypted.h +++ b/src/Disks/DiskEncrypted.h @@ -60,9 +60,9 @@ class DiskEncrypted : public IDisk void createDirectories(const String & path) override { - auto tx = createEncryptedTransaction(); - tx->createDirectories(path); - tx->commit(); + auto wrapped_path = wrappedPath(path); + /// Delegate disk can have retry logic for recursive directory creation. Let it handle it. + delegate->createDirectories(wrapped_path); } void clearDirectory(const String & path) override From cdfc0e07e5161dc2d381c0ad74205c0514e94850 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 17 Nov 2023 11:06:03 +0000 Subject: [PATCH 143/274] Update version_date.tsv and changelogs after v23.10.4.25-stable --- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v23.10.4.25-stable.md | 28 +++++++++++++++++++++++++++ utils/list-versions/version_date.tsv | 2 ++ 5 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 docs/changelogs/v23.10.4.25-stable.md diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 1f4fd39bc261..63de9f6c4629 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-keeper" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 41be7e611a3b..d26bb344fef1 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 0ff6ae2e2270..53a368181213 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -30,7 +30,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v23.10.4.25-stable.md b/docs/changelogs/v23.10.4.25-stable.md new file mode 100644 index 000000000000..2d7d2a38e047 --- /dev/null +++ b/docs/changelogs/v23.10.4.25-stable.md @@ -0,0 +1,28 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.10.4.25-stable (330fd687d41) FIXME as compared to v23.10.3.5-stable (b2ba7637a41) + +#### Build/Testing/Packaging Improvement +* Backported in [#56633](https://github.com/ClickHouse/ClickHouse/issues/56633): In [#54043](https://github.com/ClickHouse/ClickHouse/issues/54043) the setup plan started to appear in the logs. It should be only in the `runner_get_all_tests.log` only. As well, send the failed infrastructure event to CI db. [#56214](https://github.com/ClickHouse/ClickHouse/pull/56214) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#56737](https://github.com/ClickHouse/ClickHouse/issues/56737): Do not fetch changed submodules in the builder container. [#56689](https://github.com/ClickHouse/ClickHouse/pull/56689) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Select from system tables when table based on table function. [#55540](https://github.com/ClickHouse/ClickHouse/pull/55540) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Fix restore from backup with `flatten_nested` and `data_type_default_nullable` [#56306](https://github.com/ClickHouse/ClickHouse/pull/56306) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix segfault during Kerberos initialization [#56401](https://github.com/ClickHouse/ClickHouse/pull/56401) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix: RabbitMQ OpenSSL dynamic loading issue [#56703](https://github.com/ClickHouse/ClickHouse/pull/56703) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix crash in GCD codec in case when zeros present in data [#56704](https://github.com/ClickHouse/ClickHouse/pull/56704) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix crash in FPC codec [#56795](https://github.com/ClickHouse/ClickHouse/pull/56795) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Rewrite jobs to use callable workflow [#56385](https://github.com/ClickHouse/ClickHouse/pull/56385) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Continue rewriting workflows to reusable tests [#56501](https://github.com/ClickHouse/ClickHouse/pull/56501) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Better exception messages [#56854](https://github.com/ClickHouse/ClickHouse/pull/56854) ([Antonio Andelic](https://github.com/antonio2368)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 0f2684cd91d9..ace5546aadb5 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v23.10.4.25-stable 2023-11-17 v23.10.3.5-stable 2023-11-10 v23.10.2.13-stable 2023-11-08 v23.10.1.1976-stable 2023-11-02 @@ -31,6 +32,7 @@ v23.4.4.16-stable 2023-06-17 v23.4.3.48-stable 2023-06-12 v23.4.2.11-stable 2023-05-02 v23.4.1.1943-stable 2023-04-27 +v23.3.17.13-lts 2023-11-17 v23.3.16.7-lts 2023-11-08 v23.3.15.29-lts 2023-10-31 v23.3.14.78-lts 2023-10-18 From c695405c85454e5f0266342fdbe02ab000a670cb Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 17 Nov 2023 11:08:05 +0000 Subject: [PATCH 144/274] Update version_date.tsv and changelogs after v23.3.17.13-lts --- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v23.3.17.13-lts.md | 23 +++++++++++++++++++++++ utils/list-versions/version_date.tsv | 2 ++ 5 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 docs/changelogs/v23.3.17.13-lts.md diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 1f4fd39bc261..63de9f6c4629 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-keeper" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 41be7e611a3b..d26bb344fef1 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 0ff6ae2e2270..53a368181213 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -30,7 +30,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v23.3.17.13-lts.md b/docs/changelogs/v23.3.17.13-lts.md new file mode 100644 index 000000000000..a18ced70d467 --- /dev/null +++ b/docs/changelogs/v23.3.17.13-lts.md @@ -0,0 +1,23 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.3.17.13-lts (e867d59020f) FIXME as compared to v23.3.16.7-lts (fb4125cc92a) + +#### Build/Testing/Packaging Improvement +* Backported in [#56731](https://github.com/ClickHouse/ClickHouse/issues/56731): Do not fetch changed submodules in the builder container. [#56689](https://github.com/ClickHouse/ClickHouse/pull/56689) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Fix segfault during Kerberos initialization [#56401](https://github.com/ClickHouse/ClickHouse/pull/56401) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix crash in FPC codec [#56795](https://github.com/ClickHouse/ClickHouse/pull/56795) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Rewrite jobs to use callable workflow [#56385](https://github.com/ClickHouse/ClickHouse/pull/56385) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Continue rewriting workflows to reusable tests [#56501](https://github.com/ClickHouse/ClickHouse/pull/56501) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Better exception messages [#56854](https://github.com/ClickHouse/ClickHouse/pull/56854) ([Antonio Andelic](https://github.com/antonio2368)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 0f2684cd91d9..ace5546aadb5 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v23.10.4.25-stable 2023-11-17 v23.10.3.5-stable 2023-11-10 v23.10.2.13-stable 2023-11-08 v23.10.1.1976-stable 2023-11-02 @@ -31,6 +32,7 @@ v23.4.4.16-stable 2023-06-17 v23.4.3.48-stable 2023-06-12 v23.4.2.11-stable 2023-05-02 v23.4.1.1943-stable 2023-04-27 +v23.3.17.13-lts 2023-11-17 v23.3.16.7-lts 2023-11-08 v23.3.15.29-lts 2023-10-31 v23.3.14.78-lts 2023-10-18 From 09b3f5e541935a1dd15ae1cab056ae680cec46ed Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 17 Nov 2023 11:10:10 +0000 Subject: [PATCH 145/274] Update version_date.tsv and changelogs after v23.8.7.24-lts --- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v23.8.7.24-lts.md | 31 ++++++++++++++++++++++++++++ utils/list-versions/version_date.tsv | 3 +++ 5 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 docs/changelogs/v23.8.7.24-lts.md diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 1f4fd39bc261..63de9f6c4629 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-keeper" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 41be7e611a3b..d26bb344fef1 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 0ff6ae2e2270..53a368181213 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -30,7 +30,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v23.8.7.24-lts.md b/docs/changelogs/v23.8.7.24-lts.md new file mode 100644 index 000000000000..37862c173154 --- /dev/null +++ b/docs/changelogs/v23.8.7.24-lts.md @@ -0,0 +1,31 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.8.7.24-lts (812b95e14ba) FIXME as compared to v23.8.6.16-lts (077df679bed) + +#### Build/Testing/Packaging Improvement +* Backported in [#56733](https://github.com/ClickHouse/ClickHouse/issues/56733): Do not fetch changed submodules in the builder container. [#56689](https://github.com/ClickHouse/ClickHouse/pull/56689) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Select from system tables when table based on table function. [#55540](https://github.com/ClickHouse/ClickHouse/pull/55540) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Fix incomplete query result for UNION in view() function. [#56274](https://github.com/ClickHouse/ClickHouse/pull/56274) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in case of adding a column with type Object(JSON) [#56307](https://github.com/ClickHouse/ClickHouse/pull/56307) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix segfault during Kerberos initialization [#56401](https://github.com/ClickHouse/ClickHouse/pull/56401) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix: RabbitMQ OpenSSL dynamic loading issue [#56703](https://github.com/ClickHouse/ClickHouse/pull/56703) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix crash in FPC codec [#56795](https://github.com/ClickHouse/ClickHouse/pull/56795) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NO CL CATEGORY + +* Backported in [#56601](https://github.com/ClickHouse/ClickHouse/issues/56601):. [#56598](https://github.com/ClickHouse/ClickHouse/pull/56598) ([Maksim Kita](https://github.com/kitaisreal)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Rewrite jobs to use callable workflow [#56385](https://github.com/ClickHouse/ClickHouse/pull/56385) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Continue rewriting workflows to reusable tests [#56501](https://github.com/ClickHouse/ClickHouse/pull/56501) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Better exception messages [#56854](https://github.com/ClickHouse/ClickHouse/pull/56854) ([Antonio Andelic](https://github.com/antonio2368)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 0f2684cd91d9..c0b9fd229646 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v23.10.4.25-stable 2023-11-17 v23.10.3.5-stable 2023-11-10 v23.10.2.13-stable 2023-11-08 v23.10.1.1976-stable 2023-11-02 @@ -5,6 +6,7 @@ v23.9.4.11-stable 2023-11-08 v23.9.3.12-stable 2023-10-31 v23.9.2.56-stable 2023-10-19 v23.9.1.1854-stable 2023-09-29 +v23.8.7.24-lts 2023-11-17 v23.8.6.16-lts 2023-11-08 v23.8.5.16-lts 2023-10-31 v23.8.4.69-lts 2023-10-19 @@ -31,6 +33,7 @@ v23.4.4.16-stable 2023-06-17 v23.4.3.48-stable 2023-06-12 v23.4.2.11-stable 2023-05-02 v23.4.1.1943-stable 2023-04-27 +v23.3.17.13-lts 2023-11-17 v23.3.16.7-lts 2023-11-08 v23.3.15.29-lts 2023-10-31 v23.3.14.78-lts 2023-10-18 From af2f06db5282fa801b82d1b9def56284f49ce77c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 17 Nov 2023 11:12:16 +0000 Subject: [PATCH 146/274] Update version_date.tsv and changelogs after v23.9.5.29-stable --- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v23.9.5.29-stable.md | 34 ++++++++++++++++++++++++++++ utils/list-versions/version_date.tsv | 4 ++++ 5 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 docs/changelogs/v23.9.5.29-stable.md diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 1f4fd39bc261..63de9f6c4629 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-keeper" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index 41be7e611a3b..d26bb344fef1 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 0ff6ae2e2270..53a368181213 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -30,7 +30,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="23.10.3.5" +ARG VERSION="23.10.4.25" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v23.9.5.29-stable.md b/docs/changelogs/v23.9.5.29-stable.md new file mode 100644 index 000000000000..02572d0e5629 --- /dev/null +++ b/docs/changelogs/v23.9.5.29-stable.md @@ -0,0 +1,34 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.9.5.29-stable (f8554c1a1ff) FIXME as compared to v23.9.4.11-stable (74c1f49dd6a) + +#### Build/Testing/Packaging Improvement +* Backported in [#56631](https://github.com/ClickHouse/ClickHouse/issues/56631): In [#54043](https://github.com/ClickHouse/ClickHouse/issues/54043) the setup plan started to appear in the logs. It should be only in the `runner_get_all_tests.log` only. As well, send the failed infrastructure event to CI db. [#56214](https://github.com/ClickHouse/ClickHouse/pull/56214) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#56735](https://github.com/ClickHouse/ClickHouse/issues/56735): Do not fetch changed submodules in the builder container. [#56689](https://github.com/ClickHouse/ClickHouse/pull/56689) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Select from system tables when table based on table function. [#55540](https://github.com/ClickHouse/ClickHouse/pull/55540) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Fix incomplete query result for UNION in view() function. [#56274](https://github.com/ClickHouse/ClickHouse/pull/56274) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in case of adding a column with type Object(JSON) [#56307](https://github.com/ClickHouse/ClickHouse/pull/56307) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix segfault during Kerberos initialization [#56401](https://github.com/ClickHouse/ClickHouse/pull/56401) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix: RabbitMQ OpenSSL dynamic loading issue [#56703](https://github.com/ClickHouse/ClickHouse/pull/56703) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix crash in GCD codec in case when zeros present in data [#56704](https://github.com/ClickHouse/ClickHouse/pull/56704) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix crash in FPC codec [#56795](https://github.com/ClickHouse/ClickHouse/pull/56795) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### NO CL CATEGORY + +* Backported in [#56603](https://github.com/ClickHouse/ClickHouse/issues/56603):. [#56598](https://github.com/ClickHouse/ClickHouse/pull/56598) ([Maksim Kita](https://github.com/kitaisreal)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Improve enrich image [#55793](https://github.com/ClickHouse/ClickHouse/pull/55793) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Rewrite jobs to use callable workflow [#56385](https://github.com/ClickHouse/ClickHouse/pull/56385) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Continue rewriting workflows to reusable tests [#56501](https://github.com/ClickHouse/ClickHouse/pull/56501) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Better exception messages [#56854](https://github.com/ClickHouse/ClickHouse/pull/56854) ([Antonio Andelic](https://github.com/antonio2368)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 0f2684cd91d9..014ee5e9a17d 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,10 +1,13 @@ +v23.10.4.25-stable 2023-11-17 v23.10.3.5-stable 2023-11-10 v23.10.2.13-stable 2023-11-08 v23.10.1.1976-stable 2023-11-02 +v23.9.5.29-stable 2023-11-17 v23.9.4.11-stable 2023-11-08 v23.9.3.12-stable 2023-10-31 v23.9.2.56-stable 2023-10-19 v23.9.1.1854-stable 2023-09-29 +v23.8.7.24-lts 2023-11-17 v23.8.6.16-lts 2023-11-08 v23.8.5.16-lts 2023-10-31 v23.8.4.69-lts 2023-10-19 @@ -31,6 +34,7 @@ v23.4.4.16-stable 2023-06-17 v23.4.3.48-stable 2023-06-12 v23.4.2.11-stable 2023-05-02 v23.4.1.1943-stable 2023-04-27 +v23.3.17.13-lts 2023-11-17 v23.3.16.7-lts 2023-11-08 v23.3.15.29-lts 2023-10-31 v23.3.14.78-lts 2023-10-18 From d6fdfdd45f590860a12023c25b2af735f2853796 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 12:16:54 +0100 Subject: [PATCH 147/274] Remove outdated instructions --- tests/instructions/coverity.txt | 28 ----------- tests/instructions/cppcheck.txt | 22 --------- tests/instructions/heap-profiler.txt | 14 ------ tests/instructions/kafka.txt | 45 ----------------- tests/instructions/sanitizers.md | 72 ---------------------------- tests/instructions/syntax.txt | 5 -- tests/instructions/tscancode.txt | 26 ---------- 7 files changed, 212 deletions(-) delete mode 100644 tests/instructions/coverity.txt delete mode 100644 tests/instructions/cppcheck.txt delete mode 100644 tests/instructions/heap-profiler.txt delete mode 100644 tests/instructions/kafka.txt delete mode 100644 tests/instructions/sanitizers.md delete mode 100644 tests/instructions/syntax.txt delete mode 100644 tests/instructions/tscancode.txt diff --git a/tests/instructions/coverity.txt b/tests/instructions/coverity.txt deleted file mode 100644 index f8d6d68d3267..000000000000 --- a/tests/instructions/coverity.txt +++ /dev/null @@ -1,28 +0,0 @@ -# Download tool at https://scan.coverity.com/download?tab=cxx - -tar xf cov-analysis-linux64-2017.07.tar.gz -export PATH=$PATH:/home/milovidov/cov-analysis-linux64-2017.07/bin - -mkdir ClickHouse_coverity -cd ClickHouse_coverity -git clone --recursive git@github.com:yandex/ClickHouse.git . - -mkdir build -cd build - -# "Debug" is for faster build -CC=gcc-7 CXX=g++-7 cmake -D CMAKE_BUILD_TYPE=Debug -D CCACHE_FOUND=0 .. - -# Build all targets that we don't want to analyze. -cd contrib && make -j24 && cd .. - -cov-configure --comptype gcc --compiler gcc-7 --template - -cov-build --dir cov-int make -j24 - -# Build is painful slow. Some targets compile in about one hour. Total time is about 4..5 hours. - -tar czvf clickhouse.tgz cov-int - -# tarball is 1.2 GB. -# Upload result at https://scan.coverity.com/projects/yandex-clickhouse/builds/new diff --git a/tests/instructions/cppcheck.txt b/tests/instructions/cppcheck.txt deleted file mode 100644 index 1bc6d1f6c099..000000000000 --- a/tests/instructions/cppcheck.txt +++ /dev/null @@ -1,22 +0,0 @@ -# Install cppcheck - -mkdir cppcheck && cd cppcheck -git clone git@github.com:danmar/cppcheck.git . -mkdir build && cd build -CC=gcc-7 CXX=g++-7 cmake -D CMAKE_BUILD_TYPE=Release .. -make -j24 -sudo make install - -# Perform analysis - -cd ClickHouse_clean/build -cppcheck -j24 --project=compile_commands.json --enable=all 2> cppcheck-errors.txt - -# or (from directory with sources) -# cppcheck -i contrib -i build --enable=all . 2> cppcheck-errors.txt - -# Check is pretty fast. -# It gives many false positives. -# But the result is worth looking and at least few real errors found. - -grep -v -F 'contrib/' cppcheck-errors.txt diff --git a/tests/instructions/heap-profiler.txt b/tests/instructions/heap-profiler.txt deleted file mode 100644 index 3c35e9cf518a..000000000000 --- a/tests/instructions/heap-profiler.txt +++ /dev/null @@ -1,14 +0,0 @@ -Build clickhouse without tcmalloc. cmake -D ENABLE_TCMALLOC=0 - -Copy clickhouse binary to your server. -scp programs/clickhouse server:~ - -ssh to your server - -Stop clickhouse: -sudo service clickhouse-server stop - -Run clickhouse with heap profiler from the terminal: -sudo -u clickhouse LD_PRELOAD=/usr/lib/libtcmalloc.so HEAPPROFILE=/var/log/clickhouse-server/heap.hprof ./clickhouse server --config /etc/clickhouse-server/config.xml - -Profiles will appear in /var/log/clickhouse-server/ diff --git a/tests/instructions/kafka.txt b/tests/instructions/kafka.txt deleted file mode 100644 index 69e87f38b242..000000000000 --- a/tests/instructions/kafka.txt +++ /dev/null @@ -1,45 +0,0 @@ -Use this config for docker-compose: - - version: '3' - - services: - - kafka: - depends_on: - - zookeeper - hostname: kafka - image: wurstmeister/kafka - environment: - KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - ports: - - "9092:9092" - - "9094:9094" - - security_opt: - - label:disable - - zookeeper: - hostname: zookeeper - image: zookeeper - - security_opt: - - label:disable - -Start containers with `docker-compose up`. - -In clickhouse-client create table like: - - CREATE TABLE kafka ( a UInt8, b String) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'CSV') SETTINGS kafka_row_delimiter = '\n'; - -Login inside Kafka container and stream some data: - - docker exec -it bash --login - vi data.csv - cat data.csv | /opt/kafka/bin/kafka-console-producer.sh --topic topic --broker-list localhost:9092 - -Read data in clickhouse: - - SELECT * FROM kafka; diff --git a/tests/instructions/sanitizers.md b/tests/instructions/sanitizers.md deleted file mode 100644 index 3c50f6cbab78..000000000000 --- a/tests/instructions/sanitizers.md +++ /dev/null @@ -1,72 +0,0 @@ -# How to use Address Sanitizer - -Note: We use Address Sanitizer to run functional tests for every commit automatically. - -``` -mkdir build_asan && cd build_asan -``` - -Note: using clang instead of gcc is strongly recommended. Make sure you have installed required packages (`clang`, `lld`). It may be required to specify non-standard `lld` binary using `LINKER_NAME` option (e.g. `-D LINKER_NAME=lld-8`). - -``` -CC=clang CXX=clang++ cmake -D SANITIZE=address .. -ninja -``` - -## Copy binary to your server - -``` -scp ./programs/clickhouse yourserver:~/clickhouse-asan -``` - -## Start ClickHouse and run tests - -``` -sudo -u clickhouse ./clickhouse-asan server --config /etc/clickhouse-server/config.xml -``` - - -# How to use Thread Sanitizer - -``` -mkdir build_tsan && cd build_tsan -``` - -``` -CC=clang CXX=clang++ cmake -D SANITIZE=thread .. -ninja -``` - -## Start ClickHouse and run tests - -``` -sudo -u clickhouse TSAN_OPTIONS='halt_on_error=1' ./clickhouse-tsan server --config /etc/clickhouse-server/config.xml -``` - - -# How to use Undefined Behaviour Sanitizer - -``` -mkdir build_ubsan && cd build_ubsan -``` - -Note: clang is mandatory, because gcc (in version 8) has false positives due to devirtualization and it has less amount of checks. - -``` -CC=clang CXX=clang++ cmake -D SANITIZE=undefined .. -ninja -``` - -## Start ClickHouse and run tests - -``` -sudo -u clickhouse UBSAN_OPTIONS='print_stacktrace=1' ./clickhouse-ubsan server --config /etc/clickhouse-server/config.xml -``` - - -# How to use Memory Sanitizer - -``` -CC=clang CXX=clang++ cmake -D SANITIZE=memory .. -ninja -``` diff --git a/tests/instructions/syntax.txt b/tests/instructions/syntax.txt deleted file mode 100644 index 228b0eb60457..000000000000 --- a/tests/instructions/syntax.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Relatively quick syntax check (20 minutes on 16-core server) - -mkdir build && cd build -cmake -D CMAKE_BUILD_TYPE=Debug .. -time jq --raw-output '.[] | .command' compile_commands.json | grep -P -- ' -o [^ ]+\.o' | grep -v -P -- '-c .+/contrib/' | grep -vP '\.(s|asm)$' | sed -r -e 's/ -o [^ ]+\.o/ -fsyntax-only/' | sort -R | xargs -I{} -P$(nproc) sh -c '{}' diff --git a/tests/instructions/tscancode.txt b/tests/instructions/tscancode.txt deleted file mode 100644 index 33a4eb34f35f..000000000000 --- a/tests/instructions/tscancode.txt +++ /dev/null @@ -1,26 +0,0 @@ -# TScanCode is a static analyzer from Tencent -# It looks like to be based on CppCheck - -git clone git@github.com:Tencent/TscanCode.git -cd TscanCode/trunk -make -j4 - -# It looks weird that TScanCode itself compiles with multiple warnings like 'unused-but-set-variable' and 'misleading-indentation' - -# Run analysis: - -./tscancode -j4 --enable=all ~/work/ClickHouse 2> result.txt - -# It has no way to remove specific directories. We have to checkout ClickHouse to separate directory and manually remove "contrib". -# Otherwise it segfaults when analysing llvm submodule. - -# It works quite fast: - -real 0m17.174s -user 0m45.498s -sys 0m0.496s - -wc -l result.txt -61 result.txt - -# It gives almost all false positives. From 9523bd0ad8d0841c2b76a00e18e09c20f50e46d7 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Fri, 17 Nov 2023 12:17:01 +0100 Subject: [PATCH 148/274] Fix config --- tests/clickhouse-test | 2 +- tests/config/users.d/s3_cache_new.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 36ac409a4cb5..1fad1583fa5c 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -577,7 +577,7 @@ class SettingsRandomizer: ), "remote_filesystem_read_method": lambda: random.choice(["read", "threadpool"]), "local_filesystem_read_prefetch": lambda: random.randint(0, 1), - "filesystem_cache_getorset_batch_size": lambda: random.choice([0, 3, 10, 50]), + "filesystem_cache_segments_batch_size": lambda: random.choice([0, 3, 10, 50]), "read_from_filesystem_cache_if_exists_otherwise_bypass_cache": lambda: random.randint( 0, 1 ), diff --git a/tests/config/users.d/s3_cache_new.xml b/tests/config/users.d/s3_cache_new.xml index 638b72679600..0afa3d68fc66 100644 --- a/tests/config/users.d/s3_cache_new.xml +++ b/tests/config/users.d/s3_cache_new.xml @@ -1,7 +1,7 @@ - 10 + 10 From 216450e789f62b509bbef9f451f852e8527ac034 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 12:23:06 +0100 Subject: [PATCH 149/274] Some tasks are done --- tests/instructions/easy_tasks_sorted_ru.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/instructions/easy_tasks_sorted_ru.md b/tests/instructions/easy_tasks_sorted_ru.md index 17e9708eef58..bc95e6b1c372 100644 --- a/tests/instructions/easy_tasks_sorted_ru.md +++ b/tests/instructions/easy_tasks_sorted_ru.md @@ -201,9 +201,9 @@ https://clickhouse.com/docs/en/operations/table_engines/external_data/ ## ВозможноŃŃ‚ŃŚ ATTACH партиции Ń ĐĽĐµĐ˝ŃŚŃим или больŃим количеŃтвом Ńтолбцов. -## Поддержка неконŃтантного аргŃмента Ń Ń‚Đ°ĐąĐĽ-зоной Ń Đ˝ĐµĐşĐľŃ‚ĐľŃ€Ń‹Ń… Ń„Ńнкций для работы Ń Đ´Đ°Ń‚ĐľĐą и временем. +## + Поддержка неконŃтантного аргŃмента Ń Ń‚Đ°ĐąĐĽ-зоной Ń Đ˝ĐµĐşĐľŃ‚ĐľŃ€Ń‹Ń… Ń„Ńнкций для работы Ń Đ´Đ°Ń‚ĐľĐą и временем. -## ВозможноŃŃ‚ŃŚ задавать параметры Ńоединений для табличных Ń„Ńнкций, движков таблиц и для реплик из отдельных разделов конфигŃрации. +## + ВозможноŃŃ‚ŃŚ задавать параметры Ńоединений для табличных Ń„Ńнкций, движков таблиц и для реплик из отдельных разделов конфигŃрации. ## + НаŃтройка rollup_use_nulls. From 86119dbc3f56a78234eaa67ea40760bb0e30e7fa Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Fri, 17 Nov 2023 12:26:50 +0100 Subject: [PATCH 150/274] fix data race --- src/Storages/MergeTree/DataPartsExchange.cpp | 10 ++++++++-- .../02916_replication_protocol_wait_for_part.sql | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index c39263a0b73c..0192fb1868bf 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -349,7 +349,7 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( return data_checksums; } -bool wait_loop(UInt32 wait_timeout_ms, std::function pred) +bool wait_loop(UInt32 wait_timeout_ms, const std::function & pred) { static const UInt32 loop_delay_ms = 5; @@ -360,6 +360,7 @@ bool wait_loop(UInt32 wait_timeout_ms, std::function pred) return true; Stopwatch timer; + sleepForMilliseconds(loop_delay_ms); while (!pred() && timer.elapsedMilliseconds() < wait_timeout_ms) { sleepForMilliseconds(loop_delay_ms); @@ -387,8 +388,13 @@ MergeTreeData::DataPartPtr Service::findPart(const String & name) /// do not expose PreActive parts for zero-copy static const UInt32 wait_timeout_ms = 1000; - bool pred_result = wait_loop(wait_timeout_ms, [&] () { return part->getState() != MergeTreeDataPartState::PreActive; }); + auto pred = [&] () + { + auto lock = data.lockParts(); + return part->getState() != MergeTreeDataPartState::PreActive; + }; + bool pred_result = wait_loop(wait_timeout_ms, pred); if (!pred_result) throw Exception( ErrorCodes::ABORTED, diff --git a/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql index 97ef33f96e8a..010e29a34e85 100644 --- a/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql +++ b/tests/queries/0_stateless/02916_replication_protocol_wait_for_part.sql @@ -22,5 +22,5 @@ insert into tableIn values(2); system sync replica tableOut; select count() from tableOut; -drop table tableIn -drop table tableOut +drop table tableIn; +drop table tableOut; From b2dc5ada6e1702332d15fbd515c728e5d06cb7d2 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 17 Nov 2023 11:31:52 +0000 Subject: [PATCH 151/274] Fix tryDecodeBase64() with invalid input --- src/Functions/FunctionBase64Conversion.h | 10 ++++----- .../00732_base64_functions.reference | 6 ++--- .../0_stateless/00732_base64_functions.sql | 22 +++++++++++++------ 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/Functions/FunctionBase64Conversion.h b/src/Functions/FunctionBase64Conversion.h index f52dec0eaf76..de922747ccdd 100644 --- a/src/Functions/FunctionBase64Conversion.h +++ b/src/Functions/FunctionBase64Conversion.h @@ -76,12 +76,10 @@ struct TryBase64Decode static size_t perform(const std::span src, UInt8 * dst) { size_t outlen = 0; - base64_decode(reinterpret_cast(src.data()), src.size(), reinterpret_cast(dst), &outlen, 0); + int rc = base64_decode(reinterpret_cast(src.data()), src.size(), reinterpret_cast(dst), &outlen, 0); - // during decoding character array can be partially polluted - // if fail, revert back and clean - if (!outlen) - *dst = 0; + if (rc != 1) + outlen = 0; return outlen; } @@ -147,7 +145,7 @@ class FunctionBase64Conversion : public IFunction for (size_t row = 0; row < src_row_count; ++row) { const size_t src_length = src_offsets[row] - src_offset_prev - 1; - const auto outlen = Func::perform({src, src_length}, dst_pos); + const size_t outlen = Func::perform({src, src_length}, dst_pos); /// Base64 library is using AVX-512 with some shuffle operations. /// Memory sanitizer don't understand if there was uninitialized memory in SIMD register but it was not used in the result of shuffle. diff --git a/tests/queries/0_stateless/00732_base64_functions.reference b/tests/queries/0_stateless/00732_base64_functions.reference index f97c19427e7b..8f91ffa74aba 100644 --- a/tests/queries/0_stateless/00732_base64_functions.reference +++ b/tests/queries/0_stateless/00732_base64_functions.reference @@ -21,9 +21,9 @@ fooba foobar 1 1 1 1 -fooba -~Š + + + Zm9v foo foo -TEcgT3B0aW11cw== diff --git a/tests/queries/0_stateless/00732_base64_functions.sql b/tests/queries/0_stateless/00732_base64_functions.sql index 99268004003d..3c60bf939fe3 100644 --- a/tests/queries/0_stateless/00732_base64_functions.sql +++ b/tests/queries/0_stateless/00732_base64_functions.sql @@ -2,17 +2,23 @@ SET send_logs_level = 'fatal'; -SELECT base64Encode(val) FROM (select arrayJoin(['', 'f', 'fo', 'foo', 'foob', 'fooba', 'foobar']) val); +SELECT base64Encode(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT base64Decode(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT tryBase64Decode(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT base64Encode('foo', 'excess argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT base64Decode('foo', 'excess argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +SELECT tryBase64Decode('foo', 'excess argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +-- test with valid inputs +SELECT base64Encode(val) FROM (select arrayJoin(['', 'f', 'fo', 'foo', 'foob', 'fooba', 'foobar']) val); SELECT base64Decode(val) FROM (select arrayJoin(['', 'Zg==', 'Zm8=', 'Zm9v', 'Zm9vYg==', 'Zm9vYmE=', 'Zm9vYmFy']) val); SELECT tryBase64Decode(val) FROM (select arrayJoin(['', 'Zg==', 'Zm8=', 'Zm9v', 'Zm9vYg==', 'Zm9vYmE=', 'Zm9vYmFy']) val); SELECT base64Decode(base64Encode('foo')) = 'foo', base64Encode(base64Decode('Zm9v')) == 'Zm9v'; SELECT tryBase64Decode(base64Encode('foo')) = 'foo', base64Encode(tryBase64Decode('Zm9v')) == 'Zm9v'; -SELECT base64Encode('foo', 'excess argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } -SELECT base64Decode('foo', 'excess argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } -SELECT tryBase64Decode('foo', 'excess argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +-- test with invalid inputs SELECT base64Decode('Zm9vYmF=Zm9v'); -- { serverError INCORRECT_DATA } SELECT tryBase64Decode('Zm9vYmF=Zm9v'); @@ -20,9 +26,11 @@ SELECT tryBase64Decode('Zm9vYmF=Zm9v'); SELECT base64Decode('foo'); -- { serverError INCORRECT_DATA } SELECT tryBase64Decode('foo'); +SELECT base64Decode('aoeo054640eu='); -- { serverError INCORRECT_DATA } +SELECT tryBase64Decode('aoeo054640eu='); + +-- test FixedString arguments + select base64Encode(toFixedString('foo', 3)); select base64Decode(toFixedString('Zm9v', 4)); select tryBase64Decode(toFixedString('Zm9v', 4)); - --- This query reproduces a bug in TurboBase64 library (which we no longer use) -select distinct base64Encode(materialize('LG Optimus')) from numbers(100); From 318c7a06f92db53d1cf7389a37387193da187ff6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 12:39:00 +0100 Subject: [PATCH 152/274] Avoid dependencies with no fixed versions --- docker/packager/binary/Dockerfile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index fb033e289595..65f79da44a09 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -6,29 +6,27 @@ FROM clickhouse/test-util:latest AS cctools ENV CC=clang-${LLVM_VERSION} ENV CXX=clang++-${LLVM_VERSION} # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -# DO NOT PUT ANYTHING BEFORE THREE NEXT `RUN` DIRECTIVES +# DO NOT PUT ANYTHING BEFORE THE NEXT TWO `RUN` DIRECTIVES # THE MOST HEAVY OPERATION MUST BE THE FIRST IN THE CACHE # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # libtapi is required to support .tbh format from recent MacOS SDKs RUN git clone --depth 1 https://github.com/tpoechtrager/apple-libtapi.git \ && cd apple-libtapi \ + && git checkout 15dfc2a8c9a2a89d06ff227560a69f5265b692f9 \ && INSTALLPREFIX=/cctools ./build.sh \ && ./install.sh \ && cd .. \ && rm -rf apple-libtapi # Build and install tools for cross-linking to Darwin (x86-64) +# Build and install tools for cross-linking to Darwin (aarch64) RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ && cd cctools-port/cctools \ + && git checkout 59f5fb87a3d2c6fd2ba3df6533015cd6172001c6 \ && ./configure --prefix=/cctools --with-libtapi=/cctools \ --target=x86_64-apple-darwin \ && make install -j$(nproc) \ - && cd ../.. \ - && rm -rf cctools-port - -# Build and install tools for cross-linking to Darwin (aarch64) -RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ - && cd cctools-port/cctools \ + && make clean \ && ./configure --prefix=/cctools --with-libtapi=/cctools \ --target=aarch64-apple-darwin \ && make install -j$(nproc) \ From 45b69566012ece56f9e284f2521c4bb13fb43c7f Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Fri, 17 Nov 2023 12:42:06 +0100 Subject: [PATCH 153/274] Fix race on zk_log --- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index fd845016f8af..4335ea4655f1 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1147,7 +1147,8 @@ void ZooKeeper::pushRequest(RequestInfo && info) { checkSessionDeadline(); info.time = clock::now(); - if (zk_log) + auto maybe_zk_log = std::atomic_load(&zk_log); + if (maybe_zk_log) { info.request->thread_id = getThreadId(); info.request->query_id = String(CurrentThread::getQueryId()); From 1d5bc13e2aa18dd30553eede612df350973425c8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 12:50:51 +0100 Subject: [PATCH 154/274] Check what will happen if I remove some lines --- docker/packager/binary/Dockerfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index fb033e289595..2d3b83814b8a 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -62,19 +62,12 @@ RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \ rustup target add aarch64-unknown-linux-musl && \ rustup target add riscv64gc-unknown-linux-gnu -# NOTE: Seems like gcc-11 is too new for ubuntu20 repository # A cross-linker for RISC-V 64 (we need it, because LLVM's LLD does not work): RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \ && apt-get update \ && apt-get install --yes \ binutils-riscv64-linux-gnu \ build-essential \ - g++-11 \ - gcc-11 \ - gcc-aarch64-linux-gnu \ - libc6 \ - libc6-dev \ - libc6-dev-arm64-cross \ python3-boto3 \ yasm \ zstd \ From 2a9d05e24541d098dfe375567d5d369698a859bf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 12:59:04 +0100 Subject: [PATCH 155/274] Remove more lines --- docker/packager/packager | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/docker/packager/packager b/docker/packager/packager index e63a4912e7cc..b5bcbada1da2 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -236,16 +236,14 @@ def parse_env_variables( cc = compiler result.append("DEB_ARCH=amd64") - cxx = cc.replace("gcc", "g++").replace("clang", "clang++") + cxx = cc.replace("clang", "clang++") if package_type == "deb": - # NOTE: This are the env for packages/build script + # NOTE: This is the env for packages/build script result.append("MAKE_DEB=true") cmake_flags.append("-DENABLE_TESTS=0") cmake_flags.append("-DENABLE_UTILS=0") - cmake_flags.append("-DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON") cmake_flags.append("-DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON") - cmake_flags.append("-DCMAKE_AUTOGEN_VERBOSE=ON") cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr") cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc") cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var") @@ -265,12 +263,7 @@ def parse_env_variables( elif package_type == "fuzzers": cmake_flags.append("-DENABLE_FUZZING=1") cmake_flags.append("-DENABLE_PROTOBUF=1") - cmake_flags.append("-DUSE_INTERNAL_PROTOBUF_LIBRARY=1") cmake_flags.append("-DWITH_COVERAGE=1") - cmake_flags.append("-DCMAKE_AUTOGEN_VERBOSE=ON") - # cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr") - # cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc") - # cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var") # Reduce linking and building time by avoid *install/all dependencies cmake_flags.append("-DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON") From 354fb5182bfc15c37687da9918680430e6478eb8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 13:01:33 +0100 Subject: [PATCH 156/274] Remove some code that I don't understand --- cmake/darwin/toolchain-aarch64.cmake | 6 ------ cmake/darwin/toolchain-x86_64.cmake | 6 ------ cmake/freebsd/toolchain-aarch64.cmake | 6 ------ cmake/freebsd/toolchain-ppc64le.cmake | 6 ------ cmake/freebsd/toolchain-x86_64.cmake | 6 ------ cmake/linux/toolchain-aarch64.cmake | 6 ------ cmake/linux/toolchain-ppc64le.cmake | 6 ------ cmake/linux/toolchain-riscv64.cmake | 6 ------ cmake/linux/toolchain-s390x.cmake | 6 ------ cmake/linux/toolchain-x86_64-musl.cmake | 6 ------ cmake/linux/toolchain-x86_64.cmake | 6 ------ 11 files changed, 66 deletions(-) diff --git a/cmake/darwin/toolchain-aarch64.cmake b/cmake/darwin/toolchain-aarch64.cmake index 569b02bb642d..178153c10989 100644 --- a/cmake/darwin/toolchain-aarch64.cmake +++ b/cmake/darwin/toolchain-aarch64.cmake @@ -9,9 +9,3 @@ set (CMAKE_ASM_COMPILER_TARGET "aarch64-apple-darwin") set (CMAKE_OSX_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/darwin-aarch64") set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/darwin/toolchain-x86_64.cmake b/cmake/darwin/toolchain-x86_64.cmake index c4527d2fc0d2..b9cbe72a2b63 100644 --- a/cmake/darwin/toolchain-x86_64.cmake +++ b/cmake/darwin/toolchain-x86_64.cmake @@ -9,9 +9,3 @@ set (CMAKE_ASM_COMPILER_TARGET "x86_64-apple-darwin") set (CMAKE_OSX_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/darwin-x86_64") set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/freebsd/toolchain-aarch64.cmake b/cmake/freebsd/toolchain-aarch64.cmake index 8a8da00f3bef..0d7eba7c1982 100644 --- a/cmake/freebsd/toolchain-aarch64.cmake +++ b/cmake/freebsd/toolchain-aarch64.cmake @@ -13,9 +13,3 @@ set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it # Will be changed later, but somehow needed to be set here. set (CMAKE_AR "ar") set (CMAKE_RANLIB "ranlib") - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/freebsd/toolchain-ppc64le.cmake b/cmake/freebsd/toolchain-ppc64le.cmake index c3f6594204dc..f9878bb47be3 100644 --- a/cmake/freebsd/toolchain-ppc64le.cmake +++ b/cmake/freebsd/toolchain-ppc64le.cmake @@ -13,9 +13,3 @@ set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it # Will be changed later, but somehow needed to be set here. set (CMAKE_AR "ar") set (CMAKE_RANLIB "ranlib") - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/freebsd/toolchain-x86_64.cmake b/cmake/freebsd/toolchain-x86_64.cmake index 460de6a7d39c..60489da1d65c 100644 --- a/cmake/freebsd/toolchain-x86_64.cmake +++ b/cmake/freebsd/toolchain-x86_64.cmake @@ -13,9 +13,3 @@ set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it # Will be changed later, but somehow needed to be set here. set (CMAKE_AR "ar") set (CMAKE_RANLIB "ranlib") - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/linux/toolchain-aarch64.cmake b/cmake/linux/toolchain-aarch64.cmake index 2dedef8859f8..954f3da43312 100644 --- a/cmake/linux/toolchain-aarch64.cmake +++ b/cmake/linux/toolchain-aarch64.cmake @@ -20,9 +20,3 @@ set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/aarch64-linux-gnu/libc") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/linux/toolchain-ppc64le.cmake b/cmake/linux/toolchain-ppc64le.cmake index c46ea954b710..ae10cac9a557 100644 --- a/cmake/linux/toolchain-ppc64le.cmake +++ b/cmake/linux/toolchain-ppc64le.cmake @@ -20,9 +20,3 @@ set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/powerpc64le-linux-gnu/libc") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/linux/toolchain-riscv64.cmake b/cmake/linux/toolchain-riscv64.cmake index 7f876f88d72f..7f0e30869fc6 100644 --- a/cmake/linux/toolchain-riscv64.cmake +++ b/cmake/linux/toolchain-riscv64.cmake @@ -27,9 +27,3 @@ set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=bfd") # ld.lld: error: section size decrease is too large # But GNU BinUtils work. set (LINKER_NAME "riscv64-linux-gnu-ld.bfd" CACHE STRING "Linker name" FORCE) - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/linux/toolchain-s390x.cmake b/cmake/linux/toolchain-s390x.cmake index 945eb9affa45..b89275d58129 100644 --- a/cmake/linux/toolchain-s390x.cmake +++ b/cmake/linux/toolchain-s390x.cmake @@ -23,9 +23,3 @@ set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=mold -Wl,-L${CMAKE_SYSROOT}/usr/lib64") set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=mold -Wl,-L${CMAKE_SYSROOT}/usr/lib64") set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=mold -Wl,-L${CMAKE_SYSROOT}/usr/lib64") - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/linux/toolchain-x86_64-musl.cmake b/cmake/linux/toolchain-x86_64-musl.cmake index bc327e5ac258..250e52d5f588 100644 --- a/cmake/linux/toolchain-x86_64-musl.cmake +++ b/cmake/linux/toolchain-x86_64-musl.cmake @@ -21,11 +21,5 @@ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - set (USE_MUSL 1) add_definitions(-DUSE_MUSL=1) diff --git a/cmake/linux/toolchain-x86_64.cmake b/cmake/linux/toolchain-x86_64.cmake index 55b9df79f700..8f54cbb0b487 100644 --- a/cmake/linux/toolchain-x86_64.cmake +++ b/cmake/linux/toolchain-x86_64.cmake @@ -32,9 +32,3 @@ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") - -set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - -set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) -set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) From a3083f305ba50de44e33bcd97a089cb41e24ce22 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 13:06:38 +0100 Subject: [PATCH 157/274] Remove more code that I don't understand --- cmake/freebsd/toolchain-aarch64.cmake | 4 ---- cmake/freebsd/toolchain-ppc64le.cmake | 4 ---- cmake/freebsd/toolchain-x86_64.cmake | 4 ---- cmake/linux/toolchain-aarch64.cmake | 4 ---- cmake/linux/toolchain-ppc64le.cmake | 4 ---- cmake/linux/toolchain-riscv64.cmake | 4 ---- cmake/linux/toolchain-s390x.cmake | 4 ---- cmake/linux/toolchain-x86_64-musl.cmake | 4 ---- cmake/linux/toolchain-x86_64.cmake | 4 ---- 9 files changed, 36 deletions(-) diff --git a/cmake/freebsd/toolchain-aarch64.cmake b/cmake/freebsd/toolchain-aarch64.cmake index 0d7eba7c1982..53b7856ed03e 100644 --- a/cmake/freebsd/toolchain-aarch64.cmake +++ b/cmake/freebsd/toolchain-aarch64.cmake @@ -9,7 +9,3 @@ set (CMAKE_ASM_COMPILER_TARGET "aarch64-unknown-freebsd12") set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-aarch64") set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake - -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") diff --git a/cmake/freebsd/toolchain-ppc64le.cmake b/cmake/freebsd/toolchain-ppc64le.cmake index f9878bb47be3..bb23f0fbafce 100644 --- a/cmake/freebsd/toolchain-ppc64le.cmake +++ b/cmake/freebsd/toolchain-ppc64le.cmake @@ -9,7 +9,3 @@ set (CMAKE_ASM_COMPILER_TARGET "powerpc64le-unknown-freebsd13") set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-ppc64le") set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake - -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") diff --git a/cmake/freebsd/toolchain-x86_64.cmake b/cmake/freebsd/toolchain-x86_64.cmake index 60489da1d65c..4635880b4a66 100644 --- a/cmake/freebsd/toolchain-x86_64.cmake +++ b/cmake/freebsd/toolchain-x86_64.cmake @@ -9,7 +9,3 @@ set (CMAKE_ASM_COMPILER_TARGET "x86_64-pc-freebsd11") set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-x86_64") set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake - -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") diff --git a/cmake/linux/toolchain-aarch64.cmake b/cmake/linux/toolchain-aarch64.cmake index 954f3da43312..b80cc01296d7 100644 --- a/cmake/linux/toolchain-aarch64.cmake +++ b/cmake/linux/toolchain-aarch64.cmake @@ -9,10 +9,6 @@ set (CMAKE_C_COMPILER_TARGET "aarch64-linux-gnu") set (CMAKE_CXX_COMPILER_TARGET "aarch64-linux-gnu") set (CMAKE_ASM_COMPILER_TARGET "aarch64-linux-gnu") -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") - set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-aarch64") set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/aarch64-linux-gnu/libc") diff --git a/cmake/linux/toolchain-ppc64le.cmake b/cmake/linux/toolchain-ppc64le.cmake index ae10cac9a557..98e8f7e84894 100644 --- a/cmake/linux/toolchain-ppc64le.cmake +++ b/cmake/linux/toolchain-ppc64le.cmake @@ -9,10 +9,6 @@ set (CMAKE_C_COMPILER_TARGET "powerpc64le-linux-gnu") set (CMAKE_CXX_COMPILER_TARGET "powerpc64le-linux-gnu") set (CMAKE_ASM_COMPILER_TARGET "powerpc64le-linux-gnu") -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") - set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-powerpc64le") set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/powerpc64le-linux-gnu/libc") diff --git a/cmake/linux/toolchain-riscv64.cmake b/cmake/linux/toolchain-riscv64.cmake index 7f0e30869fc6..ae5a38f08eb9 100644 --- a/cmake/linux/toolchain-riscv64.cmake +++ b/cmake/linux/toolchain-riscv64.cmake @@ -9,10 +9,6 @@ set (CMAKE_C_COMPILER_TARGET "riscv64-linux-gnu") set (CMAKE_CXX_COMPILER_TARGET "riscv64-linux-gnu") set (CMAKE_ASM_COMPILER_TARGET "riscv64-linux-gnu") -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") - set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-riscv64") set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}") diff --git a/cmake/linux/toolchain-s390x.cmake b/cmake/linux/toolchain-s390x.cmake index b89275d58129..d34329fb3bbf 100644 --- a/cmake/linux/toolchain-s390x.cmake +++ b/cmake/linux/toolchain-s390x.cmake @@ -9,10 +9,6 @@ set (CMAKE_C_COMPILER_TARGET "s390x-linux-gnu") set (CMAKE_CXX_COMPILER_TARGET "s390x-linux-gnu") set (CMAKE_ASM_COMPILER_TARGET "s390x-linux-gnu") -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") - set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-s390x") set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/s390x-linux-gnu/libc") diff --git a/cmake/linux/toolchain-x86_64-musl.cmake b/cmake/linux/toolchain-x86_64-musl.cmake index 250e52d5f588..fa7b3eaf0d1a 100644 --- a/cmake/linux/toolchain-x86_64-musl.cmake +++ b/cmake/linux/toolchain-x86_64-musl.cmake @@ -9,10 +9,6 @@ set (CMAKE_C_COMPILER_TARGET "x86_64-linux-musl") set (CMAKE_CXX_COMPILER_TARGET "x86_64-linux-musl") set (CMAKE_ASM_COMPILER_TARGET "x86_64-linux-musl") -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") - set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-x86_64-musl") set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}") diff --git a/cmake/linux/toolchain-x86_64.cmake b/cmake/linux/toolchain-x86_64.cmake index 8f54cbb0b487..e341219a7e59 100644 --- a/cmake/linux/toolchain-x86_64.cmake +++ b/cmake/linux/toolchain-x86_64.cmake @@ -19,10 +19,6 @@ set (CMAKE_C_COMPILER_TARGET "x86_64-linux-gnu") set (CMAKE_CXX_COMPILER_TARGET "x86_64-linux-gnu") set (CMAKE_ASM_COMPILER_TARGET "x86_64-linux-gnu") -# Will be changed later, but somehow needed to be set here. -set (CMAKE_AR "ar") -set (CMAKE_RANLIB "ranlib") - set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-x86_64") set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/x86_64-linux-gnu/libc") From 215cd7b9f285bbfa5d07882dfd64820e3428c402 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 15:13:16 +0300 Subject: [PATCH 158/274] Update build.sh --- docker/packager/binary/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 37440fe8202a..f943011df9df 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -38,7 +38,7 @@ rm -f CMakeCache.txt # To check it, find and delete them. grep -o -P '"contrib/[^"]+"' ../.gitmodules | - grep -v -P 'llvm-project|google-protobuf|grpc|corrosion' | + grep -v -P 'llvm-project|google-protobuf|grpc|abseil-cpp|corrosion' | xargs -I@ find ../@ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | xargs rm From 19dd29e8af5f79323a8f86c6a72f68ae2b45fa6e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 15:19:00 +0300 Subject: [PATCH 159/274] Update Dockerfile --- docker/packager/binary/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 65f79da44a09..d7864a116725 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -22,7 +22,7 @@ RUN git clone --depth 1 https://github.com/tpoechtrager/apple-libtapi.git \ # Build and install tools for cross-linking to Darwin (aarch64) RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ && cd cctools-port/cctools \ - && git checkout 59f5fb87a3d2c6fd2ba3df6533015cd6172001c6 \ + && git checkout 2ea20c36c10fa1ec70ada3d5aeb2c205d4aa591e \ && ./configure --prefix=/cctools --with-libtapi=/cctools \ --target=x86_64-apple-darwin \ && make install -j$(nproc) \ From ea3cd71225794724ed63213d3567a9167133eade Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 15:42:10 +0300 Subject: [PATCH 160/274] Update Dockerfile --- docker/packager/binary/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index d7864a116725..8b5049c0fc76 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -10,7 +10,7 @@ ENV CXX=clang++-${LLVM_VERSION} # THE MOST HEAVY OPERATION MUST BE THE FIRST IN THE CACHE # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # libtapi is required to support .tbh format from recent MacOS SDKs -RUN git clone --depth 1 https://github.com/tpoechtrager/apple-libtapi.git \ +RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \ && cd apple-libtapi \ && git checkout 15dfc2a8c9a2a89d06ff227560a69f5265b692f9 \ && INSTALLPREFIX=/cctools ./build.sh \ @@ -20,9 +20,9 @@ RUN git clone --depth 1 https://github.com/tpoechtrager/apple-libtapi.git \ # Build and install tools for cross-linking to Darwin (x86-64) # Build and install tools for cross-linking to Darwin (aarch64) -RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ +RUN git clone https://github.com/tpoechtrager/cctools-port.git \ && cd cctools-port/cctools \ - && git checkout 2ea20c36c10fa1ec70ada3d5aeb2c205d4aa591e \ + && git checkout 319ef50ea51a73acfc3d691396c05005e48647da \ && ./configure --prefix=/cctools --with-libtapi=/cctools \ --target=x86_64-apple-darwin \ && make install -j$(nproc) \ From b42db2ec298d67ad0c4cb5f190fc7c1f1815e781 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 13:50:13 +0100 Subject: [PATCH 161/274] Update fasttest --- docker/test/fasttest/run.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 1b72dab5e3c6..d3695ba26133 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -206,7 +206,7 @@ function build ( cd "$FASTTEST_BUILD" TIMEFORMAT=$'\nreal\t%3R\nuser\t%3U\nsys\t%3S' - ( time ninja clickhouse-bundle) |& ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/build_log.txt" + ( time ninja clickhouse-bundle clickhouse-stripped) |& ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/build_log.txt" BUILD_SECONDS_ELAPSED=$(awk '/^....-..-.. ..:..:.. real\t[0-9]/ {print $4}' < "$FASTTEST_OUTPUT/build_log.txt") echo "build_clickhouse_fasttest_binary: [ OK ] $BUILD_SECONDS_ELAPSED sec." \ | ts '%Y-%m-%d %H:%M:%S' \ @@ -215,7 +215,6 @@ function build mkdir -p "$FASTTEST_OUTPUT/binaries/" cp programs/clickhouse "$FASTTEST_OUTPUT/binaries/clickhouse" - strip programs/clickhouse -o programs/clickhouse-stripped zstd --threads=0 programs/clickhouse-stripped -o "$FASTTEST_OUTPUT/binaries/clickhouse-stripped.zst" fi ccache_status From 9eb0b74167b6b72343fba25bbc31fd42f919ef1b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 15:54:32 +0300 Subject: [PATCH 162/274] Update Dockerfile --- docker/packager/binary/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 8b5049c0fc76..5e630ea5a3f0 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -22,7 +22,7 @@ RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \ # Build and install tools for cross-linking to Darwin (aarch64) RUN git clone https://github.com/tpoechtrager/cctools-port.git \ && cd cctools-port/cctools \ - && git checkout 319ef50ea51a73acfc3d691396c05005e48647da \ + && git checkout 2a3e1c2a6ff54a30f898b70cfb9ba1692a55fad7 \ && ./configure --prefix=/cctools --with-libtapi=/cctools \ --target=x86_64-apple-darwin \ && make install -j$(nproc) \ From 332e7f565e613ab3a519749534472d14b30845dd Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Fri, 17 Nov 2023 14:41:26 +0100 Subject: [PATCH 163/274] Fix concat tests --- tests/queries/0_stateless/00727_concat.reference | 2 +- tests/queries/0_stateless/00727_concat.sql | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/00727_concat.reference b/tests/queries/0_stateless/00727_concat.reference index 7c48ba97c2b1..1e102051fd0f 100644 --- a/tests/queries/0_stateless/00727_concat.reference +++ b/tests/queries/0_stateless/00727_concat.reference @@ -46,7 +46,7 @@ With [[[(0,0),(10,0),(10,10),(0,10)]],[[(20,20),(50,20),(50,50),(20,50)],[(30,30 With 42 With 4 -- Nested -With [(\'foo\',\'qaz\'),(\'bar\',\'qux\')] +With [\'foo\',\'bar\'][\'qaz\',\'qux\'] -- NULL arguments \N \N diff --git a/tests/queries/0_stateless/00727_concat.sql b/tests/queries/0_stateless/00727_concat.sql index 7d901514aea6..edeaf9340ddc 100644 --- a/tests/queries/0_stateless/00727_concat.sql +++ b/tests/queries/0_stateless/00727_concat.sql @@ -50,15 +50,19 @@ SELECT concat('With ', materialize([[(20, 20), (50, 20), (50, 50), (20, 50)], [( SELECT concat('With ', materialize([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]] :: MultiPolygon)); SELECT '-- SimpleAggregateFunction'; -CREATE OR REPLACE TABLE concat_saf_test(x SimpleAggregateFunction(max, Int32)) ENGINE=MergeTree ORDER BY tuple(); +DROP TABLE IF EXISTS concat_saf_test; +CREATE TABLE concat_saf_test(x SimpleAggregateFunction(max, Int32)) ENGINE=MergeTree ORDER BY tuple(); INSERT INTO concat_saf_test VALUES (42); INSERT INTO concat_saf_test SELECT max(number) FROM numbers(5); SELECT concat('With ', x) FROM concat_saf_test ORDER BY x DESC; +DROP TABLE concat_saf_test; SELECT '-- Nested'; -CREATE OR REPLACE TABLE concat_nested_test(kv Nested(k String, v String)) ENGINE = MergeTree ORDER BY tuple(); +DROP TABLE IF EXISTS concat_nested_test; +CREATE TABLE concat_nested_test(attrs Nested(k String, v String)) ENGINE = MergeTree ORDER BY tuple(); INSERT INTO concat_nested_test VALUES (['foo', 'bar'], ['qaz', 'qux']); -SELECT concat('With ', kv) FROM concat_nested_test; +SELECT concat('With ', attrs.k, attrs.v) FROM concat_nested_test; +DROP TABLE concat_nested_test; SELECT '-- NULL arguments'; SELECT concat(NULL, NULL); From 571a35c84d70fc54ce6852b1fc2084ab2e5adf47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 17 Nov 2023 14:45:19 +0100 Subject: [PATCH 164/274] Make some tests independent on macro settings --- ...licated_minimalistic_part_header_zookeeper.sh | 11 +++++++---- .../00953_zookeeper_suetin_deduplication_bug.sh | 14 +++++++------- ...6_inactive_replica_cleanup_nodes_zookeeper.sh | 11 +++++++---- ...1586_replicated_mutations_empty_partition.sql | 4 ++-- ...nt_alter_mutations_kill_many_replicas_long.sh | 7 +++++-- .../01700_system_zookeeper_path_in.reference | 2 -- .../01700_system_zookeeper_path_in.sql | 16 ++++++++-------- .../02439_merge_selecting_partitions.sql | 2 +- 8 files changed, 37 insertions(+), 30 deletions(-) diff --git a/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sh b/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sh index bab2304cec21..12d889a71372 100755 --- a/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sh +++ b/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sh @@ -7,6 +7,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +SHARD=$($CLICKHOUSE_CLIENT --query "Select getMacro('shard')") +REPLICA=$($CLICKHOUSE_CLIENT --query "Select getMacro('replica')") + $CLICKHOUSE_CLIENT -nm -q " DROP TABLE IF EXISTS part_header_r1; @@ -54,8 +57,8 @@ elapsed=1 until [ $elapsed -eq 5 ]; do sleep $(( elapsed++ )) - count1=$($CLICKHOUSE_CLIENT --query="SELECT count(name) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/s1/replicas/1r1/parts'") - count2=$($CLICKHOUSE_CLIENT --query="SELECT count(name) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/s1/replicas/2r1/parts'") + count1=$($CLICKHOUSE_CLIENT --query="SELECT count(name) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/$SHARD/replicas/1$REPLICA/parts'") + count2=$($CLICKHOUSE_CLIENT --query="SELECT count(name) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/$SHARD/replicas/2$REPLICA/parts'") [[ $count1 == 1 && $count2 == 1 ]] && break done @@ -64,10 +67,10 @@ $CLICKHOUSE_CLIENT -nm -q " SELECT '*** Test part removal ***'; SELECT '*** replica 1 ***'; SELECT name FROM system.parts WHERE active AND database = currentDatabase() AND table = 'part_header_r1'; -SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/s1/replicas/1r1/parts'; +SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/$SHARD/replicas/1$REPLICA/parts'; SELECT '*** replica 2 ***'; SELECT name FROM system.parts WHERE active AND database = currentDatabase() AND table = 'part_header_r2'; -SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/s1/replicas/2r1/parts'; +SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_00814/part_header/$SHARD/replicas/2$REPLICA/parts'; SELECT '*** Test ALTER ***'; ALTER TABLE part_header_r1 MODIFY COLUMN y String; diff --git a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh index ad0146b9d994..57a415269004 100755 --- a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh +++ b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh @@ -9,7 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh CLICKHOUSE_TEST_ZOOKEEPER_PREFIX="${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_DATABASE}" - +SHARD=$($CLICKHOUSE_CLIENT --query "Select getMacro('shard')") $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS elog;" @@ -30,33 +30,33 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 3, 'h $CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 3 rows -count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") +count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/$SHARD/blocks'") while [[ $count != 2 ]] do sleep 1 - count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") + count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/$SHARD/blocks'") done $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 1, 'hello')" $CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 4 rows -count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") +count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/$SHARD/blocks'") while [[ $count != 2 ]] do sleep 1 - count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") + count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/$SHARD/blocks'") done $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'hello')" $CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 5 rows -count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") +count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/$SHARD/blocks'") while [[ $count != 2 ]] do sleep 1 - count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'") + count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/$SHARD/blocks'") done $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'hello')" diff --git a/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh b/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh index 2d761df998e5..67a2a70b5099 100755 --- a/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh +++ b/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh @@ -5,6 +5,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +SHARD=$($CLICKHOUSE_CLIENT --query "Select getMacro('shard')") +REPLICA=$($CLICKHOUSE_CLIENT --query "Select getMacro('replica')") + # Check that if we have one inactive replica and a huge number of INSERTs to active replicas, # the number of nodes in ZooKeeper does not grow unbounded. @@ -32,16 +35,16 @@ for _ in {1..60}; do done -$CLICKHOUSE_CLIENT --query "SELECT numChildren < $((SCALE / 4)) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1' AND name = 'log'"; +$CLICKHOUSE_CLIENT --query "SELECT numChildren < $((SCALE / 4)) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/$SHARD' AND name = 'log'"; echo -e '\n---\n'; -$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1/replicas/1r1' AND name = 'is_lost'"; -$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1/replicas/2r1' AND name = 'is_lost'"; +$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/$SHARD/replicas/1$REPLICA' AND name = 'is_lost'"; +$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/$SHARD/replicas/2$REPLICA' AND name = 'is_lost'"; echo -e '\n---\n'; $CLICKHOUSE_CLIENT --query "ATTACH TABLE r2" $CLICKHOUSE_CLIENT --receive_timeout 600 --query "SYSTEM SYNC REPLICA r2" # Need to increase timeout, otherwise it timed out in debug build -$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1/replicas/2r1' AND name = 'is_lost'"; +$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/$SHARD/replicas/2$REPLICA' AND name = 'is_lost'"; $CLICKHOUSE_CLIENT -n --query " DROP TABLE IF EXISTS r1; diff --git a/tests/queries/0_stateless/01586_replicated_mutations_empty_partition.sql b/tests/queries/0_stateless/01586_replicated_mutations_empty_partition.sql index b5ad6c06e96c..c4a3c939c265 100644 --- a/tests/queries/0_stateless/01586_replicated_mutations_empty_partition.sql +++ b/tests/queries/0_stateless/01586_replicated_mutations_empty_partition.sql @@ -16,7 +16,7 @@ INSERT INTO replicated_mutations_empty_partitions SETTINGS insert_keeper_fault_i SELECT count(distinct value) FROM replicated_mutations_empty_partitions; -SELECT count() FROM system.zookeeper WHERE path = '/clickhouse/test/'||currentDatabase()||'/01586_replicated_mutations_empty_partitions/s1/block_numbers'; +SELECT count() FROM system.zookeeper WHERE path = '/clickhouse/test/'||currentDatabase()||'/01586_replicated_mutations_empty_partitions/'||getMacro('shard')||'/block_numbers'; ALTER TABLE replicated_mutations_empty_partitions DROP PARTITION '3'; ALTER TABLE replicated_mutations_empty_partitions DROP PARTITION '4'; @@ -24,7 +24,7 @@ ALTER TABLE replicated_mutations_empty_partitions DROP PARTITION '5'; ALTER TABLE replicated_mutations_empty_partitions DROP PARTITION '9'; -- still ten records -SELECT count() FROM system.zookeeper WHERE path = '/clickhouse/test/'||currentDatabase()||'/01586_replicated_mutations_empty_partitions/s1/block_numbers'; +SELECT count() FROM system.zookeeper WHERE path = '/clickhouse/test/'||currentDatabase()||'/01586_replicated_mutations_empty_partitions/'||getMacro('shard')||'/block_numbers'; ALTER TABLE replicated_mutations_empty_partitions MODIFY COLUMN value UInt64 SETTINGS replication_alter_partitions_sync=2; diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh index f8f3ccd6dd68..2762f918d72a 100755 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas_long.sh @@ -7,6 +7,9 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=./replication.lib . "$CURDIR"/replication.lib +SHARD=$($CLICKHOUSE_CLIENT --query "Select getMacro('shard')") +REPLICA=$($CLICKHOUSE_CLIENT --query "Select getMacro('replica')") + REPLICAS=5 for i in $(seq $REPLICAS); do @@ -79,9 +82,9 @@ while true; do done -metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1/replicas/r11/' and name = 'metadata_version'") +metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/$SHARD/replicas/${REPLICA}1/' and name = 'metadata_version'") for i in $(seq $REPLICAS); do - replica_metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1/replicas/r1$i/' and name = 'metadata_version'") + replica_metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/$SHARD/replicas/${REPLICA}$i/' and name = 'metadata_version'") if [ "$metadata_version" != "$replica_metadata_version" ]; then echo "Metadata version on replica $i differs from the first replica, FAIL" diff --git a/tests/queries/0_stateless/01700_system_zookeeper_path_in.reference b/tests/queries/0_stateless/01700_system_zookeeper_path_in.reference index 664d8e84f27f..b4eaf2261062 100644 --- a/tests/queries/0_stateless/01700_system_zookeeper_path_in.reference +++ b/tests/queries/0_stateless/01700_system_zookeeper_path_in.reference @@ -14,5 +14,3 @@ abandonable_lock-other failed_parts last_part parallel -shared -shared diff --git a/tests/queries/0_stateless/01700_system_zookeeper_path_in.sql b/tests/queries/0_stateless/01700_system_zookeeper_path_in.sql index cf4bc7650e76..3b321d3cea5c 100644 --- a/tests/queries/0_stateless/01700_system_zookeeper_path_in.sql +++ b/tests/queries/0_stateless/01700_system_zookeeper_path_in.sql @@ -8,17 +8,17 @@ CREATE TABLE sample_table ( ENGINE ReplicatedMergeTree('/clickhouse/{database}/01700_system_zookeeper_path_in/{shard}', '{replica}') ORDER BY tuple(); -SELECT name FROM system.zookeeper WHERE path = '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1' AND name like 'block%' ORDER BY name; -SELECT name FROM system.zookeeper WHERE path = '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1/replicas' AND name LIKE '%r1%' ORDER BY name; +SELECT name FROM system.zookeeper WHERE path = '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard') AND name like 'block%' ORDER BY name; +SELECT 'r1' FROM system.zookeeper WHERE path = '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard') || '/replicas' AND name LIKE '%'|| getMacro('replica') ||'%' ORDER BY name; SELECT '========'; -SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1') AND name LIKE 'block%' ORDER BY name; -SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1/replicas') AND name LIKE '%r1%' ORDER BY name; +SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard')) AND name LIKE 'block%' ORDER BY name; +SELECT 'r1' FROM system.zookeeper WHERE path IN ('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard') || '/replicas') AND name LIKE '%' || getMacro('replica') || '%' ORDER BY name; SELECT '========'; -SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1', - '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1/replicas') AND name LIKE 'block%' ORDER BY name; +SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard'), + '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard') || '/replicas') AND name LIKE 'block%' ORDER BY name; SELECT '========'; -SELECT name FROM system.zookeeper WHERE path IN (SELECT concat('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1/', name) - FROM system.zookeeper WHERE (name != 'replicas' AND name NOT LIKE 'leader_election%' AND path = '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/s1')) ORDER BY name; +SELECT name FROM system.zookeeper WHERE path IN (SELECT concat('/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard') || '/', name) + FROM system.zookeeper WHERE (name != 'replicas' AND name NOT LIKE 'leader_election%' AND name NOT LIKE 'zero_copy_%' AND path = '/clickhouse/' || currentDatabase() || '/01700_system_zookeeper_path_in/' || getMacro('shard'))) ORDER BY name; DROP TABLE IF EXISTS sample_table; diff --git a/tests/queries/0_stateless/02439_merge_selecting_partitions.sql b/tests/queries/0_stateless/02439_merge_selecting_partitions.sql index 1d01fde56d66..0142afba7f25 100644 --- a/tests/queries/0_stateless/02439_merge_selecting_partitions.sql +++ b/tests/queries/0_stateless/02439_merge_selecting_partitions.sql @@ -21,7 +21,7 @@ select sleepEachRow(3) as higher_probability_of_reproducing_the_issue format Nul system flush logs; -- it should not list unneeded partitions where we cannot merge anything -select * from system.zookeeper_log where path like '/test/02439/s1/' || currentDatabase() || '/block_numbers/%' +select * from system.zookeeper_log where path like '/test/02439/' || getMacro('shard') || '/' || currentDatabase() || '/block_numbers/%' and op_num in ('List', 'SimpleList', 'FilteredList') and path not like '%/block_numbers/1' and path not like '%/block_numbers/123' and event_time >= now() - interval 1 minute From 01000e8b9edddbbe337d5e6287c20b9b88a64cc2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 17 Nov 2023 14:57:02 +0100 Subject: [PATCH 165/274] Merge with master --- contrib/abseil-cpp-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/abseil-cpp-cmake/CMakeLists.txt b/contrib/abseil-cpp-cmake/CMakeLists.txt index e84b4d46c4a6..e6c3268c57a2 100644 --- a/contrib/abseil-cpp-cmake/CMakeLists.txt +++ b/contrib/abseil-cpp-cmake/CMakeLists.txt @@ -2683,6 +2683,7 @@ absl_cc_library( "${DIR}/status.h" SRCS "${DIR}/internal/status_internal.h" + "${DIR}/internal/status_internal.cc" "${DIR}/status.cc" "${DIR}/status_payload_printer.h" "${DIR}/status_payload_printer.cc" @@ -2761,7 +2762,6 @@ absl_cc_library( "${DIR}/has_absl_stringify.h" "${DIR}/internal/damerau_levenshtein_distance.h" "${DIR}/internal/string_constant.h" - "${DIR}/internal/has_absl_stringify.h" "${DIR}/match.h" "${DIR}/numbers.h" "${DIR}/str_cat.h" From 4d5becb4deff91f4e9675624175f2207ea2d9ccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 17 Nov 2023 15:01:54 +0100 Subject: [PATCH 166/274] Adapt test_storage_s3/test.py::test_predefined_connection_configuration --- tests/integration/test_storage_s3/test.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 01ade1acc4d5..3dd3c9e39d02 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -944,13 +944,6 @@ def test_predefined_connection_configuration(started_cluster): instance.query("GRANT SELECT ON *.* TO user") instance.query(f"drop table if exists {name}", user="user") - error = instance.query_and_get_error( - f"CREATE TABLE {name} (id UInt32) ENGINE = S3(s3_conf1, format='CSV')" - ) - assert ( - "To execute this query, it's necessary to have the grant NAMED COLLECTION ON s3_conf1" - in error - ) error = instance.query_and_get_error( f"CREATE TABLE {name} (id UInt32) ENGINE = S3(s3_conf1, format='CSV')", user="user", @@ -975,11 +968,6 @@ def test_predefined_connection_configuration(started_cluster): ) assert result == instance.query("SELECT number FROM numbers(10)") - error = instance.query_and_get_error("SELECT * FROM s3(no_collection)") - assert ( - "To execute this query, it's necessary to have the grant NAMED COLLECTION ON no_collection" - in error - ) error = instance.query_and_get_error("SELECT * FROM s3(no_collection)", user="user") assert ( "To execute this query, it's necessary to have the grant NAMED COLLECTION ON no_collection" From a530d8c80db5cd03e567f7eb6962824cfc12b9f2 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 17 Nov 2023 15:25:43 +0000 Subject: [PATCH 167/274] Fix flaky test #56926 --- .../02494_query_cache_events.reference | 3 --- .../0_stateless/02494_query_cache_events.sql | 15 ++------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/tests/queries/0_stateless/02494_query_cache_events.reference b/tests/queries/0_stateless/02494_query_cache_events.reference index 9bcd2820f270..00510f3a0c6e 100644 --- a/tests/queries/0_stateless/02494_query_cache_events.reference +++ b/tests/queries/0_stateless/02494_query_cache_events.reference @@ -1,7 +1,4 @@ ---- 1 -0 1 ---- 1 0 1 1 0 diff --git a/tests/queries/0_stateless/02494_query_cache_events.sql b/tests/queries/0_stateless/02494_query_cache_events.sql index 05c0acad4b8b..f92e71cb50f1 100644 --- a/tests/queries/0_stateless/02494_query_cache_events.sql +++ b/tests/queries/0_stateless/02494_query_cache_events.sql @@ -4,20 +4,7 @@ -- Start with empty query cache QC SYSTEM DROP QUERY CACHE; --- Run a query with QC on. The first execution is a QC miss. -SELECT '---'; SELECT 1 SETTINGS use_query_cache = true; - -SYSTEM FLUSH LOGS; -SELECT ProfileEvents['QueryCacheHits'], ProfileEvents['QueryCacheMisses'] -FROM system.query_log -WHERE type = 'QueryFinish' - AND current_database = currentDatabase() - AND query = 'SELECT 1 SETTINGS use_query_cache = true;'; - - --- Run previous query again with query cache on -SELECT '---'; SELECT 1 SETTINGS use_query_cache = true; SYSTEM FLUSH LOGS; @@ -28,4 +15,6 @@ WHERE type = 'QueryFinish' AND query = 'SELECT 1 SETTINGS use_query_cache = true;' ORDER BY event_time_microseconds; +-- (The 1st execution was a cache miss, the 2nd execution was a cache hit) + SYSTEM DROP QUERY CACHE; From dd626d51cbc3b7307b7d48279d31f02dca8ec302 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 17 Nov 2023 16:36:19 +0100 Subject: [PATCH 168/274] Fix perf tests report when there are no tests (#56881) * fix perf tests report when there are no tests * Automatic style fix * Update docker/test/performance-comparison/compare.sh --------- Co-authored-by: robot-clickhouse --- docker/test/performance-comparison/compare.sh | 10 +++++ docker/test/performance-comparison/report.py | 42 ++++++++++++++----- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 7d6de7324891..f10236b7135b 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -189,6 +189,8 @@ function run_tests test_prefix=right/performance fi + run_only_changed_tests=0 + # Determine which tests to run. if [ -v CHPC_TEST_GREP ] then @@ -203,6 +205,7 @@ function run_tests # tests. The lists of changed files are prepared in entrypoint.sh because # it has the repository. test_files=($(sed "s/tests\/performance/${test_prefix//\//\\/}/" changed-test-definitions.txt)) + run_only_changed_tests=1 else # The default -- run all tests found in the test dir. test_files=($(ls "$test_prefix"/*.xml)) @@ -226,6 +229,13 @@ function run_tests test_files=("${test_files[@]}") fi + if [ "$run_only_changed_tests" -ne 0 ]; then + if [ ${#test_files[@]} -eq 0 ]; then + time "$script_dir/report.py" --no-tests-run > report.html + exit 0 + fi + fi + # For PRs w/o changes in test definitons, test only a subset of queries, # and run them less times. If the corresponding environment variables are # already set, keep those values. diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 7da30ba7a083..c2bc773bd54a 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -19,6 +19,7 @@ choices=["main", "all-queries"], help="Which report to build", ) +parser.add_argument("--no-tests-run", action="store_true", default=False) args = parser.parse_args() tables = [] @@ -354,6 +355,36 @@ def add_errors_explained(): add_tested_commits() + def print_status(status, message): + print( + ( + """ + + + """.format( + status=status, message=message + ) + ) + ) + + if args.no_tests_run: + for t in tables: + print(t) + print( + "

No tests to run. Only changed tests were run, but all changed tests are from another batch.

" + ) + print( + f""" + + {os.getenv("CHPC_ADD_REPORT_LINKS") or ''} + + + """ + ) + # Why failure? Because otherwise we will not notice if we have a bug that leads to 0 tests being run + print_status("failure", "No tests changed, nothing to run") + exit(0) + run_error_rows = tsvRows("run-errors.tsv") error_tests += len(run_error_rows) addSimpleTable("Run Errors", ["Test", "Error"], run_error_rows) @@ -646,16 +677,7 @@ def add_test_times(): status = "failure" message = "Errors while building the report." - print( - ( - """ - - - """.format( - status=status, message=message - ) - ) - ) + print_status(status, message) elif args.report == "all-queries": print((header_template.format())) From 6d5a5f9fcd07bc87d558060495f9b071b2535abf Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Fri, 17 Nov 2023 17:26:53 +0100 Subject: [PATCH 169/274] buffer result if out copacity is not enough --- src/IO/Lz4DeflatingWriteBuffer.cpp | 182 ++++++++++-------- src/IO/Lz4DeflatingWriteBuffer.h | 5 +- .../test_checking_s3_blobs_paranoid/test.py | 6 +- 3 files changed, 106 insertions(+), 87 deletions(-) diff --git a/src/IO/Lz4DeflatingWriteBuffer.cpp b/src/IO/Lz4DeflatingWriteBuffer.cpp index aab8dacef389..076b8c44f913 100644 --- a/src/IO/Lz4DeflatingWriteBuffer.cpp +++ b/src/IO/Lz4DeflatingWriteBuffer.cpp @@ -2,6 +2,59 @@ #include +namespace +{ + using namespace DB; + + class SinkToOut + { + public: + SinkToOut(WriteBuffer * out_, Memory<> & mem_, size_t guaranteed_capacity) + : sink(out_) + , tmp_out(mem_) + , cur_out(sink) + { + chassert(sink); + + if (sink->available() < guaranteed_capacity) + { + mem_.resize(guaranteed_capacity); + cur_out = &tmp_out; + } + } + + size_t getCapacity() + { + return cur_out->available(); + } + + BufferBase::Position getPosition() + { + return cur_out->position(); + } + + void advancePosition(size_t size) + { + chassert(size <= cur_out->available()); + cur_out->position() += size; + } + + ~SinkToOut() noexcept(false) + { + if (cur_out == sink) + return; + + sink->write(tmp_out.buffer().begin(), tmp_out.count()); + } + + private: + WriteBuffer * sink; + BufferWithOutsideMemory tmp_out; + WriteBuffer * cur_out; + }; +} + + namespace DB { namespace ErrorCodes @@ -13,9 +66,9 @@ Lz4DeflatingWriteBuffer::Lz4DeflatingWriteBuffer( std::unique_ptr out_, int compression_level, size_t buf_size, char * existing_memory, size_t alignment) : WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment) , in_data(nullptr) - , out_data(nullptr) , in_capacity(0) - , out_capacity(0) + , tmp_memory(buf_size) + { kPrefs = { {LZ4F_max256KB, @@ -36,8 +89,8 @@ Lz4DeflatingWriteBuffer::Lz4DeflatingWriteBuffer( if (LZ4F_isError(ret)) throw Exception( ErrorCodes::LZ4_ENCODER_FAILED, - "creation of LZ4 compression context failed. LZ4F version: {}", - LZ4F_VERSION); + "creation of LZ4 compression context failed. LZ4F version: {}, error: {}", + LZ4F_VERSION, LZ4F_getErrorName(ret)); } Lz4DeflatingWriteBuffer::~Lz4DeflatingWriteBuffer() @@ -54,107 +107,76 @@ void Lz4DeflatingWriteBuffer::nextImpl() in_data = reinterpret_cast(working_buffer.begin()); in_capacity = offset(); - out_capacity = out->buffer().end() - out->position(); - out_data = reinterpret_cast(out->position()); - - try + if (first_time) { - if (first_time) - { - if (out_capacity < LZ4F_HEADER_SIZE_MAX) - { - out->next(); - out_capacity = out->buffer().end() - out->position(); - out_data = reinterpret_cast(out->position()); - } + auto sink = SinkToOut(out.get(), tmp_memory, LZ4F_HEADER_SIZE_MAX); + chassert(sink.getCapacity() >= LZ4F_HEADER_SIZE_MAX); - /// write frame header and check for errors - size_t header_size = LZ4F_compressBegin(ctx, out_data, out_capacity, &kPrefs); + /// write frame header and check for errors + size_t header_size = LZ4F_compressBegin( + ctx, sink.getPosition(), sink.getCapacity(), &kPrefs); - if (LZ4F_isError(header_size)) - throw Exception( - ErrorCodes::LZ4_ENCODER_FAILED, - "LZ4 failed to start stream encoding. LZ4F version: {}", - LZ4F_VERSION); + if (LZ4F_isError(header_size)) + throw Exception( + ErrorCodes::LZ4_ENCODER_FAILED, + "LZ4 failed to start stream encoding. LZ4F version: {}, error: {}", + LZ4F_VERSION, LZ4F_getErrorName(header_size)); - out_capacity -= header_size; - out->position() = out->buffer().end() - out_capacity; - out_data = reinterpret_cast(out->position()); + sink.advancePosition(header_size); + first_time = false; + } - first_time = false; - } + do + { + /// Ensure that there is enough space for compressed block of minimal size + size_t min_compressed_block_size = LZ4F_compressBound(1, &kPrefs); - do - { - /// Ensure that there is enough space for compressed block of minimal size - size_t min_compressed_block_size = LZ4F_compressBound(1, &kPrefs); - if (out_capacity < min_compressed_block_size) - { - out->next(); - out_capacity = out->buffer().end() - out->position(); - out_data = reinterpret_cast(out->position()); - } + auto sink = SinkToOut(out.get(), tmp_memory, min_compressed_block_size); + chassert(sink.getCapacity() >= min_compressed_block_size); - /// LZ4F_compressUpdate compresses whole input buffer at once so we need to shink it manually - size_t cur_buffer_size = in_capacity; - if (out_capacity >= min_compressed_block_size) /// We cannot shrink the input buffer if it's already too small. - { - while (out_capacity < LZ4F_compressBound(cur_buffer_size, &kPrefs)) - cur_buffer_size /= 2; - } + /// LZ4F_compressUpdate compresses whole input buffer at once so we need to shink it manually + size_t cur_buffer_size = in_capacity; + if (sink.getCapacity() >= min_compressed_block_size) /// We cannot shrink the input buffer if it's already too small. + { + while (sink.getCapacity() < LZ4F_compressBound(cur_buffer_size, &kPrefs)) + cur_buffer_size /= 2; + } - size_t compressed_size = LZ4F_compressUpdate(ctx, out_data, out_capacity, in_data, cur_buffer_size, nullptr); + size_t compressed_size = LZ4F_compressUpdate( + ctx, sink.getPosition(), sink.getCapacity(), in_data, cur_buffer_size, nullptr); - if (LZ4F_isError(compressed_size)) - throw Exception( - ErrorCodes::LZ4_ENCODER_FAILED, - "LZ4 failed to encode stream. LZ4F version: {}", - LZ4F_VERSION); + if (LZ4F_isError(compressed_size)) + throw Exception( + ErrorCodes::LZ4_ENCODER_FAILED, + "LZ4 failed to encode stream. LZ4F version: {}, error {}, out_capacity {}", + LZ4F_VERSION, LZ4F_getErrorName(compressed_size), sink.getCapacity()); - in_capacity -= cur_buffer_size; - in_data = reinterpret_cast(working_buffer.end() - in_capacity); + in_capacity -= cur_buffer_size; + in_data = reinterpret_cast(working_buffer.end() - in_capacity); - out_capacity -= compressed_size; - out->position() = out->buffer().end() - out_capacity; - out_data = reinterpret_cast(out->position()); - } - while (in_capacity > 0); + sink.advancePosition(compressed_size); } - catch (...) - { - out->position() = out->buffer().begin(); - throw; - } - out->next(); - out_capacity = out->buffer().end() - out->position(); + while (in_capacity > 0); } void Lz4DeflatingWriteBuffer::finalizeBefore() { next(); - out_capacity = out->buffer().end() - out->position(); - out_data = reinterpret_cast(out->position()); - - if (out_capacity < LZ4F_compressBound(0, &kPrefs)) - { - out->next(); - out_capacity = out->buffer().end() - out->position(); - out_data = reinterpret_cast(out->position()); - } + auto suffix_size = LZ4F_compressBound(0, &kPrefs); + auto sink = SinkToOut(out.get(), tmp_memory, suffix_size); + chassert(sink.getCapacity() >= suffix_size); /// compression end - size_t end_size = LZ4F_compressEnd(ctx, out_data, out_capacity, nullptr); + size_t end_size = LZ4F_compressEnd(ctx, sink.getPosition(), sink.getCapacity(), nullptr); if (LZ4F_isError(end_size)) throw Exception( ErrorCodes::LZ4_ENCODER_FAILED, - "LZ4 failed to end stream encoding. LZ4F version: {}", - LZ4F_VERSION); + "LZ4 failed to end stream encoding. LZ4F version: {}, error {}, out_capacity {}", + LZ4F_VERSION, LZ4F_getErrorName(end_size), sink.getCapacity()); - out_capacity -= end_size; - out->position() = out->buffer().end() - out_capacity; - out_data = reinterpret_cast(out->position()); + sink.advancePosition(end_size); } void Lz4DeflatingWriteBuffer::finalizeAfter() diff --git a/src/IO/Lz4DeflatingWriteBuffer.h b/src/IO/Lz4DeflatingWriteBuffer.h index 68873b5f8ee2..65f4f0c73498 100644 --- a/src/IO/Lz4DeflatingWriteBuffer.h +++ b/src/IO/Lz4DeflatingWriteBuffer.h @@ -33,10 +33,9 @@ class Lz4DeflatingWriteBuffer : public WriteBufferWithOwnMemoryDecorator LZ4F_compressionContext_t ctx; void * in_data; - void * out_data; - size_t in_capacity; - size_t out_capacity; + + Memory<> tmp_memory; bool first_time = true; }; diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py index d6bcb3fb8f4b..606423793667 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/test.py +++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py @@ -97,9 +97,8 @@ def get_counters(node, query_id, log_type="ExceptionWhileProcessing"): ] -# Add "lz4" compression method in the list after https://github.com/ClickHouse/ClickHouse/issues/50975 is fixed @pytest.mark.parametrize( - "compression", ["none", "gzip", "br", "xz", "zstd", "bz2", "deflate"] + "compression", ["none", "gzip", "br", "xz", "zstd", "bz2", "deflate", "lz4"] ) def test_upload_s3_fail_create_multi_part_upload(cluster, broken_s3, compression): node = cluster.instances["node"] @@ -137,9 +136,8 @@ def test_upload_s3_fail_create_multi_part_upload(cluster, broken_s3, compression assert count_s3_errors == 1 -# Add "lz4" compression method in the list after https://github.com/ClickHouse/ClickHouse/issues/50975 is fixed @pytest.mark.parametrize( - "compression", ["none", "gzip", "br", "xz", "zstd", "bz2", "deflate"] + "compression", ["none", "gzip", "br", "xz", "zstd", "bz2", "deflate", "lz4"] ) def test_upload_s3_fail_upload_part_when_multi_part_upload( cluster, broken_s3, compression From 29e64347b939638b70d4b9397b9f2d1972f75008 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Fri, 17 Nov 2023 17:37:14 +0100 Subject: [PATCH 170/274] improve exception message --- src/Storages/MergeTree/DataPartsExchange.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 0192fb1868bf..43642b3ba79a 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -398,7 +398,8 @@ MergeTreeData::DataPartPtr Service::findPart(const String & name) if (!pred_result) throw Exception( ErrorCodes::ABORTED, - "Part {} is in PreActive state for {} ms. Another host has to be asked.", + "Could not exchange part {} as it's in preActive state ({} ms) and it uses zero copy replication. " + "This is expected behaviour and the client will retry fetching the part automatically.", name, wait_timeout_ms); return part; From 6366819f120f5fd108acd60079921ff6aa595116 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 17 Nov 2023 16:52:20 +0000 Subject: [PATCH 171/274] Fix generating deep nested columns in CapnProto/Protobuf schemas --- src/Formats/StructureToFormatSchemaUtils.cpp | 4 ++ ...apnp_protobuf_auto_schema_nested.reference | 52 +++++++++++++++++++ ...02920_capnp_protobuf_auto_schema_nested.sh | 21 ++++++++ 3 files changed, 77 insertions(+) create mode 100644 tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.reference create mode 100755 tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.sh diff --git a/src/Formats/StructureToFormatSchemaUtils.cpp b/src/Formats/StructureToFormatSchemaUtils.cpp index 47701fa4f812..c56ff821a4a2 100644 --- a/src/Formats/StructureToFormatSchemaUtils.cpp +++ b/src/Formats/StructureToFormatSchemaUtils.cpp @@ -96,6 +96,10 @@ NamesAndTypesList collectNested(const NamesAndTypesList & names_and_types, bool nested[field_name].emplace_back(nested_name, type); } + /// Collect nested recursively. + for (auto & [field_name, elements] : nested) + elements = collectNested(elements, allow_split_by_underscore, format_name); + for (const auto & [field_name, elements]: nested) result.emplace_back(field_name, std::make_shared(elements.getTypes(), elements.getNames())); diff --git a/tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.reference b/tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.reference new file mode 100644 index 000000000000..9874bc571423 --- /dev/null +++ b/tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.reference @@ -0,0 +1,52 @@ + +message Message +{ + message H + { + uint32 k = 1; + } + H h = 1; + message A + { + uint32 g = 1; + message B + { + uint32 c = 1; + uint32 f = 2; + message D + { + uint32 e = 1; + } + D d = 3; + } + B b = 2; + } + A a = 2; +} +46 (45,(42,44,43)) + +struct Message +{ + struct H + { + k @0 : UInt8; + } + h @0 : H; + struct A + { + g @0 : UInt8; + struct B + { + c @0 : UInt8; + f @1 : UInt8; + struct D + { + e @0 : UInt8; + } + d @2 : D; + } + b @1 : B; + } + a @1 : A; +} +(46) (45,(42,44,(43))) diff --git a/tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.sh b/tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.sh new file mode 100755 index 000000000000..aee6b8667197 --- /dev/null +++ b/tests/queries/0_stateless/02920_capnp_protobuf_auto_schema_nested.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +SCHEMA_FILE=$CLICKHOUSE_TEST_UNIQUE_NAME-schema +FILE=$CLICKHOUSE_TEST_UNIQUE_NAME + +$CLICKHOUSE_LOCAL -q "select 42 as \`a.b.c\`, 43 as \`a.b.d.e\`, 44 as \`a.b.f\`, 45 as \`a.g\`, 46 as \`h.k\` format Protobuf settings output_format_schema='$SCHEMA_FILE.proto'" > $FILE.pb +tail -n +2 $SCHEMA_FILE.proto +$CLICKHOUSE_LOCAL -q "select * from file('$FILE.pb') settings format_schema='$SCHEMA_FILE:Message'" + +$CLICKHOUSE_LOCAL -q "select 42 as a_b_c, 43 as a_b_d_e, 44 as a_b_f, 45 as a_g, 46 as h_k format CapnProto settings output_format_schema='$SCHEMA_FILE.capnp'" > $FILE.capnp +tail -n +2 $SCHEMA_FILE.capnp +$CLICKHOUSE_LOCAL -q "select * from file('$FILE.capnp') settings format_schema='$SCHEMA_FILE:Message'" + +rm $SCHEMA_FILE* +rm $FILE.* + From fcce5409f47c4db15ffedded1789e34f9e528010 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 16 Nov 2023 15:17:19 +0000 Subject: [PATCH 172/274] Simplify --- cmake/cpu_features.cmake | 50 ++++++++++++++-------------- contrib/aws-cmake/AwsSIMD.cmake | 2 +- contrib/fastops-cmake/CMakeLists.txt | 4 +-- contrib/rocksdb-cmake/CMakeLists.txt | 6 ++-- 4 files changed, 29 insertions(+), 33 deletions(-) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index 765e36403ad3..484c139f1a7c 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -134,60 +134,60 @@ elseif (ARCH_AMD64) # ClickHouse can be cross-compiled (e.g. on an ARM host for x86) but it is also possible to build ClickHouse on x86 w/o AVX for x86 w/ # AVX. We only assume that the compiler can emit certain SIMD instructions, we don't care if the host system is able to run the binary. - SET (HAVE_SSSE3 1) - SET (HAVE_SSE41 1) - SET (HAVE_SSE42 1) - SET (HAVE_PCLMULQDQ 1) - SET (HAVE_POPCNT 1) - SET (HAVE_AVX 1) - SET (HAVE_AVX2 1) - SET (HAVE_AVX512 1) - SET (HAVE_AVX512_VBMI 1) - SET (HAVE_BMI 1) - SET (HAVE_BMI2 1) - - if (HAVE_SSSE3 AND ENABLE_SSSE3) + if (ENABLE_SSSE3) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mssse3") endif () - if (HAVE_SSE41 AND ENABLE_SSE41) + + if (ENABLE_SSE41) set (COMPILER_FLAGS "${COMPILER_FLAGS} -msse4.1") endif () - if (HAVE_SSE42 AND ENABLE_SSE42) + + if (ENABLE_SSE42) set (COMPILER_FLAGS "${COMPILER_FLAGS} -msse4.2") endif () - if (HAVE_PCLMULQDQ AND ENABLE_PCLMULQDQ) + + if (ENABLE_PCLMULQDQ) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mpclmul") endif () - if (HAVE_POPCNT AND ENABLE_POPCNT) + + if (ENABLE_POPCNT) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mpopcnt") endif () - if (HAVE_AVX AND ENABLE_AVX) + + if (ENABLE_AVX) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx") endif () - if (HAVE_AVX2 AND ENABLE_AVX2) + + if (ENABLE_AVX2) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx2") endif () - if (HAVE_AVX512 AND ENABLE_AVX512) + + if (ENABLE_AVX512) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512f -mavx512bw -mavx512vl") endif () - if (HAVE_AVX512 AND ENABLE_AVX512 AND HAVE_AVX512_VBMI AND ENABLE_AVX512_VBMI) + + if (ENABLE_AVX512 AND ENABLE_AVX512_VBMI) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512vbmi") endif () - if (HAVE_BMI AND ENABLE_BMI) + + if (ENABLE_BMI) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi") endif () - if (HAVE_BMI2 AND HAVE_AVX2 AND ENABLE_AVX2 AND ENABLE_BMI2) + + if (ENABLE_AVX2 AND ENABLE_BMI2) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi2") endif () + if (ENABLE_AVX512_FOR_SPEC_OP) set (X86_INTRINSICS_FLAGS "") - if (HAVE_BMI) + if (1) set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mbmi") endif () - if (HAVE_AVX512) + if (1) set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mavx512f -mavx512bw -mavx512vl -mprefer-vector-width=256") endif () endif () + else () # RISC-V + exotic platforms endif () diff --git a/contrib/aws-cmake/AwsSIMD.cmake b/contrib/aws-cmake/AwsSIMD.cmake index a2f50f27d4e4..24f7628e86f7 100644 --- a/contrib/aws-cmake/AwsSIMD.cmake +++ b/contrib/aws-cmake/AwsSIMD.cmake @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0. if (USE_CPU_EXTENSIONS) - if (HAVE_AVX2) + if (ENABLE_AVX2) set (AVX2_CFLAGS "-mavx -mavx2") set (HAVE_AVX2_INTRINSICS 1) set (HAVE_MM256_EXTRACT_EPI64 1) diff --git a/contrib/fastops-cmake/CMakeLists.txt b/contrib/fastops-cmake/CMakeLists.txt index e9aa4803583d..1b09b736b2a8 100644 --- a/contrib/fastops-cmake/CMakeLists.txt +++ b/contrib/fastops-cmake/CMakeLists.txt @@ -13,12 +13,10 @@ set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/fastops") set(SRCS "") -if(HAVE_AVX) +if(ARCH_AMD64) set (SRCS ${SRCS} "${LIBRARY_DIR}/fastops/avx/ops_avx.cpp") set_source_files_properties("${LIBRARY_DIR}/fastops/avx/ops_avx.cpp" PROPERTIES COMPILE_FLAGS "-mavx -DNO_AVX2") -endif() -if(HAVE_AVX2) set (SRCS ${SRCS} "${LIBRARY_DIR}/fastops/avx2/ops_avx2.cpp") set_source_files_properties("${LIBRARY_DIR}/fastops/avx2/ops_avx2.cpp" PROPERTIES COMPILE_FLAGS "-mavx2 -mfma") endif() diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt index 2b6c48f0b383..7d7666dff87d 100644 --- a/contrib/rocksdb-cmake/CMakeLists.txt +++ b/contrib/rocksdb-cmake/CMakeLists.txt @@ -93,11 +93,9 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm64|ARM64") endif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm64|ARM64") -if(HAVE_SSE42) +if(ENABLE_AVX2 AND ENABLE_PCLMULQDQ) add_definitions(-DHAVE_SSE42) add_definitions(-DHAVE_PCLMUL) -elseif(FORCE_SSE42) - message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled") endif() set (HAVE_THREAD_LOCAL 1) @@ -429,7 +427,7 @@ set(SOURCES ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/util/memarena.cc rocksdb_build_version.cc) -if(HAVE_SSE42) +if(ENABLE_SSE42 AND ENABLE_PCLMULQDQ) set_source_files_properties( "${ROCKSDB_SOURCE_DIR}/util/crc32c.cc" PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul") From c51429b1ef006b2bced059624312af5c1dc1f07f Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 16 Nov 2023 15:24:47 +0000 Subject: [PATCH 173/274] Simplify more --- cmake/cpu_features.cmake | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index 484c139f1a7c..cfa9c314bc04 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -150,6 +150,10 @@ elseif (ARCH_AMD64) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mpclmul") endif () + if (ENABLE_BMI) + set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi") + endif () + if (ENABLE_POPCNT) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mpopcnt") endif () @@ -160,32 +164,20 @@ elseif (ARCH_AMD64) if (ENABLE_AVX2) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx2") + if (ENABLE_BMI2) + set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi2") + endif () endif () if (ENABLE_AVX512) set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512f -mavx512bw -mavx512vl") - endif () - - if (ENABLE_AVX512 AND ENABLE_AVX512_VBMI) - set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512vbmi") - endif () - - if (ENABLE_BMI) - set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi") - endif () - - if (ENABLE_AVX2 AND ENABLE_BMI2) - set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi2") + if (ENABLE_AVX512_VBMI) + set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512vbmi") + endif () endif () if (ENABLE_AVX512_FOR_SPEC_OP) - set (X86_INTRINSICS_FLAGS "") - if (1) - set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mbmi") - endif () - if (1) - set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mavx512f -mavx512bw -mavx512vl -mprefer-vector-width=256") - endif () + set (X86_INTRINSICS_FLAGS "-mbmi -mavx512f -mavx512bw -mavx512vl -mprefer-vector-width=256") endif () else () From a18b7155919d7a7ffc10814043dde3e2c913f620 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Sat, 18 Nov 2023 15:44:45 +0000 Subject: [PATCH 174/274] Fix a bug --- src/Storages/AlterCommands.cpp | 17 +++++++++++++++-- .../02916_addcolumn_nested.reference | 1 + .../0_stateless/02916_addcolumn_nested.sql | 5 +++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 7eeaa2d45941..f5293c52bb06 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -33,6 +33,8 @@ #include #include +#include + namespace DB { @@ -403,10 +405,21 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) const auto transformed_columns = temporary_metadata.columns.getAll(); - for (auto it = transformed_columns.rbegin(); it != transformed_columns.rend(); it++) + auto add_column = [&](const String & name) { - const auto & transformed_column = temporary_metadata.columns.get(it->name); + const auto & transformed_column = temporary_metadata.columns.get(name); metadata.columns.add(transformed_column, after_column, first); + }; + + if (!after_column.empty() || first) + { + for (const auto & col: transformed_columns | std::views::reverse) + add_column(col.name); + } + else + { + for (const auto & col: transformed_columns) + add_column(col.name); } } else diff --git a/tests/queries/0_stateless/02916_addcolumn_nested.reference b/tests/queries/0_stateless/02916_addcolumn_nested.reference index 869d4336c62f..7d79cd8731f2 100644 --- a/tests/queries/0_stateless/02916_addcolumn_nested.reference +++ b/tests/queries/0_stateless/02916_addcolumn_nested.reference @@ -1,3 +1,4 @@ CREATE TABLE default.nested_table\n(\n `id` UInt64,\n `first` Nested(a Int8, b String)\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192 CREATE TABLE default.nested_table\n(\n `id` UInt64,\n `second.c` Array(Int8),\n `second.d` Array(String),\n `first` Nested(a Int8, b String)\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192 CREATE TABLE default.nested_table\n(\n `third` Nested(e Int8, f String),\n `id` UInt64,\n `second.c` Array(Int8),\n `second.d` Array(String),\n `first` Nested(a Int8, b String)\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.nested_table\n(\n `third` Nested(e Int8, f String),\n `id` UInt64,\n `second.c` Array(Int8),\n `second.d` Array(String),\n `first` Nested(a Int8, b String),\n `fourth.g` Array(Int8),\n `fourth.h` Array(String)\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02916_addcolumn_nested.sql b/tests/queries/0_stateless/02916_addcolumn_nested.sql index b23854824b5d..1e64fca6a154 100644 --- a/tests/queries/0_stateless/02916_addcolumn_nested.sql +++ b/tests/queries/0_stateless/02916_addcolumn_nested.sql @@ -14,4 +14,9 @@ SET flatten_nested = 0; ALTER TABLE nested_table ADD COLUMN third Nested(e Int8, f String) FIRST; SHOW CREATE nested_table; +SET flatten_nested = 1; + +ALTER TABLE nested_table ADD COLUMN fourth Nested(g Int8, h String); +SHOW CREATE nested_table; + DROP TABLE nested_table; From 773715a562cdcba9ac9dc0730a773201634c7326 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Sat, 18 Nov 2023 17:30:49 +0100 Subject: [PATCH 175/274] finalize tmp_out --- src/IO/BufferWithOwnMemory.h | 6 ++++++ src/IO/Lz4DeflatingWriteBuffer.cpp | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/IO/BufferWithOwnMemory.h b/src/IO/BufferWithOwnMemory.h index 39c83e9167c2..5c9a69893dfc 100644 --- a/src/IO/BufferWithOwnMemory.h +++ b/src/IO/BufferWithOwnMemory.h @@ -191,6 +191,12 @@ class BufferWithOutsideMemory : public Base memory.resize(2 * prev_size + 1); Base::set(memory.data() + prev_size, memory.size() - prev_size, 0); } + + void finalizeImpl() final + { + /// there is no need to allocate twice more memory at finalize() + /// So make that call no op, do not call here nextImpl() + } }; } diff --git a/src/IO/Lz4DeflatingWriteBuffer.cpp b/src/IO/Lz4DeflatingWriteBuffer.cpp index 076b8c44f913..e952e6400ecd 100644 --- a/src/IO/Lz4DeflatingWriteBuffer.cpp +++ b/src/IO/Lz4DeflatingWriteBuffer.cpp @@ -41,6 +41,8 @@ namespace ~SinkToOut() noexcept(false) { + tmp_out.finalize(); + if (cur_out == sink) return; From d1c56e3dab845baefe0514f86c1b49be02538a6c Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Sat, 18 Nov 2023 17:40:15 +0000 Subject: [PATCH 176/274] Send fatal logs by default in clickhouse-local --- programs/local/LocalServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 36020d22cc0f..f3b551b08d25 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -612,7 +612,7 @@ void LocalServer::processConfig() else if (logging || is_interactive) { config().setString("logger", "logger"); - auto log_level_default = is_interactive && !logging ? "none" : level; + auto log_level_default = is_interactive && !logging ? "fatal" : level; config().setString("logger.level", config().getString("log-level", config().getString("send_logs_level", log_level_default))); buildLoggers(config(), logger(), "clickhouse-local"); logging_initialized = true; From 6eedd1649db62fcfbeab6eda8039e6217e155c49 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Sat, 18 Nov 2023 17:55:38 +0000 Subject: [PATCH 177/274] Resubmit: Better except for SSL authentication --- src/Server/TCPHandler.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 1da9806b4f5c..1c2c16496f0d 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1431,8 +1431,11 @@ void TCPHandler::receiveHello() getClientAddress(client_info)); return; } - catch (...) + catch (const Exception & e) { + if (e.code() != DB::ErrorCodes::AUTHENTICATION_FAILED) + throw; + tryLogCurrentException(log, "SSL authentication failed, falling back to password authentication"); } } From d56cbda1850974da1f4cf7d66ef52d002d3b7244 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Nov 2023 19:07:59 +0100 Subject: [PATCH 178/274] Add metrics for the number of queued jobs, which is useful for the IO thread pool --- programs/benchmark/Benchmark.cpp | 3 +- programs/copier/ClusterCopier.cpp | 3 +- src/Backups/BackupsWorker.cpp | 7 ++- src/Common/AsyncLoader.cpp | 1 + src/Common/AsyncLoader.h | 5 ++- src/Common/CurrentMetrics.cpp | 43 +++++++++++++++++++ src/Common/ThreadPool.cpp | 40 ++++++++--------- src/Common/ThreadPool.h | 14 ++++-- src/Common/examples/parallel_aggregation.cpp | 3 +- src/Common/examples/parallel_aggregation2.cpp | 3 +- .../examples/thread_creation_latency.cpp | 7 +-- src/Common/tests/gtest_async_loader.cpp | 2 + .../gtest_thread_pool_concurrent_wait.cpp | 5 ++- .../tests/gtest_thread_pool_global_full.cpp | 7 +-- src/Common/tests/gtest_thread_pool_limit.cpp | 3 +- src/Common/tests/gtest_thread_pool_loop.cpp | 3 +- .../gtest_thread_pool_schedule_exception.cpp | 5 ++- src/Coordination/Standalone/Context.cpp | 1 + src/Databases/DatabaseOnDisk.cpp | 3 +- src/Databases/DatabaseOrdinary.cpp | 3 +- src/Databases/TablesLoader.cpp | 3 +- .../CacheDictionaryUpdateQueue.cpp | 3 +- src/Dictionaries/HashedDictionary.cpp | 5 ++- src/Disks/IDisk.h | 5 ++- src/Disks/IO/ThreadPoolReader.cpp | 3 +- src/Disks/IO/ThreadPoolRemoteFSReader.cpp | 2 + .../AzureBlobStorage/AzureObjectStorage.cpp | 4 +- ...jectStorageRemoteMetadataRestoreHelper.cpp | 5 ++- .../ObjectStorageIteratorAsync.cpp | 1 + .../ObjectStorageIteratorAsync.h | 4 +- .../ObjectStorages/S3/S3ObjectStorage.cpp | 2 + src/IO/SharedThreadPools.cpp | 20 ++++++--- src/IO/SharedThreadPools.h | 5 ++- src/Interpreters/Aggregator.cpp | 7 +-- src/Interpreters/AsynchronousInsertQueue.cpp | 3 +- src/Interpreters/Context.cpp | 9 ++-- src/Interpreters/DDLWorker.cpp | 7 +-- src/Interpreters/DatabaseCatalog.cpp | 3 +- src/Interpreters/InterpreterSystemQuery.cpp | 3 +- src/Interpreters/loadMetadata.cpp | 5 ++- src/Interpreters/threadPoolCallbackRunner.h | 6 +-- src/Processors/Executors/PipelineExecutor.cpp | 3 +- .../Formats/Impl/DWARFBlockInputFormat.cpp | 3 +- .../Impl/ParallelFormattingOutputFormat.h | 3 +- .../Formats/Impl/ParallelParsingInputFormat.h | 3 +- .../Formats/Impl/ParquetBlockInputFormat.cpp | 3 +- .../Formats/Impl/ParquetBlockOutputFormat.cpp | 5 ++- .../Transforms/AggregatingTransform.h | 2 + src/Storages/Distributed/DistributedSink.cpp | 4 +- src/Storages/Hive/StorageHive.cpp | 3 +- .../MergeTree/MergeTreeBackgroundExecutor.cpp | 3 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 2 + src/Storages/StorageAzureBlob.cpp | 3 +- src/Storages/StorageDistributed.cpp | 3 +- src/Storages/StorageS3.cpp | 7 +-- src/Storages/System/StorageSystemReplicas.cpp | 3 +- utils/keeper-bench/Runner.cpp | 5 ++- 57 files changed, 222 insertions(+), 99 deletions(-) diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index ed3d4a1ea690..d6b8b38d84df 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -46,6 +46,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } namespace DB @@ -107,7 +108,7 @@ class Benchmark : public Poco::Util::Application settings(settings_), shared_context(Context::createShared()), global_context(Context::createGlobal(shared_context.get())), - pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, concurrency) + pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, concurrency) { const auto secure = secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable; size_t connections_cnt = std::max(ports_.size(), hosts_.size()); diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index b2b4970d04f9..7d58f35f62f2 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -25,6 +25,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } namespace DB @@ -200,7 +201,7 @@ void ClusterCopier::discoverTablePartitions(const ConnectionTimeouts & timeouts, { /// Fetch partitions list from a shard { - ThreadPool thread_pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, num_threads ? num_threads : 2 * getNumberOfPhysicalCPUCores()); + ThreadPool thread_pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, num_threads ? num_threads : 2 * getNumberOfPhysicalCPUCores()); for (const TaskShardPtr & task_shard : task_table.all_shards) thread_pool.scheduleOrThrowOnError([this, timeouts, task_shard]() diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index f6020deabecf..4e24269fb254 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -31,8 +31,10 @@ namespace CurrentMetrics { extern const Metric BackupsThreads; extern const Metric BackupsThreadsActive; + extern const Metric BackupsThreadsScheduled; extern const Metric RestoreThreads; extern const Metric RestoreThreadsActive; + extern const Metric RestoreThreadsScheduled; } namespace DB @@ -264,6 +266,7 @@ class BackupsWorker::ThreadPools CurrentMetrics::Metric metric_threads; CurrentMetrics::Metric metric_active_threads; + CurrentMetrics::Metric metric_scheduled_threads; size_t max_threads = 0; /// What to do with a new job if a corresponding thread pool is already running `max_threads` jobs: @@ -279,6 +282,7 @@ class BackupsWorker::ThreadPools { metric_threads = CurrentMetrics::BackupsThreads; metric_active_threads = CurrentMetrics::BackupsThreadsActive; + metric_active_threads = CurrentMetrics::BackupsThreadsScheduled; max_threads = num_backup_threads; /// We don't use thread pool queues for thread pools with a lot of tasks otherwise that queue could be memory-wasting. use_queue = (thread_pool_id != ThreadPoolId::BACKUP_COPY_FILES); @@ -291,6 +295,7 @@ class BackupsWorker::ThreadPools { metric_threads = CurrentMetrics::RestoreThreads; metric_active_threads = CurrentMetrics::RestoreThreadsActive; + metric_active_threads = CurrentMetrics::RestoreThreadsScheduled; max_threads = num_restore_threads; use_queue = (thread_pool_id != ThreadPoolId::RESTORE_TABLES_DATA); break; @@ -301,7 +306,7 @@ class BackupsWorker::ThreadPools chassert(max_threads != 0); size_t max_free_threads = 0; size_t queue_size = use_queue ? 0 : max_threads; - auto thread_pool = std::make_unique(metric_threads, metric_active_threads, max_threads, max_free_threads, queue_size); + auto thread_pool = std::make_unique(metric_threads, metric_active_threads, metric_scheduled_threads, max_threads, max_free_threads, queue_size); auto * thread_pool_ptr = thread_pool.get(); thread_pools.emplace(thread_pool_id, std::move(thread_pool)); return *thread_pool_ptr; diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp index 2e96d3eab7d3..cff34ac036dc 100644 --- a/src/Common/AsyncLoader.cpp +++ b/src/Common/AsyncLoader.cpp @@ -179,6 +179,7 @@ AsyncLoader::AsyncLoader(std::vector pool_initializers, bool lo .thread_pool = std::make_unique( init.metric_threads, init.metric_active_threads, + init.metric_scheduled_threads, init.max_threads, /* max_free_threads = */ 0, init.max_threads), diff --git a/src/Common/AsyncLoader.h b/src/Common/AsyncLoader.h index 77905319f001..0496549001ee 100644 --- a/src/Common/AsyncLoader.h +++ b/src/Common/AsyncLoader.h @@ -271,8 +271,8 @@ inline LoadTaskPtrs joinTasks(const LoadTaskPtrs & tasks1, const LoadTaskPtrs & // Basic usage example: // // Start async_loader with two thread pools (0=fg, 1=bg): // AsyncLoader async_loader({ -// {"FgPool", CurrentMetrics::AsyncLoaderThreads, CurrentMetrics::AsyncLoaderThreadsActive, .max_threads = 2, .priority{0}} -// {"BgPool", CurrentMetrics::AsyncLoaderThreads, CurrentMetrics::AsyncLoaderThreadsActive, .max_threads = 1, .priority{1}} +// {"FgPool", CurrentMetrics::AsyncLoaderThreads, ..., .max_threads = 2, .priority{0}} +// {"BgPool", CurrentMetrics::AsyncLoaderThreads, ..., .max_threads = 1, .priority{1}} // }); // // // Create and schedule a task consisting of three jobs. Job1 has no dependencies and is run first. @@ -368,6 +368,7 @@ class AsyncLoader : private boost::noncopyable String name; Metric metric_threads; Metric metric_active_threads; + Metric metric_scheduled_threads; size_t max_threads; Priority priority; }; diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index c929f4d86e2b..5a4b6e80f75b 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -71,92 +71,135 @@ M(RWLockActiveWriters, "Number of threads holding write lock in a table RWLock.") \ M(GlobalThread, "Number of threads in global thread pool.") \ M(GlobalThreadActive, "Number of threads in global thread pool running a task.") \ + M(GlobalThreadScheduled, "Number of queued or active jobs in global thread pool.") \ M(LocalThread, "Number of threads in local thread pools. The threads in local thread pools are taken from the global thread pool.") \ M(LocalThreadActive, "Number of threads in local thread pools running a task.") \ + M(LocalThreadScheduled, "Number of queued or active jobs in local thread pools.") \ M(MergeTreeDataSelectExecutorThreads, "Number of threads in the MergeTreeDataSelectExecutor thread pool.") \ M(MergeTreeDataSelectExecutorThreadsActive, "Number of threads in the MergeTreeDataSelectExecutor thread pool running a task.") \ + M(MergeTreeDataSelectExecutorThreadsScheduled, "Number of queued or active jobs in the MergeTreeDataSelectExecutor thread pool.") \ M(BackupsThreads, "Number of threads in the thread pool for BACKUP.") \ M(BackupsThreadsActive, "Number of threads in thread pool for BACKUP running a task.") \ + M(BackupsThreadsScheduled, "Number of queued or active jobs for BACKUP.") \ M(RestoreThreads, "Number of threads in the thread pool for RESTORE.") \ M(RestoreThreadsActive, "Number of threads in the thread pool for RESTORE running a task.") \ + M(RestoreThreadsScheduled, "Number of queued or active jobs for RESTORE.") \ M(MarksLoaderThreads, "Number of threads in thread pool for loading marks.") \ M(MarksLoaderThreadsActive, "Number of threads in the thread pool for loading marks running a task.") \ + M(MarksLoaderThreadsScheduled, "Number of queued or active jobs in the thread pool for loading marks.") \ M(IOPrefetchThreads, "Number of threads in the IO prefertch thread pool.") \ M(IOPrefetchThreadsActive, "Number of threads in the IO prefetch thread pool running a task.") \ + M(IOPrefetchThreadsScheduled, "Number of queued or active jobs in the IO prefetch thread pool.") \ M(IOWriterThreads, "Number of threads in the IO writer thread pool.") \ M(IOWriterThreadsActive, "Number of threads in the IO writer thread pool running a task.") \ + M(IOWriterThreadsScheduled, "Number of queued or active jobs in the IO writer thread pool.") \ M(IOThreads, "Number of threads in the IO thread pool.") \ M(IOThreadsActive, "Number of threads in the IO thread pool running a task.") \ + M(IOThreadsScheduled, "Number of queued or active jobs in the IO thread pool.") \ M(ThreadPoolRemoteFSReaderThreads, "Number of threads in the thread pool for remote_filesystem_read_method=threadpool.") \ M(ThreadPoolRemoteFSReaderThreadsActive, "Number of threads in the thread pool for remote_filesystem_read_method=threadpool running a task.") \ + M(ThreadPoolRemoteFSReaderThreadsScheduled, "Number of queued or active jobs in the thread pool for remote_filesystem_read_method=threadpool.") \ M(ThreadPoolFSReaderThreads, "Number of threads in the thread pool for local_filesystem_read_method=threadpool.") \ M(ThreadPoolFSReaderThreadsActive, "Number of threads in the thread pool for local_filesystem_read_method=threadpool running a task.") \ + M(ThreadPoolFSReaderThreadsScheduled, "Number of queued or active jobs in the thread pool for local_filesystem_read_method=threadpool.") \ M(BackupsIOThreads, "Number of threads in the BackupsIO thread pool.") \ M(BackupsIOThreadsActive, "Number of threads in the BackupsIO thread pool running a task.") \ + M(BackupsIOThreadsScheduled, "Number of queued or active jobs in the BackupsIO thread pool.") \ M(DiskObjectStorageAsyncThreads, "Obsolete metric, shows nothing.") \ M(DiskObjectStorageAsyncThreadsActive, "Obsolete metric, shows nothing.") \ M(StorageHiveThreads, "Number of threads in the StorageHive thread pool.") \ M(StorageHiveThreadsActive, "Number of threads in the StorageHive thread pool running a task.") \ + M(StorageHiveThreadsScheduled, "Number of queued or active jobs in the StorageHive thread pool.") \ M(TablesLoaderThreads, "Number of threads in the tables loader thread pool.") \ M(TablesLoaderThreadsActive, "Number of threads in the tables loader thread pool running a task.") \ + M(TablesLoaderThreadsScheduled, "Number of queued or active jobs in the tables loader thread pool.") \ M(DatabaseOrdinaryThreads, "Number of threads in the Ordinary database thread pool.") \ M(DatabaseOrdinaryThreadsActive, "Number of threads in the Ordinary database thread pool running a task.") \ + M(DatabaseOrdinaryThreadsScheduled, "Number of queued or active jobs in the Ordinary database thread pool.") \ M(DatabaseOnDiskThreads, "Number of threads in the DatabaseOnDisk thread pool.") \ M(DatabaseOnDiskThreadsActive, "Number of threads in the DatabaseOnDisk thread pool running a task.") \ + M(DatabaseOnDiskThreadsScheduled, "Number of queued or active jobs in the DatabaseOnDisk thread pool.") \ M(DatabaseCatalogThreads, "Number of threads in the DatabaseCatalog thread pool.") \ M(DatabaseCatalogThreadsActive, "Number of threads in the DatabaseCatalog thread pool running a task.") \ + M(DatabaseCatalogThreadsScheduled, "Number of queued or active jobs in the DatabaseCatalog thread pool.") \ M(DestroyAggregatesThreads, "Number of threads in the thread pool for destroy aggregate states.") \ M(DestroyAggregatesThreadsActive, "Number of threads in the thread pool for destroy aggregate states running a task.") \ + M(DestroyAggregatesThreadsScheduled, "Number of queued or active jobs in the thread pool for destroy aggregate states.") \ M(HashedDictionaryThreads, "Number of threads in the HashedDictionary thread pool.") \ M(HashedDictionaryThreadsActive, "Number of threads in the HashedDictionary thread pool running a task.") \ + M(HashedDictionaryThreadsScheduled, "Number of queued or active jobs in the HashedDictionary thread pool.") \ M(CacheDictionaryThreads, "Number of threads in the CacheDictionary thread pool.") \ M(CacheDictionaryThreadsActive, "Number of threads in the CacheDictionary thread pool running a task.") \ + M(CacheDictionaryThreadsScheduled, "Number of queued or active jobs in the CacheDictionary thread pool.") \ M(ParallelFormattingOutputFormatThreads, "Number of threads in the ParallelFormattingOutputFormatThreads thread pool.") \ M(ParallelFormattingOutputFormatThreadsActive, "Number of threads in the ParallelFormattingOutputFormatThreads thread pool running a task.") \ + M(ParallelFormattingOutputFormatThreadsScheduled, "Number of queued or active jobs in the ParallelFormattingOutputFormatThreads thread pool.") \ M(ParallelParsingInputFormatThreads, "Number of threads in the ParallelParsingInputFormat thread pool.") \ M(ParallelParsingInputFormatThreadsActive, "Number of threads in the ParallelParsingInputFormat thread pool running a task.") \ + M(ParallelParsingInputFormatThreadsScheduled, "Number of queued or active jobs in the ParallelParsingInputFormat thread pool.") \ M(MergeTreeBackgroundExecutorThreads, "Number of threads in the MergeTreeBackgroundExecutor thread pool.") \ M(MergeTreeBackgroundExecutorThreadsActive, "Number of threads in the MergeTreeBackgroundExecutor thread pool running a task.") \ + M(MergeTreeBackgroundExecutorThreadsScheduled, "Number of queued or active jobs in the MergeTreeBackgroundExecutor thread pool.") \ M(AsynchronousInsertThreads, "Number of threads in the AsynchronousInsert thread pool.") \ M(AsynchronousInsertThreadsActive, "Number of threads in the AsynchronousInsert thread pool running a task.") \ + M(AsynchronousInsertThreadsScheduled, "Number of queued or active jobs in the AsynchronousInsert thread pool.") \ M(StartupSystemTablesThreads, "Number of threads in the StartupSystemTables thread pool.") \ M(StartupSystemTablesThreadsActive, "Number of threads in the StartupSystemTables thread pool running a task.") \ + M(StartupSystemTablesThreadsScheduled, "Number of queued or active jobs in the StartupSystemTables thread pool.") \ M(AggregatorThreads, "Number of threads in the Aggregator thread pool.") \ M(AggregatorThreadsActive, "Number of threads in the Aggregator thread pool running a task.") \ + M(AggregatorThreadsScheduled, "Number of queued or active jobs in the Aggregator thread pool.") \ M(DDLWorkerThreads, "Number of threads in the DDLWorker thread pool for ON CLUSTER queries.") \ M(DDLWorkerThreadsActive, "Number of threads in the DDLWORKER thread pool for ON CLUSTER queries running a task.") \ + M(DDLWorkerThreadsScheduled, "Number of queued or active jobs in the DDLWORKER thread pool for ON CLUSTER queries.") \ M(StorageDistributedThreads, "Number of threads in the StorageDistributed thread pool.") \ M(StorageDistributedThreadsActive, "Number of threads in the StorageDistributed thread pool running a task.") \ + M(StorageDistributedThreadsScheduled, "Number of queued or active jobs in the StorageDistributed thread pool.") \ M(DistributedInsertThreads, "Number of threads used for INSERT into Distributed.") \ M(DistributedInsertThreadsActive, "Number of threads used for INSERT into Distributed running a task.") \ + M(DistributedInsertThreadsScheduled, "Number of queued or active jobs used for INSERT into Distributed.") \ M(StorageS3Threads, "Number of threads in the StorageS3 thread pool.") \ M(StorageS3ThreadsActive, "Number of threads in the StorageS3 thread pool running a task.") \ + M(StorageS3ThreadsScheduled, "Number of queued or active jobs in the StorageS3 thread pool.") \ M(ObjectStorageS3Threads, "Number of threads in the S3ObjectStorage thread pool.") \ M(ObjectStorageS3ThreadsActive, "Number of threads in the S3ObjectStorage thread pool running a task.") \ + M(ObjectStorageS3ThreadsScheduled, "Number of queued or active jobs in the S3ObjectStorage thread pool.") \ M(ObjectStorageAzureThreads, "Number of threads in the AzureObjectStorage thread pool.") \ M(ObjectStorageAzureThreadsActive, "Number of threads in the AzureObjectStorage thread pool running a task.") \ + M(ObjectStorageAzureThreadsScheduled, "Number of queued or active jobs in the AzureObjectStorage thread pool.") \ M(MergeTreePartsLoaderThreads, "Number of threads in the MergeTree parts loader thread pool.") \ M(MergeTreePartsLoaderThreadsActive, "Number of threads in the MergeTree parts loader thread pool running a task.") \ + M(MergeTreePartsLoaderThreadsScheduled, "Number of queued or active jobs in the MergeTree parts loader thread pool.") \ M(MergeTreeOutdatedPartsLoaderThreads, "Number of threads in the threadpool for loading Outdated data parts.") \ M(MergeTreeOutdatedPartsLoaderThreadsActive, "Number of active threads in the threadpool for loading Outdated data parts.") \ + M(MergeTreeOutdatedPartsLoaderThreadsScheduled, "Number of queued or active jobs in the threadpool for loading Outdated data parts.") \ M(MergeTreePartsCleanerThreads, "Number of threads in the MergeTree parts cleaner thread pool.") \ M(MergeTreePartsCleanerThreadsActive, "Number of threads in the MergeTree parts cleaner thread pool running a task.") \ + M(MergeTreePartsCleanerThreadsScheduled, "Number of queued or active jobs in the MergeTree parts cleaner thread pool.") \ M(IDiskCopierThreads, "Number of threads for copying data between disks of different types.") \ M(IDiskCopierThreadsActive, "Number of threads for copying data between disks of different types running a task.") \ + M(IDiskCopierThreadsScheduled, "Number of queued or active jobs for copying data between disks of different types.") \ M(SystemReplicasThreads, "Number of threads in the system.replicas thread pool.") \ M(SystemReplicasThreadsActive, "Number of threads in the system.replicas thread pool running a task.") \ + M(SystemReplicasThreadsScheduled, "Number of queued or active jobs in the system.replicas thread pool.") \ M(RestartReplicaThreads, "Number of threads in the RESTART REPLICA thread pool.") \ M(RestartReplicaThreadsActive, "Number of threads in the RESTART REPLICA thread pool running a task.") \ + M(RestartReplicaThreadsScheduled, "Number of queued or active jobs in the RESTART REPLICA thread pool.") \ M(QueryPipelineExecutorThreads, "Number of threads in the PipelineExecutor thread pool.") \ M(QueryPipelineExecutorThreadsActive, "Number of threads in the PipelineExecutor thread pool running a task.") \ + M(QueryPipelineExecutorThreadsScheduled, "Number of queued or active jobs in the PipelineExecutor thread pool.") \ M(ParquetDecoderThreads, "Number of threads in the ParquetBlockInputFormat thread pool.") \ M(ParquetDecoderThreadsActive, "Number of threads in the ParquetBlockInputFormat thread pool running a task.") \ + M(ParquetDecoderThreadsScheduled, "Number of queued or active jobs in the ParquetBlockInputFormat thread pool.") \ M(ParquetEncoderThreads, "Number of threads in ParquetBlockOutputFormat thread pool.") \ M(ParquetEncoderThreadsActive, "Number of threads in ParquetBlockOutputFormat thread pool running a task.") \ + M(ParquetEncoderThreadsScheduled, "Number of queued or active jobs in ParquetBlockOutputFormat thread pool.") \ M(DWARFReaderThreads, "Number of threads in the DWARFBlockInputFormat thread pool.") \ M(DWARFReaderThreadsActive, "Number of threads in the DWARFBlockInputFormat thread pool running a task.") \ + M(DWARFReaderThreadsScheduled, "Number of queued or active jobs in the DWARFBlockInputFormat thread pool.") \ M(OutdatedPartsLoadingThreads, "Number of threads in the threadpool for loading Outdated data parts.") \ M(OutdatedPartsLoadingThreadsActive, "Number of active threads in the threadpool for loading Outdated data parts.") \ + M(OutdatedPartsLoadingThreadsScheduled, "Number of queued or active jobs in the threadpool for loading Outdated data parts.") \ M(DistributedBytesToInsert, "Number of pending bytes to process for asynchronous insertion into Distributed tables. Number of bytes for every shard is summed.") \ M(BrokenDistributedBytesToInsert, "Number of bytes for asynchronous insertion into Distributed tables that has been marked as broken. Number of bytes for every shard is summed.") \ M(DistributedFilesToInsert, "Number of pending files to process for asynchronous insertion into Distributed tables. Number of files for every shard is summed.") \ diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index 4a5bdeffcee4..8cba13373b9b 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -25,13 +25,14 @@ namespace CurrentMetrics { extern const Metric GlobalThread; extern const Metric GlobalThreadActive; + extern const Metric GlobalThreadScheduled; } static constexpr auto DEFAULT_THREAD_NAME = "ThreadPool"; template -ThreadPoolImpl::ThreadPoolImpl(Metric metric_threads_, Metric metric_active_threads_) - : ThreadPoolImpl(metric_threads_, metric_active_threads_, getNumberOfPhysicalCPUCores()) +ThreadPoolImpl::ThreadPoolImpl(Metric metric_threads_, Metric metric_active_threads_, Metric metric_scheduled_jobs_) + : ThreadPoolImpl(metric_threads_, metric_active_threads_, metric_scheduled_jobs_, getNumberOfPhysicalCPUCores()) { } @@ -40,8 +41,9 @@ template ThreadPoolImpl::ThreadPoolImpl( Metric metric_threads_, Metric metric_active_threads_, + Metric metric_scheduled_jobs_, size_t max_threads_) - : ThreadPoolImpl(metric_threads_, metric_active_threads_, max_threads_, max_threads_, max_threads_) + : ThreadPoolImpl(metric_threads_, metric_active_threads_, metric_scheduled_jobs_, max_threads_, max_threads_, max_threads_) { } @@ -49,12 +51,14 @@ template ThreadPoolImpl::ThreadPoolImpl( Metric metric_threads_, Metric metric_active_threads_, + Metric metric_scheduled_jobs_, size_t max_threads_, size_t max_free_threads_, size_t queue_size_, bool shutdown_on_exception_) : metric_threads(metric_threads_) , metric_active_threads(metric_active_threads_) + , metric_scheduled_jobs(metric_scheduled_jobs_) , max_threads(max_threads_) , max_free_threads(std::min(max_free_threads_, max_threads)) , queue_size(queue_size_ ? std::max(queue_size_, max_threads) : 0 /* zero means the queue is unlimited */) @@ -187,6 +191,7 @@ ReturnType ThreadPoolImpl::scheduleImpl(Job job, Priority priority, std: jobs.emplace(std::move(job), priority, + metric_scheduled_jobs, /// Tracing context on this thread is used as parent context for the sub-thread that runs the job propagate_opentelemetry_tracing_context ? DB::OpenTelemetry::CurrentContext() : DB::OpenTelemetry::TracingContextOnThread(), /// capture_frame_pointers @@ -346,13 +351,8 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ /// This is inside the loop to also reset previous thread names set inside the jobs. setThreadName(DEFAULT_THREAD_NAME); - /// A copy of parent trace context - DB::OpenTelemetry::TracingContextOnThread parent_thread_trace_context; - - std::vector thread_frame_pointers; - /// Get a job from the queue. - Job job; + std::optional job_data; { std::unique_lock lock(mutex); @@ -393,12 +393,8 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ } /// boost::priority_queue does not provide interface for getting non-const reference to an element - /// to prevent us from modifying its priority. We have to use const_cast to force move semantics on JobWithPriority::job. - job = std::move(const_cast(jobs.top().job)); - parent_thread_trace_context = std::move(const_cast(jobs.top().thread_trace_context)); - DB::Exception::enable_job_stack_trace = jobs.top().enable_job_stack_trace; - if (DB::Exception::enable_job_stack_trace) - thread_frame_pointers = std::move(const_cast &>(jobs.top().frame_pointers)); + /// to prevent us from modifying its priority. We have to use const_cast to force move semantics on JobWithPriority. + job_data = std::move(const_cast(jobs.top())); jobs.pop(); /// We don't run jobs after `shutdown` is set, but we have to properly dequeue all jobs and finish them. @@ -412,18 +408,17 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ ALLOW_ALLOCATIONS_IN_SCOPE; /// Set up tracing context for this thread by its parent context. - DB::OpenTelemetry::TracingContextHolder thread_trace_context("ThreadPool::worker()", parent_thread_trace_context); + DB::OpenTelemetry::TracingContextHolder thread_trace_context("ThreadPool::worker()", job_data->thread_trace_context); /// Run the job. try { if (DB::Exception::enable_job_stack_trace) - DB::Exception::thread_frame_pointers = std::move(thread_frame_pointers); - + DB::Exception::thread_frame_pointers = std::move(job_data->frame_pointers); CurrentMetrics::Increment metric_active_pool_threads(metric_active_threads); - job(); + job_data->job(); if (thread_trace_context.root_span.isTraceEnabled()) { @@ -437,13 +432,13 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ else { /// If the thread name is not set, use the type name of the job instead - thread_trace_context.root_span.operation_name = demangle(job.target_type().name()); + thread_trace_context.root_span.operation_name = demangle(job_data->job.target_type().name()); } } /// job should be reset before decrementing scheduled_jobs to /// ensure that the Job destroyed before wait() returns. - job = {}; + job_data.reset(); } catch (...) { @@ -452,7 +447,7 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ /// job should be reset before decrementing scheduled_jobs to /// ensure that the Job destroyed before wait() returns. - job = {}; + job_data.reset(); } job_is_done = true; @@ -475,6 +470,7 @@ GlobalThreadPool::GlobalThreadPool( : FreeThreadPool( CurrentMetrics::GlobalThread, CurrentMetrics::GlobalThreadActive, + CurrentMetrics::GlobalThreadScheduled, max_threads_, max_free_threads_, queue_size_, diff --git a/src/Common/ThreadPool.h b/src/Common/ThreadPool.h index f5721146e095..c8eefedd838d 100644 --- a/src/Common/ThreadPool.h +++ b/src/Common/ThreadPool.h @@ -41,18 +41,20 @@ class ThreadPoolImpl using Metric = CurrentMetrics::Metric; /// Maximum number of threads is based on the number of physical cores. - ThreadPoolImpl(Metric metric_threads_, Metric metric_active_threads_); + ThreadPoolImpl(Metric metric_threads_, Metric metric_active_threads_, Metric metric_scheduled_jobs_); /// Size is constant. Up to num_threads are created on demand and then run until shutdown. explicit ThreadPoolImpl( Metric metric_threads_, Metric metric_active_threads_, + Metric metric_scheduled_jobs_, size_t max_threads_); /// queue_size - maximum number of running plus scheduled jobs. It can be greater than max_threads. Zero means unlimited. ThreadPoolImpl( Metric metric_threads_, Metric metric_active_threads_, + Metric metric_scheduled_jobs_, size_t max_threads_, size_t max_free_threads_, size_t queue_size_, @@ -113,6 +115,7 @@ class ThreadPoolImpl Metric metric_threads; Metric metric_active_threads; + Metric metric_scheduled_jobs; size_t max_threads; size_t max_free_threads; @@ -127,14 +130,19 @@ class ThreadPoolImpl { Job job; Priority priority; + CurrentMetrics::Increment metric_increment; DB::OpenTelemetry::TracingContextOnThread thread_trace_context; /// Call stacks of all jobs' schedulings leading to this one std::vector frame_pointers; bool enable_job_stack_trace = false; - JobWithPriority(Job job_, Priority priority_, const DB::OpenTelemetry::TracingContextOnThread & thread_trace_context_, bool capture_frame_pointers = false) - : job(job_), priority(priority_), thread_trace_context(thread_trace_context_), enable_job_stack_trace(capture_frame_pointers) + JobWithPriority( + Job job_, Priority priority_, CurrentMetrics::Metric metric, + const DB::OpenTelemetry::TracingContextOnThread & thread_trace_context_, + bool capture_frame_pointers) + : job(job_), priority(priority_), metric_increment(metric), + thread_trace_context(thread_trace_context_), enable_job_stack_trace(capture_frame_pointers) { if (!capture_frame_pointers) return; diff --git a/src/Common/examples/parallel_aggregation.cpp b/src/Common/examples/parallel_aggregation.cpp index cf7a3197fef1..20f5f1c5224b 100644 --- a/src/Common/examples/parallel_aggregation.cpp +++ b/src/Common/examples/parallel_aggregation.cpp @@ -33,6 +33,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } struct SmallLock @@ -254,7 +255,7 @@ int main(int argc, char ** argv) std::cerr << std::fixed << std::setprecision(2); - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, num_threads); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, num_threads); Source data(n); diff --git a/src/Common/examples/parallel_aggregation2.cpp b/src/Common/examples/parallel_aggregation2.cpp index 1b0ad760490a..e7136707dbd9 100644 --- a/src/Common/examples/parallel_aggregation2.cpp +++ b/src/Common/examples/parallel_aggregation2.cpp @@ -29,6 +29,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } template @@ -281,7 +282,7 @@ int main(int argc, char ** argv) std::cerr << std::fixed << std::setprecision(2); - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, num_threads); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, num_threads); Source data(n); diff --git a/src/Common/examples/thread_creation_latency.cpp b/src/Common/examples/thread_creation_latency.cpp index 2434759c968a..60fb27dc3453 100644 --- a/src/Common/examples/thread_creation_latency.cpp +++ b/src/Common/examples/thread_creation_latency.cpp @@ -19,6 +19,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } namespace DB @@ -72,7 +73,7 @@ int main(int argc, char ** argv) test(n, "Create and destroy ThreadPool each iteration", [] { - ThreadPool tp(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 1); + ThreadPool tp(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 1); tp.scheduleOrThrowOnError(f); tp.wait(); }); @@ -93,7 +94,7 @@ int main(int argc, char ** argv) }); { - ThreadPool tp(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 1); + ThreadPool tp(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 1); test(n, "Schedule job for Threadpool each iteration", [&tp] { @@ -103,7 +104,7 @@ int main(int argc, char ** argv) } { - ThreadPool tp(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 128); + ThreadPool tp(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 128); test(n, "Schedule job for Threadpool with 128 threads each iteration", [&tp] { diff --git a/src/Common/tests/gtest_async_loader.cpp b/src/Common/tests/gtest_async_loader.cpp index dfcbf27b9fcd..28d47593e78a 100644 --- a/src/Common/tests/gtest_async_loader.cpp +++ b/src/Common/tests/gtest_async_loader.cpp @@ -21,6 +21,7 @@ namespace CurrentMetrics { extern const Metric TablesLoaderThreads; extern const Metric TablesLoaderThreadsActive; + extern const Metric TablesLoaderThreadsScheduled; } namespace DB::ErrorCodes @@ -62,6 +63,7 @@ struct AsyncLoaderTest .name = fmt::format("Pool{}", pool_id), .metric_threads = CurrentMetrics::TablesLoaderThreads, .metric_active_threads = CurrentMetrics::TablesLoaderThreadsActive, + .metric_scheduled_threads = CurrentMetrics::TablesLoaderThreadsScheduled, .max_threads = desc.max_threads, .priority = desc.priority }); diff --git a/src/Common/tests/gtest_thread_pool_concurrent_wait.cpp b/src/Common/tests/gtest_thread_pool_concurrent_wait.cpp index f93017129dd5..ddaff3382dbb 100644 --- a/src/Common/tests/gtest_thread_pool_concurrent_wait.cpp +++ b/src/Common/tests/gtest_thread_pool_concurrent_wait.cpp @@ -12,6 +12,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } TEST(ThreadPool, ConcurrentWait) @@ -25,14 +26,14 @@ TEST(ThreadPool, ConcurrentWait) constexpr size_t num_threads = 4; constexpr size_t num_jobs = 4; - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, num_threads); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, num_threads); for (size_t i = 0; i < num_jobs; ++i) pool.scheduleOrThrowOnError(worker); constexpr size_t num_waiting_threads = 4; - ThreadPool waiting_pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, num_waiting_threads); + ThreadPool waiting_pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, num_waiting_threads); for (size_t i = 0; i < num_waiting_threads; ++i) waiting_pool.scheduleOrThrowOnError([&pool] { pool.wait(); }); diff --git a/src/Common/tests/gtest_thread_pool_global_full.cpp b/src/Common/tests/gtest_thread_pool_global_full.cpp index 1b2ded9c7e10..4507998be3c0 100644 --- a/src/Common/tests/gtest_thread_pool_global_full.cpp +++ b/src/Common/tests/gtest_thread_pool_global_full.cpp @@ -11,6 +11,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } /// Test what happens if local ThreadPool cannot create a ThreadFromGlobalPool. @@ -34,7 +35,7 @@ TEST(ThreadPool, GlobalFull1) auto func = [&] { ++counter; while (counter != num_jobs) {} }; - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, num_jobs); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, num_jobs); for (size_t i = 0; i < capacity; ++i) pool.scheduleOrThrowOnError(func); @@ -72,11 +73,11 @@ TEST(ThreadPool, GlobalFull2) std::atomic counter = 0; auto func = [&] { ++counter; while (counter != capacity + 1) {} }; - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, capacity, 0, capacity); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, capacity, 0, capacity); for (size_t i = 0; i < capacity; ++i) pool.scheduleOrThrowOnError(func); - ThreadPool another_pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 1); + ThreadPool another_pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 1); EXPECT_THROW(another_pool.scheduleOrThrowOnError(func), DB::Exception); ++counter; diff --git a/src/Common/tests/gtest_thread_pool_limit.cpp b/src/Common/tests/gtest_thread_pool_limit.cpp index b47c8cdad180..66c6f8dc1223 100644 --- a/src/Common/tests/gtest_thread_pool_limit.cpp +++ b/src/Common/tests/gtest_thread_pool_limit.cpp @@ -8,6 +8,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } /// Test for thread self-removal when number of free threads in pool is too large. @@ -16,7 +17,7 @@ namespace CurrentMetrics template int test() { - Pool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 10, 2, 10); + Pool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 10, 2, 10); std::atomic counter{0}; for (size_t i = 0; i < 10; ++i) diff --git a/src/Common/tests/gtest_thread_pool_loop.cpp b/src/Common/tests/gtest_thread_pool_loop.cpp index 170a888ff725..4257c0b73a5f 100644 --- a/src/Common/tests/gtest_thread_pool_loop.cpp +++ b/src/Common/tests/gtest_thread_pool_loop.cpp @@ -9,6 +9,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } TEST(ThreadPool, Loop) @@ -18,7 +19,7 @@ TEST(ThreadPool, Loop) for (size_t i = 0; i < 1000; ++i) { size_t threads = 16; - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, threads); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, threads); for (size_t j = 0; j < threads; ++j) pool.scheduleOrThrowOnError([&] { ++res; }); pool.wait(); diff --git a/src/Common/tests/gtest_thread_pool_schedule_exception.cpp b/src/Common/tests/gtest_thread_pool_schedule_exception.cpp index d8e00b5314ca..1b4f27e59e0f 100644 --- a/src/Common/tests/gtest_thread_pool_schedule_exception.cpp +++ b/src/Common/tests/gtest_thread_pool_schedule_exception.cpp @@ -9,11 +9,12 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } static bool check() { - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 10); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 10); /// The throwing thread. pool.scheduleOrThrowOnError([] { throw std::runtime_error("Hello, world!"); }); @@ -53,7 +54,7 @@ TEST(ThreadPool, ExceptionFromSchedule) static bool check2() { - ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, 2); + ThreadPool pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, 2); try { diff --git a/src/Coordination/Standalone/Context.cpp b/src/Coordination/Standalone/Context.cpp index ef23e85e54b7..4001673e01a8 100644 --- a/src/Coordination/Standalone/Context.cpp +++ b/src/Coordination/Standalone/Context.cpp @@ -25,6 +25,7 @@ namespace CurrentMetrics extern const Metric BackgroundSchedulePoolSize; extern const Metric IOWriterThreads; extern const Metric IOWriterThreadsActive; + extern const Metric IOWriterThreadsScheduled; } namespace DB diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 96c084a261cd..549711150b8e 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -30,6 +30,7 @@ namespace CurrentMetrics { extern const Metric DatabaseOnDiskThreads; extern const Metric DatabaseOnDiskThreadsActive; + extern const Metric DatabaseOnDiskThreadsScheduled; } namespace DB @@ -627,7 +628,7 @@ void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const Iterat } /// Read and parse metadata in parallel - ThreadPool pool(CurrentMetrics::DatabaseOnDiskThreads, CurrentMetrics::DatabaseOnDiskThreadsActive); + ThreadPool pool(CurrentMetrics::DatabaseOnDiskThreads, CurrentMetrics::DatabaseOnDiskThreadsActive, CurrentMetrics::DatabaseOnDiskThreadsScheduled); for (const auto & file : metadata_files) { pool.scheduleOrThrowOnError([&]() diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 51d37b84e14f..1b2a7b9d5e2e 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -34,6 +34,7 @@ namespace CurrentMetrics { extern const Metric DatabaseOrdinaryThreads; extern const Metric DatabaseOrdinaryThreadsActive; + extern const Metric DatabaseOrdinaryThreadsScheduled; } namespace DB @@ -106,7 +107,7 @@ void DatabaseOrdinary::loadStoredObjects(ContextMutablePtr local_context, Loadin std::atomic dictionaries_processed{0}; std::atomic tables_processed{0}; - ThreadPool pool(CurrentMetrics::DatabaseOrdinaryThreads, CurrentMetrics::DatabaseOrdinaryThreadsActive); + ThreadPool pool(CurrentMetrics::DatabaseOrdinaryThreads, CurrentMetrics::DatabaseOrdinaryThreadsActive, CurrentMetrics::DatabaseOrdinaryThreadsScheduled); /// We must attach dictionaries before attaching tables /// because while we're attaching tables we may need to have some dictionaries attached diff --git a/src/Databases/TablesLoader.cpp b/src/Databases/TablesLoader.cpp index f8b4e7fe33b5..5cc7e77a6de2 100644 --- a/src/Databases/TablesLoader.cpp +++ b/src/Databases/TablesLoader.cpp @@ -15,6 +15,7 @@ namespace CurrentMetrics { extern const Metric TablesLoaderThreads; extern const Metric TablesLoaderThreadsActive; + extern const Metric TablesLoaderThreadsScheduled; } namespace DB @@ -32,7 +33,7 @@ TablesLoader::TablesLoader(ContextMutablePtr global_context_, Databases database , referential_dependencies("ReferentialDeps") , loading_dependencies("LoadingDeps") , all_loading_dependencies("LoadingDeps") - , pool(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive) + , pool(CurrentMetrics::TablesLoaderThreads, CurrentMetrics::TablesLoaderThreadsActive, CurrentMetrics::TablesLoaderThreadsScheduled) { metadata.default_database = global_context->getCurrentDatabase(); log = &Poco::Logger::get("TablesLoader"); diff --git a/src/Dictionaries/CacheDictionaryUpdateQueue.cpp b/src/Dictionaries/CacheDictionaryUpdateQueue.cpp index 3aa3cbb9a4b9..1e9b1da390a9 100644 --- a/src/Dictionaries/CacheDictionaryUpdateQueue.cpp +++ b/src/Dictionaries/CacheDictionaryUpdateQueue.cpp @@ -9,6 +9,7 @@ namespace CurrentMetrics { extern const Metric CacheDictionaryThreads; extern const Metric CacheDictionaryThreadsActive; + extern const Metric CacheDictionaryThreadsScheduled; } namespace DB @@ -33,7 +34,7 @@ CacheDictionaryUpdateQueue::CacheDictionaryUpdateQueue( , configuration(configuration_) , update_func(std::move(update_func_)) , update_queue(configuration.max_update_queue_size) - , update_pool(CurrentMetrics::CacheDictionaryThreads, CurrentMetrics::CacheDictionaryThreadsActive, configuration.max_threads_for_updates) + , update_pool(CurrentMetrics::CacheDictionaryThreads, CurrentMetrics::CacheDictionaryThreadsActive, CurrentMetrics::CacheDictionaryThreadsScheduled, configuration.max_threads_for_updates) { for (size_t i = 0; i < configuration.max_threads_for_updates; ++i) update_pool.scheduleOrThrowOnError([this] { updateThreadFunction(); }); diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp index 0556e2bb266d..9c5dfeef6ca8 100644 --- a/src/Dictionaries/HashedDictionary.cpp +++ b/src/Dictionaries/HashedDictionary.cpp @@ -32,6 +32,7 @@ namespace CurrentMetrics { extern const Metric HashedDictionaryThreads; extern const Metric HashedDictionaryThreadsActive; + extern const Metric HashedDictionaryThreadsScheduled; } namespace DB @@ -59,7 +60,7 @@ class ParallelDictionaryLoader : public boost::noncopyable explicit ParallelDictionaryLoader(HashedDictionary & dictionary_) : dictionary(dictionary_) , shards(dictionary.configuration.shards) - , pool(CurrentMetrics::HashedDictionaryThreads, CurrentMetrics::HashedDictionaryThreadsActive, shards) + , pool(CurrentMetrics::HashedDictionaryThreads, CurrentMetrics::HashedDictionaryThreadsActive, CurrentMetrics::HashedDictionaryThreadsScheduled, shards) , shards_queues(shards) { UInt64 backlog = dictionary.configuration.shard_load_queue_backlog; @@ -229,7 +230,7 @@ HashedDictionary::~HashedDictionary() return; size_t shards = std::max(configuration.shards, 1); - ThreadPool pool(CurrentMetrics::HashedDictionaryThreads, CurrentMetrics::HashedDictionaryThreadsActive, shards); + ThreadPool pool(CurrentMetrics::HashedDictionaryThreads, CurrentMetrics::HashedDictionaryThreadsActive, CurrentMetrics::HashedDictionaryThreadsScheduled, shards); size_t hash_tables_count = 0; auto schedule_destroy = [&hash_tables_count, &pool](auto & container) diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 6911fd86db27..5fcf9a9dce4d 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -38,6 +38,7 @@ namespace CurrentMetrics { extern const Metric IDiskCopierThreads; extern const Metric IDiskCopierThreadsActive; + extern const Metric IDiskCopierThreadsScheduled; } namespace DB @@ -117,13 +118,13 @@ class IDisk : public Space /// Default constructor. IDisk(const String & name_, const Poco::Util::AbstractConfiguration & config, const String & config_prefix) : name(name_) - , copying_thread_pool(CurrentMetrics::IDiskCopierThreads, CurrentMetrics::IDiskCopierThreadsActive, config.getUInt(config_prefix + ".thread_pool_size", 16)) + , copying_thread_pool(CurrentMetrics::IDiskCopierThreads, CurrentMetrics::IDiskCopierThreadsActive, CurrentMetrics::IDiskCopierThreadsScheduled, config.getUInt(config_prefix + ".thread_pool_size", 16)) { } explicit IDisk(const String & name_) : name(name_) - , copying_thread_pool(CurrentMetrics::IDiskCopierThreads, CurrentMetrics::IDiskCopierThreadsActive, 16) + , copying_thread_pool(CurrentMetrics::IDiskCopierThreads, CurrentMetrics::IDiskCopierThreadsActive, CurrentMetrics::IDiskCopierThreadsScheduled, 16) { } diff --git a/src/Disks/IO/ThreadPoolReader.cpp b/src/Disks/IO/ThreadPoolReader.cpp index cd3f2d8dea09..15cbcdf5b9b2 100644 --- a/src/Disks/IO/ThreadPoolReader.cpp +++ b/src/Disks/IO/ThreadPoolReader.cpp @@ -64,6 +64,7 @@ namespace CurrentMetrics extern const Metric Read; extern const Metric ThreadPoolFSReaderThreads; extern const Metric ThreadPoolFSReaderThreadsActive; + extern const Metric ThreadPoolFSReaderThreadsScheduled; } @@ -88,7 +89,7 @@ static bool hasBugInPreadV2() #endif ThreadPoolReader::ThreadPoolReader(size_t pool_size, size_t queue_size_) - : pool(std::make_unique(CurrentMetrics::ThreadPoolFSReaderThreads, CurrentMetrics::ThreadPoolFSReaderThreadsActive, pool_size, pool_size, queue_size_)) + : pool(std::make_unique(CurrentMetrics::ThreadPoolFSReaderThreads, CurrentMetrics::ThreadPoolFSReaderThreadsActive, CurrentMetrics::ThreadPoolFSReaderThreadsScheduled, pool_size, pool_size, queue_size_)) { } diff --git a/src/Disks/IO/ThreadPoolRemoteFSReader.cpp b/src/Disks/IO/ThreadPoolRemoteFSReader.cpp index 0ec5e0fd6c14..ac599bb35477 100644 --- a/src/Disks/IO/ThreadPoolRemoteFSReader.cpp +++ b/src/Disks/IO/ThreadPoolRemoteFSReader.cpp @@ -29,6 +29,7 @@ namespace CurrentMetrics extern const Metric RemoteRead; extern const Metric ThreadPoolRemoteFSReaderThreads; extern const Metric ThreadPoolRemoteFSReaderThreadsActive; + extern const Metric ThreadPoolRemoteFSReaderThreadsScheduled; } namespace DB @@ -59,6 +60,7 @@ namespace ThreadPoolRemoteFSReader::ThreadPoolRemoteFSReader(size_t pool_size, size_t queue_size_) : pool(std::make_unique(CurrentMetrics::ThreadPoolRemoteFSReaderThreads, CurrentMetrics::ThreadPoolRemoteFSReaderThreadsActive, + CurrentMetrics::ThreadPoolRemoteFSReaderThreadsScheduled, pool_size, pool_size, queue_size_)) { } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp index fcb82daca95b..068e2aebab1b 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp @@ -14,11 +14,12 @@ #include #include + namespace CurrentMetrics { extern const Metric ObjectStorageAzureThreads; extern const Metric ObjectStorageAzureThreadsActive; - + extern const Metric ObjectStorageAzureThreadsScheduled; } namespace DB @@ -45,6 +46,7 @@ class AzureIteratorAsync final : public IObjectStorageIteratorAsync : IObjectStorageIteratorAsync( CurrentMetrics::ObjectStorageAzureThreads, CurrentMetrics::ObjectStorageAzureThreadsActive, + CurrentMetrics::ObjectStorageAzureThreadsScheduled, "ListObjectAzure") , client(client_) { diff --git a/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp b/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp index 33b98cd328c6..0314e0a7e920 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp @@ -13,6 +13,7 @@ namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } namespace DB @@ -156,7 +157,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::migrateToRestorableSchema() { LOG_INFO(disk->log, "Start migration to restorable schema for disk {}", disk->name); - ThreadPool pool{CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive}; + ThreadPool pool{CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled}; for (const auto & root : data_roots) if (disk->exists(root)) @@ -355,7 +356,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFiles(IObjectStorage * { LOG_INFO(disk->log, "Starting restore files for disk {}", disk->name); - ThreadPool pool{CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive}; + ThreadPool pool{CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled}; auto restore_files = [this, &source_object_storage, &restore_information, &pool](const RelativePathsWithMetadata & objects) { std::vector keys_names; diff --git a/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.cpp b/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.cpp index 7425f629a5a5..990e66fc4e5b 100644 --- a/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.cpp +++ b/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.cpp @@ -2,6 +2,7 @@ #include + namespace DB { diff --git a/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.h b/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.h index b0dd3cef39c3..a6abe03bac96 100644 --- a/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.h +++ b/src/Disks/ObjectStorages/ObjectStorageIteratorAsync.h @@ -6,6 +6,7 @@ #include #include + namespace DB { @@ -15,8 +16,9 @@ class IObjectStorageIteratorAsync : public IObjectStorageIterator IObjectStorageIteratorAsync( CurrentMetrics::Metric threads_metric, CurrentMetrics::Metric threads_active_metric, + CurrentMetrics::Metric threads_scheduled_metric, const std::string & thread_name) - : list_objects_pool(threads_metric, threads_active_metric, 1) + : list_objects_pool(threads_metric, threads_active_metric, threads_scheduled_metric, 1) , list_objects_scheduler(threadPoolCallbackRunner(list_objects_pool, thread_name)) { } diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index b36185249af7..3af316bf0cf8 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -37,6 +37,7 @@ namespace CurrentMetrics { extern const Metric ObjectStorageS3Threads; extern const Metric ObjectStorageS3ThreadsActive; + extern const Metric ObjectStorageS3ThreadsScheduled; } @@ -105,6 +106,7 @@ class S3IteratorAsync final : public IObjectStorageIteratorAsync : IObjectStorageIteratorAsync( CurrentMetrics::ObjectStorageS3Threads, CurrentMetrics::ObjectStorageS3ThreadsActive, + CurrentMetrics::ObjectStorageS3ThreadsScheduled, "ListObjectS3") , client(client_) { diff --git a/src/IO/SharedThreadPools.cpp b/src/IO/SharedThreadPools.cpp index 6a0e953f0efe..4a0ef1736697 100644 --- a/src/IO/SharedThreadPools.cpp +++ b/src/IO/SharedThreadPools.cpp @@ -7,14 +7,19 @@ namespace CurrentMetrics { extern const Metric IOThreads; extern const Metric IOThreadsActive; + extern const Metric IOThreadsScheduled; extern const Metric BackupsIOThreads; extern const Metric BackupsIOThreadsActive; + extern const Metric BackupsIOThreadsScheduled; extern const Metric MergeTreePartsLoaderThreads; extern const Metric MergeTreePartsLoaderThreadsActive; + extern const Metric MergeTreePartsLoaderThreadsScheduled; extern const Metric MergeTreePartsCleanerThreads; extern const Metric MergeTreePartsCleanerThreadsActive; + extern const Metric MergeTreePartsCleanerThreadsScheduled; extern const Metric MergeTreeOutdatedPartsLoaderThreads; extern const Metric MergeTreeOutdatedPartsLoaderThreadsActive; + extern const Metric MergeTreeOutdatedPartsLoaderThreadsScheduled; } namespace DB @@ -29,10 +34,12 @@ namespace ErrorCodes StaticThreadPool::StaticThreadPool( const String & name_, CurrentMetrics::Metric threads_metric_, - CurrentMetrics::Metric threads_active_metric_) + CurrentMetrics::Metric threads_active_metric_, + CurrentMetrics::Metric threads_scheduled_metric_) : name(name_) , threads_metric(threads_metric_) , threads_active_metric(threads_active_metric_) + , threads_scheduled_metric(threads_scheduled_metric_) { } @@ -47,6 +54,7 @@ void StaticThreadPool::initialize(size_t max_threads, size_t max_free_threads, s instance = std::make_unique( threads_metric, threads_active_metric, + threads_scheduled_metric, max_threads, max_free_threads, queue_size, @@ -110,31 +118,31 @@ void StaticThreadPool::setMaxTurboThreads(size_t max_threads_turbo_) StaticThreadPool & getIOThreadPool() { - static StaticThreadPool instance("IOThreadPool", CurrentMetrics::IOThreads, CurrentMetrics::IOThreadsActive); + static StaticThreadPool instance("IOThreadPool", CurrentMetrics::IOThreads, CurrentMetrics::IOThreadsActive, CurrentMetrics::IOThreadsScheduled); return instance; } StaticThreadPool & getBackupsIOThreadPool() { - static StaticThreadPool instance("BackupsIOThreadPool", CurrentMetrics::BackupsIOThreads, CurrentMetrics::BackupsIOThreadsActive); + static StaticThreadPool instance("BackupsIOThreadPool", CurrentMetrics::BackupsIOThreads, CurrentMetrics::BackupsIOThreadsActive, CurrentMetrics::BackupsIOThreadsScheduled); return instance; } StaticThreadPool & getActivePartsLoadingThreadPool() { - static StaticThreadPool instance("MergeTreePartsLoaderThreadPool", CurrentMetrics::MergeTreePartsLoaderThreads, CurrentMetrics::MergeTreePartsLoaderThreadsActive); + static StaticThreadPool instance("MergeTreePartsLoaderThreadPool", CurrentMetrics::MergeTreePartsLoaderThreads, CurrentMetrics::MergeTreePartsLoaderThreadsActive, CurrentMetrics::MergeTreeOutdatedPartsLoaderThreadsScheduled); return instance; } StaticThreadPool & getPartsCleaningThreadPool() { - static StaticThreadPool instance("MergeTreePartsCleanerThreadPool", CurrentMetrics::MergeTreePartsCleanerThreads, CurrentMetrics::MergeTreePartsCleanerThreadsActive); + static StaticThreadPool instance("MergeTreePartsCleanerThreadPool", CurrentMetrics::MergeTreePartsCleanerThreads, CurrentMetrics::MergeTreePartsCleanerThreadsActive, CurrentMetrics::MergeTreePartsCleanerThreadsScheduled); return instance; } StaticThreadPool & getOutdatedPartsLoadingThreadPool() { - static StaticThreadPool instance("MergeTreeOutdatedPartsLoaderThreadPool", CurrentMetrics::MergeTreeOutdatedPartsLoaderThreads, CurrentMetrics::MergeTreeOutdatedPartsLoaderThreadsActive); + static StaticThreadPool instance("MergeTreeOutdatedPartsLoaderThreadPool", CurrentMetrics::MergeTreeOutdatedPartsLoaderThreads, CurrentMetrics::MergeTreeOutdatedPartsLoaderThreadsActive, CurrentMetrics::MergeTreeOutdatedPartsLoaderThreadsScheduled); return instance; } diff --git a/src/IO/SharedThreadPools.h b/src/IO/SharedThreadPools.h index 188a2a4f003c..f37f3acefe7d 100644 --- a/src/IO/SharedThreadPools.h +++ b/src/IO/SharedThreadPools.h @@ -8,6 +8,7 @@ #include #include + namespace DB { @@ -17,7 +18,8 @@ class StaticThreadPool StaticThreadPool( const String & name_, CurrentMetrics::Metric threads_metric_, - CurrentMetrics::Metric threads_active_metric_); + CurrentMetrics::Metric threads_active_metric_, + CurrentMetrics::Metric threads_scheduled_metric_); ThreadPool & get(); @@ -34,6 +36,7 @@ class StaticThreadPool const String name; const CurrentMetrics::Metric threads_metric; const CurrentMetrics::Metric threads_active_metric; + const CurrentMetrics::Metric threads_scheduled_metric; std::unique_ptr instance; std::mutex mutex; diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 129c02f032b5..e2ddfbe3418c 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -63,6 +63,7 @@ namespace CurrentMetrics extern const Metric TemporaryFilesForAggregation; extern const Metric AggregatorThreads; extern const Metric AggregatorThreadsActive; + extern const Metric AggregatorThreadsScheduled; } namespace DB @@ -2466,7 +2467,7 @@ BlocksList Aggregator::convertToBlocks(AggregatedDataVariants & data_variants, b std::unique_ptr thread_pool; if (max_threads > 1 && data_variants.sizeWithoutOverflowRow() > 100000 /// TODO Make a custom threshold. && data_variants.isTwoLevel()) /// TODO Use the shared thread pool with the `merge` function. - thread_pool = std::make_unique(CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, max_threads); + thread_pool = std::make_unique(CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, CurrentMetrics::AggregatorThreadsScheduled, max_threads); if (data_variants.without_key) blocks.emplace_back(prepareBlockAndFillWithoutKey( @@ -2656,7 +2657,7 @@ void NO_INLINE Aggregator::mergeDataOnlyExistingKeysImpl( void NO_INLINE Aggregator::mergeWithoutKeyDataImpl( ManyAggregatedDataVariants & non_empty_data) const { - ThreadPool thread_pool{CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, params.max_threads}; + ThreadPool thread_pool{CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, CurrentMetrics::AggregatorThreadsScheduled, params.max_threads}; AggregatedDataVariantsPtr & res = non_empty_data[0]; @@ -3144,7 +3145,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari std::unique_ptr thread_pool; if (max_threads > 1 && total_input_rows > 100000) /// TODO Make a custom threshold. - thread_pool = std::make_unique(CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, max_threads); + thread_pool = std::make_unique(CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, CurrentMetrics::AggregatorThreadsScheduled, max_threads); for (const auto & bucket_blocks : bucket_to_blocks) { diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 04285a06a65f..a0750122a5cc 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -39,6 +39,7 @@ namespace CurrentMetrics extern const Metric PendingAsyncInsert; extern const Metric AsynchronousInsertThreads; extern const Metric AsynchronousInsertThreadsActive; + extern const Metric AsynchronousInsertThreadsScheduled; } namespace ProfileEvents @@ -175,7 +176,7 @@ AsynchronousInsertQueue::AsynchronousInsertQueue(ContextPtr context_, size_t poo , pool_size(pool_size_) , flush_on_shutdown(flush_on_shutdown_) , queue_shards(pool_size) - , pool(CurrentMetrics::AsynchronousInsertThreads, CurrentMetrics::AsynchronousInsertThreadsActive, pool_size) + , pool(CurrentMetrics::AsynchronousInsertThreads, CurrentMetrics::AsynchronousInsertThreadsActive, CurrentMetrics::AsynchronousInsertThreadsScheduled, pool_size) { if (!pool_size) throw Exception(ErrorCodes::BAD_ARGUMENTS, "pool_size cannot be zero"); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index c6640d582a27..c13be0209333 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -135,10 +135,13 @@ namespace CurrentMetrics extern const Metric BackgroundCommonPoolSize; extern const Metric MarksLoaderThreads; extern const Metric MarksLoaderThreadsActive; + extern const Metric MarksLoaderThreadsScheduled; extern const Metric IOPrefetchThreads; extern const Metric IOPrefetchThreadsActive; + extern const Metric IOPrefetchThreadsScheduled; extern const Metric IOWriterThreads; extern const Metric IOWriterThreadsActive; + extern const Metric IOWriterThreadsScheduled; } @@ -2542,7 +2545,7 @@ ThreadPool & Context::getLoadMarksThreadpool() const auto pool_size = config.getUInt(".load_marks_threadpool_pool_size", 50); auto queue_size = config.getUInt(".load_marks_threadpool_queue_size", 1000000); shared->load_marks_threadpool = std::make_unique( - CurrentMetrics::MarksLoaderThreads, CurrentMetrics::MarksLoaderThreadsActive, pool_size, pool_size, queue_size); + CurrentMetrics::MarksLoaderThreads, CurrentMetrics::MarksLoaderThreadsActive, CurrentMetrics::MarksLoaderThreadsScheduled, pool_size, pool_size, queue_size); }); return *shared->load_marks_threadpool; @@ -2725,7 +2728,7 @@ ThreadPool & Context::getPrefetchThreadpool() const auto pool_size = config.getUInt(".prefetch_threadpool_pool_size", 100); auto queue_size = config.getUInt(".prefetch_threadpool_queue_size", 1000000); shared->prefetch_threadpool = std::make_unique( - CurrentMetrics::IOPrefetchThreads, CurrentMetrics::IOPrefetchThreadsActive, pool_size, pool_size, queue_size); + CurrentMetrics::IOPrefetchThreads, CurrentMetrics::IOPrefetchThreadsActive, CurrentMetrics::IOPrefetchThreadsScheduled, pool_size, pool_size, queue_size); }); return *shared->prefetch_threadpool; @@ -4765,7 +4768,7 @@ ThreadPool & Context::getThreadPoolWriter() const auto queue_size = config.getUInt(".threadpool_writer_queue_size", 1000000); shared->threadpool_writer = std::make_unique( - CurrentMetrics::IOWriterThreads, CurrentMetrics::IOWriterThreadsActive, pool_size, pool_size, queue_size); + CurrentMetrics::IOWriterThreads, CurrentMetrics::IOWriterThreadsActive, CurrentMetrics::IOWriterThreadsScheduled, pool_size, pool_size, queue_size); }); return *shared->threadpool_writer; diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index a9930036e7ed..a43c61f3d66a 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -46,6 +46,7 @@ namespace CurrentMetrics { extern const Metric DDLWorkerThreads; extern const Metric DDLWorkerThreadsActive; + extern const Metric DDLWorkerThreadsScheduled; } namespace DB @@ -93,7 +94,7 @@ DDLWorker::DDLWorker( { LOG_WARNING(log, "DDLWorker is configured to use multiple threads. " "It's not recommended because queries can be reordered. Also it may cause some unknown issues to appear."); - worker_pool = std::make_unique(CurrentMetrics::DDLWorkerThreads, CurrentMetrics::DDLWorkerThreadsActive, pool_size); + worker_pool = std::make_unique(CurrentMetrics::DDLWorkerThreads, CurrentMetrics::DDLWorkerThreadsActive, CurrentMetrics::DDLWorkerThreadsScheduled, pool_size); } queue_dir = zk_root_dir; @@ -1113,7 +1114,7 @@ void DDLWorker::runMainThread() /// It will wait for all threads in pool to finish and will not rethrow exceptions (if any). /// We create new thread pool to forget previous exceptions. if (1 < pool_size) - worker_pool = std::make_unique(CurrentMetrics::DDLWorkerThreads, CurrentMetrics::DDLWorkerThreadsActive, pool_size); + worker_pool = std::make_unique(CurrentMetrics::DDLWorkerThreads, CurrentMetrics::DDLWorkerThreadsActive, CurrentMetrics::DDLWorkerThreadsScheduled, pool_size); /// Clear other in-memory state, like server just started. current_tasks.clear(); last_skipped_entry_name.reset(); @@ -1152,7 +1153,7 @@ void DDLWorker::runMainThread() initialized = false; /// Wait for pending async tasks if (1 < pool_size) - worker_pool = std::make_unique(CurrentMetrics::DDLWorkerThreads, CurrentMetrics::DDLWorkerThreadsActive, pool_size); + worker_pool = std::make_unique(CurrentMetrics::DDLWorkerThreads, CurrentMetrics::DDLWorkerThreadsActive, CurrentMetrics::DDLWorkerThreadsScheduled, pool_size); LOG_INFO(log, "Lost ZooKeeper connection, will try to connect again: {}", getCurrentExceptionMessage(true)); } else diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 8dc90e1a2f17..e0ce2b99b14e 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -44,6 +44,7 @@ namespace CurrentMetrics extern const Metric TablesToDropQueueSize; extern const Metric DatabaseCatalogThreads; extern const Metric DatabaseCatalogThreadsActive; + extern const Metric DatabaseCatalogThreadsScheduled; } namespace DB @@ -1024,7 +1025,7 @@ void DatabaseCatalog::loadMarkedAsDroppedTables() LOG_INFO(log, "Found {} partially dropped tables. Will load them and retry removal.", dropped_metadata.size()); - ThreadPool pool(CurrentMetrics::DatabaseCatalogThreads, CurrentMetrics::DatabaseCatalogThreadsActive); + ThreadPool pool(CurrentMetrics::DatabaseCatalogThreads, CurrentMetrics::DatabaseCatalogThreadsActive, CurrentMetrics::DatabaseCatalogThreadsScheduled); for (const auto & elem : dropped_metadata) { pool.scheduleOrThrowOnError([&]() diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index b42745b726c5..ff95b3014dc3 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -76,6 +76,7 @@ namespace CurrentMetrics { extern const Metric RestartReplicaThreads; extern const Metric RestartReplicaThreadsActive; + extern const Metric RestartReplicaThreadsScheduled; } namespace DB @@ -809,7 +810,7 @@ void InterpreterSystemQuery::restartReplicas(ContextMutablePtr system_context) size_t threads = std::min(static_cast(getNumberOfPhysicalCPUCores()), replica_names.size()); LOG_DEBUG(log, "Will restart {} replicas using {} threads", replica_names.size(), threads); - ThreadPool pool(CurrentMetrics::RestartReplicaThreads, CurrentMetrics::RestartReplicaThreadsActive, threads); + ThreadPool pool(CurrentMetrics::RestartReplicaThreads, CurrentMetrics::RestartReplicaThreadsActive, CurrentMetrics::RestartReplicaThreadsScheduled, threads); for (auto & replica : replica_names) { diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index 3612dbfdc4ec..44b57396bd5b 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -32,6 +32,7 @@ namespace CurrentMetrics { extern const Metric StartupSystemTablesThreads; extern const Metric StartupSystemTablesThreadsActive; + extern const Metric StartupSystemTablesThreadsScheduled; } namespace DB @@ -377,7 +378,7 @@ static void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, cons if (!tables_started) { /// It's not quite correct to run DDL queries while database is not started up. - ThreadPool pool(CurrentMetrics::StartupSystemTablesThreads, CurrentMetrics::StartupSystemTablesThreadsActive); + ThreadPool pool(CurrentMetrics::StartupSystemTablesThreads, CurrentMetrics::StartupSystemTablesThreadsActive, CurrentMetrics::StartupSystemTablesThreadsScheduled); DatabaseCatalog::instance().getSystemDatabase()->startupTables(pool, LoadingStrictnessLevel::FORCE_RESTORE); } @@ -472,7 +473,7 @@ void convertDatabasesEnginesIfNeed(ContextMutablePtr context) void startupSystemTables() { - ThreadPool pool(CurrentMetrics::StartupSystemTablesThreads, CurrentMetrics::StartupSystemTablesThreadsActive); + ThreadPool pool(CurrentMetrics::StartupSystemTablesThreads, CurrentMetrics::StartupSystemTablesThreadsActive, CurrentMetrics::StartupSystemTablesThreadsScheduled); DatabaseCatalog::instance().getSystemDatabase()->startupTables(pool, LoadingStrictnessLevel::FORCE_RESTORE); } diff --git a/src/Interpreters/threadPoolCallbackRunner.h b/src/Interpreters/threadPoolCallbackRunner.h index eb90b61cf31e..2b9431102732 100644 --- a/src/Interpreters/threadPoolCallbackRunner.h +++ b/src/Interpreters/threadPoolCallbackRunner.h @@ -24,9 +24,10 @@ ThreadPoolCallbackRunner threadPoolCallbackRunner(ThreadPool & if (thread_group) CurrentThread::attachToGroup(thread_group); - SCOPE_EXIT_SAFE({ + SCOPE_EXIT_SAFE( + { { - /// Release all captutred resources before detaching thread group + /// Release all captured resources before detaching thread group /// Releasing has to use proper memory tracker which has been set here before callback [[maybe_unused]] auto tmp = std::move(my_callback); @@ -34,7 +35,6 @@ ThreadPoolCallbackRunner threadPoolCallbackRunner(ThreadPool & if (thread_group) CurrentThread::detachFromGroupIfNotDetached(); - }); setThreadName(thread_name.data()); diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 37af391fba31..812b64ccdb86 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -22,6 +22,7 @@ namespace CurrentMetrics { extern const Metric QueryPipelineExecutorThreads; extern const Metric QueryPipelineExecutorThreadsActive; + extern const Metric QueryPipelineExecutorThreadsScheduled; } namespace DB @@ -332,7 +333,7 @@ void PipelineExecutor::initializeExecution(size_t num_threads, bool concurrency_ tasks.fill(queue); if (num_threads > 1) - pool = std::make_unique(CurrentMetrics::QueryPipelineExecutorThreads, CurrentMetrics::QueryPipelineExecutorThreadsActive, num_threads); + pool = std::make_unique(CurrentMetrics::QueryPipelineExecutorThreads, CurrentMetrics::QueryPipelineExecutorThreadsActive, CurrentMetrics::QueryPipelineExecutorThreadsScheduled, num_threads); } void PipelineExecutor::spawnThreads() diff --git a/src/Processors/Formats/Impl/DWARFBlockInputFormat.cpp b/src/Processors/Formats/Impl/DWARFBlockInputFormat.cpp index c5f8059f93ca..19ed5c94dfdf 100644 --- a/src/Processors/Formats/Impl/DWARFBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/DWARFBlockInputFormat.cpp @@ -28,6 +28,7 @@ namespace CurrentMetrics { extern const Metric DWARFReaderThreads; extern const Metric DWARFReaderThreadsActive; + extern const Metric DWARFReaderThreadsScheduled; } namespace DB @@ -238,7 +239,7 @@ void DWARFBlockInputFormat::initializeIfNeeded() LOG_DEBUG(&Poco::Logger::get("DWARF"), "{} units, reading in {} threads", units_queue.size(), num_threads); - pool.emplace(CurrentMetrics::DWARFReaderThreads, CurrentMetrics::DWARFReaderThreadsActive, num_threads); + pool.emplace(CurrentMetrics::DWARFReaderThreads, CurrentMetrics::DWARFReaderThreadsActive, CurrentMetrics::DWARFReaderThreadsScheduled, num_threads); for (size_t i = 0; i < num_threads; ++i) pool->scheduleOrThrowOnError( [this, thread_group = CurrentThread::getGroup()]() diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index bf8968dd376b..c2f08479730e 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -21,6 +21,7 @@ namespace CurrentMetrics { extern const Metric ParallelFormattingOutputFormatThreads; extern const Metric ParallelFormattingOutputFormatThreadsActive; + extern const Metric ParallelFormattingOutputFormatThreadsScheduled; } namespace DB @@ -80,7 +81,7 @@ class ParallelFormattingOutputFormat : public IOutputFormat explicit ParallelFormattingOutputFormat(Params params) : IOutputFormat(params.header, params.out) , internal_formatter_creator(params.internal_formatter_creator) - , pool(CurrentMetrics::ParallelFormattingOutputFormatThreads, CurrentMetrics::ParallelFormattingOutputFormatThreadsActive, params.max_threads_for_parallel_formatting) + , pool(CurrentMetrics::ParallelFormattingOutputFormatThreads, CurrentMetrics::ParallelFormattingOutputFormatThreadsActive, CurrentMetrics::ParallelFormattingOutputFormatThreadsScheduled, params.max_threads_for_parallel_formatting) { LOG_TEST(&Poco::Logger::get("ParallelFormattingOutputFormat"), "Parallel formatting is being used"); diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index f61dc3fbc780..be9e50b854b7 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -18,6 +18,7 @@ namespace CurrentMetrics { extern const Metric ParallelParsingInputFormatThreads; extern const Metric ParallelParsingInputFormatThreadsActive; + extern const Metric ParallelParsingInputFormatThreadsScheduled; } namespace DB @@ -101,7 +102,7 @@ class ParallelParsingInputFormat : public IInputFormat , min_chunk_bytes(params.min_chunk_bytes) , max_block_size(params.max_block_size) , is_server(params.is_server) - , pool(CurrentMetrics::ParallelParsingInputFormatThreads, CurrentMetrics::ParallelParsingInputFormatThreadsActive, params.max_threads) + , pool(CurrentMetrics::ParallelParsingInputFormatThreads, CurrentMetrics::ParallelParsingInputFormatThreadsActive, CurrentMetrics::ParallelParsingInputFormatThreadsScheduled, params.max_threads) { // One unit for each thread, including segmentator and reader, plus a // couple more units so that the segmentation thread doesn't spuriously diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp index c9ac2438fc0d..d37c2dc11604 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp @@ -28,6 +28,7 @@ namespace CurrentMetrics { extern const Metric ParquetDecoderThreads; extern const Metric ParquetDecoderThreadsActive; + extern const Metric ParquetDecoderThreadsScheduled; } namespace DB @@ -377,7 +378,7 @@ ParquetBlockInputFormat::ParquetBlockInputFormat( , pending_chunks(PendingChunk::Compare { .row_group_first = format_settings_.parquet.preserve_order }) { if (max_decoding_threads > 1) - pool = std::make_unique(CurrentMetrics::ParquetDecoderThreads, CurrentMetrics::ParquetDecoderThreadsActive, max_decoding_threads); + pool = std::make_unique(CurrentMetrics::ParquetDecoderThreads, CurrentMetrics::ParquetDecoderThreadsActive, CurrentMetrics::ParquetDecoderThreadsScheduled, max_decoding_threads); } ParquetBlockInputFormat::~ParquetBlockInputFormat() diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index fbf8b3a7c878..fb9f853dc01b 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -13,6 +13,7 @@ namespace CurrentMetrics { extern const Metric ParquetEncoderThreads; extern const Metric ParquetEncoderThreadsActive; + extern const Metric ParquetEncoderThreadsScheduled; } namespace DB @@ -79,7 +80,9 @@ ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, const Blo { if (format_settings.parquet.parallel_encoding && format_settings.max_threads > 1) pool = std::make_unique( - CurrentMetrics::ParquetEncoderThreads, CurrentMetrics::ParquetEncoderThreadsActive, + CurrentMetrics::ParquetEncoderThreads, + CurrentMetrics::ParquetEncoderThreadsActive, + CurrentMetrics::ParquetEncoderThreadsScheduled, format_settings.max_threads); using C = FormatSettings::ParquetCompression; diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 3420cdeaa501..61a6acd6bc80 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -13,6 +13,7 @@ namespace CurrentMetrics { extern const Metric DestroyAggregatesThreads; extern const Metric DestroyAggregatesThreadsActive; + extern const Metric DestroyAggregatesThreadsScheduled; } namespace DB @@ -95,6 +96,7 @@ struct ManyAggregatedData const auto pool = std::make_unique( CurrentMetrics::DestroyAggregatesThreads, CurrentMetrics::DestroyAggregatesThreadsActive, + CurrentMetrics::DestroyAggregatesThreadsScheduled, variants.size()); for (auto && variant : variants) diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index 65a4aa2741a7..8c58e3042036 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -31,8 +31,6 @@ #include #include #include -#include -#include #include #include @@ -43,6 +41,7 @@ namespace CurrentMetrics extern const Metric DistributedSend; extern const Metric DistributedInsertThreads; extern const Metric DistributedInsertThreadsActive; + extern const Metric DistributedInsertThreadsScheduled; } namespace ProfileEvents @@ -465,6 +464,7 @@ void DistributedSink::writeSync(const Block & block) pool.emplace( CurrentMetrics::DistributedInsertThreads, CurrentMetrics::DistributedInsertThreadsActive, + CurrentMetrics::DistributedInsertThreadsScheduled, max_threads, max_threads, jobs_count); if (!throttler && (settings.max_network_bandwidth || settings.max_network_bytes)) diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index 1587354452eb..f03136e4edf3 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -46,6 +46,7 @@ namespace CurrentMetrics { extern const Metric StorageHiveThreads; extern const Metric StorageHiveThreadsActive; + extern const Metric StorageHiveThreadsScheduled; } namespace DB @@ -861,7 +862,7 @@ HiveFiles StorageHive::collectHiveFiles( Int64 hive_max_query_partitions = context_->getSettings().max_partitions_to_read; /// Mutext to protect hive_files, which maybe appended in multiple threads std::mutex hive_files_mutex; - ThreadPool pool{CurrentMetrics::StorageHiveThreads, CurrentMetrics::StorageHiveThreadsActive, max_threads}; + ThreadPool pool{CurrentMetrics::StorageHiveThreads, CurrentMetrics::StorageHiveThreadsActive, CurrentMetrics::StorageHiveThreadsScheduled, max_threads}; if (!partitions.empty()) { for (const auto & partition : partitions) diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp index 1a7a0b5b2c15..a5f503718b6d 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp @@ -14,6 +14,7 @@ namespace CurrentMetrics { extern const Metric MergeTreeBackgroundExecutorThreads; extern const Metric MergeTreeBackgroundExecutorThreadsActive; + extern const Metric MergeTreeBackgroundExecutorThreadsScheduled; } namespace DB @@ -40,7 +41,7 @@ MergeTreeBackgroundExecutor::MergeTreeBackgroundExecutor( , metric(metric_) , max_tasks_metric(max_tasks_metric_, 2 * max_tasks_count) // active + pending , pool(std::make_unique( - CurrentMetrics::MergeTreeBackgroundExecutorThreads, CurrentMetrics::MergeTreeBackgroundExecutorThreadsActive)) + CurrentMetrics::MergeTreeBackgroundExecutorThreads, CurrentMetrics::MergeTreeBackgroundExecutorThreadsActive, CurrentMetrics::MergeTreeBackgroundExecutorThreadsScheduled)) { if (max_tasks_count == 0) throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Task count for MergeTreeBackgroundExecutor must not be zero"); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index e521491c2d51..d1a285b8818f 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -53,6 +53,7 @@ namespace CurrentMetrics { extern const Metric MergeTreeDataSelectExecutorThreads; extern const Metric MergeTreeDataSelectExecutorThreadsActive; + extern const Metric MergeTreeDataSelectExecutorThreadsScheduled; } namespace DB @@ -1075,6 +1076,7 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd ThreadPool pool( CurrentMetrics::MergeTreeDataSelectExecutorThreads, CurrentMetrics::MergeTreeDataSelectExecutorThreadsActive, + CurrentMetrics::MergeTreeDataSelectExecutorThreadsScheduled, num_threads); for (size_t part_index = 0; part_index < parts.size(); ++part_index) diff --git a/src/Storages/StorageAzureBlob.cpp b/src/Storages/StorageAzureBlob.cpp index 2e0703a8df3e..b43f25b0fff3 100644 --- a/src/Storages/StorageAzureBlob.cpp +++ b/src/Storages/StorageAzureBlob.cpp @@ -53,6 +53,7 @@ namespace CurrentMetrics { extern const Metric ObjectStorageAzureThreads; extern const Metric ObjectStorageAzureThreadsActive; + extern const Metric ObjectStorageAzureThreadsScheduled; } namespace ProfileEvents @@ -1087,7 +1088,7 @@ StorageAzureBlobSource::StorageAzureBlobSource( , file_iterator(file_iterator_) , need_only_count(need_only_count_) , query_info(query_info_) - , create_reader_pool(CurrentMetrics::ObjectStorageAzureThreads, CurrentMetrics::ObjectStorageAzureThreadsActive, 1) + , create_reader_pool(CurrentMetrics::ObjectStorageAzureThreads, CurrentMetrics::ObjectStorageAzureThreadsActive, CurrentMetrics::ObjectStorageAzureThreadsScheduled, 1) , create_reader_scheduler(threadPoolCallbackRunner(create_reader_pool, "AzureReader")) { reader = createReader(); diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 94ce525bc386..2dedc8abdda5 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -134,6 +134,7 @@ namespace CurrentMetrics { extern const Metric StorageDistributedThreads; extern const Metric StorageDistributedThreadsActive; + extern const Metric StorageDistributedThreadsScheduled; } namespace DB @@ -1214,7 +1215,7 @@ void StorageDistributed::initializeFromDisk() const auto & disks = data_volume->getDisks(); /// Make initialization for large number of disks parallel. - ThreadPool pool(CurrentMetrics::StorageDistributedThreads, CurrentMetrics::StorageDistributedThreadsActive, disks.size()); + ThreadPool pool(CurrentMetrics::StorageDistributedThreads, CurrentMetrics::StorageDistributedThreadsActive, CurrentMetrics::StorageDistributedThreadsScheduled, disks.size()); for (const DiskPtr & disk : disks) { diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 63ed84680c98..80ee1e9339d1 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -77,6 +77,7 @@ namespace CurrentMetrics { extern const Metric StorageS3Threads; extern const Metric StorageS3ThreadsActive; + extern const Metric StorageS3ThreadsScheduled; } namespace ProfileEvents @@ -147,7 +148,7 @@ class StorageS3Source::DisclosedGlobIterator::Impl : WithContext , virtual_columns(virtual_columns_) , read_keys(read_keys_) , request_settings(request_settings_) - , list_objects_pool(CurrentMetrics::StorageS3Threads, CurrentMetrics::StorageS3ThreadsActive, 1) + , list_objects_pool(CurrentMetrics::StorageS3Threads, CurrentMetrics::StorageS3ThreadsActive, CurrentMetrics::StorageS3ThreadsScheduled, 1) , list_objects_scheduler(threadPoolCallbackRunner(list_objects_pool, "ListObjects")) , file_progress_callback(file_progress_callback_) { @@ -499,7 +500,7 @@ StorageS3Source::ReadTaskIterator::ReadTaskIterator( size_t max_threads_count) : callback(callback_) { - ThreadPool pool(CurrentMetrics::StorageS3Threads, CurrentMetrics::StorageS3ThreadsActive, max_threads_count); + ThreadPool pool(CurrentMetrics::StorageS3Threads, CurrentMetrics::StorageS3ThreadsActive, CurrentMetrics::StorageS3ThreadsScheduled, max_threads_count); auto pool_scheduler = threadPoolCallbackRunner(pool, "S3ReadTaskItr"); std::vector> keys; @@ -564,7 +565,7 @@ StorageS3Source::StorageS3Source( , file_iterator(file_iterator_) , max_parsing_threads(max_parsing_threads_) , need_only_count(need_only_count_) - , create_reader_pool(CurrentMetrics::StorageS3Threads, CurrentMetrics::StorageS3ThreadsActive, 1) + , create_reader_pool(CurrentMetrics::StorageS3Threads, CurrentMetrics::StorageS3ThreadsActive, CurrentMetrics::StorageS3ThreadsScheduled, 1) , create_reader_scheduler(threadPoolCallbackRunner(create_reader_pool, "CreateS3Reader")) { } diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index ffefd41327d3..d9a120954434 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -23,6 +23,7 @@ namespace CurrentMetrics { extern const Metric SystemReplicasThreads; extern const Metric SystemReplicasThreadsActive; + extern const Metric SystemReplicasThreadsScheduled; } namespace DB @@ -59,7 +60,7 @@ class StatusRequestsPool public: explicit StatusRequestsPool(size_t max_threads) - : thread_pool(CurrentMetrics::SystemReplicasThreads, CurrentMetrics::SystemReplicasThreadsActive, max_threads) + : thread_pool(CurrentMetrics::SystemReplicasThreads, CurrentMetrics::SystemReplicasThreadsActive, CurrentMetrics::SystemReplicasThreadsScheduled, max_threads) , log(&Poco::Logger::get("StatusRequestsPool")) {} diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index 13855c6d94e9..611ca948c53c 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -10,10 +10,12 @@ #include #include + namespace CurrentMetrics { extern const Metric LocalThread; extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; } namespace DB::ErrorCodes @@ -106,7 +108,7 @@ Runner::Runner( std::cerr << "---- Run options ----\n" << std::endl; - pool.emplace(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, concurrency); + pool.emplace(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, CurrentMetrics::LocalThreadScheduled, concurrency); queue.emplace(concurrency); } @@ -461,4 +463,3 @@ Runner::~Runner() pool->wait(); generator->cleanup(*connections[0]); } - From 593f04a6b5825182940e1f241989d801b57ed93d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Nov 2023 20:19:24 +0100 Subject: [PATCH 179/274] Fix style --- src/IO/SharedThreadPools.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/SharedThreadPools.cpp b/src/IO/SharedThreadPools.cpp index 4a0ef1736697..6af5aab7a385 100644 --- a/src/IO/SharedThreadPools.cpp +++ b/src/IO/SharedThreadPools.cpp @@ -130,7 +130,7 @@ StaticThreadPool & getBackupsIOThreadPool() StaticThreadPool & getActivePartsLoadingThreadPool() { - static StaticThreadPool instance("MergeTreePartsLoaderThreadPool", CurrentMetrics::MergeTreePartsLoaderThreads, CurrentMetrics::MergeTreePartsLoaderThreadsActive, CurrentMetrics::MergeTreeOutdatedPartsLoaderThreadsScheduled); + static StaticThreadPool instance("MergeTreePartsLoaderThreadPool", CurrentMetrics::MergeTreePartsLoaderThreads, CurrentMetrics::MergeTreePartsLoaderThreadsActive, CurrentMetrics::MergeTreePartsLoaderThreadsScheduled); return instance; } From 75cebb3c29c54979218f0f0f80f46609851059b9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 18 Nov 2023 20:20:21 +0100 Subject: [PATCH 180/274] Fix build --- src/Coordination/Standalone/Context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Coordination/Standalone/Context.cpp b/src/Coordination/Standalone/Context.cpp index 4001673e01a8..6942e866f234 100644 --- a/src/Coordination/Standalone/Context.cpp +++ b/src/Coordination/Standalone/Context.cpp @@ -263,7 +263,7 @@ ThreadPool & Context::getThreadPoolWriter() const auto queue_size = config.getUInt(".threadpool_writer_queue_size", 1000000); shared->threadpool_writer = std::make_unique( - CurrentMetrics::IOWriterThreads, CurrentMetrics::IOWriterThreadsActive, pool_size, pool_size, queue_size); + CurrentMetrics::IOWriterThreads, CurrentMetrics::IOWriterThreadsActive, CurrentMetrics::IOWriterThreadsScheduled, pool_size, pool_size, queue_size); }); return *shared->threadpool_writer; From 7c81d9b2b73ee35f8cf317397c2bd23dcd2336ea Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Sat, 18 Nov 2023 19:38:33 +0000 Subject: [PATCH 181/274] Fix test_keeper_auth --- tests/integration/test_keeper_auth/test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_keeper_auth/test.py b/tests/integration/test_keeper_auth/test.py index e247984cc6a6..78fbf84bbe2c 100644 --- a/tests/integration/test_keeper_auth/test.py +++ b/tests/integration/test_keeper_auth/test.py @@ -1,6 +1,7 @@ import pytest import time from helpers.cluster import ClickHouseCluster +from helpers import keeper_utils from kazoo.client import KazooClient, KazooState from kazoo.security import ACL, make_digest_acl, make_acl from kazoo.exceptions import ( @@ -26,6 +27,7 @@ def started_cluster(): try: cluster.start() + keeper_utils.wait_until_connected(cluster, node) yield cluster From 053b20a255d33dca78c8260e6f49cd94ac2545c7 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Sun, 19 Nov 2023 00:44:39 +0100 Subject: [PATCH 182/274] fix in_data pointer --- src/IO/Lz4DeflatingWriteBuffer.cpp | 16 +++++----------- src/IO/Lz4DeflatingWriteBuffer.h | 3 --- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/src/IO/Lz4DeflatingWriteBuffer.cpp b/src/IO/Lz4DeflatingWriteBuffer.cpp index e952e6400ecd..2dbb7f684ed1 100644 --- a/src/IO/Lz4DeflatingWriteBuffer.cpp +++ b/src/IO/Lz4DeflatingWriteBuffer.cpp @@ -43,9 +43,6 @@ namespace { tmp_out.finalize(); - if (cur_out == sink) - return; - sink->write(tmp_out.buffer().begin(), tmp_out.count()); } @@ -67,8 +64,6 @@ namespace ErrorCodes Lz4DeflatingWriteBuffer::Lz4DeflatingWriteBuffer( std::unique_ptr out_, int compression_level, size_t buf_size, char * existing_memory, size_t alignment) : WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment) - , in_data(nullptr) - , in_capacity(0) , tmp_memory(buf_size) { @@ -106,9 +101,6 @@ void Lz4DeflatingWriteBuffer::nextImpl() if (!offset()) return; - in_data = reinterpret_cast(working_buffer.begin()); - in_capacity = offset(); - if (first_time) { auto sink = SinkToOut(out.get(), tmp_memory, LZ4F_HEADER_SIZE_MAX); @@ -128,7 +120,10 @@ void Lz4DeflatingWriteBuffer::nextImpl() first_time = false; } - do + auto in_data = working_buffer.begin(); + auto in_capacity = offset(); + + while (in_capacity > 0) { /// Ensure that there is enough space for compressed block of minimal size size_t min_compressed_block_size = LZ4F_compressBound(1, &kPrefs); @@ -154,11 +149,10 @@ void Lz4DeflatingWriteBuffer::nextImpl() LZ4F_VERSION, LZ4F_getErrorName(compressed_size), sink.getCapacity()); in_capacity -= cur_buffer_size; - in_data = reinterpret_cast(working_buffer.end() - in_capacity); + in_data += cur_buffer_size; sink.advancePosition(compressed_size); } - while (in_capacity > 0); } void Lz4DeflatingWriteBuffer::finalizeBefore() diff --git a/src/IO/Lz4DeflatingWriteBuffer.h b/src/IO/Lz4DeflatingWriteBuffer.h index 65f4f0c73498..7bb8a5e6c0ec 100644 --- a/src/IO/Lz4DeflatingWriteBuffer.h +++ b/src/IO/Lz4DeflatingWriteBuffer.h @@ -32,9 +32,6 @@ class Lz4DeflatingWriteBuffer : public WriteBufferWithOwnMemoryDecorator LZ4F_preferences_t kPrefs; /// NOLINT LZ4F_compressionContext_t ctx; - void * in_data; - size_t in_capacity; - Memory<> tmp_memory; bool first_time = true; From a894671e8af10d4b5b70d79beccdba6be2e12174 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Sat, 18 Nov 2023 17:53:32 -0800 Subject: [PATCH 183/274] [Docs] Add perf tip for COUNT(DISTINCT expr) --- docs/en/sql-reference/aggregate-functions/reference/count.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/en/sql-reference/aggregate-functions/reference/count.md b/docs/en/sql-reference/aggregate-functions/reference/count.md index a98c8e50174d..a40108a331ab 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/count.md +++ b/docs/en/sql-reference/aggregate-functions/reference/count.md @@ -34,6 +34,10 @@ The `SELECT count() FROM table` query is optimized by default using metadata fro However `SELECT count(nullable_column) FROM table` query can be optimized by enabling the [optimize_functions_to_subcolumns](../../../operations/settings/settings.md#optimize-functions-to-subcolumns) setting. With `optimize_functions_to_subcolumns = 1` the function reads only [null](../../../sql-reference/data-types/nullable.md#finding-null) subcolumn instead of reading and processing the whole column data. The query `SELECT count(n) FROM table` transforms to `SELECT sum(NOT n.null) FROM table`. +**Improving COUNT(DISTINCT expr) performance** + +If your `COUNT(DISTINCT expr)` query is slow, consider adding a [`GROUP BY`](../../../sql-reference/statements/select/group-by.md) clause as this improves parallelization. You can also use a [projection](../../../sql-reference/statements/alter/projection.md) to create an index on the target column used with `COUNT(DISTINCT target_col)`. + **Examples** Example 1: From 96e87322b525351b7d241c9e2c6ca125d2178b20 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Sat, 18 Nov 2023 18:10:43 -0800 Subject: [PATCH 184/274] [Docs] Add insert_distributed_sync to Core Settings docs --- docs/en/operations/settings/settings.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index a22bd6e33e54..e61934d21686 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2716,6 +2716,10 @@ Default value: `0`. - [Distributed Table Engine](../../engines/table-engines/special/distributed.md/#distributed) - [Managing Distributed Tables](../../sql-reference/statements/system.md/#query-language-system-distributed) +## insert_distributed_sync {#insert_distributed_sync} + +Alias for [`distributed_foreground_insert`](#distributed_foreground_insert). + ## insert_shard_id {#insert_shard_id} If not `0`, specifies the shard of [Distributed](../../engines/table-engines/special/distributed.md/#distributed) table into which the data will be inserted synchronously. From 5e1da38720d16672bba07da1b9b54fc081b1464e Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Sat, 18 Nov 2023 18:28:29 -0800 Subject: [PATCH 185/274] [Docs] Add details on why partitions improve query perf --- .../table-engines/mergetree-family/custom-partitioning-key.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md index 7e564b236768..97d37e476aee 100644 --- a/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -14,7 +14,7 @@ You should never use too granular of partitioning. Don't partition your data by Partitioning is available for the [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) family tables (including [replicated](../../../engines/table-engines/mergetree-family/replication.md) tables). [Materialized views](../../../engines/table-engines/special/materializedview.md#materializedview) based on MergeTree tables support partitioning, as well. -A partition is a logical combination of records in a table by a specified criterion. You can set a partition by an arbitrary criterion, such as by month, by day, or by event type. Each partition is stored separately to simplify manipulations of this data. When accessing the data, ClickHouse uses the smallest subset of partitions possible. +A partition is a logical combination of records in a table by a specified criterion. You can set a partition by an arbitrary criterion, such as by month, by day, or by event type. Each partition is stored separately to simplify manipulations of this data. When accessing the data, ClickHouse uses the smallest subset of partitions possible. Partitions improve performance for queries containing a partitioning key because ClickHouse will filter for that partition before selecting the parts and granules within the partition. The partition is specified in the `PARTITION BY expr` clause when [creating a table](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table). The partition key can be any expression from the table columns. For example, to specify partitioning by month, use the expression `toYYYYMM(date_column)`: From 8062fb578c7b6c65d4d24efa28bb47e8c8783ae8 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Sat, 18 Nov 2023 18:38:49 -0800 Subject: [PATCH 186/274] [Docs] Recommend ReplacingMergeTree for frequent updates --- docs/en/engines/table-engines/mergetree-family/mergetree.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index e615c9ad9d36..d250cfd1a086 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -6,7 +6,7 @@ sidebar_label: MergeTree # MergeTree -The `MergeTree` engine and other engines of this family (`*MergeTree`) are the most robust ClickHouse table engines. +The `MergeTree` engine and other engines of this family (`*MergeTree`) are the most common and most robust ClickHouse table engines. Engines in the `MergeTree` family are designed for inserting a very large amount of data into a table. The data is quickly written to the table part by part, then rules are applied for merging the parts in the background. This method is much more efficient than continually rewriting the data in storage during insert. @@ -32,6 +32,8 @@ Main features: The [Merge](/docs/en/engines/table-engines/special/merge.md/#merge) engine does not belong to the `*MergeTree` family. ::: +If you need to update rows frequently, we recommend using the [`ReplacingMergeTree`](/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md). Using `ALTER TALBE my_table UPDATE` to update rows triggers a mutation, which causes parts to be re-written and uses IO/resources. With `ReplacingMergeTree`, you can simply insert the updated rows and the old rows will be replaced according to the table sorting key. + ## Creating a Table {#table_engine-mergetree-creating-a-table} ``` sql From a398e3f51e499c5523e62ac4c296b0203304e672 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Sat, 18 Nov 2023 18:40:14 -0800 Subject: [PATCH 187/274] [Docs] Fix typo --- docs/en/engines/table-engines/mergetree-family/mergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index d250cfd1a086..810b75961507 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -6,7 +6,7 @@ sidebar_label: MergeTree # MergeTree -The `MergeTree` engine and other engines of this family (`*MergeTree`) are the most common and most robust ClickHouse table engines. +The `MergeTree` engine and other engines of this family (`*MergeTree`) are the most commonly used and most robust ClickHouse table engines. Engines in the `MergeTree` family are designed for inserting a very large amount of data into a table. The data is quickly written to the table part by part, then rules are applied for merging the parts in the background. This method is much more efficient than continually rewriting the data in storage during insert. From f29777db2d77aee2d17034e6844377f65e5ddfe7 Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Sat, 18 Nov 2023 18:41:20 -0800 Subject: [PATCH 188/274] [Docs] More typos --- docs/en/engines/table-engines/mergetree-family/mergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 810b75961507..f0bc45b9f531 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -32,7 +32,7 @@ Main features: The [Merge](/docs/en/engines/table-engines/special/merge.md/#merge) engine does not belong to the `*MergeTree` family. ::: -If you need to update rows frequently, we recommend using the [`ReplacingMergeTree`](/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md). Using `ALTER TALBE my_table UPDATE` to update rows triggers a mutation, which causes parts to be re-written and uses IO/resources. With `ReplacingMergeTree`, you can simply insert the updated rows and the old rows will be replaced according to the table sorting key. +If you need to update rows frequently, we recommend using the [`ReplacingMergeTree`](/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md) table engine. Using `ALTER TABLE my_table UPDATE` to update rows triggers a mutation, which causes parts to be re-written and uses IO/resources. With `ReplacingMergeTree`, you can simply insert the updated rows and the old rows will be replaced according to the table sorting key. ## Creating a Table {#table_engine-mergetree-creating-a-table} From 9d3c62ec418802371fcf7f466163d683c1ec262a Mon Sep 17 00:00:00 2001 From: Justin de Guzman Date: Sat, 18 Nov 2023 18:51:35 -0800 Subject: [PATCH 189/274] [Docs] Recommend against OPTIMIZE FINAL in OPTIMIZE page --- docs/en/sql-reference/statements/optimize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/optimize.md b/docs/en/sql-reference/statements/optimize.md index 49843eaff9a5..07b5a196096d 100644 --- a/docs/en/sql-reference/statements/optimize.md +++ b/docs/en/sql-reference/statements/optimize.md @@ -5,7 +5,7 @@ sidebar_label: OPTIMIZE title: "OPTIMIZE Statement" --- -This query tries to initialize an unscheduled merge of data parts for tables. +This query tries to initialize an unscheduled merge of data parts for tables. Note that we generally recommend against using `OPTIMIZE TABLE ... FINAL` (see these [docs](/docs/en/optimize/avoidoptimizefinal)) as its use case is meant for administration, not for daily operations. :::note `OPTIMIZE` can’t fix the `Too many parts` error. From 097f80657c35c83ca522b412a040dfeaa36011d5 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Sun, 19 Nov 2023 11:14:24 +0100 Subject: [PATCH 190/274] Fewer concurrent requests in 02908_many_requests_to_system_replicas --- .../02908_many_requests_to_system_replicas.reference | 2 +- .../0_stateless/02908_many_requests_to_system_replicas.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference index d7850e59dec4..af0e50ec3320 100644 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference @@ -1,5 +1,5 @@ Creating 300 tables -Making making 500 requests to system.replicas +Making making 200 requests to system.replicas Query system.replicas while waiting for other concurrent requests to finish 0 900 diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh index c620fcf4beaf..f93175529c00 100755 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e NUM_TABLES=300 -CONCURRENCY=500 +CONCURRENCY=200 echo "Creating $NUM_TABLES tables" From 24fbe620d32ef624ed0f6003c0d9650b03086544 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Sun, 19 Nov 2023 12:14:53 +0100 Subject: [PATCH 191/274] fix build --- src/IO/Lz4DeflatingWriteBuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/Lz4DeflatingWriteBuffer.cpp b/src/IO/Lz4DeflatingWriteBuffer.cpp index 2dbb7f684ed1..7def2da104fa 100644 --- a/src/IO/Lz4DeflatingWriteBuffer.cpp +++ b/src/IO/Lz4DeflatingWriteBuffer.cpp @@ -120,7 +120,7 @@ void Lz4DeflatingWriteBuffer::nextImpl() first_time = false; } - auto in_data = working_buffer.begin(); + auto * in_data = working_buffer.begin(); auto in_capacity = offset(); while (in_capacity > 0) From cacc23b8b745f15eb6d84db002987b6229010932 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Sun, 19 Nov 2023 12:25:42 +0100 Subject: [PATCH 192/274] safe SinkToOut d-tor --- src/IO/Lz4DeflatingWriteBuffer.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/IO/Lz4DeflatingWriteBuffer.cpp b/src/IO/Lz4DeflatingWriteBuffer.cpp index 7def2da104fa..0af205a426dd 100644 --- a/src/IO/Lz4DeflatingWriteBuffer.cpp +++ b/src/IO/Lz4DeflatingWriteBuffer.cpp @@ -39,10 +39,9 @@ namespace cur_out->position() += size; } - ~SinkToOut() noexcept(false) + void finalize() { tmp_out.finalize(); - sink->write(tmp_out.buffer().begin(), tmp_out.count()); } @@ -117,6 +116,7 @@ void Lz4DeflatingWriteBuffer::nextImpl() LZ4F_VERSION, LZ4F_getErrorName(header_size)); sink.advancePosition(header_size); + sink.finalize(); first_time = false; } @@ -152,6 +152,7 @@ void Lz4DeflatingWriteBuffer::nextImpl() in_data += cur_buffer_size; sink.advancePosition(compressed_size); + sink.finalize(); } } @@ -173,6 +174,7 @@ void Lz4DeflatingWriteBuffer::finalizeBefore() LZ4F_VERSION, LZ4F_getErrorName(end_size), sink.getCapacity()); sink.advancePosition(end_size); + sink.finalize(); } void Lz4DeflatingWriteBuffer::finalizeAfter() From f27018c14196e938aa3b16cae387e46d2578501a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Nov 2023 15:21:11 +0100 Subject: [PATCH 193/274] Own CMake for GRPC --- cmake/limit_jobs.cmake | 4 +- contrib/grpc-cmake/CMakeLists.txt | 40 +- contrib/grpc-cmake/grpc.cmake | 1864 +++++++++++++++++++++++++++++ 3 files changed, 1868 insertions(+), 40 deletions(-) create mode 100644 contrib/grpc-cmake/grpc.cmake diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index 28ccb62e10c6..a43e208ff0dd 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -21,7 +21,7 @@ if (NOT PARALLEL_COMPILE_JOBS AND MAX_COMPILER_MEMORY) set (PARALLEL_COMPILE_JOBS 1) endif () if (PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES) - message(WARNING "The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.") + message(INFO "The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.") endif() endif () @@ -32,7 +32,7 @@ if (NOT PARALLEL_LINK_JOBS AND MAX_LINKER_MEMORY) set (PARALLEL_LINK_JOBS 1) endif () if (PARALLEL_LINK_JOBS LESS NUMBER_OF_LOGICAL_CORES) - message(WARNING "The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.") + message(INFO "The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.") endif() endif () diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index 09ed2fe3f80c..b8b5f5580c4e 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -9,50 +9,14 @@ endif() set(_gRPC_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/grpc") set(_gRPC_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/grpc") -# Use re2 from ClickHouse contrib, not from gRPC third_party. -set(gRPC_RE2_PROVIDER "clickhouse" CACHE STRING "" FORCE) -set(_gRPC_RE2_INCLUDE_DIR "") -set(_gRPC_RE2_LIBRARIES ch_contrib::re2) - -# Use zlib from ClickHouse contrib, not from gRPC third_party. -set(gRPC_ZLIB_PROVIDER "clickhouse" CACHE STRING "" FORCE) -set(_gRPC_ZLIB_INCLUDE_DIR "") -set(_gRPC_ZLIB_LIBRARIES ch_contrib::zlib) - -# Use protobuf from ClickHouse contrib, not from gRPC third_party. -set(gRPC_PROTOBUF_PROVIDER "clickhouse" CACHE STRING "" FORCE) -set(_gRPC_PROTOBUF_LIBRARIES ch_contrib::protobuf) -set(_gRPC_PROTOBUF_PROTOC "protoc") -set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE $) -set(_gRPC_PROTOBUF_PROTOC_LIBRARIES ch_contrib::protoc) - if(TARGET OpenSSL::SSL) set(gRPC_USE_UNSECURE_LIBRARIES FALSE) else() set(gRPC_USE_UNSECURE_LIBRARIES TRUE) endif() -# Use OpenSSL from ClickHouse contrib, not from gRPC third_party. -set(gRPC_SSL_PROVIDER "clickhouse" CACHE STRING "" FORCE) -set(_gRPC_SSL_INCLUDE_DIR "") -set(_gRPC_SSL_LIBRARIES OpenSSL::Crypto OpenSSL::SSL) - -# Use abseil-cpp from ClickHouse contrib, not from gRPC third_party. -set(gRPC_ABSL_PROVIDER "clickhouse" CACHE STRING "" FORCE) - -# We don't want to build C# extensions. -set(gRPC_BUILD_CSHARP_EXT OFF) - -# TODO: Remove this. We generally like to compile with C++23 but grpc isn't ready yet. -set (CMAKE_CXX_STANDARD 20) - -set(_gRPC_CARES_LIBRARIES ch_contrib::c-ares) -set(gRPC_CARES_PROVIDER "clickhouse" CACHE STRING "" FORCE) -add_subdirectory("${_gRPC_SOURCE_DIR}" "${_gRPC_BINARY_DIR}") - -# The contrib/grpc/CMakeLists.txt redefined the PROTOBUF_GENERATE_GRPC_CPP() function for its own purposes, -# so we need to redefine it back. -include("${ClickHouse_SOURCE_DIR}/contrib/grpc-cmake/protobuf_generate_grpc.cmake") +include(grpc.cmake) +include(protobuf_generate_grpc.cmake) set(gRPC_CPP_PLUGIN $) set(gRPC_PYTHON_PLUGIN $) diff --git a/contrib/grpc-cmake/grpc.cmake b/contrib/grpc-cmake/grpc.cmake new file mode 100644 index 000000000000..43d4edd191e5 --- /dev/null +++ b/contrib/grpc-cmake/grpc.cmake @@ -0,0 +1,1864 @@ +# This file was edited for ClickHouse. + +# GRPC global cmake file +# This currently builds C and C++ code. +# This file has been automatically generated from a template file. +# Please look at the templates directory instead. +# This file can be regenerated from the template by running +# tools/buildgen/generate_projects.sh +# +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# We want to use C++23, but GRPC is not ready +set (CMAKE_CXX_STANDARD 20) + +set(_gRPC_ZLIB_INCLUDE_DIR "") +set(_gRPC_ZLIB_LIBRARIES ch_contrib::zlib) + +set(_gRPC_CARES_LIBRARIES ch_contrib::c-ares) + +set(_gRPC_RE2_INCLUDE_DIR "") +set(_gRPC_RE2_LIBRARIES ch_contrib::re2) + +set(_gRPC_SSL_INCLUDE_DIR "") +set(_gRPC_SSL_LIBRARIES OpenSSL::Crypto OpenSSL::SSL) + +set(_gRPC_PROTOBUF_LIBRARIES ch_contrib::protobuf) +set(_gRPC_PROTOBUF_PROTOC "protoc") +set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE $) +set(_gRPC_PROTOBUF_PROTOC_LIBRARIES ch_contrib::protoc) + + +if(UNIX) + if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") + set(_gRPC_PLATFORM_LINUX ON) + elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set(_gRPC_PLATFORM_MAC ON) + elseif(${CMAKE_SYSTEM_NAME} MATCHES "iOS") + set(_gRPC_PLATFORM_IOS ON) + elseif(${CMAKE_SYSTEM_NAME} MATCHES "Android") + set(_gRPC_PLATFORM_ANDROID ON) + else() + set(_gRPC_PLATFORM_POSIX ON) + endif() +endif() + +if(UNIX AND NOT HAIKU) + # -pthread does more than -lpthread + set(THREADS_PREFER_PTHREAD_FLAG ON) + find_package(Threads) + set(_gRPC_ALLTARGETS_LIBRARIES ${CMAKE_DL_LIBS} Threads::Threads) + if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) + set(_gRPC_ALLTARGETS_LIBRARIES ${_gRPC_ALLTARGETS_LIBRARIES} rt) + endif() +endif() + +set(_gRPC_ADDRESS_SORTING_INCLUDE_DIR "${_gRPC_SOURCE_DIR}/third_party/address_sorting/include") +set(_gRPC_ADDRESS_SORTING_LIBRARIES address_sorting) + +set(UPB_ROOT_DIR ${_gRPC_SOURCE_DIR}/third_party/upb) + +set(_gRPC_UPB_INCLUDE_DIR "${UPB_ROOT_DIR}" "${_gRPC_SOURCE_DIR}/third_party/utf8_range") +set(_gRPC_UPB_GRPC_GENERATED_DIR "${_gRPC_SOURCE_DIR}/src//core/ext/upb-generated" "${_gRPC_SOURCE_DIR}/src//core/ext/upbdefs-generated") + +set(_gRPC_UPB_LIBRARIES upb) + +set(_gRPC_XXHASH_INCLUDE_DIR "${_gRPC_SOURCE_DIR}/third_party/xxhash") + +add_library(address_sorting + ${_gRPC_SOURCE_DIR}/third_party/address_sorting/address_sorting.c + ${_gRPC_SOURCE_DIR}/third_party/address_sorting/address_sorting_posix.c + ${_gRPC_SOURCE_DIR}/third_party/address_sorting/address_sorting_windows.c +) + +target_compile_features(address_sorting PUBLIC cxx_std_14) + +target_include_directories(address_sorting + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(address_sorting + ${_gRPC_ALLTARGETS_LIBRARIES} +) + + +add_library(gpr + ${_gRPC_SOURCE_DIR}/src/core/lib/config/config_vars.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/config/config_vars_non_generated.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/config/load_config.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thread_local.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/alloc.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/android/log.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/atm.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/iphone/cpu.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/linux/cpu.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/linux/log.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/log.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/msys/tmpfile.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/posix/cpu.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/posix/log.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/posix/string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/posix/sync.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/posix/time.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/posix/tmpfile.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/sync.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/sync_abseil.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/time.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/time_precise.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/windows/cpu.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/windows/log.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/windows/string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/windows/string_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/windows/sync.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/windows/time.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/windows/tmpfile.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gpr/wrap_memcpy.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/crash.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/examine_stack.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/fork.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/host_port.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/linux/env.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/mpscq.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/posix/env.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/posix/stat.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/posix/thd.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/strerror.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/tchar.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/time_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/windows/env.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/windows/stat.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/windows/thd.cc +) + +target_compile_features(gpr PUBLIC cxx_std_14) + +target_include_directories(gpr + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(gpr + ${_gRPC_ALLTARGETS_LIBRARIES} + absl::base + absl::core_headers + absl::flags + absl::flags_marshalling + absl::any_invocable + absl::memory + absl::random_random + absl::status + absl::cord + absl::str_format + absl::strings + absl::synchronization + absl::time + absl::optional + absl::variant +) +if(_gRPC_PLATFORM_ANDROID) + target_link_libraries(gpr + android + log + ) +endif() + + +add_library(grpc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/backend_metrics/backend_metric_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/census/grpc_context.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/channel_idle/channel_idle_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/channel_idle/idle_filter_state.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backend_metric.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backup_poller.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/channel_connectivity.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_channelz.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_service_config.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/config_selector.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/dynamic_filters.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/global_subchannel_pool.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/http_proxy_mapper.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/address_filtering.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/endpoint_list.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/health_check_client.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/rls/rls.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/static_stride_scheduler.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/weighted_round_robin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/local_subchannel_pool.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/binder/binder_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/event_engine/event_engine_client_channel_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/event_engine/service_config_helper.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/polling_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_filter_legacy_call_data.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_service_config.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_throttle.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel_pool_interface.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel_stream_client.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/deadline/deadline_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/fault_injection/fault_injection_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/client/http_client_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/client_authority_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/http_filters_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/message_compress/compression_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/server/http_server_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/message_size/message_size_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/rbac/rbac_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/rbac/rbac_service_config_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/server_config_selector/server_config_selector_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/stateful_session/stateful_session_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/gcp/metadata_query.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/alpn/alpn.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/chttp2_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/server/chttp2_server.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_decoder.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_encoder.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_transport.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/decode_huff.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/flow_control.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_data.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_goaway.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_ping.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_settings.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_window_update.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parse_result.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_settings.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/huffsyms.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/max_concurrent_streams_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/parsing.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_abuse_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_callbacks.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_rate_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_lists.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/varint.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/write_size_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/writing.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/inproc/inproc_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/inproc/inproc_transport.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/certs.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/clusters.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/config_dump.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/config_dump_shared.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/init_dump.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/listeners.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/memory.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/metrics.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/mutex_stats.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/server_info.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/admin/v3/tap.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/annotations/deprecation.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/annotations/resource.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/accesslog/v3/accesslog.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/bootstrap/v3/bootstrap.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/cluster/v3/circuit_breaker.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/cluster/v3/cluster.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/cluster/v3/filter.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/cluster/v3/outlier_detection.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/common/matcher/v3/matcher.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/address.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/backoff.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/base.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/config_source.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/event_service_config.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/extension.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/grpc_method_list.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/grpc_service.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/health_check.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/http_uri.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/protocol.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/proxy_protocol.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/resolver.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/socket_option.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/substitution_format_string.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/core/v3/udp_socket_config.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint_components.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/endpoint/v3/load_report.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/listener/v3/api_listener.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/listener/v3/listener.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/listener/v3/listener_components.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/listener/v3/quic_config.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/listener/v3/udp_listener_config.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/metrics/v3/metrics_service.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/metrics/v3/stats.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/overload/v3/overload.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/rbac/v3/rbac.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/route/v3/route.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/route/v3/route_components.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/route/v3/scoped_route.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/tap/v3/common.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/datadog.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/dynamic_ot.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/http_tracer.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/lightstep.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/opencensus.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/opentelemetry.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/service.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/skywalking.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/trace.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/xray.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/config/trace/v3/zipkin.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/data/accesslog/v3/accesslog.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/clusters/aggregate/v3/cluster.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/filters/common/fault/v3/fault.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/filters/http/fault/v3/fault.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/filters/http/rbac/v3/rbac.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/filters/http/router/v3/router.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/common/v3/common.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/cert.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/common.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/secret.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/service/discovery/v3/ads.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/service/discovery/v3/discovery.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/service/load_stats/v3/lrs.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/service/status/v3/csds.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/http/v3/cookie.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/http/v3/path_transformation.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/filter_state.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/http_inputs.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/metadata.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/node.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/number.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/path.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/regex.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/status_code_input.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/string.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/struct.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/matcher/v3/value.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/metadata/v3/metadata.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/tracing/v3/custom_tag.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/hash_policy.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/http.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/http_status.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/percent.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/range.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/ratelimit_strategy.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/ratelimit_unit.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/semantic_version.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/v3/token_bucket.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/annotations.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/expr/v1alpha1/checked.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/http.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/httpbody.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/any.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/duration.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/empty.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/struct.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/timestamp.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/wrappers.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/rpc/status.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/opencensus/proto/trace/v1/trace_config.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/altscontext.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/lb/v1/load_balancer.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls_config.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/udpa/annotations/migrate.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/udpa/annotations/security.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/udpa/annotations/sensitive.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/udpa/annotations/status.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/udpa/annotations/versioning.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/validate/validate.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/annotations/v3/migrate.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/annotations/v3/security.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/annotations/v3/sensitive.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/annotations/v3/status.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/annotations/v3/versioning.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/authority.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/cidr.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/collection_entry.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/context_params.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/extension.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/resource.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/resource_locator.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/core/v3/resource_name.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/service/orca/v3/orca.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/cel.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/domain.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/http_inputs.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/ip.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/matcher.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/range.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/regex.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/matcher/v3/string.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/v3/cel.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/v3/range.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/type/v3/typed_struct.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/certs.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/clusters.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump_shared.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/init_dump.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/listeners.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/memory.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/metrics.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/mutex_stats.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/server_info.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/admin/v3/tap.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/annotations/deprecation.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/annotations/resource.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/accesslog/v3/accesslog.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/bootstrap/v3/bootstrap.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/circuit_breaker.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/cluster.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/filter.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/outlier_detection.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/common/matcher/v3/matcher.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/address.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/backoff.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/base.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/config_source.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/event_service_config.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/extension.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/grpc_method_list.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/grpc_service.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/health_check.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/http_uri.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/protocol.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/proxy_protocol.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/resolver.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/socket_option.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/substitution_format_string.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/core/v3/udp_socket_config.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/endpoint/v3/endpoint.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/endpoint/v3/endpoint_components.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/endpoint/v3/load_report.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/listener/v3/api_listener.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener_components.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/listener/v3/quic_config.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/listener/v3/udp_listener_config.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/metrics/v3/metrics_service.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/metrics/v3/stats.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/overload/v3/overload.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/rbac/v3/rbac.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/route/v3/route.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/route/v3/route_components.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/route/v3/scoped_route.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/tap/v3/common.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/datadog.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/dynamic_ot.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/http_tracer.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/lightstep.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/opencensus.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/opentelemetry.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/service.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/skywalking.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/trace.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/xray.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/config/trace/v3/zipkin.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/data/accesslog/v3/accesslog.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/clusters/aggregate/v3/cluster.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/filters/common/fault/v3/fault.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/fault/v3/fault.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/rbac/v3/rbac.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/router/v3/router.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/cert.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/common.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/secret.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/service/discovery/v3/ads.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/service/discovery/v3/discovery.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/service/load_stats/v3/lrs.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/service/status/v3/csds.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/http/v3/cookie.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/http/v3/path_transformation.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/filter_state.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/http_inputs.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/metadata.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/node.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/number.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/path.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/regex.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/status_code_input.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/string.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/struct.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/value.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/metadata/v3/metadata.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/tracing/v3/custom_tag.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/hash_policy.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/http.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/http_status.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/percent.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/range.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/ratelimit_strategy.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/ratelimit_unit.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/semantic_version.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/envoy/type/v3/token_bucket.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/api/annotations.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/api/expr/v1alpha1/checked.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/api/expr/v1alpha1/syntax.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/api/http.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/api/httpbody.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/protobuf/any.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/protobuf/descriptor.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/protobuf/duration.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/protobuf/empty.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/protobuf/struct.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/protobuf/timestamp.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/protobuf/wrappers.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/google/rpc/status.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/opencensus/proto/trace/v1/trace_config.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/src/proto/grpc/lookup/v1/rls_config.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/udpa/annotations/migrate.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/udpa/annotations/security.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/udpa/annotations/sensitive.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/udpa/annotations/status.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/udpa/annotations/versioning.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/validate/validate.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/annotations/v3/migrate.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/annotations/v3/security.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/annotations/v3/sensitive.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/annotations/v3/status.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/annotations/v3/versioning.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/authority.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/cidr.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/collection_entry.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/context_params.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/extension.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/resource.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/resource_locator.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/core/v3/resource_name.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/cel.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/domain.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/http_inputs.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/ip.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/matcher.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/range.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/regex.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/matcher/v3/string.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/v3/cel.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/v3/range.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-generated/xds/type/v3/typed_struct.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/certificate_provider_store.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/file_watcher_certificate_provider_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_api.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_audit_logger_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_bootstrap.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_bootstrap_grpc.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_certificate_provider.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_channel_stack_modifier.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_client.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_client_grpc.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_client_stats.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_cluster.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_cluster_specifier_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_common_types.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_health_status.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_http_fault_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_http_filters.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_http_rbac_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_http_stateful_session_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_lb_policy_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_listener.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_route_config.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_routing.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_server_config_fetcher.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/xds/xds_transport_grpc.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/address_utils/parse_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/address_utils/sockaddr_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/backoff/backoff.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/backoff/random_early_detection.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/call_tracer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_args.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_args_preconditioning.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack_builder.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack_builder_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channelz.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channelz_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/connected_channel.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/promise_based_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/server_call_tracer_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/status_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/compression/compression.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/compression/compression_internal.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/compression/message_compress.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/config/core_configuration.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/event_log.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/histogram_view.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/stats.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/stats_data.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/ares_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/cf_engine/cf_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/cf_engine/cfstream_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/cf_engine/dns_service_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/channel_args_endpoint_config.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/default_event_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/default_event_engine_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/event_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/forkable.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/memory_allocator.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/ev_poll_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/internal_errqueue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/lockfree_event.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_engine_listener.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/timer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/timer_heap.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/timer_manager.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/traced_buffer_list.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/resolved_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/shim.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/slice.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/slice_buffer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/tcp_socket_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thread_pool/thread_count.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thread_pool/thread_pool_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thready_event_engine/thready_event_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/time_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/iocp.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/win_socket.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/windows_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/windows_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/windows_listener.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/work_queue/basic_work_queue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/experiments/config.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/experiments/experiments.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/load_file.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/per_cpu.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/ref_counted_string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/status_helper.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/time.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/time_averaged_stats.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/validation_errors.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/work_serializer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/handshaker/proxy_mapper_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/http/format_request.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/http/httpcli.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/http/httpcli_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/http/parser.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/buffer_list.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/call_combiner.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/cfstream_handle.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/closure.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/combiner.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/dualstack_socket_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/error.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/error_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_apple.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_epoll1_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_poll_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/event_engine_shims/closure.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/event_engine_shims/endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/event_engine_shims/tcp_client.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/exec_ctx.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/executor.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_fallback.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_host_name_max.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_sysconf.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/internal_errqueue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iocp_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_internal.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/load_file.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/lockfree_event.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/polling_entity.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/sockaddr_utils_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_factory_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_mutator.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_common_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/systemd_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_common.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_generic.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_heap.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_manager.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix_noop.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/vsock.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_eventfd.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_nospecial.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_pipe.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/json/json_object_loader.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/json/json_reader.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/json/json_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/json/json_writer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/load_balancing/lb_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/load_balancing/lb_policy_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/matchers/matchers.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/activity.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/party.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/sleep.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resolver/resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resolver/resolver_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resolver/server_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/api.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/arena.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/memory_quota.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/periodic_update.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/resource_quota.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/thread_quota.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/audit_logging.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/evaluate_args.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/grpc_authorization_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/grpc_server_authz_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/matchers.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/rbac_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/stdout_logger.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/certificate_provider/certificate_provider_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/context/security_context.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/alts_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/call_creds_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/channel_creds_registry_init.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/composite/composite_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/external/aws_external_account_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/external/aws_request_signer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/external/external_account_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/external/file_external_account_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/external/url_external_account_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/fake/fake_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/google_default/credentials_generic.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/google_default/google_default_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/iam/iam_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/insecure/insecure_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/json_token.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/jwt_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/jwt_verifier.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/local/local_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/plugin/plugin_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/ssl/ssl_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/grpc_tls_certificate_match.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/tls_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/tls_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/xds/xds_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/alts/alts_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/fake/fake_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/insecure/insecure_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_fallback.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_supported.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/local/local_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/tls/tls_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/client_auth_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/secure_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/security_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/server_auth_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/tsi_error.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/util/json_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/service_config/service_config_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/service_config/service_config_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/b64.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/percent_encoding.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice_buffer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice_refcount.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice_string_helpers.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/api_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/builtins.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer_reader.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call_details.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call_log_batch.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel_init.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel_ping.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel_stack_type.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/event_string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/init.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/init_internally.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/lame_client.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/metadata_array.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/server.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/validate_metadata.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/version.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/batch_builder.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/bdp_estimator.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/connectivity_state.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/error_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/handshaker_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/http_connect_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/metadata_batch.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/parsed_metadata.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/pid_controller.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/status_conversion.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/tcp_connect_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/timeout_encoding.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/transport.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/transport_op_string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/uri/uri_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry_extra.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/crypt/aes_gcm.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/crypt/gsec.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_counter.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_crypter.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_frame_protector.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_seal_privacy_integrity_crypter.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_unseal_privacy_integrity_crypter.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/frame_handler.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_handshaker_client.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_shared_resource.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_tsi_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/transport_security_common_api.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/fake_transport_security.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/local_transport_security.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/ssl/key_logging/ssl_key_logging.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_boringssl.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_cache.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_openssl.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/ssl_transport_security.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/ssl_transport_security_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/transport_security.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/transport_security_grpc.cc +) + +target_compile_features(grpc PUBLIC cxx_std_14) + +target_include_directories(grpc + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(grpc + ${_gRPC_ALLTARGETS_LIBRARIES} + ${_gRPC_RE2_LIBRARIES} + upb_json_lib + upb_textformat_lib + ${_gRPC_ZLIB_LIBRARIES} + absl::algorithm_container + absl::cleanup + absl::flat_hash_map + absl::flat_hash_set + absl::inlined_vector + absl::bind_front + absl::function_ref + absl::hash + absl::type_traits + absl::random_bit_gen_ref + absl::random_distributions + absl::statusor + absl::span + absl::utility + ${_gRPC_CARES_LIBRARIES} + gpr + ${_gRPC_SSL_LIBRARIES} + ${_gRPC_ADDRESS_SORTING_LIBRARIES} +) +if(_gRPC_PLATFORM_IOS OR _gRPC_PLATFORM_MAC) + target_link_libraries(grpc "-framework CoreFoundation") +endif() + +add_library(grpc_unsecure + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/backend_metrics/backend_metric_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/census/grpc_context.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/channel_idle/channel_idle_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/channel_idle/idle_filter_state.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backend_metric.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backup_poller.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/channel_connectivity.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_channelz.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_service_config.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/config_selector.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/dynamic_filters.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/global_subchannel_pool.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/http_proxy_mapper.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/address_filtering.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/endpoint_list.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/health_check_client.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/rls/rls.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/static_stride_scheduler.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/weighted_round_robin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/local_subchannel_pool.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/binder/binder_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/event_engine/event_engine_client_channel_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/event_engine/service_config_helper.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/polling_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_filter_legacy_call_data.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_service_config.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_throttle.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel_pool_interface.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel_stream_client.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/deadline/deadline_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/fault_injection/fault_injection_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/client/http_client_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/client_authority_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/http_filters_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/message_compress/compression_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/server/http_server_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/filters/message_size/message_size_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/chttp2_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/server/chttp2_server.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_decoder.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_encoder.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_transport.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/decode_huff.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/flow_control.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_data.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_goaway.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_ping.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_settings.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_window_update.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parse_result.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_settings.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/huffsyms.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/max_concurrent_streams_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/parsing.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_abuse_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_callbacks.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_rate_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_lists.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/varint.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/write_size_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/writing.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/inproc/inproc_plugin.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/inproc/inproc_transport.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/annotations.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/http.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/any.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/duration.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/empty.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/struct.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/timestamp.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/wrappers.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/rpc/status.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/altscontext.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/lb/v1/load_balancer.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/validate/validate.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/xds/service/orca/v3/orca.upb.c + ${_gRPC_SOURCE_DIR}/src/core/lib/address_utils/parse_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/address_utils/sockaddr_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/backoff/backoff.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/backoff/random_early_detection.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/call_tracer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_args.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_args_preconditioning.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack_builder.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack_builder_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channel_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channelz.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/channelz_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/connected_channel.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/promise_based_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/server_call_tracer_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/channel/status_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/compression/compression.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/compression/compression_internal.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/compression/message_compress.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/config/core_configuration.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/event_log.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/histogram_view.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/stats.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/stats_data.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/debug/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/ares_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/cf_engine/cf_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/cf_engine/cfstream_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/cf_engine/dns_service_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/channel_args_endpoint_config.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/default_event_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/default_event_engine_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/event_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/forkable.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/memory_allocator.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/ev_poll_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/internal_errqueue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/lockfree_event.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_engine_listener.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/timer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/timer_heap.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/timer_manager.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/traced_buffer_list.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/resolved_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/shim.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/slice.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/slice_buffer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/tcp_socket_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thread_pool/thread_count.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thread_pool/thread_pool_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thready_event_engine/thready_event_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/time_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/iocp.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/win_socket.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/windows_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/windows_engine.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/windows/windows_listener.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/work_queue/basic_work_queue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/experiments/config.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/experiments/experiments.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/load_file.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/per_cpu.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/ref_counted_string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/status_helper.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/time.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/time_averaged_stats.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/validation_errors.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/gprpp/work_serializer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/handshaker/proxy_mapper_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/http/format_request.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/http/httpcli.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/http/parser.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/buffer_list.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/call_combiner.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/cfstream_handle.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/closure.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/combiner.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/dualstack_socket_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/error.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/error_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_apple.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_epoll1_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_poll_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/event_engine_shims/closure.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/event_engine_shims/endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/event_engine_shims/tcp_client.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/exec_ctx.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/executor.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_fallback.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_host_name_max.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_sysconf.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/internal_errqueue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iocp_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_internal.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/load_file.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/lockfree_event.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/polling_entity.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/sockaddr_utils_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_factory_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_mutator.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_common_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/systemd_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_cfstream.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_common.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_generic.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_heap.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_manager.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix_noop.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/vsock.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_eventfd.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_nospecial.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_pipe.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/json/json_object_loader.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/json/json_reader.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/json/json_writer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/load_balancing/lb_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/load_balancing/lb_policy_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/activity.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/party.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/sleep.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resolver/resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resolver/resolver_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resolver/server_address.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/api.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/arena.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/memory_quota.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/periodic_update.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/resource_quota.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/thread_quota.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/evaluate_args.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/grpc_server_authz_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/certificate_provider/certificate_provider_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/context/security_context.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/call_creds_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/composite/composite_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/fake/fake_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/insecure/insecure_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/plugin/plugin_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/tls_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/fake/fake_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/insecure/insecure_security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_fallback.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_supported.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/security_connector/security_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/client_auth_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/secure_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/security_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/server_auth_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/transport/tsi_error.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/security/util/json_util.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/service_config/service_config_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/service_config/service_config_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/b64.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/percent_encoding.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice_buffer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice_refcount.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice_string_helpers.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/api_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/builtins.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer_reader.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call_details.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call_log_batch.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/call_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel_init.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel_ping.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/channel_stack_type.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/event_string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/init.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/init_internally.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/lame_client.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/metadata_array.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/server.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/validate_metadata.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/surface/version.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/batch_builder.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/bdp_estimator.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/connectivity_state.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/error_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/handshaker_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/http_connect_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/metadata_batch.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/parsed_metadata.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/pid_controller.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/status_conversion.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/tcp_connect_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/timeout_encoding.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/transport.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/transport_op_string.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/uri/uri_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry_noextra.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/transport_security_common_api.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/fake_transport_security.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/local_transport_security.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/transport_security.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/transport_security_grpc.cc + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/message/accessors.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/build_enum.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/decode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/internal/base92.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/link.c + ${gRPC_ADDITIONAL_DLL_SRC} +) + +target_compile_features(grpc_unsecure PUBLIC cxx_std_14) + +target_include_directories(grpc_unsecure + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(grpc_unsecure + ${_gRPC_ALLTARGETS_LIBRARIES} + upb_collections_lib + upb + ${_gRPC_ZLIB_LIBRARIES} + absl::algorithm_container + absl::cleanup + absl::flat_hash_map + absl::flat_hash_set + absl::inlined_vector + absl::bind_front + absl::function_ref + absl::hash + absl::type_traits + absl::random_bit_gen_ref + absl::random_distributions + absl::statusor + absl::span + absl::utility + ${_gRPC_CARES_LIBRARIES} + gpr + ${_gRPC_ADDRESS_SORTING_LIBRARIES} +) +if(_gRPC_PLATFORM_IOS OR _gRPC_PLATFORM_MAC) + target_link_libraries(grpc_unsecure "-framework CoreFoundation") +endif() + +add_library(upb + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/base/status.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/collections/array.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/collections/map.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/collections/map_sorter.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/hash/common.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/lex/atoi.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/lex/round_trip.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/lex/strtod.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/lex/unicode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mem/alloc.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mem/arena.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/message/message.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/extension_registry.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/internal/message.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/message.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/decode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/decode_fast.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/encode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/eps_copy_input_stream.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/reader.c +) + +target_compile_features(upb PUBLIC cxx_std_14) + +target_include_directories(upb + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(upb + ${_gRPC_ALLTARGETS_LIBRARIES} + utf8_range_lib +) + + +add_library(upb_collections_lib + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/base/status.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/collections/array.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/collections/map.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/collections/map_sorter.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/hash/common.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mem/alloc.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mem/arena.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/message/message.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/extension_registry.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/internal/message.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/message.c +) + +target_compile_features(upb_collections_lib PUBLIC cxx_std_14) + +target_include_directories(upb_collections_lib + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(upb_collections_lib + ${_gRPC_ALLTARGETS_LIBRARIES} +) + + + +add_library(upb_json_lib + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/json/decode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/json/encode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/message/accessors.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/build_enum.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/decode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/internal/base92.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/internal/encode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/link.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/def_builder.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/def_pool.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/def_type.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/desc_state.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/enum_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/enum_reserved_range.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/enum_value_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/extension_range.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/field_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/file_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/message.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/message_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/message_reserved_range.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/method_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/oneof_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/service_def.c +) + +target_compile_features(upb_json_lib PUBLIC cxx_std_14) + +target_include_directories(upb_json_lib + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(upb_json_lib + ${_gRPC_ALLTARGETS_LIBRARIES} + upb_collections_lib + upb +) + + +add_library(upb_textformat_lib + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/message/accessors.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/build_enum.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/decode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/internal/base92.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/internal/encode.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/link.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/def_builder.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/def_pool.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/def_type.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/desc_state.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/enum_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/enum_reserved_range.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/enum_value_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/extension_range.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/field_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/file_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/message.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/message_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/message_reserved_range.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/method_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/oneof_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/reflection/service_def.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/text/encode.c +) + +target_compile_features(upb_textformat_lib PUBLIC cxx_std_14) + +target_include_directories(upb_textformat_lib + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(upb_textformat_lib + ${_gRPC_ALLTARGETS_LIBRARIES} + upb_collections_lib + upb +) + + +add_library(utf8_range_lib + ${_gRPC_SOURCE_DIR}/third_party/utf8_range/naive.c + ${_gRPC_SOURCE_DIR}/third_party/utf8_range/range2-neon.c + ${_gRPC_SOURCE_DIR}/third_party/utf8_range/range2-sse.c +) + +target_compile_features(utf8_range_lib PUBLIC cxx_std_14) + +target_include_directories(utf8_range_lib + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(utf8_range_lib + ${_gRPC_ALLTARGETS_LIBRARIES} +) + + +add_library(grpc++ + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/client/binder_connector.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/client/channel_create.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/client/channel_create_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/client/connection_id_generator.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/client/endpoint_binder_pool.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/client/jni_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/client/security_policy_setting.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/security_policy/binder_security_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/server/binder_server.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/server/binder_server_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/transport/binder_transport.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/utils/ndk_binder.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/utils/transport_stream_receiver_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/wire_format/binder_android.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/wire_format/binder_constants.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/wire_format/transaction.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/wire_format/wire_reader_impl.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/binder/wire_format/wire_writer.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/channel_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_callback.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_context.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_interceptor.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_stats_interceptor.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/create_channel.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/create_channel_internal.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/create_channel_posix.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/insecure_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/secure_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/xds_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/alarm.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/auth_property_iterator.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/channel_arguments.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/channel_filter.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/completion_queue_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/resource_quota_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/rpc_method.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/secure_auth_context.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/secure_channel_arguments.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/secure_create_auth_context.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/tls_certificate_provider.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/tls_certificate_verifier.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/tls_credentials_options.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/validate_service_config.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/version_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/async_generic_service.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/backend_metric_recorder.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/channel_argument_option.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/create_default_thread_pool.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/external_connection_acceptor_impl.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/health/default_health_check_service.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/health/health_check_service.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/health/health_check_service_server_builder_option.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/insecure_server_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/secure_server_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_builder.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_callback.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_context.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_posix.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/xds_server_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/thread_manager/thread_manager.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/byte_buffer_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/status.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/string_ref.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/time_cc.cc + ${gRPC_UPB_GEN_DUPL_SRC} +) + +target_compile_features(grpc++ PUBLIC cxx_std_14) + +target_include_directories(grpc++ + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(grpc++ + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc + ${_gRPC_PROTOBUF_LIBRARIES} +) + +add_library(grpc++_unsecure + ${_gRPC_SOURCE_DIR}/src/cpp/client/channel_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_callback.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_context.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_interceptor.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/client_stats_interceptor.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/create_channel.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/create_channel_internal.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/create_channel_posix.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/insecure_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/alarm.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/channel_arguments.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/channel_filter.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/completion_queue_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/insecure_create_auth_context.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/resource_quota_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/rpc_method.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/validate_service_config.cc + ${_gRPC_SOURCE_DIR}/src/cpp/common/version_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/async_generic_service.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/backend_metric_recorder.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/channel_argument_option.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/create_default_thread_pool.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/external_connection_acceptor_impl.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/health/default_health_check_service.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/health/health_check_service.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/health/health_check_service_server_builder_option.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/insecure_server_credentials.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_builder.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_callback.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_context.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_posix.cc + ${_gRPC_SOURCE_DIR}/src/cpp/thread_manager/thread_manager.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/byte_buffer_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/status.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/string_ref.cc + ${_gRPC_SOURCE_DIR}/src/cpp/util/time_cc.cc + ${gRPC_UPB_GEN_DUPL_SRC} +) + +target_compile_features(grpc++_unsecure PUBLIC cxx_std_14) + +target_include_directories(grpc++_unsecure + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(grpc++_unsecure + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_unsecure + ${_gRPC_PROTOBUF_LIBRARIES} +) + +add_library(grpc_plugin_support + ${_gRPC_SOURCE_DIR}/src/compiler/cpp_generator.cc + ${_gRPC_SOURCE_DIR}/src/compiler/csharp_generator.cc + ${_gRPC_SOURCE_DIR}/src/compiler/node_generator.cc + ${_gRPC_SOURCE_DIR}/src/compiler/objective_c_generator.cc + ${_gRPC_SOURCE_DIR}/src/compiler/php_generator.cc + ${_gRPC_SOURCE_DIR}/src/compiler/proto_parser_helper.cc + ${_gRPC_SOURCE_DIR}/src/compiler/python_generator.cc + ${_gRPC_SOURCE_DIR}/src/compiler/ruby_generator.cc +) + +target_compile_features(grpc_plugin_support PUBLIC cxx_std_14) + +target_include_directories(grpc_plugin_support + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(grpc_plugin_support + ${_gRPC_ALLTARGETS_LIBRARIES} + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_PROTOBUF_PROTOC_LIBRARIES} +) + + +add_executable(grpc_cpp_plugin + ${_gRPC_SOURCE_DIR}/src/compiler/cpp_plugin.cc +) +target_compile_features(grpc_cpp_plugin PUBLIC cxx_std_14) +target_include_directories(grpc_cpp_plugin + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_SOURCE_DIR}/include + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) + +target_link_libraries(grpc_cpp_plugin + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_plugin_support +) From b117d8ed12f2f3839a03fc4882f5228ff7d010b5 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Sun, 19 Nov 2023 14:28:33 +0000 Subject: [PATCH 194/274] create interpreter with changed select_query_options --- src/Interpreters/InterpreterSelectQueryAnalyzer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp index abcc20310474..eed9d03ab5a6 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp @@ -209,7 +209,7 @@ Block InterpreterSelectQueryAnalyzer::getSampleBlock(const QueryTreeNodePtr & qu { auto select_query_options_copy = select_query_options; select_query_options_copy.only_analyze = true; - InterpreterSelectQueryAnalyzer interpreter(query_tree, context, select_query_options); + InterpreterSelectQueryAnalyzer interpreter(query_tree, context, select_query_options_copy); return interpreter.getSampleBlock(); } From 3213443ee2a53699f341b72916ca6011fe6f87cd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Nov 2023 15:42:35 +0100 Subject: [PATCH 195/274] Own CMake for GRPC --- cmake/limit_jobs.cmake | 4 ++-- contrib/grpc-cmake/grpc.cmake | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index a43e208ff0dd..9d693f655281 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -21,7 +21,7 @@ if (NOT PARALLEL_COMPILE_JOBS AND MAX_COMPILER_MEMORY) set (PARALLEL_COMPILE_JOBS 1) endif () if (PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES) - message(INFO "The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.") + message(INFORMATION "The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.") endif() endif () @@ -32,7 +32,7 @@ if (NOT PARALLEL_LINK_JOBS AND MAX_LINKER_MEMORY) set (PARALLEL_LINK_JOBS 1) endif () if (PARALLEL_LINK_JOBS LESS NUMBER_OF_LOGICAL_CORES) - message(INFO "The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.") + message(INFORMATION "The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.") endif() endif () diff --git a/contrib/grpc-cmake/grpc.cmake b/contrib/grpc-cmake/grpc.cmake index 43d4edd191e5..c24885392117 100644 --- a/contrib/grpc-cmake/grpc.cmake +++ b/contrib/grpc-cmake/grpc.cmake @@ -55,16 +55,6 @@ if(UNIX) endif() endif() -if(UNIX AND NOT HAIKU) - # -pthread does more than -lpthread - set(THREADS_PREFER_PTHREAD_FLAG ON) - find_package(Threads) - set(_gRPC_ALLTARGETS_LIBRARIES ${CMAKE_DL_LIBS} Threads::Threads) - if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) - set(_gRPC_ALLTARGETS_LIBRARIES ${_gRPC_ALLTARGETS_LIBRARIES} rt) - endif() -endif() - set(_gRPC_ADDRESS_SORTING_INCLUDE_DIR "${_gRPC_SOURCE_DIR}/third_party/address_sorting/include") set(_gRPC_ADDRESS_SORTING_LIBRARIES address_sorting) From 907f168e0dea4df5ac305fecf63c9c9a3820e433 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Nov 2023 15:43:09 +0100 Subject: [PATCH 196/274] Own CMake for GRPC --- cmake/limit_jobs.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index 9d693f655281..8e48fc9b9d83 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -21,7 +21,7 @@ if (NOT PARALLEL_COMPILE_JOBS AND MAX_COMPILER_MEMORY) set (PARALLEL_COMPILE_JOBS 1) endif () if (PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES) - message(INFORMATION "The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.") + message("The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.") endif() endif () @@ -32,7 +32,7 @@ if (NOT PARALLEL_LINK_JOBS AND MAX_LINKER_MEMORY) set (PARALLEL_LINK_JOBS 1) endif () if (PARALLEL_LINK_JOBS LESS NUMBER_OF_LOGICAL_CORES) - message(INFORMATION "The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.") + message("The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.") endif() endif () From cf4621444042c9dde066ddfe5b1dc50af04e2a8b Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Sun, 19 Nov 2023 15:09:52 +0000 Subject: [PATCH 197/274] Always send fatal level logs --- programs/client/Client.cpp | 1 - programs/local/LocalServer.cpp | 15 ++------------- src/Client/ClientBase.h | 2 -- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index d29824581fa2..8029f97a9920 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1241,7 +1241,6 @@ void Client::processConfig() global_context->setCurrentQueryId(query_id); } print_stack_trace = config().getBool("stacktrace", false); - logging_initialized = true; if (config().has("multiquery")) is_multiquery = true; diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index f3b551b08d25..9a3f948a01c4 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -563,9 +563,6 @@ catch (...) void LocalServer::updateLoggerLevel(const String & logs_level) { - if (!logging_initialized) - return; - config().setString("logger.level", logs_level); updateLevels(config(), logger()); } @@ -607,21 +604,13 @@ void LocalServer::processConfig() Poco::AutoPtr pf = new OwnPatternFormatter; Poco::AutoPtr log = new OwnFormattingChannel(pf, new Poco::SimpleFileChannel(server_logs_file)); Poco::Logger::root().setChannel(log); - logging_initialized = true; } - else if (logging || is_interactive) + else { config().setString("logger", "logger"); - auto log_level_default = is_interactive && !logging ? "fatal" : level; + auto log_level_default = logging ? level : "fatal"; config().setString("logger.level", config().getString("log-level", config().getString("send_logs_level", log_level_default))); buildLoggers(config(), logger(), "clickhouse-local"); - logging_initialized = true; - } - else - { - Poco::Logger::root().setLevel("none"); - Poco::Logger::root().setChannel(Poco::AutoPtr(new Poco::NullChannel())); - logging_initialized = false; } shared_context = Context::createShared(); diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index 2156aae71817..9fde23cf775e 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -321,8 +321,6 @@ class ClientBase : public Poco::Util::Application, public IHints<2> bool allow_merge_tree_settings = false; bool cancelled = false; - - bool logging_initialized = false; }; } From 60a17ee397bf98e07ece79bef06d6b025bc0dfe0 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 19 Nov 2023 15:27:59 +0000 Subject: [PATCH 198/274] Fix build --- src/Backups/BackupIO_S3.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 76b9fba7b836..ea3f57c27ffc 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -253,7 +253,6 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, { LOG_TRACE(log, "Copying file inside backup from {} to {} ", source, destination); copyS3File( - client, client, /* src_bucket */ s3_uri.bucket, /* src_key= */ fs::path(s3_uri.key) / source, From 4fc658cd1fe751e082ed4cf5bbda581c1b5a145a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Nov 2023 16:31:03 +0100 Subject: [PATCH 199/274] Fix build --- src/Backups/BackupIO_S3.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 76b9fba7b836..ea3f57c27ffc 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -253,7 +253,6 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, { LOG_TRACE(log, "Copying file inside backup from {} to {} ", source, destination); copyS3File( - client, client, /* src_bucket */ s3_uri.bucket, /* src_key= */ fs::path(s3_uri.key) / source, From 62a87665c551f4efa9ae5b98000604e64590e79e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 19 Nov 2023 16:31:03 +0100 Subject: [PATCH 200/274] Fix build --- src/Backups/BackupIO_S3.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 76b9fba7b836..ea3f57c27ffc 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -253,7 +253,6 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, { LOG_TRACE(log, "Copying file inside backup from {} to {} ", source, destination); copyS3File( - client, client, /* src_bucket */ s3_uri.bucket, /* src_key= */ fs::path(s3_uri.key) / source, From a3c9f13ac916ddecf74fba96d7c85cfe9c3f7468 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 19 Nov 2023 15:33:58 +0000 Subject: [PATCH 201/274] Add exclude for tryBase64Decode to backward compat test (follow-up to #56913) Fixes #56969 --- .../integration/test_backward_compatibility/test_functions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/test_backward_compatibility/test_functions.py b/tests/integration/test_backward_compatibility/test_functions.py index 94771a624e25..b6b6ef28de53 100644 --- a/tests/integration/test_backward_compatibility/test_functions.py +++ b/tests/integration/test_backward_compatibility/test_functions.py @@ -153,6 +153,9 @@ def test_string_functions(start_cluster): # mandatory or optional). The former lib produces a value based on implicit padding, the latter lib throws an error. "FROM_BASE64", "base64Decode", + # PR #56913 (in v23.11) corrected the way tryBase64Decode() behaved with invalid inputs. Old versions return garbage, new versions + # return an empty string (as it was always documented). + "tryBase64Decode", # Removed in 23.9 "meiliMatch", ] From 0513c93829d8c6d5f3226d98593b325d2d68f9df Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sun, 19 Nov 2023 23:11:05 +0100 Subject: [PATCH 202/274] Prefer sccache to ccache by default --- cmake/ccache.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/ccache.cmake b/cmake/ccache.cmake index e8bf856332a5..0df70d82d2c5 100644 --- a/cmake/ccache.cmake +++ b/cmake/ccache.cmake @@ -9,10 +9,10 @@ if (CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" OR CMAKE_C_COMPILER_LAUNCHER MA return() endif() -set(COMPILER_CACHE "auto" CACHE STRING "Speedup re-compilations using the caching tools; valid options are 'auto' (ccache, then sccache), 'ccache', 'sccache', or 'disabled'") +set(COMPILER_CACHE "auto" CACHE STRING "Speedup re-compilations using the caching tools; valid options are 'auto' (sccache, then ccache), 'ccache', 'sccache', or 'disabled'") if(COMPILER_CACHE STREQUAL "auto") - find_program (CCACHE_EXECUTABLE NAMES ccache sccache) + find_program (CCACHE_EXECUTABLE NAMES sccache ccache) elseif (COMPILER_CACHE STREQUAL "ccache") find_program (CCACHE_EXECUTABLE ccache) elseif(COMPILER_CACHE STREQUAL "sccache") @@ -21,7 +21,7 @@ elseif(COMPILER_CACHE STREQUAL "disabled") message(STATUS "Using *ccache: no (disabled via configuration)") return() else() - message(${RECONFIGURE_MESSAGE_LEVEL} "The COMPILER_CACHE must be one of (auto|ccache|sccache|disabled), value: '${COMPILER_CACHE}'") + message(${RECONFIGURE_MESSAGE_LEVEL} "The COMPILER_CACHE must be one of (auto|sccache|ccache|disabled), value: '${COMPILER_CACHE}'") endif() From 33df68cd0149a901812cfb0276a097c042ed800f Mon Sep 17 00:00:00 2001 From: Bharat Nallan Chakravarthy Date: Sun, 19 Nov 2023 14:48:21 -0800 Subject: [PATCH 203/274] update 02003_memory_limit_in_client.sh --- tests/queries/0_stateless/02003_memory_limit_in_client.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02003_memory_limit_in_client.sh b/tests/queries/0_stateless/02003_memory_limit_in_client.sh index 2d2493828c8f..4017c3771a61 100755 --- a/tests/queries/0_stateless/02003_memory_limit_in_client.sh +++ b/tests/queries/0_stateless/02003_memory_limit_in_client.sh @@ -1,4 +1,4 @@ -#!/usr/bin/bash -f +#!/usr/bin/env bash CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From d385217012723ba7f380a5caf1348f429e25b14e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Nov 2023 01:15:04 +0100 Subject: [PATCH 204/274] One step back --- base/glibc-compatibility/CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/base/glibc-compatibility/CMakeLists.txt b/base/glibc-compatibility/CMakeLists.txt index c967fa5b11ba..59f29093d5fe 100644 --- a/base/glibc-compatibility/CMakeLists.txt +++ b/base/glibc-compatibility/CMakeLists.txt @@ -35,6 +35,13 @@ if (GLIBC_COMPATIBILITY) target_link_libraries(global-libs INTERFACE glibc-compatibility ${MEMCPY_LIBRARY}) + # TODO: remove it + install( + TARGETS glibc-compatibility ${MEMCPY_LIBRARY} + EXPORT global + ARCHIVE DESTINATION lib + ) + message (STATUS "Some symbols from glibc will be replaced for compatibility") elseif (CLICKHOUSE_OFFICIAL_BUILD) From b205d4919d8ce88e7776bcc0e6ff3de16460ec9f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Nov 2023 01:24:28 +0100 Subject: [PATCH 205/274] Remove garbage --- CMakeLists.txt | 8 -------- base/glibc-compatibility/CMakeLists.txt | 7 ------- cmake/darwin/default_libs.cmake | 6 ------ cmake/freebsd/default_libs.cmake | 6 ------ cmake/linux/default_libs.cmake | 6 ------ contrib/llvm-project-cmake/CMakeLists.txt | 3 +++ 6 files changed, 3 insertions(+), 33 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4fe7a1e05e74..2486b970c59d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -460,14 +460,6 @@ endif () message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE}") -include (GNUInstallDirs) - -# When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc. - -if (TARGET global-group) - install (EXPORT global DESTINATION cmake) -endif () - add_subdirectory (contrib EXCLUDE_FROM_ALL) if (NOT ENABLE_JEMALLOC) diff --git a/base/glibc-compatibility/CMakeLists.txt b/base/glibc-compatibility/CMakeLists.txt index 59f29093d5fe..c967fa5b11ba 100644 --- a/base/glibc-compatibility/CMakeLists.txt +++ b/base/glibc-compatibility/CMakeLists.txt @@ -35,13 +35,6 @@ if (GLIBC_COMPATIBILITY) target_link_libraries(global-libs INTERFACE glibc-compatibility ${MEMCPY_LIBRARY}) - # TODO: remove it - install( - TARGETS glibc-compatibility ${MEMCPY_LIBRARY} - EXPORT global - ARCHIVE DESTINATION lib - ) - message (STATUS "Some symbols from glibc will be replaced for compatibility") elseif (CLICKHOUSE_OFFICIAL_BUILD) diff --git a/cmake/darwin/default_libs.cmake b/cmake/darwin/default_libs.cmake index 42b8473cb755..cf0210d9b455 100644 --- a/cmake/darwin/default_libs.cmake +++ b/cmake/darwin/default_libs.cmake @@ -22,9 +22,3 @@ link_libraries(global-group) target_link_libraries(global-group INTERFACE $ ) - -# FIXME: remove when all contribs will get custom cmake lists -install( - TARGETS global-group global-libs - EXPORT global -) diff --git a/cmake/freebsd/default_libs.cmake b/cmake/freebsd/default_libs.cmake index 65bf296ee09a..1eeb1a872bd2 100644 --- a/cmake/freebsd/default_libs.cmake +++ b/cmake/freebsd/default_libs.cmake @@ -25,9 +25,3 @@ link_libraries(global-group) target_link_libraries(global-group INTERFACE $ ) - -# FIXME: remove when all contribs will get custom cmake lists -install( - TARGETS global-group global-libs - EXPORT global -) diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake index 56a663a708eb..8552097fa571 100644 --- a/cmake/linux/default_libs.cmake +++ b/cmake/linux/default_libs.cmake @@ -50,9 +50,3 @@ target_link_libraries(global-group INTERFACE $ -Wl,--end-group ) - -# FIXME: remove when all contribs will get custom cmake lists -install( - TARGETS global-group global-libs - EXPORT global -) diff --git a/contrib/llvm-project-cmake/CMakeLists.txt b/contrib/llvm-project-cmake/CMakeLists.txt index d6133f145bcc..406bac73e90d 100644 --- a/contrib/llvm-project-cmake/CMakeLists.txt +++ b/contrib/llvm-project-cmake/CMakeLists.txt @@ -61,6 +61,9 @@ set (REQUIRED_LLVM_LIBRARIES LLVMDemangle ) +# Skip useless "install" instructions from CMake: +set (LLVM_INSTALL_TOOLCHAIN_ONLY 1 CACHE INTERNAL "") + if (ARCH_AMD64) set (LLVM_TARGETS_TO_BUILD "X86" CACHE INTERNAL "") list(APPEND REQUIRED_LLVM_LIBRARIES LLVMX86Info LLVMX86Desc LLVMX86CodeGen) From f6e4c29669f85b15ae6c84938d5e5cc8ca2628d1 Mon Sep 17 00:00:00 2001 From: Peignon Melvyn Date: Mon, 20 Nov 2023 01:49:17 +0100 Subject: [PATCH 206/274] MaterializedMysql doc Add experimental flag for materializedMysql --- docs/en/engines/database-engines/materialized-mysql.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/en/engines/database-engines/materialized-mysql.md b/docs/en/engines/database-engines/materialized-mysql.md index b7e567c7b6cd..f32698f84f60 100644 --- a/docs/en/engines/database-engines/materialized-mysql.md +++ b/docs/en/engines/database-engines/materialized-mysql.md @@ -7,7 +7,10 @@ sidebar_position: 70 # [experimental] MaterializedMySQL :::note -This is an experimental feature that should not be used in production. +This database engine is experimental. To use it, set `allow_experimental_database_materialized_mysql` to 1 in your configuration files or by using the `SET` command: +```sql +SET allow_experimental_database_materialized_mysql=1 +``` ::: Creates a ClickHouse database with all the tables existing in MySQL, and all the data in those tables. The ClickHouse server works as MySQL replica. It reads `binlog` and performs DDL and DML queries. From 6dc3efb3f9aef0bf65360a1b9a1413e9e8ead71b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Nov 2023 01:55:34 +0100 Subject: [PATCH 207/274] Fix build --- CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2486b970c59d..063cfc77302e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,8 +21,11 @@ include (cmake/clang_tidy.cmake) include (cmake/git.cmake) include (cmake/utils.cmake) +# This is needed to set up the CMAKE_INSTALL_BINDIR variable. +include (GNUInstallDirs) + # Ignore export() since we don't use it, -# but it gets broken with a global targets via link_libraries() +# but it gets broken with global targets via link_libraries() macro (export) endmacro () From ab5f3d12b63388f403ce7d1758076a47839a7b23 Mon Sep 17 00:00:00 2001 From: santrancisco Date: Mon, 20 Nov 2023 12:51:30 +1100 Subject: [PATCH 208/274] Fix sqlite file path validation to make sure it does not skip validation on relative path --- src/Databases/SQLite/SQLiteUtils.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Databases/SQLite/SQLiteUtils.cpp b/src/Databases/SQLite/SQLiteUtils.cpp index 152370050f1c..4fe7c6f1707a 100644 --- a/src/Databases/SQLite/SQLiteUtils.cpp +++ b/src/Databases/SQLite/SQLiteUtils.cpp @@ -26,10 +26,12 @@ void processSQLiteError(const String & message, bool throw_on_error) String validateSQLiteDatabasePath(const String & path, const String & user_files_path, bool need_check, bool throw_on_error) { + + String absolute_path = fs::absolute(path).lexically_normal(); + if (fs::path(path).is_relative()) - return fs::absolute(fs::path(user_files_path) / path).lexically_normal(); + absolute_path = fs::absolute(fs::path(user_files_path) / path).lexically_normal(); - String absolute_path = fs::absolute(path).lexically_normal(); String absolute_user_files_path = fs::absolute(user_files_path).lexically_normal(); if (need_check && !absolute_path.starts_with(absolute_user_files_path)) From f05f572b3c34bd437218f23fc1140e30cf7a4c62 Mon Sep 17 00:00:00 2001 From: santrancisco Date: Mon, 20 Nov 2023 13:37:27 +1100 Subject: [PATCH 209/274] Fixed trailing whitespace --- src/Databases/SQLite/SQLiteUtils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Databases/SQLite/SQLiteUtils.cpp b/src/Databases/SQLite/SQLiteUtils.cpp index 4fe7c6f1707a..ddc2fb911e99 100644 --- a/src/Databases/SQLite/SQLiteUtils.cpp +++ b/src/Databases/SQLite/SQLiteUtils.cpp @@ -28,7 +28,7 @@ String validateSQLiteDatabasePath(const String & path, const String & user_files { String absolute_path = fs::absolute(path).lexically_normal(); - + if (fs::path(path).is_relative()) absolute_path = fs::absolute(fs::path(user_files_path) / path).lexically_normal(); From 96c603ef97ce63a617637b0c1f45053ce0b5899c Mon Sep 17 00:00:00 2001 From: Chuan-Zheng Lee Date: Mon, 20 Nov 2023 17:17:27 +1300 Subject: [PATCH 210/274] [Docs] MaterializedPostgreSQL: Change DETACH to DETACH PERMANENTLY If I'm not mistaken, ClickHouse/ClickHouse#35158 changed the syntax for dynamically removing tables from MaterializedPostgreSQL databases from `DETACH` to `DETACH PERMANENTLY`. Currently when just running `DETACH TABLE postgres_database.table_to_remove`, it shows an error: ``` DETACH TABLE not allowed, use DETACH PERMANENTLY. (NOT_IMPLEMENTED) ``` This adds the keyword `PERMANENTLY` to both places where `DETACH` occurs on the MaterializedPostgreSQL database engine page. --- docs/en/engines/database-engines/materialized-postgresql.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/engines/database-engines/materialized-postgresql.md b/docs/en/engines/database-engines/materialized-postgresql.md index 4e978947e36b..3aa6dd01ea3e 100644 --- a/docs/en/engines/database-engines/materialized-postgresql.md +++ b/docs/en/engines/database-engines/materialized-postgresql.md @@ -8,7 +8,7 @@ sidebar_position: 60 Creates a ClickHouse database with tables from PostgreSQL database. Firstly, database with engine `MaterializedPostgreSQL` creates a snapshot of PostgreSQL database and loads required tables. Required tables can include any subset of tables from any subset of schemas from specified database. Along with the snapshot database engine acquires LSN and once initial dump of tables is performed - it starts pulling updates from WAL. After database is created, newly added tables to PostgreSQL database are not automatically added to replication. They have to be added manually with `ATTACH TABLE db.table` query. -Replication is implemented with PostgreSQL Logical Replication Protocol, which does not allow to replicate DDL, but allows to know whether replication breaking changes happened (column type changes, adding/removing columns). Such changes are detected and according tables stop receiving updates. In this case you should use `ATTACH`/ `DETACH` queries to reload table completely. If DDL does not break replication (for example, renaming a column) table will still receive updates (insertion is done by position). +Replication is implemented with PostgreSQL Logical Replication Protocol, which does not allow to replicate DDL, but allows to know whether replication breaking changes happened (column type changes, adding/removing columns). Such changes are detected and according tables stop receiving updates. In this case you should use `ATTACH`/ `DETACH PERMANENTLY` queries to reload table completely. If DDL does not break replication (for example, renaming a column) table will still receive updates (insertion is done by position). :::note This database engine is experimental. To use it, set `allow_experimental_database_materialized_postgresql` to 1 in your configuration files or by using the `SET` command: @@ -63,7 +63,7 @@ Before version 22.1, adding a table to replication left a non-removed temporary It is possible to remove specific tables from replication: ``` sql -DETACH TABLE postgres_database.table_to_remove; +DETACH TABLE postgres_database.table_to_remove PERMANENTLY; ``` ## PostgreSQL schema {#schema} From af0c954c9e14c23bad56670b5b4c626e32a08c57 Mon Sep 17 00:00:00 2001 From: Kevin Mingtarja Date: Mon, 20 Nov 2023 15:05:12 +0800 Subject: [PATCH 211/274] Add implicit constraint for CollapsingMergeTree sign column --- src/Storages/MergeTree/MergeTreeSettings.h | 1 + .../MergeTree/registerStorageMergeTree.cpp | 27 +++++++++++++--- ...onstraints_for_collapsing_engine.reference | 2 ++ ...olumn_constraints_for_collapsing_engine.sh | 31 +++++++++++++++++++ 4 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.reference create mode 100644 tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.sh diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 69307e74d1d9..df1e8a17e18f 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -84,6 +84,7 @@ struct Settings; M(UInt64, min_delay_to_insert_ms, 10, "Min delay of inserting data into MergeTree table in milliseconds, if there are a lot of unmerged parts in single partition.", 0) \ M(UInt64, max_parts_in_total, 100000, "If more than this number active parts in all partitions in total, throw 'Too many parts ...' exception.", 0) \ M(Bool, async_insert, false, "If true, data from INSERT query is stored in queue and later flushed to table in background.", 0) \ + M(Bool, add_implicit_sign_column_constraint_for_collapsing_engine, false, "If true, add implicit constraint for sign column for CollapsingMergeTree engine.", 0) \ \ /* Part removal settings. */ \ M(UInt64, simultaneous_parts_removal_limit, 0, "Maximum number of parts to remove during one CleanupThread iteration (0 means unlimited).", 0) \ diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index f22d86499c22..9285dfcdd91f 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -598,11 +598,6 @@ static StoragePtr create(const StorageFactory::Arguments & args) metadata.projections.add(std::move(projection)); } - auto constraints = metadata.constraints.getConstraints(); - if (args.query.columns_list && args.query.columns_list->constraints) - for (auto & constraint : args.query.columns_list->constraints->children) - constraints.push_back(constraint); - metadata.constraints = ConstraintsDescription(constraints); auto column_ttl_asts = columns.getColumnTTLs(); for (const auto & [name, ast] : column_ttl_asts) @@ -620,6 +615,28 @@ static StoragePtr create(const StorageFactory::Arguments & args) args.getLocalContext()->checkMergeTreeSettingsConstraints(initial_storage_settings, storage_settings->changes()); metadata.settings_changes = args.storage_def->settings->ptr(); } + + auto constraints = metadata.constraints.getConstraints(); + if (args.query.columns_list && args.query.columns_list->constraints) + for (auto & constraint : args.query.columns_list->constraints->children) + constraints.push_back(constraint); + if (merging_params.mode == MergeTreeData::MergingParams::Collapsing && storage_settings->add_implicit_sign_column_constraint_for_collapsing_engine) + { + auto sign_column_check_constraint = std::make_unique(); + sign_column_check_constraint->name = "check_sign_column"; + sign_column_check_constraint->type = ASTConstraintDeclaration::Type::CHECK; + + Array valid_values_array; + valid_values_array.emplace_back(-1); + valid_values_array.emplace_back(1); + + auto valid_values_ast = std::make_unique(std::move(valid_values_array)); + auto sign_column_ast = std::make_unique(merging_params.sign_column); + sign_column_check_constraint->set(sign_column_check_constraint->expr, makeASTFunction("in", std::move(sign_column_ast), std::move(valid_values_ast))); + + constraints.push_back(std::move(sign_column_check_constraint)); + } + metadata.constraints = ConstraintsDescription(constraints); } else { diff --git a/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.reference b/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.reference new file mode 100644 index 000000000000..5c6c001014d3 --- /dev/null +++ b/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.reference @@ -0,0 +1,2 @@ +1 2504 1 +ok diff --git a/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.sh b/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.sh new file mode 100644 index 000000000000..bee12afc5112 --- /dev/null +++ b/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +EXCEPTION_SUCCESS_TEXT=ok +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS collapsing_merge_tree;" + +# CollapsingSortedAlgorithm::merge() also has a check for sign column value +# optimize_on_insert = 0 is required to avoid this automatic merge behavior +$CLICKHOUSE_CLIENT --query="SET optimize_on_insert = 0;" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE collapsing_merge_tree +( + Key UInt32, + Count UInt16, + Sign Int8 +) +ENGINE=CollapsingMergeTree(Sign) ORDER BY Key +SETTINGS add_implicit_sign_column_constraint_for_collapsing_engine=1;" + +# Should succeed +$CLICKHOUSE_CLIENT --query="INSERT INTO collapsing_merge_tree VALUES (1, 2504, 1);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM collapsing_merge_tree;" + +# Should throw an exception +$CLICKHOUSE_CLIENT --query="INSERT INTO collapsing_merge_tree VALUES (1, 2504, 5);" 2>&1 \ + | grep -q VIOLATED_CONSTRAINT && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not throw an exception" + +$CLICKHOUSE_CLIENT --query="DROP TABLE collapsing_merge_tree;" From f9a8df4296b9c4cc657cb14d6cd2cfaa84d1a6ba Mon Sep 17 00:00:00 2001 From: Aleksandr Musorin Date: Thu, 16 Nov 2023 13:57:58 +0100 Subject: [PATCH 212/274] Added comment to prevent using --remote to update submodules --- contrib/update-submodules.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/update-submodules.sh b/contrib/update-submodules.sh index b612d25352b3..6d187f124a4e 100755 --- a/contrib/update-submodules.sh +++ b/contrib/update-submodules.sh @@ -9,4 +9,8 @@ cd $GIT_DIR contrib/sparse-checkout/setup-sparse-checkout.sh git submodule init git submodule sync -git config --file .gitmodules --get-regexp .*path | sed 's/[^ ]* //' | xargs -I _ --max-procs 64 git submodule update --depth=1 --single-branch _ +# NOTE: do not use --remote for `git submodule update`[1] command, since the submodule references to the specific commit SHA1 in the subproject. +# It may cause unexpected behavior. Instead you need to commit a new SHA1 for a submodule. +# +# [1] - https://git-scm.com/book/en/v2/Git-Tools-Submodules +git config --file .gitmodules --get-regexp '.*path' | sed 's/[^ ]* //' | xargs -I _ --max-procs 64 git submodule update --depth=1 --single-branch _ From ebb66c1a9e33ccea792ee4ff64519767ea82cd67 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Mon, 20 Nov 2023 12:13:24 +0100 Subject: [PATCH 213/274] add comments --- src/IO/Lz4DeflatingWriteBuffer.cpp | 3 +++ src/IO/WriteBuffer.h | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/IO/Lz4DeflatingWriteBuffer.cpp b/src/IO/Lz4DeflatingWriteBuffer.cpp index 0af205a426dd..1f937ac545dd 100644 --- a/src/IO/Lz4DeflatingWriteBuffer.cpp +++ b/src/IO/Lz4DeflatingWriteBuffer.cpp @@ -6,6 +6,9 @@ namespace { using namespace DB; + /// SinkToOut provides the safe way to do direct write into buffer's memory + /// When out->capacity() is not less that guaranteed_capacity, SinkToOut is pointing directly to out_'s memory. + /// Otherwise the writes are directed to the temporary memory. That data is copied to out_ at finalize call. class SinkToOut { public: diff --git a/src/IO/WriteBuffer.h b/src/IO/WriteBuffer.h index d29ca6d5c6c3..67dbb9b2e7af 100644 --- a/src/IO/WriteBuffer.h +++ b/src/IO/WriteBuffer.h @@ -34,7 +34,12 @@ class WriteBuffer : public BufferBase void set(Position ptr, size_t size) { BufferBase::set(ptr, size, 0); } /** write the data in the buffer (from the beginning of the buffer to the current position); - * set the position to the beginning; throw an exception, if something is wrong + * set the position to the beginning; throw an exception, if something is wrong. + * + * Next call doesn't guarantee that buffer capacity is regained after. + * Some buffers (i.g WriteBufferFromS3) flush its data only after certain amount of consumed data. + * If direct write is performed into [position(), buffer().end()) and its length is not enough, + * you need to fill it first (i.g with write call), after it the capacity is regained. */ inline void next() { From 088022df5ddad0347d29d31062a861af0c4b9d88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 20 Nov 2023 12:20:26 +0100 Subject: [PATCH 214/274] Add test --- .../0_stateless/02918_sqlite_path_check.reference | 2 ++ .../queries/0_stateless/02918_sqlite_path_check.sh | 13 +++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 tests/queries/0_stateless/02918_sqlite_path_check.reference create mode 100755 tests/queries/0_stateless/02918_sqlite_path_check.sh diff --git a/tests/queries/0_stateless/02918_sqlite_path_check.reference b/tests/queries/0_stateless/02918_sqlite_path_check.reference new file mode 100644 index 000000000000..56b832a64697 --- /dev/null +++ b/tests/queries/0_stateless/02918_sqlite_path_check.reference @@ -0,0 +1,2 @@ +SQLite database file path '/etc/passwd' must be inside 'user_files' directory. (PATH_ACCESS_DENIED) +SQLite database file path '../../../../etc/passwd' must be inside 'user_files' directory. (PATH_ACCESS_DENIED) diff --git a/tests/queries/0_stateless/02918_sqlite_path_check.sh b/tests/queries/0_stateless/02918_sqlite_path_check.sh new file mode 100755 index 000000000000..1f250387a71d --- /dev/null +++ b/tests/queries/0_stateless/02918_sqlite_path_check.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +function get_exception_message() +{ + $CLICKHOUSE_CLIENT --query "$1" |& head -n1 | sed 's/.*DB::Exception: \(.*\) (version.*/\1/g' +} + +get_exception_message "Select * from sqlite('/etc/passwd', 'something');" +get_exception_message "Select * from sqlite('../../../../etc/passwd', 'something'); From 44874859bb0a27635a590617baa8fc3d957b8dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 20 Nov 2023 12:21:12 +0100 Subject: [PATCH 215/274] Fix style --- src/Databases/SQLite/SQLiteUtils.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Databases/SQLite/SQLiteUtils.cpp b/src/Databases/SQLite/SQLiteUtils.cpp index ddc2fb911e99..19b8662707b1 100644 --- a/src/Databases/SQLite/SQLiteUtils.cpp +++ b/src/Databases/SQLite/SQLiteUtils.cpp @@ -26,7 +26,6 @@ void processSQLiteError(const String & message, bool throw_on_error) String validateSQLiteDatabasePath(const String & path, const String & user_files_path, bool need_check, bool throw_on_error) { - String absolute_path = fs::absolute(path).lexically_normal(); if (fs::path(path).is_relative()) From 10fb40ece964947fc7e8d32641d8caaa66cd2a81 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 20 Nov 2023 12:34:16 +0100 Subject: [PATCH 216/274] Fix --- src/Storages/S3Queue/S3QueueMetadataFactory.cpp | 1 + src/Storages/S3Queue/StorageS3Queue.cpp | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Storages/S3Queue/S3QueueMetadataFactory.cpp b/src/Storages/S3Queue/S3QueueMetadataFactory.cpp index bd01bd52425b..92cdab6355de 100644 --- a/src/Storages/S3Queue/S3QueueMetadataFactory.cpp +++ b/src/Storages/S3Queue/S3QueueMetadataFactory.cpp @@ -43,6 +43,7 @@ void S3QueueMetadataFactory::remove(const std::string & zookeeper_path) if (it == metadata_by_path.end()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Metadata with zookeeper path {} does not exist", zookeeper_path); + chassert(it->second.ref_count > 0); if (--it->second.ref_count == 0) { try diff --git a/src/Storages/S3Queue/StorageS3Queue.cpp b/src/Storages/S3Queue/StorageS3Queue.cpp index 72e74d3c2a02..d2cf074e6a1c 100644 --- a/src/Storages/S3Queue/StorageS3Queue.cpp +++ b/src/Storages/S3Queue/StorageS3Queue.cpp @@ -112,7 +112,6 @@ StorageS3Queue::StorageS3Queue( , s3queue_settings(std::move(s3queue_settings_)) , zk_path(chooseZooKeeperPath(table_id_, context_->getSettingsRef(), *s3queue_settings)) , after_processing(s3queue_settings->after_processing) - , files_metadata(S3QueueMetadataFactory::instance().getOrCreate(zk_path, *s3queue_settings)) , configuration{configuration_} , format_settings(format_settings_) , reschedule_processing_interval_ms(s3queue_settings->s3queue_polling_min_timeout_ms) @@ -157,6 +156,13 @@ StorageS3Queue::StorageS3Queue( void StorageS3Queue::startup() { + if (!files_metadata) + { + /// Get metadata manager from S3QueueMetadataFactory, + /// it will increase the ref count for the metadata object. + /// The ref count is decreased when StorageS3Queue::drop() method is called. + files_metadata = S3QueueMetadataFactory::instance().getOrCreate(zk_path, *s3queue_settings); + } if (task) task->activateAndSchedule(); } From 1ade4b797b1e18d82b52a113c74dad9b4e203487 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 20 Nov 2023 12:48:49 +0100 Subject: [PATCH 217/274] Add a comment --- src/Storages/S3Queue/S3QueueMetadataFactory.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/S3Queue/S3QueueMetadataFactory.h b/src/Storages/S3Queue/S3QueueMetadataFactory.h index 55c2dfad5dd5..c5e94d59050b 100644 --- a/src/Storages/S3Queue/S3QueueMetadataFactory.h +++ b/src/Storages/S3Queue/S3QueueMetadataFactory.h @@ -25,6 +25,7 @@ class S3QueueMetadataFactory final : private boost::noncopyable explicit Metadata(std::shared_ptr metadata_) : metadata(metadata_), ref_count(1) {} std::shared_ptr metadata; + /// TODO: the ref count should be kept in keeper, because of the case with distributed processing. size_t ref_count = 0; }; using MetadataByPath = std::unordered_map; From c3a3cf8d24d5ca1205c2fe9cb2d8d266ea276b00 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 20 Nov 2023 12:57:10 +0100 Subject: [PATCH 218/274] Make check for the limited cmake dependencies the part of sparse checkout --- contrib/update-submodules.sh | 8 ++++++++ docker/packager/binary/build.sh | 9 --------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/contrib/update-submodules.sh b/contrib/update-submodules.sh index 6d187f124a4e..b12f3f924dc6 100755 --- a/contrib/update-submodules.sh +++ b/contrib/update-submodules.sh @@ -14,3 +14,11 @@ git submodule sync # # [1] - https://git-scm.com/book/en/v2/Git-Tools-Submodules git config --file .gitmodules --get-regexp '.*path' | sed 's/[^ ]* //' | xargs -I _ --max-procs 64 git submodule update --depth=1 --single-branch _ + +# We don't want to depend on any third-party CMake files. +# To check it, find and delete them. +grep -o -P '"contrib/[^"]+"' .gitmodules | + grep -v -P 'contrib/(llvm-project|google-protobuf|grpc|abseil-cpp|corrosion)' | + xargs -I@ find @ \ + -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' \ + -delete diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index f943011df9df..fd9bfcaabb2e 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -34,15 +34,6 @@ cd /build/build_docker rm -f CMakeCache.txt -# We don't want to depend on any third-party CMake files. -# To check it, find and delete them. - -grep -o -P '"contrib/[^"]+"' ../.gitmodules | - grep -v -P 'llvm-project|google-protobuf|grpc|abseil-cpp|corrosion' | - xargs -I@ find ../@ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' | - xargs rm - - if [ -n "$MAKE_DEB" ]; then rm -rf /build/packages/root # NOTE: this is for backward compatibility with previous releases, From 28bb76d56861750a12a223a1da3401bc159fde02 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 20 Nov 2023 13:21:46 +0100 Subject: [PATCH 219/274] address review comments --- .../Passes/LogicalExpressionOptimizerPass.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index e667b6030206..dfaccbc5cdbf 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -59,9 +59,11 @@ class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWi static bool tryOptimizeIsNotDistinctOrIsNull(QueryTreeNodePtr & node, const ContextPtr & context) { auto & function_node = node->as(); - assert(function_node.getFunctionName() == "or"); + chassert(function_node.getFunctionName() == "or"); + QueryTreeNodes or_operands; + or_operands.reserve(function_node.getArguments()->getNodes().size()); /// Indices of `equals` or `isNotDistinctFrom` functions in the vector above std::vector equals_functions_indices; @@ -88,9 +90,10 @@ class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWi const auto & func_name = argument_function->getFunctionName(); if (func_name == "equals" || func_name == "isNotDistinctFrom") + { equals_functions_indices.push_back(or_operands.size() - 1); - - if (func_name == "and") + } + else if (func_name == "and") { for (const auto & and_argument : argument_function->getArguments().getNodes()) { @@ -169,7 +172,7 @@ class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWi if (function->getFunctionName() == "equals") { /// We should replace `a = b` with `a <=> b` because we removed checks for IS NULL - need_reresolve = need_reresolve || function->getResultType()->isNullable(); + need_reresolve |= function->getResultType()->isNullable(); function->resolveAsFunction(strict_equals_function_resolver); new_or_operands.emplace_back(std::move(or_operands[i])); } From 779a8971e05cfbc0dc43aecb8e25e98a1f8c94c6 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 20 Nov 2023 13:33:39 +0100 Subject: [PATCH 220/274] Disable settings randomisation for `02896_memory_accounting_for_user.sh` (#56709) --- tests/queries/0_stateless/02896_memory_accounting_for_user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02896_memory_accounting_for_user.sh b/tests/queries/0_stateless/02896_memory_accounting_for_user.sh index 72f4be1475df..f3016671420b 100755 --- a/tests/queries/0_stateless/02896_memory_accounting_for_user.sh +++ b/tests/queries/0_stateless/02896_memory_accounting_for_user.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-parallel, long +# Tags: no-parallel, long, no-random-settings CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From aa07403f2773d4abc50beb9f2674867f16a8b5ea Mon Sep 17 00:00:00 2001 From: Kevin Mingtarja Date: Mon, 20 Nov 2023 12:37:23 +0000 Subject: [PATCH 221/274] Make test 02918 executable --- ...2918_implicit_sign_column_constraints_for_collapsing_engine.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.sh diff --git a/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.sh b/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.sh old mode 100644 new mode 100755 From 06c6282eb74d9b9292548f698f5c73b0362bb339 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 20 Nov 2023 13:46:37 +0100 Subject: [PATCH 222/274] fix build --- src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index dfaccbc5cdbf..081a27eb8fad 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -63,7 +63,7 @@ class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWi QueryTreeNodes or_operands; - or_operands.reserve(function_node.getArguments()->getNodes().size()); + or_operands.reserve(function_node.getArguments().getNodes().size()); /// Indices of `equals` or `isNotDistinctFrom` functions in the vector above std::vector equals_functions_indices; From fafd169e7b313c5e0a5046afe0183778e828edeb Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:12:52 +0100 Subject: [PATCH 223/274] Update src/IO/Lz4DeflatingWriteBuffer.cpp Co-authored-by: Antonio Andelic --- src/IO/Lz4DeflatingWriteBuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/Lz4DeflatingWriteBuffer.cpp b/src/IO/Lz4DeflatingWriteBuffer.cpp index 1f937ac545dd..8241bfd4f3cb 100644 --- a/src/IO/Lz4DeflatingWriteBuffer.cpp +++ b/src/IO/Lz4DeflatingWriteBuffer.cpp @@ -7,7 +7,7 @@ namespace using namespace DB; /// SinkToOut provides the safe way to do direct write into buffer's memory - /// When out->capacity() is not less that guaranteed_capacity, SinkToOut is pointing directly to out_'s memory. + /// When out->capacity() is not less than guaranteed_capacity, SinkToOut is pointing directly to out_'s memory. /// Otherwise the writes are directed to the temporary memory. That data is copied to out_ at finalize call. class SinkToOut { From 5031f239c3ecb92d6da853ae2e14b82eadc0ba43 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 20 Nov 2023 14:28:59 +0100 Subject: [PATCH 224/274] Revert "s3 adaptive timeouts" --- base/poco/Net/src/HTTPServerSession.cpp | 1 + base/poco/Net/src/HTTPSession.cpp | 31 +---- docs/en/operations/settings/settings.md | 7 - src/Backups/BackupIO_S3.cpp | 9 +- src/Coordination/KeeperSnapshotManagerS3.cpp | 1 + src/Core/Settings.h | 3 +- .../ObjectStorages/S3/S3ObjectStorage.cpp | 58 +++++--- src/Disks/ObjectStorages/S3/S3ObjectStorage.h | 15 +- src/Disks/ObjectStorages/S3/diskSettings.cpp | 8 +- src/IO/ConnectionTimeouts.cpp | 82 ----------- src/IO/ConnectionTimeouts.h | 2 - src/IO/HTTPCommon.cpp | 12 +- src/IO/HTTPCommon.h | 2 - src/IO/ReadBufferFromS3.cpp | 24 ++-- src/IO/ReadBufferFromS3.h | 4 +- src/IO/S3/Client.cpp | 12 +- src/IO/S3/Client.h | 10 +- src/IO/S3/PocoHTTPClient.cpp | 111 ++++----------- src/IO/S3/PocoHTTPClient.h | 5 - src/IO/S3/copyS3File.cpp | 26 ++-- src/IO/S3/copyS3File.h | 7 + src/IO/S3/tests/gtest_aws_s3_client.cpp | 3 +- src/IO/WriteBufferFromS3.cpp | 4 +- src/IO/WriteBufferFromS3.h | 3 + src/IO/tests/gtest_writebuffer_s3.cpp | 1 + src/Storages/StorageS3.cpp | 3 + src/Storages/StorageS3.h | 1 + src/Storages/StorageS3Settings.h | 3 +- .../configs/inf_s3_retries.xml | 1 - .../configs/s3_retries.xml | 1 - .../configs/storage_conf.xml | 16 +-- .../test_checking_s3_blobs_paranoid/test.py | 129 ++++-------------- .../configs/config.d/storage_conf.xml | 2 - .../test_storage_s3/configs/defaultS3.xml | 5 + .../test_storage_s3/configs/s3_retry.xml | 4 +- .../s3_mocks/unstable_server.py | 17 +-- tests/integration/test_storage_s3/test.py | 9 -- 37 files changed, 202 insertions(+), 430 deletions(-) diff --git a/base/poco/Net/src/HTTPServerSession.cpp b/base/poco/Net/src/HTTPServerSession.cpp index d4f2b24879e4..f6d3c4e5b923 100644 --- a/base/poco/Net/src/HTTPServerSession.cpp +++ b/base/poco/Net/src/HTTPServerSession.cpp @@ -26,6 +26,7 @@ HTTPServerSession::HTTPServerSession(const StreamSocket& socket, HTTPServerParam _maxKeepAliveRequests(pParams->getMaxKeepAliveRequests()) { setTimeout(pParams->getTimeout()); + this->socket().setReceiveTimeout(pParams->getTimeout()); } diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index 8f951b3102cd..d2663baaf9fe 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -93,34 +93,9 @@ void HTTPSession::setTimeout(const Poco::Timespan& timeout) void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco::Timespan& sendTimeout, const Poco::Timespan& receiveTimeout) { - try - { - _connectionTimeout = connectionTimeout; - - if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) { - _sendTimeout = sendTimeout; - - if (connected()) - _socket.setSendTimeout(_sendTimeout); - } - - if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) { - _receiveTimeout = receiveTimeout; - - if (connected()) - _socket.setReceiveTimeout(_receiveTimeout); - } - } - catch (NetException &) - { -#ifndef NDEBUG - throw; -#else - // mute exceptions in release - // just in case when changing settings on socket is not allowed - // however it should be OK for timeouts -#endif - } + _connectionTimeout = connectionTimeout; + _sendTimeout = sendTimeout; + _receiveTimeout = receiveTimeout; } diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index edc1c9bdfd77..e61934d21686 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -4826,10 +4826,3 @@ When set to `true` the metadata files are written with `VERSION_FULL_OBJECT_KEY` When set to `false` the metadata files are written with the previous format version, `VERSION_INLINE_DATA`. With that format only suffixes of object storage key names are are written to the metadata files. The prefix for all of object storage key names is set in configurations files at `storage_configuration.disks` section. Default value: `false`. - -## s3_use_adaptive_timeouts {#s3_use_adaptive_timeouts} - -When set to `true` than for all s3 requests first two attempts are made with low send and receive timeouts. -When set to `false` than all attempts are made with identical timeouts. - -Default value: `true`. diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index ea3f57c27ffc..eb9dcf6b45af 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -55,9 +55,7 @@ namespace static_cast(context->getGlobalContext()->getSettingsRef().s3_max_redirects), static_cast(context->getGlobalContext()->getSettingsRef().s3_retry_attempts), context->getGlobalContext()->getSettingsRef().enable_s3_requests_logging, - /* for_disk_s3 = */ false, - request_settings.get_request_throttler, - request_settings.put_request_throttler, + /* for_disk_s3 = */ false, request_settings.get_request_throttler, request_settings.put_request_throttler, s3_uri.uri.getScheme()); client_configuration.endpointOverride = s3_uri.endpoint; @@ -169,6 +167,7 @@ void BackupReaderS3::copyFileToDisk(const String & path_in_backup, size_t file_s blob_path.size(), mode); copyS3File( + client, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, @@ -230,6 +229,7 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src { LOG_TRACE(log, "Copying file {} from disk {} to S3", src_path, src_disk->getName()); copyS3File( + client, client, /* src_bucket */ blob_path[1], /* src_key= */ blob_path[0], @@ -268,7 +268,7 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, void BackupWriterS3::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) { - copyDataToS3File(create_read_buffer, start_pos, length, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, s3_settings.request_settings, {}, + copyDataToS3File(create_read_buffer, start_pos, length, client, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, s3_settings.request_settings, {}, threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterS3")); } @@ -298,6 +298,7 @@ std::unique_ptr BackupWriterS3::writeFile(const String & file_name) { return std::make_unique( client, + client, // already has long timeout s3_uri.bucket, fs::path(s3_uri.key) / file_name, DBMS_DEFAULT_BUFFER_SIZE, diff --git a/src/Coordination/KeeperSnapshotManagerS3.cpp b/src/Coordination/KeeperSnapshotManagerS3.cpp index bedde0d7b39c..302e05c84184 100644 --- a/src/Coordination/KeeperSnapshotManagerS3.cpp +++ b/src/Coordination/KeeperSnapshotManagerS3.cpp @@ -148,6 +148,7 @@ void KeeperSnapshotManagerS3::uploadSnapshotImpl(const SnapshotFileInfo & snapsh const auto create_writer = [&](const auto & key) { return WriteBufferFromS3( + s3_client->client, s3_client->client, s3_client->uri.bucket, key, diff --git a/src/Core/Settings.h b/src/Core/Settings.h index bb5e43224852..ac4c6b6c17fc 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -94,7 +94,6 @@ class IColumn; M(UInt64, s3_max_put_rps, 0, "Limit on S3 PUT request per second rate before throttling. Zero means unlimited.", 0) \ M(UInt64, s3_max_put_burst, 0, "Max number of requests that can be issued simultaneously before hitting request per second limit. By default (0) equals to `s3_max_put_rps`", 0) \ M(UInt64, s3_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ - M(Bool, s3_use_adaptive_timeouts, true, "When adaptive timeouts are enabled first two attempts are made with low receive and send timeout", 0) \ M(UInt64, azure_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ M(Bool, s3_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables.", 0) \ M(Bool, azure_truncate_on_insert, false, "Enables or disables truncate before insert in azure engine tables.", 0) \ @@ -105,7 +104,7 @@ class IColumn; M(Bool, s3_allow_parallel_part_upload, true, "Use multiple threads for s3 multipart upload. It may lead to slightly higher memory usage", 0) \ M(Bool, s3_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \ M(UInt64, s3_retry_attempts, 100, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries", 0) \ - M(UInt64, s3_request_timeout_ms, 30000, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \ + M(UInt64, s3_request_timeout_ms, 3000, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \ M(UInt64, s3_http_connection_pool_size, 1000, "How many reusable open connections to keep per S3 endpoint. Only applies to the S3 table engine and table function, not to S3 disks (for disks, use disk config instead). Global setting, can only be set in config, overriding it per session or per query has no effect.", 0) \ M(Bool, enable_s3_requests_logging, false, "Enable very explicit logging of S3 requests. Makes sense for debug only.", 0) \ M(String, s3queue_default_zookeeper_path, "/clickhouse/s3queue/", "Default zookeeper path prefix for S3Queue engine", 0) \ diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 308db389ee13..3af316bf0cf8 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -155,7 +155,7 @@ class S3IteratorAsync final : public IObjectStorageIteratorAsync bool S3ObjectStorage::exists(const StoredObject & object) const { auto settings_ptr = s3_settings.get(); - return S3::objectExists(*client.get(), bucket, object.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + return S3::objectExists(*clients.get()->client, bucket, object.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); } std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT @@ -174,7 +174,7 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT (const std::string & path, size_t read_until_position) -> std::unique_ptr { return std::make_unique( - client.get(), + clients.get()->client, bucket, path, version_id, @@ -224,7 +224,7 @@ std::unique_ptr S3ObjectStorage::readObject( /// NOLINT { auto settings_ptr = s3_settings.get(); return std::make_unique( - client.get(), + clients.get()->client, bucket, object.remote_path, version_id, @@ -249,8 +249,10 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN if (write_settings.s3_allow_parallel_part_upload) scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "VFSWrite"); + auto clients_ = clients.get(); return std::make_unique( - client.get(), + clients_->client, + clients_->client_with_long_timeout, bucket, object.remote_path, buf_size, @@ -264,12 +266,15 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN ObjectStorageIteratorPtr S3ObjectStorage::iterate(const std::string & path_prefix) const { auto settings_ptr = s3_settings.get(); - return std::make_shared(bucket, path_prefix, client.get(), settings_ptr->list_object_keys_size); + auto client_ptr = clients.get()->client; + + return std::make_shared(bucket, path_prefix, client_ptr, settings_ptr->list_object_keys_size); } void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const { auto settings_ptr = s3_settings.get(); + auto client_ptr = clients.get()->client; S3::ListObjectsV2Request request; request.SetBucket(bucket); @@ -284,7 +289,7 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet { ProfileEvents::increment(ProfileEvents::S3ListObjects); ProfileEvents::increment(ProfileEvents::DiskS3ListObjects); - outcome = client.get()->ListObjectsV2(request); + outcome = client_ptr->ListObjectsV2(request); throwIfError(outcome); auto result = outcome.GetResult(); @@ -315,12 +320,14 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet void S3ObjectStorage::removeObjectImpl(const StoredObject & object, bool if_exists) { + auto client_ptr = clients.get()->client; + ProfileEvents::increment(ProfileEvents::S3DeleteObjects); ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); S3::DeleteObjectRequest request; request.SetBucket(bucket); request.SetKey(object.remote_path); - auto outcome = client.get()->DeleteObject(request); + auto outcome = client_ptr->DeleteObject(request); throwIfUnexpectedError(outcome, if_exists); @@ -339,6 +346,7 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e } else { + auto client_ptr = clients.get()->client; auto settings_ptr = s3_settings.get(); size_t chunk_size_limit = settings_ptr->objects_chunk_size_to_delete; @@ -367,7 +375,7 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e S3::DeleteObjectsRequest request; request.SetBucket(bucket); request.SetDelete(delkeys); - auto outcome = client.get()->DeleteObjects(request); + auto outcome = client_ptr->DeleteObjects(request); throwIfUnexpectedError(outcome, if_exists); @@ -399,7 +407,7 @@ void S3ObjectStorage::removeObjectsIfExist(const StoredObjects & objects) std::optional S3ObjectStorage::tryGetObjectMetadata(const std::string & path) const { auto settings_ptr = s3_settings.get(); - auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true, /* throw_on_error= */ false); + auto object_info = S3::getObjectInfo(*clients.get()->client, bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true, /* throw_on_error= */ false); if (object_info.size == 0 && object_info.last_modification_time == 0 && object_info.metadata.empty()) return {}; @@ -415,7 +423,7 @@ std::optional S3ObjectStorage::tryGetObjectMetadata(const std::s ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path) const { auto settings_ptr = s3_settings.get(); - auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true); + auto object_info = S3::getObjectInfo(*clients.get()->client, bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true); ObjectMetadata result; result.size_bytes = object_info.size; @@ -436,12 +444,12 @@ void S3ObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT /// Shortcut for S3 if (auto * dest_s3 = dynamic_cast(&object_storage_to); dest_s3 != nullptr) { - auto client_ = client.get(); + auto clients_ = clients.get(); auto settings_ptr = s3_settings.get(); - auto size = S3::getObjectSize(*client_, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + auto size = S3::getObjectSize(*clients_->client, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); auto scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "S3ObjStor_copy"); - copyS3File( - client.get(), + copyS3File(clients_->client, + clients_->client_with_long_timeout, bucket, object_from.remote_path, 0, @@ -465,11 +473,12 @@ void S3ObjectStorage::copyObject( // NOLINT const WriteSettings &, std::optional object_to_attributes) { - auto client_ = client.get(); + auto clients_ = clients.get(); auto settings_ptr = s3_settings.get(); - auto size = S3::getObjectSize(*client_, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + auto size = S3::getObjectSize(*clients_->client, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); auto scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "S3ObjStor_copy"); - copyS3File(client_, + copyS3File(clients_->client, + clients_->client_with_long_timeout, bucket, object_from.remote_path, 0, @@ -490,25 +499,31 @@ void S3ObjectStorage::setNewSettings(std::unique_ptr && void S3ObjectStorage::shutdown() { + auto clients_ptr = clients.get(); /// This call stops any next retry attempts for ongoing S3 requests. /// If S3 request is failed and the method below is executed S3 client immediately returns the last failed S3 request outcome. /// If S3 is healthy nothing wrong will be happened and S3 requests will be processed in a regular way without errors. /// This should significantly speed up shutdown process if S3 is unhealthy. - const_cast(*client.get()).DisableRequestProcessing(); + const_cast(*clients_ptr->client).DisableRequestProcessing(); + const_cast(*clients_ptr->client_with_long_timeout).DisableRequestProcessing(); } void S3ObjectStorage::startup() { + auto clients_ptr = clients.get(); + /// Need to be enabled if it was disabled during shutdown() call. - const_cast(*client.get()).EnableRequestProcessing(); + const_cast(*clients_ptr->client).EnableRequestProcessing(); + const_cast(*clients_ptr->client_with_long_timeout).EnableRequestProcessing(); } void S3ObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) { auto new_s3_settings = getSettings(config, config_prefix, context); auto new_client = getClient(config, config_prefix, context, *new_s3_settings); + auto new_clients = std::make_unique(std::move(new_client), *new_s3_settings); s3_settings.set(std::move(new_s3_settings)); - client.set(std::move(new_client)); + clients.set(std::move(new_clients)); } std::unique_ptr S3ObjectStorage::cloneObjectStorage( @@ -523,6 +538,9 @@ std::unique_ptr S3ObjectStorage::cloneObjectStorage( endpoint, object_key_prefix); } +S3ObjectStorage::Clients::Clients(std::shared_ptr client_, const S3ObjectStorageSettings & settings) + : client(std::move(client_)), client_with_long_timeout(client->clone(std::nullopt, settings.request_settings.long_request_timeout_ms)) {} + ObjectStorageKey S3ObjectStorage::generateObjectKeyForPath(const std::string &) const { /// Path to store the new S3 object. diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index 7d14482311f0..b1b3fb22366f 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -39,6 +39,16 @@ struct S3ObjectStorageSettings class S3ObjectStorage : public IObjectStorage { +public: + struct Clients + { + std::shared_ptr client; + std::shared_ptr client_with_long_timeout; + + Clients() = default; + Clients(std::shared_ptr client, const S3ObjectStorageSettings & settings); + }; + private: friend class S3PlainObjectStorage; @@ -53,7 +63,7 @@ class S3ObjectStorage : public IObjectStorage String object_key_prefix_) : bucket(std::move(bucket_)) , object_key_prefix(std::move(object_key_prefix_)) - , client(std::move(client_)) + , clients(std::make_unique(std::move(client_), *s3_settings_)) , s3_settings(std::move(s3_settings_)) , s3_capabilities(s3_capabilities_) , version_id(std::move(version_id_)) @@ -174,8 +184,7 @@ class S3ObjectStorage : public IObjectStorage std::string bucket; String object_key_prefix; - - MultiVersion client; + MultiVersion clients; MultiVersion s3_settings; S3Capabilities s3_capabilities; diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 0232a6eb0706..de88c876922f 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -60,15 +60,13 @@ std::unique_ptr getClient( uri.uri.getScheme()); client_configuration.connectTimeoutMs = config.getUInt(config_prefix + ".connect_timeout_ms", 1000); - client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", 30000); + client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", 3000); client_configuration.maxConnections = config.getUInt(config_prefix + ".max_connections", 100); client_configuration.endpointOverride = uri.endpoint; - client_configuration.http_keep_alive_timeout_ms = config.getUInt( - config_prefix + ".http_keep_alive_timeout_ms", DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT * 1000); + client_configuration.http_keep_alive_timeout_ms + = config.getUInt(config_prefix + ".http_keep_alive_timeout_ms", DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT * 1000); client_configuration.http_connection_pool_size = config.getUInt(config_prefix + ".http_connection_pool_size", 1000); client_configuration.wait_on_pool_size_limit = false; - client_configuration.s3_use_adaptive_timeouts = config.getBool( - config_prefix + ".use_adaptive_timeouts", client_configuration.s3_use_adaptive_timeouts); /* * Override proxy configuration for backwards compatibility with old configuration format. diff --git a/src/IO/ConnectionTimeouts.cpp b/src/IO/ConnectionTimeouts.cpp index 970afc75ec3e..01fbaa4f8172 100644 --- a/src/IO/ConnectionTimeouts.cpp +++ b/src/IO/ConnectionTimeouts.cpp @@ -133,86 +133,4 @@ ConnectionTimeouts ConnectionTimeouts::getHTTPTimeouts(const Settings & settings settings.http_receive_timeout); } -class SendReceiveTimeoutsForFirstAttempt -{ -private: - static constexpr size_t known_methods_count = 6; - using KnownMethodsArray = std::array; - static const KnownMethodsArray known_methods; - - /// HTTP_POST is used for CompleteMultipartUpload requests. Its latency could be high. - /// These requests need longer timeout, especially when minio is used. - /// The same assumption are made for HTTP_DELETE, HTTP_PATCH - /// That requests are more heavy that HTTP_GET, HTTP_HEAD, HTTP_PUT - - static constexpr Poco::Timestamp::TimeDiff first_byte_ms[known_methods_count][2] = - { - /* GET */ {200, 200}, - /* POST */ {200, 200}, - /* DELETE */ {200, 200}, - /* PUT */ {200, 200}, - /* HEAD */ {200, 200}, - /* PATCH */ {200, 200}, - }; - - static constexpr Poco::Timestamp::TimeDiff rest_bytes_ms[known_methods_count][2] = - { - /* GET */ {500, 500}, - /* POST */ {1000, 30000}, - /* DELETE */ {1000, 10000}, - /* PUT */ {1000, 3000}, - /* HEAD */ {500, 500}, - /* PATCH */ {1000, 10000}, - }; - - static_assert(sizeof(first_byte_ms) == sizeof(rest_bytes_ms)); - static_assert(sizeof(first_byte_ms) == known_methods_count * sizeof(Poco::Timestamp::TimeDiff) * 2); - - static size_t getMethodIndex(const String & method) - { - KnownMethodsArray::const_iterator it = std::find(known_methods.begin(), known_methods.end(), method); - chassert(it != known_methods.end()); - if (it == known_methods.end()) - return 0; - return std::distance(known_methods.begin(), it); - } - -public: - static std::pair getSendReceiveTimeout(const String & method, bool first_byte) - { - auto idx = getMethodIndex(method); - - if (first_byte) - return std::make_pair( - Poco::Timespan(first_byte_ms[idx][0] * 1000), - Poco::Timespan(first_byte_ms[idx][1] * 1000) - ); - - return std::make_pair( - Poco::Timespan(rest_bytes_ms[idx][0] * 1000), - Poco::Timespan(rest_bytes_ms[idx][1] * 1000) - ); - } -}; - -const SendReceiveTimeoutsForFirstAttempt::KnownMethodsArray SendReceiveTimeoutsForFirstAttempt::known_methods = -{ - "GET", "POST", "DELETE", "PUT", "HEAD", "PATCH" -}; - - -ConnectionTimeouts ConnectionTimeouts::getAdaptiveTimeouts(const String & method, bool first_attempt, bool first_byte) const -{ - if (!first_attempt) - return *this; - - auto [send, recv] = SendReceiveTimeoutsForFirstAttempt::getSendReceiveTimeout(method, first_byte); - - auto aggressive = *this; - aggressive.send_timeout = saturate(send, send_timeout); - aggressive.receive_timeout = saturate(recv, receive_timeout); - - return aggressive; -} - } diff --git a/src/IO/ConnectionTimeouts.h b/src/IO/ConnectionTimeouts.h index aabebdb836d5..684af42827f5 100644 --- a/src/IO/ConnectionTimeouts.h +++ b/src/IO/ConnectionTimeouts.h @@ -67,8 +67,6 @@ struct ConnectionTimeouts /// Timeouts for the case when we will try many addresses in a loop. static ConnectionTimeouts getTCPTimeoutsWithFailover(const Settings & settings); static ConnectionTimeouts getHTTPTimeouts(const Settings & settings, Poco::Timespan http_keep_alive_timeout); - - ConnectionTimeouts getAdaptiveTimeouts(const String & method, bool first_attempt, bool first_byte) const; }; } diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index cce394c67c98..65ffa51a466a 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -50,6 +50,12 @@ namespace ErrorCodes namespace { + void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts) + { + session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout); + session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout); + } + Poco::Net::HTTPClientSession::ProxyConfig proxyConfigurationToPocoProxyConfig(const ProxyConfiguration & proxy_configuration) { Poco::Net::HTTPClientSession::ProxyConfig poco_proxy_config; @@ -353,12 +359,6 @@ namespace }; } -void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts) -{ - session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout); - session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout); -} - void setResponseDefaultHeaders(HTTPServerResponse & response, size_t keep_alive_timeout) { if (!response.getKeepAlive()) diff --git a/src/IO/HTTPCommon.h b/src/IO/HTTPCommon.h index c9968fc6915e..de62b5d5c165 100644 --- a/src/IO/HTTPCommon.h +++ b/src/IO/HTTPCommon.h @@ -113,6 +113,4 @@ std::istream * receiveResponse( void assertResponseIsOk( const Poco::Net::HTTPRequest & request, Poco::Net::HTTPResponse & response, std::istream & istr, bool allow_redirects = false); - -void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts); } diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index c9c9319c44ce..f19978ccb471 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -167,9 +167,9 @@ bool ReadBufferFromS3::nextImpl() } size_t sleep_time_with_backoff_milliseconds = 100; - for (size_t attempt = 1; !next_result; ++attempt) + for (size_t attempt = 0; !next_result; ++attempt) { - bool last_attempt = attempt >= request_settings.max_single_read_retries; + bool last_attempt = attempt + 1 >= request_settings.max_single_read_retries; ProfileEventTimeIncrement watch(ProfileEvents::ReadBufferFromS3Microseconds); @@ -177,7 +177,7 @@ bool ReadBufferFromS3::nextImpl() { if (!impl) { - impl = initialize(attempt); + impl = initialize(); if (use_external_buffer) { @@ -232,9 +232,9 @@ size_t ReadBufferFromS3::readBigAt(char * to, size_t n, size_t range_begin, cons { size_t initial_n = n; size_t sleep_time_with_backoff_milliseconds = 100; - for (size_t attempt = 1; n > 0; ++attempt) + for (size_t attempt = 0; n > 0; ++attempt) { - bool last_attempt = attempt >= request_settings.max_single_read_retries; + bool last_attempt = attempt + 1 >= request_settings.max_single_read_retries; size_t bytes_copied = 0; ProfileEventTimeIncrement watch(ProfileEvents::ReadBufferFromS3Microseconds); @@ -266,7 +266,7 @@ size_t ReadBufferFromS3::readBigAt(char * to, size_t n, size_t range_begin, cons try { - result = sendRequest(attempt, range_begin, range_begin + n - 1); + result = sendRequest(range_begin, range_begin + n - 1); std::istream & istr = result->GetBody(); copyFromIStreamWithProgressCallback(istr, to, n, progress_callback, &bytes_copied); @@ -304,8 +304,8 @@ bool ReadBufferFromS3::processException(Poco::Exception & e, size_t read_offset, LOG_DEBUG( log, "Caught exception while reading S3 object. Bucket: {}, Key: {}, Version: {}, Offset: {}, " - "Attempt: {}/{}, Message: {}", - bucket, key, version_id.empty() ? "Latest" : version_id, read_offset, attempt, request_settings.max_single_read_retries, e.message()); + "Attempt: {}, Message: {}", + bucket, key, version_id.empty() ? "Latest" : version_id, read_offset, attempt, e.message()); if (auto * s3_exception = dynamic_cast(&e)) @@ -463,7 +463,7 @@ ReadBufferFromS3::~ReadBufferFromS3() } } -std::unique_ptr ReadBufferFromS3::initialize(size_t attempt) +std::unique_ptr ReadBufferFromS3::initialize() { resetSessionIfNeeded(readAllRangeSuccessfully(), read_result); read_all_range_successfully = false; @@ -475,13 +475,13 @@ std::unique_ptr ReadBufferFromS3::initialize(size_t attempt) if (read_until_position && offset >= read_until_position) throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read beyond right offset ({} > {})", offset, read_until_position - 1); - read_result = sendRequest(attempt, offset, read_until_position ? std::make_optional(read_until_position - 1) : std::nullopt); + read_result = sendRequest(offset, read_until_position ? std::make_optional(read_until_position - 1) : std::nullopt); size_t buffer_size = use_external_buffer ? 0 : read_settings.remote_fs_buffer_size; return std::make_unique(read_result->GetBody(), buffer_size); } -Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t attempt, size_t range_begin, std::optional range_end_incl) const +Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t range_begin, std::optional range_end_incl) const { S3::GetObjectRequest req; req.SetBucket(bucket); @@ -489,8 +489,6 @@ Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t attempt, si if (!version_id.empty()) req.SetVersionId(version_id); - req.SetAdditionalCustomHeaderValue("clickhouse-request", fmt::format("attempt={}", attempt)); - if (range_end_incl) { req.SetRange(fmt::format("bytes={}-{}", range_begin, *range_end_incl)); diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h index 101e25f8b436..0835e52a5b2e 100644 --- a/src/IO/ReadBufferFromS3.h +++ b/src/IO/ReadBufferFromS3.h @@ -79,7 +79,7 @@ class ReadBufferFromS3 : public ReadBufferFromFileBase bool supportsReadAt() override { return true; } private: - std::unique_ptr initialize(size_t attempt); + std::unique_ptr initialize(); /// If true, if we destroy impl now, no work was wasted. Just for metrics. bool atEndOfRequestedRangeGuess(); @@ -88,7 +88,7 @@ class ReadBufferFromS3 : public ReadBufferFromFileBase /// Returns true if the error looks retriable. bool processException(Poco::Exception & e, size_t read_offset, size_t attempt) const; - Aws::S3::Model::GetObjectResult sendRequest(size_t attempt, size_t range_begin, std::optional range_end_incl) const; + Aws::S3::Model::GetObjectResult sendRequest(size_t range_begin, std::optional range_end_incl) const; bool readAllRangeSuccessfully() const; diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 4630e68fbb6c..ceb7d2752992 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -118,9 +118,16 @@ std::unique_ptr Client::create( new Client(max_redirects_, std::move(sse_kms_config_), credentials_provider, client_configuration, sign_payloads, use_virtual_addressing)); } -std::unique_ptr Client::clone() const +std::unique_ptr Client::clone( + std::optional> override_retry_strategy, + std::optional override_request_timeout_ms) const { - return std::unique_ptr(new Client(*this, client_configuration)); + PocoHTTPClientConfiguration new_configuration = client_configuration; + if (override_retry_strategy.has_value()) + new_configuration.retryStrategy = *override_retry_strategy; + if (override_request_timeout_ms.has_value()) + new_configuration.requestTimeoutMs = *override_request_timeout_ms; + return std::unique_ptr(new Client(*this, new_configuration)); } namespace @@ -898,7 +905,6 @@ PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT s3_retry_attempts, enable_s3_requests_logging, for_disk_s3, - context->getGlobalContext()->getSettingsRef().s3_use_adaptive_timeouts, get_request_throttler, put_request_throttler, error_report); diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index 5ad57a9d8272..48310bc21af9 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -118,7 +118,15 @@ class Client : private Aws::S3::S3Client Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy sign_payloads, bool use_virtual_addressing); - std::unique_ptr clone() const; + /// Create a client with adjusted settings: + /// * override_retry_strategy can be used to disable retries to avoid nested retries when we have + /// a retry loop outside of S3 client. Specifically, for read and write buffers. Currently not + /// actually used. + /// * override_request_timeout_ms is used to increase timeout for CompleteMultipartUploadRequest + /// because it often sits idle for 10 seconds: https://github.com/ClickHouse/ClickHouse/pull/42321 + std::unique_ptr clone( + std::optional> override_retry_strategy = std::nullopt, + std::optional override_request_timeout_ms = std::nullopt) const; Client & operator=(const Client &) = delete; diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 4a1b6def1331..d0f248f48a69 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -99,7 +99,6 @@ PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( unsigned int s3_retry_attempts_, bool enable_s3_requests_logging_, bool for_disk_s3_, - bool s3_use_adaptive_timeouts_, const ThrottlerPtr & get_request_throttler_, const ThrottlerPtr & put_request_throttler_, std::function error_report_) @@ -112,7 +111,6 @@ PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( , for_disk_s3(for_disk_s3_) , get_request_throttler(get_request_throttler_) , put_request_throttler(put_request_throttler_) - , s3_use_adaptive_timeouts(s3_use_adaptive_timeouts_) , error_report(error_report_) { } @@ -159,7 +157,6 @@ PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & client_config Poco::Timespan(client_configuration.http_keep_alive_timeout_ms * 1000))) /// flag indicating whether keep-alive is enabled is set to each session upon creation , remote_host_filter(client_configuration.remote_host_filter) , s3_max_redirects(client_configuration.s3_max_redirects) - , s3_use_adaptive_timeouts(client_configuration.s3_use_adaptive_timeouts) , enable_s3_requests_logging(client_configuration.enable_s3_requests_logging) , for_disk_s3(client_configuration.for_disk_s3) , get_request_throttler(client_configuration.get_request_throttler) @@ -271,38 +268,6 @@ void PocoHTTPClient::addMetric(const Aws::Http::HttpRequest & request, S3MetricT ProfileEvents::increment(disk_s3_events_map[static_cast(type)][static_cast(kind)], amount); } -String extractAttemptFromInfo(const Aws::String & request_info) -{ - static auto key = Aws::String("attempt="); - - auto key_begin = request_info.find(key, 0); - if (key_begin == Aws::String::npos) - return "1"; - - auto val_begin = key_begin + key.size(); - auto val_end = request_info.find(';', val_begin); - if (val_end == Aws::String::npos) - val_end = request_info.size(); - - return request_info.substr(val_begin, val_end-val_begin); -} - -String getOrEmpty(const Aws::Http::HeaderValueCollection & map, const String & key) -{ - auto it = map.find(key); - if (it == map.end()) - return {}; - return it->second; -} - -ConnectionTimeouts PocoHTTPClient::getTimeouts(const String & method, bool first_attempt, bool first_byte) const -{ - if (!s3_use_adaptive_timeouts) - return timeouts; - - return timeouts.getAdaptiveTimeouts(method, first_attempt, first_byte); -} - void PocoHTTPClient::makeRequestInternal( Aws::Http::HttpRequest & request, std::shared_ptr & response, @@ -317,25 +282,6 @@ void PocoHTTPClient::makeRequestInternal( makeRequestInternalImpl(request, request_configuration, response, readLimiter, writeLimiter); } -String getMethod(const Aws::Http::HttpRequest & request) -{ - switch (request.GetMethod()) - { - case Aws::Http::HttpMethod::HTTP_GET: - return Poco::Net::HTTPRequest::HTTP_GET; - case Aws::Http::HttpMethod::HTTP_POST: - return Poco::Net::HTTPRequest::HTTP_POST; - case Aws::Http::HttpMethod::HTTP_DELETE: - return Poco::Net::HTTPRequest::HTTP_DELETE; - case Aws::Http::HttpMethod::HTTP_PUT: - return Poco::Net::HTTPRequest::HTTP_PUT; - case Aws::Http::HttpMethod::HTTP_HEAD: - return Poco::Net::HTTPRequest::HTTP_HEAD; - case Aws::Http::HttpMethod::HTTP_PATCH: - return Poco::Net::HTTPRequest::HTTP_PATCH; - } -} - template void PocoHTTPClient::makeRequestInternalImpl( Aws::Http::HttpRequest & request, @@ -349,14 +295,9 @@ void PocoHTTPClient::makeRequestInternalImpl( Poco::Logger * log = &Poco::Logger::get("AWSClient"); auto uri = request.GetUri().GetURIString(); - auto method = getMethod(request); - - auto sdk_attempt = extractAttemptFromInfo(getOrEmpty(request.GetHeaders(), Aws::Http::SDK_REQUEST_HEADER)); - auto ch_attempt = extractAttemptFromInfo(getOrEmpty(request.GetHeaders(), "clickhouse-request")); - bool first_attempt = ch_attempt == "1" && sdk_attempt == "1"; if (enable_s3_requests_logging) - LOG_TEST(log, "Make request to: {}, aws sdk attempt: {}, clickhouse attempt: {}", uri, sdk_attempt, ch_attempt); + LOG_TEST(log, "Make request to: {}", uri); switch (request.GetMethod()) { @@ -407,29 +348,17 @@ void PocoHTTPClient::makeRequestInternalImpl( /// This can lead to request signature difference on S3 side. if constexpr (pooled) session = makePooledHTTPSession( - target_uri, - getTimeouts(method, first_attempt, /*first_byte*/ true), - http_connection_pool_size, - wait_on_pool_size_limit, - proxy_configuration); + target_uri, timeouts, http_connection_pool_size, wait_on_pool_size_limit, proxy_configuration); else - session = makeHTTPSession( - target_uri, - getTimeouts(method, first_attempt, /*first_byte*/ true), - proxy_configuration); + session = makeHTTPSession(target_uri, timeouts, proxy_configuration); } else { if constexpr (pooled) session = makePooledHTTPSession( - target_uri, - getTimeouts(method, first_attempt, /*first_byte*/ true), - http_connection_pool_size, - wait_on_pool_size_limit); + target_uri, timeouts, http_connection_pool_size, wait_on_pool_size_limit); else - session = makeHTTPSession( - target_uri, - getTimeouts(method, first_attempt, /*first_byte*/ true)); + session = makeHTTPSession(target_uri, timeouts); } /// In case of error this address will be written to logs @@ -463,7 +392,28 @@ void PocoHTTPClient::makeRequestInternalImpl( path_and_query = "/"; poco_request.setURI(path_and_query); - poco_request.setMethod(method); + + switch (request.GetMethod()) + { + case Aws::Http::HttpMethod::HTTP_GET: + poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_GET); + break; + case Aws::Http::HttpMethod::HTTP_POST: + poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_POST); + break; + case Aws::Http::HttpMethod::HTTP_DELETE: + poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_DELETE); + break; + case Aws::Http::HttpMethod::HTTP_PUT: + poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_PUT); + break; + case Aws::Http::HttpMethod::HTTP_HEAD: + poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_HEAD); + break; + case Aws::Http::HttpMethod::HTTP_PATCH: + poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_PATCH); + break; + } /// Headers coming from SDK are lower-cased. for (const auto & [header_name, header_value] : request.GetHeaders()) @@ -488,7 +438,6 @@ void PocoHTTPClient::makeRequestInternalImpl( request.GetContentBody()->clear(); request.GetContentBody()->seekg(0); - setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); auto size = Poco::StreamCopier::copyStream(*request.GetContentBody(), request_body_stream); if (enable_s3_requests_logging) LOG_TEST(log, "Written {} bytes to request body", size); @@ -498,8 +447,6 @@ void PocoHTTPClient::makeRequestInternalImpl( LOG_TEST(log, "Receiving response..."); auto & response_body_stream = session->receiveResponse(poco_response); - setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); - watch.stop(); addMetric(request, S3MetricType::Microseconds, watch.elapsedMicroseconds()); @@ -551,7 +498,6 @@ void PocoHTTPClient::makeRequestInternalImpl( /// Request is successful but for some special requests we can have actual error message in body if (status_code >= SUCCESS_RESPONSE_MIN && status_code <= SUCCESS_RESPONSE_MAX && checkRequestCanReturn2xxAndErrorInBody(request)) { - /// reading the full response std::string response_string((std::istreambuf_iterator(response_body_stream)), std::istreambuf_iterator()); @@ -566,6 +512,7 @@ void PocoHTTPClient::makeRequestInternalImpl( addMetric(request, S3MetricType::Errors); if (error_report) error_report(proxy_configuration); + } /// Set response from string @@ -584,8 +531,6 @@ void PocoHTTPClient::makeRequestInternalImpl( if (status_code >= 500 && error_report) error_report(proxy_configuration); } - - /// expose stream, after that client reads data from that stream without built-in retries response->SetResponseBody(response_body_stream, session); } diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 5178d75e7b6d..2a449458360c 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -55,7 +55,6 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration size_t http_connection_pool_size = 0; /// See PoolBase::BehaviourOnLimit bool wait_on_pool_size_limit = true; - bool s3_use_adaptive_timeouts = true; std::function error_report; @@ -70,7 +69,6 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration unsigned int s3_retry_attempts, bool enable_s3_requests_logging_, bool for_disk_s3_, - bool s3_use_adaptive_timeouts_, const ThrottlerPtr & get_request_throttler_, const ThrottlerPtr & put_request_throttler_, std::function error_report_ @@ -171,8 +169,6 @@ class PocoHTTPClient : public Aws::Http::HttpClient Aws::Utils::RateLimits::RateLimiterInterface * readLimiter, Aws::Utils::RateLimits::RateLimiterInterface * writeLimiter) const; - ConnectionTimeouts getTimeouts(const String & method, bool first_attempt, bool first_byte) const; - protected: static S3MetricKind getMetricKind(const Aws::Http::HttpRequest & request); void addMetric(const Aws::Http::HttpRequest & request, S3MetricType type, ProfileEvents::Count amount = 1) const; @@ -182,7 +178,6 @@ class PocoHTTPClient : public Aws::Http::HttpClient ConnectionTimeouts timeouts; const RemoteHostFilter & remote_host_filter; unsigned int s3_max_redirects; - bool s3_use_adaptive_timeouts = true; bool enable_s3_requests_logging; bool for_disk_s3; diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index 30da1c580c13..a16a1a415058 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -53,6 +53,7 @@ namespace public: UploadHelper( const std::shared_ptr & client_ptr_, + const std::shared_ptr & client_with_long_timeout_ptr_, const String & dest_bucket_, const String & dest_key_, const S3Settings::RequestSettings & request_settings_, @@ -61,6 +62,7 @@ namespace bool for_disk_s3_, const Poco::Logger * log_) : client_ptr(client_ptr_) + , client_with_long_timeout_ptr(client_with_long_timeout_ptr_) , dest_bucket(dest_bucket_) , dest_key(dest_key_) , request_settings(request_settings_) @@ -76,6 +78,7 @@ namespace protected: std::shared_ptr client_ptr; + std::shared_ptr client_with_long_timeout_ptr; const String & dest_bucket; const String & dest_key; const S3Settings::RequestSettings & request_settings; @@ -176,7 +179,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); - auto outcome = client_ptr->CompleteMultipartUpload(request); + auto outcome = client_with_long_timeout_ptr->CompleteMultipartUpload(request); if (outcome.IsSuccess()) { @@ -430,13 +433,14 @@ namespace size_t offset_, size_t size_, const std::shared_ptr & client_ptr_, + const std::shared_ptr & client_with_long_timeout_ptr_, const String & dest_bucket_, const String & dest_key_, const S3Settings::RequestSettings & request_settings_, const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, bool for_disk_s3_) - : UploadHelper(client_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyDataToS3File")) + : UploadHelper(client_ptr_, client_with_long_timeout_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyDataToS3File")) , create_read_buffer(create_read_buffer_) , offset(offset_) , size(size_) @@ -598,6 +602,7 @@ namespace public: CopyFileHelper( const std::shared_ptr & client_ptr_, + const std::shared_ptr & client_with_long_timeout_ptr_, const String & src_bucket_, const String & src_key_, size_t src_offset_, @@ -609,7 +614,7 @@ namespace const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, bool for_disk_s3_) - : UploadHelper(client_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyS3File")) + : UploadHelper(client_ptr_, client_with_long_timeout_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyS3File")) , src_bucket(src_bucket_) , src_key(src_key_) , offset(src_offset_) @@ -672,7 +677,7 @@ namespace /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 request.SetContentType("binary/octet-stream"); - client_ptr->setKMSHeaders(request); + client_with_long_timeout_ptr->setKMSHeaders(request); } void processCopyRequest(const S3::CopyObjectRequest & request) @@ -684,7 +689,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3CopyObject); - auto outcome = client_ptr->CopyObject(request); + auto outcome = client_with_long_timeout_ptr->CopyObject(request); if (outcome.IsSuccess()) { LOG_TRACE( @@ -709,6 +714,7 @@ namespace offset, size, client_ptr, + client_with_long_timeout_ptr, dest_bucket, dest_key, request_settings, @@ -782,7 +788,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3UploadPartCopy); - auto outcome = client_ptr->UploadPartCopy(req); + auto outcome = client_with_long_timeout_ptr->UploadPartCopy(req); if (!outcome.IsSuccess()) { abortMultipartUpload(); @@ -800,6 +806,7 @@ void copyDataToS3File( size_t offset, size_t size, const std::shared_ptr & dest_s3_client, + const std::shared_ptr & dest_s3_client_with_long_timeout, const String & dest_bucket, const String & dest_key, const S3Settings::RequestSettings & settings, @@ -807,13 +814,14 @@ void copyDataToS3File( ThreadPoolCallbackRunner schedule, bool for_disk_s3) { - CopyDataToFileHelper helper{create_read_buffer, offset, size, dest_s3_client, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3}; + CopyDataToFileHelper helper{create_read_buffer, offset, size, dest_s3_client, dest_s3_client_with_long_timeout, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3}; helper.performCopy(); } void copyS3File( const std::shared_ptr & s3_client, + const std::shared_ptr & s3_client_with_long_timeout, const String & src_bucket, const String & src_key, size_t src_offset, @@ -828,7 +836,7 @@ void copyS3File( { if (settings.allow_native_copy) { - CopyFileHelper helper{s3_client, src_bucket, src_key, src_offset, src_size, dest_bucket, dest_key, settings, read_settings, object_metadata, schedule, for_disk_s3}; + CopyFileHelper helper{s3_client, s3_client_with_long_timeout, src_bucket, src_key, src_offset, src_size, dest_bucket, dest_key, settings, read_settings, object_metadata, schedule, for_disk_s3}; helper.performCopy(); } else @@ -837,7 +845,7 @@ void copyS3File( { return std::make_unique(s3_client, src_bucket, src_key, "", settings, read_settings); }; - copyDataToS3File(create_read_buffer, src_offset, src_size, s3_client, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3); + copyDataToS3File(create_read_buffer, src_offset, src_size, s3_client, s3_client_with_long_timeout, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3); } } diff --git a/src/IO/S3/copyS3File.h b/src/IO/S3/copyS3File.h index 33e22fdfba22..1bcbfd7735e4 100644 --- a/src/IO/S3/copyS3File.h +++ b/src/IO/S3/copyS3File.h @@ -27,9 +27,15 @@ using CreateReadBuffer = std::function()>; /// because it is a known issue, it is fallbacks to read-write copy /// (copyDataToS3File()). /// +/// s3_client_with_long_timeout (may be equal to s3_client) is used for native copy and +/// CompleteMultipartUpload requests. These requests need longer timeout because S3 servers often +/// block on them for multiple seconds without sending or receiving data from us (maybe the servers +/// are copying data internally, or maybe throttling, idk). +/// /// read_settings - is used for throttling in case of native copy is not possible void copyS3File( const std::shared_ptr & s3_client, + const std::shared_ptr & s3_client_with_long_timeout, const String & src_bucket, const String & src_key, size_t src_offset, @@ -52,6 +58,7 @@ void copyDataToS3File( size_t offset, size_t size, const std::shared_ptr & dest_s3_client, + const std::shared_ptr & dest_s3_client_with_long_timeout, const String & dest_bucket, const String & dest_key, const S3Settings::RequestSettings & settings, diff --git a/src/IO/S3/tests/gtest_aws_s3_client.cpp b/src/IO/S3/tests/gtest_aws_s3_client.cpp index bff9ca6fa7b3..c42f14e9a53e 100644 --- a/src/IO/S3/tests/gtest_aws_s3_client.cpp +++ b/src/IO/S3/tests/gtest_aws_s3_client.cpp @@ -91,6 +91,7 @@ void doWriteRequest(std::shared_ptr client, const DB::S3:: DB::S3Settings::RequestSettings request_settings; request_settings.max_unexpected_write_error_retries = max_unexpected_write_error_retries; DB::WriteBufferFromS3 write_buffer( + client, client, uri.bucket, uri.key, @@ -170,7 +171,6 @@ TEST(IOTestAwsS3Client, AppendExtraSSECHeadersRead) "authorization: ... SignedHeaders=" "amz-sdk-invocation-id;" "amz-sdk-request;" - "clickhouse-request;" "content-type;" "host;" "x-amz-api-version;" @@ -216,7 +216,6 @@ TEST(IOTestAwsS3Client, AppendExtraSSEKMSHeadersRead) "authorization: ... SignedHeaders=" "amz-sdk-invocation-id;" "amz-sdk-request;" - "clickhouse-request;" "content-type;" "host;" "x-amz-api-version;" diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 62d0c80f1f28..e1b9c17efe97 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -77,6 +77,7 @@ struct WriteBufferFromS3::PartData WriteBufferFromS3::WriteBufferFromS3( std::shared_ptr client_ptr_, + std::shared_ptr client_with_long_timeout_ptr_, const String & bucket_, const String & key_, size_t buf_size_, @@ -91,6 +92,7 @@ WriteBufferFromS3::WriteBufferFromS3( , upload_settings(request_settings.getUploadSettings()) , write_settings(write_settings_) , client_ptr(std::move(client_ptr_)) + , client_with_long_timeout_ptr(std::move(client_with_long_timeout_ptr_)) , object_metadata(std::move(object_metadata_)) , buffer_allocation_policy(ChooseBufferPolicy(upload_settings)) , task_tracker( @@ -564,7 +566,7 @@ void WriteBufferFromS3::completeMultipartUpload() ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Stopwatch watch; - auto outcome = client_ptr->CompleteMultipartUpload(req); + auto outcome = client_with_long_timeout_ptr->CompleteMultipartUpload(req); watch.stop(); ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Microseconds, watch.elapsedMicroseconds()); diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 590342cc997f..95148c497791 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -30,6 +30,8 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase public: WriteBufferFromS3( std::shared_ptr client_ptr_, + /// for CompleteMultipartUploadRequest, because it blocks on recv() for a few seconds on big uploads + std::shared_ptr client_with_long_timeout_ptr_, const String & bucket_, const String & key_, size_t buf_size_, @@ -88,6 +90,7 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase const S3Settings::RequestSettings::PartUploadSettings & upload_settings; const WriteSettings write_settings; const std::shared_ptr client_ptr; + const std::shared_ptr client_with_long_timeout_ptr; const std::optional> object_metadata; Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3"); LogSeriesLimiterPtr limitedLog = std::make_shared(log, 1, 5); diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp index c82f97f8b201..21bdd9a6f26d 100644 --- a/src/IO/tests/gtest_writebuffer_s3.cpp +++ b/src/IO/tests/gtest_writebuffer_s3.cpp @@ -549,6 +549,7 @@ class WBS3Test : public ::testing::Test getAsyncPolicy().setAutoExecute(false); return std::make_unique( + client, client, bucket, file_name, diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index bdbba5abd965..80ee1e9339d1 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -825,6 +825,7 @@ class StorageS3Sink : public SinkToStorage write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique( configuration_.client, + configuration_.client_with_long_timeout, bucket, key, DBMS_DEFAULT_BUFFER_SIZE, @@ -1329,6 +1330,8 @@ void StorageS3::Configuration::connect(ContextPtr context) context->getConfigRef().getUInt64("s3.expiration_window_seconds", S3::DEFAULT_EXPIRATION_WINDOW_SECONDS)), auth_settings.no_sign_request.value_or(context->getConfigRef().getBool("s3.no_sign_request", false)), }); + + client_with_long_timeout = client->clone(std::nullopt, request_settings.long_request_timeout_ms); } void StorageS3::processNamedCollectionResult(StorageS3::Configuration & configuration, const NamedCollection & collection) diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 3f35c578e19b..3330ac6c210e 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -311,6 +311,7 @@ class StorageS3 : public IStorage HTTPHeaderEntries headers_from_ast; std::shared_ptr client; + std::shared_ptr client_with_long_timeout; std::vector keys; }; diff --git a/src/Storages/StorageS3Settings.h b/src/Storages/StorageS3Settings.h index 728972c948c9..e3d577ca0b36 100644 --- a/src/Storages/StorageS3Settings.h +++ b/src/Storages/StorageS3Settings.h @@ -69,7 +69,8 @@ struct S3Settings ThrottlerPtr get_request_throttler; ThrottlerPtr put_request_throttler; size_t retry_attempts = 10; - size_t request_timeout_ms = 30000; + size_t request_timeout_ms = 3000; + size_t long_request_timeout_ms = 30000; // TODO: Take this from config like request_timeout_ms bool allow_native_copy = true; bool throw_on_zero_files_match = false; diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml index 4210c13b727c..206eb4f2badb 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml @@ -4,7 +4,6 @@ 1000000 - 1 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml index 95a313ea4f27..556bf60d3853 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml @@ -4,7 +4,6 @@ 5 - 0 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml index 7b1f503ed553..b77e72d808b6 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml @@ -7,18 +7,11 @@ - - s3 - http://minio1:9001/root/data/ - minio - minio123 - s3 http://resolver:8083/root/data/ minio minio123 - 1 @@ -30,16 +23,9 @@ - - -
- s3 -
- - - s3 + broken_s3 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py index b000ccabcf47..d6bcb3fb8f4b 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/test.py +++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py @@ -64,8 +64,6 @@ def test_upload_after_check_works(cluster, broken_s3): data String ) ENGINE=MergeTree() ORDER BY id - SETTINGS - storage_policy='broken_s3' """ ) @@ -80,7 +78,7 @@ def test_upload_after_check_works(cluster, broken_s3): assert "suddenly disappeared" in error, error -def get_multipart_counters(node, query_id, log_type="ExceptionWhileProcessing"): +def get_counters(node, query_id, log_type="ExceptionWhileProcessing"): node.query("SYSTEM FLUSH LOGS") return [ int(x) @@ -89,25 +87,7 @@ def get_multipart_counters(node, query_id, log_type="ExceptionWhileProcessing"): SELECT ProfileEvents['S3CreateMultipartUpload'], ProfileEvents['S3UploadPart'], - ProfileEvents['S3WriteRequestsErrors'], - FROM system.query_log - WHERE query_id='{query_id}' - AND type='{log_type}' - """ - ).split() - if x - ] - - -def get_put_counters(node, query_id, log_type="ExceptionWhileProcessing"): - node.query("SYSTEM FLUSH LOGS") - return [ - int(x) - for x in node.query( - f""" - SELECT - ProfileEvents['S3PutObject'], - ProfileEvents['S3WriteRequestsErrors'], + ProfileEvents['S3WriteRequestsErrors'] FROM system.query_log WHERE query_id='{query_id}' AND type='{log_type}' @@ -149,12 +129,12 @@ def test_upload_s3_fail_create_multi_part_upload(cluster, broken_s3, compression assert "Code: 499" in error, error assert "mock s3 injected error" in error, error - create_multipart, upload_parts, s3_errors = get_multipart_counters( + count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( node, insert_query_id ) - assert create_multipart == 1 - assert upload_parts == 0 - assert s3_errors == 1 + assert count_create_multi_part_uploads == 1 + assert count_upload_parts == 0 + assert count_s3_errors == 1 # Add "lz4" compression method in the list after https://github.com/ClickHouse/ClickHouse/issues/50975 is fixed @@ -192,12 +172,12 @@ def test_upload_s3_fail_upload_part_when_multi_part_upload( assert "Code: 499" in error, error assert "mock s3 injected error" in error, error - create_multipart, upload_parts, s3_errors = get_multipart_counters( + count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( node, insert_query_id ) - assert create_multipart == 1 - assert upload_parts >= 2 - assert s3_errors >= 2 + assert count_create_multi_part_uploads == 1 + assert count_upload_parts >= 2 + assert count_s3_errors >= 2 def test_when_s3_connection_refused_is_retried(cluster, broken_s3): @@ -227,12 +207,12 @@ def test_when_s3_connection_refused_is_retried(cluster, broken_s3): query_id=insert_query_id, ) - create_multipart, upload_parts, s3_errors = get_multipart_counters( + count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( node, insert_query_id, log_type="QueryFinish" ) - assert create_multipart == 1 - assert upload_parts == 39 - assert s3_errors == 3 + assert count_create_multi_part_uploads == 1 + assert count_upload_parts == 39 + assert count_s3_errors == 3 broken_s3.setup_at_part_upload(count=1000, after=2, action="connection_refused") insert_query_id = f"INSERT_INTO_TABLE_FUNCTION_CONNECTION_REFUSED_RETRIED_1" @@ -299,13 +279,13 @@ def test_when_s3_connection_reset_by_peer_at_upload_is_retried( query_id=insert_query_id, ) - create_multipart, upload_parts, s3_errors = get_multipart_counters( + count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( node, insert_query_id, log_type="QueryFinish" ) - assert create_multipart == 1 - assert upload_parts == 39 - assert s3_errors == 3 + assert count_create_multi_part_uploads == 1 + assert count_upload_parts == 39 + assert count_s3_errors == 3 broken_s3.setup_at_part_upload( count=1000, @@ -381,13 +361,13 @@ def test_when_s3_connection_reset_by_peer_at_create_mpu_retried( query_id=insert_query_id, ) - create_multipart, upload_parts, s3_errors = get_multipart_counters( + count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( node, insert_query_id, log_type="QueryFinish" ) - assert create_multipart == 1 - assert upload_parts == 39 - assert s3_errors == 3 + assert count_create_multi_part_uploads == 1 + assert count_upload_parts == 39 + assert count_s3_errors == 3 broken_s3.setup_at_create_multi_part_upload( count=1000, @@ -458,13 +438,13 @@ def test_when_s3_broken_pipe_at_upload_is_retried(cluster, broken_s3): query_id=insert_query_id, ) - create_multipart, upload_parts, s3_errors = get_multipart_counters( + count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( node, insert_query_id, log_type="QueryFinish" ) - assert create_multipart == 1 - assert upload_parts == 7 - assert s3_errors == 3 + assert count_create_multi_part_uploads == 1 + assert count_upload_parts == 7 + assert count_s3_errors == 3 broken_s3.setup_at_part_upload( count=1000, @@ -553,60 +533,3 @@ def test_query_is_canceled_with_inf_retries(cluster, broken_s3): retry_count=120, sleep_time=1, ) - - -@pytest.mark.parametrize("node_name", ["node", "node_with_inf_s3_retries"]) -def test_adaptive_timeouts(cluster, broken_s3, node_name): - node = cluster.instances[node_name] - - broken_s3.setup_fake_puts(part_length=1) - broken_s3.setup_slow_answers( - timeout=5, - count=1000000, - ) - - insert_query_id = f"TEST_ADAPTIVE_TIMEOUTS_{node_name}" - node.query( - f""" - INSERT INTO - TABLE FUNCTION s3( - 'http://resolver:8083/root/data/adaptive_timeouts', - 'minio', 'minio123', - 'CSV', auto, 'none' - ) - SELECT - * - FROM system.numbers - LIMIT 1 - SETTINGS - s3_request_timeout_ms=30000, - s3_check_objects_after_upload=0 - """, - query_id=insert_query_id, - ) - - broken_s3.reset() - - put_objects, s3_errors = get_put_counters( - node, insert_query_id, log_type="QueryFinish" - ) - - assert put_objects == 1 - - s3_use_adaptive_timeouts = node.query( - f""" - SELECT - value - FROM system.settings - WHERE - name='s3_use_adaptive_timeouts' - """ - ).strip() - - if node_name == "node_with_inf_s3_retries": - # first 2 attempts failed - assert s3_use_adaptive_timeouts == "1" - assert s3_errors == 1 - else: - assert s3_use_adaptive_timeouts == "0" - assert s3_errors == 0 diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml index 6303e9273fc3..235b9a7b7a12 100644 --- a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml @@ -11,7 +11,6 @@ true 0 - 0 20000 @@ -34,7 +33,6 @@ true 1 - 0 1 20000 diff --git a/tests/integration/test_storage_s3/configs/defaultS3.xml b/tests/integration/test_storage_s3/configs/defaultS3.xml index 7dac6d9fbb57..37454ef6781f 100644 --- a/tests/integration/test_storage_s3/configs/defaultS3.xml +++ b/tests/integration/test_storage_s3/configs/defaultS3.xml @@ -1,4 +1,9 @@ + + + 5 + + http://resolver:8080 diff --git a/tests/integration/test_storage_s3/configs/s3_retry.xml b/tests/integration/test_storage_s3/configs/s3_retry.xml index 3171da051d0c..727e23273cf3 100644 --- a/tests/integration/test_storage_s3/configs/s3_retry.xml +++ b/tests/integration/test_storage_s3/configs/s3_retry.xml @@ -1,9 +1,7 @@ - 1 - 10 - 5 + 5 diff --git a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py index 5ef781bdc9e2..103dd30340ce 100644 --- a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py +++ b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py @@ -4,7 +4,6 @@ import socket import struct import sys -import time def gen_n_digit_number(n): @@ -40,14 +39,14 @@ def add_number(): # Generating some "random" data and append a line which contains sum of numbers in column 4. lines = ( - b"".join([gen_line() for _ in range(500000)]) + b"".join((gen_line() for _ in range(500000))) + f"0,0,0,{-sum_in_4_column}\n".encode() ) class RequestHandler(http.server.BaseHTTPRequestHandler): def do_HEAD(self): - if self.path == "/root/test.csv" or self.path == "/root/slow_send_test.csv": + if self.path == "/root/test.csv": self.from_bytes = 0 self.end_bytes = len(lines) self.size = self.end_bytes @@ -102,18 +101,6 @@ def do_GET(self): print("Dropping connection") break - if self.path == "/root/slow_send_test.csv": - self.send_block_size = 81920 - - for c, i in enumerate( - range(self.from_bytes, self.end_bytes, self.send_block_size) - ): - self.wfile.write( - lines[i : min(i + self.send_block_size, self.end_bytes)] - ) - self.wfile.flush() - time.sleep(1) - elif self.path == "/": self.wfile.write(b"OK") diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 835c8b908f0f..3dd3c9e39d02 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -818,15 +818,6 @@ def test_storage_s3_get_unstable(started_cluster): assert result.splitlines() == ["500001,500000,0"] -def test_storage_s3_get_slow(started_cluster): - bucket = started_cluster.minio_bucket - instance = started_cluster.instances["dummy"] - table_format = "column1 Int64, column2 Int64, column3 Int64, column4 Int64" - get_query = f"SELECT count(), sum(column3), sum(column4) FROM s3('http://resolver:8081/{started_cluster.minio_bucket}/slow_send_test.csv', 'CSV', '{table_format}') FORMAT CSV" - result = run_query(instance, get_query) - assert result.splitlines() == ["500001,500000,0"] - - def test_storage_s3_put_uncompressed(started_cluster): bucket = started_cluster.minio_bucket instance = started_cluster.instances["dummy"] From f999337daee891499a78cadec1cc562cf29ebcaa Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:53:22 +0100 Subject: [PATCH 225/274] Revert "Revert "s3 adaptive timeouts"" --- base/poco/Net/src/HTTPServerSession.cpp | 1 - base/poco/Net/src/HTTPSession.cpp | 31 ++++- docs/en/operations/settings/settings.md | 7 + src/Backups/BackupIO_S3.cpp | 9 +- src/Coordination/KeeperSnapshotManagerS3.cpp | 1 - src/Core/Settings.h | 3 +- .../ObjectStorages/S3/S3ObjectStorage.cpp | 58 +++----- src/Disks/ObjectStorages/S3/S3ObjectStorage.h | 15 +- src/Disks/ObjectStorages/S3/diskSettings.cpp | 8 +- src/IO/ConnectionTimeouts.cpp | 82 +++++++++++ src/IO/ConnectionTimeouts.h | 2 + src/IO/HTTPCommon.cpp | 12 +- src/IO/HTTPCommon.h | 2 + src/IO/ReadBufferFromS3.cpp | 24 ++-- src/IO/ReadBufferFromS3.h | 4 +- src/IO/S3/Client.cpp | 12 +- src/IO/S3/Client.h | 10 +- src/IO/S3/PocoHTTPClient.cpp | 111 +++++++++++---- src/IO/S3/PocoHTTPClient.h | 5 + src/IO/S3/copyS3File.cpp | 26 ++-- src/IO/S3/copyS3File.h | 7 - src/IO/S3/tests/gtest_aws_s3_client.cpp | 3 +- src/IO/WriteBufferFromS3.cpp | 4 +- src/IO/WriteBufferFromS3.h | 3 - src/IO/tests/gtest_writebuffer_s3.cpp | 1 - src/Storages/StorageS3.cpp | 3 - src/Storages/StorageS3.h | 1 - src/Storages/StorageS3Settings.h | 3 +- .../configs/inf_s3_retries.xml | 1 + .../configs/s3_retries.xml | 1 + .../configs/storage_conf.xml | 16 ++- .../test_checking_s3_blobs_paranoid/test.py | 129 ++++++++++++++---- .../configs/config.d/storage_conf.xml | 2 + .../test_storage_s3/configs/defaultS3.xml | 5 - .../test_storage_s3/configs/s3_retry.xml | 4 +- .../s3_mocks/unstable_server.py | 17 ++- tests/integration/test_storage_s3/test.py | 9 ++ 37 files changed, 430 insertions(+), 202 deletions(-) diff --git a/base/poco/Net/src/HTTPServerSession.cpp b/base/poco/Net/src/HTTPServerSession.cpp index f6d3c4e5b923..d4f2b24879e4 100644 --- a/base/poco/Net/src/HTTPServerSession.cpp +++ b/base/poco/Net/src/HTTPServerSession.cpp @@ -26,7 +26,6 @@ HTTPServerSession::HTTPServerSession(const StreamSocket& socket, HTTPServerParam _maxKeepAliveRequests(pParams->getMaxKeepAliveRequests()) { setTimeout(pParams->getTimeout()); - this->socket().setReceiveTimeout(pParams->getTimeout()); } diff --git a/base/poco/Net/src/HTTPSession.cpp b/base/poco/Net/src/HTTPSession.cpp index d2663baaf9fe..8f951b3102cd 100644 --- a/base/poco/Net/src/HTTPSession.cpp +++ b/base/poco/Net/src/HTTPSession.cpp @@ -93,9 +93,34 @@ void HTTPSession::setTimeout(const Poco::Timespan& timeout) void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco::Timespan& sendTimeout, const Poco::Timespan& receiveTimeout) { - _connectionTimeout = connectionTimeout; - _sendTimeout = sendTimeout; - _receiveTimeout = receiveTimeout; + try + { + _connectionTimeout = connectionTimeout; + + if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) { + _sendTimeout = sendTimeout; + + if (connected()) + _socket.setSendTimeout(_sendTimeout); + } + + if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) { + _receiveTimeout = receiveTimeout; + + if (connected()) + _socket.setReceiveTimeout(_receiveTimeout); + } + } + catch (NetException &) + { +#ifndef NDEBUG + throw; +#else + // mute exceptions in release + // just in case when changing settings on socket is not allowed + // however it should be OK for timeouts +#endif + } } diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index e61934d21686..edc1c9bdfd77 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -4826,3 +4826,10 @@ When set to `true` the metadata files are written with `VERSION_FULL_OBJECT_KEY` When set to `false` the metadata files are written with the previous format version, `VERSION_INLINE_DATA`. With that format only suffixes of object storage key names are are written to the metadata files. The prefix for all of object storage key names is set in configurations files at `storage_configuration.disks` section. Default value: `false`. + +## s3_use_adaptive_timeouts {#s3_use_adaptive_timeouts} + +When set to `true` than for all s3 requests first two attempts are made with low send and receive timeouts. +When set to `false` than all attempts are made with identical timeouts. + +Default value: `true`. diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index eb9dcf6b45af..ea3f57c27ffc 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -55,7 +55,9 @@ namespace static_cast(context->getGlobalContext()->getSettingsRef().s3_max_redirects), static_cast(context->getGlobalContext()->getSettingsRef().s3_retry_attempts), context->getGlobalContext()->getSettingsRef().enable_s3_requests_logging, - /* for_disk_s3 = */ false, request_settings.get_request_throttler, request_settings.put_request_throttler, + /* for_disk_s3 = */ false, + request_settings.get_request_throttler, + request_settings.put_request_throttler, s3_uri.uri.getScheme()); client_configuration.endpointOverride = s3_uri.endpoint; @@ -167,7 +169,6 @@ void BackupReaderS3::copyFileToDisk(const String & path_in_backup, size_t file_s blob_path.size(), mode); copyS3File( - client, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, @@ -229,7 +230,6 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src { LOG_TRACE(log, "Copying file {} from disk {} to S3", src_path, src_disk->getName()); copyS3File( - client, client, /* src_bucket */ blob_path[1], /* src_key= */ blob_path[0], @@ -268,7 +268,7 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, void BackupWriterS3::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) { - copyDataToS3File(create_read_buffer, start_pos, length, client, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, s3_settings.request_settings, {}, + copyDataToS3File(create_read_buffer, start_pos, length, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, s3_settings.request_settings, {}, threadPoolCallbackRunner(getBackupsIOThreadPool().get(), "BackupWriterS3")); } @@ -298,7 +298,6 @@ std::unique_ptr BackupWriterS3::writeFile(const String & file_name) { return std::make_unique( client, - client, // already has long timeout s3_uri.bucket, fs::path(s3_uri.key) / file_name, DBMS_DEFAULT_BUFFER_SIZE, diff --git a/src/Coordination/KeeperSnapshotManagerS3.cpp b/src/Coordination/KeeperSnapshotManagerS3.cpp index 302e05c84184..bedde0d7b39c 100644 --- a/src/Coordination/KeeperSnapshotManagerS3.cpp +++ b/src/Coordination/KeeperSnapshotManagerS3.cpp @@ -148,7 +148,6 @@ void KeeperSnapshotManagerS3::uploadSnapshotImpl(const SnapshotFileInfo & snapsh const auto create_writer = [&](const auto & key) { return WriteBufferFromS3( - s3_client->client, s3_client->client, s3_client->uri.bucket, key, diff --git a/src/Core/Settings.h b/src/Core/Settings.h index ac4c6b6c17fc..bb5e43224852 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -94,6 +94,7 @@ class IColumn; M(UInt64, s3_max_put_rps, 0, "Limit on S3 PUT request per second rate before throttling. Zero means unlimited.", 0) \ M(UInt64, s3_max_put_burst, 0, "Max number of requests that can be issued simultaneously before hitting request per second limit. By default (0) equals to `s3_max_put_rps`", 0) \ M(UInt64, s3_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ + M(Bool, s3_use_adaptive_timeouts, true, "When adaptive timeouts are enabled first two attempts are made with low receive and send timeout", 0) \ M(UInt64, azure_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ M(Bool, s3_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables.", 0) \ M(Bool, azure_truncate_on_insert, false, "Enables or disables truncate before insert in azure engine tables.", 0) \ @@ -104,7 +105,7 @@ class IColumn; M(Bool, s3_allow_parallel_part_upload, true, "Use multiple threads for s3 multipart upload. It may lead to slightly higher memory usage", 0) \ M(Bool, s3_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \ M(UInt64, s3_retry_attempts, 100, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries", 0) \ - M(UInt64, s3_request_timeout_ms, 3000, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \ + M(UInt64, s3_request_timeout_ms, 30000, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \ M(UInt64, s3_http_connection_pool_size, 1000, "How many reusable open connections to keep per S3 endpoint. Only applies to the S3 table engine and table function, not to S3 disks (for disks, use disk config instead). Global setting, can only be set in config, overriding it per session or per query has no effect.", 0) \ M(Bool, enable_s3_requests_logging, false, "Enable very explicit logging of S3 requests. Makes sense for debug only.", 0) \ M(String, s3queue_default_zookeeper_path, "/clickhouse/s3queue/", "Default zookeeper path prefix for S3Queue engine", 0) \ diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 3af316bf0cf8..308db389ee13 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -155,7 +155,7 @@ class S3IteratorAsync final : public IObjectStorageIteratorAsync bool S3ObjectStorage::exists(const StoredObject & object) const { auto settings_ptr = s3_settings.get(); - return S3::objectExists(*clients.get()->client, bucket, object.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + return S3::objectExists(*client.get(), bucket, object.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); } std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT @@ -174,7 +174,7 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT (const std::string & path, size_t read_until_position) -> std::unique_ptr { return std::make_unique( - clients.get()->client, + client.get(), bucket, path, version_id, @@ -224,7 +224,7 @@ std::unique_ptr S3ObjectStorage::readObject( /// NOLINT { auto settings_ptr = s3_settings.get(); return std::make_unique( - clients.get()->client, + client.get(), bucket, object.remote_path, version_id, @@ -249,10 +249,8 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN if (write_settings.s3_allow_parallel_part_upload) scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "VFSWrite"); - auto clients_ = clients.get(); return std::make_unique( - clients_->client, - clients_->client_with_long_timeout, + client.get(), bucket, object.remote_path, buf_size, @@ -266,15 +264,12 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN ObjectStorageIteratorPtr S3ObjectStorage::iterate(const std::string & path_prefix) const { auto settings_ptr = s3_settings.get(); - auto client_ptr = clients.get()->client; - - return std::make_shared(bucket, path_prefix, client_ptr, settings_ptr->list_object_keys_size); + return std::make_shared(bucket, path_prefix, client.get(), settings_ptr->list_object_keys_size); } void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMetadata & children, int max_keys) const { auto settings_ptr = s3_settings.get(); - auto client_ptr = clients.get()->client; S3::ListObjectsV2Request request; request.SetBucket(bucket); @@ -289,7 +284,7 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet { ProfileEvents::increment(ProfileEvents::S3ListObjects); ProfileEvents::increment(ProfileEvents::DiskS3ListObjects); - outcome = client_ptr->ListObjectsV2(request); + outcome = client.get()->ListObjectsV2(request); throwIfError(outcome); auto result = outcome.GetResult(); @@ -320,14 +315,12 @@ void S3ObjectStorage::listObjects(const std::string & path, RelativePathsWithMet void S3ObjectStorage::removeObjectImpl(const StoredObject & object, bool if_exists) { - auto client_ptr = clients.get()->client; - ProfileEvents::increment(ProfileEvents::S3DeleteObjects); ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); S3::DeleteObjectRequest request; request.SetBucket(bucket); request.SetKey(object.remote_path); - auto outcome = client_ptr->DeleteObject(request); + auto outcome = client.get()->DeleteObject(request); throwIfUnexpectedError(outcome, if_exists); @@ -346,7 +339,6 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e } else { - auto client_ptr = clients.get()->client; auto settings_ptr = s3_settings.get(); size_t chunk_size_limit = settings_ptr->objects_chunk_size_to_delete; @@ -375,7 +367,7 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e S3::DeleteObjectsRequest request; request.SetBucket(bucket); request.SetDelete(delkeys); - auto outcome = client_ptr->DeleteObjects(request); + auto outcome = client.get()->DeleteObjects(request); throwIfUnexpectedError(outcome, if_exists); @@ -407,7 +399,7 @@ void S3ObjectStorage::removeObjectsIfExist(const StoredObjects & objects) std::optional S3ObjectStorage::tryGetObjectMetadata(const std::string & path) const { auto settings_ptr = s3_settings.get(); - auto object_info = S3::getObjectInfo(*clients.get()->client, bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true, /* throw_on_error= */ false); + auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true, /* throw_on_error= */ false); if (object_info.size == 0 && object_info.last_modification_time == 0 && object_info.metadata.empty()) return {}; @@ -423,7 +415,7 @@ std::optional S3ObjectStorage::tryGetObjectMetadata(const std::s ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path) const { auto settings_ptr = s3_settings.get(); - auto object_info = S3::getObjectInfo(*clients.get()->client, bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true); + auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, settings_ptr->request_settings, /* with_metadata= */ true, /* for_disk_s3= */ true); ObjectMetadata result; result.size_bytes = object_info.size; @@ -444,12 +436,12 @@ void S3ObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT /// Shortcut for S3 if (auto * dest_s3 = dynamic_cast(&object_storage_to); dest_s3 != nullptr) { - auto clients_ = clients.get(); + auto client_ = client.get(); auto settings_ptr = s3_settings.get(); - auto size = S3::getObjectSize(*clients_->client, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + auto size = S3::getObjectSize(*client_, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); auto scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "S3ObjStor_copy"); - copyS3File(clients_->client, - clients_->client_with_long_timeout, + copyS3File( + client.get(), bucket, object_from.remote_path, 0, @@ -473,12 +465,11 @@ void S3ObjectStorage::copyObject( // NOLINT const WriteSettings &, std::optional object_to_attributes) { - auto clients_ = clients.get(); + auto client_ = client.get(); auto settings_ptr = s3_settings.get(); - auto size = S3::getObjectSize(*clients_->client, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); + auto size = S3::getObjectSize(*client_, bucket, object_from.remote_path, {}, settings_ptr->request_settings, /* for_disk_s3= */ true); auto scheduler = threadPoolCallbackRunner(getThreadPoolWriter(), "S3ObjStor_copy"); - copyS3File(clients_->client, - clients_->client_with_long_timeout, + copyS3File(client_, bucket, object_from.remote_path, 0, @@ -499,31 +490,25 @@ void S3ObjectStorage::setNewSettings(std::unique_ptr && void S3ObjectStorage::shutdown() { - auto clients_ptr = clients.get(); /// This call stops any next retry attempts for ongoing S3 requests. /// If S3 request is failed and the method below is executed S3 client immediately returns the last failed S3 request outcome. /// If S3 is healthy nothing wrong will be happened and S3 requests will be processed in a regular way without errors. /// This should significantly speed up shutdown process if S3 is unhealthy. - const_cast(*clients_ptr->client).DisableRequestProcessing(); - const_cast(*clients_ptr->client_with_long_timeout).DisableRequestProcessing(); + const_cast(*client.get()).DisableRequestProcessing(); } void S3ObjectStorage::startup() { - auto clients_ptr = clients.get(); - /// Need to be enabled if it was disabled during shutdown() call. - const_cast(*clients_ptr->client).EnableRequestProcessing(); - const_cast(*clients_ptr->client_with_long_timeout).EnableRequestProcessing(); + const_cast(*client.get()).EnableRequestProcessing(); } void S3ObjectStorage::applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) { auto new_s3_settings = getSettings(config, config_prefix, context); auto new_client = getClient(config, config_prefix, context, *new_s3_settings); - auto new_clients = std::make_unique(std::move(new_client), *new_s3_settings); s3_settings.set(std::move(new_s3_settings)); - clients.set(std::move(new_clients)); + client.set(std::move(new_client)); } std::unique_ptr S3ObjectStorage::cloneObjectStorage( @@ -538,9 +523,6 @@ std::unique_ptr S3ObjectStorage::cloneObjectStorage( endpoint, object_key_prefix); } -S3ObjectStorage::Clients::Clients(std::shared_ptr client_, const S3ObjectStorageSettings & settings) - : client(std::move(client_)), client_with_long_timeout(client->clone(std::nullopt, settings.request_settings.long_request_timeout_ms)) {} - ObjectStorageKey S3ObjectStorage::generateObjectKeyForPath(const std::string &) const { /// Path to store the new S3 object. diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index b1b3fb22366f..7d14482311f0 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -39,16 +39,6 @@ struct S3ObjectStorageSettings class S3ObjectStorage : public IObjectStorage { -public: - struct Clients - { - std::shared_ptr client; - std::shared_ptr client_with_long_timeout; - - Clients() = default; - Clients(std::shared_ptr client, const S3ObjectStorageSettings & settings); - }; - private: friend class S3PlainObjectStorage; @@ -63,7 +53,7 @@ class S3ObjectStorage : public IObjectStorage String object_key_prefix_) : bucket(std::move(bucket_)) , object_key_prefix(std::move(object_key_prefix_)) - , clients(std::make_unique(std::move(client_), *s3_settings_)) + , client(std::move(client_)) , s3_settings(std::move(s3_settings_)) , s3_capabilities(s3_capabilities_) , version_id(std::move(version_id_)) @@ -184,7 +174,8 @@ class S3ObjectStorage : public IObjectStorage std::string bucket; String object_key_prefix; - MultiVersion clients; + + MultiVersion client; MultiVersion s3_settings; S3Capabilities s3_capabilities; diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index de88c876922f..0232a6eb0706 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -60,13 +60,15 @@ std::unique_ptr getClient( uri.uri.getScheme()); client_configuration.connectTimeoutMs = config.getUInt(config_prefix + ".connect_timeout_ms", 1000); - client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", 3000); + client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", 30000); client_configuration.maxConnections = config.getUInt(config_prefix + ".max_connections", 100); client_configuration.endpointOverride = uri.endpoint; - client_configuration.http_keep_alive_timeout_ms - = config.getUInt(config_prefix + ".http_keep_alive_timeout_ms", DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT * 1000); + client_configuration.http_keep_alive_timeout_ms = config.getUInt( + config_prefix + ".http_keep_alive_timeout_ms", DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT * 1000); client_configuration.http_connection_pool_size = config.getUInt(config_prefix + ".http_connection_pool_size", 1000); client_configuration.wait_on_pool_size_limit = false; + client_configuration.s3_use_adaptive_timeouts = config.getBool( + config_prefix + ".use_adaptive_timeouts", client_configuration.s3_use_adaptive_timeouts); /* * Override proxy configuration for backwards compatibility with old configuration format. diff --git a/src/IO/ConnectionTimeouts.cpp b/src/IO/ConnectionTimeouts.cpp index 01fbaa4f8172..970afc75ec3e 100644 --- a/src/IO/ConnectionTimeouts.cpp +++ b/src/IO/ConnectionTimeouts.cpp @@ -133,4 +133,86 @@ ConnectionTimeouts ConnectionTimeouts::getHTTPTimeouts(const Settings & settings settings.http_receive_timeout); } +class SendReceiveTimeoutsForFirstAttempt +{ +private: + static constexpr size_t known_methods_count = 6; + using KnownMethodsArray = std::array; + static const KnownMethodsArray known_methods; + + /// HTTP_POST is used for CompleteMultipartUpload requests. Its latency could be high. + /// These requests need longer timeout, especially when minio is used. + /// The same assumption are made for HTTP_DELETE, HTTP_PATCH + /// That requests are more heavy that HTTP_GET, HTTP_HEAD, HTTP_PUT + + static constexpr Poco::Timestamp::TimeDiff first_byte_ms[known_methods_count][2] = + { + /* GET */ {200, 200}, + /* POST */ {200, 200}, + /* DELETE */ {200, 200}, + /* PUT */ {200, 200}, + /* HEAD */ {200, 200}, + /* PATCH */ {200, 200}, + }; + + static constexpr Poco::Timestamp::TimeDiff rest_bytes_ms[known_methods_count][2] = + { + /* GET */ {500, 500}, + /* POST */ {1000, 30000}, + /* DELETE */ {1000, 10000}, + /* PUT */ {1000, 3000}, + /* HEAD */ {500, 500}, + /* PATCH */ {1000, 10000}, + }; + + static_assert(sizeof(first_byte_ms) == sizeof(rest_bytes_ms)); + static_assert(sizeof(first_byte_ms) == known_methods_count * sizeof(Poco::Timestamp::TimeDiff) * 2); + + static size_t getMethodIndex(const String & method) + { + KnownMethodsArray::const_iterator it = std::find(known_methods.begin(), known_methods.end(), method); + chassert(it != known_methods.end()); + if (it == known_methods.end()) + return 0; + return std::distance(known_methods.begin(), it); + } + +public: + static std::pair getSendReceiveTimeout(const String & method, bool first_byte) + { + auto idx = getMethodIndex(method); + + if (first_byte) + return std::make_pair( + Poco::Timespan(first_byte_ms[idx][0] * 1000), + Poco::Timespan(first_byte_ms[idx][1] * 1000) + ); + + return std::make_pair( + Poco::Timespan(rest_bytes_ms[idx][0] * 1000), + Poco::Timespan(rest_bytes_ms[idx][1] * 1000) + ); + } +}; + +const SendReceiveTimeoutsForFirstAttempt::KnownMethodsArray SendReceiveTimeoutsForFirstAttempt::known_methods = +{ + "GET", "POST", "DELETE", "PUT", "HEAD", "PATCH" +}; + + +ConnectionTimeouts ConnectionTimeouts::getAdaptiveTimeouts(const String & method, bool first_attempt, bool first_byte) const +{ + if (!first_attempt) + return *this; + + auto [send, recv] = SendReceiveTimeoutsForFirstAttempt::getSendReceiveTimeout(method, first_byte); + + auto aggressive = *this; + aggressive.send_timeout = saturate(send, send_timeout); + aggressive.receive_timeout = saturate(recv, receive_timeout); + + return aggressive; +} + } diff --git a/src/IO/ConnectionTimeouts.h b/src/IO/ConnectionTimeouts.h index 684af42827f5..aabebdb836d5 100644 --- a/src/IO/ConnectionTimeouts.h +++ b/src/IO/ConnectionTimeouts.h @@ -67,6 +67,8 @@ struct ConnectionTimeouts /// Timeouts for the case when we will try many addresses in a loop. static ConnectionTimeouts getTCPTimeoutsWithFailover(const Settings & settings); static ConnectionTimeouts getHTTPTimeouts(const Settings & settings, Poco::Timespan http_keep_alive_timeout); + + ConnectionTimeouts getAdaptiveTimeouts(const String & method, bool first_attempt, bool first_byte) const; }; } diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 65ffa51a466a..cce394c67c98 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -50,12 +50,6 @@ namespace ErrorCodes namespace { - void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts) - { - session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout); - session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout); - } - Poco::Net::HTTPClientSession::ProxyConfig proxyConfigurationToPocoProxyConfig(const ProxyConfiguration & proxy_configuration) { Poco::Net::HTTPClientSession::ProxyConfig poco_proxy_config; @@ -359,6 +353,12 @@ namespace }; } +void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts) +{ + session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout); + session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout); +} + void setResponseDefaultHeaders(HTTPServerResponse & response, size_t keep_alive_timeout) { if (!response.getKeepAlive()) diff --git a/src/IO/HTTPCommon.h b/src/IO/HTTPCommon.h index de62b5d5c165..c9968fc6915e 100644 --- a/src/IO/HTTPCommon.h +++ b/src/IO/HTTPCommon.h @@ -113,4 +113,6 @@ std::istream * receiveResponse( void assertResponseIsOk( const Poco::Net::HTTPRequest & request, Poco::Net::HTTPResponse & response, std::istream & istr, bool allow_redirects = false); + +void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts); } diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index f19978ccb471..c9c9319c44ce 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -167,9 +167,9 @@ bool ReadBufferFromS3::nextImpl() } size_t sleep_time_with_backoff_milliseconds = 100; - for (size_t attempt = 0; !next_result; ++attempt) + for (size_t attempt = 1; !next_result; ++attempt) { - bool last_attempt = attempt + 1 >= request_settings.max_single_read_retries; + bool last_attempt = attempt >= request_settings.max_single_read_retries; ProfileEventTimeIncrement watch(ProfileEvents::ReadBufferFromS3Microseconds); @@ -177,7 +177,7 @@ bool ReadBufferFromS3::nextImpl() { if (!impl) { - impl = initialize(); + impl = initialize(attempt); if (use_external_buffer) { @@ -232,9 +232,9 @@ size_t ReadBufferFromS3::readBigAt(char * to, size_t n, size_t range_begin, cons { size_t initial_n = n; size_t sleep_time_with_backoff_milliseconds = 100; - for (size_t attempt = 0; n > 0; ++attempt) + for (size_t attempt = 1; n > 0; ++attempt) { - bool last_attempt = attempt + 1 >= request_settings.max_single_read_retries; + bool last_attempt = attempt >= request_settings.max_single_read_retries; size_t bytes_copied = 0; ProfileEventTimeIncrement watch(ProfileEvents::ReadBufferFromS3Microseconds); @@ -266,7 +266,7 @@ size_t ReadBufferFromS3::readBigAt(char * to, size_t n, size_t range_begin, cons try { - result = sendRequest(range_begin, range_begin + n - 1); + result = sendRequest(attempt, range_begin, range_begin + n - 1); std::istream & istr = result->GetBody(); copyFromIStreamWithProgressCallback(istr, to, n, progress_callback, &bytes_copied); @@ -304,8 +304,8 @@ bool ReadBufferFromS3::processException(Poco::Exception & e, size_t read_offset, LOG_DEBUG( log, "Caught exception while reading S3 object. Bucket: {}, Key: {}, Version: {}, Offset: {}, " - "Attempt: {}, Message: {}", - bucket, key, version_id.empty() ? "Latest" : version_id, read_offset, attempt, e.message()); + "Attempt: {}/{}, Message: {}", + bucket, key, version_id.empty() ? "Latest" : version_id, read_offset, attempt, request_settings.max_single_read_retries, e.message()); if (auto * s3_exception = dynamic_cast(&e)) @@ -463,7 +463,7 @@ ReadBufferFromS3::~ReadBufferFromS3() } } -std::unique_ptr ReadBufferFromS3::initialize() +std::unique_ptr ReadBufferFromS3::initialize(size_t attempt) { resetSessionIfNeeded(readAllRangeSuccessfully(), read_result); read_all_range_successfully = false; @@ -475,13 +475,13 @@ std::unique_ptr ReadBufferFromS3::initialize() if (read_until_position && offset >= read_until_position) throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read beyond right offset ({} > {})", offset, read_until_position - 1); - read_result = sendRequest(offset, read_until_position ? std::make_optional(read_until_position - 1) : std::nullopt); + read_result = sendRequest(attempt, offset, read_until_position ? std::make_optional(read_until_position - 1) : std::nullopt); size_t buffer_size = use_external_buffer ? 0 : read_settings.remote_fs_buffer_size; return std::make_unique(read_result->GetBody(), buffer_size); } -Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t range_begin, std::optional range_end_incl) const +Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t attempt, size_t range_begin, std::optional range_end_incl) const { S3::GetObjectRequest req; req.SetBucket(bucket); @@ -489,6 +489,8 @@ Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t range_begin if (!version_id.empty()) req.SetVersionId(version_id); + req.SetAdditionalCustomHeaderValue("clickhouse-request", fmt::format("attempt={}", attempt)); + if (range_end_incl) { req.SetRange(fmt::format("bytes={}-{}", range_begin, *range_end_incl)); diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h index 0835e52a5b2e..101e25f8b436 100644 --- a/src/IO/ReadBufferFromS3.h +++ b/src/IO/ReadBufferFromS3.h @@ -79,7 +79,7 @@ class ReadBufferFromS3 : public ReadBufferFromFileBase bool supportsReadAt() override { return true; } private: - std::unique_ptr initialize(); + std::unique_ptr initialize(size_t attempt); /// If true, if we destroy impl now, no work was wasted. Just for metrics. bool atEndOfRequestedRangeGuess(); @@ -88,7 +88,7 @@ class ReadBufferFromS3 : public ReadBufferFromFileBase /// Returns true if the error looks retriable. bool processException(Poco::Exception & e, size_t read_offset, size_t attempt) const; - Aws::S3::Model::GetObjectResult sendRequest(size_t range_begin, std::optional range_end_incl) const; + Aws::S3::Model::GetObjectResult sendRequest(size_t attempt, size_t range_begin, std::optional range_end_incl) const; bool readAllRangeSuccessfully() const; diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index ceb7d2752992..4630e68fbb6c 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -118,16 +118,9 @@ std::unique_ptr Client::create( new Client(max_redirects_, std::move(sse_kms_config_), credentials_provider, client_configuration, sign_payloads, use_virtual_addressing)); } -std::unique_ptr Client::clone( - std::optional> override_retry_strategy, - std::optional override_request_timeout_ms) const +std::unique_ptr Client::clone() const { - PocoHTTPClientConfiguration new_configuration = client_configuration; - if (override_retry_strategy.has_value()) - new_configuration.retryStrategy = *override_retry_strategy; - if (override_request_timeout_ms.has_value()) - new_configuration.requestTimeoutMs = *override_request_timeout_ms; - return std::unique_ptr(new Client(*this, new_configuration)); + return std::unique_ptr(new Client(*this, client_configuration)); } namespace @@ -905,6 +898,7 @@ PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT s3_retry_attempts, enable_s3_requests_logging, for_disk_s3, + context->getGlobalContext()->getSettingsRef().s3_use_adaptive_timeouts, get_request_throttler, put_request_throttler, error_report); diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index 48310bc21af9..5ad57a9d8272 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -118,15 +118,7 @@ class Client : private Aws::S3::S3Client Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy sign_payloads, bool use_virtual_addressing); - /// Create a client with adjusted settings: - /// * override_retry_strategy can be used to disable retries to avoid nested retries when we have - /// a retry loop outside of S3 client. Specifically, for read and write buffers. Currently not - /// actually used. - /// * override_request_timeout_ms is used to increase timeout for CompleteMultipartUploadRequest - /// because it often sits idle for 10 seconds: https://github.com/ClickHouse/ClickHouse/pull/42321 - std::unique_ptr clone( - std::optional> override_retry_strategy = std::nullopt, - std::optional override_request_timeout_ms = std::nullopt) const; + std::unique_ptr clone() const; Client & operator=(const Client &) = delete; diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index d0f248f48a69..4a1b6def1331 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -99,6 +99,7 @@ PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( unsigned int s3_retry_attempts_, bool enable_s3_requests_logging_, bool for_disk_s3_, + bool s3_use_adaptive_timeouts_, const ThrottlerPtr & get_request_throttler_, const ThrottlerPtr & put_request_throttler_, std::function error_report_) @@ -111,6 +112,7 @@ PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( , for_disk_s3(for_disk_s3_) , get_request_throttler(get_request_throttler_) , put_request_throttler(put_request_throttler_) + , s3_use_adaptive_timeouts(s3_use_adaptive_timeouts_) , error_report(error_report_) { } @@ -157,6 +159,7 @@ PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & client_config Poco::Timespan(client_configuration.http_keep_alive_timeout_ms * 1000))) /// flag indicating whether keep-alive is enabled is set to each session upon creation , remote_host_filter(client_configuration.remote_host_filter) , s3_max_redirects(client_configuration.s3_max_redirects) + , s3_use_adaptive_timeouts(client_configuration.s3_use_adaptive_timeouts) , enable_s3_requests_logging(client_configuration.enable_s3_requests_logging) , for_disk_s3(client_configuration.for_disk_s3) , get_request_throttler(client_configuration.get_request_throttler) @@ -268,6 +271,38 @@ void PocoHTTPClient::addMetric(const Aws::Http::HttpRequest & request, S3MetricT ProfileEvents::increment(disk_s3_events_map[static_cast(type)][static_cast(kind)], amount); } +String extractAttemptFromInfo(const Aws::String & request_info) +{ + static auto key = Aws::String("attempt="); + + auto key_begin = request_info.find(key, 0); + if (key_begin == Aws::String::npos) + return "1"; + + auto val_begin = key_begin + key.size(); + auto val_end = request_info.find(';', val_begin); + if (val_end == Aws::String::npos) + val_end = request_info.size(); + + return request_info.substr(val_begin, val_end-val_begin); +} + +String getOrEmpty(const Aws::Http::HeaderValueCollection & map, const String & key) +{ + auto it = map.find(key); + if (it == map.end()) + return {}; + return it->second; +} + +ConnectionTimeouts PocoHTTPClient::getTimeouts(const String & method, bool first_attempt, bool first_byte) const +{ + if (!s3_use_adaptive_timeouts) + return timeouts; + + return timeouts.getAdaptiveTimeouts(method, first_attempt, first_byte); +} + void PocoHTTPClient::makeRequestInternal( Aws::Http::HttpRequest & request, std::shared_ptr & response, @@ -282,6 +317,25 @@ void PocoHTTPClient::makeRequestInternal( makeRequestInternalImpl(request, request_configuration, response, readLimiter, writeLimiter); } +String getMethod(const Aws::Http::HttpRequest & request) +{ + switch (request.GetMethod()) + { + case Aws::Http::HttpMethod::HTTP_GET: + return Poco::Net::HTTPRequest::HTTP_GET; + case Aws::Http::HttpMethod::HTTP_POST: + return Poco::Net::HTTPRequest::HTTP_POST; + case Aws::Http::HttpMethod::HTTP_DELETE: + return Poco::Net::HTTPRequest::HTTP_DELETE; + case Aws::Http::HttpMethod::HTTP_PUT: + return Poco::Net::HTTPRequest::HTTP_PUT; + case Aws::Http::HttpMethod::HTTP_HEAD: + return Poco::Net::HTTPRequest::HTTP_HEAD; + case Aws::Http::HttpMethod::HTTP_PATCH: + return Poco::Net::HTTPRequest::HTTP_PATCH; + } +} + template void PocoHTTPClient::makeRequestInternalImpl( Aws::Http::HttpRequest & request, @@ -295,9 +349,14 @@ void PocoHTTPClient::makeRequestInternalImpl( Poco::Logger * log = &Poco::Logger::get("AWSClient"); auto uri = request.GetUri().GetURIString(); + auto method = getMethod(request); + + auto sdk_attempt = extractAttemptFromInfo(getOrEmpty(request.GetHeaders(), Aws::Http::SDK_REQUEST_HEADER)); + auto ch_attempt = extractAttemptFromInfo(getOrEmpty(request.GetHeaders(), "clickhouse-request")); + bool first_attempt = ch_attempt == "1" && sdk_attempt == "1"; if (enable_s3_requests_logging) - LOG_TEST(log, "Make request to: {}", uri); + LOG_TEST(log, "Make request to: {}, aws sdk attempt: {}, clickhouse attempt: {}", uri, sdk_attempt, ch_attempt); switch (request.GetMethod()) { @@ -348,17 +407,29 @@ void PocoHTTPClient::makeRequestInternalImpl( /// This can lead to request signature difference on S3 side. if constexpr (pooled) session = makePooledHTTPSession( - target_uri, timeouts, http_connection_pool_size, wait_on_pool_size_limit, proxy_configuration); + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true), + http_connection_pool_size, + wait_on_pool_size_limit, + proxy_configuration); else - session = makeHTTPSession(target_uri, timeouts, proxy_configuration); + session = makeHTTPSession( + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true), + proxy_configuration); } else { if constexpr (pooled) session = makePooledHTTPSession( - target_uri, timeouts, http_connection_pool_size, wait_on_pool_size_limit); + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true), + http_connection_pool_size, + wait_on_pool_size_limit); else - session = makeHTTPSession(target_uri, timeouts); + session = makeHTTPSession( + target_uri, + getTimeouts(method, first_attempt, /*first_byte*/ true)); } /// In case of error this address will be written to logs @@ -392,28 +463,7 @@ void PocoHTTPClient::makeRequestInternalImpl( path_and_query = "/"; poco_request.setURI(path_and_query); - - switch (request.GetMethod()) - { - case Aws::Http::HttpMethod::HTTP_GET: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_GET); - break; - case Aws::Http::HttpMethod::HTTP_POST: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_POST); - break; - case Aws::Http::HttpMethod::HTTP_DELETE: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_DELETE); - break; - case Aws::Http::HttpMethod::HTTP_PUT: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_PUT); - break; - case Aws::Http::HttpMethod::HTTP_HEAD: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_HEAD); - break; - case Aws::Http::HttpMethod::HTTP_PATCH: - poco_request.setMethod(Poco::Net::HTTPRequest::HTTP_PATCH); - break; - } + poco_request.setMethod(method); /// Headers coming from SDK are lower-cased. for (const auto & [header_name, header_value] : request.GetHeaders()) @@ -438,6 +488,7 @@ void PocoHTTPClient::makeRequestInternalImpl( request.GetContentBody()->clear(); request.GetContentBody()->seekg(0); + setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); auto size = Poco::StreamCopier::copyStream(*request.GetContentBody(), request_body_stream); if (enable_s3_requests_logging) LOG_TEST(log, "Written {} bytes to request body", size); @@ -447,6 +498,8 @@ void PocoHTTPClient::makeRequestInternalImpl( LOG_TEST(log, "Receiving response..."); auto & response_body_stream = session->receiveResponse(poco_response); + setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); + watch.stop(); addMetric(request, S3MetricType::Microseconds, watch.elapsedMicroseconds()); @@ -498,6 +551,7 @@ void PocoHTTPClient::makeRequestInternalImpl( /// Request is successful but for some special requests we can have actual error message in body if (status_code >= SUCCESS_RESPONSE_MIN && status_code <= SUCCESS_RESPONSE_MAX && checkRequestCanReturn2xxAndErrorInBody(request)) { + /// reading the full response std::string response_string((std::istreambuf_iterator(response_body_stream)), std::istreambuf_iterator()); @@ -512,7 +566,6 @@ void PocoHTTPClient::makeRequestInternalImpl( addMetric(request, S3MetricType::Errors); if (error_report) error_report(proxy_configuration); - } /// Set response from string @@ -531,6 +584,8 @@ void PocoHTTPClient::makeRequestInternalImpl( if (status_code >= 500 && error_report) error_report(proxy_configuration); } + + /// expose stream, after that client reads data from that stream without built-in retries response->SetResponseBody(response_body_stream, session); } diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 2a449458360c..5178d75e7b6d 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -55,6 +55,7 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration size_t http_connection_pool_size = 0; /// See PoolBase::BehaviourOnLimit bool wait_on_pool_size_limit = true; + bool s3_use_adaptive_timeouts = true; std::function error_report; @@ -69,6 +70,7 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration unsigned int s3_retry_attempts, bool enable_s3_requests_logging_, bool for_disk_s3_, + bool s3_use_adaptive_timeouts_, const ThrottlerPtr & get_request_throttler_, const ThrottlerPtr & put_request_throttler_, std::function error_report_ @@ -169,6 +171,8 @@ class PocoHTTPClient : public Aws::Http::HttpClient Aws::Utils::RateLimits::RateLimiterInterface * readLimiter, Aws::Utils::RateLimits::RateLimiterInterface * writeLimiter) const; + ConnectionTimeouts getTimeouts(const String & method, bool first_attempt, bool first_byte) const; + protected: static S3MetricKind getMetricKind(const Aws::Http::HttpRequest & request); void addMetric(const Aws::Http::HttpRequest & request, S3MetricType type, ProfileEvents::Count amount = 1) const; @@ -178,6 +182,7 @@ class PocoHTTPClient : public Aws::Http::HttpClient ConnectionTimeouts timeouts; const RemoteHostFilter & remote_host_filter; unsigned int s3_max_redirects; + bool s3_use_adaptive_timeouts = true; bool enable_s3_requests_logging; bool for_disk_s3; diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index a16a1a415058..30da1c580c13 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -53,7 +53,6 @@ namespace public: UploadHelper( const std::shared_ptr & client_ptr_, - const std::shared_ptr & client_with_long_timeout_ptr_, const String & dest_bucket_, const String & dest_key_, const S3Settings::RequestSettings & request_settings_, @@ -62,7 +61,6 @@ namespace bool for_disk_s3_, const Poco::Logger * log_) : client_ptr(client_ptr_) - , client_with_long_timeout_ptr(client_with_long_timeout_ptr_) , dest_bucket(dest_bucket_) , dest_key(dest_key_) , request_settings(request_settings_) @@ -78,7 +76,6 @@ namespace protected: std::shared_ptr client_ptr; - std::shared_ptr client_with_long_timeout_ptr; const String & dest_bucket; const String & dest_key; const S3Settings::RequestSettings & request_settings; @@ -179,7 +176,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); - auto outcome = client_with_long_timeout_ptr->CompleteMultipartUpload(request); + auto outcome = client_ptr->CompleteMultipartUpload(request); if (outcome.IsSuccess()) { @@ -433,14 +430,13 @@ namespace size_t offset_, size_t size_, const std::shared_ptr & client_ptr_, - const std::shared_ptr & client_with_long_timeout_ptr_, const String & dest_bucket_, const String & dest_key_, const S3Settings::RequestSettings & request_settings_, const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, bool for_disk_s3_) - : UploadHelper(client_ptr_, client_with_long_timeout_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyDataToS3File")) + : UploadHelper(client_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyDataToS3File")) , create_read_buffer(create_read_buffer_) , offset(offset_) , size(size_) @@ -602,7 +598,6 @@ namespace public: CopyFileHelper( const std::shared_ptr & client_ptr_, - const std::shared_ptr & client_with_long_timeout_ptr_, const String & src_bucket_, const String & src_key_, size_t src_offset_, @@ -614,7 +609,7 @@ namespace const std::optional> & object_metadata_, ThreadPoolCallbackRunner schedule_, bool for_disk_s3_) - : UploadHelper(client_ptr_, client_with_long_timeout_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyS3File")) + : UploadHelper(client_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, for_disk_s3_, &Poco::Logger::get("copyS3File")) , src_bucket(src_bucket_) , src_key(src_key_) , offset(src_offset_) @@ -677,7 +672,7 @@ namespace /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 request.SetContentType("binary/octet-stream"); - client_with_long_timeout_ptr->setKMSHeaders(request); + client_ptr->setKMSHeaders(request); } void processCopyRequest(const S3::CopyObjectRequest & request) @@ -689,7 +684,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3CopyObject); - auto outcome = client_with_long_timeout_ptr->CopyObject(request); + auto outcome = client_ptr->CopyObject(request); if (outcome.IsSuccess()) { LOG_TRACE( @@ -714,7 +709,6 @@ namespace offset, size, client_ptr, - client_with_long_timeout_ptr, dest_bucket, dest_key, request_settings, @@ -788,7 +782,7 @@ namespace if (for_disk_s3) ProfileEvents::increment(ProfileEvents::DiskS3UploadPartCopy); - auto outcome = client_with_long_timeout_ptr->UploadPartCopy(req); + auto outcome = client_ptr->UploadPartCopy(req); if (!outcome.IsSuccess()) { abortMultipartUpload(); @@ -806,7 +800,6 @@ void copyDataToS3File( size_t offset, size_t size, const std::shared_ptr & dest_s3_client, - const std::shared_ptr & dest_s3_client_with_long_timeout, const String & dest_bucket, const String & dest_key, const S3Settings::RequestSettings & settings, @@ -814,14 +807,13 @@ void copyDataToS3File( ThreadPoolCallbackRunner schedule, bool for_disk_s3) { - CopyDataToFileHelper helper{create_read_buffer, offset, size, dest_s3_client, dest_s3_client_with_long_timeout, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3}; + CopyDataToFileHelper helper{create_read_buffer, offset, size, dest_s3_client, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3}; helper.performCopy(); } void copyS3File( const std::shared_ptr & s3_client, - const std::shared_ptr & s3_client_with_long_timeout, const String & src_bucket, const String & src_key, size_t src_offset, @@ -836,7 +828,7 @@ void copyS3File( { if (settings.allow_native_copy) { - CopyFileHelper helper{s3_client, s3_client_with_long_timeout, src_bucket, src_key, src_offset, src_size, dest_bucket, dest_key, settings, read_settings, object_metadata, schedule, for_disk_s3}; + CopyFileHelper helper{s3_client, src_bucket, src_key, src_offset, src_size, dest_bucket, dest_key, settings, read_settings, object_metadata, schedule, for_disk_s3}; helper.performCopy(); } else @@ -845,7 +837,7 @@ void copyS3File( { return std::make_unique(s3_client, src_bucket, src_key, "", settings, read_settings); }; - copyDataToS3File(create_read_buffer, src_offset, src_size, s3_client, s3_client_with_long_timeout, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3); + copyDataToS3File(create_read_buffer, src_offset, src_size, s3_client, dest_bucket, dest_key, settings, object_metadata, schedule, for_disk_s3); } } diff --git a/src/IO/S3/copyS3File.h b/src/IO/S3/copyS3File.h index 1bcbfd7735e4..33e22fdfba22 100644 --- a/src/IO/S3/copyS3File.h +++ b/src/IO/S3/copyS3File.h @@ -27,15 +27,9 @@ using CreateReadBuffer = std::function()>; /// because it is a known issue, it is fallbacks to read-write copy /// (copyDataToS3File()). /// -/// s3_client_with_long_timeout (may be equal to s3_client) is used for native copy and -/// CompleteMultipartUpload requests. These requests need longer timeout because S3 servers often -/// block on them for multiple seconds without sending or receiving data from us (maybe the servers -/// are copying data internally, or maybe throttling, idk). -/// /// read_settings - is used for throttling in case of native copy is not possible void copyS3File( const std::shared_ptr & s3_client, - const std::shared_ptr & s3_client_with_long_timeout, const String & src_bucket, const String & src_key, size_t src_offset, @@ -58,7 +52,6 @@ void copyDataToS3File( size_t offset, size_t size, const std::shared_ptr & dest_s3_client, - const std::shared_ptr & dest_s3_client_with_long_timeout, const String & dest_bucket, const String & dest_key, const S3Settings::RequestSettings & settings, diff --git a/src/IO/S3/tests/gtest_aws_s3_client.cpp b/src/IO/S3/tests/gtest_aws_s3_client.cpp index c42f14e9a53e..bff9ca6fa7b3 100644 --- a/src/IO/S3/tests/gtest_aws_s3_client.cpp +++ b/src/IO/S3/tests/gtest_aws_s3_client.cpp @@ -91,7 +91,6 @@ void doWriteRequest(std::shared_ptr client, const DB::S3:: DB::S3Settings::RequestSettings request_settings; request_settings.max_unexpected_write_error_retries = max_unexpected_write_error_retries; DB::WriteBufferFromS3 write_buffer( - client, client, uri.bucket, uri.key, @@ -171,6 +170,7 @@ TEST(IOTestAwsS3Client, AppendExtraSSECHeadersRead) "authorization: ... SignedHeaders=" "amz-sdk-invocation-id;" "amz-sdk-request;" + "clickhouse-request;" "content-type;" "host;" "x-amz-api-version;" @@ -216,6 +216,7 @@ TEST(IOTestAwsS3Client, AppendExtraSSEKMSHeadersRead) "authorization: ... SignedHeaders=" "amz-sdk-invocation-id;" "amz-sdk-request;" + "clickhouse-request;" "content-type;" "host;" "x-amz-api-version;" diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index e1b9c17efe97..62d0c80f1f28 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -77,7 +77,6 @@ struct WriteBufferFromS3::PartData WriteBufferFromS3::WriteBufferFromS3( std::shared_ptr client_ptr_, - std::shared_ptr client_with_long_timeout_ptr_, const String & bucket_, const String & key_, size_t buf_size_, @@ -92,7 +91,6 @@ WriteBufferFromS3::WriteBufferFromS3( , upload_settings(request_settings.getUploadSettings()) , write_settings(write_settings_) , client_ptr(std::move(client_ptr_)) - , client_with_long_timeout_ptr(std::move(client_with_long_timeout_ptr_)) , object_metadata(std::move(object_metadata_)) , buffer_allocation_policy(ChooseBufferPolicy(upload_settings)) , task_tracker( @@ -566,7 +564,7 @@ void WriteBufferFromS3::completeMultipartUpload() ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Stopwatch watch; - auto outcome = client_with_long_timeout_ptr->CompleteMultipartUpload(req); + auto outcome = client_ptr->CompleteMultipartUpload(req); watch.stop(); ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Microseconds, watch.elapsedMicroseconds()); diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 95148c497791..590342cc997f 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -30,8 +30,6 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase public: WriteBufferFromS3( std::shared_ptr client_ptr_, - /// for CompleteMultipartUploadRequest, because it blocks on recv() for a few seconds on big uploads - std::shared_ptr client_with_long_timeout_ptr_, const String & bucket_, const String & key_, size_t buf_size_, @@ -90,7 +88,6 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase const S3Settings::RequestSettings::PartUploadSettings & upload_settings; const WriteSettings write_settings; const std::shared_ptr client_ptr; - const std::shared_ptr client_with_long_timeout_ptr; const std::optional> object_metadata; Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3"); LogSeriesLimiterPtr limitedLog = std::make_shared(log, 1, 5); diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp index 21bdd9a6f26d..c82f97f8b201 100644 --- a/src/IO/tests/gtest_writebuffer_s3.cpp +++ b/src/IO/tests/gtest_writebuffer_s3.cpp @@ -549,7 +549,6 @@ class WBS3Test : public ::testing::Test getAsyncPolicy().setAutoExecute(false); return std::make_unique( - client, client, bucket, file_name, diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 80ee1e9339d1..bdbba5abd965 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -825,7 +825,6 @@ class StorageS3Sink : public SinkToStorage write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique( configuration_.client, - configuration_.client_with_long_timeout, bucket, key, DBMS_DEFAULT_BUFFER_SIZE, @@ -1330,8 +1329,6 @@ void StorageS3::Configuration::connect(ContextPtr context) context->getConfigRef().getUInt64("s3.expiration_window_seconds", S3::DEFAULT_EXPIRATION_WINDOW_SECONDS)), auth_settings.no_sign_request.value_or(context->getConfigRef().getBool("s3.no_sign_request", false)), }); - - client_with_long_timeout = client->clone(std::nullopt, request_settings.long_request_timeout_ms); } void StorageS3::processNamedCollectionResult(StorageS3::Configuration & configuration, const NamedCollection & collection) diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 3330ac6c210e..3f35c578e19b 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -311,7 +311,6 @@ class StorageS3 : public IStorage HTTPHeaderEntries headers_from_ast; std::shared_ptr client; - std::shared_ptr client_with_long_timeout; std::vector keys; }; diff --git a/src/Storages/StorageS3Settings.h b/src/Storages/StorageS3Settings.h index e3d577ca0b36..728972c948c9 100644 --- a/src/Storages/StorageS3Settings.h +++ b/src/Storages/StorageS3Settings.h @@ -69,8 +69,7 @@ struct S3Settings ThrottlerPtr get_request_throttler; ThrottlerPtr put_request_throttler; size_t retry_attempts = 10; - size_t request_timeout_ms = 3000; - size_t long_request_timeout_ms = 30000; // TODO: Take this from config like request_timeout_ms + size_t request_timeout_ms = 30000; bool allow_native_copy = true; bool throw_on_zero_files_match = false; diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml index 206eb4f2badb..4210c13b727c 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/inf_s3_retries.xml @@ -4,6 +4,7 @@ 1000000 + 1 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml index 556bf60d3853..95a313ea4f27 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/s3_retries.xml @@ -4,6 +4,7 @@ 5 + 0 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml b/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml index b77e72d808b6..7b1f503ed553 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml +++ b/tests/integration/test_checking_s3_blobs_paranoid/configs/storage_conf.xml @@ -7,11 +7,18 @@ + + s3 + http://minio1:9001/root/data/ + minio + minio123 + s3 http://resolver:8083/root/data/ minio minio123 + 1 @@ -23,9 +30,16 @@ + + +
+ s3 +
+ + - broken_s3 + s3 diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py index d6bcb3fb8f4b..b000ccabcf47 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/test.py +++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py @@ -64,6 +64,8 @@ def test_upload_after_check_works(cluster, broken_s3): data String ) ENGINE=MergeTree() ORDER BY id + SETTINGS + storage_policy='broken_s3' """ ) @@ -78,7 +80,7 @@ def test_upload_after_check_works(cluster, broken_s3): assert "suddenly disappeared" in error, error -def get_counters(node, query_id, log_type="ExceptionWhileProcessing"): +def get_multipart_counters(node, query_id, log_type="ExceptionWhileProcessing"): node.query("SYSTEM FLUSH LOGS") return [ int(x) @@ -87,7 +89,25 @@ def get_counters(node, query_id, log_type="ExceptionWhileProcessing"): SELECT ProfileEvents['S3CreateMultipartUpload'], ProfileEvents['S3UploadPart'], - ProfileEvents['S3WriteRequestsErrors'] + ProfileEvents['S3WriteRequestsErrors'], + FROM system.query_log + WHERE query_id='{query_id}' + AND type='{log_type}' + """ + ).split() + if x + ] + + +def get_put_counters(node, query_id, log_type="ExceptionWhileProcessing"): + node.query("SYSTEM FLUSH LOGS") + return [ + int(x) + for x in node.query( + f""" + SELECT + ProfileEvents['S3PutObject'], + ProfileEvents['S3WriteRequestsErrors'], FROM system.query_log WHERE query_id='{query_id}' AND type='{log_type}' @@ -129,12 +149,12 @@ def test_upload_s3_fail_create_multi_part_upload(cluster, broken_s3, compression assert "Code: 499" in error, error assert "mock s3 injected error" in error, error - count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( + create_multipart, upload_parts, s3_errors = get_multipart_counters( node, insert_query_id ) - assert count_create_multi_part_uploads == 1 - assert count_upload_parts == 0 - assert count_s3_errors == 1 + assert create_multipart == 1 + assert upload_parts == 0 + assert s3_errors == 1 # Add "lz4" compression method in the list after https://github.com/ClickHouse/ClickHouse/issues/50975 is fixed @@ -172,12 +192,12 @@ def test_upload_s3_fail_upload_part_when_multi_part_upload( assert "Code: 499" in error, error assert "mock s3 injected error" in error, error - count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( + create_multipart, upload_parts, s3_errors = get_multipart_counters( node, insert_query_id ) - assert count_create_multi_part_uploads == 1 - assert count_upload_parts >= 2 - assert count_s3_errors >= 2 + assert create_multipart == 1 + assert upload_parts >= 2 + assert s3_errors >= 2 def test_when_s3_connection_refused_is_retried(cluster, broken_s3): @@ -207,12 +227,12 @@ def test_when_s3_connection_refused_is_retried(cluster, broken_s3): query_id=insert_query_id, ) - count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( + create_multipart, upload_parts, s3_errors = get_multipart_counters( node, insert_query_id, log_type="QueryFinish" ) - assert count_create_multi_part_uploads == 1 - assert count_upload_parts == 39 - assert count_s3_errors == 3 + assert create_multipart == 1 + assert upload_parts == 39 + assert s3_errors == 3 broken_s3.setup_at_part_upload(count=1000, after=2, action="connection_refused") insert_query_id = f"INSERT_INTO_TABLE_FUNCTION_CONNECTION_REFUSED_RETRIED_1" @@ -279,13 +299,13 @@ def test_when_s3_connection_reset_by_peer_at_upload_is_retried( query_id=insert_query_id, ) - count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( + create_multipart, upload_parts, s3_errors = get_multipart_counters( node, insert_query_id, log_type="QueryFinish" ) - assert count_create_multi_part_uploads == 1 - assert count_upload_parts == 39 - assert count_s3_errors == 3 + assert create_multipart == 1 + assert upload_parts == 39 + assert s3_errors == 3 broken_s3.setup_at_part_upload( count=1000, @@ -361,13 +381,13 @@ def test_when_s3_connection_reset_by_peer_at_create_mpu_retried( query_id=insert_query_id, ) - count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( + create_multipart, upload_parts, s3_errors = get_multipart_counters( node, insert_query_id, log_type="QueryFinish" ) - assert count_create_multi_part_uploads == 1 - assert count_upload_parts == 39 - assert count_s3_errors == 3 + assert create_multipart == 1 + assert upload_parts == 39 + assert s3_errors == 3 broken_s3.setup_at_create_multi_part_upload( count=1000, @@ -438,13 +458,13 @@ def test_when_s3_broken_pipe_at_upload_is_retried(cluster, broken_s3): query_id=insert_query_id, ) - count_create_multi_part_uploads, count_upload_parts, count_s3_errors = get_counters( + create_multipart, upload_parts, s3_errors = get_multipart_counters( node, insert_query_id, log_type="QueryFinish" ) - assert count_create_multi_part_uploads == 1 - assert count_upload_parts == 7 - assert count_s3_errors == 3 + assert create_multipart == 1 + assert upload_parts == 7 + assert s3_errors == 3 broken_s3.setup_at_part_upload( count=1000, @@ -533,3 +553,60 @@ def test_query_is_canceled_with_inf_retries(cluster, broken_s3): retry_count=120, sleep_time=1, ) + + +@pytest.mark.parametrize("node_name", ["node", "node_with_inf_s3_retries"]) +def test_adaptive_timeouts(cluster, broken_s3, node_name): + node = cluster.instances[node_name] + + broken_s3.setup_fake_puts(part_length=1) + broken_s3.setup_slow_answers( + timeout=5, + count=1000000, + ) + + insert_query_id = f"TEST_ADAPTIVE_TIMEOUTS_{node_name}" + node.query( + f""" + INSERT INTO + TABLE FUNCTION s3( + 'http://resolver:8083/root/data/adaptive_timeouts', + 'minio', 'minio123', + 'CSV', auto, 'none' + ) + SELECT + * + FROM system.numbers + LIMIT 1 + SETTINGS + s3_request_timeout_ms=30000, + s3_check_objects_after_upload=0 + """, + query_id=insert_query_id, + ) + + broken_s3.reset() + + put_objects, s3_errors = get_put_counters( + node, insert_query_id, log_type="QueryFinish" + ) + + assert put_objects == 1 + + s3_use_adaptive_timeouts = node.query( + f""" + SELECT + value + FROM system.settings + WHERE + name='s3_use_adaptive_timeouts' + """ + ).strip() + + if node_name == "node_with_inf_s3_retries": + # first 2 attempts failed + assert s3_use_adaptive_timeouts == "1" + assert s3_errors == 1 + else: + assert s3_use_adaptive_timeouts == "0" + assert s3_errors == 0 diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml index 235b9a7b7a12..6303e9273fc3 100644 --- a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml @@ -11,6 +11,7 @@ true 0 + 0 20000 @@ -33,6 +34,7 @@ true 1 + 0 1 20000 diff --git a/tests/integration/test_storage_s3/configs/defaultS3.xml b/tests/integration/test_storage_s3/configs/defaultS3.xml index 37454ef6781f..7dac6d9fbb57 100644 --- a/tests/integration/test_storage_s3/configs/defaultS3.xml +++ b/tests/integration/test_storage_s3/configs/defaultS3.xml @@ -1,9 +1,4 @@ - - - 5 - - http://resolver:8080 diff --git a/tests/integration/test_storage_s3/configs/s3_retry.xml b/tests/integration/test_storage_s3/configs/s3_retry.xml index 727e23273cf3..3171da051d0c 100644 --- a/tests/integration/test_storage_s3/configs/s3_retry.xml +++ b/tests/integration/test_storage_s3/configs/s3_retry.xml @@ -1,7 +1,9 @@ - 5 + 1 + 10 + 5 diff --git a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py index 103dd30340ce..5ef781bdc9e2 100644 --- a/tests/integration/test_storage_s3/s3_mocks/unstable_server.py +++ b/tests/integration/test_storage_s3/s3_mocks/unstable_server.py @@ -4,6 +4,7 @@ import socket import struct import sys +import time def gen_n_digit_number(n): @@ -39,14 +40,14 @@ def add_number(): # Generating some "random" data and append a line which contains sum of numbers in column 4. lines = ( - b"".join((gen_line() for _ in range(500000))) + b"".join([gen_line() for _ in range(500000)]) + f"0,0,0,{-sum_in_4_column}\n".encode() ) class RequestHandler(http.server.BaseHTTPRequestHandler): def do_HEAD(self): - if self.path == "/root/test.csv": + if self.path == "/root/test.csv" or self.path == "/root/slow_send_test.csv": self.from_bytes = 0 self.end_bytes = len(lines) self.size = self.end_bytes @@ -101,6 +102,18 @@ def do_GET(self): print("Dropping connection") break + if self.path == "/root/slow_send_test.csv": + self.send_block_size = 81920 + + for c, i in enumerate( + range(self.from_bytes, self.end_bytes, self.send_block_size) + ): + self.wfile.write( + lines[i : min(i + self.send_block_size, self.end_bytes)] + ) + self.wfile.flush() + time.sleep(1) + elif self.path == "/": self.wfile.write(b"OK") diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 3dd3c9e39d02..835c8b908f0f 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -818,6 +818,15 @@ def test_storage_s3_get_unstable(started_cluster): assert result.splitlines() == ["500001,500000,0"] +def test_storage_s3_get_slow(started_cluster): + bucket = started_cluster.minio_bucket + instance = started_cluster.instances["dummy"] + table_format = "column1 Int64, column2 Int64, column3 Int64, column4 Int64" + get_query = f"SELECT count(), sum(column3), sum(column4) FROM s3('http://resolver:8081/{started_cluster.minio_bucket}/slow_send_test.csv', 'CSV', '{table_format}') FORMAT CSV" + result = run_query(instance, get_query) + assert result.splitlines() == ["500001,500000,0"] + + def test_storage_s3_put_uncompressed(started_cluster): bucket = started_cluster.minio_bucket instance = started_cluster.instances["dummy"] From 4c7daf51675fda3f1ef02481b25ba9f572132f58 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:55:23 +0100 Subject: [PATCH 226/274] Fix --- tests/integration/test_storage_s3_queue/test.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index ec27b7326344..b1163a549b11 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -717,6 +717,8 @@ def test_multiple_tables_streaming_sync_distributed(started_cluster, mode): keeper_path = f"/clickhouse/test_{table_name}" files_path = f"{table_name}_data" files_to_generate = 300 + row_num = 50 + total_rows = row_num * files_to_generate for instance in [node, node_2]: create_table( @@ -734,7 +736,7 @@ def test_multiple_tables_streaming_sync_distributed(started_cluster, mode): create_mv(instance, table_name, dst_table_name) total_values = generate_random_files( - started_cluster, files_path, files_to_generate, row_num=50 + started_cluster, files_path, files_to_generate, row_num=row_num ) def get_count(node, table_name): @@ -743,13 +745,13 @@ def get_count(node, table_name): for _ in range(150): if ( get_count(node, dst_table_name) + get_count(node_2, dst_table_name) - ) == files_to_generate: + ) == total_rows: break time.sleep(1) if ( get_count(node, dst_table_name) + get_count(node_2, dst_table_name) - ) != files_to_generate: + ) != total_rows: info = node.query( f"SELECT * FROM system.s3queue WHERE zookeeper_path like '%{table_name}' ORDER BY file_name FORMAT Vertical" ) @@ -762,7 +764,7 @@ def get_count(node, table_name): list(map(int, l.split())) for l in run_query(node_2, get_query).splitlines() ] - assert len(res1) + len(res2) == files_to_generate + assert len(res1) + len(res2) == total_rows # Checking that all engines have made progress assert len(res1) > 0 @@ -774,7 +776,7 @@ def get_count(node, table_name): time.sleep(10) assert ( get_count(node, dst_table_name) + get_count(node_2, dst_table_name) - ) == files_to_generate + ) == total_rows def test_max_set_age(started_cluster): From a7fc8d4b997359da08cc46fbf66f8aad1de42ed9 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Mon, 20 Nov 2023 15:04:14 +0100 Subject: [PATCH 227/274] test_merge_tree_s3 counts errors, turn off s3_use_adaptive_timeouts --- tests/integration/test_merge_tree_s3/configs/config.d/users.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_merge_tree_s3/configs/config.d/users.xml b/tests/integration/test_merge_tree_s3/configs/config.d/users.xml index 3daa6f06a789..79e5091b28ae 100644 --- a/tests/integration/test_merge_tree_s3/configs/config.d/users.xml +++ b/tests/integration/test_merge_tree_s3/configs/config.d/users.xml @@ -3,6 +3,7 @@ 1 20 + 0 From 6da51942ebe3e01b3ff2c565e87dc97c3441fc79 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 20 Nov 2023 11:57:46 +0100 Subject: [PATCH 228/274] Follow up the fix from #44311 --- tests/ci/build_check.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 3a20ca846a19..34a5956b777e 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -128,18 +128,16 @@ def check_for_success_run( version: ClickHouseVersion, ) -> None: # TODO: Remove after S3 artifacts - # the final empty argument is necessary for distinguish build and build_suffix - logged_prefix = "/".join((S3_BUILDS_BUCKET, s3_prefix, "")) - logging.info("Checking for artifacts in %s", logged_prefix) + logging.info("Checking for artifacts %s in bucket %s", s3_prefix, S3_BUILDS_BUCKET) try: # Performance artifacts are now part of regular build, so we're safe build_results = s3_helper.list_prefix(s3_prefix) except Exception as ex: - logging.info("Got exception while listing %s: %s\nRerun", logged_prefix, ex) + logging.info("Got exception while listing %s: %s\nRerun", s3_prefix, ex) return if build_results is None or len(build_results) == 0: - logging.info("Nothing found in %s, rerun", logged_prefix) + logging.info("Nothing found in %s, rerun", s3_prefix) return logging.info("Some build results found:\n%s", build_results) @@ -254,7 +252,9 @@ def main(): # If this is rerun, then we try to find already created artifacts and just # put them as github actions artifact (result) - check_for_success_run(s3_helper, s3_path_prefix, build_name, version) + # The s3_path_prefix has additional "/" in the end to prevent finding + # e.g. `binary_darwin_aarch64/clickhouse` for `binary_darwin` + check_for_success_run(s3_helper, f"{s3_path_prefix}/", build_name, version) docker_image = get_image_with_version(IMAGES_PATH, IMAGE_NAME) image_version = docker_image.version From 94824a01945edf76cc8a7e9af3611f97b9620ff9 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 20 Nov 2023 14:21:38 +0100 Subject: [PATCH 229/274] Fix shellcheck for time-trace --- utils/prepare-time-trace/prepare-time-trace.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/prepare-time-trace/prepare-time-trace.sh b/utils/prepare-time-trace/prepare-time-trace.sh index 5f4aad4c0b97..812928e8bd8c 100755 --- a/utils/prepare-time-trace/prepare-time-trace.sh +++ b/utils/prepare-time-trace/prepare-time-trace.sh @@ -8,7 +8,7 @@ # See also https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview -< Date: Mon, 20 Nov 2023 14:22:02 +0100 Subject: [PATCH 230/274] Fix logging for profile JSON files --- tests/ci/build_check.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 34a5956b777e..adbd67bd95a5 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -426,7 +426,9 @@ def main(): url = f"https://{ci_logs_credentials.host}/" profiles_dir = temp_path / "profiles_source" profiles_dir.mkdir(parents=True, exist_ok=True) - logging.info("Processing profile JSON files from {GIT_REPO_ROOT}/build_docker") + logging.info( + "Processing profile JSON files from %s", repo_path / "build_docker" + ) git_runner( "./utils/prepare-time-trace/prepare-time-trace.sh " f"build_docker {profiles_dir.absolute()}" From 8fef863b95a6812097355558c57d04231624c030 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 20 Nov 2023 21:46:32 +0800 Subject: [PATCH 231/274] Fix flaky and slow tests. --- .../01710_projection_aggregation_in_order.sql | 6 ++---- .../0_stateless/02516_projections_with_rollup.sql | 12 +++++------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql b/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql index e4fb1816c897..06f192adb57f 100644 --- a/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql +++ b/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql @@ -1,5 +1,3 @@ --- Tags: disabled --- FIXME https://github.com/ClickHouse/ClickHouse/issues/49552 -- Test that check the correctness of the result for optimize_aggregation_in_order and projections, -- not that this optimization will take place. @@ -20,7 +18,7 @@ CREATE TABLE normal ) ) ENGINE = MergeTree -ORDER BY (key, ts); +ORDER BY tuple(); INSERT INTO normal SELECT number, @@ -52,7 +50,7 @@ CREATE TABLE agg ) ) ENGINE = MergeTree -ORDER BY (key, ts); +ORDER BY tuple(); INSERT INTO agg SELECT 1, diff --git a/tests/queries/0_stateless/02516_projections_with_rollup.sql b/tests/queries/0_stateless/02516_projections_with_rollup.sql index 038caf592647..a87621073af8 100644 --- a/tests/queries/0_stateless/02516_projections_with_rollup.sql +++ b/tests/queries/0_stateless/02516_projections_with_rollup.sql @@ -1,6 +1,3 @@ --- Tags: disabled --- FIXME https://github.com/ClickHouse/ClickHouse/issues/49552 - DROP TABLE IF EXISTS video_log; DROP TABLE IF EXISTS video_log_result__fuzz_0; DROP TABLE IF EXISTS rng; @@ -16,7 +13,8 @@ CREATE TABLE video_log ) ENGINE = MergeTree PARTITION BY toDate(datetime) -ORDER BY (user_id, device_id); +ORDER BY (user_id, device_id) +SETTINGS index_granularity_bytes=10485760, index_granularity=8192; CREATE TABLE video_log_result__fuzz_0 ( @@ -62,7 +60,7 @@ LIMIT 10; ALTER TABLE video_log ADD PROJECTION p_norm ( - SELECT + SELECT datetime, device_id, bytes, @@ -77,12 +75,12 @@ SETTINGS mutations_sync = 1; ALTER TABLE video_log ADD PROJECTION p_agg ( - SELECT + SELECT toStartOfHour(datetime) AS hour, domain, sum(bytes), avg(duration) - GROUP BY + GROUP BY hour, domain ); From 8bff421c279e38ecd6bc5a77dd7933398fe2140e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 20 Nov 2023 16:04:21 +0100 Subject: [PATCH 232/274] Skip test in fast tests --- tests/queries/0_stateless/02918_sqlite_path_check.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02918_sqlite_path_check.sh b/tests/queries/0_stateless/02918_sqlite_path_check.sh index 1f250387a71d..798efda6ec1c 100755 --- a/tests/queries/0_stateless/02918_sqlite_path_check.sh +++ b/tests/queries/0_stateless/02918_sqlite_path_check.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: Fast tests don't build external libraries (SQLite) CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From 59c251bf88628951b2da442780ac91b19e9a3600 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Mon, 20 Nov 2023 16:17:31 +0100 Subject: [PATCH 233/274] Allow to use concat with a single argument --- src/Functions/concat.cpp | 6 ++++-- tests/queries/0_stateless/00727_concat.reference | 2 ++ tests/queries/0_stateless/00727_concat.sql | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index f426f662868e..6403c4b8416a 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -207,6 +207,8 @@ class ConcatOverloadResolver : public IFunctionOverloadResolver FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & return_type) const override { + if (arguments.size() == 1) + return FunctionFactory::instance().getImpl("toString", context)->build(arguments); if (std::ranges::all_of(arguments, [](const auto & elem) { return isArray(elem.type); })) return FunctionFactory::instance().getImpl("arrayConcat", context)->build(arguments); if (std::ranges::all_of(arguments, [](const auto & elem) { return isMap(elem.type); })) @@ -221,10 +223,10 @@ class ConcatOverloadResolver : public IFunctionOverloadResolver DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - if (arguments.size() < 2) + if (arguments.empty()) throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Number of arguments for function {} doesn't match: passed {}, should be at least 2.", + "Number of arguments for function {} doesn't match: passed {}, should be at least 1.", getName(), arguments.size()); diff --git a/tests/queries/0_stateless/00727_concat.reference b/tests/queries/0_stateless/00727_concat.reference index 1e102051fd0f..9659405ea3b5 100644 --- a/tests/queries/0_stateless/00727_concat.reference +++ b/tests/queries/0_stateless/00727_concat.reference @@ -64,4 +64,6 @@ Three arguments test 42144255 42144 42144255 +42 +foo Testing the alias diff --git a/tests/queries/0_stateless/00727_concat.sql b/tests/queries/0_stateless/00727_concat.sql index edeaf9340ddc..94cdc83d51fd 100644 --- a/tests/queries/0_stateless/00727_concat.sql +++ b/tests/queries/0_stateless/00727_concat.sql @@ -82,8 +82,9 @@ SELECT concat(materialize(42 :: Int32), materialize(144 :: UInt64)); SELECT concat(materialize(42 :: Int32), materialize(144 :: UInt64), materialize(255 :: UInt32)); SELECT concat(42, 144); SELECT concat(42, 144, 255); +SELECT concat(42); +SELECT concat('foo'); SELECT CONCAT('Testing the ', 'alias'); SELECT concat(); -- { serverError 42 } -SELECT concat(1); -- { serverError 42 } From 9f96b5897979effb67ff9b1eb233622494bb5e1c Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Mon, 20 Nov 2023 16:22:47 +0100 Subject: [PATCH 234/274] Update docs, add more tests. --- docs/en/sql-reference/functions/string-functions.md | 2 +- tests/queries/0_stateless/00727_concat.reference | 3 +++ tests/queries/0_stateless/00727_concat.sql | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 4b6e03563013..1940993ce0b1 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -439,7 +439,7 @@ concat(s1, s2, ...) **Arguments** -At least two values of arbitrary type. +At least one value of arbitrary type. Arguments which are not of types [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md) are converted to strings using their default serialization. As this decreases performance, it is not recommended to use non-String/FixedString arguments. diff --git a/tests/queries/0_stateless/00727_concat.reference b/tests/queries/0_stateless/00727_concat.reference index 9659405ea3b5..0ce6ac247c5f 100644 --- a/tests/queries/0_stateless/00727_concat.reference +++ b/tests/queries/0_stateless/00727_concat.reference @@ -64,6 +64,9 @@ Three arguments test 42144255 42144 42144255 +-- Single argument tests 42 foo +\N +\N Testing the alias diff --git a/tests/queries/0_stateless/00727_concat.sql b/tests/queries/0_stateless/00727_concat.sql index 94cdc83d51fd..2d0e6fe6a1f8 100644 --- a/tests/queries/0_stateless/00727_concat.sql +++ b/tests/queries/0_stateless/00727_concat.sql @@ -82,8 +82,12 @@ SELECT concat(materialize(42 :: Int32), materialize(144 :: UInt64)); SELECT concat(materialize(42 :: Int32), materialize(144 :: UInt64), materialize(255 :: UInt32)); SELECT concat(42, 144); SELECT concat(42, 144, 255); + +SELECT '-- Single argument tests'; SELECT concat(42); SELECT concat('foo'); +SELECT concat(NULL); +SELECT concat(materialize(NULL :: Nullable(UInt64))); SELECT CONCAT('Testing the ', 'alias'); From 9b5e180995c39b4f6c6549a500d0a05f749c4257 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 20 Nov 2023 16:27:02 +0100 Subject: [PATCH 235/274] Fix --- src/Storages/S3Queue/StorageS3Queue.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Storages/S3Queue/StorageS3Queue.cpp b/src/Storages/S3Queue/StorageS3Queue.cpp index 72e74d3c2a02..5d1e69f8b157 100644 --- a/src/Storages/S3Queue/StorageS3Queue.cpp +++ b/src/Storages/S3Queue/StorageS3Queue.cpp @@ -118,7 +118,11 @@ StorageS3Queue::StorageS3Queue( , reschedule_processing_interval_ms(s3queue_settings->s3queue_polling_min_timeout_ms) , log(&Poco::Logger::get("StorageS3Queue (" + table_id_.table_name + ")")) { - if (configuration.url.key.ends_with('/')) + if (configuration.url.key.empty()) + { + configuration.url.key = "/*"; + } + else if (configuration.url.key.ends_with('/')) { configuration.url.key += '*'; } From 23ea802bd7fe333ab72265d9bb2cac49b83e2495 Mon Sep 17 00:00:00 2001 From: slvrtrn Date: Mon, 20 Nov 2023 16:28:04 +0100 Subject: [PATCH 236/274] Add more concat tests --- tests/queries/0_stateless/00727_concat.reference | 2 ++ tests/queries/0_stateless/00727_concat.sql | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/00727_concat.reference b/tests/queries/0_stateless/00727_concat.reference index 0ce6ac247c5f..6fb23c072d33 100644 --- a/tests/queries/0_stateless/00727_concat.reference +++ b/tests/queries/0_stateless/00727_concat.reference @@ -66,6 +66,8 @@ Three arguments test 42144255 -- Single argument tests 42 +42 +foo foo \N \N diff --git a/tests/queries/0_stateless/00727_concat.sql b/tests/queries/0_stateless/00727_concat.sql index 2d0e6fe6a1f8..f5048dcaaae9 100644 --- a/tests/queries/0_stateless/00727_concat.sql +++ b/tests/queries/0_stateless/00727_concat.sql @@ -85,7 +85,9 @@ SELECT concat(42, 144, 255); SELECT '-- Single argument tests'; SELECT concat(42); +SELECT concat(materialize(42)); SELECT concat('foo'); +SELECT concat(materialize('foo')); SELECT concat(NULL); SELECT concat(materialize(NULL :: Nullable(UInt64))); From d84d5692ef3d78195559ae9988da61ea72026e00 Mon Sep 17 00:00:00 2001 From: Kevin Mingtarja Date: Mon, 20 Nov 2023 23:52:22 +0800 Subject: [PATCH 237/274] Enable implicit constraint for VersionedCollapsing --- .../MergeTree/registerStorageMergeTree.cpp | 4 ++- ...onstraints_for_collapsing_engine.reference | 2 ++ ...olumn_constraints_for_collapsing_engine.sh | 32 +++++++++++++++++-- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 9285dfcdd91f..ce394d682fc8 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -620,7 +620,9 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (args.query.columns_list && args.query.columns_list->constraints) for (auto & constraint : args.query.columns_list->constraints->children) constraints.push_back(constraint); - if (merging_params.mode == MergeTreeData::MergingParams::Collapsing && storage_settings->add_implicit_sign_column_constraint_for_collapsing_engine) + if ((merging_params.mode == MergeTreeData::MergingParams::Collapsing || + merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing) && + storage_settings->add_implicit_sign_column_constraint_for_collapsing_engine) { auto sign_column_check_constraint = std::make_unique(); sign_column_check_constraint->name = "check_sign_column"; diff --git a/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.reference b/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.reference index 5c6c001014d3..87fb3b0e8c47 100644 --- a/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.reference +++ b/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.reference @@ -1,2 +1,4 @@ 1 2504 1 ok +1 200 1 1 +ok \ No newline at end of file diff --git a/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.sh b/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.sh index bee12afc5112..43594a45a1e2 100755 --- a/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.sh +++ b/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.sh @@ -4,13 +4,16 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +EXCEPTION_TEXT="VIOLATED_CONSTRAINT" EXCEPTION_SUCCESS_TEXT=ok -$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS collapsing_merge_tree;" # CollapsingSortedAlgorithm::merge() also has a check for sign column value # optimize_on_insert = 0 is required to avoid this automatic merge behavior -$CLICKHOUSE_CLIENT --query="SET optimize_on_insert = 0;" +$CLICKHOUSE_CLIENT --query="SET optimize_on_insert=0;" + +# CollapsingMergeTree +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS collapsing_merge_tree;" $CLICKHOUSE_CLIENT --query="CREATE TABLE collapsing_merge_tree ( Key UInt32, @@ -26,6 +29,29 @@ $CLICKHOUSE_CLIENT --query="SELECT * FROM collapsing_merge_tree;" # Should throw an exception $CLICKHOUSE_CLIENT --query="INSERT INTO collapsing_merge_tree VALUES (1, 2504, 5);" 2>&1 \ - | grep -q VIOLATED_CONSTRAINT && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not throw an exception" + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not throw an exception" $CLICKHOUSE_CLIENT --query="DROP TABLE collapsing_merge_tree;" + + +# VersionedCollapsingMergeTree +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS versioned_collapsing_merge_tree;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE versioned_collapsing_merge_tree +( + Key UInt32, + Count UInt8, + Sign Int8, + Version UInt8 +) +ENGINE=VersionedCollapsingMergeTree(Sign, Version) ORDER BY Key +SETTINGS add_implicit_sign_column_constraint_for_collapsing_engine=1;" + +# Should succeed +$CLICKHOUSE_CLIENT --query="INSERT INTO versioned_collapsing_merge_tree VALUES (1, 2504, 1, 1);" +$CLICKHOUSE_CLIENT --query="SELECT * FROM versioned_collapsing_merge_tree;" + +# Should throw an exception +$CLICKHOUSE_CLIENT --query="INSERT INTO versioned_collapsing_merge_tree VALUES (1, 2504, 5, 1);" 2>&1 \ + | grep -q "$EXCEPTION_TEXT" && echo "$EXCEPTION_SUCCESS_TEXT" || echo "Did not throw an exception" + +$CLICKHOUSE_CLIENT --query="DROP TABLE versioned_collapsing_merge_tree;" From 1d668264e11eafa184dd16a927a1897b2297dfd0 Mon Sep 17 00:00:00 2001 From: Kevin Mingtarja Date: Mon, 20 Nov 2023 23:54:36 +0800 Subject: [PATCH 238/274] Fix style --- src/Storages/MergeTree/registerStorageMergeTree.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index ce394d682fc8..3310b8cb72df 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -620,8 +620,8 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (args.query.columns_list && args.query.columns_list->constraints) for (auto & constraint : args.query.columns_list->constraints->children) constraints.push_back(constraint); - if ((merging_params.mode == MergeTreeData::MergingParams::Collapsing || - merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing) && + if ((merging_params.mode == MergeTreeData::MergingParams::Collapsing || + merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing) && storage_settings->add_implicit_sign_column_constraint_for_collapsing_engine) { auto sign_column_check_constraint = std::make_unique(); From d5a4580236b98bbcf1ecfed4d61b6b32c8de34ec Mon Sep 17 00:00:00 2001 From: Kevin Mingtarja Date: Tue, 21 Nov 2023 00:13:53 +0800 Subject: [PATCH 239/274] Add newline in test reference --- ...icit_sign_column_constraints_for_collapsing_engine.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.reference b/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.reference index 87fb3b0e8c47..323b12c173a2 100644 --- a/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.reference +++ b/tests/queries/0_stateless/02918_implicit_sign_column_constraints_for_collapsing_engine.reference @@ -1,4 +1,4 @@ 1 2504 1 ok 1 200 1 1 -ok \ No newline at end of file +ok From ff438112e95568bb322c9ff8283e6b7f2e1b412a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 20 Nov 2023 18:08:39 +0100 Subject: [PATCH 240/274] Style --- tests/queries/0_stateless/02918_sqlite_path_check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02918_sqlite_path_check.sh b/tests/queries/0_stateless/02918_sqlite_path_check.sh index 798efda6ec1c..fa74b9ecfc88 100755 --- a/tests/queries/0_stateless/02918_sqlite_path_check.sh +++ b/tests/queries/0_stateless/02918_sqlite_path_check.sh @@ -12,4 +12,4 @@ function get_exception_message() } get_exception_message "Select * from sqlite('/etc/passwd', 'something');" -get_exception_message "Select * from sqlite('../../../../etc/passwd', 'something'); +get_exception_message "Select * from sqlite('../../../../etc/passwd', 'something');" From 897cd06bcf2a7e75998133cad9a5ad668366231b Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Mon, 20 Nov 2023 19:15:49 +0100 Subject: [PATCH 241/274] Fix dropping tables in test "test_create_or_drop_tables_during_backup". --- .../test_backup_restore_on_cluster/test_concurrency.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_backup_restore_on_cluster/test_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_concurrency.py index aea82c6b559d..ab37846db9a0 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_concurrency.py +++ b/tests/integration/test_backup_restore_on_cluster/test_concurrency.py @@ -214,7 +214,13 @@ def drop_tables(): while time.time() < end_time: table_name = f"mydb.tbl{randint(1, num_nodes)}" node = nodes[randint(0, num_nodes - 1)] - node.query(f"DROP TABLE IF EXISTS {table_name} SYNC") + # "DROP TABLE IF EXISTS" still can throw some errors (e.g. "WRITE locking attempt on node0 has timed out!") + # So we use query_and_get_answer_with_error() to ignore any errors. + # `lock_acquire_timeout` is also reduced because we don't wait our test to wait too long. + node.query_and_get_answer_with_error( + f"DROP TABLE IF EXISTS {table_name} SYNC", + settings={"lock_acquire_timeout": 10}, + ) def rename_tables(): while time.time() < end_time: From 852e983fe3086a6a6aa7b2ba3461b124fe30a6f8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Nov 2023 21:34:22 +0100 Subject: [PATCH 242/274] Follow-up --- packages/clickhouse-server.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/clickhouse-server.yaml b/packages/clickhouse-server.yaml index 5e2bc7c74125..7894129b8e38 100644 --- a/packages/clickhouse-server.yaml +++ b/packages/clickhouse-server.yaml @@ -52,8 +52,6 @@ contents: dst: /lib/systemd/system/clickhouse-server.service - src: root/usr/bin/clickhouse-copier dst: /usr/bin/clickhouse-copier -- src: root/usr/bin/clickhouse-report - dst: /usr/bin/clickhouse-report - src: root/usr/bin/clickhouse-server dst: /usr/bin/clickhouse-server # clickhouse-keeper part From 3c2cf5dc7018b23cbb465ed9f906768fb1314a19 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Nov 2023 21:52:47 +0100 Subject: [PATCH 243/274] Enable Analyzer in Stress and Fuzz tests --- docker/test/fuzzer/query-fuzzer-tweaks-users.xml | 5 ----- docker/test/stateless/stress_tests.lib | 15 --------------- 2 files changed, 20 deletions(-) diff --git a/docker/test/fuzzer/query-fuzzer-tweaks-users.xml b/docker/test/fuzzer/query-fuzzer-tweaks-users.xml index ecd7aae2e4a1..023f257253a4 100644 --- a/docker/test/fuzzer/query-fuzzer-tweaks-users.xml +++ b/docker/test/fuzzer/query-fuzzer-tweaks-users.xml @@ -23,11 +23,6 @@ 10G - - - - - 200 diff --git a/docker/test/stateless/stress_tests.lib b/docker/test/stateless/stress_tests.lib index 551461b6eca5..8f89c1b80dda 100644 --- a/docker/test/stateless/stress_tests.lib +++ b/docker/test/stateless/stress_tests.lib @@ -140,21 +140,6 @@ EOL --> $PWD -EOL - - # Analyzer is not yet ready for testing - cat > /etc/clickhouse-server/users.d/no_analyzer.xml < - - - - - - - - - - EOL } From ff6dfd2490cee6300147ddea4f5eb1ffcfacb08b Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Mon, 20 Nov 2023 21:47:17 +0000 Subject: [PATCH 244/274] Run CI for PRs with missing documentation if 'can be tested' label is present --- tests/ci/run_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index db98a2c1ab5d..231e2617a3f1 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -150,7 +150,7 @@ def main(): DOCS_NAME, pr_info, ) - sys.exit(1) + sys.exit(0) if description_error: print( From 15234474d7a570f6b4fc0c4eae5cfb3718449316 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 7 Nov 2023 10:03:57 +0000 Subject: [PATCH 245/274] Implement system table blob_storage_log --- .../settings.md | 2 +- .../system-tables/blob_storage_log.md | 59 +++++++++++ programs/server/config.xml | 10 ++ src/Backups/BackupIO_S3.cpp | 39 +++++++- src/Backups/BackupIO_S3.h | 6 +- src/Common/SystemLogBase.cpp | 1 + src/Common/SystemLogBase.h | 3 +- src/Coordination/KeeperSnapshotManagerS3.cpp | 5 +- src/Coordination/Standalone/Context.cpp | 5 + src/Coordination/Standalone/Context.h | 2 + src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 2 +- .../ObjectStorages/DiskObjectStorage.cpp | 9 +- .../DiskObjectStorageTransaction.cpp | 4 +- src/Disks/ObjectStorages/IObjectStorage.h | 4 - .../MetadataStorageFromDisk.cpp | 2 +- .../MetadataStorageFromPlainObjectStorage.cpp | 2 +- .../ObjectStorages/S3/S3ObjectStorage.cpp | 28 +++++- src/Disks/ObjectStorages/S3/S3ObjectStorage.h | 6 +- .../ObjectStorages/S3/registerDiskS3.cpp | 11 ++- src/Disks/ObjectStorages/StoredObject.h | 21 +--- ...etadataStorageFromStaticFilesWebServer.cpp | 2 +- src/IO/S3/BlobStorageLogWriter.cpp | 72 ++++++++++++++ src/IO/S3/BlobStorageLogWriter.h | 57 +++++++++++ src/IO/S3/copyS3File.cpp | 53 ++++++++-- src/IO/S3/copyS3File.h | 3 + src/IO/S3/tests/gtest_aws_s3_client.cpp | 3 +- src/IO/WriteBufferFromS3.cpp | 26 ++++- src/IO/WriteBufferFromS3.h | 4 + src/IO/tests/gtest_writebuffer_s3.cpp | 3 +- src/Interpreters/BlobStorageLog.cpp | 92 +++++++++++++++++ src/Interpreters/BlobStorageLog.h | 57 +++++++++++ src/Interpreters/Context.cpp | 11 ++- src/Interpreters/Context.h | 2 + src/Interpreters/InterpreterSystemQuery.cpp | 1 + src/Interpreters/SystemLog.cpp | 4 + src/Interpreters/SystemLog.h | 3 + src/Storages/S3Queue/StorageS3Queue.cpp | 7 +- src/Storages/S3Queue/StorageS3Queue.h | 3 + src/Storages/StorageAzureBlob.cpp | 2 +- src/Storages/StorageS3.cpp | 19 +++- src/Storages/StorageS3.h | 1 + tests/config/config.d/blob_storage_log.xml | 9 ++ tests/config/install.sh | 1 + .../configs/blob_log.xml | 9 ++ .../test_backup_restore_s3/test.py | 28 +++++- .../configs/config.d/blob_log.xml | 9 ++ tests/integration/test_merge_tree_s3/test.py | 99 ++++++++++++++++--- .../test_storage_s3/configs/blob_log.xml | 9 ++ tests/integration/test_storage_s3/test.py | 58 +++++++++-- 49 files changed, 782 insertions(+), 86 deletions(-) create mode 100644 docs/en/operations/system-tables/blob_storage_log.md create mode 100644 src/IO/S3/BlobStorageLogWriter.cpp create mode 100644 src/IO/S3/BlobStorageLogWriter.h create mode 100644 src/Interpreters/BlobStorageLog.cpp create mode 100644 src/Interpreters/BlobStorageLog.h create mode 100644 tests/config/config.d/blob_storage_log.xml create mode 100644 tests/integration/test_backup_restore_s3/configs/blob_log.xml create mode 100644 tests/integration/test_merge_tree_s3/configs/config.d/blob_log.xml create mode 100644 tests/integration/test_storage_s3/configs/blob_log.xml diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index cfc5a939a0ee..3e4f1f4313f5 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -2740,7 +2740,7 @@ ClickHouse will use it to form the proxy URI using the following template: `{pro 10 - + http://resolver:8080/hostname diff --git a/docs/en/operations/system-tables/blob_storage_log.md b/docs/en/operations/system-tables/blob_storage_log.md new file mode 100644 index 000000000000..db08b0c583d8 --- /dev/null +++ b/docs/en/operations/system-tables/blob_storage_log.md @@ -0,0 +1,59 @@ +--- +slug: /en/operations/system-tables/blob_storage_log +--- +# Blob Storage Operations Log + +Contains logging entries with information about various blob storage operations such as uploads and deletes. + +Columns: + +- `event_date` ([Date](../../sql-reference/data-types/date.md)) — Date of the event. +- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Time of the event. +- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Time of the event with microseconds precision. +- `event_type` ([Enum8](../../sql-reference/data-types/enum.md)) — Type of the event. Possible values: + - `'Upload'` + - `'Delete'` + - `'MultiPartUploadCreate'` + - `'MultiPartUploadWrite'` + - `'MultiPartUploadComplete'` + - `'MultiPartUploadAbort'` +- `query_id` ([String](../../sql-reference/data-types/string.md)) — Identifier of the query associated with the event, if any. +- `thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Identifier of the thread performing the operation. +- `thread_name` ([String](../../sql-reference/data-types/string.md)) — Name of the thread performing the operation. +- `disk_name` ([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md)) — Name of the associated disk. +- `bucket` ([String](../../sql-reference/data-types/string.md)) — Name of the bucket. +- `remote_path` ([String](../../sql-reference/data-types/string.md)) — Path to the remote resource. +- `local_path` ([String](../../sql-reference/data-types/string.md)) — Path to the metadata file on the local system, which references the remote resource. +- `data_size` ([UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Size of the data involved in the upload event. +- `error` ([String](../../sql-reference/data-types/string.md)) — Error message associated with the event, if any. + +**Example** + +Suppose a blob storage operation uploads a file, and an event is logged: + +```sql +SELECT * FROM system.blob_storage_log WHERE query_id = '7afe0450-504d-4e4b-9a80-cd9826047972' ORDER BY event_date, event_time_microseconds \G +``` + +```text +Row 1: +────── +event_date: 2023-10-31 +event_time: 2023-10-31 16:03:40 +event_time_microseconds: 2023-10-31 16:03:40.481437 +event_type: Upload +query_id: 7afe0450-504d-4e4b-9a80-cd9826047972 +thread_id: 2381740 +disk_name: disk_s3 +bucket: bucket1 +remote_path: rrr/kxo/tbnqtrghgtnxkzgtcrlutwuslgawe +local_path: store/654/6549e8b3-d753-4447-8047-d462df6e6dbe/tmp_insert_all_1_1_0/checksums.txt +data_size: 259 +error: +``` + +In this example, upload operation was associated with the `INSERT` query with ID `7afe0450-504d-4e4b-9a80-cd9826047972`. The local metadata file `store/654/6549e8b3-d753-4447-8047-d462df6e6dbe/tmp_insert_all_1_1_0/checksums.txt` refers to remote path `rrr/kxo/tbnqtrghgtnxkzgtcrlutwuslgawe` in bucket `bucket1` on disk `disk_s3`, with a size of 259 bytes. + +**See Also** + +- [External Disks for Storing Data](../../operations/storing-data.md) diff --git a/programs/server/config.xml b/programs/server/config.xml index 7800aa511666..4d7d9ab4d5a3 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -1248,6 +1248,16 @@ 7500 + + + system + blob_storage_log
+ toYYYYMM(event_date) + 7500 + event_date + INTERVAL 30 DAY + + + + system + s3queue_log
+ toYYYYMM(event_date) + 7500 + +